Commit f8387b50 authored by Martin Bauer's avatar Martin Bauer
Browse files

Added CI and test files

parent a8dd0c50
[flake8]
max-line-length=120
exclude=lbmpy/plot2d.py
lbmpy/session.py
ignore = W293 W503 W291
__pycache__
.ipynb_checkpoints
.coverage
*.pyc
*.vti
/build
/dist
/*.egg-info
.cache
_build
/.idea
.cache
_local_tmp
\ No newline at end of file
stages:
- test
- deploy
# -------------------------- Tests ------------------------------------------------------------------------------------
# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
tags:
- docker
- cuda
- AVX
artifacts:
when: always
paths:
- coverage_report
# Nightly test - runs "long run" jobs only
test-longrun:
stage: test
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
tags:
- docker
- cuda
- AVX
artifacts:
paths:
- coverage_report
# Minimal tests in windows environment
minimal-windows:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
tags:
- win
script:
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils_dev
- env
- conda env list
- python -c "import numpy"
- python setup.py quicktest
minimal-ubuntu:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
script:
- python3 setup.py quicktest
tags:
- docker
minimal-conda:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
script:
- python setup.py quicktest
tags:
- docker
# -------------------- Linter & Documentation --------------------------------------------------------------------------
flake8-lint:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- flake8 lbmpy
tags:
- docker
- cuda
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
- sphinx-build -W -b html doc html_doc
tags:
- docker
- cuda
artifacts:
paths:
- html_doc
pages:
image: i10git.cs.fau.de:5005/software/pystencils/full
stage: deploy
script:
- ls -l
- mv coverage_report html_doc
- mv html_doc public # folder has to be named "public" for gitlab to publish it
artifacts:
paths:
- public
tags:
- docker
only:
- master@pycodegen/lbmpy
import os
import pytest
import tempfile
import runpy
import sys
# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
# at the same time
from pystencils.cpu import cpujit
# trigger cython imports - there seems to be a problem when multiple processes try to compile the same cython file
# at the same time
try:
import pyximport
pyximport.install(language_level=3)
except ImportError:
pass
SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.abspath('lbmpy'))
def add_path_to_ignore(path):
if not os.path.exists(path):
return
global collect_ignore
collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py"),
os.path.join(SCRIPT_FOLDER, "doc", "img", "mb_discretization", "maxwell_boltzmann_stencil_plot.py")]
add_path_to_ignore('pystencils_tests/benchmark')
add_path_to_ignore('_local_tmp')
try:
import pycuda
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "lbmpy_tests/test_cpu_gpu_equivalence.py")]
try:
import waLBerla
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "lbmpy_tests/test_serial_scenarios.py")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, "lbmpy_tests/test_datahandling_parallel.py")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
for root, sub_dirs, files in os.walk('.'):
for f in files:
if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
collect_ignore.append(f)
import nbformat
from nbconvert import PythonExporter
class IPythonMockup:
def run_line_magic(self, *args, **kwargs):
pass
def run_cell_magic(self, *args, **kwargs):
pass
def magic(self, *args, **kwargs):
pass
def __bool__(self):
return False
class IPyNbTest(pytest.Item):
def __init__(self, name, parent, code):
super(IPyNbTest, self).__init__(name, parent)
self.code = code
self.add_marker('notebook')
def runtest(self):
global_dict = {'get_ipython': lambda: IPythonMockup(),
'is_test_run': True}
# disable matplotlib output
exec("import matplotlib.pyplot as p; "
"p.switch_backend('Template')", global_dict)
# in notebooks there is an implicit plt.show() - if this is not called a warning is shown when the next
# plot is created. This warning is suppressed here
exec("import warnings;"
"warnings.filterwarnings('ignore', 'Adding an axes using the same arguments as a previous.*')",
global_dict)
with tempfile.NamedTemporaryFile() as f:
f.write(self.code.encode())
f.flush()
runpy.run_path(f.name, init_globals=global_dict, run_name=self.name)
class IPyNbFile(pytest.File):
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
def teardown(self):
pass
def pytest_collect_file(path, parent):
glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
if any(path.fnmatch(g) for g in glob_exprs):
return IPyNbFile(path, parent)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
import datetime
import sphinx_rtd_theme
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('../../pystencils'))
sys.path.insert(0, os.path.abspath('../..'))
from sphinx_doc_conf import *
sys.path.insert(0, os.path.abspath('.'))
from version_from_git import version_number_from_git
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.mathjax',
'sphinx.ext.napoleon',
'nbsphinx',
'sphinxcontrib.bibtex',
'sphinx_autodoc_typehints',
]
add_module_names = False
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
copyright = '{}, Martin Bauer'.format(datetime.datetime.now().year)
author = 'Martin Bauer'
version = version_number_from_git()
release = version_number_from_git()
language = None
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
default_role = 'any'
pygments_style = 'sphinx'
todo_include_todos = False
# Options for HTML output
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_theme = 'sphinx_rtd_theme'
htmlhelp_basename = 'pystencilsdoc'
html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
# NbSphinx configuration
nbsphinx_execute = 'never'
nbsphinx_codecell_lexer = 'python3'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
'numpy': ('https://docs.scipy.org/doc/numpy/', None),
'matplotlib': ('https://matplotlib.org/', None),
'sympy': ('https://docs.sympy.org/latest/', None),
}
autodoc_member_order = 'bysource'
project = 'lbmpy'
html_logo = "img/logo.png"
......@@ -480,6 +480,158 @@
effect="spiro"
id="path-effect4188-7"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-3"
style="overflow:visible"
inkscape:isstock="true">
<path
id="path1421-6"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
inkscape:connector-curvature="0" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-75"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-3"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-5"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-6"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-6-2"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-7-9"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-7-1"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-2-27"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-2-0"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-3-9"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-2-6-36"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-2-5-0"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-3-7-6"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-2-6-3-2"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-2-5-9-61"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-3-7-4-8"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-2-2-7"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-2-8-9"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-3-4-2"
is_visible="true" />
<marker
inkscape:stockid="Arrow1Send"
orient="auto"
refY="0"
refX="0"
id="Arrow1Send-8-6-9-0"
style="overflow:visible"
inkscape:isstock="true">
<path
inkscape:connector-curvature="0"
id="path1421-2-7-4-2"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
style="fill:#dddddd;fill-opacity:1;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.00000003pt;stroke-opacity:1"
transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
</marker>
<inkscape:path-effect
effect="spiro"
id="path-effect1404-7-7-5-3"
is_visible="true" />
</defs>
<sodipodi:namedview
id="base"
......@@ -488,16 +640,16 @@
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="0.98994949"
inkscape:cx="159.4121"
inkscape:cy="-32.506835"
inkscape:zoom="1.4"
inkscape:cx="158.26067"
inkscape:cy="-4.9825309"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
inkscape:window-width="498"
inkscape:window-height="394"
inkscape:window-x="2210"
inkscape:window-y="646"
inkscape:window-width="1214"
inkscape:window-height="1052"
inkscape:window-x="1482"
inkscape:window-y="524"
inkscape:window-maximized="0"
fit-margin-top="0"
fit-margin-left="0"
......@@ -622,5 +774,87 @@
d="m 36.797679,33.475 11.90625,13.229166"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-2-2)" />
</g>
<rect
style="opacity:1;fill:#646ecb;fill-opacity:1;stroke:#d2d2d2;stroke-width:0.5091567;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
id="rect1396-7"
width="53.404633"
height="53.404633"
x="9.9747782"
y="82.509102"
ry="3.0735996"
inkscape:export-xdpi="188.45"
inkscape:export-ydpi="188.45" />
<path
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:1.22452438;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-6-9-0)"
d="M 36.299116,109.56879 H 17.132601"
id="path1402-8-0-4-2"
inkscape:connector-curvature="0"
inkscape:path-effect="#path-effect1404-7-7-5-3"
inkscape:original-d="m 36.299116,109.56879 c -6.171351,0.0481 -12.995819,-0.0487 -19.166515,0"
sodipodi:nodetypes="cc"
inkscape:export-xdpi="188.45"
inkscape:export-ydpi="188.45" />
<g
id="g9842-8"
inkscape:export-xdpi="188.45"
inkscape:export-ydpi="188.45"
transform="matrix(1.4488076,0,0,1.4488076,-17.013641,61.069964)">
<path
sodipodi:nodetypes="cc"
inkscape:original-d="m 36.797679,33.475 c 2.23e-4,-4.259735 2.23e-4,-8.969879 0,-13.229167"
inkscape:path-effect="#path-effect1404-75"
inkscape:connector-curvature="0"
id="path1402-9"
d="M 36.797679,33.475 V 20.245833"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-3)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="m 36.797679,33.475 c 4.259736,2.23e-4 8.969879,2.23e-4 13.229167,0"
inkscape:path-effect="#path-effect1404-7-6"
inkscape:connector-curvature="0"
id="path1402-8-7"
d="M 36.797679,33.475 H 50.026846"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-3)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="m 36.797679,33.475 c 0.03317,4.259607 -0.03362,8.97001 0,13.229166"
inkscape:path-effect="#path-effect1404-7-7-1"
inkscape:connector-curvature="0"
id="path1402-8-0-3"
d="M 36.797679,33.475 V 46.704166"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-6-2)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="m 36.797679,33.475 c 4.259736,2.23e-4 8.13389,-13.228083 12.393178,-13.228306"
inkscape:path-effect="#path-effect1404-7-3-9"
inkscape:connector-curvature="0"
id="path1402-8-9-6"
d="M 36.797679,33.475 49.190857,20.246694"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-2-27)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="M 36.797679,33.475 C 32.537943,33.475223 27.827801,20.246056 23.568513,20.245833"
inkscape:path-effect="#path-effect1404-7-3-7-6"
inkscape:connector-curvature="0"
id="path1402-8-9-4-1"
d="M 36.797679,33.475 23.568513,20.245833"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-2-6-36)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="M 36.797679,33.475 C 32.537943,33.474776 27.827801,46.703943 23.568513,46.704166"
inkscape:path-effect="#path-effect1404-7-3-7-4-8"
inkscape:connector-curvature="0"
id="path1402-8-9-4-4-2"
d="M 36.797679,33.475 23.568513,46.704166"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-2-6-3-2)" />
<path
sodipodi:nodetypes="cc"
inkscape:original-d="m 36.797679,33.475 c 4.259736,-2.23e-4 7.646962,13.228943 11.90625,13.229166"
inkscape:path-effect="#path-effect1404-7-3-4-2"
inkscape:connector-curvature="0"
id="path1402-8-9-3-9"
d="m 36.797679,33.475 11.90625,13.229166"
style="fill:none;fill-rule:evenodd;stroke:#dddddd;stroke-width:0.84519458;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-8-2-2-7)" />
</g>
</g>
</svg>
import subprocess
def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
def get_released_versions():
tags = sorted(subprocess.getoutput('git tag').split('\n'))
versions = [t[len(tag_prefix):] for t in tags if t.startswith(tag_prefix)]
return versions
def tag_from_version(v):
return tag_prefix + v
def increment_version(v):
parsed_version = [int(i) for i in v.split('.')]
parsed_version[-1] += 1
return '.'.join(str(i) for i in parsed_version)
latest_release = get_released_versions()[-1]
commits_since_tag = subprocess.getoutput('git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
is_dirty = len(subprocess.getoutput("git status --untracked-files=no -s")) > 0
if int(commits_since_tag) == 0: