Commit bc82de86 authored by Martin Bauer's avatar Martin Bauer
Browse files

Added CI and test files

parent 277eca3b
[flake8]
max-line-length=120
exclude=pystencils/jupytersetup.py,
pystencils/plot2d.py
pystencils/session.py
ignore = W293 W503 W291
__pycache__
.ipynb_checkpoints
.coverage
*.pyc
*.vti
/build
/dist
/*.egg-info
.cache
_build
/.idea
.cache
_local_tmp
\ No newline at end of file
stages:
- test
- deploy
# -------------------------- Tests ------------------------------------------------------------------------------------
# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
tags:
- docker
- cuda
- AVX
artifacts:
when: always
paths:
- coverage_report
# Nightly test - runs "long run" jobs only
test-longrun:
stage: test
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
tags:
- docker
- cuda
- AVX
artifacts:
paths:
- coverage_report
# Minimal tests in windows environment
minimal-windows:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
tags:
- win
script:
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils_dev
- env
- conda env list
- python -c "import numpy"
- python setup.py quicktest
minimal-ubuntu:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
script:
- python3 setup.py quicktest
tags:
- docker
minimal-conda:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
script:
- python setup.py quicktest
tags:
- docker
# -------------------- Linter & Documentation --------------------------------------------------------------------------
flake8-lint:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- flake8 pystencils
tags:
- docker
- cuda
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
- sphinx-build -W -b html doc html_doc
tags:
- docker
- cuda
artifacts:
paths:
- html_doc
pages:
image: i10git.cs.fau.de:5005/software/pystencils/full
stage: deploy
script:
- ls -l
- mv coverage_report html_doc
- mv html_doc public # folder has to be named "public" for gitlab to publish it
artifacts:
paths:
- public
tags:
- docker
only:
- master@software/pystencils
import os
import pytest
import tempfile
import runpy
import sys
# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
# at the same time
from pystencils.cpu import cpujit
# trigger cython imports - there seems to be a problem when multiple processes try to compile the same cython file
# at the same time
try:
import pyximport
pyximport.install(language_level=3)
except ImportError:
pass
from pystencils.boundaries.createindexlistcython import * # NOQA
SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.abspath('pystencils'))
def add_path_to_ignore(path):
if not os.path.exists(path):
return
global collect_ignore
collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py")]
add_path_to_ignore('pystencils_tests/benchmark')
add_path_to_ignore('_local_tmp')
try:
import pycuda
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/pystencils_tests/test_cudagpu.py")]
add_path_to_ignore('pystencils/gpucuda')
try:
import llvmlite
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/backends/llvm.py')]
add_path_to_ignore('pystencils/llvm')
try:
import kerncraft
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py")]
add_path_to_ignore('pystencils/kerncraft_coupling')
try:
import blitzdb
except ImportError:
add_path_to_ignore('pystencils/runhelper')
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
for root, sub_dirs, files in os.walk('.'):
for f in files:
if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
collect_ignore.append(f)
import nbformat
from nbconvert import PythonExporter
class IPythonMockup:
def run_line_magic(self, *args, **kwargs):
pass
def run_cell_magic(self, *args, **kwargs):
pass
def magic(self, *args, **kwargs):
pass
def __bool__(self):
return False
class IPyNbTest(pytest.Item):
def __init__(self, name, parent, code):
super(IPyNbTest, self).__init__(name, parent)
self.code = code
self.add_marker('notebook')
def runtest(self):
global_dict = {'get_ipython': lambda: IPythonMockup(),
'is_test_run': True}
# disable matplotlib output
exec("import matplotlib.pyplot as p; "
"p.switch_backend('Template')", global_dict)
# in notebooks there is an implicit plt.show() - if this is not called a warning is shown when the next
# plot is created. This warning is suppressed here
exec("import warnings;"
"warnings.filterwarnings('ignore', 'Adding an axes using the same arguments as a previous.*')",
global_dict)
with tempfile.NamedTemporaryFile() as f:
f.write(self.code.encode())
f.flush()
runpy.run_path(f.name, init_globals=global_dict, run_name=self.name)
class IPyNbFile(pytest.File):
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
def teardown(self):
pass
def pytest_collect_file(path, parent):
glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
if any(path.fnmatch(g) for g in glob_exprs):
return IPyNbFile(path, parent)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
import datetime
import sphinx_rtd_theme
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('../..'))
from sphinx_doc_conf import *
sys.path.insert(0, os.path.abspath('.'))
from version_from_git import version_number_from_git
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.mathjax',
'sphinx.ext.napoleon',
'nbsphinx',
'sphinxcontrib.bibtex',
'sphinx_autodoc_typehints',
]
add_module_names = False
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
copyright = '{}, Martin Bauer'.format(datetime.datetime.now().year)
author = 'Martin Bauer'
version = version_number_from_git()
release = version_number_from_git()
language = None
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
default_role = 'any'
pygments_style = 'sphinx'
todo_include_todos = False
# Options for HTML output
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_theme = 'sphinx_rtd_theme'
htmlhelp_basename = 'pystencilsdoc'
html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
# NbSphinx configuration
nbsphinx_execute = 'never'
nbsphinx_codecell_lexer = 'python3'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
'numpy': ('https://docs.scipy.org/doc/numpy/', None),
'matplotlib': ('https://matplotlib.org/', None),
'sympy': ('https://docs.sympy.org/latest/', None),
}
autodoc_member_order = 'bysource'
project = 'pystencils'
html_logo = "img/logo.png"
import subprocess
def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
def get_released_versions():
tags = sorted(subprocess.getoutput('git tag').split('\n'))
versions = [t[len(tag_prefix):] for t in tags if t.startswith(tag_prefix)]
return versions
def tag_from_version(v):
return tag_prefix + v
def increment_version(v):
parsed_version = [int(i) for i in v.split('.')]
parsed_version[-1] += 1
return '.'.join(str(i) for i in parsed_version)
latest_release = get_released_versions()[-1]
commits_since_tag = subprocess.getoutput('git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
is_dirty = len(subprocess.getoutput("git status --untracked-files=no -s")) > 0
if int(commits_since_tag) == 0:
version_string = latest_release
else:
next_version = increment_version(latest_release)
version_string = version_format.format(version=next_version, commits=commits_since_tag, sha=sha)
if is_dirty:
version_string += ".dirty"
return version_string
import numpy as np
import os
from tempfile import TemporaryDirectory
import pystencils as ps
from pystencils import create_kernel, create_data_handling
......@@ -196,11 +195,15 @@ def test_kernel():
for domain_shape in [(4, 5), (3, 4, 5)]:
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=True)
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
reduction(dh)
try:
import pycuda
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
except ImportError:
pass
def test_vtk_output():
for domain_shape in [(4, 5), (3, 4, 5)]:
......
This diff is collapsed.
%% Cell type:code id: tags:
``` python
from pystencils.session import *
from pystencils.data_types import cast_func
```
%% Cell type:markdown id: tags:
## Test field equality behaviour
%% Cell type:markdown id: tags:
Fields create with same parameters are equal
%% Cell type:code id: tags:
``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
assert f1 == f2
```
%% Cell type:code id: tags:
``` python
print("Field ids equal in accesses: ", id(f1.center._field) == id(f2.center._field))
print("Field accesses equal: ", f1.center == f2.center)
```
%%%% Output: stream
Field ids equal in accesses: True
Field accesses equal: True
%% Cell type:code id: tags:
``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
assert f1 != f2
```
%% Cell type:code id: tags:
``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32)
assert f1 != f2
```
%% Cell type:markdown id: tags:
Properties of fields:
- `field_type`: enum distinguishing normal from index or buffer fields
- `_dtype`: data type of field elements
- `_layout`: tuple indicating the memory linearization order
- `shape`: size of field for each dimension
- `strides`: number of elements to jump over to increase coordinate of this dimension by one
- `latex_name`: optional display name when field is printed as latex
Equality compare of fields:
- field has `__eq__` and ``__hash__`` overridden
- all parameter but `latex_name` are considered for equality
%% Cell type:markdown id: tags:
## Test field access equality behaviour
%% Cell type:code id: tags:
``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
assert f1.center == f2.center
```
%% Cell type:code id: tags:
``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32)
assert f1.center != f2.center
```
%% Cell type:code id: tags:
``` python
def print_field_accesses_debug(expr):
from pystencils import Field
fas = list(expr.atoms(Field.Access))
fields = list({e.field for e in fas})
print("Field Accesses:")
for fa in fas:
print(f" - {fa}, hash {hash(fa)}, offsets {fa._offsets}, index {fa._index}, {fa._hashable_content()}")
print("")
for i in range(len(fas)):
for j in range(len(fas)):
if not i < j:
continue
print( f" -> {i},{j} {fas[i]} == {fas[j]}: {fas[i] == {fas[j]}}")
print("Fields")
for f in fields:
print(f" - {f}, {id(f)}, shape {f.shape}, strides {f.strides}, {f._dtype}, {f.field_type}, layout {f.layout}")
print("")
for i in range(len(fields)):
for j in range(len(fields)):
if not i < j:
continue
print(f" - {fields[i]} == {fields[j]}: {fields[i] == fields[j]}, ids equal {id(fields[i])==id(fields[j])}, hash equal {hash(fields[i])==hash(fields[j])}")
```
%% Cell type:code id: tags:
``` python
print_field_accesses_debug(f1.center * f2.center)
```
%%%% Output: stream
Field Accesses:
- f[0], hash 2177071761647211096, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (fshape_f[0],), (fstride_f[0],), -2638709558778433189, <FieldType.GENERIC: 0>, 'f'), 0)
- f[0], hash -219035921004479174, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (fshape_f[0],), (fstride_f[0],), 1379426851108887372, <FieldType.GENERIC: 0>, 'f'), 0)
- f[0], hash -3276894289571194847, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (_size_f_0,), (_stride_f_0,), 3146377891102027609, <FieldType.GENERIC: 0>, 'f', None), 0)
- f[0], hash -1516451775709390846, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (_size_f_0,), (_stride_f_0,), -1421177580377734245, <FieldType.GENERIC: 0>, 'f', None), 0)
-> 0,1 f[0] == f[0]: False
Fields
- f, 139911303819560, shape (fshape_f[0],), strides (fstride_f[0],), double, FieldType.GENERIC, layout (0,)
- f, 139911303820008, shape (fshape_f[0],), strides (fstride_f[0],), float, FieldType.GENERIC, layout (0,)
- f, 140548694371968, shape (_size_f_0,), strides (_stride_f_0,), double, FieldType.GENERIC, layout (0,)
- f, 140548693963104, shape (_size_f_0,), strides (_stride_f_0,), float, FieldType.GENERIC, layout (0,)
- f == f: False, ids equal False, hash equal False
%% Cell type:markdown id: tags:
## Custom fields
%% Cell type:code id: tags:
``` python
from lbmpy.sparse.update_rule_sparse import *
```
%% Cell type:code id: tags:
``` python
list_field = create_symbolic_list('l', 10, 2, np.float64)
normal_field = ps.fields("f: [2D]")
normal_field.field_type = ps.FieldType.CUSTOM
t1 = normal_field.absolute_access( (list_field[1](0),), (1,))
t2 = normal_field.absolute_access( (list_field[1](1),), (1,))
t1 + t2
```
%%%% Output: execute_result
![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEcAAAAgBAMAAACyb3jOAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEJl2IquJVETdZu8yu83OyatpAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABN0lEQVQ4EcWQsUoDQRCGJ6A5CcFAsNDqLGJhF3yC4AukEITgE9iIICmCRV4hjZVIAr6AkDoQ8gByTao0aWwPLZR0OjO7MzdLbkErr7j5d75v93YOwD2vvm6V+lpbi57GMNRPRkVjUMQwJf8jNc86UHonAnKny+FtuUTAS5XNIVjpTsZj4KXkHbv3QgBUYpC8MKmOAM7HF2qpRGDn9LONpHb1cKwCBZEC0CDVPCKBBd25MTCqZMGQnFovY/Umzx/z/I0zgm/XBbjmvLtmgC89CUEq0pJzieSA2zuNSQ6wVNlEJA9Y2vuISB6wVMW7pVmymk14WVzcAR67v49QhyDvwMkCaKKnI+wFknMUkPTcikgC9N+kmfxxf4yU8ReoZLNwqSphw2bhVM3YccluiJ70Zyn4dLD7t4sf071mgtNBAg8AAAAASUVORK5CYII=)
$${{f}_{\mathbf{{l}_{1}^{1}}}^{1}} + {{f}_{\mathbf{{l}_{1}}}^{1}}$$
f_000035D373 + f_000035D373
......
......@@ -25,6 +25,7 @@ def test_spatial_2d_unit_sum():
_, coefficients = stencil_coefficients(discretized)
assert sum(coefficients) == 0
def test_spatial_1d_unit_sum():
f = ps.fields("f: double[1D]")
h = sp.symbols("h")
......
[pytest]
python_files = test_*.py *_test.py scenario_*.py
norecursedirs = *.egg-info .git .cache .ipynb_checkpoints htmlcov
addopts = --doctest-modules --durations=20 --cov-config pytest.ini
[run]
branch = True
source = pystencils
pystencils_tests
omit = doc/*
pystencils_tests/*
setup.py
conftest.py
pystencils/jupytersetup.py
pystencils/cpu/msvc_detection.py
pystencils/sympy_gmpy_bug_workaround.py
pystencils/cache.py
pystencils/pacxx/benchmark.py
[report]
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
def __repr__
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise NotImplementedError
#raise ValueError
# Don't complain if non-runnable code isn't run:
if 0:
if False:
if __name__ == .__main__.: