Commit bc82de86 authored by Martin Bauer's avatar Martin Bauer
Browse files

Added CI and test files

parent 277eca3b
[flake8]
max-line-length=120
exclude=pystencils/jupytersetup.py,
pystencils/plot2d.py
pystencils/session.py
ignore = W293 W503 W291
__pycache__
.ipynb_checkpoints
.coverage
*.pyc
*.vti
/build
/dist
/*.egg-info
.cache
_build
/.idea
.cache
_local_tmp
\ No newline at end of file
stages:
- test
- deploy
# -------------------------- Tests ------------------------------------------------------------------------------------
# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
tags:
- docker
- cuda
- AVX
artifacts:
when: always
paths:
- coverage_report
# Nightly test - runs "long run" jobs only
test-longrun:
stage: test
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
tags:
- docker
- cuda
- AVX
artifacts:
paths:
- coverage_report
# Minimal tests in windows environment
minimal-windows:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
tags:
- win
script:
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils_dev
- env
- conda env list
- python -c "import numpy"
- python setup.py quicktest
minimal-ubuntu:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
script:
- python3 setup.py quicktest
tags:
- docker
minimal-conda:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
script:
- python setup.py quicktest
tags:
- docker
# -------------------- Linter & Documentation --------------------------------------------------------------------------
flake8-lint:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- flake8 pystencils
tags:
- docker
- cuda
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
- sphinx-build -W -b html doc html_doc
tags:
- docker
- cuda
artifacts:
paths:
- html_doc
pages:
image: i10git.cs.fau.de:5005/software/pystencils/full
stage: deploy
script:
- ls -l
- mv coverage_report html_doc
- mv html_doc public # folder has to be named "public" for gitlab to publish it
artifacts:
paths:
- public
tags:
- docker
only:
- master@software/pystencils
import os
import pytest
import tempfile
import runpy
import sys
# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
# at the same time
from pystencils.cpu import cpujit
# trigger cython imports - there seems to be a problem when multiple processes try to compile the same cython file
# at the same time
try:
import pyximport
pyximport.install(language_level=3)
except ImportError:
pass
from pystencils.boundaries.createindexlistcython import * # NOQA
SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.abspath('pystencils'))
def add_path_to_ignore(path):
if not os.path.exists(path):
return
global collect_ignore
collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py")]
add_path_to_ignore('pystencils_tests/benchmark')
add_path_to_ignore('_local_tmp')
try:
import pycuda
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/pystencils_tests/test_cudagpu.py")]
add_path_to_ignore('pystencils/gpucuda')
try:
import llvmlite
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/backends/llvm.py')]
add_path_to_ignore('pystencils/llvm')
try:
import kerncraft
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py")]
add_path_to_ignore('pystencils/kerncraft_coupling')
try:
import blitzdb
except ImportError:
add_path_to_ignore('pystencils/runhelper')
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
for root, sub_dirs, files in os.walk('.'):
for f in files:
if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
collect_ignore.append(f)
import nbformat
from nbconvert import PythonExporter
class IPythonMockup:
def run_line_magic(self, *args, **kwargs):
pass
def run_cell_magic(self, *args, **kwargs):
pass
def magic(self, *args, **kwargs):
pass
def __bool__(self):
return False
class IPyNbTest(pytest.Item):
def __init__(self, name, parent, code):
super(IPyNbTest, self).__init__(name, parent)
self.code = code
self.add_marker('notebook')
def runtest(self):
global_dict = {'get_ipython': lambda: IPythonMockup(),
'is_test_run': True}
# disable matplotlib output
exec("import matplotlib.pyplot as p; "
"p.switch_backend('Template')", global_dict)
# in notebooks there is an implicit plt.show() - if this is not called a warning is shown when the next
# plot is created. This warning is suppressed here
exec("import warnings;"
"warnings.filterwarnings('ignore', 'Adding an axes using the same arguments as a previous.*')",
global_dict)
with tempfile.NamedTemporaryFile() as f:
f.write(self.code.encode())
f.flush()
runpy.run_path(f.name, init_globals=global_dict, run_name=self.name)
class IPyNbFile(pytest.File):
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
def teardown(self):
pass
def pytest_collect_file(path, parent):
glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
if any(path.fnmatch(g) for g in glob_exprs):
return IPyNbFile(path, parent)
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
import datetime
import sphinx_rtd_theme
import os import os
import sys import sys
sys.path.insert(0, os.path.abspath('..')) sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('../..')) from version_from_git import version_number_from_git
from sphinx_doc_conf import *
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.mathjax',
'sphinx.ext.napoleon',
'nbsphinx',
'sphinxcontrib.bibtex',
'sphinx_autodoc_typehints',
]
add_module_names = False
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
copyright = '{}, Martin Bauer'.format(datetime.datetime.now().year)
author = 'Martin Bauer'
version = version_number_from_git()
release = version_number_from_git()
language = None
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
default_role = 'any'
pygments_style = 'sphinx'
todo_include_todos = False
# Options for HTML output
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_theme = 'sphinx_rtd_theme'
htmlhelp_basename = 'pystencilsdoc'
html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
# NbSphinx configuration
nbsphinx_execute = 'never'
nbsphinx_codecell_lexer = 'python3'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
'numpy': ('https://docs.scipy.org/doc/numpy/', None),
'matplotlib': ('https://matplotlib.org/', None),
'sympy': ('https://docs.sympy.org/latest/', None),
}
autodoc_member_order = 'bysource'
project = 'pystencils' project = 'pystencils'
html_logo = "img/logo.png" html_logo = "img/logo.png"
import subprocess
def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
def get_released_versions():
tags = sorted(subprocess.getoutput('git tag').split('\n'))
versions = [t[len(tag_prefix):] for t in tags if t.startswith(tag_prefix)]
return versions
def tag_from_version(v):
return tag_prefix + v
def increment_version(v):
parsed_version = [int(i) for i in v.split('.')]
parsed_version[-1] += 1
return '.'.join(str(i) for i in parsed_version)
latest_release = get_released_versions()[-1]
commits_since_tag = subprocess.getoutput('git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
is_dirty = len(subprocess.getoutput("git status --untracked-files=no -s")) > 0
if int(commits_since_tag) == 0:
version_string = latest_release
else:
next_version = increment_version(latest_release)
version_string = version_format.format(version=next_version, commits=commits_since_tag, sha=sha)
if is_dirty:
version_string += ".dirty"
return version_string
import numpy as np import numpy as np
import os import os
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
import pystencils as ps import pystencils as ps
from pystencils import create_kernel, create_data_handling from pystencils import create_kernel, create_data_handling
...@@ -196,11 +195,15 @@ def test_kernel(): ...@@ -196,11 +195,15 @@ def test_kernel():
for domain_shape in [(4, 5), (3, 4, 5)]: for domain_shape in [(4, 5), (3, 4, 5)]:
dh = create_data_handling(domain_size=domain_shape, periodicity=True) dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=True) kernel_execution_jacobi(dh, test_gpu=True)
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
reduction(dh) reduction(dh)
try:
import pycuda
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
except ImportError:
pass
def test_vtk_output(): def test_vtk_output():
for domain_shape in [(4, 5), (3, 4, 5)]: for domain_shape in [(4, 5), (3, 4, 5)]:
......
This diff is collapsed.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from pystencils.session import * from pystencils.session import *
from pystencils.data_types import cast_func from pystencils.data_types import cast_func
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Test field equality behaviour ## Test field equality behaviour
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Fields create with same parameters are equal Fields create with same parameters are equal
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0) f1 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0) f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
assert f1 == f2 assert f1 == f2
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print("Field ids equal in accesses: ", id(f1.center._field) == id(f2.center._field)) print("Field ids equal in accesses: ", id(f1.center._field) == id(f2.center._field))
print("Field accesses equal: ", f1.center == f2.center) print("Field accesses equal: ", f1.center == f2.center)
``` ```
%%%% Output: stream %%%% Output: stream
Field ids equal in accesses: True Field ids equal in accesses: True
Field accesses equal: True Field accesses equal: True
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0) f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0) f2 = ps.Field.create_generic('f', spatial_dimensions=2, index_dimensions=0)
assert f1 != f2 assert f1 != f2
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0) f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32) f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32)
assert f1 != f2 assert f1 != f2
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Properties of fields: Properties of fields:
- `field_type`: enum distinguishing normal from index or buffer fields - `field_type`: enum distinguishing normal from index or buffer fields
- `_dtype`: data type of field elements - `_dtype`: data type of field elements
- `_layout`: tuple indicating the memory linearization order - `_layout`: tuple indicating the memory linearization order
- `shape`: size of field for each dimension - `shape`: size of field for each dimension
- `strides`: number of elements to jump over to increase coordinate of this dimension by one - `strides`: number of elements to jump over to increase coordinate of this dimension by one
- `latex_name`: optional display name when field is printed as latex - `latex_name`: optional display name when field is printed as latex
Equality compare of fields: Equality compare of fields:
- field has `__eq__` and ``__hash__`` overridden - field has `__eq__` and ``__hash__`` overridden
- all parameter but `latex_name` are considered for equality - all parameter but `latex_name` are considered for equality
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Test field access equality behaviour ## Test field access equality behaviour
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0) f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0) f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
assert f1.center == f2.center assert f1.center == f2.center
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0) f1 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0)
f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32) f2 = ps.Field.create_generic('f', spatial_dimensions=1, index_dimensions=0, dtype=np.float32)
assert f1.center != f2.center assert f1.center != f2.center
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def print_field_accesses_debug(expr): def print_field_accesses_debug(expr):
from pystencils import Field from pystencils import Field
fas = list(expr.atoms(Field.Access)) fas = list(expr.atoms(Field.Access))
fields = list({e.field for e in fas}) fields = list({e.field for e in fas})
print("Field Accesses:") print("Field Accesses:")
for fa in fas: for fa in fas:
print(f" - {fa}, hash {hash(fa)}, offsets {fa._offsets}, index {fa._index}, {fa._hashable_content()}") print(f" - {fa}, hash {hash(fa)}, offsets {fa._offsets}, index {fa._index}, {fa._hashable_content()}")
print("") print("")
for i in range(len(fas)): for i in range(len(fas)):
for j in range(len(fas)): for j in range(len(fas)):
if not i < j: if not i < j:
continue continue
print( f" -> {i},{j} {fas[i]} == {fas[j]}: {fas[i] == {fas[j]}}") print( f" -> {i},{j} {fas[i]} == {fas[j]}: {fas[i] == {fas[j]}}")
print("Fields") print("Fields")
for f in fields: for f in fields:
print(f" - {f}, {id(f)}, shape {f.shape}, strides {f.strides}, {f._dtype}, {f.field_type}, layout {f.layout}") print(f" - {f}, {id(f)}, shape {f.shape}, strides {f.strides}, {f._dtype}, {f.field_type}, layout {f.layout}")
print("") print("")