Commit bc82de86 authored by Martin Bauer's avatar Martin Bauer
Browse files

Added CI and test files

parent 277eca3b
[flake8]
max-line-length=120
exclude=pystencils/jupytersetup.py,
pystencils/plot2d.py
pystencils/session.py
ignore = W293 W503 W291
__pycache__
.ipynb_checkpoints
.coverage
*.pyc
*.vti
/build
/dist
/*.egg-info
.cache
_build
/.idea
.cache
_local_tmp
\ No newline at end of file
stages:
- test
- deploy
# -------------------------- Tests ------------------------------------------------------------------------------------
# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
tags:
- docker
- cuda
- AVX
artifacts:
when: always
paths:
- coverage_report
# Nightly test - runs "long run" jobs only
test-longrun:
stage: test
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
tags:
- docker
- cuda
- AVX
artifacts:
paths:
- coverage_report
# Minimal tests in windows environment
minimal-windows:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
tags:
- win
script:
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils_dev
- env
- conda env list
- python -c "import numpy"
- python setup.py quicktest
minimal-ubuntu:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
script:
- python3 setup.py quicktest
tags:
- docker
minimal-conda:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
script:
- python setup.py quicktest
tags:
- docker
# -------------------- Linter & Documentation --------------------------------------------------------------------------
flake8-lint:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- flake8 pystencils
tags:
- docker
- cuda
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/software/pystencils/full
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
- sphinx-build -W -b html doc html_doc
tags:
- docker
- cuda
artifacts:
paths:
- html_doc
pages:
image: i10git.cs.fau.de:5005/software/pystencils/full
stage: deploy
script:
- ls -l
- mv coverage_report html_doc
- mv html_doc public # folder has to be named "public" for gitlab to publish it
artifacts:
paths:
- public
tags:
- docker
only:
- master@software/pystencils
import os
import pytest
import tempfile
import runpy
import sys
# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
# at the same time
from pystencils.cpu import cpujit
# trigger cython imports - there seems to be a problem when multiple processes try to compile the same cython file
# at the same time
try:
import pyximport
pyximport.install(language_level=3)
except ImportError:
pass
from pystencils.boundaries.createindexlistcython import * # NOQA
SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.abspath('pystencils'))
def add_path_to_ignore(path):
if not os.path.exists(path):
return
global collect_ignore
collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py")]
add_path_to_ignore('pystencils_tests/benchmark')
add_path_to_ignore('_local_tmp')
try:
import pycuda
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/pystencils_tests/test_cudagpu.py")]
add_path_to_ignore('pystencils/gpucuda')
try:
import llvmlite
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/backends/llvm.py')]
add_path_to_ignore('pystencils/llvm')
try:
import kerncraft
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py")]
add_path_to_ignore('pystencils/kerncraft_coupling')
try:
import blitzdb
except ImportError:
add_path_to_ignore('pystencils/runhelper')
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
for root, sub_dirs, files in os.walk('.'):
for f in files:
if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
collect_ignore.append(f)
import nbformat
from nbconvert import PythonExporter
class IPythonMockup:
def run_line_magic(self, *args, **kwargs):
pass
def run_cell_magic(self, *args, **kwargs):
pass
def magic(self, *args, **kwargs):
pass
def __bool__(self):
return False
class IPyNbTest(pytest.Item):
def __init__(self, name, parent, code):
super(IPyNbTest, self).__init__(name, parent)
self.code = code
self.add_marker('notebook')
def runtest(self):
global_dict = {'get_ipython': lambda: IPythonMockup(),
'is_test_run': True}
# disable matplotlib output
exec("import matplotlib.pyplot as p; "
"p.switch_backend('Template')", global_dict)
# in notebooks there is an implicit plt.show() - if this is not called a warning is shown when the next
# plot is created. This warning is suppressed here
exec("import warnings;"
"warnings.filterwarnings('ignore', 'Adding an axes using the same arguments as a previous.*')",
global_dict)
with tempfile.NamedTemporaryFile() as f:
f.write(self.code.encode())
f.flush()
runpy.run_path(f.name, init_globals=global_dict, run_name=self.name)
class IPyNbFile(pytest.File):
def collect(self):
exporter = PythonExporter()
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
def teardown(self):
pass
def pytest_collect_file(path, parent):
glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
if any(path.fnmatch(g) for g in glob_exprs):
return IPyNbFile(path, parent)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
import datetime
import sphinx_rtd_theme
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('../..'))
from sphinx_doc_conf import *
sys.path.insert(0, os.path.abspath('.'))
from version_from_git import version_number_from_git
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.mathjax',
'sphinx.ext.napoleon',
'nbsphinx',
'sphinxcontrib.bibtex',
'sphinx_autodoc_typehints',
]
add_module_names = False
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
copyright = '{}, Martin Bauer'.format(datetime.datetime.now().year)
author = 'Martin Bauer'
version = version_number_from_git()
release = version_number_from_git()
language = None
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
default_role = 'any'
pygments_style = 'sphinx'
todo_include_todos = False
# Options for HTML output
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
html_theme = 'sphinx_rtd_theme'
htmlhelp_basename = 'pystencilsdoc'
html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
# NbSphinx configuration
nbsphinx_execute = 'never'
nbsphinx_codecell_lexer = 'python3'
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
'numpy': ('https://docs.scipy.org/doc/numpy/', None),
'matplotlib': ('https://matplotlib.org/', None),
'sympy': ('https://docs.sympy.org/latest/', None),
}
autodoc_member_order = 'bysource'
project = 'pystencils'
html_logo = "img/logo.png"
import subprocess
def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
def get_released_versions():
tags = sorted(subprocess.getoutput('git tag').split('\n'))
versions = [t[len(tag_prefix):] for t in tags if t.startswith(tag_prefix)]
return versions
def tag_from_version(v):
return tag_prefix + v
def increment_version(v):
parsed_version = [int(i) for i in v.split('.')]
parsed_version[-1] += 1
return '.'.join(str(i) for i in parsed_version)
latest_release = get_released_versions()[-1]
commits_since_tag = subprocess.getoutput('git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
is_dirty = len(subprocess.getoutput("git status --untracked-files=no -s")) > 0
if int(commits_since_tag) == 0:
version_string = latest_release
else:
next_version = increment_version(latest_release)
version_string = version_format.format(version=next_version, commits=commits_since_tag, sha=sha)
if is_dirty:
version_string += ".dirty"
return version_string
import numpy as np
import os
from tempfile import TemporaryDirectory
import pystencils as ps
from pystencils import create_kernel, create_data_handling
......@@ -196,11 +195,15 @@ def test_kernel():
for domain_shape in [(4, 5), (3, 4, 5)]:
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=True)
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
reduction(dh)
try:
import pycuda
dh = create_data_handling(domain_size=domain_shape, periodicity=True)
kernel_execution_jacobi(dh, test_gpu=False)
except ImportError:
pass
def test_vtk_output():
for domain_shape in [(4, 5), (3, 4, 5)]:
......
This diff is collapsed.
......@@ -111,33 +111,5 @@
%% Cell type:code id: tags:
``` python
print_field_accesses_debug(f1.center * f2.center)
```
%% Cell type:markdown id: tags:
## Custom fields
%% Cell type:code id: tags:
``` python
from lbmpy.sparse.update_rule_sparse import *
```
%% Cell type:code id: tags:
``` python
list_field = create_symbolic_list('l', 10, 2, np.float64)
normal_field = ps.fields("f: [2D]")
normal_field.field_type = ps.FieldType.CUSTOM
t1 = normal_field.absolute_access( (list_field[1](0),), (1,))
t2 = normal_field.absolute_access( (list_field[1](1),), (1,))
t1 + t2
```
%%%% Output: execute_result
![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEcAAAAgBAMAAACyb3jOAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEJl2IquJVETdZu8yu83OyatpAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABN0lEQVQ4EcWQsUoDQRCGJ6A5CcFAsNDqLGJhF3yC4AukEITgE9iIICmCRV4hjZVIAr6AkDoQ8gByTao0aWwPLZR0OjO7MzdLbkErr7j5d75v93YOwD2vvm6V+lpbi57GMNRPRkVjUMQwJf8jNc86UHonAnKny+FtuUTAS5XNIVjpTsZj4KXkHbv3QgBUYpC8MKmOAM7HF2qpRGDn9LONpHb1cKwCBZEC0CDVPCKBBd25MTCqZMGQnFovY/Umzx/z/I0zgm/XBbjmvLtmgC89CUEq0pJzieSA2zuNSQ6wVNlEJA9Y2vuISB6wVMW7pVmymk14WVzcAR67v49QhyDvwMkCaKKnI+wFknMUkPTcikgC9N+kmfxxf4yU8ReoZLNwqSphw2bhVM3YccluiJ70Zyn4dLD7t4sf071mgtNBAg8AAAAASUVORK5CYII=)
$${{f}_{\mathbf{{l}_{1}^{1}}}^{1}} + {{f}_{\mathbf{{l}_{1}}}^{1}}$$
f_000035D373 + f_000035D373
......
......@@ -25,6 +25,7 @@ def test_spatial_2d_unit_sum():
_, coefficients = stencil_coefficients(discretized)
assert sum(coefficients) == 0
def test_spatial_1d_unit_sum():
f = ps.fields("f: double[1D]")
h = sp.symbols("h")
......
[pytest]
python_files = test_*.py *_test.py scenario_*.py
norecursedirs = *.egg-info .git .cache .ipynb_checkpoints htmlcov
addopts = --doctest-modules --durations=20 --cov-config pytest.ini
[run]
branch = True
source = pystencils
pystencils_tests
omit = doc/*
pystencils_tests/*
setup.py
conftest.py
pystencils/jupytersetup.py
pystencils/cpu/msvc_detection.py
pystencils/sympy_gmpy_bug_workaround.py
pystencils/cache.py
pystencils/pacxx/benchmark.py
[report]
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
def __repr__
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise NotImplementedError
#raise ValueError
# Don't complain if non-runnable code isn't run:
if 0:
if False:
if __name__ == .__main__.:
skip_covered = True
fail_under = 74
[html]
directory = coverage_report
import os
import sys
import io
from setuptools import setup, find_packages
sys.path.insert(0, os.path.abspath('..'))
from custom_pypi_index.pypi_index import get_current_dev_version_from_git
import distutils
from contextlib import redirect_stdout
from importlib import import_module
sys.path.insert(0, os.path.abspath('doc'))
from version_from_git import version_number_from_git
quick_tests = [
'test_datahandling.test_kernel',
'test_blocking_staggered.test_blocking_staggered',
'test_blocking_staggered.test_blocking_staggered',
'test_vectorization.test_vectorization_variable_size',
]
class SimpleTestRunner(distutils.cmd.Command):
"""A custom command to run selected tests"""
description = 'run some quick tests'
user_options = []
@staticmethod
def _run_tests_in_module(test):
"""Short test runner function - to work also if py.test is not installed."""
test = 'pystencils_tests.' + test
mod, function_name = test.rsplit('.', 1)
if isinstance(mod, str):
mod = import_module(mod)
func = getattr(mod, function_name)
print(" -> %s in %s" % (function_name, mod.__name__))
with redirect_stdout(io.StringIO()):
func()
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
"""Run command."""
for test in quick_tests:
self._run_tests_in_module(test)
setup(name='pystencils',
version=get_current_dev_version_from_git(),
version=version_number_from_git(),
description='Python Stencil Compiler based on sympy as numpy',
author='Martin Bauer',
license='AGPLv3',
......@@ -14,6 +57,7 @@ setup(name='pystencils',
url='https://i10git.cs.fau.de/software/pystencils/',
packages=['pystencils'] + ['pystencils.' + s for s in find_packages('pystencils')],
install_requires=['sympy>=1.1', 'numpy', 'appdirs', 'joblib'],
package_data={'pystencils': ['include/*.h']},
classifiers=[
'Development Status :: 4 - Beta',
'Framework :: Jupyter',
......@@ -31,6 +75,9 @@ setup(name='pystencils',
'doc': ['sphinx', 'sphinx_rtd_theme', 'nbsphinx',
'sphinxcontrib-bibtex', 'sphinx_autodoc_typehints', 'pandoc'],
},
tests_require=['pytest', 'pytest-cov', 'pytest-xdist', 'flake8'],
tests_require=['pytest', 'pytest-cov', 'pytest-xdist', 'flake8', 'nbformat', 'nbconvert', 'ipython'],
python_requires=">=3.6",
cmdclass={
'quicktest': SimpleTestRunner
}
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment