Commit f8ea9fc4 authored by Dominik Thoennes's avatar Dominik Thoennes
Browse files

Merge branch 'master' into hyteg

parents da54b3ec b2b2e912
[flake8]
max-line-length=120
exclude=pystencils/jupytersetup.py,
pystencils/plot2d.py
exclude=pystencils/jupyter.py,
pystencils/plot.py
pystencils/session.py
ignore = W293 W503 W291
ignore = W293 W503 W291 C901 E741
......@@ -10,4 +10,11 @@ __pycache__
_build
/.idea
.cache
_local_tmp
\ No newline at end of file
_local_tmp
RELEASE-VERSION
test-report
pystencils/boundaries/createindexlistcython.c
pystencils/boundaries/createindexlistcython.*.so
pystencils_tests/tmp
pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
\ No newline at end of file
......@@ -11,21 +11,24 @@ tests-and-coverage:
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script:
- env
- pip list
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun" --html test-report/index.html
tags:
- docker
- cuda
- cuda11
- AVX
artifacts:
when: always
paths:
- coverage_report
- test-report
# Nightly test - runs "long run" jobs only
test-longrun:
......@@ -33,19 +36,20 @@ test-longrun:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
- py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --html test-report/index.html
tags:
- docker
- cuda
- cuda11
- AVX
artifacts:
paths:
- coverage_report
- test-report
# Minimal tests in windows environment
minimal-windows:
......@@ -63,29 +67,91 @@ minimal-windows:
- python -c "import numpy"
- python setup.py quicktest
minimal-ubuntu:
ubuntu:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
script:
- python3 setup.py quicktest
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- sed -i 's/--doctest-modules //g' pytest.ini
- pytest-3 -v -m "not longrun"
tags:
- docker
- cuda11
- AVX
minimal-conda:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
script:
- python setup.py quicktest
tags:
- docker
- cuda
minimal-sympy-master:
stage: test
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
script:
- python -m pip install --upgrade git+https://github.com/sympy/sympy.git
- python setup.py quicktest
allow_failure: true
tags:
- docker
- cuda
pycodegen-integration:
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
stage: test
when: manual
allow_failure: true
script:
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@i10git.cs.fau.de/pycodegen/pycodegen.git
- cd pycodegen
- git submodule sync --recursive
- git submodule update --init --recursive
- git submodule foreach git fetch origin # compare the latest master version!
- git submodule foreach git reset --hard origin/master
- cd pystencils
- git remote add test $CI_REPOSITORY_URL
- git fetch test
- git reset --hard $CI_COMMIT_SHA
- cd ..
- pip install -e pystencils/
- pip install -e lbmpy/
- pip install -e pygrandchem/
- ./install_walberla.sh
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- cd pystencils
- py.test -v -n $NUM_CORES .
- cd ../lbmpy
- py.test -v -n $NUM_CORES .
- cd ../pygrandchem
- py.test -v -n $NUM_CORES .
- cd ../walberla/build/
- make CodegenJacobiCPU CodegenJacobiGPU CodegenPoissonCPU CodegenPoissonGPU MicroBenchmarkGpuLbm LbCodeGenerationExample UniformGridBenchmarkGPU_trt UniformGridBenchmarkGPU_entropic_kbc_n4 FluctuatingMRT
- cd apps/benchmarks/UniformGridGPU
- make -j $NUM_CORES
- cd ../UniformGridGenerated
- make -j $NUM_CORES
tags:
- docker
- cuda11
- AVX
# -------------------- Linter & Documentation --------------------------------------------------------------------------
......@@ -94,31 +160,30 @@ flake8-lint:
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/software/pystencils/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script:
- flake8 pystencils
tags:
- docker
- cuda
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/software/pystencils/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
- sphinx-build -b html doc html_doc
- sphinx-build -W -b html doc html_doc
tags:
- docker
- cuda
artifacts:
paths:
- html_doc
pages:
image: i10git.cs.fau.de:5005/software/pystencils/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
stage: deploy
script:
- ls -l
......
[settings]
line_length=100
balanced_wrapping=True
multi_line_output=4
known_third_party=sympy
Contributors:
-------------
- Martin Bauer <martin.bauer@fau.de>
- Stephan Seitz <stephan.seitz@fau.de>
- Jan Hönig <jan.hoenig@fau.de>
- Nils Kohl <nils.kohl@fau.de>
- Julian Hammer <julian.hammer@fau.de>
- Christian Godenschwager <christian.godenschwager@fau.de>
- Markus Holzer <markus.holzer@fau.de>
- Michael Kuron <mkuron@icp.uni-stuttgart.de>
- Dominik Ernst <dominik.ernst@fau.de>
- João Victor Tozatti Risso <joaovictortr@protonmail.com>
include README.md
include COPYING.txt
include RELEASE-VERSION
global-include *.pyx
global-exclude boundaries/createindexlistcython.c
pystencils
==========
[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mabau/pystencils/master?filepath=doc%2Fnotebooks)
[![Docs](https://img.shields.io/badge/read-the_docs-brightgreen.svg)](https://pycodegen.pages.i10git.cs.fau.de/pystencils)
[![pypi-package](https://badge.fury.io/py/pystencils.svg)](https://badge.fury.io/py/pystencils)
[![pipeline status](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/pipeline.svg)](https://i10git.cs.fau.de/pycodegen/pystencils/commits/master)
[![coverage report](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/coverage.svg)](https://i10git.cs.fau.de/pycodegen/pystencils/commits/master)
[coverage report](http://pycodegen.pages.walberla.net/pystencils/coverage_report)
[![coverage report](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/coverage.svg)](http://pycodegen.pages.i10git.cs.fau.de/pystencils/coverage_report)
Run blazingly fast stencil codes on numpy arrays.
*pystencils* uses sympy to define stencil operations, that can be executed on numpy array.
It runs faster than normal numpy code and even as Cython and numba.
*pystencils* uses sympy to define stencil operations, that can be executed on numpy arrays.
Exploiting the stencil structure makes *pystencils* run faster than normal numpy code and even as Cython and numba,
[as demonstrated in this notebook](https://pycodegen.pages.i10git.cs.fau.de/pystencils/notebooks/demo_benchmark.html).
Here is a code snippet that computes the average of neighboring cells:
```python
......@@ -17,7 +21,7 @@ import numpy as np
f, g = ps.fields("f, g : [2D]")
stencil = ps.Assignment(g[0, 0],
(f[1, 0] + f[-1, 0] + f[0, 1] + f[0, -1]) / 4)
(f[1, 0] + f[-1, 0] + f[0, 1] + f[0, -1]) / 4)
kernel = ps.create_kernel(stencil).compile()
f_arr = np.random.rand(1000, 1000)
......@@ -35,9 +39,6 @@ discretize = ps.fd.Discretization2ndOrder(dx=1, dt=0.01)
discretization = discretize(adv_diff_pde)
```
Look at the [documentation](http://pycodegen.pages.walberla.net/pystencils) to learn more.
Installation
------------
......@@ -48,10 +49,12 @@ pip install pystencils[interactive]
Without `[interactive]` you get a minimal version with very little dependencies.
All options:
- `gpu`: use this if nVidia GPU is available and CUDA is installed
- `gpu`: use this if an NVIDIA GPU is available and CUDA is installed
- `opencl`: basic OpenCL support (experimental)
- `alltrafos`: pulls in additional dependencies for loop simplification e.g. libisl
- `bench_db`: functionality to store benchmark result in object databases
- `interactive`: installs dependencies to work in Jupyter including image I/O, plotting etc.
- `autodiff`: enable derivation of adjoint kernels and generation of Torch/Tensorflow operations
- `doc`: packages to build documentation
Options can be combined e.g.
......@@ -63,5 +66,5 @@ pip install pystencils[interactive,gpu,doc]
Documentation
-------------
Read the docs [here](http://pycodegen.pages.walberla.net/pystencils) and
Read the docs [here](https://pycodegen.pages.i10git.cs.fau.de/pystencils) and
check out the Jupyter notebooks in `doc/notebooks`.
build-essential
graphviz
ffmpeg
# ----------------------------------------------------------------------------------------------------------------------
# Environment with all dependencies to use pystencils
#
#
# Download conda at https://conda.io/miniconda.html and create this environment by running:
# conda env create -f conda_environment_user.yml
# . activate pystencils
#
# If you have CUDA installed and want to use your GPU, uncomment the last line to install pycuda
#
# ----------------------------------------------------------------------------------------------------------------------
name: pystencils
dependencies:
# Basic dependencies:
- python >= 3.6
- numpy
- sympy >= 1.1
- appdirs # to find default cache directory on each platform
- joblib # caching on hard-disk, this is optional, but if not installed lbmpy is really slow
- cython # speed up boundary list computation (optional)
- matplotlib
- imageio
- pandas
- scipy
- pip
- pip:
- islpy # used to optimize staggered kernels
- py-cpuinfo # get cpu info like cache sizes, supported vector instruction sets, ...
- graphviz # can show abstract syntax trees as formatted graphs
- ipy_table # HTML tables for jupyter notebooks
- pyevtk # VTK output for serial simulations
- blitzdb # file-based No-SQL database to store simulation results
#- pycuda # add this if you have CUDA installed
#!/bin/bash
python3 setup.py develop
exec "$@"
\ No newline at end of file
import os
import pytest
import tempfile
import runpy
import sys
import tempfile
import warnings
import nbformat
import pytest
from nbconvert import PythonExporter
from pystencils.boundaries.createindexlistcython import * # NOQA
# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
# at the same time
from pystencils.cpu import cpujit
......@@ -14,12 +20,17 @@ try:
pyximport.install(language_level=3)
except ImportError:
pass
from pystencils.boundaries.createindexlistcython import * # NOQA
SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.abspath('pystencils'))
# the Ubuntu pipeline uses an older version of pytest which uses deprecated functionality.
# This leads to many warinings in the test and coverage pipeline.
pytest_numeric_version = [int(x, 10) for x in pytest.__version__.split('.')]
pytest_numeric_version.reverse()
pytest_version = sum(x * (100 ** i) for i, x in enumerate(pytest_numeric_version))
def add_path_to_ignore(path):
if not os.path.exists(path):
......@@ -28,34 +39,68 @@ def add_path_to_ignore(path):
collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py")]
collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py"),
os.path.join(SCRIPT_FOLDER, "pystencils", "opencl", "opencl.autoinit")]
add_path_to_ignore('pystencils_tests/benchmark')
add_path_to_ignore('_local_tmp')
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/autodiff.py")]
try:
import pycuda
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/pystencils_tests/test_cudagpu.py")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_cudagpu.py")]
add_path_to_ignore('pystencils/gpucuda')
try:
import llvmlite
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/backends/llvm.py')]
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/test_basic_usage_llvm.ipynb')]
add_path_to_ignore('pystencils/llvm')
try:
import kerncraft
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py"),
os.path.join(SCRIPT_FOLDER, "pystencils_tests/benchmark/benchmark.py")]
add_path_to_ignore('pystencils/kerncraft_coupling')
try:
import waLBerla
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_aligned_array.py"),
os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_datahandling_parallel.py"),
os.path.join(SCRIPT_FOLDER, "doc/notebooks/03_tutorial_datahandling.ipynb"),
os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/parallel_datahandling.py"),
os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_small_block_benchmark.ipynb")]
try:
import blitzdb
except ImportError:
add_path_to_ignore('pystencils/runhelper')
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_parameterstudy.py")]
try:
import islpy
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/integer_set_analysis.py")]
try:
import graphviz
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/backends/dot.py")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, "doc/notebooks/01_tutorial_getting_started.ipynb")]
try:
import pyevtk
except ImportError:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/vtk.py")]
# TODO: Remove if Ubuntu 18.04 is no longer supported
if pytest_version < 50403:
collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_jupyter_extensions.ipynb")]
collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
......@@ -65,10 +110,6 @@ for root, sub_dirs, files in os.walk('.'):
collect_ignore.append(f)
import nbformat
from nbconvert import PythonExporter
class IPythonMockup:
def run_line_magic(self, *args, **kwargs):
pass
......@@ -89,6 +130,7 @@ class IPyNbTest(pytest.Item):
self.code = code
self.add_marker('notebook')
@pytest.mark.filterwarnings("ignore:IPython.core.inputsplitter is deprecated")
def runtest(self):
global_dict = {'get_ipython': lambda: IPythonMockup(),
'is_test_run': True}
......@@ -114,10 +156,16 @@ class IPyNbFile(pytest.File):
exporter.exclude_markdown = True
exporter.exclude_input_prompt = True
notebook_contents = self.fspath.open()
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
yield IPyNbTest(self.name, self, code)
notebook_contents = self.fspath.open(encoding='utf-8')
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "IPython.core.inputsplitter is deprecated")
notebook = nbformat.read(notebook_contents, 4)
code, _ = exporter.from_notebook_node(notebook)
if pytest_version >= 50403:
yield IPyNbTest.from_parent(name=self.name, parent=self, code=code)
else:
yield IPyNbTest(self.name, self, code)
def teardown(self):
pass
......@@ -126,4 +174,7 @@ class IPyNbFile(pytest.File):
def pytest_collect_file(path, parent):
glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
if any(path.fnmatch(g) for g in glob_exprs):
return IPyNbFile(path, parent)
if pytest_version >= 50403:
return IPyNbFile.from_parent(fspath=path, parent=parent)
else:
return IPyNbFile(path, parent)
......@@ -54,6 +54,7 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
}
autodoc_member_order = 'bysource'
bibtex_bibfiles = ['sphinx/pystencils.bib']
project = 'pystencils'
html_logo = "img/logo.png"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="379.82614mm"
height="189.91307mm"
viewBox="0 0 1345.8407 672.92033"
id="svg2"
version="1.1"
inkscape:version="0.92.3 (2405546, 2018-03-11)"
sodipodi:docname="logo_large.svg"
inkscape:export-filename="/home/martin/code/pycodegen/pystencils/doc/img/github_repo_card.png"
inkscape:export-xdpi="85.599998"
inkscape:export-ydpi="85.599998">
<defs
id="defs4">
<inkscape:path-effect
effect="spiro"
id="path-effect4188"
is_visible="true" />
<inkscape:path-effect
effect="spiro"
id="path-effect4188-5"
is_visible="true" />
<filter
y="-0.25"
height="1.5"
inkscape:menu-tooltip="Darkens the edge with an inner blur and adds a flexible glow"
inkscape:menu="Shadows and Glows"
inkscape:label="Dark And Glow"
style="color-interpolation-filters:sRGB"
id="filter4596">
<feGaussianBlur
stdDeviation="5"
result="result6"
id="feGaussianBlur4598" />
<feComposite
result="result8"
in="SourceGraphic"
operator="atop"
in2="result6"
id="feComposite4600" />
<feComposite
result="result9"
operator="over"
in2="SourceAlpha"
in="result8"
id="feComposite4602" />
<feColorMatrix
values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 1 0 "
result="result10"
id="feColorMatrix4604" />
<feBlend
in="result10"
mode="normal"
in2="result6"
id="feBlend4606" />
</filter>
<filter
y="-0.25"
height="1.5"
inkscape:menu-tooltip="Darkens the edge with an inner blur and adds a flexible glow"
inkscape:menu="Shadows and Glows"
inkscape:label="Dark And Glow"
style="color-interpolation-filters:sRGB"
id="filter4608">
<feGaussianBlur
stdDeviation="5"
result="result6"
id="feGaussianBlur4610" />
<feComposite
result="result8"
in="SourceGraphic"
operator="atop"
in2="result6"
id="feComposite4612" />
<feComposite
result="result9"
operator="over"
in2="SourceAlpha"
in="result8"
id="feComposite4614" />
<feColorMatrix
values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 1 0 "
result="result10"
id="feColorMatrix4616" />
<feBlend
in="result10"
mode="normal"
in2="result6"
id="feBlend4618" />
</filter>
<filter
y="-0.25"
height="1.5"
inkscape:menu-tooltip="Darkens the edge with an inner blur and adds a flexible glow"
inkscape:menu="Shadows and Glows"
inkscape:label="Dark And Glow"
style="color-interpolation-filters:sRGB"
id="filter4620">
<feGaussianBlur
stdDeviation="5"
result="result6"
id="feGaussianBlur4622" />
<feComposite
result="result8"
in="SourceGraphic"
operator="atop"
in2="result6"
id="feComposite4624" />
<feComposite
result="result9"
operator="over"
in2="SourceAlpha"
in="result8"
id="feComposite4626" />
<feColorMatrix
values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 1 0 "
result="result10"
id="feColorMatrix4628" />
<feBlend