Commit 9bad685f authored by Helen Schottenhamml's avatar Helen Schottenhamml
Browse files

Merge branch 'Fixes' into 'master'

Clean up and Bug Fixes

Closes #38

See merge request !264
parents a3cec2ce 6f74f2ab
Pipeline #35136 failed with stages
in 23 minutes and 26 seconds
......@@ -19,3 +19,7 @@ pystencils/boundaries/createindexlistcython.*.so
pystencils_tests/tmp
pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
# macOS
**/.DS_Store
\ No newline at end of file
......@@ -14,6 +14,7 @@ tests-and-coverage:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script:
- pip install sympy --upgrade
- env
- pip list
- export NUM_CORES=$(nproc --all)
......@@ -94,6 +95,7 @@ minimal-windows:
- export NUM_CORES=$(nproc --all)
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils
- pip install joblib
- pip list
- python -c "import numpy"
- py.test -v -m "not (notebook or longrun)"
......@@ -105,9 +107,9 @@ ubuntu:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
before_script:
- apt-get -y remove python3-sympy
# - apt-get -y remove python3-sympy
- ln -s /usr/include/locale.h /usr/include/xlocale.h
- pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
# - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
script:
- export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
......@@ -293,7 +295,7 @@ flake8-lint:
build-documentation:
stage: test
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
script:
- export PYTHONPATH=`pwd`
- mkdir html_doc
......
......@@ -322,6 +322,8 @@ class CBackend:
offset = sp.Add(*[sp.Symbol(LoopOverCoordinate.get_loop_counter_name(i))
* node.lhs.args[0].field.spatial_strides[i] for i in
range(len(node.lhs.args[0].field.spatial_strides))])
if stride == 1:
offset = offset.subs({node.lhs.args[0].field.spatial_strides[0]: 1})
size = sp.Mul(*node.lhs.args[0].field.spatial_shape)
element_size = 8 if data_type.base_type.base_name == 'double' else 4
size_cond = f"({offset} + {CachelineSize.symbol/element_size}) < {size}"
......
......@@ -8,22 +8,15 @@ try:
except ImportError:
from backports.functools_lru_cache import lru_cache as memorycache
from joblib import Memory
from appdirs import user_cache_dir
try:
from joblib import Memory
from appdirs import user_cache_dir
if 'PYSTENCILS_CACHE_DIR' in os.environ:
cache_dir = os.environ['PYSTENCILS_CACHE_DIR']
else:
cache_dir = user_cache_dir('pystencils')
disk_cache = Memory(cache_dir, verbose=False).cache
disk_cache_no_fallback = disk_cache
except ImportError:
# fallback to in-memory caching if joblib is not available
disk_cache = memorycache(maxsize=64)
def disk_cache_no_fallback(o):
return o
if 'PYSTENCILS_CACHE_DIR' in os.environ:
cache_dir = os.environ['PYSTENCILS_CACHE_DIR']
else:
cache_dir = user_cache_dir('pystencils')
disk_cache = Memory(cache_dir, verbose=False).cache
disk_cache_no_fallback = disk_cache
def _wrapper(wrapped_func, cached_func, *args, **kwargs):
......@@ -34,7 +27,6 @@ def _wrapper(wrapped_func, cached_func, *args, **kwargs):
def memorycache_if_hashable(maxsize=128, typed=False):
def wrapper(func):
return partial(_wrapper, func, memorycache(maxsize, typed)(func))
......
......@@ -75,6 +75,7 @@ def make_python_function(kernel_function_node, custom_backend=None):
- all symbols which are not defined in the kernel itself are expected as parameters
:param kernel_function_node: the abstract syntax tree
:param custom_backend: use own custom printer for code generation
:return: kernel functor
"""
result = compile_and_load(kernel_function_node, custom_backend)
......@@ -183,6 +184,10 @@ def read_config():
if os.path.exists(libomp):
default_compiler_config['flags'] += ' ' + libomp
break
else:
raise ValueError("The detection of the platform with platform.system() did not work. "
"Pystencils is only supported for linux, windows, and darwin platforms.")
default_cache_config = OrderedDict([
('object_cache', os.path.join(user_cache_dir('pystencils'), 'objectcache')),
('clear_cache_on_start', False),
......@@ -205,19 +210,19 @@ def read_config():
if config['cache']['object_cache'] is not False:
config['cache']['object_cache'] = os.path.expanduser(config['cache']['object_cache']).format(pid=os.getpid())
clear_cache = False
clear_cache_on_start = False
cache_status_file = os.path.join(config['cache']['object_cache'], 'last_config.json')
if os.path.exists(cache_status_file):
# check if compiler config has changed
last_config = json.load(open(cache_status_file, 'r'))
if set(last_config.items()) != set(config['compiler'].items()):
clear_cache = True
clear_cache_on_start = True
else:
for key in last_config.keys():
if last_config[key] != config['compiler'][key]:
clear_cache = True
clear_cache_on_start = True
if config['cache']['clear_cache_on_start'] or clear_cache:
if config['cache']['clear_cache_on_start'] or clear_cache_on_start:
shutil.rmtree(config['cache']['object_cache'], ignore_errors=True)
create_folder(config['cache']['object_cache'], False)
......@@ -578,7 +583,10 @@ class ExtensionModuleCode:
print(self._code_string, file=file)
def compile_module(code, code_hash, base_dir, compile_flags=[]):
def compile_module(code, code_hash, base_dir, compile_flags=None):
if compile_flags is None:
compile_flags = []
compiler_config = get_compiler_config()
extra_flags = ['-I' + get_paths()['include'], '-I' + get_pystencils_include_path()] + compile_flags
......
......@@ -56,7 +56,7 @@ class CachelineSize(ast.Node):
@property
def symbols_defined(self):
return set([self.symbol, self.mask_symbol, self.last_symbol])
return {self.symbol, self.mask_symbol, self.last_symbol}
@property
def undefined_symbols(self):
......
......@@ -450,17 +450,12 @@ def collate_types(types,
Uses the collation rules from numpy.
"""
if forbid_collation_to_complex:
types = [
t for t in types
if not np.issubdtype(t.numpy_dtype, np.complexfloating)
]
types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.complexfloating)]
if not types:
return create_type(default_float_type)
if forbid_collation_to_float:
types = [
t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)
]
types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)]
if not types:
return create_type(default_int_type)
......@@ -567,10 +562,17 @@ def get_type_of_expression(expr,
expr: sp.Expr
if expr.args:
types = tuple(get_type(a) for a in expr.args)
# collate_types checks numpy_dtype in the special cases
if any(not hasattr(t, 'numpy_dtype') for t in types):
forbid_collation_to_complex = False
forbid_collation_to_float = False
else:
forbid_collation_to_complex = expr.is_real is True
forbid_collation_to_float = expr.is_integer is True
return collate_types(
types,
forbid_collation_to_complex=expr.is_real is True,
forbid_collation_to_float=expr.is_integer is True,
forbid_collation_to_complex=forbid_collation_to_complex,
forbid_collation_to_float=forbid_collation_to_float,
default_float_type=default_float_type,
default_int_type=default_int_type)
else:
......
......@@ -21,6 +21,10 @@ class DataHandling(ABC):
_GPU_LIKE_BACKENDS = [Backend.CUDA, Backend.OPENCL]
# ---------------------------- Adding and accessing data -----------------------------------------------------------
@property
@abstractmethod
def default_target(self) -> Target:
"""Target Enum indicating the target of the computation"""
@property
@abstractmethod
......
......@@ -35,13 +35,13 @@ class ParallelDataHandling(DataHandling):
"""
super(ParallelDataHandling, self).__init__()
assert dim in (2, 3)
self.blocks = blocks
self.default_ghost_layers = default_ghost_layers
self.default_layout = default_layout
self._blocks = blocks
self._default_ghost_layers = default_ghost_layers
self._default_layout = default_layout
self._fields = DotDict() # maps name to symbolic pystencils field
self._field_name_to_cpu_data_name = {}
self._field_name_to_gpu_data_name = {}
self.data_names = set()
self._data_names = set()
self._dim = dim
self._fieldInformation = {}
self._cpu_gpu_pairs = []
......@@ -55,7 +55,11 @@ class ParallelDataHandling(DataHandling):
if self._dim == 2:
assert self.blocks.getDomainCellBB().size[2] == 1
self.default_target = default_target
self._default_target = default_target
@property
def default_target(self):
return self._default_target
@property
def dim(self):
......@@ -73,6 +77,22 @@ class ParallelDataHandling(DataHandling):
def fields(self):
return self._fields
@property
def blocks(self):
return self._blocks
@property
def default_ghost_layers(self):
return self._default_ghost_layers
@property
def default_layout(self):
return self._default_layout
@property
def data_names(self):
return self.data_names
def ghost_layers_of_field(self, name):
return self._fieldInformation[name]['ghost_layers']
......
......@@ -69,9 +69,13 @@ class SerialDataHandling(DataHandling):
self._periodicity = periodicity
self._field_information = {}
self.default_target = default_target
self._default_target = default_target
self._start_time = time.perf_counter()
@property
def default_target(self):
return self._default_target
@property
def dim(self):
return len(self._domainSize)
......
......@@ -261,7 +261,7 @@ class FiniteDifferenceStaggeredStencilDerivation:
main_points = [neighbor / 2, neighbor / -2, flipped(neighbor / 2, nonzero_indices[0]),
flipped(neighbor / -2, nonzero_indices[0])]
else:
main_points = [neighbor.multiply_elementwise(sp.Matrix(c) / 2)
main_points = [sp.Matrix(np.multiply(neighbor, sp.Matrix(c) / 2))
for c in itertools.product([-1, 1], repeat=3)]
points += main_points
zero_indices = [i for i, v in enumerate(neighbor) if v == 0 and i < dim]
......
......@@ -7,9 +7,6 @@ from IPython.display import HTML
import pystencils.plot as plt
__all__ = ['make_imshow_animation', 'display_animation', 'set_display_mode']
VIDEO_TAG = """<video controls width="80%">
<source src="data:video/x-m4v;base64,{0}" type="video/mp4">
Your browser does not support the video tag.
......
"""
Default Sympy optimizations applied in pystencils kernels using :func:`sympy.codegen.rewriting.optimize`.
See :func:`sympy.codegen.rewriting.optimize`.
"""
import itertools
from pystencils import Assignment
from pystencils.astnodes import SympyAssignment
try:
from sympy.codegen.rewriting import optims_c99, optimize
from sympy.codegen.rewriting import ReplaceOptim
HAS_REWRITING = True
# Evaluates all constant terms
evaluate_constant_terms = ReplaceOptim(
lambda e: hasattr(e, 'is_constant') and e.is_constant and not e.is_integer,
lambda p: p.evalf()
)
optims_pystencils_cpu = [evaluate_constant_terms] + list(optims_c99)
optims_pystencils_gpu = [evaluate_constant_terms] + list(optims_c99)
except ImportError:
from warnings import warn
warn("Could not import ReplaceOptim, optims_c99, optimize from sympy.codegen.rewriting."
"Please update your sympy installation!")
optims_c99 = []
optims_pystencils_cpu = []
optims_pystencils_gpu = []
HAS_REWRITING = False
def optimize_assignments(assignments, optimizations):
if HAS_REWRITING:
assignments = [Assignment(a.lhs, optimize(a.rhs, optimizations))
if hasattr(a, 'lhs')
else a for a in assignments]
assignments_nodes = [a.atoms(SympyAssignment) for a in assignments]
for a in itertools.chain.from_iterable(assignments_nodes):
a.optimize(optimizations)
return assignments
def optimize_ast(ast, optimizations):
if HAS_REWRITING:
assignments_nodes = ast.atoms(SympyAssignment)
for a in assignments_nodes:
a.optimize(optimizations)
return ast
......@@ -2,7 +2,7 @@ import numpy as np
import sympy as sp
import pystencils as ps
import pystencils.jupyter
from pystencils.jupyter import make_imshow_animation, display_animation, set_display_mode
import pystencils.plot as plt
__all__ = ['sp', 'np', 'ps', 'plt']
__all__ = ['sp', 'np', 'ps', 'plt', 'make_imshow_animation', 'display_animation', 'set_display_mode']
import pytest
from pystencils import create_data_handling
from pystencils.alignedarray import *
from pystencils.field import create_numpy_array_with_layout
......@@ -11,45 +13,45 @@ def is_aligned(arr, alignment, byte_offset=0):
return rest == 0
def test_1d_arrays():
for alignment in [8, 8*4, True]:
for shape in [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)]:
arrays = [
aligned_zeros(shape, alignment),
aligned_ones(shape, alignment),
aligned_empty(shape, alignment),
]
for arr in arrays:
assert is_aligned(arr, alignment)
@pytest.mark.parametrize("alignment", [8, 8*4, True])
@pytest.mark.parametrize("shape", [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)])
def test_1d_arrays(alignment, shape):
arrays = [
aligned_zeros(shape, alignment),
aligned_ones(shape, alignment),
aligned_empty(shape, alignment),
]
for arr in arrays:
assert is_aligned(arr, alignment)
def test_3d_arrays():
for order in ('C', 'F'):
for alignment in [8, 8*4, True]:
for shape in [(16, 16), (17, 17), (18, 18), (19, 19)]:
arrays = [
aligned_zeros(shape, alignment, order=order),
aligned_ones(shape, alignment, order=order),
aligned_empty(shape, alignment, order=order),
]
for arr in arrays:
assert is_aligned(arr, alignment)
if order == 'C':
assert is_aligned(arr[1], alignment)
assert is_aligned(arr[5], alignment)
else:
assert is_aligned(arr[..., 1], alignment)
assert is_aligned(arr[..., 5], alignment)
@pytest.mark.parametrize("order", ['C', 'F'])
@pytest.mark.parametrize("alignment", [8, 8*4, True])
@pytest.mark.parametrize("shape", [(16, 16), (17, 17), (18, 18), (19, 19)])
def test_3d_arrays(order, alignment, shape):
arrays = [
aligned_zeros(shape, alignment, order=order),
aligned_ones(shape, alignment, order=order),
aligned_empty(shape, alignment, order=order),
]
for arr in arrays:
assert is_aligned(arr, alignment)
if order == 'C':
assert is_aligned(arr[1], alignment)
assert is_aligned(arr[5], alignment)
else:
assert is_aligned(arr[..., 1], alignment)
assert is_aligned(arr[..., 5], alignment)
def test_data_handling():
for parallel in (False, True):
for tries in range(16): # try a few times, since we might get lucky and get randomly a correct alignment
dh = create_data_handling((6, 7), default_ghost_layers=1, parallel=parallel)
dh.add_array('test', alignment=8 * 4)
for b in dh.iterate(ghost_layers=True, inner_ghost_layers=True):
arr = b['test']
assert is_aligned(arr[1:, 3:], 8*4)
@pytest.mark.parametrize("parallel", [False, True])
def test_data_handling(parallel):
for tries in range(16): # try a few times, since we might get lucky and get randomly a correct alignment
dh = create_data_handling((6, 7), default_ghost_layers=1, parallel=parallel)
dh.add_array('test', alignment=8 * 4, values_per_cell=1)
for b in dh.iterate(ghost_layers=True, inner_ghost_layers=True):
arr = b['test']
assert is_aligned(arr[1:, 3:], 8*4)
def test_alignment_of_different_layouts():
......
import numpy as np
import sympy as sp
from pystencils import Field, Assignment, create_kernel
from pystencils.bit_masks import flag_cond
from pystencils import TypedSymbol
def test_flag_condition():
f_arr = np.zeros((2,2,2), dtype=np.float64)
mask_arr = np.zeros((2,2), dtype=np.uint64)
f_arr = np.zeros((2, 2, 2), dtype=np.float64)
mask_arr = np.zeros((2, 2), dtype=np.uint64)
mask_arr[0,1] = (1<<3)
mask_arr[1,0] = (1<<5)
mask_arr[1,1] = (1<<3) + (1 << 5)
mask_arr[0, 1] = (1 << 3)
mask_arr[1, 0] = (1 << 5)
mask_arr[1, 1] = (1 << 3) + (1 << 5)
f = Field.create_from_numpy_array('f', f_arr, index_dimensions=1)
mask = Field.create_from_numpy_array('mask', mask_arr)
......@@ -28,14 +26,14 @@ def test_flag_condition():
kernel = create_kernel(assignments).compile()
kernel(f=f_arr, mask=mask_arr)
reference = np.zeros((2,2,2), dtype=np.float64)
reference[0,1,0] = v1
reference[1,1,0] = v1
reference = np.zeros((2, 2, 2), dtype=np.float64)
reference[0, 1, 0] = v1
reference[1, 1, 0] = v1
reference[0,0,1] = v3
reference[0,1,1] = v3
reference[0, 0, 1] = v3
reference[0, 1, 1] = v3
reference[1,0,1] = v2
reference[1,1,1] = v2
reference[1, 0, 1] = v2
reference[1, 1, 1] = v2
np.testing.assert_array_equal(f_arr, reference)
......@@ -4,13 +4,15 @@ import pystencils.boundaries.createindexlist as cil
import pytest
@pytest.mark.parametrize('single_link', [False, True])
@pytest.mark.skipif(not cil.cython_funcs_available, reason='Cython functions are not available')
def test_equivalence_cython_python_version(single_link):
# D2Q9
stencil_2d = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
stencil_2d = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
# D3Q19
stencil_3d = tuple((x,y,z) for x,y,z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
stencil_3d = tuple(
(x, y, z) for x, y, z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
for dtype in [int, np.int16, np.uint32]:
fluid_mask = dtype(1)
......@@ -40,13 +42,15 @@ def test_equivalence_cython_python_version(single_link):
np.testing.assert_equal(result_python_2d, result_cython_2d)
np.testing.assert_equal(result_python_3d, result_cython_3d)
@pytest.mark.parametrize('single_link', [False, True])
@pytest.mark.skipif(not cil.cython_funcs_available, reason='Cython functions are not available')
def test_equivalence_cell_idx_list_cython_python_version(single_link):
# D2Q9
stencil_2d = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
stencil_2d = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
# D3Q19
stencil_3d = tuple((x,y,z) for x,y,z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
stencil_3d = tuple(
(x, y, z) for x, y, z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
for dtype in [int, np.int16, np.uint32]:
fluid_mask = dtype(1)
......@@ -76,9 +80,10 @@ def test_equivalence_cell_idx_list_cython_python_version(single_link):
np.testing.assert_equal(result_python_2d, result_cython_2d)
np.testing.assert_equal(result_python_3d, result_cython_3d)
@pytest.mark.parametrize('inner_or_boundary', [False, True])
def test_normal_calculation(inner_or_boundary):
stencil = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
stencil = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
domain_size = (32, 32)
dtype = np.uint32
fluid_mask = dtype(1)
......
import pytest
import pystencils as ps
import numpy as np
# This test aims to trigger deprication warnings. Thus the warnings should not be displayed in the warning summary.
def test_create_kernel_backwards_compatibility():
size = (30, 20)
......@@ -17,9 +20,11 @@ def test_create_kernel_backwards_compatibility():
jacobi = ps.Assignment(d[0, 0], (f[1, 0] + f[-1, 0] + f[0, 1] + f[0, -1]) / 4)
ast_enum = ps.create_kernel(jacobi, target=ps.Target.CPU).compile()
ast_string = ps.create_kernel(jacobi, target='cpu').compile()
with pytest.warns(DeprecationWarning):
ast_string = ps.create_kernel(jacobi, target='cpu').compile()
# noinspection PyTypeChecker
ast_config = ps.create_kernel(jacobi, config=ps.CreateKernelConfig(target='cpu')).compile()
with pytest.warns(DeprecationWarning):
ast_config = ps.create_kernel(jacobi, config=ps.CreateKernelConfig(target='cpu')).compile()
ast_enum(f=src_field_enum, d=dst_field_enum)
ast_string(f=src_field_string, d=dst_field_string)
ast_config(f=src_field_config, d=dst_field_config)
......