Merge branch 'Fixes' into 'master'

Clean up and Bug Fixes

Closes #38

See merge request pycodegen/pystencils!264

.gitignore

@@ -19,3 +19,7 @@ pystencils/boundaries/createindexlistcython.*.so
 pystencils_tests/tmp
 pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
 pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
+
+
+# macOS
+**/.DS_Store
\ No newline at end of file

.gitlab-ci.yml

@@ -14,6 +14,7 @@ tests-and-coverage:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
   script:
+    - pip install sympy --upgrade
     - env
     - pip list
     - export NUM_CORES=$(nproc --all)
@@ -94,6 +95,7 @@ minimal-windows:
     - export NUM_CORES=$(nproc --all)
     - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
     - source activate pystencils
+    - pip install joblib
     - pip list
     - python -c "import numpy"
     - py.test -v -m "not (notebook or longrun)"
@@ -105,9 +107,9 @@ ubuntu:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
   before_script:
-    - apt-get -y remove python3-sympy
+    # - apt-get -y remove python3-sympy
     - ln -s /usr/include/locale.h /usr/include/xlocale.h
-    - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
+    # - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
   script:
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
@@ -293,7 +295,7 @@ flake8-lint:
 
 build-documentation:
   stage: test
-  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
+  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
   script:
     - export PYTHONPATH=`pwd`
     - mkdir html_doc

pystencils/backends/cbackend.py

@@ -322,6 +322,8 @@ class CBackend:
                     offset = sp.Add(*[sp.Symbol(LoopOverCoordinate.get_loop_counter_name(i))
                                       * node.lhs.args[0].field.spatial_strides[i] for i in
                                       range(len(node.lhs.args[0].field.spatial_strides))])
+                    if stride == 1:
+                        offset = offset.subs({node.lhs.args[0].field.spatial_strides[0]: 1})
                     size = sp.Mul(*node.lhs.args[0].field.spatial_shape)
                     element_size = 8 if data_type.base_type.base_name == 'double' else 4
                     size_cond = f"({offset} + {CachelineSize.symbol/element_size}) < {size}"

pystencils/cache.py

@@ -8,22 +8,15 @@ try:
 except ImportError:
     from backports.functools_lru_cache import lru_cache as memorycache
 
+from joblib import Memory
+from appdirs import user_cache_dir
 
-try:
-    from joblib import Memory
-    from appdirs import user_cache_dir
-    if 'PYSTENCILS_CACHE_DIR' in os.environ:
+if 'PYSTENCILS_CACHE_DIR' in os.environ:
     cache_dir = os.environ['PYSTENCILS_CACHE_DIR']
-    else:
+else:
     cache_dir = user_cache_dir('pystencils')
-    disk_cache = Memory(cache_dir, verbose=False).cache
-    disk_cache_no_fallback = disk_cache
-except ImportError:
-    # fallback to in-memory caching if joblib is not available
-    disk_cache = memorycache(maxsize=64)
-
-    def disk_cache_no_fallback(o):
-        return o
+disk_cache = Memory(cache_dir, verbose=False).cache
+disk_cache_no_fallback = disk_cache
 
 
 def _wrapper(wrapped_func, cached_func, *args, **kwargs):
@@ -34,7 +27,6 @@ def _wrapper(wrapped_func, cached_func, *args, **kwargs):
 
 
 def memorycache_if_hashable(maxsize=128, typed=False):
-
     def wrapper(func):
         return partial(_wrapper, func, memorycache(maxsize, typed)(func))
 

pystencils/cpu/cpujit.py

@@ -75,6 +75,7 @@ def make_python_function(kernel_function_node, custom_backend=None):
         - all symbols which are not defined in the kernel itself are expected as parameters
 
     :param kernel_function_node: the abstract syntax tree
+    :param custom_backend: use own custom printer for code generation
     :return: kernel functor
     """
     result = compile_and_load(kernel_function_node, custom_backend)
@@ -183,6 +184,10 @@ def read_config():
             if os.path.exists(libomp):
                 default_compiler_config['flags'] += ' ' + libomp
                 break
+    else:
+        raise ValueError("The detection of the platform with platform.system() did not work. "
+                         "Pystencils is only supported for linux, windows, and darwin platforms.")
+
     default_cache_config = OrderedDict([
         ('object_cache', os.path.join(user_cache_dir('pystencils'), 'objectcache')),
         ('clear_cache_on_start', False),
@@ -205,19 +210,19 @@ def read_config():
     if config['cache']['object_cache'] is not False:
         config['cache']['object_cache'] = os.path.expanduser(config['cache']['object_cache']).format(pid=os.getpid())
 
-        clear_cache = False
+        clear_cache_on_start = False
         cache_status_file = os.path.join(config['cache']['object_cache'], 'last_config.json')
         if os.path.exists(cache_status_file):
             # check if compiler config has changed
             last_config = json.load(open(cache_status_file, 'r'))
             if set(last_config.items()) != set(config['compiler'].items()):
-                clear_cache = True
+                clear_cache_on_start = True
             else:
                 for key in last_config.keys():
                     if last_config[key] != config['compiler'][key]:
-                        clear_cache = True
+                        clear_cache_on_start = True
 
-        if config['cache']['clear_cache_on_start'] or clear_cache:
+        if config['cache']['clear_cache_on_start'] or clear_cache_on_start:
             shutil.rmtree(config['cache']['object_cache'], ignore_errors=True)
 
         create_folder(config['cache']['object_cache'], False)
@@ -578,7 +583,10 @@ class ExtensionModuleCode:
         print(self._code_string, file=file)
 
 
-def compile_module(code, code_hash, base_dir, compile_flags=[]):
+def compile_module(code, code_hash, base_dir, compile_flags=None):
+    if compile_flags is None:
+        compile_flags = []
+
     compiler_config = get_compiler_config()
     extra_flags = ['-I' + get_paths()['include'], '-I' + get_pystencils_include_path()] + compile_flags
 

pystencils/cpu/vectorization.py

@@ -56,7 +56,7 @@ class CachelineSize(ast.Node):
 
     @property
     def symbols_defined(self):
-        return set([self.symbol, self.mask_symbol, self.last_symbol])
+        return {self.symbol, self.mask_symbol, self.last_symbol}
 
     @property
     def undefined_symbols(self):

pystencils/data_types.py

@@ -450,17 +450,12 @@ def collate_types(types,
     Uses the collation rules from numpy.
     """
     if forbid_collation_to_complex:
-        types = [
-            t for t in types
-            if not np.issubdtype(t.numpy_dtype, np.complexfloating)
-        ]
+        types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.complexfloating)]
         if not types:
             return create_type(default_float_type)
 
     if forbid_collation_to_float:
-        types = [
-            t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)
-        ]
+        types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)]
         if not types:
             return create_type(default_int_type)
 
@@ -567,10 +562,17 @@ def get_type_of_expression(expr,
         expr: sp.Expr
         if expr.args:
             types = tuple(get_type(a) for a in expr.args)
+            # collate_types checks numpy_dtype in the special cases
+            if any(not hasattr(t, 'numpy_dtype') for t in types):
+                forbid_collation_to_complex = False
+                forbid_collation_to_float = False
+            else:
+                forbid_collation_to_complex = expr.is_real is True
+                forbid_collation_to_float = expr.is_integer is True
             return collate_types(
                 types,
-                forbid_collation_to_complex=expr.is_real is True,
-                forbid_collation_to_float=expr.is_integer is True,
+                forbid_collation_to_complex=forbid_collation_to_complex,
+                forbid_collation_to_float=forbid_collation_to_float,
                 default_float_type=default_float_type,
                 default_int_type=default_int_type)
         else:

pystencils/datahandling/datahandling_interface.py

@@ -21,6 +21,10 @@ class DataHandling(ABC):
     _GPU_LIKE_BACKENDS = [Backend.CUDA, Backend.OPENCL]
 
     # ---------------------------- Adding and accessing data -----------------------------------------------------------
+    @property
+    @abstractmethod
+    def default_target(self) -> Target:
+        """Target Enum indicating the target of the computation"""
 
     @property
     @abstractmethod

pystencils/datahandling/parallel_datahandling.py

@@ -35,13 +35,13 @@ class ParallelDataHandling(DataHandling):
         """
         super(ParallelDataHandling, self).__init__()
         assert dim in (2, 3)
-        self.blocks = blocks
-        self.default_ghost_layers = default_ghost_layers
-        self.default_layout = default_layout
+        self._blocks = blocks
+        self._default_ghost_layers = default_ghost_layers
+        self._default_layout = default_layout
         self._fields = DotDict()  # maps name to symbolic pystencils field
         self._field_name_to_cpu_data_name = {}
         self._field_name_to_gpu_data_name = {}
-        self.data_names = set()
+        self._data_names = set()
         self._dim = dim
         self._fieldInformation = {}
         self._cpu_gpu_pairs = []
@@ -55,7 +55,11 @@ class ParallelDataHandling(DataHandling):
 
         if self._dim == 2:
             assert self.blocks.getDomainCellBB().size[2] == 1
-        self.default_target = default_target
+        self._default_target = default_target
+
+    @property
+    def default_target(self):
+        return self._default_target
 
     @property
     def dim(self):
@@ -73,6 +77,22 @@ class ParallelDataHandling(DataHandling):
     def fields(self):
         return self._fields
 
+    @property
+    def blocks(self):
+        return self._blocks
+
+    @property
+    def default_ghost_layers(self):
+        return self._default_ghost_layers
+
+    @property
+    def default_layout(self):
+        return self._default_layout
+
+    @property
+    def data_names(self):
+        return self.data_names
+
     def ghost_layers_of_field(self, name):
         return self._fieldInformation[name]['ghost_layers']
 

pystencils/datahandling/serial_datahandling.py

@@ -69,9 +69,13 @@ class SerialDataHandling(DataHandling):
 
         self._periodicity = periodicity
         self._field_information = {}
-        self.default_target = default_target
+        self._default_target = default_target
         self._start_time = time.perf_counter()
 
+    @property
+    def default_target(self):
+        return self._default_target
+
     @property
     def dim(self):
         return len(self._domainSize)

pystencils/fd/derivation.py

@@ -261,7 +261,7 @@ class FiniteDifferenceStaggeredStencilDerivation:
             main_points = [neighbor / 2, neighbor / -2, flipped(neighbor / 2, nonzero_indices[0]),
                            flipped(neighbor / -2, nonzero_indices[0])]
         else:
-            main_points = [neighbor.multiply_elementwise(sp.Matrix(c) / 2)
+            main_points = [sp.Matrix(np.multiply(neighbor, sp.Matrix(c) / 2))
                            for c in itertools.product([-1, 1], repeat=3)]
         points += main_points
         zero_indices = [i for i, v in enumerate(neighbor) if v == 0 and i < dim]

pystencils/jupyter.py

@@ -7,9 +7,6 @@ from IPython.display import HTML
 
 import pystencils.plot as plt
 
-__all__ = ['make_imshow_animation', 'display_animation', 'set_display_mode']
-
-
 VIDEO_TAG = """<video controls width="80%">
  <source src="data:video/x-m4v;base64,{0}" type="video/mp4">
  Your browser does not support the video tag.

pystencils/math_optimizations.py

-"""
-Default Sympy optimizations applied in pystencils kernels using :func:`sympy.codegen.rewriting.optimize`.
-
-See :func:`sympy.codegen.rewriting.optimize`.
-"""
-
-
-import itertools
-
-from pystencils import Assignment
-from pystencils.astnodes import SympyAssignment
-
-try:
-    from sympy.codegen.rewriting import optims_c99, optimize
-    from sympy.codegen.rewriting import ReplaceOptim
-    HAS_REWRITING = True
-
-    # Evaluates all constant terms
-    evaluate_constant_terms = ReplaceOptim(
-        lambda e: hasattr(e, 'is_constant') and e.is_constant and not e.is_integer,
-        lambda p: p.evalf()
-    )
-
-    optims_pystencils_cpu = [evaluate_constant_terms] + list(optims_c99)
-    optims_pystencils_gpu = [evaluate_constant_terms] + list(optims_c99)
-except ImportError:
-    from warnings import warn
-    warn("Could not import ReplaceOptim, optims_c99, optimize from sympy.codegen.rewriting."
-         "Please update your sympy installation!")
-    optims_c99 = []
-    optims_pystencils_cpu = []
-    optims_pystencils_gpu = []
-    HAS_REWRITING = False
-
-
-def optimize_assignments(assignments, optimizations):
-
-    if HAS_REWRITING:
-        assignments = [Assignment(a.lhs, optimize(a.rhs, optimizations))
-                       if hasattr(a, 'lhs')
-                       else a for a in assignments]
-        assignments_nodes = [a.atoms(SympyAssignment) for a in assignments]
-        for a in itertools.chain.from_iterable(assignments_nodes):
-            a.optimize(optimizations)
-
-    return assignments
-
-
-def optimize_ast(ast, optimizations):
-
-    if HAS_REWRITING:
-        assignments_nodes = ast.atoms(SympyAssignment)
-        for a in assignments_nodes:
-            a.optimize(optimizations)
-
-    return ast

pystencils/session.py

@@ -2,7 +2,7 @@ import numpy as np
 import sympy as sp
 
 import pystencils as ps
-import pystencils.jupyter
+from pystencils.jupyter import make_imshow_animation, display_animation, set_display_mode
 import pystencils.plot as plt
 
-__all__ = ['sp', 'np', 'ps', 'plt']
+__all__ = ['sp', 'np', 'ps', 'plt', 'make_imshow_animation', 'display_animation', 'set_display_mode']

pystencils_tests/test_aligned_array.py

+import pytest
+
 from pystencils import create_data_handling
 from pystencils.alignedarray import *
 from pystencils.field import create_numpy_array_with_layout
@@ -11,9 +13,9 @@ def is_aligned(arr, alignment, byte_offset=0):
     return rest == 0
 
 
-def test_1d_arrays():
-    for alignment in [8, 8*4, True]:
-        for shape in [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)]:
+@pytest.mark.parametrize("alignment", [8, 8*4, True])
+@pytest.mark.parametrize("shape", [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)])
+def test_1d_arrays(alignment, shape):
     arrays = [
         aligned_zeros(shape, alignment),
         aligned_ones(shape, alignment),
@@ -23,10 +25,10 @@ def test_1d_arrays():
         assert is_aligned(arr, alignment)
 
 
-def test_3d_arrays():
-    for order in ('C', 'F'):
-        for alignment in [8, 8*4, True]:
-            for shape in [(16, 16), (17, 17), (18, 18), (19, 19)]:
+@pytest.mark.parametrize("order", ['C', 'F'])
+@pytest.mark.parametrize("alignment", [8, 8*4, True])
+@pytest.mark.parametrize("shape", [(16, 16), (17, 17), (18, 18), (19, 19)])
+def test_3d_arrays(order, alignment, shape):
     arrays = [
         aligned_zeros(shape, alignment, order=order),
         aligned_ones(shape, alignment, order=order),
@@ -42,11 +44,11 @@ def test_3d_arrays():
             assert is_aligned(arr[..., 5], alignment)
 
 
-def test_data_handling():
-    for parallel in (False, True):
+@pytest.mark.parametrize("parallel", [False, True])
+def test_data_handling(parallel):
     for tries in range(16):  # try a few times, since we might get lucky and get randomly a correct alignment
         dh = create_data_handling((6, 7), default_ghost_layers=1, parallel=parallel)
-            dh.add_array('test', alignment=8 * 4)
+        dh.add_array('test', alignment=8 * 4, values_per_cell=1)
         for b in dh.iterate(ghost_layers=True, inner_ghost_layers=True):
             arr = b['test']
             assert is_aligned(arr[1:, 3:], 8*4)

pystencils_tests/test_bit_masks.py

 import numpy as np
-import sympy as sp
 from pystencils import Field, Assignment, create_kernel
 from pystencils.bit_masks import flag_cond
-from pystencils import TypedSymbol
 
 
 def test_flag_condition():
-    f_arr = np.zeros((2,2,2), dtype=np.float64)
-    mask_arr = np.zeros((2,2), dtype=np.uint64)
+    f_arr = np.zeros((2, 2, 2), dtype=np.float64)
+    mask_arr = np.zeros((2, 2), dtype=np.uint64)
 
-    mask_arr[0,1] = (1<<3)
-    mask_arr[1,0] = (1<<5)
-    mask_arr[1,1] = (1<<3) + (1 << 5)
+    mask_arr[0, 1] = (1 << 3)
+    mask_arr[1, 0] = (1 << 5)
+    mask_arr[1, 1] = (1 << 3) + (1 << 5)
 
     f = Field.create_from_numpy_array('f', f_arr, index_dimensions=1)
     mask = Field.create_from_numpy_array('mask', mask_arr)
@@ -28,14 +26,14 @@ def test_flag_condition():
     kernel = create_kernel(assignments).compile()
     kernel(f=f_arr, mask=mask_arr)
 
-    reference = np.zeros((2,2,2), dtype=np.float64)
-    reference[0,1,0] = v1
-    reference[1,1,0] = v1
+    reference = np.zeros((2, 2, 2), dtype=np.float64)
+    reference[0, 1, 0] = v1
+    reference[1, 1, 0] = v1
 
-    reference[0,0,1] = v3
-    reference[0,1,1] = v3
+    reference[0, 0, 1] = v3
+    reference[0, 1, 1] = v3
 
-    reference[1,0,1] = v2
-    reference[1,1,1] = v2
+    reference[1, 0, 1] = v2
+    reference[1, 1, 1] = v2
 
     np.testing.assert_array_equal(f_arr, reference)

pystencils_tests/test_boundary_indexlist_creation.py

@@ -4,13 +4,15 @@ import pystencils.boundaries.createindexlist as cil
 
 import pytest
 
+
 @pytest.mark.parametrize('single_link', [False, True])
 @pytest.mark.skipif(not cil.cython_funcs_available, reason='Cython functions are not available')
 def test_equivalence_cython_python_version(single_link):
     #   D2Q9
-    stencil_2d = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
+    stencil_2d = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
     #   D3Q19
-    stencil_3d = tuple((x,y,z) for x,y,z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
+    stencil_3d = tuple(
+        (x, y, z) for x, y, z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
 
     for dtype in [int, np.int16, np.uint32]:
         fluid_mask = dtype(1)
@@ -40,13 +42,15 @@ def test_equivalence_cython_python_version(single_link):
         np.testing.assert_equal(result_python_2d, result_cython_2d)
         np.testing.assert_equal(result_python_3d, result_cython_3d)
 
+
 @pytest.mark.parametrize('single_link', [False, True])
 @pytest.mark.skipif(not cil.cython_funcs_available, reason='Cython functions are not available')
 def test_equivalence_cell_idx_list_cython_python_version(single_link):
     #   D2Q9
-    stencil_2d = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
+    stencil_2d = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
     #   D3Q19
-    stencil_3d = tuple((x,y,z) for x,y,z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
+    stencil_3d = tuple(
+        (x, y, z) for x, y, z in product([-1, 0, 1], [-1, 0, 1], [-1, 0, 1]) if abs(x) + abs(y) + abs(z) < 3)
 
     for dtype in [int, np.int16, np.uint32]:
         fluid_mask = dtype(1)
@@ -76,9 +80,10 @@ def test_equivalence_cell_idx_list_cython_python_version(single_link):
         np.testing.assert_equal(result_python_2d, result_cython_2d)
         np.testing.assert_equal(result_python_3d, result_cython_3d)
 
+
 @pytest.mark.parametrize('inner_or_boundary', [False, True])
 def test_normal_calculation(inner_or_boundary):
-    stencil = tuple((x,y) for x,y in product([-1, 0, 1], [-1, 0, 1]))
+    stencil = tuple((x, y) for x, y in product([-1, 0, 1], [-1, 0, 1]))
     domain_size = (32, 32)
     dtype = np.uint32
     fluid_mask = dtype(1)

pystencils_tests/test_create_kernel_backwards_compability.py

+import pytest
+
 import pystencils as ps
 import numpy as np
 
 
+# This test aims to trigger deprication warnings. Thus the warnings should not be displayed in the warning summary.
 def test_create_kernel_backwards_compatibility():
     size = (30, 20)
 
@@ -17,8 +20,10 @@ def test_create_kernel_backwards_compatibility():
 
     jacobi = ps.Assignment(d[0, 0], (f[1, 0] + f[-1, 0] + f[0, 1] + f[0, -1]) / 4)
     ast_enum = ps.create_kernel(jacobi, target=ps.Target.CPU).compile()
+    with pytest.warns(DeprecationWarning):
         ast_string = ps.create_kernel(jacobi, target='cpu').compile()
     # noinspection PyTypeChecker
+    with pytest.warns(DeprecationWarning):
         ast_config = ps.create_kernel(jacobi, config=ps.CreateKernelConfig(target='cpu')).compile()
     ast_enum(f=src_field_enum, d=dst_field_enum)
     ast_string(f=src_field_string, d=dst_field_string)