Merge branch 'master' into 'eliminate-the-dialects'

# Conflicts:
#   pystencils/cpu/cpujit.py

.isort.cfg

+[settings]
+line_length=100
+balanced_wrapping=True
+multi_line_output=4

MANIFEST.in

 include README.md
 include COPYING.txt
+include RELEASE-VERSION

conftest.py

@@ -89,6 +89,7 @@ class IPyNbTest(pytest.Item):
         self.code = code
         self.add_marker('notebook')
 
+    @pytest.mark.filterwarnings("ignore:IPython.core.inputsplitter is deprecated")
     def runtest(self):
         global_dict = {'get_ipython': lambda: IPythonMockup(),
                        'is_test_run': True}

pystencils/__init__.py

 """Module to generate stencil kernels in C or CUDA using sympy expressions and call them as Python functions"""
 from . import sympy_gmpy_bug_workaround  # NOQA
-from .field import Field, FieldType, fields
+from . import fd
+from . import stencil as stencil
+from .assignment import Assignment, assignment_from_stencil
 from .data_types import TypedSymbol
-from .slicing import make_slice
-from .kernelcreation import create_kernel, create_indexed_kernel, create_staggered_kernel
+from .datahandling import create_data_handling
 from .display_utils import show_code, to_dot
+from .field import Field, FieldType, fields
+from .kernel_decorator import kernel
+from .kernelcreation import create_indexed_kernel, create_kernel, create_staggered_kernel
 from .simp import AssignmentCollection
-from .assignment import Assignment, assignment_from_stencil
+from .slicing import make_slice
 from .sympyextensions import SymbolCreator
-from .datahandling import create_data_handling
-from .kernel_decorator import kernel
-from . import fd
-from . import stencil as stencil
-
 
 __all__ = ['Field', 'FieldType', 'fields',
            'TypedSymbol',

pystencils/assignment.py

 # -*- coding: utf-8 -*-
+import numpy as np
 import sympy as sp
 from sympy.printing.latex import LatexPrinter
 
@@ -6,7 +7,6 @@ try:
     from sympy.codegen.ast import Assignment
 except ImportError:
     Assignment = None
-import numpy as np
 
 __all__ = ['Assignment', 'assignment_from_stencil']
 
@@ -84,7 +84,7 @@ def assignment_from_stencil(stencil_array, input_field, output_field,
         >>> assignment_from_stencil(stencil, f[1, 0], g[2, 0])
         Assignment(g_2E, 3*f_C + 6*f_SE + 4*f_E + 2*f_NE + 5*f_2E)
     """
-    from pystencils import Field
+    from pystencils.field import Field
 
     stencil_array = np.array(stencil_array)
     if order == 'visual':

pystencils/astnodes.py

@@ -6,8 +6,7 @@ import sympy as sp
 
 from pystencils.data_types import TypedSymbol, cast_func, create_type
 from pystencils.field import Field
-from pystencils.kernelparameters import (FieldPointerSymbol, FieldShapeSymbol,
-                                         FieldStrideSymbol)
+from pystencils.kernelparameters import FieldPointerSymbol, FieldShapeSymbol, FieldStrideSymbol
 from pystencils.sympyextensions import fast_subs
 
 NodeOrExpr = Union['Node', sp.Expr]
@@ -163,18 +162,30 @@ class KernelFunction(Node):
         def field_name(self):
             return self.fields[0].name
 
-    def __init__(self, body, ghost_layers=None, function_name="kernel", backend=""):
+    def __init__(self, body, target, backend, compile_function, ghost_layers, function_name="kernel"):
         super(KernelFunction, self).__init__()
         self._body = body
         body.parent = self
         self.function_name = function_name
         self._body.parent = self
-        self.compile = None
         self.ghost_layers = ghost_layers
+        self._target = target
+        self._backend = backend
         # these variables are assumed to be global, so no automatic parameter is generated for them
         self.global_variables = set()
-        self.backend = backend
         self.instruction_set = None  # used in `vectorize` function to tell the backend which i.s. (SSE,AVX) to use
+        # function that compiles the node to a Python callable, is set by the backends
+        self._compile_function = compile_function
+
+    @property
+    def target(self):
+        """Currently either 'cpu' or 'gpu' """
+        return self._target
+
+    @property
+    def backend(self):
+        """Backend for generating the code e.g. 'llvm', 'c', 'cuda' """
+        return self._backend
 
     @property
     def symbols_defined(self):
@@ -195,7 +206,7 @@ class KernelFunction(Node):
 
     @property
     def args(self):
-        return [self._body]
+        return self._body,
 
     @property
     def fields_accessed(self) -> Set['ResolvedFieldAccess']:
@@ -232,6 +243,11 @@ class KernelFunction(Node):
         params = [p.symbol for p in self.get_parameters()]
         return '{0} {1}({2})'.format(type(self).__name__, self.function_name, params)
 
+    def compile(self, *args, **kwargs):
+        if self._compile_function is None:
+            raise ValueError("No compile-function provided for this KernelFunction node")
+        return self._compile_function(self, *args, **kwargs)
+
 
 class SkipIteration(Node):
     @property

pystencils/backends/cbackend.py

@@ -6,18 +6,15 @@ import sympy as sp
 from sympy.core import S
 from sympy.printing.ccode import C89CodePrinter
 
-from pystencils.astnodes import (DestructuringBindingsForFieldClass,
-                                 KernelFunction, Node)
+from pystencils.astnodes import DestructuringBindingsForFieldClass, KernelFunction, Node
 from pystencils.cpu.vectorization import vec_all, vec_any
-from pystencils.data_types import (PointerType, VectorType, address_of,
-                                   cast_func, create_type,
-                                   get_type_of_expression,
-                                   reinterpret_cast_func, vector_memory_access)
-from pystencils.fast_approximation import (fast_division, fast_inv_sqrt,
-                                           fast_sqrt)
-from pystencils.integer_functions import (bit_shift_left, bit_shift_right,
-                                          bitwise_and, bitwise_or, bitwise_xor,
-                                          int_div, int_power_of_2, modulo_ceil)
+from pystencils.data_types import (
+    PointerType, VectorType, address_of, cast_func, create_type, get_type_of_expression,
+    reinterpret_cast_func, vector_memory_access)
+from pystencils.fast_approximation import fast_division, fast_inv_sqrt, fast_sqrt
+from pystencils.integer_functions import (
+    bit_shift_left, bit_shift_right, bitwise_and, bitwise_or, bitwise_xor,
+    int_div, int_power_of_2, modulo_ceil)
 from pystencils.kernelparameters import FieldPointerSymbol
 
 try:

pystencils/backends/dot.py

-from sympy.printing.printer import Printer
-from graphviz import Digraph, lang
 import graphviz
+from graphviz import Digraph, lang
+from sympy.printing.printer import Printer
 
 
 # noinspection PyPep8Naming

pystencils/boundaries/__init__.py

+from pystencils.boundaries.boundaryconditions import Dirichlet, Neumann
 from pystencils.boundaries.boundaryhandling import BoundaryHandling
-from pystencils.boundaries.boundaryconditions import Neumann, Dirichlet
 from pystencils.boundaries.inkernel import add_neumann_boundary
 
 __all__ = ['BoundaryHandling', 'Neumann', 'Dirichlet', 'add_neumann_boundary']

pystencils/boundaries/boundaryconditions.py

-from typing import List, Tuple, Any
+from typing import Any, List, Tuple
+
 from pystencils import Assignment
 from pystencils.boundaries.boundaryhandling import BoundaryOffsetInfo
 from pystencils.data_types import create_type

pystencils/boundaries/boundaryhandling.py

 import numpy as np
 import sympy as sp
+
+from pystencils import create_indexed_kernel
 from pystencils.assignment import Assignment
-from pystencils import Field, TypedSymbol, create_indexed_kernel
 from pystencils.backends.cbackend import CustomCodeNode
-from pystencils.boundaries.createindexlist import numpy_data_type_for_boundary_object, create_boundary_index_array
+from pystencils.boundaries.createindexlist import (
+    create_boundary_index_array, numpy_data_type_for_boundary_object)
 from pystencils.cache import memorycache
-from pystencils.data_types import create_type
+from pystencils.data_types import TypedSymbol, create_type
+from pystencils.field import Field
 from pystencils.kernelparameters import FieldPointerSymbol
 
 DEFAULT_FLAG_TYPE = np.uint32

pystencils/boundaries/createindexlist.py

-import numpy as np
 import itertools
 import warnings
 
-try:
-    import pyximport
+import numpy as np
 
-    pyximport.install(language_level=3)
+try:
+    # Try to import right away - assume compiled code is available
+    # compile with: python setup.py build_ext --inplace --use-cython
     from pystencils.boundaries.createindexlistcython import create_boundary_neighbor_index_list_2d, \
         create_boundary_neighbor_index_list_3d, create_boundary_cell_index_list_2d, create_boundary_cell_index_list_3d
 
     cython_funcs_available = True
-except Exception:
-    cython_funcs_available = False
-    create_boundary_index_list_2d = None
-    create_boundary_index_list_3d = None
+except ImportError:
+    try:
+        # If not, try development mode and import via pyximport
+        import pyximport
+
+        pyximport.install(language_level=3)
+        cython_funcs_available = True
+    except ImportError:
+        cython_funcs_available = False
+    if cython_funcs_available:
+        from pystencils.boundaries.createindexlistcython import create_boundary_neighbor_index_list_2d, \
+            create_boundary_neighbor_index_list_3d, create_boundary_cell_index_list_2d, \
+            create_boundary_cell_index_list_3d
 
 boundary_index_array_coordinate_names = ["x", "y", "z"]
 direction_member_name = "dir"

pystencils/boundaries/createindexlistcython.pyx

+# distutils: language=c
 # Workaround for cython bug
 # see https://stackoverflow.com/questions/8024805/cython-compiled-c-extension-importerror-dynamic-module-does-not-define-init-fu
 WORKAROUND = "Something"

pystencils/boundaries/inkernel.py

 import sympy as sp
-from pystencils import Field, TypedSymbol
-from pystencils.integer_functions import bitwise_and
+
 from pystencils.boundaries.boundaryhandling import DEFAULT_FLAG_TYPE
-from pystencils.data_types import create_type
+from pystencils.data_types import TypedSymbol, create_type
+from pystencils.field import Field
+from pystencils.integer_functions import bitwise_and
 
 
 def add_neumann_boundary(eqs, fields, flag_field, boundary_flag="neumann_flag", inverse_flag=False):
     """
     Replaces all neighbor accesses by flag field guarded accesses.
     If flag in neighboring cell is set, the center value is used instead
-    :param eqs: list of equations containing field accesses to direct neighbors
-    :param fields: fields for which the Neumann boundary should be applied
-    :param flag_field: integer field marking boundary cells
-    :param boundary_flag: if flag field has value 'boundary_flag' (no bit operations yet)
-                          the cell is assumed to be boundary
-    :param inverse_flag: if true, boundary cells are where flag field has not the value of boundary_flag
-    :return: list of equations with guarded field accesses
+
+    Args:
+        eqs: list of equations containing field accesses to direct neighbors
+        fields: fields for which the Neumann boundary should be applied
+        flag_field: integer field marking boundary cells
+        boundary_flag: if flag field has value 'boundary_flag' (no bit operations yet)
+                       the cell is assumed to be boundary
+        inverse_flag: if true, boundary cells are where flag field has not the value of boundary_flag
+
+    Returns:
+        list of equations with guarded field accesses
     """
     if not hasattr(fields, "__len__"):
         fields = [fields]

pystencils/cpu/__init__.py

-from pystencils.cpu.kernelcreation import create_kernel, create_indexed_kernel, add_openmp
 from pystencils.cpu.cpujit import make_python_function
+from pystencils.cpu.kernelcreation import add_openmp, create_indexed_kernel, create_kernel
 
 __all__ = ['create_kernel', 'create_indexed_kernel', 'add_openmp', 'make_python_function']

pystencils/cpu/cpujit.py

@@ -402,9 +402,19 @@ def create_module_boilerplate_code(module_name, names):
 
 def load_kernel_from_file(module_name, function_name, path):
     from importlib.util import spec_from_file_location, module_from_spec
-    spec = spec_from_file_location(name=module_name, location=path)
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    try:
+        spec = spec_from_file_location(name=module_name, location=path)
+        mod = module_from_spec(spec)
+        spec.loader.exec_module(mod)
+    except ImportError:
+        import time
+        import warnings
+        warnings.warn("Could not load " + path + ", trying on more time...")
+        time.sleep(1)
+        spec = spec_from_file_location(name=module_name, location=path)
+        mod = module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
     return getattr(mod, function_name)
 
 

pystencils/cpu/kernelcreation.py

+from typing import List, Union
+
 import sympy as sp
-from functools import partial
-from pystencils.astnodes import SympyAssignment, Block, LoopOverCoordinate, KernelFunction
-from pystencils.transformations import resolve_buffer_accesses, resolve_field_accesses, make_loop_over_domain, \
-    add_types, get_optimal_loop_ordering, parse_base_pointer_info, move_constants_before_loop, \
-    split_inner_loop, get_base_buffer_index, filtered_tree_iteration
-from pystencils.data_types import TypedSymbol, BasicType, StructType, create_type
-from pystencils.field import Field, FieldType
+
 import pystencils.astnodes as ast
-from pystencils.cpu.cpujit import make_python_function
 from pystencils.assignment import Assignment
-from typing import List, Union
+from pystencils.astnodes import Block, KernelFunction, LoopOverCoordinate, SympyAssignment
+from pystencils.cpu.cpujit import make_python_function
+from pystencils.data_types import BasicType, StructType, TypedSymbol, create_type
+from pystencils.field import Field, FieldType
+from pystencils.transformations import (
+    add_types, filtered_tree_iteration, get_base_buffer_index, get_optimal_loop_ordering,
+    make_loop_over_domain, move_constants_before_loop, parse_base_pointer_info,
+    resolve_buffer_accesses, resolve_field_accesses, split_inner_loop)
 
 AssignmentOrAstNodeList = List[Union[Assignment, ast.Node]]
 
@@ -61,9 +63,10 @@ def create_kernel(assignments: AssignmentOrAstNodeList, function_name: str = "ke
 
     body = ast.Block(assignments)
     loop_order = get_optimal_loop_ordering(fields_without_buffers)
-    ast_node = make_loop_over_domain(body, function_name, iteration_slice=iteration_slice,
-                                     ghost_layers=ghost_layers, loop_order=loop_order)
-    ast_node.target = 'cpu'
+    loop_node, ghost_layer_info = make_loop_over_domain(body, iteration_slice=iteration_slice,
+                                                        ghost_layers=ghost_layers, loop_order=loop_order)
+    ast_node = KernelFunction(loop_node, 'cpu', 'c', compile_function=make_python_function,
+                              ghost_layers=ghost_layer_info, function_name=function_name)
 
     if split_groups:
         typed_split_groups = [[type_symbol(s) for s in split_group] for split_group in split_groups]
@@ -83,7 +86,6 @@ def create_kernel(assignments: AssignmentOrAstNodeList, function_name: str = "ke
         resolve_buffer_accesses(ast_node, get_base_buffer_index(ast_node), read_only_fields)
     resolve_field_accesses(ast_node, read_only_fields, field_to_base_pointer_info=base_pointer_info)
     move_constants_before_loop(ast_node)
-    ast_node.compile = partial(make_python_function, ast_node)
     return ast_node
 
 
@@ -141,14 +143,14 @@ def create_indexed_kernel(assignments: AssignmentOrAstNodeList, index_fields, fu
         loop_body.append(assignment)
 
     function_body = Block([loop_node])
-    ast_node = KernelFunction(function_body, backend="cpu", function_name=function_name)
+    ast_node = KernelFunction(function_body, "cpu", "c", make_python_function,
+                              ghost_layers=None, function_name=function_name)
 
     fixed_coordinate_mapping = {f.name: coordinate_typed_symbols for f in non_index_fields}
 
     read_only_fields = set([f.name for f in fields_read - fields_written])
     resolve_field_accesses(ast_node, read_only_fields, field_to_fixed_coordinates=fixed_coordinate_mapping)
     move_constants_before_loop(ast_node)
-    ast_node.compile = partial(make_python_function, ast_node)
     return ast_node
 
 

pystencils/cpu/msvc_detection.py

-import subprocess
 import os
+import subprocess
 
 
 def get_environment(version_specifier, arch='x64'):