Commit 9bad685f authored by Helen Schottenhamml's avatar Helen Schottenhamml
Browse files

Merge branch 'Fixes' into 'master'

Clean up and Bug Fixes

Closes #38

See merge request pycodegen/pystencils!264
parents a3cec2ce 6f74f2ab
Pipeline #35071 passed with stages
in 17 minutes and 57 seconds
...@@ -19,3 +19,7 @@ pystencils/boundaries/createindexlistcython.*.so ...@@ -19,3 +19,7 @@ pystencils/boundaries/createindexlistcython.*.so
pystencils_tests/tmp pystencils_tests/tmp
pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/ pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/ pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
# macOS
**/.DS_Store
\ No newline at end of file
...@@ -14,6 +14,7 @@ tests-and-coverage: ...@@ -14,6 +14,7 @@ tests-and-coverage:
- $ENABLE_NIGHTLY_BUILDS - $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
script: script:
- pip install sympy --upgrade
- env - env
- pip list - pip list
- export NUM_CORES=$(nproc --all) - export NUM_CORES=$(nproc --all)
...@@ -94,6 +95,7 @@ minimal-windows: ...@@ -94,6 +95,7 @@ minimal-windows:
- export NUM_CORES=$(nproc --all) - export NUM_CORES=$(nproc --all)
- source /cygdrive/c/Users/build/Miniconda3/Scripts/activate - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
- source activate pystencils - source activate pystencils
- pip install joblib
- pip list - pip list
- python -c "import numpy" - python -c "import numpy"
- py.test -v -m "not (notebook or longrun)" - py.test -v -m "not (notebook or longrun)"
...@@ -105,9 +107,9 @@ ubuntu: ...@@ -105,9 +107,9 @@ ubuntu:
- $ENABLE_NIGHTLY_BUILDS - $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
before_script: before_script:
- apt-get -y remove python3-sympy # - apt-get -y remove python3-sympy
- ln -s /usr/include/locale.h /usr/include/xlocale.h - ln -s /usr/include/locale.h /usr/include/xlocale.h
- pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'` # - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
script: script:
- export NUM_CORES=$(nproc --all) - export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib - mkdir -p ~/.config/matplotlib
...@@ -293,7 +295,7 @@ flake8-lint: ...@@ -293,7 +295,7 @@ flake8-lint:
build-documentation: build-documentation:
stage: test stage: test
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
script: script:
- export PYTHONPATH=`pwd` - export PYTHONPATH=`pwd`
- mkdir html_doc - mkdir html_doc
......
...@@ -322,6 +322,8 @@ class CBackend: ...@@ -322,6 +322,8 @@ class CBackend:
offset = sp.Add(*[sp.Symbol(LoopOverCoordinate.get_loop_counter_name(i)) offset = sp.Add(*[sp.Symbol(LoopOverCoordinate.get_loop_counter_name(i))
* node.lhs.args[0].field.spatial_strides[i] for i in * node.lhs.args[0].field.spatial_strides[i] for i in
range(len(node.lhs.args[0].field.spatial_strides))]) range(len(node.lhs.args[0].field.spatial_strides))])
if stride == 1:
offset = offset.subs({node.lhs.args[0].field.spatial_strides[0]: 1})
size = sp.Mul(*node.lhs.args[0].field.spatial_shape) size = sp.Mul(*node.lhs.args[0].field.spatial_shape)
element_size = 8 if data_type.base_type.base_name == 'double' else 4 element_size = 8 if data_type.base_type.base_name == 'double' else 4
size_cond = f"({offset} + {CachelineSize.symbol/element_size}) < {size}" size_cond = f"({offset} + {CachelineSize.symbol/element_size}) < {size}"
......
...@@ -8,22 +8,15 @@ try: ...@@ -8,22 +8,15 @@ try:
except ImportError: except ImportError:
from backports.functools_lru_cache import lru_cache as memorycache from backports.functools_lru_cache import lru_cache as memorycache
from joblib import Memory
from appdirs import user_cache_dir
try: if 'PYSTENCILS_CACHE_DIR' in os.environ:
from joblib import Memory cache_dir = os.environ['PYSTENCILS_CACHE_DIR']
from appdirs import user_cache_dir else:
if 'PYSTENCILS_CACHE_DIR' in os.environ: cache_dir = user_cache_dir('pystencils')
cache_dir = os.environ['PYSTENCILS_CACHE_DIR'] disk_cache = Memory(cache_dir, verbose=False).cache
else: disk_cache_no_fallback = disk_cache
cache_dir = user_cache_dir('pystencils')
disk_cache = Memory(cache_dir, verbose=False).cache
disk_cache_no_fallback = disk_cache
except ImportError:
# fallback to in-memory caching if joblib is not available
disk_cache = memorycache(maxsize=64)
def disk_cache_no_fallback(o):
return o
def _wrapper(wrapped_func, cached_func, *args, **kwargs): def _wrapper(wrapped_func, cached_func, *args, **kwargs):
...@@ -34,7 +27,6 @@ def _wrapper(wrapped_func, cached_func, *args, **kwargs): ...@@ -34,7 +27,6 @@ def _wrapper(wrapped_func, cached_func, *args, **kwargs):
def memorycache_if_hashable(maxsize=128, typed=False): def memorycache_if_hashable(maxsize=128, typed=False):
def wrapper(func): def wrapper(func):
return partial(_wrapper, func, memorycache(maxsize, typed)(func)) return partial(_wrapper, func, memorycache(maxsize, typed)(func))
......
...@@ -75,6 +75,7 @@ def make_python_function(kernel_function_node, custom_backend=None): ...@@ -75,6 +75,7 @@ def make_python_function(kernel_function_node, custom_backend=None):
- all symbols which are not defined in the kernel itself are expected as parameters - all symbols which are not defined in the kernel itself are expected as parameters
:param kernel_function_node: the abstract syntax tree :param kernel_function_node: the abstract syntax tree
:param custom_backend: use own custom printer for code generation
:return: kernel functor :return: kernel functor
""" """
result = compile_and_load(kernel_function_node, custom_backend) result = compile_and_load(kernel_function_node, custom_backend)
...@@ -183,6 +184,10 @@ def read_config(): ...@@ -183,6 +184,10 @@ def read_config():
if os.path.exists(libomp): if os.path.exists(libomp):
default_compiler_config['flags'] += ' ' + libomp default_compiler_config['flags'] += ' ' + libomp
break break
else:
raise ValueError("The detection of the platform with platform.system() did not work. "
"Pystencils is only supported for linux, windows, and darwin platforms.")
default_cache_config = OrderedDict([ default_cache_config = OrderedDict([
('object_cache', os.path.join(user_cache_dir('pystencils'), 'objectcache')), ('object_cache', os.path.join(user_cache_dir('pystencils'), 'objectcache')),
('clear_cache_on_start', False), ('clear_cache_on_start', False),
...@@ -205,19 +210,19 @@ def read_config(): ...@@ -205,19 +210,19 @@ def read_config():
if config['cache']['object_cache'] is not False: if config['cache']['object_cache'] is not False:
config['cache']['object_cache'] = os.path.expanduser(config['cache']['object_cache']).format(pid=os.getpid()) config['cache']['object_cache'] = os.path.expanduser(config['cache']['object_cache']).format(pid=os.getpid())
clear_cache = False clear_cache_on_start = False
cache_status_file = os.path.join(config['cache']['object_cache'], 'last_config.json') cache_status_file = os.path.join(config['cache']['object_cache'], 'last_config.json')
if os.path.exists(cache_status_file): if os.path.exists(cache_status_file):
# check if compiler config has changed # check if compiler config has changed
last_config = json.load(open(cache_status_file, 'r')) last_config = json.load(open(cache_status_file, 'r'))
if set(last_config.items()) != set(config['compiler'].items()): if set(last_config.items()) != set(config['compiler'].items()):
clear_cache = True clear_cache_on_start = True
else: else:
for key in last_config.keys(): for key in last_config.keys():
if last_config[key] != config['compiler'][key]: if last_config[key] != config['compiler'][key]:
clear_cache = True clear_cache_on_start = True
if config['cache']['clear_cache_on_start'] or clear_cache: if config['cache']['clear_cache_on_start'] or clear_cache_on_start:
shutil.rmtree(config['cache']['object_cache'], ignore_errors=True) shutil.rmtree(config['cache']['object_cache'], ignore_errors=True)
create_folder(config['cache']['object_cache'], False) create_folder(config['cache']['object_cache'], False)
...@@ -578,7 +583,10 @@ class ExtensionModuleCode: ...@@ -578,7 +583,10 @@ class ExtensionModuleCode:
print(self._code_string, file=file) print(self._code_string, file=file)
def compile_module(code, code_hash, base_dir, compile_flags=[]): def compile_module(code, code_hash, base_dir, compile_flags=None):
if compile_flags is None:
compile_flags = []
compiler_config = get_compiler_config() compiler_config = get_compiler_config()
extra_flags = ['-I' + get_paths()['include'], '-I' + get_pystencils_include_path()] + compile_flags extra_flags = ['-I' + get_paths()['include'], '-I' + get_pystencils_include_path()] + compile_flags
......
...@@ -56,7 +56,7 @@ class CachelineSize(ast.Node): ...@@ -56,7 +56,7 @@ class CachelineSize(ast.Node):
@property @property
def symbols_defined(self): def symbols_defined(self):
return set([self.symbol, self.mask_symbol, self.last_symbol]) return {self.symbol, self.mask_symbol, self.last_symbol}
@property @property
def undefined_symbols(self): def undefined_symbols(self):
......
...@@ -450,17 +450,12 @@ def collate_types(types, ...@@ -450,17 +450,12 @@ def collate_types(types,
Uses the collation rules from numpy. Uses the collation rules from numpy.
""" """
if forbid_collation_to_complex: if forbid_collation_to_complex:
types = [ types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.complexfloating)]
t for t in types
if not np.issubdtype(t.numpy_dtype, np.complexfloating)
]
if not types: if not types:
return create_type(default_float_type) return create_type(default_float_type)
if forbid_collation_to_float: if forbid_collation_to_float:
types = [ types = [t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)]
t for t in types if not np.issubdtype(t.numpy_dtype, np.floating)
]
if not types: if not types:
return create_type(default_int_type) return create_type(default_int_type)
...@@ -567,10 +562,17 @@ def get_type_of_expression(expr, ...@@ -567,10 +562,17 @@ def get_type_of_expression(expr,
expr: sp.Expr expr: sp.Expr
if expr.args: if expr.args:
types = tuple(get_type(a) for a in expr.args) types = tuple(get_type(a) for a in expr.args)
# collate_types checks numpy_dtype in the special cases
if any(not hasattr(t, 'numpy_dtype') for t in types):
forbid_collation_to_complex = False
forbid_collation_to_float = False
else:
forbid_collation_to_complex = expr.is_real is True
forbid_collation_to_float = expr.is_integer is True
return collate_types( return collate_types(
types, types,
forbid_collation_to_complex=expr.is_real is True, forbid_collation_to_complex=forbid_collation_to_complex,
forbid_collation_to_float=expr.is_integer is True, forbid_collation_to_float=forbid_collation_to_float,
default_float_type=default_float_type, default_float_type=default_float_type,
default_int_type=default_int_type) default_int_type=default_int_type)
else: else:
......
...@@ -21,6 +21,10 @@ class DataHandling(ABC): ...@@ -21,6 +21,10 @@ class DataHandling(ABC):
_GPU_LIKE_BACKENDS = [Backend.CUDA, Backend.OPENCL] _GPU_LIKE_BACKENDS = [Backend.CUDA, Backend.OPENCL]
# ---------------------------- Adding and accessing data ----------------------------------------------------------- # ---------------------------- Adding and accessing data -----------------------------------------------------------
@property
@abstractmethod
def default_target(self) -> Target:
"""Target Enum indicating the target of the computation"""
@property @property
@abstractmethod @abstractmethod
......
...@@ -35,13 +35,13 @@ class ParallelDataHandling(DataHandling): ...@@ -35,13 +35,13 @@ class ParallelDataHandling(DataHandling):
""" """
super(ParallelDataHandling, self).__init__() super(ParallelDataHandling, self).__init__()
assert dim in (2, 3) assert dim in (2, 3)
self.blocks = blocks self._blocks = blocks
self.default_ghost_layers = default_ghost_layers self._default_ghost_layers = default_ghost_layers
self.default_layout = default_layout self._default_layout = default_layout
self._fields = DotDict() # maps name to symbolic pystencils field self._fields = DotDict() # maps name to symbolic pystencils field
self._field_name_to_cpu_data_name = {} self._field_name_to_cpu_data_name = {}
self._field_name_to_gpu_data_name = {} self._field_name_to_gpu_data_name = {}
self.data_names = set() self._data_names = set()
self._dim = dim self._dim = dim
self._fieldInformation = {} self._fieldInformation = {}
self._cpu_gpu_pairs = [] self._cpu_gpu_pairs = []
...@@ -55,7 +55,11 @@ class ParallelDataHandling(DataHandling): ...@@ -55,7 +55,11 @@ class ParallelDataHandling(DataHandling):
if self._dim == 2: if self._dim == 2:
assert self.blocks.getDomainCellBB().size[2] == 1 assert self.blocks.getDomainCellBB().size[2] == 1
self.default_target = default_target self._default_target = default_target
@property
def default_target(self):
return self._default_target
@property @property
def dim(self): def dim(self):
...@@ -73,6 +77,22 @@ class ParallelDataHandling(DataHandling): ...@@ -73,6 +77,22 @@ class ParallelDataHandling(DataHandling):
def fields(self): def fields(self):
return self._fields return self._fields
@property
def blocks(self):
return self._blocks
@property
def default_ghost_layers(self):
return self._default_ghost_layers
@property
def default_layout(self):
return self._default_layout
@property
def data_names(self):
return self.data_names
def ghost_layers_of_field(self, name): def ghost_layers_of_field(self, name):
return self._fieldInformation[name]['ghost_layers'] return self._fieldInformation[name]['ghost_layers']
......
...@@ -69,9 +69,13 @@ class SerialDataHandling(DataHandling): ...@@ -69,9 +69,13 @@ class SerialDataHandling(DataHandling):
self._periodicity = periodicity self._periodicity = periodicity
self._field_information = {} self._field_information = {}
self.default_target = default_target self._default_target = default_target
self._start_time = time.perf_counter() self._start_time = time.perf_counter()
@property
def default_target(self):
return self._default_target
@property @property
def dim(self): def dim(self):
return len(self._domainSize) return len(self._domainSize)
......
...@@ -261,7 +261,7 @@ class FiniteDifferenceStaggeredStencilDerivation: ...@@ -261,7 +261,7 @@ class FiniteDifferenceStaggeredStencilDerivation:
main_points = [neighbor / 2, neighbor / -2, flipped(neighbor / 2, nonzero_indices[0]), main_points = [neighbor / 2, neighbor / -2, flipped(neighbor / 2, nonzero_indices[0]),
flipped(neighbor / -2, nonzero_indices[0])] flipped(neighbor / -2, nonzero_indices[0])]
else: else:
main_points = [neighbor.multiply_elementwise(sp.Matrix(c) / 2) main_points = [sp.Matrix(np.multiply(neighbor, sp.Matrix(c) / 2))
for c in itertools.product([-1, 1], repeat=3)] for c in itertools.product([-1, 1], repeat=3)]
points += main_points points += main_points
zero_indices = [i for i, v in enumerate(neighbor) if v == 0 and i < dim] zero_indices = [i for i, v in enumerate(neighbor) if v == 0 and i < dim]
......
...@@ -7,9 +7,6 @@ from IPython.display import HTML ...@@ -7,9 +7,6 @@ from IPython.display import HTML
import pystencils.plot as plt import pystencils.plot as plt
__all__ = ['make_imshow_animation', 'display_animation', 'set_display_mode']
VIDEO_TAG = """<video controls width="80%"> VIDEO_TAG = """<video controls width="80%">
<source src="data:video/x-m4v;base64,{0}" type="video/mp4"> <source src="data:video/x-m4v;base64,{0}" type="video/mp4">
Your browser does not support the video tag. Your browser does not support the video tag.
......
"""
Default Sympy optimizations applied in pystencils kernels using :func:`sympy.codegen.rewriting.optimize`.
See :func:`sympy.codegen.rewriting.optimize`.
"""
import itertools
from pystencils import Assignment
from pystencils.astnodes import SympyAssignment
try:
from sympy.codegen.rewriting import optims_c99, optimize
from sympy.codegen.rewriting import ReplaceOptim
HAS_REWRITING = True
# Evaluates all constant terms
evaluate_constant_terms = ReplaceOptim(
lambda e: hasattr(e, 'is_constant') and e.is_constant and not e.is_integer,
lambda p: p.evalf()
)
optims_pystencils_cpu = [evaluate_constant_terms] + list(optims_c99)
optims_pystencils_gpu = [evaluate_constant_terms] + list(optims_c99)
except ImportError:
from warnings import warn
warn("Could not import ReplaceOptim, optims_c99, optimize from sympy.codegen.rewriting."
"Please update your sympy installation!")
optims_c99 = []
optims_pystencils_cpu = []
optims_pystencils_gpu = []
HAS_REWRITING = False
def optimize_assignments(assignments, optimizations):
if HAS_REWRITING:
assignments = [Assignment(a.lhs, optimize(a.rhs, optimizations))
if hasattr(a, 'lhs')
else a for a in assignments]
assignments_nodes = [a.atoms(SympyAssignment) for a in assignments]
for a in itertools.chain.from_iterable(assignments_nodes):
a.optimize(optimizations)
return assignments
def optimize_ast(ast, optimizations):
if HAS_REWRITING:
assignments_nodes = ast.atoms(SympyAssignment)
for a in assignments_nodes:
a.optimize(optimizations)
return ast
...@@ -2,7 +2,7 @@ import numpy as np ...@@ -2,7 +2,7 @@ import numpy as np
import sympy as sp import sympy as sp
import pystencils as ps import pystencils as ps
import pystencils.jupyter from pystencils.jupyter import make_imshow_animation, display_animation, set_display_mode
import pystencils.plot as plt import pystencils.plot as plt
__all__ = ['sp', 'np', 'ps', 'plt'] __all__ = ['sp', 'np', 'ps', 'plt', 'make_imshow_animation', 'display_animation', 'set_display_mode']
import pytest
from pystencils import create_data_handling from pystencils import create_data_handling
from pystencils.alignedarray import * from pystencils.alignedarray import *
from pystencils.field import create_numpy_array_with_layout from pystencils.field import create_numpy_array_with_layout
...@@ -11,45 +13,45 @@ def is_aligned(arr, alignment, byte_offset=0): ...@@ -11,45 +13,45 @@ def is_aligned(arr, alignment, byte_offset=0):
return rest == 0 return rest == 0
def test_1d_arrays(): @pytest.mark.parametrize("alignment", [8, 8*4, True])
for alignment in [8, 8*4, True]: @pytest.mark.parametrize("shape", [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)])
for shape in [17, 16, (16, 16), (17, 17), (18, 18), (19, 19)]: def test_1d_arrays(alignment, shape):
arrays = [ arrays = [
aligned_zeros(shape, alignment), aligned_zeros(shape, alignment),
aligned_ones(shape, alignment), aligned_ones(shape, alignment),
aligned_empty(shape, alignment), aligned_empty(shape, alignment),
] ]
for arr in arrays: for arr in arrays:
assert is_aligned(arr, alignment) assert is_aligned(arr, alignment)
def test_3d_arrays(): @pytest.mark.parametrize("order", ['C', 'F'])
for order in ('C', 'F'): @pytest.mark.parametrize("alignment", [8, 8*4, True])
for alignment in [8, 8*4, True]: @pytest.mark.parametrize("shape", [(16, 16), (17, 17), (18, 18), (19, 19)])
for shape in [(16, 16), (17, 17), (18, 18), (19, 19)]: def test_3d_arrays(order, alignment, shape):
arrays = [ arrays = [
aligned_zeros(shape, alignment, order=order), aligned_zeros(shape, alignment, order=order),
aligned_ones(shape, alignment, order=order), aligned_ones(shape, alignment, order=order),
aligned_empty(shape, alignment, order=order), aligned_empty(shape, alignment, order=order),
] ]
for arr in arrays: for arr in arrays:
assert is_aligned(arr, alignment) assert is_aligned(arr, alignment)
if order == 'C': if order == 'C':
assert is_aligned(arr[1], alignment) assert is_aligned(arr[1], alignment)
assert is_aligned(arr[5], alignment) assert is_aligned(arr[5], alignment)
else: else:
assert is_aligned(arr[..., 1], alignment) assert is_aligned(arr[..., 1], alignment)
assert is_aligned(arr[..., 5], alignment) assert is_aligned(arr[..., 5], alignment)
def test_data_handling(): @pytest.mark.parametrize("parallel", [False, True])
for parallel in (False, True): def test_data_handling(parallel):
for tries in range(16): # try a few times, since we might get lucky and get randomly a correct alignment for tries in range(16): # try a few times, since we might get lucky and get randomly a correct alignment
dh = create_data_handling((6, 7), default_ghost_layers=1, parallel=parallel) dh = create_data_handling((6, 7), default_ghost_layers=1, parallel=parallel)
dh.add_array('test', alignment=8 * 4) dh.add_array('test', alignment=8 * 4, values_per_cell=1)
for b in dh.iterate(ghost_layers=True, inner_ghost_layers=True): for b in dh.iterate(ghost_layers=True, inner_ghost_layers=True):
arr = b['test'] arr = b['test']
assert is_aligned(arr[1:, 3:], 8*4) assert is_aligned(arr[1:, 3:], 8*4)
def test_alignment_of_different_layouts():