Commit 956c89a0 authored by Martin Bauer's avatar Martin Bauer
Browse files

Bug fix for shared library cache -> switched to atomic filesystem write

- when running multiple pystencils instances, sometimes errors happened
  because one process might have partially written a cached file, which
  is already read before writing was finished
-> switched to "atomic write" (only on linux yet) that uses os.rename
   which is guaranteed to be atomic
parent 82bfa691
......@@ -76,7 +76,7 @@ from collections import OrderedDict
from pystencils.transformations import symbol_name_to_variable_name
from pystencils.data_types import to_ctypes, get_base_type, StructType
from pystencils.field import FieldType
from pystencils.utils import recursive_dict_update
from pystencils.utils import recursive_dict_update, file_handle_for_atomic_write, atomic_file_write
def make_python_function(kernel_function_node, argument_dict={}):
......@@ -263,11 +263,10 @@ def compile_object_cache_to_shared_library():
atexit.register(compile_object_cache_to_shared_library)
def generate_code(ast, restrict_qualifier, function_prefix, target_file):
def generate_code(ast, restrict_qualifier, function_prefix, source_file):
headers = get_headers(ast)
headers.update(['<cmath>', '<cstdint>'])
with open(target_file, 'w') as source_file:
code = generate_c(ast)
includes = "\n".join(["#include %s" % (include_file,) for include_file in headers])
print(includes, file=source_file)
......@@ -298,15 +297,17 @@ def compile_linux(ast, code_hash_str, src_file, lib_file):
compiler_config = get_compiler_config()
object_file = os.path.join(cache_config['object_cache'], code_hash_str + '.o')
# Compilation
if not os.path.exists(object_file):
generate_code(ast, compiler_config['restrict_qualifier'], '', src_file)
with file_handle_for_atomic_write(src_file) as f:
generate_code(ast, compiler_config['restrict_qualifier'], '', f)
with atomic_file_write(object_file) as file_name:
compile_cmd = [compiler_config['command'], '-c'] + compiler_config['flags'].split()
compile_cmd += ['-o', object_file, src_file]
compile_cmd += ['-o', file_name, src_file]
run_compile_step(compile_cmd)
# Linking
run_compile_step([compiler_config['command'], '-shared', object_file, '-o', lib_file] +
with atomic_file_write(lib_file) as file_name:
run_compile_step([compiler_config['command'], '-shared', object_file, '-o', file_name] +
compiler_config['flags'].split())
......@@ -317,8 +318,8 @@ def compile_windows(ast, code_hash_str, src_file, lib_file):
object_file = os.path.join(cache_config['object_cache'], code_hash_str + '.obj')
# Compilation
if not os.path.exists(object_file):
generate_code(ast, compiler_config['restrict_qualifier'],
'__declspec(dllexport)', src_file)
with file_handle_for_atomic_write(src_file) as f:
generate_code(ast, compiler_config['restrict_qualifier'], '__declspec(dllexport)', f)
# /c compiles only, /EHsc turns of exception handling in c code
compile_cmd = ['cl.exe', '/c', '/EHsc'] + compiler_config['flags'].split()
......
import os
from tempfile import NamedTemporaryFile
from contextlib import contextmanager
from typing import Mapping
......@@ -34,3 +37,32 @@ def recursive_dict_update(d, u):
else:
d[k] = u[k]
return d
@contextmanager
def file_handle_for_atomic_write(file_path):
"""Open temporary file object that atomically moves to destination upon exiting.
Allows reading and writing to and from the same filename.
The file will not be moved to destination in case of an exception.
Args:
file_path: path to file to be opened
"""
target_folder = os.path.dirname(os.path.abspath(file_path))
with NamedTemporaryFile(delete=False, dir=target_folder, mode='w') as f:
try:
yield f
finally:
f.flush()
os.fsync(f.fileno())
os.rename(f.name, file_path)
@contextmanager
def atomic_file_write(file_path):
target_folder = os.path.dirname(os.path.abspath(file_path))
with NamedTemporaryFile(delete=False, dir=target_folder) as f:
f.file.close()
yield f.name
os.rename(f.name, file_path)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment