############################################################################################################################
##
## waLBerla's main cmake file
##
## Contents:
##   - definition of build options
##   - compiler variables ( c++ standard, warnings etc. )
##   - Finding of service libraries. Required: none, Optional: MPI, FFTW3, METIS, OpenMesh, Python
##     the include paths are set, and the libraries are added to variable SERVICE_LIBS
##   - Subdirectory cmake lists are called
##       -> src/   this folder contains all modules, each module (that contains c or cpp files) is linked to a
##                 static library.  Dependencies between these shared libraries are tracked manually,
##                 for more information see waLBerlaModuleDependencySystem.cmake
##       -> tests/ Same subdirectories as src/ folder. Contains tests for each module
##   - Export of variables into internal cache variables, for usage in applications or projects that use walberla as
##     subdirectory. Variables containing the service-libs,
##
############################################################################################################################



############################################################################################################################
##
## Project name, version, Custom CMake functions
##
############################################################################################################################

CMAKE_MINIMUM_REQUIRED (VERSION 3.14)


PROJECT ( walberla )

set ( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${walberla_SOURCE_DIR}/cmake )

include ( waLBerlaFunctions )

set_version( 7 0 )

include( CMakeParseArguments )

# Enable CTest
enable_testing()
include( CTest )


############################################################################################################################




############################################################################################################################
##
## Definition of build options
##
############################################################################################################################


# Build options
option ( WALBERLA_DOUBLE_ACCURACY           "Floating point accuracy, defaults to double"     ON )
option ( WALBERLA_ENABLE_GUI                "Compile with GUI"                                   )

option ( WALBERLA_BUILD_TESTS               "Build Testcases"                                 ON )
option ( WALBERLA_BUILD_BENCHMARKS          "Build Benchmarks"                                ON )
option ( WALBERLA_BUILD_TOOLS               "Build Tools"                                     ON )
option ( WALBERLA_BUILD_TUTORIALS           "Build Tutorials"                                 ON )
option ( WALBERLA_BUILD_SHOWCASES           "Build Showcases"                                 ON )
option ( WALBERLA_BUILD_DOC                 "Build Documentation"                             ON )

option ( WALBERLA_BUILD_WITH_MPI            "Build with MPI"                                  ON )
option ( WALBERLA_BUILD_WITH_METIS          "Build with metis graph partitioner"             OFF )
option ( WALBERLA_BUILD_WITH_PARMETIS       "Build with ParMetis graph partitioner"          OFF )
option ( WALBERLA_BUILD_WITH_FFTW           "Build with FFTW Fourier Transform library"      OFF )

option ( WALBERLA_BUILD_WITH_GPROF          "Enables gprof"                                      )
option ( WALBERLA_BUILD_WITH_GCOV           "Enables gcov"                                       )
option ( WALBERLA_BUILD_WITH_LTO            "Enable link time optimizations"                     )
option ( WALBERLA_BUILD_WITH_OPENMP         "Enable OpenMP support"                              )
option ( WALBERLA_BUILD_WITH_PYTHON         "Support for embedding Python"                       )
option ( WALBERLA_BUILD_WITH_CODEGEN        "Enable pystencils code generation"              OFF )


option ( WALBERLA_BUILD_WITH_LIKWID_MARKERS "Compile in markers for likwid-perfctr"              )

option ( WALBERLA_BUILD_WITH_CUDA	        "Enable CUDA support"                                )
option ( WALBERLA_BUILD_WITH_HIP	           "Enable ROCm HIP support"                            )


option ( WALBERLA_BUILD_WITH_FASTMATH       "Fast math"                                          )

option ( WALBERLA_SIMD_FORCE_SCALAR         "Do not use SIMD operations even when available" OFF )

option ( WALBERLA_BUFFER_DEBUG              "Type checking for BufferSystem ( slow )"        OFF )

# Profile guided optimization
option ( WALBERLA_PROFILE_GENERATE          "Generates Profile for Optimization"                 )
option ( WALBERLA_PROFILE_USE               "Uses Profile to optimize"                           )

# Compiler Optimization
option ( WALBERLA_OPTIMIZE_FOR_LOCALHOST    "Enable compiler optimizations spcific to localhost" )

option ( WALBERLA_LOG_SKIPPED               "Log skipped cmake targets"                      ON  )
option ( WALBERLA_DEPS_ERROR                "Fail if module dependencies are not met"        OFF )

option ( WALBERLA_GIT_SUBMODULE_AUTO        "Check submodules during cmake run"               ON )

option ( WALBERLA_BUILD_WITH_HALF_PRECISION_SUPPORT "Experimental half precision support"    OFF )

# Installation Directory
set ( CMAKE_INSTALL_PREFIX /usr/local/waLBerla CACHE STRING "The default installation directory."   )

# Default build type
if ( NOT CMAKE_BUILD_TYPE )
    set ( CMAKE_BUILD_TYPE Release CACHE STRING "Build Types: Debug Release DebugOptimized RelWithDebInfo MinSizeRel."  FORCE )
endif()
SET_PROPERTY( CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Release DebugOptimized RelWithDebInfo MinSizeRel )
set( CMAKE_TRY_COMPILE_CONFIGURATION ${CMAKE_BUILD_TYPE} )

# Debugging options                                      )
option ( WALBERLA_STL_BOUNDS_CHECKS  "Use debug capabilites of libstd++: iterator and bounds checks" )

# Warning options
option ( WARNING_DISABLE    "Disables additional compiler warnings"          OFF )
option ( WARNING_PEDANTIC   "Enables pedantic compiler warnings"             ON  )
option ( WARNING_ERROR      "Convert warnings to errors compiler warnings"   OFF )
option ( WARNING_DEPRECATED "Show warning when deprecated features are used" ON  )

# Sanitizer options
option ( WALBERLA_SANITIZE_ADDRESS    "Enables address sanitizer in gcc and clang"            )
option ( WALBERLA_SANITIZE_UNDEFINED  "Enables undefined behavior sanitizer in gcc and clang" )

# Every folder that is listed here can contain modules or tests
# this can be extended by applications to have own modules
# Here the src/ folder is added to this list, where all modules are located
list( APPEND WALBERLA_MODULE_DIRS "${walberla_SOURCE_DIR}/src" "${walberla_SOURCE_DIR}/tests" )
list( REMOVE_DUPLICATES  WALBERLA_MODULE_DIRS )
set ( WALBERLA_MODULE_DIRS  ${WALBERLA_MODULE_DIRS} CACHE INTERNAL "All folders that contain modules or tests" )

# target_link_libraries needs to called with keywords everywhere if it is called with keywords once
if( DEFINED CUDA_LINK_LIBRARIES_KEYWORD AND NOT CUDA_LINK_LIBRARIES_KEYWORD STREQUAL "" )
    set( WALBERLA_LINK_LIBRARIES_KEYWORD PUBLIC )
endif()

############################################################################################################################

set( CMAKE_CXX_STANDARD 17 )
set( CMAKE_CXX_STANDARD_REQUIRED ON )
set( CMAKE_CXX_EXTENSIONS OFF )

############################################################################################################################
##
## Compiler detection
##
############################################################################################################################

# Check for intel compiler
if( CMAKE_CXX_COMPILER MATCHES "icpc" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "icpc" )
    option ( WALBERLA_CXX_COMPILER_IS_INTEL "Use Intel compiler" ON  )
    # Intel(R) Compiler has its own library archiver,
    # if you build libraries and do not use xiar,
    # the Intel compiler will complain about invalid
    # archives at the link phase.
    # The Intel(R) archiver is "xiar" usually
    # located in the same folder as the compiler,
    FIND_PROGRAM(XIAR xiar)
    IF(XIAR)
        SET(CMAKE_AR "${XIAR}")
    ENDIF(XIAR)
    MARK_AS_ADVANCED(XIAR)

    # Intel(R) Compiler also comes with its own linker
    # which provides a number of additional benefits when
    # linking code compiled with the Intel(R) compiler.
    # Again, usually in the same place as icc itself,
    FIND_PROGRAM(XILD xild)
    IF(XILD)
       SET(CMAKE_LINKER "${XILD}")
    ENDIF(XILD)
    MARK_AS_ADVANCED(XILD)

    if( CMAKE_VERSION VERSION_LESS 3.11.0 )
      set( CMAKE_CXX17_STANDARD_COMPILE_OPTION "-std=c++17" )
      set( CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CMAKE_CXX17_STANDARD_COMPILE_OPTION )
      add_flag ( CMAKE_CXX_FLAGS ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION} )
    endif()
    if( CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.0.5" )
      # std::filesystem uses ABI tags, which don't work 19.0.2 but do in 19.0.5
      add_flag ( CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0" )
    endif()
else()
    option ( WALBERLA_CXX_COMPILER_IS_INTEL "Use Intel compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_INTEL )

# Check for Gnu compiler
if ( CMAKE_COMPILER_IS_GNUCXX  AND NOT WALBERLA_CXX_COMPILER_IS_INTEL )
     option ( WALBERLA_CXX_COMPILER_IS_GNU "Use gnu compiler" ON  )
else()
     option ( WALBERLA_CXX_COMPILER_IS_GNU "Use gnu compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_GNU )

# Check for Visual Studio
if ( MSVC )
     option ( WALBERLA_CXX_COMPILER_IS_MSVC "Use Visual Studio compiler" ON  )
else()
     option ( WALBERLA_CXX_COMPILER_IS_MSVC "Use Visual Studio compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_MSVC )

# Check for IBM compiler
if( CMAKE_CXX_COMPILER MATCHES "xlc" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "xlc" )
    option ( WALBERLA_CXX_COMPILER_IS_IBM "Use IBM compiler" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_IBM "Use IBM compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_IBM )

# Check for NEC SX compiler
if( CMAKE_CXX_COMPILER MATCHES "/sxc" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "/sxc" OR CMAKE_CXX_COMPILER MATCHES "/sxmpic" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "/sxmpic" )
    option ( WALBERLA_CXX_COMPILER_IS_NEC "Use NEC compiler" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_NEC "Use NEC compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_NEC )

if( CMAKE_CXX_COMPILER MATCHES "clang" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "clang" OR CMAKE_CXX_COMPILER MATCHES "hipcc" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" )
    option ( WALBERLA_CXX_COMPILER_IS_CLANG "Use clang compiler" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_CLANG "Use clang compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_CLANG )

# Check for Cray compiler
if( CMAKE_CXX_COMPILER_ID MATCHES Cray )
    option ( WALBERLA_CXX_COMPILER_IS_CRAY "Use Cray compiler" ON   )
else()
    option ( WALBERLA_CXX_COMPILER_IS_CRAY "Use Cray compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_CRAY )

if( CMAKE_CXX_COMPILER MATCHES "pgc\\+\\+" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "pgc\\+\\+" )
    option ( WALBERLA_CXX_COMPILER_IS_PGI "Use PGI compiler" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_PGI "Use PGI compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_PGI )

# Check for Fujitsu compiler
if( CMAKE_CXX_COMPILER_ID MATCHES FujitsuClang )
    option ( WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG "Use FujitsuClang compiler" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG "Use FujitsuClang compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG )

# Check for MPI wrapper
get_filename_component( CXX_COMPILER_WITHOUT_PATH ${CMAKE_CXX_COMPILER} NAME )
if( CXX_COMPILER_WITHOUT_PATH MATCHES "mpi" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "mpi" )
    option ( WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER "Compiler is MPI wrapper" ON  )
else()
    option ( WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER "Compiler is MPI wrapper" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )

# Check for intel llvm compiler
if( CMAKE_CXX_COMPILER MATCHES "icpx" OR CMAKE_CXX_COMPILER_ARG1 MATCHES "icpx" )
   option ( WALBERLA_CXX_COMPILER_IS_INTELLLVM "Use Intel LLVM compiler" ON  )
else()
   option ( WALBERLA_CXX_COMPILER_IS_INTELLLVM "Use Intel LLVM compiler" OFF  )
endif()
mark_as_advanced ( WALBERLA_CXX_COMPILER_IS_INTELLLVM )

############################################################################################################################



############################################################################################################################
##
## Compiler Setup
##
############################################################################################################################

# Profile guided optimization
if ( WALBERLA_PROFILE_GENERATE )
    if( WALBERLA_CXX_COMPILER_IS_INTEL )
        add_flag( CMAKE_CXX_FLAGS "-prof-gen" )
        file( MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/profile" )
        add_flag( CMAKE_CXX_FLAGS "-prof-dir${CMAKE_BINARY_DIR}/profile" )
    elseif( WALBERLA_CXX_COMPILER_IS_GNU )
        add_flag( CMAKE_CXX_FLAGS "-fprofile-generate" )
    elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
      add_flag ( CMAKE_CXX_FLAGS           "/GL"                )
      add_flag ( CMAKE_MODULE_LINKER_FLAGS "/LTCG:PGINSTRUMENT" )
      add_flag ( CMAKE_SHARED_LINKER_FLAGS "/LTCG:PGINSTRUMENT" )
      add_flag ( CMAKE_EXE_LINKER_FLAGS    "/LTCG:PGINSTRUMENT" )
    endif()
endif()

if ( WALBERLA_PROFILE_USE )
    if( WALBERLA_CXX_COMPILER_IS_INTEL )
       add_flag( CMAKE_CXX_FLAGS "-prof-use" )
       add_flag( CMAKE_CXX_FLAGS "-prof-dir${CMAKE_BINARY_DIR}/profile" )
    elseif( WALBERLA_CXX_COMPILER_IS_GNU )
       add_flag( CMAKE_CXX_FLAGS "-fprofile-use" )
    elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
      add_flag ( CMAKE_CXX_FLAGS           "/GL"              )
      add_flag ( CMAKE_MODULE_LINKER_FLAGS "/LTCG:PGOPTIMIZE" )
      add_flag ( CMAKE_SHARED_LINKER_FLAGS "/LTCG:PGOPTIMIZE" )
      add_flag ( CMAKE_EXE_LINKER_FLAGS    "/LTCG:PGOPTIMIZE" )
    endif()
endif()

# common flags for intel and g++
if ( NOT WARNING_DISABLE AND ( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL ) )
   add_flag ( CMAKE_CXX_FLAGS "-Wall -Wconversion -Wshadow" )
endif()

# C++ language features for NEC compiler
if( WALBERLA_CXX_COMPILER_IS_NEC )
   set( CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION "-Kcpp${CMAKE_CXX_STANDARD}" )
   set( CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION )
   add_flag ( CMAKE_CXX_FLAGS "${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION} -Krtti -Kexceptions -size_t64 -Kgcc" )
   add_flag ( CMAKE_CXX_FLAGS "-D__BIG_ENDIAN -D__BYTE_ORDER=__BIG_ENDIAN" )
   add_flag ( CMAKE_CXX_FLAGS "-Tnoauto,used" )
   add_flag ( CMAKE_EXE_LINKER_FLAGS "-Wl,-h,muldefs" )
   add_flag ( CMAKE_C_FLAGS "-size_t64 -Kgcc" )
   add_flag ( CMAKE_C_FLAGS "-D__BIG_ENDIAN -D__BYTE_ORDER=__BIG_ENDIAN" )
   add_flag ( CMAKE_C_FLAGS "-DSQLITE_OMIT_WAL -DHAVE_UTIME -DTHREADSAFE=0" )
   set( CMAKE_RANLIB /bin/true )
   set( CMAKE_SKIP_BUILD_RPATH TRUE )
   set( CMAKE_C_FLAGS_DEBUGOPTIMIZED    "-Chopt -g"                               )
   set( CMAKE_C_FLAGS_DEBUG             "-Cdebug -g"                              )
   set( CMAKE_CXX_FLAGS_DEBUGOPTIMIZED  "-Chopt -g"                               )
   set( CMAKE_CXX_FLAGS_DEBUG           "-Cdebug -g"                              )
endif()

# Fixes linker errors with IBM compiler
if( WALBERLA_CXX_COMPILER_IS_IBM )
   add_flag ( CMAKE_CXX_FLAGS "-qpic=large" )
endif()
# Fixes linker errors with Cray compiler
if( WALBERLA_CXX_COMPILER_IS_CRAY )
   add_flag ( CMAKE_EXE_LINKER_FLAGS  "-dynamic -L/opt/gcc/4.9.3/snos/lib64" )
endif()

# Silences compiler and linker warnings and information with the IBM compiler
if( WALBERLA_CXX_COMPILER_IS_IBM )
   add_flag ( CMAKE_CXX_FLAGS "-qsuppress=1586-267" )  # 1586-267 (I) Inlining of specified subprogram failed due to the presence of a C++ exception handler
   add_flag ( CMAKE_CXX_FLAGS "-qsuppress=1586-266" )  # 1586-266 (I) Inlining of specified subprogram failed due to the presence of a global label
   add_flag ( CMAKE_CXX_FLAGS "-qsuppress=1500-030" )  # 1500-030: (I) INFORMATION: [...] Additional optimization may be attained by recompiling and specifying MAXMEM option with a value greater than 8192.
   add_flag ( CMAKE_C_FLAGS "-qsuppress=1500-030" )    # 1500-030: (I) INFORMATION: [...] Additional optimization may be attained by recompiling and specifying MAXMEM option with a value greater than 8192.
endif()

# Silences compiler and linker warnings and information with the Cray compiler
if( WALBERLA_CXX_COMPILER_IS_CRAY )
   set( CMAKE_INCLUDE_SYSTEM_FLAG_CXX "-isystem " )
   add_flag ( CMAKE_CXX_FLAGS "-h nomessage=1" )      # CC-1    The source file does not end with a new-line character.
   add_flag ( CMAKE_C_FLAGS   "-DSQLITE_HAVE_ISNAN" ) # SQLite will not work correctly with the -ffast-math option of GCC.
   add_flag ( CMAKE_CXX_FLAGS "-DSQLITE_HAVE_ISNAN" ) # SQLite will not work correctly with the -ffast-math option of GCC.
endif()

# Silences compiler and linker warnings and information with the PGI compiler
if( WALBERLA_CXX_COMPILER_IS_PGI )
   add_flag ( CMAKE_CXX_FLAGS "--display_error_number" )
   add_flag ( CMAKE_C_FLAGS "--display_error_number" )
   if( CMAKE_VERSION VERSION_LESS "3.19" )
      #  https://github.com/Kitware/CMake/commit/52eee1938919deb59cc2b51d44f365f0d9a418e5
      set( CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION "--c++${CMAKE_CXX_STANDARD}" )
   endif()
   add_flag ( CMAKE_CXX_FLAGS "--diag_suppress=1" )   # last line of file ends without a newline
   add_flag ( CMAKE_CXX_FLAGS "--diag_suppress=111" ) # statement is unreachable
   add_flag ( CMAKE_C_FLAGS "--diag_suppress=111" )   # statement is unreachable
   add_flag ( CMAKE_C_FLAGS "--diag_suppress=550" )   # variable [...] was set but never used
   add_flag ( CMAKE_C_FLAGS "--diag_suppress=191" )   # type qualifier is meaningless on cast type
endif()

# architecture optimization
if( WALBERLA_OPTIMIZE_FOR_LOCALHOST )
   if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
      if(( CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang" ) AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "arm64" )
        # no -march=native available on this compiler, but there is currently only one such processor
      else()
        add_flag ( CMAKE_CXX_FLAGS "-march=native" )
        add_flag ( CMAKE_C_FLAGS   "-march=native" )
      endif()

      if( WALBERLA_CXX_COMPILER_IS_INTEL )
        add_flag ( CMAKE_CXX_FLAGS "-xhost" )
        add_flag ( CMAKE_C_FLAGS   "-xhost" )
      endif()

      if( EXISTS "/proc/sys/abi/sve_default_vector_length" )
        file( READ "/proc/sys/abi/sve_default_vector_length" SVE_LENGTH_BYTES )
        string(STRIP "${SVE_LENGTH_BYTES}" SVE_LENGTH_BYTES)
        math(EXPR SVE_LENGTH "${SVE_LENGTH_BYTES} * 8")
        add_flag ( CMAKE_CXX_FLAGS "-msve-vector-bits=${SVE_LENGTH}" )
        add_flag ( CMAKE_C_FLAGS   "-msve-vector-bits=${SVE_LENGTH}" )
      endif()
   endif()
endif()

# warning flags
if( WALBERLA_CXX_COMPILER_IS_INTEL )
   # system headers are also supported by intel, but cmake does not recognize that
   set( CMAKE_INCLUDE_SYSTEM_FLAG_CXX "-isystem " )
   add_flag ( CMAKE_CXX_FLAGS "-wd2928,2504,2259,1682,597" )
   # disable icc/icpc deprecation warning
   add_flag ( CMAKE_CXX_FLAGS "-diag-disable=10441" )
elseif( WALBERLA_CXX_COMPILER_IS_GNU )
   add_flag ( CMAKE_CXX_FLAGS "-Wfloat-equal -Wextra" )
elseif( WALBERLA_CXX_COMPILER_IS_NEC )
   add_flag ( CMAKE_CXX_FLAGS "-wall" )
endif()

if ( WARNING_PEDANTIC AND WALBERLA_CXX_COMPILER_IS_GNU )
   add_flag ( CMAKE_CXX_FLAGS "-pedantic" )
endif ( )

 # omit deprecated warnings
if( NOT WARNING_DEPRECATED)
   if( WALBERLA_CXX_COMPILER_IS_INTEL )
       add_flag( CMAKE_CXX_FLAGS "-wd1478" )  # Disable compiler warning # 1478: "declared as deprecated"
   elseif( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_CLANG )
       add_flag ( CMAKE_CXX_FLAGS "-Wno-deprecated-declarations")
   endif()
endif()


if ( WALBERLA_CXX_COMPILER_IS_CLANG OR WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG )
    add_flag ( CMAKE_CXX_FLAGS "-Wall -Wconversion -Wshadow -Wno-c++11-extensions -Qunused-arguments" )
endif ( )


if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
    if ( WALBERLA_STL_BOUNDS_CHECKS )
        add_definitions ( "-D_GLIBCXX_DEBUG" )
        add_definitions ( "-D_LIBCPP_DEBUG=1" )
    endif()
endif()

# Omit maybe-uninitialized for gcc 12 for now. Check if it is a bug or a real problem:
if( WALBERLA_CXX_COMPILER_IS_GNU AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.0" )
   add_flag( CMAKE_CXX_FLAGS "-Wno-maybe-uninitialized" )
endif()

# GCC 12 reports a "array bounds" warning at UniformBufferedScheme.h:297 (error: array subscript 26 is above array bounds of)
# when e.g. compiling the GhostLayerCommTest.
# Since this is most probably a bug in GCC disable the warning for now
if( WALBERLA_CXX_COMPILER_IS_GNU AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.0" )
   add_flag( CMAKE_CXX_FLAGS "-Wno-array-bounds" )
endif()

#fastmath
if ( WALBERLA_BUILD_WITH_FASTMATH )
    if ( WALBERLA_CXX_COMPILER_IS_INTEL )
        add_flag( CMAKE_CXX_FLAGS "-fp-model fast=2 -no-prec-sqrt -no-prec-div" )
    endif()

    if ( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_CLANG )
        add_flag( CMAKE_CXX_FLAGS "-ffast-math")
    endif()

    if( WALBERLA_CXX_COMPILER_IS_MSVC )
        add_flag( CMAKE_CXX_FLAGS "/fp:fast" )
    endif()
else()
   if( WALBERLA_CXX_COMPILER_IS_INTELLLVM )
      add_flag( CMAKE_CXX_FLAGS "-fp-model=precise")
   endif()
endif()

# Xcode generator disables -isystem flag, even though current versions of Xcode support it
if(CMAKE_GENERATOR STREQUAL "Xcode")
    set(CMAKE_INCLUDE_SYSTEM_FLAG_C "-isystem ")
    set(CMAKE_INCLUDE_SYSTEM_FLAG_CXX "-isystem ")
endif()


#GCC 5+ ABI selection
if( WALBERLA_CXX_COMPILER_IS_GNU )
   if( NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0.0 )
      option ( WALBERLA_USE_CPP11_ABI "On GCC 5+ use the C++11 ABI" ON )
      if( WALBERLA_USE_CPP11_ABI )
         add_flag( CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=1" )
      else()
         add_flag( CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0" )
      endif()
   endif()
endif()


# disable Xcode 7.3+ linker deduplication pass to speed up linking in debug mode
#if ( APPLE )
#   execute_process( COMMAND ${CMAKE_LINKER} -v OUTPUT_VARIABLE LINKER_VERSION ERROR_VARIABLE LINKER_VERSION )
#   string( REGEX MATCH "ld64-[0-9\\.\\-]+" LINKER_VERSION ${LINKER_VERSION} )
#   string( REGEX MATCHALL "[^\\-]+" LINKER_VERSION ${LINKER_VERSION} )
#   list( GET LINKER_VERSION 0 LINKER_TYPE )
#   list( GET LINKER_VERSION 1 LINKER_VERSION )
#   if( LINKER_TYPE STREQUAL "ld64" AND LINKER_VERSION VERSION_GREATER 264.3.101 )
#       add_flag( CMAKE_EXE_LINKER_FLAGS_DEBUG    "-Wl,-no_deduplicate")
#       add_flag( CMAKE_MODULE_LINKER_FLAGS_DEBUG "-Wl,-no_deduplicate")
#       add_flag( CMAKE_SHARED_LINKER_FLAGS_DEBUG "-Wl,-no_deduplicate")
#   endif()
#endif()


############################################################################################################################



############################################################################################################################
##
##  Find newer C++ libraries, which may only be available in std::experimental on some compilers
##
############################################################################################################################

try_compile( WALBERLA_USE_STD_FILESYSTEM "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestStdFilesystem.cpp"
      COMPILE_DEFINITIONS -DWALBERLA_USE_STD_FILESYSTEM COPY_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestStdFilesystem" OUTPUT_VARIABLE TRY_COMPILE_OUTPUT)
if( WALBERLA_USE_STD_FILESYSTEM )
  # detect https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90050 by checking whether it segfaults
  execute_process( COMMAND "${CMAKE_CURRENT_BINARY_DIR}/TestStdFilesystem" OUTPUT_QUIET RESULT_VARIABLE WALBERLA_STD_FILESYSTEM_WORKS )
endif()
if( WALBERLA_USE_STD_FILESYSTEM AND WALBERLA_STD_FILESYSTEM_WORKS EQUAL 0 )
   message( STATUS "Found std::filesystem")
else()
   try_compile( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestStdFilesystem.cpp"
         COMPILE_DEFINITIONS -DWALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM OUTPUT_VARIABLE TRY_COMPILE_OUTPUT)
   if( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM )
      message( STATUS "Found std::experimental::filesystem")
   endif()
   if( NOT WALBERLA_CXX_COMPILER_IS_MSVC AND NOT WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM )
      unset( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM CACHE )
      try_compile( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestStdFilesystem.cpp"
            COMPILE_DEFINITIONS -DWALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM LINK_LIBRARIES stdc++fs OUTPUT_VARIABLE TRY_COMPILE_OUTPUT)
      if( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM )
         message( STATUS "Found std::experimental::filesystem in libstdc++fs")
      list ( APPEND SERVICE_LIBS -lstdc++fs )
      endif()
   endif()
   if( NOT WALBERLA_CXX_COMPILER_IS_MSVC AND NOT WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM )
      unset( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM CACHE )
      try_compile( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestStdFilesystem.cpp"
            COMPILE_DEFINITIONS -DWALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM LINK_LIBRARIES c++experimental OUTPUT_VARIABLE TRY_COMPILE_OUTPUT)
      if( WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM )
         message( STATUS "Found std::experimental::filesystem in libc++experimental")
         list ( APPEND SERVICE_LIBS -lc++experimental )
      endif()
   endif()
   if( NOT WALBERLA_USE_STD_EXPERIMENTAL_FILESYSTEM AND NOT WALBERLA_USE_STD_FILESYSTEM)
      message( WARNING ${TRY_COMPILE_OUTPUT} )
      message( FATAL_ERROR "Neither std::filesystem nor std::experimental::filesystem are available" )
   endif()
endif()

############################################################################################################################
##
##  Visual Studio Setup
##
############################################################################################################################
if ( WALBERLA_CXX_COMPILER_IS_MSVC )
   string( REGEX REPLACE "[/-]W[0-4]" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} ) # remove default warning flags

   option ( WALBERLA_GROUP_PROJECTS   "Flag if the projects are grouped or in a flat hierarchy"    ON )
   option ( WALBERLA_GROUP_FILES      "Flag if the files are grouped or in a flat hierarchy"       ON )
   set_property ( GLOBAL PROPERTY USE_FOLDERS ${WALBERLA_GROUP_PROJECTS} )

   option ( WALBERLA_VS_MULTI_PROCESS_BUILD "Use the /mp option for VS builds" ON )
   if( WALBERLA_VS_MULTI_PROCESS_BUILD )
      add_flag ( CMAKE_CXX_FLAGS "-MP" ) # enable multi-threaded compiling
   endif()

   add_definitions ( "-DNOMINMAX" )                # Disable Min/Max-Macros
   add_definitions ( "-D_WIN32_WINNT=0x501" )      # Minimum Windows versions is Windows XP
   add_definitions ( "-DWINVER=0x501" )            # Minimum Windows versions is Windows XP
   add_definitions ( "-D_CRT_SECURE_NO_WARNINGS" ) # disable warnings promoting Microsoft's security enhanced CRT
   add_definitions ( "-D_SCL_SECURE_NO_WARNINGS" ) # disable warnings triggered by Microsoft's checked iterators
   add_flag ( CMAKE_CXX_FLAGS "-W4" )              # set warning level to maximum
   add_flag ( CMAKE_CXX_FLAGS "-bigobj" )          # enable big object files
   add_flag ( CMAKE_CXX_FLAGS "-wd4127" )          # disable compiler warning C4127: "conditional expression is constant"
   add_flag ( CMAKE_CXX_FLAGS "-wd4512" )          # disable compiler warning C4512: "assignment operator could not be generated"
   add_flag ( CMAKE_CXX_FLAGS "-wd4913" )          # disable compiler warning C4512: "user defined binary operator ',' exists but
                                                   # no overload could convert all operands, default built-in binary operator ','
                                                   # used"
   add_flag ( CMAKE_CXX_FLAGS "-wd4702" )          # disable compiler warning C4702: "unreachable code"
   add_flag ( CMAKE_CXX_FLAGS "-wd4505" )          # disable compiler warning C4505: "unreferenced local function has been removed"
   add_flag ( CMAKE_CXX_FLAGS "-wd4503" )          # disable compiler warning C4503: "'identifier' : decorated name length exceeded, name was truncated"

   if ( WARNING_ERROR )
      add_flag ( CMAKE_CXX_FLAGS "-WX" )           # Treat warnings as errors
   endif ( )

   if( NOT WARNING_DEPRECATED)
      add_definitions( "-D_CRT_SECURE_NO_DEPRECATE" )
      add_definitions( "-D_SCL_SECURE_NO_DEPRECATE" )
      add_flag       ( CMAKE_CXX_FLAGS "-wd4996"    ) # Disable compiler warning C4996: "declared as deprecated"
   endif()

endif ( )
############################################################################################################################

############################################################################################################################
##
## Python
##
#############################################################################################################################

if ( WALBERLA_BUILD_WITH_CODEGEN OR WALBERLA_BUILD_WITH_PYTHON )
   if ( DEFINED PYTHON_ROOT_DIR OR DEFINED PYTHON_EXECUTABLE )
      message( WARNING "Setting PYTHON_ROOT_DIR or PYTHON_EXECUTABLE will likely not work. Use Python_ROOT_DIR instead." )
   endif ()
   cmake_policy( SET CMP0094 NEW )
   set( Python_FIND_FRAMEWORK LAST )
   find_package( Python COMPONENTS Interpreter Development )
endif ()

############################################################################################################################
##
## Code Generation (pystencils)
##
#############################################################################################################################
if ( WALBERLA_BUILD_WITH_CODEGEN )
   set(LBMPY_MIN_VERSION 1.3.6)
   execute_process(COMMAND ${Python_EXECUTABLE} -c "import lbmpy; print(lbmpy.__version__)"
         RESULT_VARIABLE LBMPY_FOUND OUTPUT_VARIABLE LBMPY_VERSION)
    if(NOT LBMPY_FOUND EQUAL 0)
       message(FATAL_ERROR "WALBERLA_BUILD_WITH_CODEGEN activated but pystencils or lbmpy package not found.
                            Please install lbmpy e.g.: 'pip3 install lbmpy'")
    elseif(LBMPY_VERSION VERSION_LESS LBMPY_MIN_VERSION)
       string(STRIP ${LBMPY_VERSION} LBMPY_VERSION_STRIP)
       message(WARNING
             "lbmpy version ${LBMPY_VERSION_STRIP} was found.\n"
             "We recommend to use at least version ${LBMPY_MIN_VERSION}.")
    endif()
    execute_process(COMMAND ${Python_EXECUTABLE} -c "from pystencils.include import get_pystencils_include_path; print(get_pystencils_include_path())"
                    OUTPUT_VARIABLE PYSTENCILS_INCLUDE_PATH)
    include_directories( ${PYSTENCILS_INCLUDE_PATH} )

    execute_process(COMMAND ${Python_EXECUTABLE} -c "import jinja2"
          RESULT_VARIABLE JINJA2_FOUND )
    if(NOT JINJA2_FOUND EQUAL 0)
       message(FATAL_ERROR "WALBERLA_BUILD_WITH_CODEGEN activated and jinja2 package not found.
       Please install jinja2 e.g.: 'pip3 install jinja2'")
    endif()
endif()
############################################################################################################################




############################################################################################################################
##
## Python Libraries
##
#############################################################################################################################
if ( WALBERLA_BUILD_WITH_PYTHON )
    if(WALBERLA_CXX_COMPILER_IS_INTEL)
        # Intel C++17 support introduced in 2.6.2 (https://github.com/pybind/pybind11/pull/2729)
        set(PYBIND11_MINIMUM_VERSION "2.6.2")
    else()
        set(PYBIND11_MINIMUM_VERSION "2.6.0")
    endif()

    execute_process(COMMAND ${Python_EXECUTABLE} -c "import pybind11; print(pybind11._version.__version__)"
                    OUTPUT_VARIABLE pybind11_VERSION ERROR_QUIET RESULT_VARIABLE pybind11_VERSION_RESULT)
    string(STRIP "${pybind11_VERSION}" pybind11_VERSION)
    if(pybind11_VERSION_RESULT EQUAL "0" AND pybind11_VERSION VERSION_GREATER_EQUAL "${PYBIND11_MINIMUM_VERSION}")
        execute_process(COMMAND ${Python_EXECUTABLE} -m pybind11 --cmakedir
                        OUTPUT_VARIABLE PYBIND11_CMAKE_PATH)
        string(STRIP "${PYBIND11_CMAKE_PATH}" PYBIND11_CMAKE_PATH)
        find_package(pybind11 PATHS "${PYBIND11_CMAKE_PATH}" NO_DEFAULT_PATH REQUIRED)
    else()
        find_package(Git QUIET)
        if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
           # Update submodules as needed
           if(WALBERLA_GIT_SUBMODULE_AUTO)
              message(STATUS "Submodule update")
              execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
                    RESULT_VARIABLE GIT_SUBMOD_RESULT)
              if(NOT GIT_SUBMOD_RESULT EQUAL "0")
                 message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
              endif()
           endif()
        endif()

        if(NOT EXISTS "${PROJECT_SOURCE_DIR}/extern/pybind11/CMakeLists.txt")
            if(EXISTS "${PROJECT_SOURCE_DIR}/.git")
                message(FATAL_ERROR "Please update git submodules or install pybind11 via pip.")
            else()
                message(FATAL_ERROR "Please install pybind11 via pip or download it to ${PROJECT_SOURCE_DIR}/extern/pybind11.")
            endif()
        endif()

        add_subdirectory(extern/pybind11)

        if(pybind11_VERSION VERSION_LESS "2.6.2")
            # if pybind11 was installed into ${Python_INCLUDE_DIRS} (e.g. by pip), that will have a higher priority than the Git submodule unless we reorder the search path
            # introduced in 2.6.0 (https://github.com/pybind/pybind11/issues/2709), fixed in 2.6.2 (https://github.com/pybind/pybind11/pull/2716)
            set_property( TARGET pybind11::pybind11
                          PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/extern/pybind11/include" "${Python_INCLUDE_DIRS}")
        endif()
    endif()

    # a python module is a shared library - so everything has to be compiled to position independent code
    # otherwise linking the static libs into the shared lib will result in errors
    set(CMAKE_POSITION_INDEPENDENT_CODE ON)


    if(WALBERLA_BUILD_DOC)
      # Sphinx documentation
      # to build documentation make sure to have sphinx and read-the-docs theme installed
      # Install with: "pip install sphinx sphinx_rtd_theme"
      add_custom_target( docPython sphinx-build -b html "${walberla_SOURCE_DIR}/python/waLBerla_docs" "${walberla_BINARY_DIR}/doc/python"
                         COMMENT "Building HTML documentation for Python extension with Sphinx")
  endif()
endif()



############################################################################################################################
##
## PThread is required in Linux environments by std::thread
##
############################################################################################################################

set( THREADS_PREFER_PTHREAD_FLAG TRUE )
find_package(Threads)
if ( Threads_FOUND )
   if( CMAKE_USE_PTHREADS_INIT )
      add_flag( CMAKE_CXX_FLAGS "-pthread" )
   else()
      add_flag( CMAKE_CXX_FLAGS "${CMAKE_THREAD_LIBS_INIT}" )
   endif()
endif()



############################################################################################################################
##
## backtrace may be in a separate library
##
############################################################################################################################

if ( NOT WIN32 AND (WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG))
   find_package ( Backtrace QUIET )
   if ( Backtrace_FOUND )
      list ( APPEND SERVICE_LIBS ${Backtrace_LIBRARIES} )
      set ( WALBERLA_BUILD_WITH_BACKTRACE ON )
      set ( WALBERLA_BACKTRACE_HEADER ${Backtrace_HEADER} )
   endif ( Backtrace_FOUND )
endif()



############################################################################################################################
##
## MPI
##
############################################################################################################################

if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
   find_package ( MPI )

   # FindMPI does not really work under windows, because it expects linux formatted strings from the mpi compiler.
   # Nevertheless for Microsoft MPI and MPICH there are workarounds included, but not for OpenMPI.
   # Here is a workaround for windows with OpenMPI (use configure twice to see correct output).
   # The workaround bases on the elseif(try_libs) case in the interrogate_* function of FindMPI.
   # For this workaround we assume, that the compiler was found in any PATH or ENV variable
   # but FindMPI was not able to interpret the command line outputs.
   if ( MPI_CXX_COMPILER AND NOT MPI_CXX_LIBRARIES )
      if ( WIN32 )
         message ( STATUS "Enter Workaround Routine for Windows and OpenMPI: PRESS CONFIGURE ONE MORE TIME!" )
         string ( REGEX REPLACE "(.*)/bin/.*" "\\1" MPI_PATH ${MPI_CXX_COMPILER} )
         find_path ( MPI_C_INCLUDE_PATH mpi.h
            HINTS ${MPI_PATH}
            PATH_SUFFIXES include Inc)
         set ( MPI_CXX_INCLUDE_PATH ${MPI_C_INCLUDE_PATH} CACHE FILEPATH "" FORCE )

         set ( MPI_CXX_LIBRARIES "MPI_CXX_LIBRARIES-NOTFOUND" CACHE FILEPATH "Cleared" FORCE )
         find_library ( MPI_CXX_LIBRARIES
            NAMES         mpi++ mpicxx cxx mpi_cxx libmpi++ libmpicxx libcxx libmpi_cxx
            HINTS         ${MPI_PATH}
            PATH_SUFFIXES lib )

         if ( NOT MPI_CXX_LIBRARIES STREQUAL "MPI_CXX_LIBRARIES-NOTFOUND" )
            set ( MPI_CXX_FOUND ON FORCE )
         endif ( )

         set ( MPI_C_LIBRARIES "MPI_C_LIBRARIES-NOTFOUND" CACHE FILEPATH "Cleared" FORCE )
         find_library ( MPI_C_LIBRARIES
           NAMES         mpi mpich mpich2 msmpi libmpi libmpich libmpich2 libmsmpi
           HINTS         ${MPI_PATH}
           PATH_SUFFIXES lib )

         if ( NOT MPI_C_LIBRARIES STREQUAL "MPI_C_LIBRARIES-NOTFOUND" )
            set ( MPI_C_FOUND ON FORCE )
         endif ( )

         if ( MPI_PATH MATCHES ".*OpenMPI.*" )
            set ( MPI_CXX_COMPILE_FLAGS "/DOMPI_IMPORTS" CACHE STRING "" FORCE )
            set ( MPI_C_COMPILE_FLAGS   "/DOMPI_IMPORTS" CACHE STRING "" FORCE )
         endif ( )
      elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
      else ( )
         message ( WARNING "Found MPI Compiler but no Libraries -> invent a new workaround" )
      endif ( )
   endif ( )

   if ( MPI_FOUND )
     include_directories ( SYSTEM ${MPI_CXX_INCLUDE_PATH} ${MPI_C_INCLUDE_PATH} )
     foreach( LIB ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES} )
         if ( LIB )
            list ( APPEND SERVICE_LIBS ${LIB} )
         endif ( )
     endforeach ( )
     add_flag ( CMAKE_CXX_FLAGS "${MPI_CXX_COMPILE_FLAGS}" )
     add_flag ( CMAKE_C_FLAGS   "${MPI_C_COMPILE_FLAGS}" )

     add_flag ( CMAKE_MODULE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
     add_flag ( CMAKE_EXE_LINKER_FLAGS    "${MPI_CXX_LINK_FLAGS}" )
     add_flag ( CMAKE_SHARED_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
   endif ( )
endif ( )

# OpenMPI 3.0 and higher checks the number of processes against the number of CPUs
execute_process(COMMAND ${MPIEXEC} --version RESULT_VARIABLE mpi_version_result OUTPUT_VARIABLE mpi_version_output)
if (mpi_version_result EQUAL 0 AND mpi_version_output MATCHES "\\(Open(RTE| MPI)\\) ([3-9]\\.|1[0-9])")
    set ( MPIEXEC_PREFLAGS "${MPIEXEC_PREFLAGS}" "-oversubscribe" CACHE STRING "" FORCE)
endif()
############################################################################################################################



############################################################################################################################
##
## Qt
##
############################################################################################################################
option (WALBERLA_ENABLE_GUI "This flag builds the graphical user interface, depends on Qt Libraries")

if ( WALBERLA_ENABLE_GUI )

    message( WARNING "The GUI is deprecated and will probably not work correctly." )
    find_package( Qt4 COMPONENTS QtCore QtGui QtOpenGL QtXml REQUIRED )
    set( OpenGL_GL_PREFERENCE LEGACY )
    find_package( OpenGL REQUIRED )

    INCLUDE( ${QT_USE_FILE} )
    list ( APPEND SERVICE_LIBS ${OPENGL_LIBRARIES} ${QT_LIBRARIES} )

endif(WALBERLA_ENABLE_GUI)

############################################################################################################################



############################################################################################################################
##
## METIS
##
############################################################################################################################

if ( WALBERLA_BUILD_WITH_PARMETIS )
   # metis is required for parmetis
   set( WALBERLA_BUILD_WITH_METIS TRUE FORCE )
endif ()

if ( WALBERLA_BUILD_WITH_METIS )
   find_package( Metis REQUIRED )

   include_directories( ${METIS_INCLUDE_DIRS} )
   list( APPEND SERVICE_LIBS ${METIS_LIBRARIES} )

endif()

if ( WALBERLA_BUILD_WITH_PARMETIS )
   find_package( Parmetis REQUIRED )

   include_directories( ${PARMETIS_INCLUDE_DIR} )
   list( APPEND SERVICE_LIBS ${PARMETIS_LIBRARY} )
endif()

############################################################################################################################



############################################################################################################################
##
## FFTW3
##
############################################################################################################################

if ( WALBERLA_BUILD_WITH_FFTW )
   if( WALBERLA_BUILD_WITH_MPI )
      find_package( PFFT )
      find_package( FFTW3 )
      set( FFT_REQUIRED_LIBRARIES pfft fftw3_mpi fftw3 )
      if( PFFT_FOUND AND FFTW3_MPI_FOUND )
         set( WALBERLA_BUILD_WITH_FFT TRUE CACHE INTERNAL "Build with FFT" )
         include_directories( SYSTEM ${PFFT_INCLUDE_DIR} ${FFTW3_MPI_INCLUDE_DIR} )
         list( APPEND SERVICE_LIBS ${PFFT_LIBRARIES} ${FFTW3_LIBRARIES} ${FFTW3_MPI_LIBRARIES} )
      endif()
   else()
      find_package( FFTW3 )
      set( FFT_REQUIRED_LIBRARIES fftw3 )
      if ( FFTW3_FOUND )
         set( WALBERLA_BUILD_WITH_FFT TRUE CACHE INTERNAL "Build with FFT" )
         include_directories( SYSTEM ${FFTW3_INCLUDE_DIR} )
         list( APPEND SERVICE_LIBS ${FFTW3_LIBRARIES} )
      endif()
   endif()
endif()



############################################################################################################################
##
## OpenMesh
##
############################################################################################################################
if( (NOT DEFINED WALBERLA_BUILD_WITH_OPENMESH) OR WALBERLA_BUILD_WITH_OPENMESH )
   find_package( OpenMesh )
   if( OPENMESH_FOUND )
      set( WALBERLA_BUILD_WITH_OPENMESH ON CACHE BOOL "Build with OpenMesh support" )
      include_directories( SYSTEM ${OPENMESH_INCLUDE_DIRS} )
      list( APPEND SERVICE_LIBS ${OPENMESH_LIBRARIES} )
      if( WALBERLA_CXX_COMPILER_IS_MSVC )
         add_definitions(-D_USE_MATH_DEFINES )
      endif()
   else()
      message("   If OpenMesh required, set OPENMESH_LIBRARY_DIR to the OpenMesh directory.")
      set( WALBERLA_BUILD_WITH_OPENMESH OFF CACHE BOOL "Build with OpenMesh support" FORCE )
   endif()
endif()

############################################################################################################################
##
## DebugOptimized Build Configuration for fast execution of tests with enabled asserts
##
############################################################################################################################

set( CMAKE_C_FLAGS_DEBUGOPTIMIZED             ${CMAKE_C_FLAGS_DEBUG}             )
set( CMAKE_CXX_FLAGS_DEBUGOPTIMIZED           ${CMAKE_CXX_FLAGS_DEBUG}           )
set( CMAKE_EXE_LINKER_FLAGS_DEBUGOPTIMIZED    ${CMAKE_EXE_LINKER_FLAGS_DEBUG}    )
set( CMAKE_SHARED_LINKER_FLAGS_DEBUGOPTIMIZED ${CMAKE_SHARED_LINKER_FLAGS_DEBUG} )
set( CMAKE_MODULE_LINKER_FLAGS_DEBUGOPTIMIZED ${CMAKE_MODULE_LINKER_FLAGS_DEBUG} )

set_property(GLOBAL PROPERTY DEBUG_CONFIGURATIONS Debug DebugOptimized)

if ( WALBERLA_CXX_COMPILER_IS_MSVC )
    string(REPLACE "/Od" "/O2"   CMAKE_C_FLAGS_DEBUGOPTIMIZED   ${CMAKE_C_FLAGS_DEBUGOPTIMIZED})
    string(REPLACE "/Ob0" "/Ob2" CMAKE_C_FLAGS_DEBUGOPTIMIZED   ${CMAKE_C_FLAGS_DEBUGOPTIMIZED})
    string(REPLACE "/RTC1" ""    CMAKE_C_FLAGS_DEBUGOPTIMIZED   ${CMAKE_C_FLAGS_DEBUGOPTIMIZED})
    string(REPLACE "/Od" "/O2"   CMAKE_CXX_FLAGS_DEBUGOPTIMIZED ${CMAKE_CXX_FLAGS_DEBUGOPTIMIZED})
    string(REPLACE "/Ob0" "/Ob2" CMAKE_CXX_FLAGS_DEBUGOPTIMIZED ${CMAKE_CXX_FLAGS_DEBUGOPTIMIZED})
    string(REPLACE "/RTC1" ""    CMAKE_CXX_FLAGS_DEBUGOPTIMIZED ${CMAKE_CXX_FLAGS_DEBUGOPTIMIZED})
elseif(    WALBERLA_CXX_COMPILER_IS_GNU
        OR WALBERLA_CXX_COMPILER_IS_INTEL
        OR WALBERLA_CXX_COMPILER_IS_CLANG
        OR WALBERLA_CXX_COMPILER_IS_INTELLLVM
        OR WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG )
   set( CMAKE_C_FLAGS_DEBUGOPTIMIZED   "${CMAKE_C_FLAGS_DEBUGOPTIMIZED} -O3" )
   set( CMAKE_CXX_FLAGS_DEBUGOPTIMIZED "${CMAKE_CXX_FLAGS_DEBUGOPTIMIZED} -O3" )
endif()

set(CMAKE_C_FLAGS_DEBUGOPTIMIZED ${CMAKE_C_FLAGS_DEBUGOPTIMIZED} CACHE STRING
    "Flags used by the compiler during DebugOptimized builds")
set(CMAKE_CXX_FLAGS_DEBUGOPTIMIZED ${CMAKE_CXX_FLAGS_DEBUGOPTIMIZED}  CACHE STRING
    "Flags used by the compiler during DebugOptimized builds")
set(CMAKE_EXE_LINKER_FLAGS_DEBUGOPTIMIZED ${CMAKE_EXE_LINKER_FLAGS_DEBUGOPTIMIZED} CACHE STRING
    "Flags used by the linker for executables during DebugOptimized builds")
set(CMAKE_SHARED_LINKER_FLAGS_DEBUGOPTIMIZED ${CMAKE_SHARED_LINKER_FLAGS_DEBUGOPTIMIZED} CACHE STRING
    "Flags used by the linker for shared libraries during DebugOptimized builds")
set(CMAKE_MODULE_LINKER_FLAGS_DEBUGOPTIMIZED ${CMAKE_MODULE_LINKER_FLAGS_DEBUGOPTIMIZED} CACHE STRING
    "Flags used by the linker for loadable modules during DebugOptimized builds")

mark_as_advanced(
    CMAKE_C_FLAGS_DEBUGOPTIMIZED
    CMAKE_CXX_FLAGS_DEBUGOPTIMIZED
    CMAKE_EXE_LINKER_FLAGS_DEBUGOPTIMIZED
    CMAKE_SHARED_LINKER_FLAGS_DEBUGOPTIMIZED
    CMAKE_MODULE_LINKER_FLAGS_DEBUGOPTIMIZED
)

# This variable is only set for multi-config IDE generators like VC
if( CMAKE_CONFIGURATION_TYPES )
    list( APPEND CMAKE_CONFIGURATION_TYPES DebugOptimized )
    list( REMOVE_DUPLICATES CMAKE_CONFIGURATION_TYPES )
    set ( CMAKE_CONFIGURATION_TYPES "${CMAKE_CONFIGURATION_TYPES}"
          CACHE STRING "Semicolon separated list of supported types
                      [Debug|Release|MinSizeRel|RelWithDebInfo|ReleaseNoOutfiles|DebugOptimized]" FORCE )
endif()
############################################################################################################################





############################################################################################################################
##
## OpenMP
##
############################################################################################################################

option ( WALBERLA_THREAD_SAFE_LOGGING "Enables/Disables thread-safe logging" ON )

if ( WALBERLA_BUILD_WITH_OPENMP )
    if( APPLE AND EXISTS /opt/local/lib/libomp AND EXISTS /opt/local/include/libomp ) # find libomp from MacPorts
        set( CMAKE_FRAMEWORK_PATH /opt/local/lib/libomp )
        set( CMAKE_INCLUDE_PATH /opt/local/include/libomp )
    endif()
    find_package( OpenMP )
    if (OpenMP_FOUND)
        add_flag ( CMAKE_C_FLAGS   "${OpenMP_C_FLAGS}" )
        add_flag ( CMAKE_CXX_FLAGS "${OpenMP_CXX_FLAGS}" )
        list ( APPEND SERVICE_LIBS ${OpenMP_CXX_LIBRARIES} )
        if( OpenMP_CXX_INCLUDE_DIRS )
            include_directories( ${OpenMP_CXX_INCLUDE_DIRS} )
        endif()
    else()
        #workarounds
        if ( WALBERLA_CXX_COMPILER_IS_NEC )
            message( STATUS "Enabling OpenMP workaround for NEC")
            add_flag ( CMAKE_C_FLAGS   "-Popenmp" )
            add_flag ( CMAKE_CXX_FLAGS "-Popenmp" )
        else()
            message(FATAL_ERROR "Could NOT enable OpenMP")
        endif()
    endif()

    if( WALBERLA_CXX_COMPILER_IS_CLANG OR WALBERLA_CXX_COMPILER_IS_INTELLLVM )
       # check for bug in combination with OpenMP and sign conversion https://bugs.llvm.org/show_bug.cgi?id=48387
       try_compile( WALBERLA_CLANG_OPENMP_BUG "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestClangOpenMPBug.cpp"
             COMPILE_DEFINITIONS -Werror )
       if( NOT ${WALBERLA_CLANG_OPENMP_BUG} )
          message( WARNING "Setting -Wno-sign-conversion due to a compiler bug in LLVM (https://bugs.llvm.org/show_bug.cgi?id=48387)" )
          add_flag( CMAKE_CXX_FLAGS "-Wno-sign-conversion" )
       endif()
    endif()

else()
    if ( WALBERLA_CXX_COMPILER_IS_CRAY )
        add_flag ( CMAKE_C_FLAGS   "-h noomp" )
        add_flag ( CMAKE_CXX_FLAGS "-h noomp" )
    endif()
endif()
############################################################################################################################



############################################################################################################################
##
## CUDA
##
############################################################################################################################
if ( WALBERLA_BUILD_WITH_CUDA )
    if (WALBERLA_BUILD_WITH_HIP)
       message(FATAL_ERROR "For GPU support either use CUDA or HIP. Both simultaneously is not supported.")
    endif()

    include(CheckLanguage)
    check_language(CUDA)
    if( CMAKE_CUDA_COMPILER )

        if(${CMAKE_VERSION} VERSION_GREATER "3.18.0" AND NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
            set(CMAKE_CUDA_ARCHITECTURES OFF)
            message(WARNING "CMAKE_CUDA_ARCHITECTURES was not set. It is automatically set to: ${CMAKE_CUDA_ARCHITECTURES}")
        endif()

        enable_language(CUDA)

        #include directories and cudart lib is needed for cpp files that use cuda headers/libs
        include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
        find_library(CUDART_LIBRARY cudart ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
        list ( APPEND SERVICE_LIBS ${CUDART_LIBRARY} )

        find_library( NVTX_LIBRARY nvToolsExt ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} )
        list ( APPEND SERVICE_LIBS ${NVTX_LIBRARY} )

        #CUDA_FOUND is need for our cmake mechanism
        set ( CUDA_FOUND TRUE )
        set (WALBERLA_BUILD_WITH_GPU_SUPPORT TRUE)
    else()
        message( WARNING "CUDA could not be enabled. The host compiler might not be compatible. Check CMakeFiles/CMakeError.log for more information" )
        set ( WALBERLA_BUILD_WITH_CUDA FALSE )
    endif ( )
endif ( )

# old nvcc compilers and newer stdlibc++ are incompatible. This needs to be checked!
if (WALBERLA_STL_BOUNDS_CHECKS AND WALBERLA_BUILD_WITH_CODEGEN AND WALBERLA_BUILD_WITH_CUDA AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS "11.0" AND WALBERLA_CXX_COMPILER_IS_GNU AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0")
    message(FATAL_ERROR "WALBERLA_STL_BOUNDS_CHECKS is not compatible with your CUDA compiler")
endif()

if (WALBERLA_BUILD_WITH_CUDA AND (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS "11.0" OR CMAKE_VERSION VERSION_LESS 3.18.0))
    # CUDA < 11 does not support C++17. std::experimental::any works with C++14, unlike std::any.
    set(CMAKE_CUDA_STANDARD 14)
    set(WALBERLA_USE_STD_EXPERIMENTAL_ANY 1)
endif()

# Can be used in CMake files containing generated files where the file suffix is dependent on the target
if (WALBERLA_BUILD_WITH_CUDA AND CUDA_FOUND)
    set(CODEGEN_FILE_SUFFIX "cu")
else()
    set(CODEGEN_FILE_SUFFIX "cpp")
endif()
############################################################################################################################



############################################################################################################################
##
## ROCm HIP
##
############################################################################################################################
if ( WALBERLA_BUILD_WITH_HIP )
    if (WALBERLA_BUILD_WITH_CUDA)
       message(FATAL_ERROR "For GPU support either use CUDA or HIP. Both simultaneously is not supported.")
    endif()
    if (${CMAKE_VERSION} VERSION_LESS "3.21.0")
       message(FATAL_ERROR "For HIP support CMake > 3.21.0 is needed. Please install a newer version")
    endif()

    include(CheckLanguage)
    check_language(HIP)

    if( CMAKE_HIP_COMPILER )
        enable_language(HIP)
        # since waLBerla also supports CUDA we only use HIP on an AMD platform
        add_compile_definitions(__HIP_PLATFORM_AMD__)
        # include_directories(${HSA_HEADER})
        set (WALBERLA_BUILD_WITH_GPU_SUPPORT TRUE)
    else()
         message("HIP compiler not found. HIP support is not possible")
        set ( WALBERLA_BUILD_WITH_HIP FALSE )
    endif ( )
endif ( )

############################################################################################################################



############################################################################################################################
##
##  Testing Coverage
##
############################################################################################################################
if (WALBERLA_BUILD_WITH_GCOV AND CMAKE_COMPILER_IS_GNUCXX  )
    add_flag ( CMAKE_CXX_FLAGS "--coverage" )
endif()
############################################################################################################################



############################################################################################################################
##
##  Profiling with gprof
##
############################################################################################################################

if ( WALBERLA_BUILD_WITH_GPROF )
    if ( WALBERLA_CXX_COMPILER_IS_INTEL )
        add_flag ( CMAKE_CXX_FLAGS        "-pg" )
        add_flag ( CMAKE_EXE_LINKER_FLAGS "-pg" )
    elseif ( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_CLANG )
        add_flag ( CMAKE_CXX_FLAGS        "-pg" )
    endif()
endif()
############################################################################################################################


############################################################################################################################
##
##  Likwid Marker API
##
############################################################################################################################


if ( WALBERLA_BUILD_WITH_LIKWID_MARKERS )
    find_library( LIKWID_LIB likwid HINTS $ENV{LIKWID_LIBDIR} $ENV{LIKWID_ROOT}/lib )
    find_path( LIKWID_INCLUDE_DIR likwid.h HINTS $ENV{LIKWID_INCDIR} $ENV{LIKWID_ROOT}/include )

    # For some reason, these turned out to be necessary when building with likwid on Fugaku
    if ( WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG )
        find_library( LIKWIDLUA_LIB NAMES likwid-lua HINTS $ENV{LIKWID_LIBDIR} $ENV{LIKWID_ROOT}/lib )
        find_library( LIKWIDHWLOC_LIB NAMES likwid-hwloc HINTS $ENV{LIKWID_LIBDIR} $ENV{LIKWID_ROOT}/lib )
    endif()

    if ( LIKWID_LIB AND LIKWID_INCLUDE_DIR)
        set( LIKWID_FOUND 1)
        include_directories( ${LIKWID_INCLUDE_DIR})
        add_definitions ( "-DLIKWID_PERFMON" )
        list ( APPEND SERVICE_LIBS ${LIKWID_LIB} )
        if( WALBERLA_CXX_COMPILER_IS_FUJITSUCLANG )
            list ( APPEND SERVICE_LIBS ${LIKWIDLUA_LIB} ${LIKWIDHWLOC_LIB} )
        endif()
    else()
        message(WARNING "likwid marker library not found. Set environment variable LIKWID_ROOT")
        set ( WALBERLA_BUILD_WITH_LIKWID_MARKERS OFF CACHE BOOL "Compile in markers for likwid-perfctr" FORCE )
    endif()
endif()



############################################################################################################################
##
##  Link time optimizations
##
############################################################################################################################
if ( WALBERLA_BUILD_WITH_LTO  )
    cmake_policy( SET CMP0069 NEW )
    include( CheckIPOSupported )
    check_ipo_supported( RESULT LTO_SUPPORTED LANGUAGES CXX )
    if( LTO_SUPPORTED )
       set( CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE )
    else()
       message( WARNING "Link-time optimization is not supported with this compiler" )
    endif()
endif ( )

############################################################################################################################
##
##  Some more compiler flags that need to happen after any try_compile (e.g. inside FindMPI)
##
############################################################################################################################

# Treat warnings as errors
if ( WARNING_ERROR )
   if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
      add_flag ( CMAKE_CXX_FLAGS "-pedantic-errors -Werror" )
   elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
      add_flag ( CMAKE_CXX_FLAGS "/WX" )
   elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
      add_flag ( CMAKE_CXX_FLAGS "-h error_on_warning" )
   endif()
endif ( )

############################################################################################################################
##
##  Sanitizer
##
############################################################################################################################
if ( WALBERLA_SANITIZE_ADDRESS )
    if ( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_CLANG )
        add_flag( CMAKE_CXX_FLAGS "-fsanitize=address")
    endif()
endif()

if ( WALBERLA_SANITIZE_UNDEFINED )
    if ( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_CLANG )
        add_flag( CMAKE_CXX_FLAGS "-fsanitize=undefined")
    endif()
endif()

############################################################################################################################
##
##  Half precision
##
############################################################################################################################
if ( WALBERLA_BUILD_WITH_HALF_PRECISION_SUPPORT )
   ### Compiler requirements:
   ### Within this project, there are several checks to ensure that the template parameter 'ValueType'
   ### is a floating point number. The check is_floating_point<ValueType> is done primarily in our MPI implementation.
   ### The IEE 754 floating type format _Float16, evaluates to true only if your compiler supports the
   ### open C++23 standard P1467R9 (Extended floating-point types and standard names).
   ### Compare:
   ###  https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1467r9.html
   ###
   ### Right now (18.12.2023) this is the case only for gcc13.
   ### For more information see:
   ###   https://gcc.gnu.org/projects/cxx-status.html#:~:text=Extended%20floating%2Dpoint%20types%20and%20standard%20names
   ###   https://clang.llvm.org/cxx_status.html#:~:text=Extended%20floating%2Dpoint%20types%20and%20standard%20names

   try_compile( WALBERLA_SUPPORT_HALF_PRECISION "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestFloat16.cpp"
         CXX_STANDARD 23 OUTPUT_VARIABLE TRY_COMPILE_OUTPUT )
   ## message( STATUS ${TRY_COMPILE_OUTPUT} )
   if ( NOT WALBERLA_SUPPORT_HALF_PRECISION )
      message( FATAL_ERROR "Compiler: ${CMAKE_CXX_COMPILER} Version: ${CMAKE_CXX_COMPILER_VERSION} does not support half precision" )
   endif ()

endif () # Check if WALBERLA_BUILD_WITH_HALF_PRECISION_SUPPORT is set

############################################################################################################################
# Documentation Generation
#
if (WALBERLA_BUILD_DOC)
  # Build documentation using Doxygen (www.doxygen.org)
  ############################################################################################################################
  find_package ( Doxygen  )
  find_package ( HTMLHelp )

  if ( HTML_HELP_COMPILER EQUAL "" )
     set ( HTML_HELP_FOUND "NO" )
  else ( )
     set ( HTML_HELP_FOUND "YES" )
  endif ( )

  if ( DOXYGEN_FOUND )
     set ( DOXYGEN_HTML_HEADER ${walberla_SOURCE_DIR}/doc/header.html )
     set ( DOXYGEN_HTML_FOOTER ${walberla_SOURCE_DIR}/doc/footer.html )
     set ( DOXYGEN_HTML_OUTPUT "html" )

     configure_file ( ${walberla_SOURCE_DIR}/doc/doxygen.in ${walberla_BINARY_DIR}/doc/doxygen.cfg @ONLY )

     add_custom_target ( doc   ${DOXYGEN_EXECUTABLE} ${walberla_BINARY_DIR}/doc/doxygen.cfg
                            COMMENT "Generating API documentation with Doxygen" VERBATIM )

  endif ( )
endif()

############################################################################################################################





############################################################################################################################
#
# Fix compiler bugs
#
############################################################################################################################

# The NEC SX has a few issues in its standard library headers
if( WALBERLA_CXX_COMPILER_IS_NEC )
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/math.h         "#include_next <math.h>\n#undef fpclassify\n#undef signbit\n#undef isfinite\n#undef isinf\n#undef isnan\n#undef isnormal\n#undef isgreater\n#undef isgreaterequal\n#undef isless\n#undef islessequal\n#undef islessgreater\n#undef isunordered\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/types.h    "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/types.h\"   \n#undef uint_t\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/acl.h      "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/acl.h\"     \n#undef uint_t\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/if_ehcpl.h "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/if_ehcpl.h\"\n#undef uint_t\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/ptms.h     "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/ptms.h\"    \n#undef uint_t\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/stream.h   "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/stream.h\"  \n#undef uint_t\n")
   file( WRITE ${walberla_BINARY_DIR}/CMakeFiles/src/sys/strsubr.h  "#define uint_t SX_UINT_T\n#include \"/SX/usr/include/sys/strsubr.h\" \n#undef uint_t\n")
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/math.h         ${walberla_BINARY_DIR}/src/math.h COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/types.h    ${walberla_BINARY_DIR}/src/sys/types.h    COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/acl.h      ${walberla_BINARY_DIR}/src/sys/acl.h      COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/if_ehcpl.h ${walberla_BINARY_DIR}/src/sys/if_ehcpl.h COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/ptms.h     ${walberla_BINARY_DIR}/src/sys/ptms.h     COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/stream.h   ${walberla_BINARY_DIR}/src/sys/stream.h   COPYONLY )
   configure_file ( ${walberla_BINARY_DIR}/CMakeFiles/src/sys/strsubr.h  ${walberla_BINARY_DIR}/src/sys/strsubr.h  COPYONLY )
endif()



############################################################################################################################
#
# Subdirectories
#
############################################################################################################################

# Add binary dir for generated headers
include_directories ( ${CMAKE_CURRENT_BINARY_DIR}/src )
# All include paths are specified relative to src/ directory
include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src )

# external
add_subdirectory( extern )

# sources
add_subdirectory ( src )

# Generate file with compile options, and add install rule for it
configure_file ( src/waLBerlaDefinitions.in.h
                 src/waLBerlaDefinitions.h    )

install( FILES ${walberla_BINARY_DIR}/src/waLBerlaDefinitions.h DESTINATION walberla/ )


# test
if ( WALBERLA_BUILD_TESTS )
    add_subdirectory ( tests )
endif()


add_subdirectory ( apps )

waLBerla_export()

############################################################################################################################

############################################################################################################################
##
## clang-tidy
##
############################################################################################################################

waLBerla_link_files_to_builddir( .clang-tidy )
add_subdirectory( utilities )