Merge branch 'master' into mogli/adaptiveRefinement_optimization

fa759e17 · Benjamin Mann · f435737e · 47f72935 · fa759e17 · fa759e17
Commit fa759e17 authored Mar 31, 2022 by Benjamin Mann
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,7 @@ stages:
   - pretest
   - test
   - benchmark
+   - no_werror

 variables:
   GIT_SUBMODULE_STRATEGY: normal
@@ -34,7 +35,7 @@ variables:
      - cd $CI_PROJECT_DIR/build
      - if dpkg --compare-versions `ompi_info | head -2 | tail -1 | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'` ge 1.10; then export MPIEXEC_PREFLAGS="--allow-run-as-root" ; fi
      - cmake ..
-        -DWARNING_ERROR=ON
+        -DWARNING_ERROR=$WARNING_ERROR
        -DWALBERLA_BUILD_WITH_MPI=$WALBERLA_BUILD_WITH_MPI
        -DWALBERLA_BUILD_WITH_OPENMP=$WALBERLA_BUILD_WITH_OPENMP
        -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
@@ -67,6 +68,7 @@ variables:
     WALBERLA_DOUBLE_ACCURACY: "ON"
     WALBERLA_BUILD_WITH_METIS: "ON"
     WALBERLA_BUILD_WITH_PARMETIS: "ON"
+     WARNING_ERROR: "ON"


 ###############################################################################
@@ -155,6 +157,23 @@ intel_19_mpionly_eigen_petsc_trilinos:
      - docker
      - intel

+intel_19_mpionly_eigen_petsc_trilinos_no_werror:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/intel:19
+   stage: no_werror
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+      WARNING_ERROR: "OFF"
+   when: manual
+   needs: [ ]
+   tags:
+      - docker
+      - intel
+
 gcc_7_serial:
   extends: .build_template
   image: i10git.cs.fau.de:5005/walberla/buildenvs/gcc:7
@@ -187,7 +206,6 @@ gcc_7_serial_dbg:
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
      CMAKE_BUILD_TYPE: "DebugOptimized"
-      WALBERLA_ENABLE_GUI: "ON"
   only:
      variables:
         - $ENABLE_NIGHTLY_BUILDS
@@ -565,6 +583,22 @@ gcc_11_mpionly_eigen_petsc_trilinos:
   tags:
      - docker

+gcc_11_mpionly_eigen_petsc_trilinos_no_werror:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/gcc:11
+   stage: no_werror
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+      WARNING_ERROR: "OFF"
+   when: manual
+   needs: [ ]
+   tags:
+      - docker
+
 clang_6.0_serial:
   extends: .build_template
   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:6.0
@@ -1014,6 +1048,9 @@ clang_11.0_serial_dbg:
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
      CMAKE_BUILD_TYPE: "DebugOptimized"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
   tags:
      - docker

@@ -1027,6 +1064,9 @@ clang_11.0_mpionly_dbg_eigen_petsc-complex_trilinos:
      HYTEG_BUILD_WITH_EIGEN: "ON"
      HYTEG_BUILD_WITH_TRILINOS: "ON"
      PETSC_DIR: "/opt/petsc-complex"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
   tags:
      - docker

@@ -1039,12 +1079,174 @@ clang_11.0_mpionly_dbg_sp:
      WALBERLA_DOUBLE_ACCURACY: "OFF"
      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
      WALBERLA_BUILD_WITH_METIS: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
   tags:
      - docker

 clang_11.0_mpionly_eigen_petsc_trilinos:
   extends: .build_template
   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:11.0
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_serial:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      WALBERLA_BUILD_WITH_MPI: "OFF"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_mpionly:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_serial_dbg:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      WALBERLA_BUILD_WITH_MPI: "OFF"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_mpionly_dbg_eigen_petsc-complex_trilinos:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc-complex"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_mpionly_dbg_sp:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_DOUBLE_ACCURACY: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+      WALBERLA_BUILD_WITH_METIS: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_12.0_mpionly_eigen_petsc_trilinos:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:12.0
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_13.0_serial:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   variables:
+      WALBERLA_BUILD_WITH_MPI: "OFF"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_13.0_mpionly:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+   only:
+      variables:
+         - $ENABLE_NIGHTLY_BUILDS
+   tags:
+      - docker
+
+clang_13.0_serial_dbg:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   variables:
+      WALBERLA_BUILD_WITH_MPI: "OFF"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+   tags:
+      - docker
+
+clang_13.0_mpionly_dbg_eigen_petsc-complex_trilinos:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   variables:
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc-complex"
+   tags:
+      - docker
+
+clang_13.0_mpionly_dbg_sp:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   variables:
+      CMAKE_BUILD_TYPE: "DebugOptimized"
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      WALBERLA_DOUBLE_ACCURACY: "OFF"
+      WALBERLA_BUILD_WITH_PARMETIS: "OFF"
+      WALBERLA_BUILD_WITH_METIS: "OFF"
+   tags:
+      - docker
+
+clang_13.0_mpionly_eigen_petsc_trilinos:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
   stage: pretest
   variables:
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
@@ -1055,9 +1257,25 @@ clang_11.0_mpionly_eigen_petsc_trilinos:
   tags:
      - docker

-inteloneapi_21.3_serial:
+clang_13.0_mpionly_eigen_petsc_trilinos_no_werror:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/clang:13.0
+   stage: no_werror
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+      WARNING_ERROR: "OFF"
+   when: manual
+   needs: [ ]
+   tags:
+      - docker
+
+inteloneapi_22.1_serial:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      WALBERLA_BUILD_WITH_MPI: "OFF"
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
@@ -1068,9 +1286,9 @@ inteloneapi_21.3_serial:
   tags:
      - docker

-inteloneapi_21.3_mpionly:
+inteloneapi_22.1_mpionly:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
   only:
@@ -1079,9 +1297,9 @@ inteloneapi_21.3_mpionly:
   tags:
      - docker

-inteloneapi_21.3_serial_dbg:
+inteloneapi_22.1_serial_dbg:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      WALBERLA_BUILD_WITH_MPI: "OFF"
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
@@ -1090,9 +1308,9 @@ inteloneapi_21.3_serial_dbg:
   tags:
      - docker

-inteloneapi_21.3_mpionly_dbg_eigen_petsc-complex_trilinos:
+inteloneapi_22.1_mpionly_dbg_eigen_petsc-complex_trilinos:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      CMAKE_BUILD_TYPE: "DebugOptimized"
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
@@ -1103,9 +1321,9 @@ inteloneapi_21.3_mpionly_dbg_eigen_petsc-complex_trilinos:
   tags:
      - docker

-inteloneapi_21.3_mpionly_dbg_sp:
+inteloneapi_22.1_mpionly_dbg_sp:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      CMAKE_BUILD_TYPE: "DebugOptimized"
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
@@ -1115,9 +1333,9 @@ inteloneapi_21.3_mpionly_dbg_sp:
   tags:
      - docker

-inteloneapi_21.3_mpionly_eigen_petsc_trilinos:
+inteloneapi_22.1_mpionly_eigen_petsc_trilinos:
   extends: .build_template
-   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:21.3
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
   variables:
      WALBERLA_BUILD_WITH_OPENMP: "OFF"
      HYTEG_BUILD_WITH_PETSC: "ON"
@@ -1127,6 +1345,22 @@ inteloneapi_21.3_mpionly_eigen_petsc_trilinos:
   tags:
      - docker

+inteloneapi_22.1_mpionly_eigen_petsc_trilinos_no_werror:
+   extends: .build_template
+   image: i10git.cs.fau.de:5005/walberla/buildenvs/inteloneapi:22.1
+   stage: no_werror
+   variables:
+      WALBERLA_BUILD_WITH_OPENMP: "OFF"
+      HYTEG_BUILD_WITH_PETSC: "ON"
+      HYTEG_BUILD_WITH_EIGEN: "ON"
+      HYTEG_BUILD_WITH_TRILINOS: "ON"
+      PETSC_DIR: "/opt/petsc"
+      WARNING_ERROR: "OFF"
+   when: manual
+   needs: [ ]
+   tags:
+      - docker
+


 ###############################################################################

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
-cmake_minimum_required (VERSION 3.10)
+cmake_minimum_required (VERSION 3.14)

 PROJECT ( hyteg )
 enable_testing()
+set_property(GLOBAL PROPERTY USE FOLDERS TRUE)

 option ( HYTEG_BUILD_WITH_PETSC       "Build with PETSc"                             OFF)
 option ( HYTEG_BUILD_WITH_TRILINOS    "Build with Trilinos"                          OFF)
@@ -13,6 +14,8 @@ set(WALBERLA_LOG_SKIPPED            ON CACHE BOOL "Log skipped cmake targets"
 set(WALBERLA_DIR ${hyteg_SOURCE_DIR}/walberla CACHE  PATH  "waLBerla path")

 set(EIGEN_DIR ${hyteg_SOURCE_DIR}/eigen CACHE  PATH  "eigen path")
+include_directories ( ${EIGEN_DIR} )
+link_directories    ( ${EIGEN_DIR} )

 include_directories ( src )

@@ -25,25 +28,6 @@ else()
    message(STATUS "Generated HyTeG kernels DISABLED! - Performance might not be optimal and some features might not be working correctly.")
 endif()

-if ( HYTEG_BUILD_WITH_PETSC )
-    find_package( PETSc )
-    include_directories ( ${PETSC_INCLUDES} )
-    link_directories    ( ${PETSC_LIBRARIES} )
-    list ( APPEND SERVICE_LIBS ${PETSC_LIBRARIES} )
-    set(WALBERLA_BUILD_WITH_MPI ON CACHE BOOL "Build with MPI" FORCE)
-    message(STATUS "WALBERLA_BUILD_WITH_MPI was force set to ON")
-endif()
-
-include_directories ( ${EIGEN_DIR} )
-link_directories    ( ${EIGEN_DIR} )
-
-if ( HYTEG_BUILD_WITH_TRILINOS)
-   find_package( Trilinos )
-   message(STATUS "Found Trilinos! Trilinos_DIR = ${Trilinos_DIR} ")
-   INCLUDE_DIRECTORIES ( ${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS})
-   list ( APPEND SERVICE_LIBS ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARY_DIRS})
-endif()
-
 find_package( waLBerla )

 if(WALBERLA_CXX_COMPILER_IS_GNU)
@@ -71,6 +55,23 @@ if(WALBERLA_CXX_COMPILER_IS_MSVC)
    add_flag( CMAKE_CXX_FLAGS "/wd4065" )
 endif()

+if ( HYTEG_BUILD_WITH_PETSC )
+    find_package( PETSc REQUIRED )
+    include_directories ( ${PETSC_INCLUDES} )
+    link_directories    ( ${PETSC_LIBRARIES} )
+    list ( APPEND SERVICE_LIBS ${PETSC_LIBRARIES} )
+    set(WALBERLA_BUILD_WITH_MPI ON CACHE BOOL "Build with MPI" FORCE)
+    message(STATUS "WALBERLA_BUILD_WITH_MPI was force set to ON")
+endif()
+
+if ( HYTEG_BUILD_WITH_TRILINOS)
+   find_package( Trilinos REQUIRED )
+   message(STATUS "Found Trilinos! Trilinos_DIR = ${Trilinos_DIR} ")
+   INCLUDE_DIRECTORIES ( ${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS})
+   list ( APPEND SERVICE_LIBS ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARY_DIRS})
+endif()
+
+
 configure_file ( ${hyteg_SOURCE_DIR}/src/hyteg/HytegDefinitions.in.hpp
                 src/hyteg/HytegDefinitions.hpp )

@@ -196,6 +197,7 @@ endif()
 string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER )

 set      ( HYTEG_BUILD_TYPE      ${CMAKE_BUILD_TYPE} )
+set      ( HYTEG_COMPILER_INFO   "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" )
 set      ( HYTEG_COMPILER_FLAGS  "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} ${CMAKE_CXX_FLAGS}" )
 string   ( REPLACE "\"" "\\\"" HYTEG_COMPILER_FLAGS ${HYTEG_COMPILER_FLAGS} )

@@ -204,4 +206,26 @@ configure_file(
        ${CMAKE_BINARY_DIR}/src/hyteg/BuildInfo.hpp
 )

+
+# this returns all targets within one folder
+function(get_all_targets _result _dir)
+   get_property(_subdirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
+   foreach (_subdir IN LISTS _subdirs)
+      get_all_targets(${_result} "${_subdir}")
+   endforeach ()
+   get_property(_sub_targets DIRECTORY "${_dir}" PROPERTY BUILDSYSTEM_TARGETS)
+   set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE)
+endfunction()
+
+get_all_targets(_tests_targets "tests")
+add_custom_target(hyteg_all_tests)
+add_dependencies(hyteg_all_tests ${_tests_targets})
+
+get_all_targets(_apps_targets "apps")
+get_all_targets(_tutorials_targets "tutorials")
+add_custom_target(hyteg_all_apps)
+add_dependencies(hyteg_all_apps ${_apps_targets} ${_tutorials_targets})
+
+add_custom_target(hyteg_all)
+add_dependencies(hyteg_all ${_apps_targets} ${_tutorials_targets} ${_tests_targets})
 ############################################################################################################################
--- a/apps/2020-moc/Helpers.cpp
+++ b/apps/2020-moc/Helpers.cpp
@@ -320,14 +320,7 @@ void solve( MeshInfo&               meshInfo,
   real_t     timeTotal   = 0;
   real_t     vMax        = 0;

-   if ( storage->hasGlobalCells() )
-   {
-      vMax = velocityMaxMagnitude( uvw[0], uvw[1], uvw[2], tmp, tmp2, level, All );
-   }
-   else
-   {
-      vMax = velocityMaxMagnitude( uvw[0], uvw[1], tmp, tmp2, level, All );
-   }
+   vMax = velocityMaxMagnitude( uvw, tmp, tmp2, level, All );

   if ( verbose )
   {
@@ -426,14 +419,7 @@ void solve( MeshInfo&               meshInfo,

      real_t advectionTimeStepRunTime;

-      if ( storage->hasGlobalCells() )
-      {
-         vMax = velocityMaxMagnitude( uvw[0], uvw[1], uvw[2], tmp, tmp2, level, All );
-      }
-      else
-      {
-         vMax = velocityMaxMagnitude( uvw[0], uvw[1], tmp, tmp2, level, All );
-      }
+      vMax = velocityMaxMagnitude( uvw, tmp, tmp2, level, All );

      if ( enableDiffusion && strangSplitting )
      {

--- a/apps/2020-moc/MantleConvection.cpp
+++ b/apps/2020-moc/MantleConvection.cpp
@@ -210,13 +210,13 @@ void calculateStokesResiduals( const StokesOperator&       A,
   r.interpolate( 0, level, All );
   A.apply( x, tmp, level, Inner | NeumannBoundary | FreeslipBoundary );
   r.assign( { 1.0, -1.0 }, { b, tmp }, level, Inner | NeumannBoundary | FreeslipBoundary );
-   residual = normL2Squared( r.uvw[0], tmp.uvw[0], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
-   residual += normL2Squared( r.uvw[1], tmp.uvw[1], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
+   residual = normL2Squared( r.uvw()[0], tmp.uvw()[0], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
+   residual += normL2Squared( r.uvw()[1], tmp.uvw()[1], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
   if ( x.getStorage()->hasGlobalCells() )
   {
-      residual += normL2Squared( r.uvw[2], tmp.uvw[2], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
+      residual += normL2Squared( r.uvw()[2], tmp.uvw()[2], Mu, level, Inner | NeumannBoundary | FreeslipBoundary );
   }
-   residual += normL2Squared( r.p, tmp.p, Mp, level, Inner | NeumannBoundary | FreeslipBoundary );
+   residual += normL2Squared( r.p(), tmp.p(), Mp, level, Inner | NeumannBoundary | FreeslipBoundary );
   residual = std::sqrt( residual );
 }

@@ -245,12 +245,12 @@ void calculateDiffusionResidual( UnsteadyDiffusion&               unsteadyDiffus
 template < typename StokesFunction, typename VelocityMass >
 real_t velocityRMS( const StokesFunction& u, const StokesFunction& tmp, const VelocityMass& M, real_t domainVolume, uint_t level )
 {
-   auto norm = std::pow( normL2( u.uvw[0], tmp.uvw[0], M, level, All ), 2.0 ) +
-               std::pow( normL2( u.uvw[1], tmp.uvw[1], M, level, All ), 2.0 );
+   auto norm = std::pow( normL2( u.uvw()[0], tmp.uvw()[0], M, level, All ), 2.0 ) +
+               std::pow( normL2( u.uvw()[1], tmp.uvw()[1], M, level, All ), 2.0 );

   if ( u.getStorage()->hasGlobalCells() )
   {
-      norm += std::pow( normL2( u.uvw[2], tmp.uvw[2], M, level, All ), 2.0 );
+      norm += std::pow( normL2( u.uvw()[2], tmp.uvw()[2], M, level, All ), 2.0 );
   }

   return std::sqrt( norm / domainVolume );
@@ -513,7 +513,7 @@ void runBenchmark( real_t     cflMax,

   for ( uint_t l = minLevel; l <= level; l++ )
   {
-      outwardNormal.uvw.interpolate( { normalX, normalY, normalZ }, l );
+      outwardNormal.uvw().interpolate( { normalX, normalY, normalZ }, l );
   }

   if ( verbose )
@@ -811,9 +811,9 @@ void runBenchmark( real_t     cflMax,
   {
      if ( outputInfo.vtkOutputVertexDoFs )
      {
-         vtkOutput.add( u.uvw[0].getVertexDoFFunction() );
-         vtkOutput.add( u.uvw[1].getVertexDoFFunction() );
-         vtkOutput.add( u.uvw[2].getVertexDoFFunction() );
+         vtkOutput.add( u.uvw()[0].getVertexDoFFunction() );
+         vtkOutput.add( u.uvw()[1].getVertexDoFFunction() );
+         vtkOutput.add( u.uvw()[2].getVertexDoFFunction() );
      }
      else
      {
@@ -854,15 +854,15 @@ void runBenchmark( real_t     cflMax,

   for ( uint_t l = minLevel; l <= level; l++ )
   {
-      MVelocity.apply( c, f.uvw[0], l, All );
-      MVelocity.apply( c, f.uvw[1], l, All );
+      MVelocity.apply( c, f.uvw()[0], l, All );
+      MVelocity.apply( c, f.uvw()[1], l, All );
      if ( storage->hasGlobalCells() )
      {
-         MVelocity.apply( c, f.uvw[2], l, All );
+         MVelocity.apply( c, f.uvw()[2], l, All );
      }

-      f.uvw.multElementwise( { f.uvw, outwardNormal.uvw }, l );
-      f.uvw.assign( { rayleighNumber }, { f.uvw }, l, All );
+      f.uvw().multElementwise( { f.uvw(), outwardNormal.uvw() }, l );
+      f.uvw().assign( { rayleighNumber }, { f.uvw() }, l, All );
   }

   calculateStokesResiduals( *A, MVelocity, MPressure, u, f, level, stokesResidual, stokesTmp, residualU );
@@ -877,14 +877,7 @@ void runBenchmark( real_t     cflMax,

   calculateStokesResiduals( *A, MVelocity, MPressure, u, f, level, stokesResidual, stokesTmp, residualU );

-   if ( storage->hasGlobalCells() )
-   {
-      vMax = velocityMaxMagnitude( u.uvw[0], u.uvw[1], u.uvw[2], uTmp, uMagnitudeSquared, level, All );
-   }
-   else
-   {
-      vMax = velocityMaxMagnitude( u.uvw[0], u.uvw[1], uTmp, uMagnitudeSquared, level, All );
-   }
+   vMax = velocityMaxMagnitude( u.uvw(), uTmp, uMagnitudeSquared, level, All );

   localTimer.start();
   if ( outputInfo.vtk )
@@ -976,15 +969,7 @@ void runBenchmark( real_t     cflMax,
      timeStep++;

      // new time step size
-
-      if ( storage->hasGlobalCells() )
-      {
-         vMax = velocityMaxMagnitude( u.uvw[0], u.uvw[1], u.uvw[2], uTmp, uTmp2, level, All );
-      }
-      else
-      {
-         vMax = velocityMaxMagnitude( u.uvw[0], u.uvw[1], uTmp, uTmp2, level, All );
-      }
+      vMax = velocityMaxMagnitude( u.uvw(), uTmp, uTmp2, level, All );

      real_t dt;
      if (