Preparing Flow around Cylinder benchmark

apps/benchmarks/NonuniformGridGenerated/NonuniformGridCodegenPerformance.cpp

@@ -383,7 +383,7 @@ class FlowAroundCylinder : public SetupBase
                   flagField->addFlag(localCell, inflowFlag);
                }
                else if(globalCell.x() >= cell_idx_c(sbfs.getNumberOfXCells(level))){
-                  flagField->addFlag(localCell, inflowFlag);
+                  flagField->addFlag(localCell, outflowFlag);
                }
                else{
                   Vector3< real_t > distanceInPlane(cellCenter - cylCenter_);
@@ -447,17 +447,21 @@ class ConcentricCylinderWakeRefinement {
 
    uint_t targetLevel(const SetupBlock * b){
       AABB blockAabb(b->getAABB());
-      Vector3< real_t > centerRel(blockAabb.center() - fs_.cylCenter_);
-      if(centerRel[0] > real_t(0.0)){
-         for(int k = int(fs_.refinementDepth_) - 1; k >= 0 ; --k){
-            if(centerRel[0] <= fs_.refinementTailLengths_[uint_c(k)] && abs(centerRel[1]) <= fs_.refinementRadii_[uint_c(k)]){
+      Vector3< real_t > aabbCenter(blockAabb.center());
+      Vector3 axisReferencePoint(std::max(aabbCenter[0], fs_.cylCenter_[0]),
+                                 fs_.cylCenter_[1],
+                                 aabbCenter[2]);
+
+      for(int k = int(fs_.refinementDepth_) - 1; k >= 0 ; --k)
+      {
+         real_t tailLength = fs_.refinementTailLengths_[uint_c(k)];
+         real_t refRadius  = fs_.refinementRadii_[uint_c(k)];
+
+         if(blockAabb.xMax() - fs_.cylCenter_[0] <= tailLength){
+            if(blockAabb.distance(axisReferencePoint) < refRadius){
                return uint_c(k + 1);
             }
          }
-      } else {
-         for(int k = int(fs_.refinementDepth_) - 1; k >= 0 ; --k){
-            if(centerRel.length() <= fs_.refinementRadii_[uint_c(k)]) return uint_c(k + 1);
-         }
       }
 
       return 0;

apps/benchmarks/NonuniformGridGenerated/Setups/FlowAroundCylinderPerformance.py

+import waLBerla as wlb
+import os
+import sys
+import sqlite3
+import numpy as np
+from waLBerla.tools.sqlitedb import sequenceValuesToScalars, checkAndUpdateSchema, storeSingle
+
+from lbmpy.relaxationrates import relaxation_rate_from_lattice_viscosity
+
+def safe_int(some_float):
+    assert int(some_float) == some_float, f"{some_float} is not an integer. :("
+    return int(some_float)
+
+class FlowAroundCylinderScenario:
+    def __init__(self, domainSize, cylinderCenter, cylinderRadius, cylinderRadius_finest_LU,
+                 cellsPerBlock,
+                 peakInflowVelocity_LU, reynoldsNumber, walls,
+                 refinementProfile, refinementDepth,
+                 refinementRadii=(0,0,0), refinementTailLengths=(0,0,0),
+                 runs=10, steps_per_run=1000, outputFrequency=0,
+                 db_file='benchmark.sqlite'):
+
+        dx_finest = cylinderRadius / cylinderRadius_finest_LU
+        dx_coarsest = dx_finest * (2 ** refinementDepth)
+        totalCells_coarsest = tuple(safe_int(ds / dx_coarsest) for ds in domainSize)
+        rootBlocks = tuple(safe_int(c / cpb) for c, cpb in zip(totalCells_coarsest, cellsPerBlock))
+
+        cylinderRadius_coarsest_LU = cylinderRadius / dx_coarsest
+
+        lattice_viscosity = (2 * cylinderRadius_coarsest_LU * peakInflowVelocity_LU) / reynoldsNumber
+        omega_shear = relaxation_rate_from_lattice_viscosity(lattice_viscosity)
+
+        self.db_file = db_file
+
+        self.params = {
+            'domainSize': domainSize,
+            'rootBlocks': rootBlocks,
+            'cellsPerBlock': cellsPerBlock,
+            'periodic': (0, 0 if walls else 1, 1),
+
+            'vtkWriteFrequency': outputFrequency,
+            "vtkBaseFolder": "FlowAroundCylinderOutput",
+
+            'runs': runs,
+            'stepsPerRun': steps_per_run,
+
+            'setup': 'FlowAroundCylinder',
+
+            'cylCenter': cylinderCenter,
+            'cylRadius': cylinderRadius,
+
+            'omegaShear': omega_shear,
+
+            'peakInflowVelocity': peakInflowVelocity_LU,
+            'inflowParabolicProfile': 1 if walls else 0,
+            'walls': walls,
+
+            'refinementProfile': refinementProfile,
+            'refinementDepth': refinementDepth,
+            'refinementRadii': refinementRadii,
+            'refinementTailLengths': refinementTailLengths
+        }
+
+    @wlb.member_callback
+    def config(self, **kwargs):
+        config_dict = {
+            "Parameters": self.params
+        }
+
+        wlb.log_info_on_root(f"Running with Parameters:\n{config_dict}")
+        return config_dict
+
+    @wlb.member_callback
+    def results_callback(self, **kwargs):
+        result = dict()
+
+        #   Parameters
+        result.update(self.params)
+
+        #   Build Information
+        result['compile_flags'] = wlb.build_info.compiler_flags
+        result['walberla_version'] = wlb.build_info.version
+        result['build_machine'] = wlb.build_info.build_machine
+
+        #   Run Information
+        result['executable'] = sys.argv[0]
+
+        #   MPI processes and OMP threads come by kwargs
+
+        #   Environment Information
+        result['num_nodes'] = int(os.environ.get('NUM_NODES', -1))
+        result['virtual_cores_per_node'] = int(os.environ.get('VIRTUAL_CORES_PER_NODE', -1))
+        result['processes_per_node'] = int(os.environ.get('PROCESSES_PER_NODE', -1))
+
+        #   Codegen Setup and Results
+        result.update(kwargs)
+
+        wlb.log_info_on_root(f"Storing Results:\n{result}")
+
+        sequenceValuesToScalars(result)
+        num_tries = 4
+        # check multiple times e.g. may fail when multiple benchmark processes are running
+        for num_try in range(num_tries):
+            try:
+                checkAndUpdateSchema(result, "runs", self.db_file)
+                storeSingle(result, "runs", self.db_file)
+                break
+            except sqlite3.OperationalError as e:
+                wlb.log_warning("Sqlite DB writing failed: try {}/{}  {}".format(num_try + 1, num_tries, str(e)))
+
+def run_flow_around_cylinder(db_file):
+    manager = wlb.ScenarioManager()
+
+    manager.add(FlowAroundCylinderScenario(
+        (16, 8, 2), (4, 4, 0), 1, 128, (64, 64, 64),
+        0.05, 100, True, 'ConcentricCylinderWake', 2,
+        refinementRadii=(2, 2, 2), refinementTailLengths=(8, 6, 6),
+        runs=1, steps_per_run=0, outputFrequency=100, db_file=db_file
+    ))
+
+if __name__ == '__main__':
+    # make runscript
+    from makeEmmyRunscript import get_script_d3q19_esotwist
+    scenario_script = "FlowAroundCylinderPerformance.py"
+    scripts_dir = 'cylinderrun'
+    os.makedirs(scripts_dir, exist_ok=True)
+    for num_nodes in [2, 4, 8, 16, 32, 64]:
+        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes.sh'
+        with open(filename, 'w') as shfile:
+            shfile.write(get_script_d3q19_esotwist(num_nodes, [1, 5, 10], scenario_script, walltime="02:00:00"))
+else:
+    run_flow_around_cylinder('FlowAroundCylinderScalingBenchmarkEmmy.sqlite')