SuperMUC benchmark stuff

apps/benchmarks/NonuniformGridGenerated/NonuniformGridCodegenPerformance.cpp

@@ -604,7 +604,13 @@ void run(std::shared_ptr< Config > & config) {
    auto sbfs = std::make_shared< StructuredBlockForest >(bfs, cellsPerBlock[0], cellsPerBlock[1], cellsPerBlock[2]);
    sbfs->createCellBoundingBoxes();
 
-   vtk::writeDomainDecomposition(sbfs, "domainDecomposition", vtkBaseFolder);
+   // vtk::writeDomainDecomposition(sbfs, "domainDecomposition", vtkBaseFolder);
+
+   WALBERLA_LOG_INFO_ON_ROOT("Blocks created: " << setupBfs.getNumberOfBlocks());
+   WALBERLA_LOG_INFO_ON_ROOT("Level 0 Blocks: " << setupBfs.getNumberOfBlocks(0));
+   WALBERLA_LOG_INFO_ON_ROOT("Level 1 Blocks: " << setupBfs.getNumberOfBlocks(1));
+   WALBERLA_LOG_INFO_ON_ROOT("Level 2 Blocks: " << setupBfs.getNumberOfBlocks(2));
+   WALBERLA_LOG_INFO_ON_ROOT("Level 3 Blocks: " << setupBfs.getNumberOfBlocks(3));
 
    // Setup Boundary Handling
    WALBERLA_LOG_INFO_ON_ROOT("Setting up boundary conditions...");
@@ -623,11 +629,6 @@ void run(std::shared_ptr< Config > & config) {
    // Maybe exit early
    if(stepsPerRun == 0) {
       WALBERLA_LOG_INFO_ON_ROOT("Domain setup complete and written to VTK. Terminating, since stepsPerRun == 0.");
-      WALBERLA_LOG_INFO_ON_ROOT("Blocks created: " << setupBfs.getNumberOfBlocks());
-      WALBERLA_LOG_INFO_ON_ROOT("Level 0 Blocks: " << setupBfs.getNumberOfBlocks(0));
-      WALBERLA_LOG_INFO_ON_ROOT("Level 1 Blocks: " << setupBfs.getNumberOfBlocks(1));
-      WALBERLA_LOG_INFO_ON_ROOT("Level 2 Blocks: " << setupBfs.getNumberOfBlocks(2));
-      WALBERLA_LOG_INFO_ON_ROOT("Level 3 Blocks: " << setupBfs.getNumberOfBlocks(3));
       return;
    }
 

apps/benchmarks/NonuniformGridGenerated/Setups/FlowAroundCylinderPerformance.py

@@ -21,7 +21,7 @@ class FlowAroundCylinderScenario:
                  refinementProfile, refinementDepth,
                  refinementRadii=(0,0,0), refinementTailLengths=(0,0,0),
                  useUniformPackInfo=False,
-                 runs=10, steps_per_run=1000, outputFrequency=0,
+                 runs=10, steps_per_run=10, outputFrequency=0,
                  db_file='benchmark.sqlite', db_table='runs', vtkOutputFolder='vtk_out'):
 
         dx_coarsest = cylinderRadius / cylinderRadius_coarsest_LU
@@ -146,9 +146,9 @@ def run_flow_around_cylinder_emmy():
     manager = wlb.ScenarioManager()
 
     cylinderRadiusCoarseLU = 64
-    
+
     ppn = os.environ['PROCESSES_PER_NODE']
-    
+
     refinementSetups = [
         {'refinementProfile': 'refineEverywhere',
          'refinementDepth': 2,
@@ -182,8 +182,7 @@ def run_flow_around_cylinder_emmy():
                 **setup
             ))
 
-
-if __name__ == '__main__':
+def emmy_runscript():
     # make runscript
     from makeEmmyRunscript import get_script_d3q19_esotwist_True
     scenario_script = "FlowAroundCylinderPerformance.py"
@@ -193,6 +192,113 @@ if __name__ == '__main__':
         filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes.sh'
         with open(filename, 'w') as shfile:
             shfile.write(get_script_d3q19_esotwist_True(num_nodes, [1, 5, 10], scenario_script, walltime="09:00:00"))
+
+
+
+def run_flow_around_cylinder_supermuc():
+    manager = wlb.ScenarioManager()
+
+    cylinderRadiusCoarseLU = 64
+
+    ppn = int(os.environ['PROCESSES_PER_NODE'])
+    num_nodes = int(os.environ['NUM_NODES'])
+
+    refinementSetups = [
+        {'refinementProfile': 'refineEverywhere',
+         'refinementDepth': 2,
+         'useUniformPackInfo': True,
+         'steps_per_run': 10},
+        {'refinementProfile': 'refineEverywhere',
+         'refinementDepth': 2,
+         'useUniformPackInfo': False,
+         'steps_per_run': 5},
+        {'refinementProfile': 'ConcentricCylinderWake',
+         'refinementDepth': 3,
+         'refinementRadii': (2, 1.5, 1.125),
+         'refinementTailLengths': (10, 8, 6),
+         'steps_per_run': 5}
+    ]
+
+    for cells_per_block in [64, 32, 16, 8]:
+        Lz_base = cells_per_block / 64
+        Lz = safe_int(num_nodes / 64) * Lz_base
+
+        for i, setup in enumerate(refinementSetups):
+            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"
+
+            manager.add(FlowAroundCylinderScenario(
+                domainSize=(18, 8, Lz),
+                cylinderCenter=(4, 4, 0),
+                cylinderRadius=1,
+                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
+                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
+                peakInflowVelocity_LU=0.05,
+                reynoldsNumber=100,
+                walls=False,
+                runs=10, outputFrequency=0,
+                db_file=f"SuperMUCCylinderBlockSizeBenchmark.sqlite",
+                db_table=f"setup{i}",
+                vtkOutputFolder=output,
+                **setup
+            ))
+
+
+
+def run_flow_around_cylinder_supermuc_scaling():
+    manager = wlb.ScenarioManager()
+
+    cylinderRadiusCoarseLU = 64
+
+    ppn = int(os.environ['PROCESSES_PER_NODE'])
+    num_nodes = int(os.environ['NUM_NODES'])
+
+    refinementSetups = [
+        {'refinementProfile': 'ConcentricCylinderWake',
+         'refinementDepth': 3,
+         'refinementRadii': (2, 1.5, 1.125),
+         'refinementTailLengths': (10, 8, 6),
+         'steps_per_run': 5}
+    ]
+
+    for cells_per_block in [64, 32, 16]:
+        Lz_base = cells_per_block / 64
+        Lz = safe_int(num_nodes / 64) * Lz_base
+
+        for i, setup in enumerate(refinementSetups):
+            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"
+
+            manager.add(FlowAroundCylinderScenario(
+                domainSize=(18, 8, Lz),
+                cylinderCenter=(4, 4, 0),
+                cylinderRadius=1,
+                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
+                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
+                peakInflowVelocity_LU=0.05,
+                reynoldsNumber=100,
+                walls=False,
+                runs=10, outputFrequency=0,
+                db_file=f"SuperMUCCylinderScalingBenchmark.sqlite",
+                db_table=f"setup{i}",
+                vtkOutputFolder=output,
+                **setup
+            ))
+
+
+def supermuc_runscripts():
+    from makeSuperMUCRunscript import get_script_d3q19_esotwist_True
+    scenario_script = "FlowAroundCylinderPerformance.py"
+    scripts_dir = 'cylinder_scaling'
+    os.makedirs(scripts_dir, exist_ok=True)
+    for num_nodes in [64, 128, 256, 512, 1024, 2048, 3072]:
+        omp_threads=1
+        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes_{omp_threads}threads.sh'
+        with open(filename, 'w') as shfile:
+            shfile.write(get_script_d3q19_esotwist_True(num_nodes, omp_threads, scenario_script, walltime="00:20:00"))
+
+
+if __name__ == '__main__':
+    supermuc_runscripts()
 else:
     #run_flow_around_cylinder_local()
-    run_flow_around_cylinder_emmy()
+    # run_flow_around_cylinder_emmy()
+    run_flow_around_cylinder_supermuc_scaling()

apps/benchmarks/NonuniformGridGenerated/Setups/makeSuperMUCRunscript.py

+
+SUPERMUC_PHYSICAL_CORES_PER_NODE = 48
+
+SCRIPT_TEMPLATE = """#!/bin/bash -l
+#
+#SBATCH -J waLBerlaRefinementCodegenBenchmark
+#
+#SBATCH -o ./%x.%j.out
+#SBATCH -e ./%x.%j.err
+#
+#
+#SBATCH --mail-type=ALL
+#SBATCH --mail-user=frederik.hennig@fau.de
+#
+#SBATCH --partition={partition}
+#SBATCH --time={walltime}
+#SBATCH --nodes={num_nodes}
+#SBATCH --ntasks={num_procs}
+#SBATCH --ntasks-per-node={procs_per_node}
+#
+#SBATCH --get-user-env
+#SBATCH --account=pn29po
+
+export NUM_NODES={num_nodes}
+export PHYSICAL_CORES_PER_NODE=48
+export PROCESSES_PER_NODE={procs_per_node}
+
+cd ~/pycodegen
+source env.sh
+cd walberla/build/apps/benchmarks/NonuniformGridGenerated/
+
+export OMP_NUM_THREADS={omp_threads}
+mpiexec --bind-to socket ./NonuniformGridCodegenPerformance_{codegen_config} Setups/{scenario_script}
+
+"""
+
+def safe_int(some_float):
+    assert int(some_float) == some_float
+    return int(some_float)
+
+def get_partition(num_nodes):
+    if num_nodes <= 16:
+        return "micro"
+    elif num_nodes <= 768:
+        return "general"
+    elif num_nodes <= 3072:
+        return "large"
+    else:
+        raise Exception("Job too large.")
+
+def get_script_d3q19_esotwist_True(num_nodes, num_omp_threads, scenario_script, name="walberlaRefinementBenchmark", walltime="00:20:00"):
+    config = 'd3q19_esotwist_True'
+    procs_per_node = safe_int(SUPERMUC_PHYSICAL_CORES_PER_NODE / num_omp_threads)
+    num_procs = num_nodes * procs_per_node
+    partition = get_partition(num_nodes)
+    script = SCRIPT_TEMPLATE.format(
+        partition=partition,
+        walltime=walltime,
+        num_nodes=num_nodes,
+        num_procs=num_procs,
+        procs_per_node=procs_per_node,
+        omp_threads=num_omp_threads,
+        codegen_config=config,
+        scenario_script=scenario_script
+    )
+
+    return script