Commit b11477b3 authored by Frederik Hennig's avatar Frederik Hennig
Browse files

SuperMUC benchmark stuff

parent a08b8c30
Pipeline #34313 failed with stages
in 15 minutes and 22 seconds
......@@ -604,7 +604,13 @@ void run(std::shared_ptr< Config > & config) {
auto sbfs = std::make_shared< StructuredBlockForest >(bfs, cellsPerBlock[0], cellsPerBlock[1], cellsPerBlock[2]);
sbfs->createCellBoundingBoxes();
vtk::writeDomainDecomposition(sbfs, "domainDecomposition", vtkBaseFolder);
// vtk::writeDomainDecomposition(sbfs, "domainDecomposition", vtkBaseFolder);
WALBERLA_LOG_INFO_ON_ROOT("Blocks created: " << setupBfs.getNumberOfBlocks());
WALBERLA_LOG_INFO_ON_ROOT("Level 0 Blocks: " << setupBfs.getNumberOfBlocks(0));
WALBERLA_LOG_INFO_ON_ROOT("Level 1 Blocks: " << setupBfs.getNumberOfBlocks(1));
WALBERLA_LOG_INFO_ON_ROOT("Level 2 Blocks: " << setupBfs.getNumberOfBlocks(2));
WALBERLA_LOG_INFO_ON_ROOT("Level 3 Blocks: " << setupBfs.getNumberOfBlocks(3));
// Setup Boundary Handling
WALBERLA_LOG_INFO_ON_ROOT("Setting up boundary conditions...");
......@@ -623,11 +629,6 @@ void run(std::shared_ptr< Config > & config) {
// Maybe exit early
if(stepsPerRun == 0) {
WALBERLA_LOG_INFO_ON_ROOT("Domain setup complete and written to VTK. Terminating, since stepsPerRun == 0.");
WALBERLA_LOG_INFO_ON_ROOT("Blocks created: " << setupBfs.getNumberOfBlocks());
WALBERLA_LOG_INFO_ON_ROOT("Level 0 Blocks: " << setupBfs.getNumberOfBlocks(0));
WALBERLA_LOG_INFO_ON_ROOT("Level 1 Blocks: " << setupBfs.getNumberOfBlocks(1));
WALBERLA_LOG_INFO_ON_ROOT("Level 2 Blocks: " << setupBfs.getNumberOfBlocks(2));
WALBERLA_LOG_INFO_ON_ROOT("Level 3 Blocks: " << setupBfs.getNumberOfBlocks(3));
return;
}
......
......@@ -21,7 +21,7 @@ class FlowAroundCylinderScenario:
refinementProfile, refinementDepth,
refinementRadii=(0,0,0), refinementTailLengths=(0,0,0),
useUniformPackInfo=False,
runs=10, steps_per_run=1000, outputFrequency=0,
runs=10, steps_per_run=10, outputFrequency=0,
db_file='benchmark.sqlite', db_table='runs', vtkOutputFolder='vtk_out'):
dx_coarsest = cylinderRadius / cylinderRadius_coarsest_LU
......@@ -146,9 +146,9 @@ def run_flow_around_cylinder_emmy():
manager = wlb.ScenarioManager()
cylinderRadiusCoarseLU = 64
ppn = os.environ['PROCESSES_PER_NODE']
refinementSetups = [
{'refinementProfile': 'refineEverywhere',
'refinementDepth': 2,
......@@ -182,8 +182,7 @@ def run_flow_around_cylinder_emmy():
**setup
))
if __name__ == '__main__':
def emmy_runscript():
# make runscript
from makeEmmyRunscript import get_script_d3q19_esotwist_True
scenario_script = "FlowAroundCylinderPerformance.py"
......@@ -193,6 +192,113 @@ if __name__ == '__main__':
filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes.sh'
with open(filename, 'w') as shfile:
shfile.write(get_script_d3q19_esotwist_True(num_nodes, [1, 5, 10], scenario_script, walltime="09:00:00"))
def run_flow_around_cylinder_supermuc():
manager = wlb.ScenarioManager()
cylinderRadiusCoarseLU = 64
ppn = int(os.environ['PROCESSES_PER_NODE'])
num_nodes = int(os.environ['NUM_NODES'])
refinementSetups = [
{'refinementProfile': 'refineEverywhere',
'refinementDepth': 2,
'useUniformPackInfo': True,
'steps_per_run': 10},
{'refinementProfile': 'refineEverywhere',
'refinementDepth': 2,
'useUniformPackInfo': False,
'steps_per_run': 5},
{'refinementProfile': 'ConcentricCylinderWake',
'refinementDepth': 3,
'refinementRadii': (2, 1.5, 1.125),
'refinementTailLengths': (10, 8, 6),
'steps_per_run': 5}
]
for cells_per_block in [64, 32, 16, 8]:
Lz_base = cells_per_block / 64
Lz = safe_int(num_nodes / 64) * Lz_base
for i, setup in enumerate(refinementSetups):
output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"
manager.add(FlowAroundCylinderScenario(
domainSize=(18, 8, Lz),
cylinderCenter=(4, 4, 0),
cylinderRadius=1,
cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
peakInflowVelocity_LU=0.05,
reynoldsNumber=100,
walls=False,
runs=10, outputFrequency=0,
db_file=f"SuperMUCCylinderBlockSizeBenchmark.sqlite",
db_table=f"setup{i}",
vtkOutputFolder=output,
**setup
))
def run_flow_around_cylinder_supermuc_scaling():
manager = wlb.ScenarioManager()
cylinderRadiusCoarseLU = 64
ppn = int(os.environ['PROCESSES_PER_NODE'])
num_nodes = int(os.environ['NUM_NODES'])
refinementSetups = [
{'refinementProfile': 'ConcentricCylinderWake',
'refinementDepth': 3,
'refinementRadii': (2, 1.5, 1.125),
'refinementTailLengths': (10, 8, 6),
'steps_per_run': 5}
]
for cells_per_block in [64, 32, 16]:
Lz_base = cells_per_block / 64
Lz = safe_int(num_nodes / 64) * Lz_base
for i, setup in enumerate(refinementSetups):
output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"
manager.add(FlowAroundCylinderScenario(
domainSize=(18, 8, Lz),
cylinderCenter=(4, 4, 0),
cylinderRadius=1,
cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
peakInflowVelocity_LU=0.05,
reynoldsNumber=100,
walls=False,
runs=10, outputFrequency=0,
db_file=f"SuperMUCCylinderScalingBenchmark.sqlite",
db_table=f"setup{i}",
vtkOutputFolder=output,
**setup
))
def supermuc_runscripts():
from makeSuperMUCRunscript import get_script_d3q19_esotwist_True
scenario_script = "FlowAroundCylinderPerformance.py"
scripts_dir = 'cylinder_scaling'
os.makedirs(scripts_dir, exist_ok=True)
for num_nodes in [64, 128, 256, 512, 1024, 2048, 3072]:
omp_threads=1
filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes_{omp_threads}threads.sh'
with open(filename, 'w') as shfile:
shfile.write(get_script_d3q19_esotwist_True(num_nodes, omp_threads, scenario_script, walltime="00:20:00"))
if __name__ == '__main__':
supermuc_runscripts()
else:
#run_flow_around_cylinder_local()
run_flow_around_cylinder_emmy()
# run_flow_around_cylinder_emmy()
run_flow_around_cylinder_supermuc_scaling()
SUPERMUC_PHYSICAL_CORES_PER_NODE = 48
SCRIPT_TEMPLATE = """#!/bin/bash -l
#
#SBATCH -J waLBerlaRefinementCodegenBenchmark
#
#SBATCH -o ./%x.%j.out
#SBATCH -e ./%x.%j.err
#
#
#SBATCH --mail-type=ALL
#SBATCH --mail-user=frederik.hennig@fau.de
#
#SBATCH --partition={partition}
#SBATCH --time={walltime}
#SBATCH --nodes={num_nodes}
#SBATCH --ntasks={num_procs}
#SBATCH --ntasks-per-node={procs_per_node}
#
#SBATCH --get-user-env
#SBATCH --account=pn29po
export NUM_NODES={num_nodes}
export PHYSICAL_CORES_PER_NODE=48
export PROCESSES_PER_NODE={procs_per_node}
cd ~/pycodegen
source env.sh
cd walberla/build/apps/benchmarks/NonuniformGridGenerated/
export OMP_NUM_THREADS={omp_threads}
mpiexec --bind-to socket ./NonuniformGridCodegenPerformance_{codegen_config} Setups/{scenario_script}
"""
def safe_int(some_float):
assert int(some_float) == some_float
return int(some_float)
def get_partition(num_nodes):
if num_nodes <= 16:
return "micro"
elif num_nodes <= 768:
return "general"
elif num_nodes <= 3072:
return "large"
else:
raise Exception("Job too large.")
def get_script_d3q19_esotwist_True(num_nodes, num_omp_threads, scenario_script, name="walberlaRefinementBenchmark", walltime="00:20:00"):
config = 'd3q19_esotwist_True'
procs_per_node = safe_int(SUPERMUC_PHYSICAL_CORES_PER_NODE / num_omp_threads)
num_procs = num_nodes * procs_per_node
partition = get_partition(num_nodes)
script = SCRIPT_TEMPLATE.format(
partition=partition,
walltime=walltime,
num_nodes=num_nodes,
num_procs=num_procs,
procs_per_node=procs_per_node,
omp_threads=num_omp_threads,
codegen_config=config,
scenario_script=scenario_script
)
return script
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment