Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Frederik Hennig
waLBerla
Commits
b11477b3
Commit
b11477b3
authored
Sep 20, 2021
by
Frederik Hennig
Browse files
SuperMUC benchmark stuff
parent
a08b8c30
Pipeline
#34313
failed with stages
in 15 minutes and 22 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
apps/benchmarks/NonuniformGridGenerated/NonuniformGridCodegenPerformance.cpp
View file @
b11477b3
...
...
@@ -604,7 +604,13 @@ void run(std::shared_ptr< Config > & config) {
auto
sbfs
=
std
::
make_shared
<
StructuredBlockForest
>
(
bfs
,
cellsPerBlock
[
0
],
cellsPerBlock
[
1
],
cellsPerBlock
[
2
]);
sbfs
->
createCellBoundingBoxes
();
vtk
::
writeDomainDecomposition
(
sbfs
,
"domainDecomposition"
,
vtkBaseFolder
);
// vtk::writeDomainDecomposition(sbfs, "domainDecomposition", vtkBaseFolder);
WALBERLA_LOG_INFO_ON_ROOT
(
"Blocks created: "
<<
setupBfs
.
getNumberOfBlocks
());
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 0 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
0
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 1 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
1
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 2 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
2
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 3 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
3
));
// Setup Boundary Handling
WALBERLA_LOG_INFO_ON_ROOT
(
"Setting up boundary conditions..."
);
...
...
@@ -623,11 +629,6 @@ void run(std::shared_ptr< Config > & config) {
// Maybe exit early
if
(
stepsPerRun
==
0
)
{
WALBERLA_LOG_INFO_ON_ROOT
(
"Domain setup complete and written to VTK. Terminating, since stepsPerRun == 0."
);
WALBERLA_LOG_INFO_ON_ROOT
(
"Blocks created: "
<<
setupBfs
.
getNumberOfBlocks
());
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 0 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
0
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 1 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
1
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 2 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
2
));
WALBERLA_LOG_INFO_ON_ROOT
(
"Level 3 Blocks: "
<<
setupBfs
.
getNumberOfBlocks
(
3
));
return
;
}
...
...
apps/benchmarks/NonuniformGridGenerated/Setups/FlowAroundCylinderPerformance.py
View file @
b11477b3
...
...
@@ -21,7 +21,7 @@ class FlowAroundCylinderScenario:
refinementProfile
,
refinementDepth
,
refinementRadii
=
(
0
,
0
,
0
),
refinementTailLengths
=
(
0
,
0
,
0
),
useUniformPackInfo
=
False
,
runs
=
10
,
steps_per_run
=
10
00
,
outputFrequency
=
0
,
runs
=
10
,
steps_per_run
=
10
,
outputFrequency
=
0
,
db_file
=
'benchmark.sqlite'
,
db_table
=
'runs'
,
vtkOutputFolder
=
'vtk_out'
):
dx_coarsest
=
cylinderRadius
/
cylinderRadius_coarsest_LU
...
...
@@ -146,9 +146,9 @@ def run_flow_around_cylinder_emmy():
manager
=
wlb
.
ScenarioManager
()
cylinderRadiusCoarseLU
=
64
ppn
=
os
.
environ
[
'PROCESSES_PER_NODE'
]
refinementSetups
=
[
{
'refinementProfile'
:
'refineEverywhere'
,
'refinementDepth'
:
2
,
...
...
@@ -182,8 +182,7 @@ def run_flow_around_cylinder_emmy():
**
setup
))
if
__name__
==
'__main__'
:
def
emmy_runscript
():
# make runscript
from
makeEmmyRunscript
import
get_script_d3q19_esotwist_True
scenario_script
=
"FlowAroundCylinderPerformance.py"
...
...
@@ -193,6 +192,113 @@ if __name__ == '__main__':
filename
=
f
'
{
scripts_dir
}
/flowAroundCylinderBM_
{
num_nodes
}
nodes.sh'
with
open
(
filename
,
'w'
)
as
shfile
:
shfile
.
write
(
get_script_d3q19_esotwist_True
(
num_nodes
,
[
1
,
5
,
10
],
scenario_script
,
walltime
=
"09:00:00"
))
def
run_flow_around_cylinder_supermuc
():
manager
=
wlb
.
ScenarioManager
()
cylinderRadiusCoarseLU
=
64
ppn
=
int
(
os
.
environ
[
'PROCESSES_PER_NODE'
])
num_nodes
=
int
(
os
.
environ
[
'NUM_NODES'
])
refinementSetups
=
[
{
'refinementProfile'
:
'refineEverywhere'
,
'refinementDepth'
:
2
,
'useUniformPackInfo'
:
True
,
'steps_per_run'
:
10
},
{
'refinementProfile'
:
'refineEverywhere'
,
'refinementDepth'
:
2
,
'useUniformPackInfo'
:
False
,
'steps_per_run'
:
5
},
{
'refinementProfile'
:
'ConcentricCylinderWake'
,
'refinementDepth'
:
3
,
'refinementRadii'
:
(
2
,
1.5
,
1.125
),
'refinementTailLengths'
:
(
10
,
8
,
6
),
'steps_per_run'
:
5
}
]
for
cells_per_block
in
[
64
,
32
,
16
,
8
]:
Lz_base
=
cells_per_block
/
64
Lz
=
safe_int
(
num_nodes
/
64
)
*
Lz_base
for
i
,
setup
in
enumerate
(
refinementSetups
):
output
=
f
"CylinderOutput_
{
ppn
}
ppn_
{
cells_per_block
}
cells_
{
i
}
"
manager
.
add
(
FlowAroundCylinderScenario
(
domainSize
=
(
18
,
8
,
Lz
),
cylinderCenter
=
(
4
,
4
,
0
),
cylinderRadius
=
1
,
cylinderRadius_coarsest_LU
=
cylinderRadiusCoarseLU
,
cellsPerBlock
=
(
cells_per_block
,
cells_per_block
,
cells_per_block
),
peakInflowVelocity_LU
=
0.05
,
reynoldsNumber
=
100
,
walls
=
False
,
runs
=
10
,
outputFrequency
=
0
,
db_file
=
f
"SuperMUCCylinderBlockSizeBenchmark.sqlite"
,
db_table
=
f
"setup
{
i
}
"
,
vtkOutputFolder
=
output
,
**
setup
))
def
run_flow_around_cylinder_supermuc_scaling
():
manager
=
wlb
.
ScenarioManager
()
cylinderRadiusCoarseLU
=
64
ppn
=
int
(
os
.
environ
[
'PROCESSES_PER_NODE'
])
num_nodes
=
int
(
os
.
environ
[
'NUM_NODES'
])
refinementSetups
=
[
{
'refinementProfile'
:
'ConcentricCylinderWake'
,
'refinementDepth'
:
3
,
'refinementRadii'
:
(
2
,
1.5
,
1.125
),
'refinementTailLengths'
:
(
10
,
8
,
6
),
'steps_per_run'
:
5
}
]
for
cells_per_block
in
[
64
,
32
,
16
]:
Lz_base
=
cells_per_block
/
64
Lz
=
safe_int
(
num_nodes
/
64
)
*
Lz_base
for
i
,
setup
in
enumerate
(
refinementSetups
):
output
=
f
"CylinderOutput_
{
ppn
}
ppn_
{
cells_per_block
}
cells_
{
i
}
"
manager
.
add
(
FlowAroundCylinderScenario
(
domainSize
=
(
18
,
8
,
Lz
),
cylinderCenter
=
(
4
,
4
,
0
),
cylinderRadius
=
1
,
cylinderRadius_coarsest_LU
=
cylinderRadiusCoarseLU
,
cellsPerBlock
=
(
cells_per_block
,
cells_per_block
,
cells_per_block
),
peakInflowVelocity_LU
=
0.05
,
reynoldsNumber
=
100
,
walls
=
False
,
runs
=
10
,
outputFrequency
=
0
,
db_file
=
f
"SuperMUCCylinderScalingBenchmark.sqlite"
,
db_table
=
f
"setup
{
i
}
"
,
vtkOutputFolder
=
output
,
**
setup
))
def
supermuc_runscripts
():
from
makeSuperMUCRunscript
import
get_script_d3q19_esotwist_True
scenario_script
=
"FlowAroundCylinderPerformance.py"
scripts_dir
=
'cylinder_scaling'
os
.
makedirs
(
scripts_dir
,
exist_ok
=
True
)
for
num_nodes
in
[
64
,
128
,
256
,
512
,
1024
,
2048
,
3072
]:
omp_threads
=
1
filename
=
f
'
{
scripts_dir
}
/flowAroundCylinderBM_
{
num_nodes
}
nodes_
{
omp_threads
}
threads.sh'
with
open
(
filename
,
'w'
)
as
shfile
:
shfile
.
write
(
get_script_d3q19_esotwist_True
(
num_nodes
,
omp_threads
,
scenario_script
,
walltime
=
"00:20:00"
))
if
__name__
==
'__main__'
:
supermuc_runscripts
()
else
:
#run_flow_around_cylinder_local()
run_flow_around_cylinder_emmy
()
# run_flow_around_cylinder_emmy()
run_flow_around_cylinder_supermuc_scaling
()
apps/benchmarks/NonuniformGridGenerated/Setups/makeSuperMUCRunscript.py
0 → 100644
View file @
b11477b3
SUPERMUC_PHYSICAL_CORES_PER_NODE
=
48
SCRIPT_TEMPLATE
=
"""#!/bin/bash -l
#
#SBATCH -J waLBerlaRefinementCodegenBenchmark
#
#SBATCH -o ./%x.%j.out
#SBATCH -e ./%x.%j.err
#
#
#SBATCH --mail-type=ALL
#SBATCH --mail-user=frederik.hennig@fau.de
#
#SBATCH --partition={partition}
#SBATCH --time={walltime}
#SBATCH --nodes={num_nodes}
#SBATCH --ntasks={num_procs}
#SBATCH --ntasks-per-node={procs_per_node}
#
#SBATCH --get-user-env
#SBATCH --account=pn29po
export NUM_NODES={num_nodes}
export PHYSICAL_CORES_PER_NODE=48
export PROCESSES_PER_NODE={procs_per_node}
cd ~/pycodegen
source env.sh
cd walberla/build/apps/benchmarks/NonuniformGridGenerated/
export OMP_NUM_THREADS={omp_threads}
mpiexec --bind-to socket ./NonuniformGridCodegenPerformance_{codegen_config} Setups/{scenario_script}
"""
def
safe_int
(
some_float
):
assert
int
(
some_float
)
==
some_float
return
int
(
some_float
)
def
get_partition
(
num_nodes
):
if
num_nodes
<=
16
:
return
"micro"
elif
num_nodes
<=
768
:
return
"general"
elif
num_nodes
<=
3072
:
return
"large"
else
:
raise
Exception
(
"Job too large."
)
def
get_script_d3q19_esotwist_True
(
num_nodes
,
num_omp_threads
,
scenario_script
,
name
=
"walberlaRefinementBenchmark"
,
walltime
=
"00:20:00"
):
config
=
'd3q19_esotwist_True'
procs_per_node
=
safe_int
(
SUPERMUC_PHYSICAL_CORES_PER_NODE
/
num_omp_threads
)
num_procs
=
num_nodes
*
procs_per_node
partition
=
get_partition
(
num_nodes
)
script
=
SCRIPT_TEMPLATE
.
format
(
partition
=
partition
,
walltime
=
walltime
,
num_nodes
=
num_nodes
,
num_procs
=
num_procs
,
procs_per_node
=
procs_per_node
,
omp_threads
=
num_omp_threads
,
codegen_config
=
config
,
scenario_script
=
scenario_script
)
return
script
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment