FlowAroundCylinderPerformance.py 10.6 KB
Newer Older
1
2
3
4
5
6
import waLBerla as wlb
import os
import sys
import sqlite3
from waLBerla.tools.sqlitedb import sequenceValuesToScalars, checkAndUpdateSchema, storeSingle

Frederik Hennig's avatar
Frederik Hennig committed
7
8
9
10

def relaxation_rate_from_lattice_viscosity(nu):
    return 2 / (6 * nu + 1)

11
12
13
14
15

def safe_int(some_float):
    assert int(some_float) == some_float, f"{some_float} is not an integer. :("
    return int(some_float)

Frederik Hennig's avatar
Frederik Hennig committed
16

17
class FlowAroundCylinderScenario:
Frederik Hennig's avatar
Frederik Hennig committed
18
    def __init__(self, domainSize, cylinderCenter, cylinderRadius, cylinderRadius_coarsest_LU,
19
20
21
22
                 cellsPerBlock,
                 peakInflowVelocity_LU, reynoldsNumber, walls,
                 refinementProfile, refinementDepth,
                 refinementRadii=(0,0,0), refinementTailLengths=(0,0,0),
Frederik Hennig's avatar
Frederik Hennig committed
23
                 useUniformPackInfo=False,
Frederik Hennig's avatar
Frederik Hennig committed
24
                 runs=10, steps_per_run=10, outputFrequency=0,
Frederik Hennig's avatar
Frederik Hennig committed
25
                 db_file='benchmark.sqlite', db_table='runs', vtkOutputFolder='vtk_out'):
26

Frederik Hennig's avatar
Frederik Hennig committed
27
        dx_coarsest = cylinderRadius / cylinderRadius_coarsest_LU
28
29
30
31
32
33
34
35
36
        totalCells_coarsest = tuple(safe_int(ds / dx_coarsest) for ds in domainSize)
        rootBlocks = tuple(safe_int(c / cpb) for c, cpb in zip(totalCells_coarsest, cellsPerBlock))

        cylinderRadius_coarsest_LU = cylinderRadius / dx_coarsest

        lattice_viscosity = (2 * cylinderRadius_coarsest_LU * peakInflowVelocity_LU) / reynoldsNumber
        omega_shear = relaxation_rate_from_lattice_viscosity(lattice_viscosity)

        self.db_file = db_file
Frederik Hennig's avatar
Frederik Hennig committed
37
        self.db_table = db_table
38

Frederik Hennig's avatar
Frederik Hennig committed
39
40
41
42
        self.num_nodes = int(os.environ['NUM_NODES'])
        self.physical_cores_per_node = int(os.environ['PHYSICAL_CORES_PER_NODE'])
        self.ppn = int(os.environ['PROCESSES_PER_NODE'])

43
44
45
46
47
48
49
        self.params = {
            'domainSize': domainSize,
            'rootBlocks': rootBlocks,
            'cellsPerBlock': cellsPerBlock,
            'periodic': (0, 0 if walls else 1, 1),

            'vtkWriteFrequency': outputFrequency,
Frederik Hennig's avatar
Frederik Hennig committed
50
            "vtkBaseFolder": vtkOutputFolder,
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

            'runs': runs,
            'stepsPerRun': steps_per_run,

            'setup': 'FlowAroundCylinder',

            'cylCenter': cylinderCenter,
            'cylRadius': cylinderRadius,

            'omegaShear': omega_shear,

            'peakInflowVelocity': peakInflowVelocity_LU,
            'inflowParabolicProfile': 1 if walls else 0,
            'walls': walls,

            'refinementProfile': refinementProfile,
            'refinementDepth': refinementDepth,
            'refinementRadii': refinementRadii,
Frederik Hennig's avatar
Frederik Hennig committed
69
            'refinementTailLengths': refinementTailLengths,
Frederik Hennig's avatar
Frederik Hennig committed
70
71
72
73
74
            'useUniformPackInfo': useUniformPackInfo,

            'numNodes': self.num_nodes,
            'physicalCoresPerNode': self.physical_cores_per_node,
            'processesPerNode': self.ppn
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
        }

    @wlb.member_callback
    def config(self, **kwargs):
        config_dict = {
            "Parameters": self.params
        }

        wlb.log_info_on_root(f"Running with Parameters:\n{config_dict}")
        return config_dict

    @wlb.member_callback
    def results_callback(self, **kwargs):
        result = dict()

        #   Parameters
        result.update(self.params)

        #   Build Information
        result['compile_flags'] = wlb.build_info.compiler_flags
        result['walberla_version'] = wlb.build_info.version
        result['build_machine'] = wlb.build_info.build_machine

        #   Run Information
        result['executable'] = sys.argv[0]

Frederik Hennig's avatar
Frederik Hennig committed
101
102
        #   Processes, Nodes, Threads, etc. come from C++
        #   Environment Setup, Codegen Setup and Results
103
104
105
106
107
108
109
110
111
        result.update(kwargs)

        wlb.log_info_on_root(f"Storing Results:\n{result}")

        sequenceValuesToScalars(result)
        num_tries = 4
        # check multiple times e.g. may fail when multiple benchmark processes are running
        for num_try in range(num_tries):
            try:
Frederik Hennig's avatar
Frederik Hennig committed
112
113
                checkAndUpdateSchema(result, self.db_table, self.db_file)
                storeSingle(result, self.db_table, self.db_file)
114
115
116
117
                break
            except sqlite3.OperationalError as e:
                wlb.log_warning("Sqlite DB writing failed: try {}/{}  {}".format(num_try + 1, num_tries, str(e)))

Frederik Hennig's avatar
Frederik Hennig committed
118
def run_flow_around_cylinder_local():
119
120
    manager = wlb.ScenarioManager()

Frederik Hennig's avatar
Frederik Hennig committed
121
122
123
124
    os.environ['NUM_NODES'] = '1'
    os.environ['PHYSICAL_CORES_PER_NODE'] = '6'
    os.environ['PROCESSES_PER_NODE'] = str(wlb.mpi.numProcesses())

Frederik Hennig's avatar
Frederik Hennig committed
125
126
    cells_per_block = 8

127
    manager.add(FlowAroundCylinderScenario(
Frederik Hennig's avatar
Frederik Hennig committed
128
129
130
        domainSize=(18, 8, 1),
        cylinderCenter=(4, 4, 0),
        cylinderRadius=1,
Frederik Hennig's avatar
Frederik Hennig committed
131
        cylinderRadius_coarsest_LU=8,
Frederik Hennig's avatar
Frederik Hennig committed
132
133
134
135
136
        cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
        peakInflowVelocity_LU=0.05,
        reynoldsNumber=100,
        walls=False,
        refinementProfile='refineEverywhere',
Frederik Hennig's avatar
Frederik Hennig committed
137
        refinementDepth=1,
Frederik Hennig's avatar
Frederik Hennig committed
138
139
        refinementRadii=(2, 1.5, 1.125), refinementTailLengths=(10, 8, 6),
        useUniformPackInfo=True,
Frederik Hennig's avatar
Frederik Hennig committed
140
141
        runs=1, steps_per_run=101, outputFrequency=20,
        db_file='localFlowAroundCylinderBenchmark.sqlite'
142
143
    ))

Frederik Hennig's avatar
Frederik Hennig committed
144

Frederik Hennig's avatar
Frederik Hennig committed
145
def run_flow_around_cylinder_emmy():
Frederik Hennig's avatar
Frederik Hennig committed
146
147
148
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64
Frederik Hennig's avatar
Frederik Hennig committed
149

Frederik Hennig's avatar
Frederik Hennig committed
150
    ppn = os.environ['PROCESSES_PER_NODE']
Frederik Hennig's avatar
Frederik Hennig committed
151

Frederik Hennig's avatar
Frederik Hennig committed
152
153
    refinementSetups = [
        {'refinementProfile': 'refineEverywhere',
Frederik Hennig's avatar
Frederik Hennig committed
154
         'refinementDepth': 2,
Frederik Hennig's avatar
Frederik Hennig committed
155
156
         'useUniformPackInfo': True},
        {'refinementProfile': 'refineEverywhere',
Frederik Hennig's avatar
Frederik Hennig committed
157
         'refinementDepth': 2,
Frederik Hennig's avatar
Frederik Hennig committed
158
159
160
161
162
163
164
165
166
         'useUniformPackInfo': False},
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6)}
    ]

    for cells_per_block in [64, 32, 16, 8]:
        for i, setup in enumerate(refinementSetups):
Frederik Hennig's avatar
Frederik Hennig committed
167
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"
Frederik Hennig's avatar
Frederik Hennig committed
168
169
170
171
172
173
174
175
176
177

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, 1),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
Frederik Hennig's avatar
Frederik Hennig committed
178
                runs=10, steps_per_run=5, outputFrequency=0,
Frederik Hennig's avatar
Frederik Hennig committed
179
180
                db_file=f"CylinderBenchmark.sqlite",
                db_table=f"setup{i}",
Frederik Hennig's avatar
Frederik Hennig committed
181
182
183
184
                vtkOutputFolder=output,
                **setup
            ))

Frederik Hennig's avatar
Frederik Hennig committed
185
def emmy_runscript():
186
    # make runscript
Frederik Hennig's avatar
Frederik Hennig committed
187
    from makeEmmyRunscript import get_script_d3q19_esotwist_True
188
189
190
    scenario_script = "FlowAroundCylinderPerformance.py"
    scripts_dir = 'cylinderrun'
    os.makedirs(scripts_dir, exist_ok=True)
Frederik Hennig's avatar
Frederik Hennig committed
191
    for num_nodes in [64]:
192
193
        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes.sh'
        with open(filename, 'w') as shfile:
Frederik Hennig's avatar
Frederik Hennig committed
194
            shfile.write(get_script_d3q19_esotwist_True(num_nodes, [1, 5, 10], scenario_script, walltime="09:00:00"))
Frederik Hennig's avatar
Frederik Hennig committed
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300



def run_flow_around_cylinder_supermuc():
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64

    ppn = int(os.environ['PROCESSES_PER_NODE'])
    num_nodes = int(os.environ['NUM_NODES'])

    refinementSetups = [
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': True,
         'steps_per_run': 10},
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': False,
         'steps_per_run': 5},
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6),
         'steps_per_run': 5}
    ]

    for cells_per_block in [64, 32, 16, 8]:
        Lz_base = cells_per_block / 64
        Lz = safe_int(num_nodes / 64) * Lz_base

        for i, setup in enumerate(refinementSetups):
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, Lz),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
                runs=10, outputFrequency=0,
                db_file=f"SuperMUCCylinderBlockSizeBenchmark.sqlite",
                db_table=f"setup{i}",
                vtkOutputFolder=output,
                **setup
            ))



def run_flow_around_cylinder_supermuc_scaling():
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64

    ppn = int(os.environ['PROCESSES_PER_NODE'])
    num_nodes = int(os.environ['NUM_NODES'])

    refinementSetups = [
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6),
         'steps_per_run': 5}
    ]

    for cells_per_block in [64, 32, 16]:
        Lz_base = cells_per_block / 64
        Lz = safe_int(num_nodes / 64) * Lz_base

        for i, setup in enumerate(refinementSetups):
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, Lz),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
                runs=10, outputFrequency=0,
                db_file=f"SuperMUCCylinderScalingBenchmark.sqlite",
                db_table=f"setup{i}",
                vtkOutputFolder=output,
                **setup
            ))


def supermuc_runscripts():
    from makeSuperMUCRunscript import get_script_d3q19_esotwist_True
    scenario_script = "FlowAroundCylinderPerformance.py"
    scripts_dir = 'cylinder_scaling'
    os.makedirs(scripts_dir, exist_ok=True)
    for num_nodes in [64, 128, 256, 512, 1024, 2048, 3072]:
        omp_threads=1
        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes_{omp_threads}threads.sh'
        with open(filename, 'w') as shfile:
            shfile.write(get_script_d3q19_esotwist_True(num_nodes, omp_threads, scenario_script, walltime="00:20:00"))


if __name__ == '__main__':
    supermuc_runscripts()
301
else:
Frederik Hennig's avatar
Frederik Hennig committed
302
    #run_flow_around_cylinder_local()
Frederik Hennig's avatar
Frederik Hennig committed
303
304
    # run_flow_around_cylinder_emmy()
    run_flow_around_cylinder_supermuc_scaling()