FlowAroundCylinderPerformance.py

import waLBerla as wlb
import os
import sys
import sqlite3
from waLBerla.tools.sqlitedb import sequenceValuesToScalars, checkAndUpdateSchema, storeSingle


def relaxation_rate_from_lattice_viscosity(nu):
    return 2 / (6 * nu + 1)


def safe_int(some_float):
    assert int(some_float) == some_float, f"{some_float} is not an integer. :("
    return int(some_float)


class FlowAroundCylinderScenario:
    def __init__(self, domainSize, cylinderCenter, cylinderRadius, cylinderRadius_coarsest_LU,
                 cellsPerBlock,
                 peakInflowVelocity_LU, reynoldsNumber, walls,
                 refinementProfile, refinementDepth,
                 refinementRadii=(0,0,0), refinementTailLengths=(0,0,0),
                 useUniformPackInfo=False,
                 runs=10, steps_per_run=10, outputFrequency=0,
                 db_file='benchmark.sqlite', db_table='runs', vtkOutputFolder='vtk_out'):

        dx_coarsest = cylinderRadius / cylinderRadius_coarsest_LU
        totalCells_coarsest = tuple(safe_int(ds / dx_coarsest) for ds in domainSize)
        rootBlocks = tuple(safe_int(c / cpb) for c, cpb in zip(totalCells_coarsest, cellsPerBlock))

        cylinderRadius_coarsest_LU = cylinderRadius / dx_coarsest

        lattice_viscosity = (2 * cylinderRadius_coarsest_LU * peakInflowVelocity_LU) / reynoldsNumber
        omega_shear = relaxation_rate_from_lattice_viscosity(lattice_viscosity)

        self.db_file = db_file
        self.db_table = db_table

        self.num_nodes = int(os.environ['NUM_NODES'])
        self.physical_cores_per_node = int(os.environ['PHYSICAL_CORES_PER_NODE'])
        self.ppn = int(os.environ['PROCESSES_PER_NODE'])

        self.params = {
            'domainSize': domainSize,
            'rootBlocks': rootBlocks,
            'cellsPerBlock': cellsPerBlock,
            'periodic': (0, 0 if walls else 1, 1),

            'vtkWriteFrequency': outputFrequency,
            "vtkBaseFolder": vtkOutputFolder,

            'runs': runs,
            'stepsPerRun': steps_per_run,

            'setup': 'FlowAroundCylinder',

            'cylCenter': cylinderCenter,
            'cylRadius': cylinderRadius,

            'omegaShear': omega_shear,

            'peakInflowVelocity': peakInflowVelocity_LU,
            'inflowParabolicProfile': 1 if walls else 0,
            'walls': walls,

            'refinementProfile': refinementProfile,
            'refinementDepth': refinementDepth,
            'refinementRadii': refinementRadii,
            'refinementTailLengths': refinementTailLengths,
            'useUniformPackInfo': useUniformPackInfo,

            'numNodes': self.num_nodes,
            'physicalCoresPerNode': self.physical_cores_per_node,
            'processesPerNode': self.ppn
        }

    @wlb.member_callback
    def config(self, **kwargs):
        config_dict = {
            "Parameters": self.params
        }

        wlb.log_info_on_root(f"Running with Parameters:\n{config_dict}")
        return config_dict

    @wlb.member_callback
    def results_callback(self, **kwargs):
        result = dict()

        #   Parameters
        result.update(self.params)

        #   Build Information
        result['compile_flags'] = wlb.build_info.compiler_flags
        result['walberla_version'] = wlb.build_info.version
        result['build_machine'] = wlb.build_info.build_machine

        #   Run Information
        result['executable'] = sys.argv[0]

        #   Processes, Nodes, Threads, etc. come from C++
        #   Environment Setup, Codegen Setup and Results
        result.update(kwargs)

        wlb.log_info_on_root(f"Storing Results:\n{result}")

        sequenceValuesToScalars(result)
        num_tries = 4
        # check multiple times e.g. may fail when multiple benchmark processes are running
        for num_try in range(num_tries):
            try:
                checkAndUpdateSchema(result, self.db_table, self.db_file)
                storeSingle(result, self.db_table, self.db_file)
                break
            except sqlite3.OperationalError as e:
                wlb.log_warning("Sqlite DB writing failed: try {}/{}  {}".format(num_try + 1, num_tries, str(e)))

def run_flow_around_cylinder_local():
    manager = wlb.ScenarioManager()

    os.environ['NUM_NODES'] = '1'
    os.environ['PHYSICAL_CORES_PER_NODE'] = '6'
    os.environ['PROCESSES_PER_NODE'] = str(wlb.mpi.numProcesses())

    cells_per_block = 8

    manager.add(FlowAroundCylinderScenario(
        domainSize=(18, 8, 1),
        cylinderCenter=(4, 4, 0),
        cylinderRadius=1,
        cylinderRadius_coarsest_LU=8,
        cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
        peakInflowVelocity_LU=0.05,
        reynoldsNumber=100,
        walls=False,
        refinementProfile='refineEverywhere',
        refinementDepth=1,
        refinementRadii=(2, 1.5, 1.125), refinementTailLengths=(10, 8, 6),
        useUniformPackInfo=True,
        runs=1, steps_per_run=101, outputFrequency=20,
        db_file='localFlowAroundCylinderBenchmark.sqlite'
    ))


def run_flow_around_cylinder_emmy():
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64

    ppn = os.environ['PROCESSES_PER_NODE']

    refinementSetups = [
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': True},
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': False},
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6)}
    ]

    for cells_per_block in [64, 32, 16, 8]:
        for i, setup in enumerate(refinementSetups):
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, 1),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
                runs=10, steps_per_run=5, outputFrequency=0,
                db_file=f"CylinderBenchmark.sqlite",
                db_table=f"setup{i}",
                vtkOutputFolder=output,
                **setup
            ))

def emmy_runscript():
    # make runscript
    from makeEmmyRunscript import get_script_d3q19_esotwist_True
    scenario_script = "FlowAroundCylinderPerformance.py"
    scripts_dir = 'cylinderrun'
    os.makedirs(scripts_dir, exist_ok=True)
    for num_nodes in [64]:
        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes.sh'
        with open(filename, 'w') as shfile:
            shfile.write(get_script_d3q19_esotwist_True(num_nodes, [1, 5, 10], scenario_script, walltime="09:00:00"))



def run_flow_around_cylinder_supermuc():
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64

    ppn = int(os.environ['PROCESSES_PER_NODE'])
    num_nodes = int(os.environ['NUM_NODES'])

    refinementSetups = [
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': True,
         'steps_per_run': 10},
        {'refinementProfile': 'refineEverywhere',
         'refinementDepth': 2,
         'useUniformPackInfo': False,
         'steps_per_run': 5},
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6),
         'steps_per_run': 5}
    ]

    for cells_per_block in [64, 32, 16, 8]:
        Lz_base = cells_per_block / 64
        Lz = safe_int(num_nodes / 64) * Lz_base

        for i, setup in enumerate(refinementSetups):
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, Lz),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
                runs=10, outputFrequency=0,
                db_file=f"SuperMUCCylinderBlockSizeBenchmark.sqlite",
                db_table=f"setup{i}",
                vtkOutputFolder=output,
                **setup
            ))



def run_flow_around_cylinder_supermuc_scaling():
    manager = wlb.ScenarioManager()

    cylinderRadiusCoarseLU = 64

    ppn = int(os.environ['PROCESSES_PER_NODE'])
    num_nodes = int(os.environ['NUM_NODES'])

    refinementSetups = [
        {'refinementProfile': 'ConcentricCylinderWake',
         'refinementDepth': 3,
         'refinementRadii': (2, 1.5, 1.125),
         'refinementTailLengths': (10, 8, 6),
         'steps_per_run': 5}
    ]

    for cells_per_block in [64, 32, 16]:
        Lz_base = cells_per_block / 64
        Lz = safe_int(num_nodes / 64) * Lz_base

        for i, setup in enumerate(refinementSetups):
            output = f"CylinderOutput_{ppn}ppn_{cells_per_block}cells_{i}"

            manager.add(FlowAroundCylinderScenario(
                domainSize=(18, 8, Lz),
                cylinderCenter=(4, 4, 0),
                cylinderRadius=1,
                cylinderRadius_coarsest_LU=cylinderRadiusCoarseLU,
                cellsPerBlock=(cells_per_block, cells_per_block, cells_per_block),
                peakInflowVelocity_LU=0.05,
                reynoldsNumber=100,
                walls=False,
                runs=10, outputFrequency=0,
                db_file=f"SuperMUCCylinderScalingBenchmark.sqlite",
                db_table=f"setup{i}",
                vtkOutputFolder=output,
                **setup
            ))


def supermuc_runscripts():
    from makeSuperMUCRunscript import get_script_d3q19_esotwist_True
    scenario_script = "FlowAroundCylinderPerformance.py"
    scripts_dir = 'cylinder_scaling'
    os.makedirs(scripts_dir, exist_ok=True)
    for num_nodes in [64, 128, 256, 512, 1024, 2048, 3072]:
        omp_threads=1
        filename = f'{scripts_dir}/flowAroundCylinderBM_{num_nodes}nodes_{omp_threads}threads.sh'
        with open(filename, 'w') as shfile:
            shfile.write(get_script_d3q19_esotwist_True(num_nodes, omp_threads, scenario_script, walltime="00:20:00"))


if __name__ == '__main__':
    supermuc_runscripts()
else:
    #run_flow_around_cylinder_local()
    # run_flow_around_cylinder_emmy()
    run_flow_around_cylinder_supermuc_scaling()