Commit 9b425340 authored by Frederik Hennig's avatar Frederik Hennig
Browse files

Updated LDC benchmark

parent 3935d76c
Pipeline #34192 failed with stages
in 14 minutes and 46 seconds
......@@ -301,6 +301,194 @@ class LDC : public SetupBase {
};
/////////////////////////////////////////
/// Flow Around Cylinder ///
/////////////////////////////////////////
class FlowAroundCylinder : public SetupBase
{
friend class ConcentricCylinderWakeRefinement;
private:
Vector3< real_t > cylCenter_;
const real_t cylRadius_;
const real_t peakInflowVelocity_;
const bool inflowParabolicProfile_;
const bool walls_;
const real_t omegaShear_;
const std::string refinementProfile_;
const uint_t refinementDepth_;
Vector3< real_t > refinementRadii_;
Vector3< real_t > refinementTailLengths_;
const FlagUID noSlipFlagUID_;
const FlagUID inflowFlagUID_;
const FlagUID outflowFlagUID_;
std::shared_ptr< lbm::GeneratedNoSlip > noSlipObject;
std::shared_ptr< lbm::GeneratedUBB > inflowObject;
std::shared_ptr< lbm::GeneratedOutflow > outflowObject;
public:
FlowAroundCylinder(const Vector3< real_t > cylCenter, const real_t cylRadius,
const real_t peakInflowVelocity, const bool inflowParabolicProfile, const bool walls,
const real_t omegaShear,
const std::string refinementProfile, const uint_t refinementDepth,
const Vector3< real_t > refinementRadii, const Vector3< real_t > refinementTailLengths)
: cylRadius_(cylRadius),
peakInflowVelocity_(peakInflowVelocity), inflowParabolicProfile_(inflowParabolicProfile), walls_(walls),
omegaShear_(omegaShear),
refinementProfile_(refinementProfile), refinementDepth_(refinementDepth),
noSlipFlagUID_("NoSlip"), inflowFlagUID_("Inflow"), outflowFlagUID_("Outflow")
{
cylCenter_ = cylCenter;
refinementRadii_ = refinementRadii;
refinementTailLengths_ = refinementTailLengths;
}
real_t shearRelaxationRate() const override {
return omegaShear_;
}
Vector3< real_t > acceleration() const override { return Vector3< real_t >(0.0); }
RefinementSelectionFunctor refinementSelector() override;
void setupBoundaryFlagField(StructuredBlockForest & sbfs, const BlockDataID flagFieldID) override {
for(auto bIt = sbfs.begin(); bIt != sbfs.end(); ++bIt)
{
Block& b = dynamic_cast< Block& >(*bIt);
uint_t level = b.getLevel();
auto flagField = b.getData< FlagField_T >(flagFieldID);
uint8_t noslipFlag = flagField->registerFlag(noSlipFlagUID_);
uint8_t inflowFlag = flagField->registerFlag(inflowFlagUID_);
uint8_t outflowFlag = flagField->registerFlag(outflowFlagUID_);
for(auto cIt = flagField->beginWithGhostLayerXYZ(2); cIt != flagField->end(); ++cIt){
Cell localCell = cIt.cell();
Cell globalCell(localCell);
sbfs.transformBlockLocalToGlobalCell(globalCell, b);
Vector3< real_t > cellCenter(sbfs.getCellCenter(globalCell, level));
if(walls_ && ( globalCell.y() < 0 || globalCell.y() >= cell_idx_c(sbfs.getNumberOfYCells(level)) )){
flagField->addFlag(localCell, noslipFlag);
} else {
// No wall -> Inflow or Outflow
if(globalCell.x() < 0){
flagField->addFlag(localCell, inflowFlag);
}
else if(globalCell.x() >= cell_idx_c(sbfs.getNumberOfXCells(level))){
flagField->addFlag(localCell, inflowFlag);
}
else{
Vector3< real_t > distanceInPlane(cellCenter - cylCenter_);
distanceInPlane[2] = real_t(0.0);
if(distanceInPlane.length() <= cylRadius_){
flagField->addFlag(localCell, noslipFlag);
}
}
}
}
}
}
void getBoundarySweeps(std::shared_ptr< StructuredBlockForest > & sbfs,
const BlockDataID pdfFieldID,
const BlockDataID flagFieldID,
const FlagUID fluidFlagUID,
std::vector< std::function< void( Block *) > > & sweeps) override {
// NoSlip Walls
noSlipObject = std::make_shared< lbm::GeneratedNoSlip >(sbfs, pdfFieldID);
noSlipObject->fillFromFlagField< FlagField_T >(sbfs, flagFieldID, noSlipFlagUID_, fluidFlagUID);
BoundarySweep< LatticeModel_T, lbm::GeneratedNoSlip, LatticeModel_T::inplace > noSlipSweep(*noSlipObject, pdfFieldID);
sweeps.emplace_back(noSlipSweep);
real_t ySize = sbfs->getDomain().ySize();
real_t R = ySize / real_t(2.0);
// Inflow Velocity Profile
VelocityCallback vCallback = [this, R](const Cell & cell, const shared_ptr< StructuredBlockForest >& blocks, IBlock& ib){
if(this->walls_){
Block & b = dynamic_cast< Block & >(ib);
Cell globalCell(cell);
blocks->transformBlockLocalToGlobalCell(globalCell, b);
Vector3< real_t > cellCenter(blocks->getCellCenter(globalCell, b.getLevel()));
real_t r = abs(cellCenter[1] - R);
real_t velocityFactor = (R*R - r*r) / (R*R);
return Vector3< real_t > (peakInflowVelocity_ * velocityFactor, real_c(0.0), real_c(0.0));
} else {
return Vector3< real_t > (this->peakInflowVelocity_, real_c(0.0), real_c(0.0));
}
};
inflowObject = std::make_shared< lbm::GeneratedUBB >(sbfs, pdfFieldID, vCallback);
inflowObject->fillFromFlagField< FlagField_T >(sbfs, flagFieldID, inflowFlagUID_, fluidFlagUID);
BoundarySweep< LatticeModel_T, lbm::GeneratedUBB, LatticeModel_T::inplace > inflowSweep(*inflowObject, pdfFieldID);
sweeps.emplace_back(inflowSweep);
// Outflow
outflowObject = std::make_shared< lbm::GeneratedOutflow >(sbfs, pdfFieldID);
outflowObject->fillFromFlagField< FlagField_T >(sbfs, flagFieldID, outflowFlagUID_, fluidFlagUID);
BoundarySweep< LatticeModel_T, lbm::GeneratedOutflow, LatticeModel_T::inplace > outflowSweep(*outflowObject, pdfFieldID);
sweeps.emplace_back(outflowSweep);
}
};
class ConcentricCylinderWakeRefinement {
private:
const FlowAroundCylinder & fs_;
uint_t targetLevel(const SetupBlock * b){
AABB blockAabb(b->getAABB());
Vector3< real_t > centerRel(blockAabb.center() - fs_.cylCenter_);
if(centerRel[0] > real_t(0.0)){
for(int k = int(fs_.refinementDepth_) - 1; k >= 0 ; --k){
if(centerRel[0] <= fs_.refinementTailLengths_[uint_c(k)] && abs(centerRel[1]) <= fs_.refinementRadii_[uint_c(k)]){
return uint_c(k + 1);
}
}
} else {
for(int k = int(fs_.refinementDepth_) - 1; k >= 0 ; --k){
if(centerRel.length() <= fs_.refinementRadii_[uint_c(k)]) return uint_c(k + 1);
}
}
return 0;
}
public:
ConcentricCylinderWakeRefinement(const FlowAroundCylinder & flowSetup) : fs_(flowSetup) {};
void operator() ( SetupBlockForest & forest ){
std::vector< SetupBlock* > blocks;
forest.getBlocks(blocks);
for (const auto & b : blocks)
{
if(b->getLevel() < targetLevel(b)){
b->setMarker(true);
}
}
}
};
RefinementSelectionFunctor FlowAroundCylinder::refinementSelector() {
if(refinementProfile_ == "refineEverywhere"){
return RefineEverywhere(refinementDepth_);
}
if(refinementProfile_ == "ConcentricCylinderWake"){
return ConcentricCylinderWakeRefinement(*this);
}
WALBERLA_ABORT("Invalid refinement profile.");
}
/////////////////////////////////////////
/// Flow Setup Selection ///
/////////////////////////////////////////
......@@ -317,6 +505,21 @@ std::shared_ptr< SetupBase > getSetup(Config::BlockHandle & cblock){
);
}
if(setupName == "FlowAroundCylinder"){
return std::make_shared< FlowAroundCylinder >(
cblock.getParameter< Vector3< real_t > >("cylCenter"),
cblock.getParameter< real_t >("cylRadius"),
cblock.getParameter< real_t >("peakInflowVelocity"),
cblock.getParameter< bool >("inflowParabolicProfile"),
cblock.getParameter< bool >("walls"),
cblock.getParameter< real_t >("omegaShear"),
cblock.getParameter< std::string >("refinementProfile"),
cblock.getParameter< uint_t >("refinementDepth"),
cblock.getParameter< Vector3< real_t > >("refinementRadii", Vector3< real_t >(real_c(0.0))),
cblock.getParameter< Vector3< real_t > >("refinementTailLengths", Vector3< real_t >(real_c(0.0)))
);
}
WALBERLA_ABORT("No valid setup was specified.");
}
......
......@@ -98,16 +98,31 @@ class LdcScenario:
except sqlite3.OperationalError as e:
wlb.log_warning("Sqlite DB writing failed: try {}/{} {}".format(num_try + 1, num_tries, str(e)))
manager = wlb.ScenarioManager()
num_nodes = int(os.environ['NUM_NODES'])
cx = 16
bx = 4
manager.add(LdcScenario(
0.05, (bx*cx, bx*cx, bx*cx), (bx, bx, bx), 100,
"LDC", 3,
runs=10, steps_per_run=500,
outputFrequency=0
))
def run_ldc():
manager = wlb.ScenarioManager()
num_nodes = int(os.environ['NUM_NODES'])
cx = 16
bx = 4
by = bx * num_nodes
bz = bx
manager.add(LdcScenario(
0.05, (bx*cx, by*cx, bz*cx), (bx, by, bz), 100,
"LDC", 3,
runs=10, steps_per_run=500,
outputFrequency=0
))
if __name__ == '__main__':
# make runscript
from makeEmmyRunscript import get_script_d3q19_esotwist
scenario_script = "LdcPerformance.py"
scripts_dir = 'ldcrun'
os.makedirs(scripts_dir, exist_ok=True)
for num_nodes in [2, 4, 8, 16, 32, 64]:
filename = f'{scripts_dir}/ldcBenchmark_{num_nodes}nodes.sh'
with open(filename, 'w') as shfile:
shfile.write(get_script_d3q19_esotwist(num_nodes, [1, 5, 10], scenario_script, walltime="01:00:00"))
else:
run_ldc()
EMMY_PHYSICAL_CORES_PER_NODE = 20
SCRIPT_HEADER_TEMPLATE = """#!/bin/bash -l
#
#PBS -l nodes={num_nodes}:ppn=40,walltime={walltime}
#PBS -N waLBerlaRefinedLidDrivenCavityBenchmark
#PBS -M frederik.hennig@fau.de
#PBS -m abe
export NUM_NODES={num_nodes}
export VIRTUAL_CORES_PER_NODE=40
cd pycodegen
source env_emmy.sh
cd walberla/build/apps/benchmarks/NonuniformGridGenerated/
"""
SCRIPT_CALL_TEMPLATE = """
export PROCESSES_PER_NODE={procs_per_node}
OMP_NUM_THREADS={omp_threads} mpirun -n {num_procs} -ppn {procs_per_node} --bind-to socket ./NonuniformGridCodegenPerformance_{codegen_config} Setups/{scenario_script}
"""
def safe_int(some_float):
assert int(some_float) == some_float
return int(some_float)
def get_script_d3q19_esotwist(num_nodes, omp_threads, scenario_script, walltime="01:00:00"):
script = SCRIPT_HEADER_TEMPLATE.format(
num_nodes=num_nodes,
walltime=walltime
)
config = 'd3q19_esotwist'
for t in omp_threads:
procs_per_node = safe_int(EMMY_PHYSICAL_CORES_PER_NODE / t)
num_procs = num_nodes * procs_per_node
script += SCRIPT_CALL_TEMPLATE.format(
num_nodes=num_nodes,
num_procs=num_procs,
procs_per_node=procs_per_node,
omp_threads=t,
codegen_config=config,
scenario_script=scenario_script
)
return script
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment