diff --git a/apps/benchmarks/GranularGas/GranularGas.cfg b/apps/benchmarks/GranularGas/GranularGas.cfg index 2c0c4755288cac12a1f2dc874330063b3fe913a7..8dea7a979b90cc5776fc1cd2cfd64f79a47080e9 100644 --- a/apps/benchmarks/GranularGas/GranularGas.cfg +++ b/apps/benchmarks/GranularGas/GranularGas.cfg @@ -1,15 +1,21 @@ GranularGas { simulationCorner < 0, 0, 0 >; - simulationDomain < 80, 80, 80 >; + simulationDomain < 40, 40, 40 >; blocks < 1,1,1 >; isPeriodic < 0, 0, 0 >; - initialRefinementLevel 3; + initialRefinementLevel 1; sorting linear; LBAlgorithm Morton; baseWeight 1; + recalculateBlockLevelsInRefresh 1; + reevaluateMinTargetLevelsAfterForcedRefinement 1; + allowRefreshChangingDepth 1; + regridMin 2000; + regridMax 100; + normal <1,1,1>; radius 0.6; spacing 1.0; diff --git a/apps/benchmarks/GranularGas/MESA_PD_KernelLoadBalancing.cpp b/apps/benchmarks/GranularGas/MESA_PD_KernelLoadBalancing.cpp index fa70bf38c092ff5e70b5e50ae42a879fde87ed29..33c97b2b931821ca2a2dfeffed0be236b6ad17a2 100644 --- a/apps/benchmarks/GranularGas/MESA_PD_KernelLoadBalancing.cpp +++ b/apps/benchmarks/GranularGas/MESA_PD_KernelLoadBalancing.cpp @@ -136,8 +136,8 @@ int main( int argc, char ** argv ) auto ic = make_shared<pe::InfoCollection>(); - // pe::amr::MinMaxLevelDetermination regrid(ic, regridMin, regridMax); - // forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); + pe::amr::MinMaxLevelDetermination regrid(ic, params.regridMin, params.regridMax); + forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); bool bRebalance = true; if (params.LBAlgorithm == "None") @@ -259,6 +259,7 @@ int main( int argc, char ** argv ) WcTimingPool tpImbalanced; WcTimingPool tpBalanced; + WcTimer timerLoadBalancing; WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING UNBALANCED SIMULATION ***"); WALBERLA_MPI_BARRIER(); @@ -372,7 +373,10 @@ int main( int argc, char ** argv ) auto maxLinkedCells = walberla::mpi::reduce(lc->cells_.size(), walberla::mpi::MAX); WALBERLA_LOG_DEVEL_ON_ROOT( "linked cells: " << minLinkedCells << " / " << maxLinkedCells ); + vtkDomainOutput->write( ); + vtkWriter->write(); WALBERLA_MPI_BARRIER(); + timerLoadBalancing.start(); if (bRebalance) { WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING LOAD BALANCING ***"); @@ -396,6 +400,9 @@ int main( int argc, char ** argv ) SNN(*ps, forest, domain); sortParticleStorage(*ps, params.sorting, lc->domain_, uint_c(lc->numCellsPerDim_[0])); } + timerLoadBalancing.end(); + vtkDomainOutput->write( ); + vtkWriter->write(); WALBERLA_MPI_BARRIER(); WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING BALANCED SIMULATION ***"); @@ -527,6 +534,8 @@ int main( int argc, char ** argv ) auto tpBalancedReduced = tpBalanced.getReduced(); WALBERLA_LOG_INFO_ON_ROOT(*tpBalancedReduced); + auto timerLoadBalancingReduced = walberla::timing::getReduced(timerLoadBalancing, REDUCE_TOTAL, 0); + numParticles = 0; int64_t numGhostParticles = 0; ps->forEachParticle(false, @@ -582,6 +591,10 @@ int main( int argc, char ** argv ) integerProperties["balancedContactsChecked"] = balancedContactsChecked; integerProperties["balancedContactsDetected"] = balancedContactsDetected; integerProperties["balancedContactsTreated"] = balancedContactsTreated; + realProperties["loadbalancing_timer_min"] = timerLoadBalancingReduced->min(); + realProperties["loadbalancing_timer_max"] = timerLoadBalancingReduced->max(); + realProperties["loadbalancing_timer_average"] = timerLoadBalancingReduced->average(); + realProperties["loadbalancing_timer_total"] = timerLoadBalancingReduced->total(); integerProperties["local_aabbs"] = int64_c(local_aabbs); integerProperties["neighbor_subdomains"] = int64_c(neighbor_subdomains); integerProperties["neighbor_processes"] = int64_c(neighbor_processes); diff --git a/apps/benchmarks/GranularGas/MESA_PD_LoadBalancing.cpp b/apps/benchmarks/GranularGas/MESA_PD_LoadBalancing.cpp index b4a09a77f919b784da5b53a13c1b5215d5844bed..856edfc886c0fa2588919325148f825495a4d319 100644 --- a/apps/benchmarks/GranularGas/MESA_PD_LoadBalancing.cpp +++ b/apps/benchmarks/GranularGas/MESA_PD_LoadBalancing.cpp @@ -132,8 +132,8 @@ int main( int argc, char ** argv ) auto ic = make_shared<pe::InfoCollection>(); - // pe::amr::MinMaxLevelDetermination regrid(ic, regridMin, regridMax); - // forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); + pe::amr::MinMaxLevelDetermination regrid(ic, params.regridMin, params.regridMax); + forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); bool bRebalance = true; if (params.LBAlgorithm == "None") @@ -226,7 +226,6 @@ int main( int argc, char ** argv ) auto vtkWriter = walberla::vtk::createVTKOutput_PointData(vtkOutput, "Bodies", 1, params.vtk_out, "simulation_step", false, false); vtkOutput->addOutput<SelectRank>("rank"); vtkOutput->addOutput<data::SelectParticleOwner>("owner"); - vtkDomainOutput->write(); WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - START ***"); // Init kernels @@ -331,6 +330,8 @@ int main( int argc, char ** argv ) } timerImbalanced.end(); + vtkDomainOutput->write( ); + vtkWriter->write(); WALBERLA_MPI_BARRIER(); timerLoadBalancing.start(); if (bRebalance) @@ -357,6 +358,8 @@ int main( int argc, char ** argv ) sortParticleStorage(*ps, params.sorting, lc->domain_, uint_c(lc->numCellsPerDim_[0])); } timerLoadBalancing.end(); + vtkDomainOutput->write( ); + vtkWriter->write(); WALBERLA_MPI_BARRIER(); WALBERLA_LOG_DEVEL_ON_ROOT("running balanced simulation"); diff --git a/apps/benchmarks/GranularGas/PE_LoadBalancing.cpp b/apps/benchmarks/GranularGas/PE_LoadBalancing.cpp index 132b6a433d28518262c20f314ebb535fc8f0883b..42eb4181a00245a17f6c3397390fa036e1c80ade 100644 --- a/apps/benchmarks/GranularGas/PE_LoadBalancing.cpp +++ b/apps/benchmarks/GranularGas/PE_LoadBalancing.cpp @@ -132,8 +132,8 @@ int main( int argc, char ** argv ) auto ic = make_shared<pe::InfoCollection>(); - // pe::amr::MinMaxLevelDetermination regrid(ic, regridMin, regridMax); - // forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); + pe::amr::MinMaxLevelDetermination regrid(ic, params.regridMin, params.regridMax); + forest->setRefreshMinTargetLevelDeterminationFunction( regrid ); bool bRebalance = true; if (params.LBAlgorithm == "None") @@ -313,16 +313,18 @@ int main( int argc, char ** argv ) syncCallWithoutTT(); tpImbalanced["Sync"].end(); - //if( i % visSpacing == 0 ) - //{ - // vtkDomainOutput->write( ); - // vtkSphereOutput->write( ); - //} + // if( i % params.visSpacing == 0 ) + // { + // vtkDomainOutput->write( ); + // vtkSphereOutput->write( ); + // } } timerImbalanced.end(); if (bRebalance) { + vtkDomainOutput->write( ); + vtkSphereOutput->write( ); WALBERLA_MPI_BARRIER(); timerLoadBalancing.start(); WALBERLA_LOG_INFO_ON_ROOT("*** Rebalance ***"); @@ -337,6 +339,8 @@ int main( int argc, char ** argv ) ccd->reloadBodies(); } timerLoadBalancing.end(); + vtkDomainOutput->write( ); + vtkSphereOutput->write( ); } WALBERLA_MPI_BARRIER(); @@ -355,11 +359,11 @@ int main( int argc, char ** argv ) syncCallWithoutTT(); tpBalanced["Sync"].end(); - //if( i % visSpacing == 0 ) - //{ - // vtkDomainOutput->write( ); - // vtkSphereOutput->write( ); - //} + // if( i % params.visSpacing == 0 ) + // { + // vtkDomainOutput->write( ); + // vtkSphereOutput->write( ); + // } } timerBalanced.end();