extended load balancing benchmarks

apps/benchmarks/GranularGas/CMakeLists.txt

@@ -5,6 +5,10 @@ waLBerla_add_executable ( NAME PE_GranularGas
                           FILES PE_GranularGas.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp
                           DEPENDS blockforest core pe postprocessing )
 
+waLBerla_add_executable ( NAME PE_LoadBalancing
+                          FILES PE_LoadBalancing.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp
+                          DEPENDS blockforest core pe postprocessing )
+
 waLBerla_add_executable ( NAME MESA_PD_LoadBalancing
                           FILES MESA_PD_LoadBalancing.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp sortParticleStorage.cpp CreateParticles.cpp
                           DEPENDS blockforest core pe mesa_pd postprocessing vtk )
@@ -16,3 +20,7 @@ waLBerla_add_executable ( NAME MESA_PD_GranularGas
 waLBerla_add_executable ( NAME MESA_PD_KernelBenchmark
                           FILES MESA_PD_KernelBenchmark.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp sortParticleStorage.cpp CreateParticles.cpp
                           DEPENDS blockforest core pe mesa_pd postprocessing vtk )
+
+waLBerla_add_executable ( NAME MESA_PD_KernelLoadBalancing
+                          FILES MESA_PD_KernelLoadBalancing.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp sortParticleStorage.cpp CreateParticles.cpp
+                          DEPENDS blockforest core pe mesa_pd postprocessing vtk )

apps/benchmarks/GranularGas/GranularGas.cfg

@@ -4,7 +4,7 @@ GranularGas
    simulationDomain < 80, 80, 80 >;
    blocks < 2,2,2 >;
    isPeriodic < 0, 0, 0 >;
-   initialRefinementLevel 2;
+   initialRefinementLevel 1;
    sorting linear;
 
    normal  <1,1,1>;

apps/benchmarks/GranularGas/MESA_PD_LoadBalancing.cpp

@@ -119,14 +119,14 @@ int main( int argc, char ** argv )
       return EXIT_SUCCESS;
    }
 
-   forest->recalculateBlockLevelsInRefresh( false );
-   forest->alwaysRebalanceInRefresh( true );
-   forest->reevaluateMinTargetLevelsAfterForcedRefinement( false );
-   forest->allowRefreshChangingDepth( false );
+   forest->recalculateBlockLevelsInRefresh( params.recalculateBlockLevelsInRefresh );
+   forest->alwaysRebalanceInRefresh( params.alwaysRebalanceInRefresh );
+   forest->reevaluateMinTargetLevelsAfterForcedRefinement( params.reevaluateMinTargetLevelsAfterForcedRefinement );
+   forest->allowRefreshChangingDepth( params.allowRefreshChangingDepth );
 
-   forest->allowMultipleRefreshCycles( false );
-   forest->checkForEarlyOutInRefresh( true );
-   forest->checkForLateOutInRefresh( true );
+   forest->allowMultipleRefreshCycles( params.allowMultipleRefreshCycles );
+   forest->checkForEarlyOutInRefresh( params.checkForEarlyOutInRefresh );
+   forest->checkForLateOutInRefresh( params.checkForLateOutInRefresh );
 
    auto ic = make_shared<pe::InfoCollection>();
 
@@ -252,6 +252,7 @@ int main( int argc, char ** argv )
    WcTimer      timerBalanced;
    WcTimingPool tpImbalanced;
    WcTimingPool tpBalanced;
+
    auto    SNNBytesSent     = SNN.getBytesSent();
    auto    SNNBytesReceived = SNN.getBytesReceived();
    auto    SNNSends         = SNN.getNumberOfSends();
@@ -493,6 +494,9 @@ int main( int argc, char ** argv )
       }
    },
    accessor);
+   auto minParticles = walberla::mpi::reduce(numParticles, walberla::mpi::MIN);
+   auto maxParticles = walberla::mpi::reduce(numParticles, walberla::mpi::MAX);
+   WALBERLA_LOG_DEVEL_ON_ROOT("particle ratio: " << minParticles << " / " << maxParticles);
    walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM);
    walberla::mpi::reduceInplace(numGhostParticles, walberla::mpi::SUM);
    walberla::mpi::reduceInplace(contactsChecked, walberla::mpi::SUM);
@@ -532,6 +536,8 @@ int main( int argc, char ** argv )
       realProperties["balanced_timer_total"]   = timerBalancedReduced->total();
       integerProperties["num_particles"]       = numParticles;
       integerProperties["num_ghost_particles"] = numGhostParticles;
+      integerProperties["minParticles"]        = minParticles;
+      integerProperties["maxParticles"]        = maxParticles;
       integerProperties["contacts_checked"]    = contactsChecked;
       integerProperties["contacts_detected"]   = contactsDetected;
       integerProperties["contacts_treated"]    = contactsTreated;

apps/benchmarks/GranularGas/Parameters.cpp

@@ -75,6 +75,27 @@ void loadFromConfig(Parameters& params, const Config::BlockHandle& cfg)
    params.sqlFile = cfg.getParameter<std::string>("sqlFile", "benchmark.sqlite" );
    WALBERLA_LOG_INFO_ON_ROOT("sqlFile: " << params.sqlFile);
    
+   params.recalculateBlockLevelsInRefresh = cfg.getParameter<bool>("recalculateBlockLevelsInRefresh", false );
+   WALBERLA_LOG_INFO_ON_ROOT("recalculateBlockLevelsInRefresh: " << params.recalculateBlockLevelsInRefresh);
+   
+   params.alwaysRebalanceInRefresh = cfg.getParameter<bool>("alwaysRebalanceInRefresh", true );
+   WALBERLA_LOG_INFO_ON_ROOT("alwaysRebalanceInRefresh: " << params.alwaysRebalanceInRefresh);
+   
+   params.reevaluateMinTargetLevelsAfterForcedRefinement = cfg.getParameter<bool>("reevaluateMinTargetLevelsAfterForcedRefinement", false );
+   WALBERLA_LOG_INFO_ON_ROOT("reevaluateMinTargetLevelsAfterForcedRefinement: " << params.reevaluateMinTargetLevelsAfterForcedRefinement);
+   
+   params.allowRefreshChangingDepth = cfg.getParameter<bool>("allowRefreshChangingDepth", false );
+   WALBERLA_LOG_INFO_ON_ROOT("allowRefreshChangingDepth: " << params.allowRefreshChangingDepth);
+   
+   params.allowMultipleRefreshCycles = cfg.getParameter<bool>("allowMultipleRefreshCycles", false );
+   WALBERLA_LOG_INFO_ON_ROOT("allowMultipleRefreshCycles: " << params.allowMultipleRefreshCycles);
+   
+   params.checkForEarlyOutInRefresh = cfg.getParameter<bool>("checkForEarlyOutInRefresh", true );
+   WALBERLA_LOG_INFO_ON_ROOT("checkForEarlyOutInRefresh: " << params.checkForEarlyOutInRefresh);
+   
+   params.checkForLateOutInRefresh = cfg.getParameter<bool>("checkForLateOutInRefresh", true );
+   WALBERLA_LOG_INFO_ON_ROOT("checkForLateOutInRefresh: " << params.checkForLateOutInRefresh);
+   
    params.regridMin = cfg.getParameter<uint_t>("regridMin", uint_c(100) );
    WALBERLA_LOG_INFO_ON_ROOT("regridMin: " << params.regridMin);
    
@@ -136,6 +157,13 @@ void saveToSQL(const Parameters& params,
    
    
    
+   
+   
+   
+   
+   
+   
+   
    realProperties["baseWeight"] = double_c(params.baseWeight);
    
    realProperties["metisipc2redist"] = double_c(params.metisipc2redist);

apps/benchmarks/GranularGas/Parameters.h

@@ -50,6 +50,13 @@ struct Parameters
    int64_t visSpacing = 1000;
    std::string path = "vtk_out";
    std::string sqlFile = "benchmark.sqlite";
+   bool recalculateBlockLevelsInRefresh = false;
+   bool alwaysRebalanceInRefresh = true;
+   bool reevaluateMinTargetLevelsAfterForcedRefinement = false;
+   bool allowRefreshChangingDepth = false;
+   bool allowMultipleRefreshCycles = false;
+   bool checkForEarlyOutInRefresh = true;
+   bool checkForLateOutInRefresh = true;
    uint_t regridMin = uint_c(100);
    uint_t regridMax = uint_c(1000);
    int maxBlocksPerProcess = int_c(1000);

apps/benchmarks/GranularGas/generateConfig.py

@@ -19,6 +19,15 @@ cfg.addParameter("visSpacing",             "int64_t",     "1000")
 cfg.addParameter("path",                   "std::string", '"vtk_out"')
 cfg.addParameter("sqlFile",                "std::string", '"benchmark.sqlite"')
 
+cfg.addParameter("recalculateBlockLevelsInRefresh",                "bool", "false");
+cfg.addParameter("alwaysRebalanceInRefresh",                       "bool", "true");
+cfg.addParameter("reevaluateMinTargetLevelsAfterForcedRefinement", "bool", "false");
+cfg.addParameter("allowRefreshChangingDepth",                      "bool", "false");
+
+cfg.addParameter("allowMultipleRefreshCycles",                     "bool", "false");
+cfg.addParameter("checkForEarlyOutInRefresh",                      "bool", "true");
+cfg.addParameter("checkForLateOutInRefresh",                       "bool", "true");
+
 cfg.addParameter("regridMin",              "uint_t",      'uint_c(100)')
 cfg.addParameter("regridMax",              "uint_t",      'uint_c(1000)')
 cfg.addParameter("maxBlocksPerProcess",    "int",         'int_c(1000)')