update agglomeration in tokamak

apps/2021-tokamak/Tokamak.cpp

@@ -75,6 +75,7 @@ struct SolverSettings
    std::string gkoExecutor;
    std::string gkoConfig;
    bool        useAgglomeration;
+   uint_t      numAgglomerationProcessors;
 
    [[nodiscard]] std::string toString() const
    {
@@ -95,7 +96,7 @@ struct SolverSettings
          }
          if ( useAgglomeration )
          {
-            ss << "  - use agglomeration on coarse grid\n";
+            ss << "  - use agglomeration on coarse grid with " << numAgglomerationProcessors << " processes\n";
          }
       }
       return ss.str();
@@ -259,6 +260,7 @@ void tokamak( TokamakDomain         tokamakDomain,
       db->setConstantEntry( "preSmooth", solverSettings.preSmooth );
       db->setConstantEntry( "postSmooth", solverSettings.postSmooth );
       db->setConstantEntry( "useAgglomeration", solverSettings.useAgglomeration );
+      db->setConstantEntry( "numAgglomerationProcessors", static_cast< int64_t >( solverSettings.numAgglomerationProcessors ) );
 
       db->setConstantEntry( "elementType", discretization.elementType );
       db->setConstantEntry( "minLevel", discretization.minLevel );
@@ -599,10 +601,29 @@ void tokamak( TokamakDomain         tokamakDomain,
       auto agglomerationWrapper = std::shared_ptr< AgglomerationWrapper< LaplaceOperator_T, LaplaceForm_T > >();
       if ( solverSettings.useAgglomeration )
       {
+         WALBERLA_LOG_INFO_ON_ROOT( "[progress] Coarse grid solver agglomeration enabled." )
+         auto globalMaxNumberOfCells = walberla::mpi::allReduce( storage->getNumberOfLocalCells(), walberla::mpi::MAX );
+         auto agglomerationNumProcesses =
+             std::min( { solverSettings.numAgglomerationProcessors - 1,
+                         static_cast< uint_t >( walberla::MPIManager::instance()->numProcesses() ) - 1 } );
+         WALBERLA_CHECK_GREATER( agglomerationNumProcesses, 0, "Cannot perform agglomeration on zero processes." )
+
+         WALBERLA_LOG_INFO_ON_ROOT( "[progress] Agglomeration from "
+                                    << uint_c( walberla::mpi::MPIManager::instance()->numProcesses() ) << " to "
+                                    << agglomerationNumProcesses + 1 << " processes." )
+         bool solveOnEmptyProcesses = true;
+         WALBERLA_LOG_INFO_ON_ROOT( "[progress] Solving on empty processes: " << ( solveOnEmptyProcesses ? "yes" : "no" ) )
+
+         WALBERLA_LOG_INFO_ON_ROOT( "[progress] Setting up agglomeration primitive storage ..." )
          agglomerationWrapper =
              std::make_shared< AgglomerationWrapper< LaplaceOperator_T, LaplaceForm_T > >( storage, minLevel, true, form );
-         agglomerationWrapper->setStrategyContinuousProcesses( 0, 0 );
+
+         WALBERLA_LOG_INFO_ON_ROOT( "[progress] Re-distribution of agglomeration primitive storage  ..." )
+         agglomerationWrapper->setStrategyContinuousProcesses( 0, agglomerationNumProcesses );
          coarse_storage = agglomerationWrapper->getAgglomerationStorage();
+
+         const auto domainInfo = coarse_storage->getGlobalInfo( true, coarse_storage->getSplitCommunicatorByPrimitiveDistribution() );
+         WALBERLA_LOG_INFO_ON_ROOT( domainInfo )
       }
 
       std::shared_ptr< Solver< LaplaceOperator_T > > coarseGridSolver;
@@ -899,15 +920,16 @@ void run( int argc, char** argv )
    discretization.minLevel    = mainConf.getParameter< uint_t >( "minLevel" );
    discretization.maxLevel    = mainConf.getParameter< uint_t >( "maxLevel" );
 
-   solverSettings.solverType                = mainConf.getParameter< std::string >( "solverType" );
-   solverSettings.coarseGridSolverType      = mainConf.getParameter< std::string >( "coarseGridSolverType" );
-   solverSettings.relativeResidualReduction = mainConf.getParameter< real_t >( "relativeResidualReduction" );
-   solverSettings.preSmooth                 = mainConf.getParameter< uint_t >( "preSmooth" );
-   solverSettings.postSmooth                = mainConf.getParameter< uint_t >( "postSmooth" );
-   solverSettings.cgHytegVerbose            = mainConf.getParameter< bool >( "cgHytegVerbose" );
-   solverSettings.gkoExecutor               = mainConf.getParameter< std::string >( "gkoExecutor" );
-   solverSettings.gkoConfig                 = mainConf.getParameter< std::string >( "gkoConfig" );
-   solverSettings.useAgglomeration          = mainConf.getParameter< bool >( "useAgglomeration" );
+   solverSettings.solverType                 = mainConf.getParameter< std::string >( "solverType" );
+   solverSettings.coarseGridSolverType       = mainConf.getParameter< std::string >( "coarseGridSolverType" );
+   solverSettings.relativeResidualReduction  = mainConf.getParameter< real_t >( "relativeResidualReduction" );
+   solverSettings.preSmooth                  = mainConf.getParameter< uint_t >( "preSmooth" );
+   solverSettings.postSmooth                 = mainConf.getParameter< uint_t >( "postSmooth" );
+   solverSettings.cgHytegVerbose             = mainConf.getParameter< bool >( "cgHytegVerbose" );
+   solverSettings.gkoExecutor                = mainConf.getParameter< std::string >( "gkoExecutor" );
+   solverSettings.gkoConfig                  = mainConf.getParameter< std::string >( "gkoConfig" );
+   solverSettings.useAgglomeration           = mainConf.getParameter< bool >( "useAgglomeration" );
+   solverSettings.numAgglomerationProcessors = mainConf.getParameter< uint_t >( "numAgglomerationProcessors" );
    // use numeric_limits<int> here to prevent overflow when casting to PetscINt
    solverSettings.maxCoarseGridSolverIterations =
        mainConf.getParameter< uint_t >( "maxCoarseGridSolverIterations", std::numeric_limits< int >::max() );

apps/2021-tokamak/Tokamak.prm

@@ -67,6 +67,7 @@ Parameters
     gkoExecutor               cuda;
     gkoConfig                 gko.json;
     useAgglomeration          false;
+    numAgglomerationProcessors 76;
 
     // ################
     // # app settings #