Commit 3a8cf4ab authored by Marcel Koch's avatar Marcel Koch
Browse files

update agglomeration in tokamak

parent 6d7530fd
......@@ -75,6 +75,7 @@ struct SolverSettings
std::string gkoExecutor;
std::string gkoConfig;
bool useAgglomeration;
uint_t numAgglomerationProcessors;
[[nodiscard]] std::string toString() const
{
......@@ -95,7 +96,7 @@ struct SolverSettings
}
if ( useAgglomeration )
{
ss << " - use agglomeration on coarse grid\n";
ss << " - use agglomeration on coarse grid with " << numAgglomerationProcessors << " processes\n";
}
}
return ss.str();
......@@ -259,6 +260,7 @@ void tokamak( TokamakDomain tokamakDomain,
db->setConstantEntry( "preSmooth", solverSettings.preSmooth );
db->setConstantEntry( "postSmooth", solverSettings.postSmooth );
db->setConstantEntry( "useAgglomeration", solverSettings.useAgglomeration );
db->setConstantEntry( "numAgglomerationProcessors", static_cast< int64_t >( solverSettings.numAgglomerationProcessors ) );
db->setConstantEntry( "elementType", discretization.elementType );
db->setConstantEntry( "minLevel", discretization.minLevel );
......@@ -599,10 +601,29 @@ void tokamak( TokamakDomain tokamakDomain,
auto agglomerationWrapper = std::shared_ptr< AgglomerationWrapper< LaplaceOperator_T, LaplaceForm_T > >();
if ( solverSettings.useAgglomeration )
{
WALBERLA_LOG_INFO_ON_ROOT( "[progress] Coarse grid solver agglomeration enabled." )
auto globalMaxNumberOfCells = walberla::mpi::allReduce( storage->getNumberOfLocalCells(), walberla::mpi::MAX );
auto agglomerationNumProcesses =
std::min( { solverSettings.numAgglomerationProcessors - 1,
static_cast< uint_t >( walberla::MPIManager::instance()->numProcesses() ) - 1 } );
WALBERLA_CHECK_GREATER( agglomerationNumProcesses, 0, "Cannot perform agglomeration on zero processes." )
WALBERLA_LOG_INFO_ON_ROOT( "[progress] Agglomeration from "
<< uint_c( walberla::mpi::MPIManager::instance()->numProcesses() ) << " to "
<< agglomerationNumProcesses + 1 << " processes." )
bool solveOnEmptyProcesses = true;
WALBERLA_LOG_INFO_ON_ROOT( "[progress] Solving on empty processes: " << ( solveOnEmptyProcesses ? "yes" : "no" ) )
WALBERLA_LOG_INFO_ON_ROOT( "[progress] Setting up agglomeration primitive storage ..." )
agglomerationWrapper =
std::make_shared< AgglomerationWrapper< LaplaceOperator_T, LaplaceForm_T > >( storage, minLevel, true, form );
agglomerationWrapper->setStrategyContinuousProcesses( 0, 0 );
WALBERLA_LOG_INFO_ON_ROOT( "[progress] Re-distribution of agglomeration primitive storage ..." )
agglomerationWrapper->setStrategyContinuousProcesses( 0, agglomerationNumProcesses );
coarse_storage = agglomerationWrapper->getAgglomerationStorage();
const auto domainInfo = coarse_storage->getGlobalInfo( true, coarse_storage->getSplitCommunicatorByPrimitiveDistribution() );
WALBERLA_LOG_INFO_ON_ROOT( domainInfo )
}
std::shared_ptr< Solver< LaplaceOperator_T > > coarseGridSolver;
......@@ -899,15 +920,16 @@ void run( int argc, char** argv )
discretization.minLevel = mainConf.getParameter< uint_t >( "minLevel" );
discretization.maxLevel = mainConf.getParameter< uint_t >( "maxLevel" );
solverSettings.solverType = mainConf.getParameter< std::string >( "solverType" );
solverSettings.coarseGridSolverType = mainConf.getParameter< std::string >( "coarseGridSolverType" );
solverSettings.relativeResidualReduction = mainConf.getParameter< real_t >( "relativeResidualReduction" );
solverSettings.preSmooth = mainConf.getParameter< uint_t >( "preSmooth" );
solverSettings.postSmooth = mainConf.getParameter< uint_t >( "postSmooth" );
solverSettings.cgHytegVerbose = mainConf.getParameter< bool >( "cgHytegVerbose" );
solverSettings.gkoExecutor = mainConf.getParameter< std::string >( "gkoExecutor" );
solverSettings.gkoConfig = mainConf.getParameter< std::string >( "gkoConfig" );
solverSettings.useAgglomeration = mainConf.getParameter< bool >( "useAgglomeration" );
solverSettings.solverType = mainConf.getParameter< std::string >( "solverType" );
solverSettings.coarseGridSolverType = mainConf.getParameter< std::string >( "coarseGridSolverType" );
solverSettings.relativeResidualReduction = mainConf.getParameter< real_t >( "relativeResidualReduction" );
solverSettings.preSmooth = mainConf.getParameter< uint_t >( "preSmooth" );
solverSettings.postSmooth = mainConf.getParameter< uint_t >( "postSmooth" );
solverSettings.cgHytegVerbose = mainConf.getParameter< bool >( "cgHytegVerbose" );
solverSettings.gkoExecutor = mainConf.getParameter< std::string >( "gkoExecutor" );
solverSettings.gkoConfig = mainConf.getParameter< std::string >( "gkoConfig" );
solverSettings.useAgglomeration = mainConf.getParameter< bool >( "useAgglomeration" );
solverSettings.numAgglomerationProcessors = mainConf.getParameter< uint_t >( "numAgglomerationProcessors" );
// use numeric_limits<int> here to prevent overflow when casting to PetscINt
solverSettings.maxCoarseGridSolverIterations =
mainConf.getParameter< uint_t >( "maxCoarseGridSolverIterations", std::numeric_limits< int >::max() );
......
......@@ -67,6 +67,7 @@ Parameters
gkoExecutor cuda;
gkoConfig gko.json;
useAgglomeration false;
numAgglomerationProcessors 76;
// ################
// # app settings #
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment