Commit 9fb874a3 authored by Marcel Koch's avatar Marcel Koch
Browse files

reduce recomputation of communication info

parent d8c5a273
Pipeline #34007 failed with stages
in 4 minutes and 15 seconds
......@@ -356,6 +356,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
: storage_( storage )
, level_( level )
, comm_( gko::mpi::communicator::create( storage->getSplitCommunicatorByPrimitiveDistribution() ) )
, part_()
, constraints_type_( constraints_type )
, host_exec_( solver_exec->get_master() )
, solver_exec_( std::move( solver_exec ) )
......@@ -393,21 +394,23 @@ class GinkgoCGSolver : public Solver< OperatorType >
auto rank = walberla::mpi::MPIManager::instance()->rank();
if ( !part_ || reassembleMatrix_ )
{
auto [start, end] = local_range( num_local_dofs, comm_ );
auto part = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
part_ = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
if ( printInfo_ )
{
WALBERLA_LOG_INFO( "[Ginkgo CG] Rank " << rank << " -> local size: " << num_local_dofs << " local range: " << start
<< "-" << end );
}
}
if ( num_local_dofs )
{
auto x_vec =
vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
auto b_vec =
vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver" );
......@@ -418,14 +421,14 @@ class GinkgoCGSolver : public Solver< OperatorType >
hyteg::petsc::createVectorFromFunction(
x,
num_,
std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ),
std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
level,
All );
hyteg::petsc::createVectorFromFunction(
b,
num_,
std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ),
std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
level,
All );
......@@ -436,7 +439,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
host_matrix_ = gko::share( mtx::create( host_exec_, comm_ ) );
x.getStorage()->getTimingTree()->start( "Ginkgo System Matrix Assembly" );
auto matrix_proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >(
host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part );
host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part_ );
hyteg::petsc::createMatrix< OperatorType >( A, num_, num_, matrix_proxy, level, All );
matrix_proxy->finalize();
x.getStorage()->getTimingTree()->stop( "Ginkgo System Matrix Assembly" );
......@@ -494,13 +497,16 @@ class GinkgoCGSolver : public Solver< OperatorType >
}
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Apply" );
auto gather_idxs = compute_gather_idxs( part );
scatter_global_vector( global_x0.get(), x0, gather_idxs, comm_ );
if ( doAssemble )
{
gather_idxs_ = compute_gather_idxs( part_ );
}
scatter_global_vector( global_x0.get(), x0, gather_idxs_, comm_ );
dir_handler->update_solution( x0 );
hyteg::petsc::createFunctionFromVector(
x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part ), level, All );
x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part_ ), level, All );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver" );
......@@ -526,6 +532,9 @@ class GinkgoCGSolver : public Solver< OperatorType >
uint_t level_;
std::shared_ptr< gko::mpi::communicator > comm_;
std::shared_ptr< gko::distributed::Partition< int32_t > > part_;
std::vector< gko::Array< gko::distributed::global_index_type > > gather_idxs_;
constraints constraints_type_;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment