Commit 9fb874a3 authored by Marcel Koch's avatar Marcel Koch
Browse files

reduce recomputation of communication info

parent d8c5a273
Pipeline #34007 failed with stages
in 4 minutes and 15 seconds
...@@ -356,6 +356,7 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -356,6 +356,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
: storage_( storage ) : storage_( storage )
, level_( level ) , level_( level )
, comm_( gko::mpi::communicator::create( storage->getSplitCommunicatorByPrimitiveDistribution() ) ) , comm_( gko::mpi::communicator::create( storage->getSplitCommunicatorByPrimitiveDistribution() ) )
, part_()
, constraints_type_( constraints_type ) , constraints_type_( constraints_type )
, host_exec_( solver_exec->get_master() ) , host_exec_( solver_exec->get_master() )
, solver_exec_( std::move( solver_exec ) ) , solver_exec_( std::move( solver_exec ) )
...@@ -393,21 +394,23 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -393,21 +394,23 @@ class GinkgoCGSolver : public Solver< OperatorType >
auto rank = walberla::mpi::MPIManager::instance()->rank(); auto rank = walberla::mpi::MPIManager::instance()->rank();
auto [start, end] = local_range( num_local_dofs, comm_ ); if ( !part_ || reassembleMatrix_ )
auto part = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
if ( printInfo_ )
{ {
WALBERLA_LOG_INFO( "[Ginkgo CG] Rank " << rank << " -> local size: " << num_local_dofs << " local range: " << start auto [start, end] = local_range( num_local_dofs, comm_ );
<< "-" << end ); part_ = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
if ( printInfo_ )
{
WALBERLA_LOG_INFO( "[Ginkgo CG] Rank " << rank << " -> local size: " << num_local_dofs << " local range: " << start
<< "-" << end );
}
} }
if ( num_local_dofs ) if ( num_local_dofs )
{ {
auto x_vec = auto x_vec =
vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } ); vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
auto b_vec = auto b_vec =
vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } ); vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver" ); x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver" );
...@@ -418,14 +421,14 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -418,14 +421,14 @@ class GinkgoCGSolver : public Solver< OperatorType >
hyteg::petsc::createVectorFromFunction( hyteg::petsc::createVectorFromFunction(
x, x,
num_, num_,
std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ), std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
level, level,
All ); All );
hyteg::petsc::createVectorFromFunction( hyteg::petsc::createVectorFromFunction(
b, b,
num_, num_,
std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ), std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
level, level,
All ); All );
...@@ -436,7 +439,7 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -436,7 +439,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
host_matrix_ = gko::share( mtx::create( host_exec_, comm_ ) ); host_matrix_ = gko::share( mtx::create( host_exec_, comm_ ) );
x.getStorage()->getTimingTree()->start( "Ginkgo System Matrix Assembly" ); x.getStorage()->getTimingTree()->start( "Ginkgo System Matrix Assembly" );
auto matrix_proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >( auto matrix_proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >(
host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part ); host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part_ );
hyteg::petsc::createMatrix< OperatorType >( A, num_, num_, matrix_proxy, level, All ); hyteg::petsc::createMatrix< OperatorType >( A, num_, num_, matrix_proxy, level, All );
matrix_proxy->finalize(); matrix_proxy->finalize();
x.getStorage()->getTimingTree()->stop( "Ginkgo System Matrix Assembly" ); x.getStorage()->getTimingTree()->stop( "Ginkgo System Matrix Assembly" );
...@@ -494,13 +497,16 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -494,13 +497,16 @@ class GinkgoCGSolver : public Solver< OperatorType >
} }
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Apply" ); x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Apply" );
auto gather_idxs = compute_gather_idxs( part ); if ( doAssemble )
scatter_global_vector( global_x0.get(), x0, gather_idxs, comm_ ); {
gather_idxs_ = compute_gather_idxs( part_ );
}
scatter_global_vector( global_x0.get(), x0, gather_idxs_, comm_ );
dir_handler->update_solution( x0 ); dir_handler->update_solution( x0 );
hyteg::petsc::createFunctionFromVector( hyteg::petsc::createFunctionFromVector(
x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part ), level, All ); x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part_ ), level, All );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver" ); x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver" );
...@@ -525,7 +531,10 @@ class GinkgoCGSolver : public Solver< OperatorType > ...@@ -525,7 +531,10 @@ class GinkgoCGSolver : public Solver< OperatorType >
std::shared_ptr< PrimitiveStorage > storage_; std::shared_ptr< PrimitiveStorage > storage_;
uint_t level_; uint_t level_;
std::shared_ptr< gko::mpi::communicator > comm_; std::shared_ptr< gko::mpi::communicator > comm_;
std::shared_ptr< gko::distributed::Partition< int32_t > > part_;
std::vector< gko::Array< gko::distributed::global_index_type > > gather_idxs_;
constraints constraints_type_; constraints constraints_type_;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment