reduce recomputation of communication info

src/hyteg/ginkgo/GinkgoCGSolver.hpp

@@ -356,6 +356,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
    : storage_( storage )
    , level_( level )
    , comm_( gko::mpi::communicator::create( storage->getSplitCommunicatorByPrimitiveDistribution() ) )
+   , part_()
    , constraints_type_( constraints_type )
    , host_exec_( solver_exec->get_master() )
    , solver_exec_( std::move( solver_exec ) )
@@ -393,21 +394,23 @@ class GinkgoCGSolver : public Solver< OperatorType >
 
       auto rank = walberla::mpi::MPIManager::instance()->rank();
 
-      auto [start, end] = local_range( num_local_dofs, comm_ );
-      auto part         = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
-
-      if ( printInfo_ )
+      if ( !part_ || reassembleMatrix_ )
       {
-         WALBERLA_LOG_INFO( "[Ginkgo CG] Rank " << rank << " -> local size: " << num_local_dofs << " local range: " << start
-                                                << "-" << end );
+         auto [start, end] = local_range( num_local_dofs, comm_ );
+         part_             = gko::share( gko::distributed::Partition<>::build_from_local_range( host_exec_, start, end, comm_ ) );
+         if ( printInfo_ )
+         {
+            WALBERLA_LOG_INFO( "[Ginkgo CG] Rank " << rank << " -> local size: " << num_local_dofs << " local range: " << start
+                                                   << "-" << end );
+         }
       }
 
       if ( num_local_dofs )
       {
          auto x_vec =
-             vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
+             vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
          auto b_vec =
-             vec::create( host_exec_, comm_, part, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
+             vec::create( host_exec_, comm_, part_, gko::dim< 2 >{ num_global_dofs, 1 }, gko::dim< 2 >{ num_local_dofs, 1 } );
 
          x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver" );
 
@@ -418,14 +421,14 @@ class GinkgoCGSolver : public Solver< OperatorType >
          hyteg::petsc::createVectorFromFunction(
              x,
              num_,
-             std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ),
+             std::make_shared< GinkgoVectorProxy >( gko::lend( x_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
              level,
              All );
 
          hyteg::petsc::createVectorFromFunction(
              b,
              num_,
-             std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part ),
+             std::make_shared< GinkgoVectorProxy >( gko::lend( b_vec ), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ),
              level,
              All );
 
@@ -436,7 +439,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
             host_matrix_ = gko::share( mtx::create( host_exec_, comm_ ) );
             x.getStorage()->getTimingTree()->start( "Ginkgo System Matrix Assembly" );
             auto matrix_proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >(
-                host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part );
+                host_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part_ );
             hyteg::petsc::createMatrix< OperatorType >( A, num_, num_, matrix_proxy, level, All );
             matrix_proxy->finalize();
             x.getStorage()->getTimingTree()->stop( "Ginkgo System Matrix Assembly" );
@@ -494,13 +497,16 @@ class GinkgoCGSolver : public Solver< OperatorType >
          }
          x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Apply" );
 
-         auto gather_idxs = compute_gather_idxs( part );
-         scatter_global_vector( global_x0.get(), x0, gather_idxs, comm_ );
+         if ( doAssemble )
+         {
+            gather_idxs_ = compute_gather_idxs( part_ );
+         }
+         scatter_global_vector( global_x0.get(), x0, gather_idxs_, comm_ );
 
          dir_handler->update_solution( x0 );
 
          hyteg::petsc::createFunctionFromVector(
-             x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part ), level, All );
+             x, num_, std::make_shared< GinkgoVectorProxy >( x0, gko::dim< 2 >{ num_global_dofs, 1 }, part_ ), level, All );
 
          x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver" );
 
@@ -525,7 +531,10 @@ class GinkgoCGSolver : public Solver< OperatorType >
    std::shared_ptr< PrimitiveStorage > storage_;
    uint_t                              level_;
 
-   std::shared_ptr< gko::mpi::communicator > comm_;
+   std::shared_ptr< gko::mpi::communicator >                 comm_;
+   std::shared_ptr< gko::distributed::Partition< int32_t > > part_;
+
+   std::vector< gko::Array< gko::distributed::global_index_type > > gather_idxs_;
 
    constraints constraints_type_;