Skip to content

GitLab

Commit 5211e66c authored by Marcel Koch's avatar Marcel Koch
Browse files

add timings

parent 57a9e19e
Hide whitespace changes
Inline Side-by-side
src/hyteg/ginkgo/GinkgoBlockSolver.hpp
View file @ 5211e66c
......@@ -101,6 +101,8 @@ class GinkgoBlockSolver : public Solver< OperatorType >
const typename OperatorType::dstType& b,
const walberla::uint_t level ) override
{
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver" );
const auto num_local_dofs = numberOfLocalDoFs< typename FunctionType::Tag >( *storage_, level );
const auto num_global_dofs = numberOfGlobalDoFs< typename FunctionType::Tag >( *storage_, level, comm_->get() );
......@@ -126,37 +128,44 @@ class GinkgoBlockSolver : public Solver< OperatorType >
hyteg::petsc::createVectorFromFunction(
b, num_, std::make_shared< GinkgoVectorProxy >( b_gko.get(), gko::dim< 2 >{ num_global_dofs, 1 }, part_ ), level, All );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Set-up distributed matrix" );
host_monolithic_matrix_ = gko::share( mtx::create( host_exec_, comm_ ) );
auto dir_handler = std::make_unique< ZeroRowsDirichletHandler >( bcIndices, b_gko.get(), host_monolithic_matrix_, true );
{
auto proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >(
host_monolithic_matrix_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part_ );
hyteg::petsc::createMatrix( A, num_, num_, proxy, level, All );
proxy->finalize();
dir_handler->update_matrix();
}
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Set-up distributed matrix" );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Set-up distributed preconditioner" );
auto monolithic_preconditioner_ = gko::share( mtx::create( host_exec_, comm_ ) );
{
auto proxy = std::make_shared< GinkgoSparseMatrixProxy< mtx > >(
monolithic_preconditioner_.get(), gko::dim< 2 >{ num_global_dofs, num_global_dofs }, part_ );
hyteg::petsc::createMatrix( blockPreconditioner_, num_, num_, proxy, level, All );
proxy->finalize();
auto dir_handler =
auto dir_handler_p =
std::make_unique< ZeroRowsDirichletHandler >( bcIndices, b_gko.get(), monolithic_preconditioner_, true );
dir_handler->update_matrix();
dir_handler_p->update_matrix();
}
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Set-up distributed preconditioner" );
auto dir_handler = std::make_unique< ZeroRowsDirichletHandler >( bcIndices, b_gko.get(), host_monolithic_matrix_, true );
dir_handler->update_matrix();
auto rhs = dir_handler->get_rhs( b_gko.get(), x_gko.get() );
auto x0 = dir_handler->get_initial_guess( x_gko.get() );
std::vector< int32_t > vIndices;
std::vector< int32_t > pIndices;
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Gather index sets" );
gatherIndices( vIndices, pIndices, *storage_, level, num_ );
std::sort( vIndices.begin(), vIndices.end() );
std::sort( pIndices.begin(), pIndices.end() );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Gather index sets" );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Set-up permutation" );
std::vector< int32_t > perm_vec;
std::vector< int32_t > recv_sizes( comm_->size() );
......@@ -177,6 +186,9 @@ class GinkgoBlockSolver : public Solver< OperatorType >
gko::mpi::gather(
pIndices.data(), local_size, perm_vec.data() + global_v_size, recv_sizes.data(), recv_offsets.data(), 0, comm_ );
gko::Array< gko::int32 > perm{ solver_exec_, perm_vec.begin(), perm_vec.end() };
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Set-up permutation" );
auto [p_v_min, p_v_max] = std::minmax_element( std::begin( vIndices ), std::end( vIndices ) );
auto [p_p_min, p_p_max] = std::minmax_element( std::begin( pIndices ), std::end( pIndices ) );
gko::span v_span( *p_v_min, *p_v_max + 1 );
......@@ -186,28 +198,35 @@ class GinkgoBlockSolver : public Solver< OperatorType >
WALBERLA_ABORT( "Indices are NOT blocked: v" << v_span << ", p" << p_span )
}
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Gather matrix" );
monolithic_matrix_ = gko::share( csr::create( solver_exec_ ) );
gko::as< mtx >( host_monolithic_matrix_ )->convert_to( monolithic_matrix_.get() );
gko::Array< gko::int32 > perm{ solver_exec_, perm_vec.begin(), perm_vec.end() };
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Gather matrix" );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Permute matrix" );
if ( comm_->size() > 1 && monolithic_matrix_->get_size() )
{
monolithic_matrix_ = gko::as< csr >( monolithic_matrix_->permute( &perm ) );
monolithic_matrix_->sort_by_column_index();
}
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Permute matrix" );
{
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Gather preconditioner" );
auto device_monolithic_pre = gko::share( csr::create( solver_exec_ ) );
monolithic_preconditioner_->convert_to( device_monolithic_pre.get() );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Gather preconditioner" );
if ( device_monolithic_pre->get_size() && device_monolithic_pre->get_size() )
{
gko::span block_v_span{ 0, global_v_size };
gko::span block_p_span{ global_v_size, global_v_size + global_p_size };
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Permute preconditioner" );
device_monolithic_pre = gko::as< csr >( device_monolithic_pre->permute( &perm ) );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Permute preconditioner" );
std::shared_ptr< gko::LinOp > b_pre_v;
std::shared_ptr< gko::LinOp > b_pre_p;
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Build block preconditioner" );
auto jac_gen =
gko::share( gko::preconditioner::Jacobi< valueType >::build().with_max_block_size( 1u ).on( solver_exec_ ) );
if ( velocityPreconditionerType_ == 0 )
......@@ -278,7 +297,9 @@ class GinkgoBlockSolver : public Solver< OperatorType >
gko::dim< 2 >{ global_v_size + global_p_size, global_v_size + global_p_size },
std::vector< std::vector< std::shared_ptr< gko::LinOp > > >{ { b_pre_v, b_pre_vp }, { b_pre_pv, b_pre_p } },
std::vector< gko::span >{ block_v_span, block_p_span } ) );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Build block preconditioner" );
}
WALBERLA_MPI_BARRIER();
}
auto log = gko::share( gko::log::Convergence< valueType >::create( solver_exec_ ) );
......@@ -303,11 +324,14 @@ class GinkgoBlockSolver : public Solver< OperatorType >
solver_->set_stop_criterion_factory( gko::share( factory ) );
}
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Gather vectors" );
auto global_rhs = dense::create( host_exec_ );
auto global_x0 = dense::create( host_exec_ );
rhs->convert_to( gko::lend( global_rhs ) );
x0->convert_to( gko::lend( global_x0 ) );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Gather vectors" );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Apply" );
if ( monolithic_matrix_->get_size() )
{
if ( comm_->size() > 1 )
......@@ -322,8 +346,13 @@ class GinkgoBlockSolver : public Solver< OperatorType >
solver_->apply( gko::lend( global_rhs ), gko::lend( global_x0 ) );
}
}
WALBERLA_MPI_BARRIER();
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Apply" );
x.getStorage()->getTimingTree()->start( "Ginkgo CG Solver Scatter vectors" );
gather_idxs_ = compute_gather_idxs( part_ );
scatter_global_vector( global_x0.get(), x0, gather_idxs_, comm_ );
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver Scatter vectors" );
dir_handler->update_solution( x0 );
......@@ -338,6 +367,7 @@ class GinkgoBlockSolver : public Solver< OperatorType >
<< solver_exec_->copy_val_to_host(
gko::as< gko::matrix::Dense< valueType > >( log->get_residual_norm() )->get_const_values() ) );
}
x.getStorage()->getTimingTree()->stop( "Ginkgo CG Solver" );
}
void setPrintInfo( bool printInfo ) { printInfo_ = printInfo; }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment