Skip to content
Snippets Groups Projects
Commit b41f9709 authored by Sebastian Eibl's avatar Sebastian Eibl
Browse files

extended checks in DynamicParMetis

parent 1b218e73
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/blockforest/loadbalancing/DynamicParMetis.cpp
+ 87
− 34
View file @ b41f9709
...@@ -37,10 +37,13 @@ ...@@ -37,10 +37,13 @@
#include <boost/algorithm/string/case_conv.hpp> #include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/trim.hpp>
#include <array>
#include <vector>
namespace walberla { namespace walberla {
namespace blockforest { namespace blockforest {
std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phantomForest, MPI_Comm comm ) std::pair<uint_t, uint_t> getBlockSequenceRange( uint_t numLocalBlocks, MPI_Comm comm )
{ {
const uint_t rank = uint_c(mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, MPIManager::instance()->rank())); const uint_t rank = uint_c(mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, MPIManager::instance()->rank()));
WALBERLA_DEBUG_SECTION() WALBERLA_DEBUG_SECTION()
...@@ -50,8 +53,6 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan ...@@ -50,8 +53,6 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan
WALBERLA_ASSERT_EQUAL(rank, rankRaw); WALBERLA_ASSERT_EQUAL(rank, rankRaw);
} }
uint_t numLocalBlocks = phantomForest.getNumberOfBlocks();
size_t sequenceStartOnProcess = 0; size_t sequenceStartOnProcess = 0;
MPI_Exscan( &numLocalBlocks, &sequenceStartOnProcess, 1, MPITrait<uint_t>::type(), MPI_SUM, comm ); MPI_Exscan( &numLocalBlocks, &sequenceStartOnProcess, 1, MPITrait<uint_t>::type(), MPI_SUM, comm );
if( rank == 0 ) if( rank == 0 )
...@@ -60,14 +61,13 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan ...@@ -60,14 +61,13 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan
return std::make_pair( sequenceStartOnProcess, sequenceStartOnProcess + numLocalBlocks ); return std::make_pair( sequenceStartOnProcess, sequenceStartOnProcess + numLocalBlocks );
} }
std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest & phantomForest, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm ) std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest& phantomForest, const std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm )
{ {
std::map< blockforest::BlockID, uint_t > mapping; std::map< blockforest::BlockID, uint_t > mapping;
const auto & blockMap = phantomForest.getBlockMap();
uint_t sequenceId = blockSequenceRange.first; uint_t sequenceId = blockSequenceRange.first;
for( auto it = blockMap.begin(); it != blockMap.end(); ++it ) for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it )
mapping.insert( std::make_pair( it->first, sequenceId++ ) ); mapping.insert( std::make_pair( it->first->getId(), sequenceId++ ) );
WALBERLA_ASSERT_EQUAL( sequenceId, blockSequenceRange.second ); WALBERLA_ASSERT_EQUAL( sequenceId, blockSequenceRange.second );
const std::vector<uint_t> neighborProcesses = phantomForest.getNeighboringProcesses(); const std::vector<uint_t> neighborProcesses = phantomForest.getNeighboringProcesses();
...@@ -124,11 +124,11 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, ...@@ -124,11 +124,11 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
globalTimer.start(); globalTimer.start();
//create new communicator which excludes processes which do not have blocks //create new communicator which excludes processes which do not have blocks
MPI_Comm subComm = MPIManager::instance()->comm(); MPI_Comm subComm = MPI_COMM_NULL;
MPI_Group allGroup, subGroup; MPI_Group allGroup, subGroup;
MPI_Comm_group( MPIManager::instance()->comm(), &allGroup ); MPI_Comm_group( MPIManager::instance()->comm(), &allGroup );
std::vector<int> ranks; std::vector<int> ranks;
if (phantomForest.getNumberOfBlocks() > 0) if (targetProcess.size() > 0)
ranks.push_back( MPIManager::instance()->rank() ); ranks.push_back( MPIManager::instance()->rank() );
ranks = mpi::allGatherv( ranks, MPIManager::instance()->comm() ); ranks = mpi::allGatherv( ranks, MPIManager::instance()->comm() );
auto numSubProcesses = ranks.size(); auto numSubProcesses = ranks.size();
...@@ -136,22 +136,51 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, ...@@ -136,22 +136,51 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
MPI_Group_incl(allGroup, int_c(ranks.size()), &ranks[0], &subGroup); MPI_Group_incl(allGroup, int_c(ranks.size()), &ranks[0], &subGroup);
MPI_Comm_create( MPIManager::instance()->comm(), subGroup, &subComm); MPI_Comm_create( MPIManager::instance()->comm(), subGroup, &subComm);
if ( targetProcess.size() != 0)
{
int subRank;
int subSize;
MPI_Comm_rank(subComm, &subRank);
MPI_Comm_size(subComm, &subSize);
WALBERLA_CHECK_GREATER_EQUAL(subRank, 0);
WALBERLA_CHECK_LESS(subRank, subSize);
} else
{
int subRank;
MPI_Comm_rank(subComm, &subRank);
WALBERLA_CHECK_EQUAL(subRank, MPI_UNDEFINED);
}
int64_t edgecut = 0; int64_t edgecut = 0;
std::vector<int64_t> part( phantomForest.getNumberOfBlocks(), int64_c( MPIManager::instance()->rank() ) ); WALBERLA_CHECK_EQUAL( phantomForest.getNumberOfBlocks(), targetProcess.size() );
std::vector<int64_t> part( targetProcess.size(), int64_c( MPIManager::instance()->rank() ) );
if (subComm != MPI_COMM_NULL) if (subComm != MPI_COMM_NULL)
{ {
const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( phantomForest, subComm ); int subRank;
const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, blockSequenceRange, subComm ); //blockid to vertex id int subSize;
MPI_Comm_rank(subComm, &subRank);
MPI_Comm_size(subComm, &subSize);
WALBERLA_CHECK_UNEQUAL(targetProcess.size(), 0);
const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( targetProcess.size(), subComm );
const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, targetProcess, blockSequenceRange, subComm ); //blockid to vertex id
std::vector<int64_t> vtxdist = mpi::allGather( int64_c( blockSequenceRange.second ), subComm ); std::vector<int64_t> vtxdist = mpi::allGather( int64_c( blockSequenceRange.second ), subComm );
vtxdist.insert( vtxdist.begin(), uint_t( 0 ) ); vtxdist.insert( vtxdist.begin(), uint_t( 0 ) );
WALBERLA_CHECK_EQUAL( vtxdist.size(), subSize + 1 );
for (size_t i = 1; i < vtxdist.size(); ++i)
{
WALBERLA_ASSERT_LESS( vtxdist[i-1], vtxdist[i] );
}
std::vector<int64_t> adjncy, xadj, vsize, vwgt, adjwgt; std::vector<int64_t> adjncy, xadj, vsize, vwgt, adjwgt;
std::vector<double> xyz; std::vector<double> xyz;
for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it ) uint_t blockIndex = 0;
for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it, ++blockIndex )
{ {
WALBERLA_CHECK_EQUAL(blockIndex, mapping.find(it->first->getId())->second - blockSequenceRange.first);
xadj.push_back( int64_c( adjncy.size() ) ); xadj.push_back( int64_c( adjncy.size() ) );
const PhantomBlock & block = *( it->first ); const PhantomBlock & block = *( it->first );
auto bi = block.getData< DynamicParMetisBlockInfo >(); auto bi = block.getData< DynamicParMetisBlockInfo >();
...@@ -163,9 +192,12 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, ...@@ -163,9 +192,12 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{ {
auto mapIt = mapping.find( nit->getId() ); auto mapIt = mapping.find( nit->getId() );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" ); WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) ); adjncy.push_back( int64_c( mapIt->second ) );
auto edgeWeightIt = bi.getEdgeWeights().find( nit->getId() ); auto edgeWeightIt = bi.getEdgeWeights().find( nit->getId() );
adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 0 ) : edgeWeightIt->second ); //WALBERLA_CHECK_UNEQUAL( edgeWeightIt->second, 0 ); // perhaps WARNING
adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 1 ) : edgeWeightIt->second );
} }
break; break;
case PARMETIS_EDGES_FROM_EDGE_WEIGHTS: case PARMETIS_EDGES_FROM_EDGE_WEIGHTS:
...@@ -173,10 +205,15 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, ...@@ -173,10 +205,15 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{ {
auto mapIt = mapping.find( edgeIt->first ); auto mapIt = mapping.find( edgeIt->first );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" ); WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) ); adjncy.push_back( int64_c( mapIt->second ) );
//WALBERLA_CHECK_UNEQUAL( edgeIt->second, 0 ); // perhaps WARNING
adjwgt.push_back( edgeIt->second ); adjwgt.push_back( edgeIt->second );
} }
break;
} }
WALBERLA_CHECK_UNEQUAL( bi.getVertexWeight(), 0 );
vwgt.push_back( bi.getVertexWeight() ); vwgt.push_back( bi.getVertexWeight() );
vsize.push_back( bi.getVertexSize() ); vsize.push_back( bi.getVertexSize() );
xyz.push_back( bi.getVertexCoords()[0] ); xyz.push_back( bi.getVertexCoords()[0] );
...@@ -185,46 +222,58 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, ...@@ -185,46 +222,58 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
} }
xadj.push_back( int64_c( adjncy.size() ) ); xadj.push_back( int64_c( adjncy.size() ) );
WALBERLA_ASSERT_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) ); WALBERLA_CHECK_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) );
WALBERLA_ASSERT_EQUAL( xadj.size(), phantomForest.getNumberOfBlocks() + 1 ); WALBERLA_CHECK_EQUAL( xadj.size(), targetProcess.size() + 1 );
WALBERLA_ASSERT_EQUAL( vwgt.size(), phantomForest.getNumberOfBlocks() ); WALBERLA_CHECK_EQUAL( xadj.front(), 0);
WALBERLA_ASSERT_EQUAL( vsize.size(), phantomForest.getNumberOfBlocks() ); WALBERLA_CHECK_EQUAL( xadj.back(), adjncy.size() );
WALBERLA_ASSERT_EQUAL( adjncy.size(), adjwgt.size() ); for (size_t i = 1; i < xadj.size(); ++i)
WALBERLA_ASSERT_EQUAL( adjwgt.size(), xadj.back() ); {
WALBERLA_ASSERT_LESS( xadj[i-1], xadj[i] );
int64_t wgtflag = weightsToUse_; }
int64_t numflag = 0; // C-style ordering WALBERLA_CHECK_EQUAL( vwgt.size(), targetProcess.size() );
int64_t ncon = 1; // Number of constraints WALBERLA_CHECK_EQUAL( vsize.size(), targetProcess.size() );
int64_t ndims = 3; // Number of dimensions WALBERLA_CHECK_EQUAL( xyz.size(), targetProcess.size() * 3 );
double ubvec[] = { real_t( 1.05 ) }; // imbalance tolerance WALBERLA_CHECK_EQUAL( adjncy.size(), adjwgt.size() );
int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains WALBERLA_CHECK_EQUAL( adjwgt.size(), xadj.back() );
auto ipc2redist = ipc2redist_;
MPI_Comm comm = subComm; //MPIManager::instance()->comm(); int64_t wgtflag = weightsToUse_;
int64_t numflag = 0; // C-style ordering
int64_t ncon = 1; // Number of constraints
int64_t ndims = 3; // Number of dimensions
std::vector<double> ubvec( uint_c(ncon), double_c( 1.05 ) ); // imbalance tolerance
int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
double ipc2redist = double_c(ipc2redist_);
MPI_Comm comm = subComm; //MPIManager::instance()->comm();
std::vector<double> tpwgts( uint_c(nparts * ncon), 1.0 / double_c( nparts ) ); // vertex weight fraction that is stored in a subdomain std::vector<double> tpwgts( uint_c(nparts * ncon), 1.0 / double_c( nparts ) ); // vertex weight fraction that is stored in a subdomain
int64_t options[] = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) }; std::vector<int64_t> options = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
int metisResult = core::METIS_OK; int metisResult = core::METIS_OK;
switch( algorithm_ ) switch( algorithm_ )
{ {
case PARMETIS_PART_GEOM:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartGeom( ptr( vtxdist ), &ndims, ptr( xyz ), ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_PART_GEOM_KWAY: case PARMETIS_PART_GEOM_KWAY:
parmetisTimer.start(); parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm ); metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ptr( ubvec ), ptr( options ), &edgecut, ptr( part ), &comm );
parmetisTimer.end(); parmetisTimer.end();
break; break;
case PARMETIS_PART_KWAY: case PARMETIS_PART_KWAY:
parmetisTimer.start(); parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm ); metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end(); parmetisTimer.end();
break; break;
case PARMETIS_ADAPTIVE_REPART: case PARMETIS_ADAPTIVE_REPART:
parmetisTimer.start(); parmetisTimer.start();
metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, &ipc2redist, options, &edgecut, ptr( part ), &comm ); metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), &ipc2redist, ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end(); parmetisTimer.end();
break; break;
case PARMETIS_REFINE_KWAY: case PARMETIS_REFINE_KWAY:
parmetisTimer.start(); parmetisTimer.start();
metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm ); metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end(); parmetisTimer.end();
break; break;
} }
...@@ -279,6 +328,8 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s ) ...@@ -279,6 +328,8 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s )
if( s == "PART_GEOM_KWAY" ) if( s == "PART_GEOM_KWAY" )
return PARMETIS_PART_GEOM_KWAY; return PARMETIS_PART_GEOM_KWAY;
else if( s == "PART_GEOM" )
return PARMETIS_PART_GEOM;
else if( s == "PART_KWAY" ) else if( s == "PART_KWAY" )
return PARMETIS_PART_KWAY; return PARMETIS_PART_KWAY;
else if( s == "PART_ADAPTIVE_REPART" ) else if( s == "PART_ADAPTIVE_REPART" )
...@@ -328,6 +379,8 @@ std::string DynamicParMetis::algorithmToString( ) const ...@@ -328,6 +379,8 @@ std::string DynamicParMetis::algorithmToString( ) const
{ {
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM_KWAY: case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM_KWAY:
return "PART_GEOM_KWAY"; return "PART_GEOM_KWAY";
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM:
return "PART_GEOM";
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_KWAY: case walberla::blockforest::DynamicParMetis::PARMETIS_PART_KWAY:
return "PART_KWAY"; return "PART_KWAY";
case walberla::blockforest::DynamicParMetis::PARMETIS_ADAPTIVE_REPART: case walberla::blockforest::DynamicParMetis::PARMETIS_ADAPTIVE_REPART:
......
src/blockforest/loadbalancing/DynamicParMetis.h
+ 1
− 1
View file @ b41f9709
...@@ -42,7 +42,7 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan ...@@ -42,7 +42,7 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
class DynamicParMetis class DynamicParMetis
{ {
public: public:
enum Algorithm { PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY }; enum Algorithm { PARMETIS_PART_GEOM, PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 }; enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 };
enum EdgeSource { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS }; enum EdgeSource { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS };
......
src/core/mpi/MPIHelper.cpp
+ 16
− 10
View file @ b41f9709
...@@ -34,6 +34,11 @@ namespace mpi { ...@@ -34,6 +34,11 @@ namespace mpi {
//! //!
int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int srcRank) int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int srcRank)
{ {
if (srcComm == destComm)
{
return srcRank;
}
int destRank = -1; int destRank = -1;
MPI_Group srcGroup, destGroup; MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup); MPI_Comm_group(srcComm, &srcGroup);
...@@ -41,11 +46,9 @@ int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int src ...@@ -41,11 +46,9 @@ int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int src
MPI_Group_translate_ranks(srcGroup, 1, const_cast<int*>(&srcRank), destGroup, &destRank); MPI_Group_translate_ranks(srcGroup, 1, const_cast<int*>(&srcRank), destGroup, &destRank);
int size; int size;
MPI_Comm_size(destComm, &size); MPI_Comm_size(destComm, &size);
if (destRank < 0 || destRank >= size) if (destRank == MPI_UNDEFINED) destRank = -1;
{ WALBERLA_CHECK_GREATER_EQUAL(destRank, -1);
WALBERLA_CHECK_EQUAL( destRank, MPI_UNDEFINED ); WALBERLA_CHECK_LESS(destRank, size);
destRank = -1;
}
MPI_Group_free(&srcGroup); MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup); MPI_Group_free(&destGroup);
return destRank; return destRank;
...@@ -60,6 +63,11 @@ int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int src ...@@ -60,6 +63,11 @@ int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int src
//! //!
std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const std::vector<int>& srcRank) std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const std::vector<int>& srcRank)
{ {
if (srcComm == destComm)
{
return srcRank;
}
std::vector<int> destRank(srcRank.size(), -1); std::vector<int> destRank(srcRank.size(), -1);
MPI_Group srcGroup, destGroup; MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup); MPI_Comm_group(srcComm, &srcGroup);
...@@ -69,11 +77,9 @@ std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, ...@@ -69,11 +77,9 @@ std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm,
MPI_Comm_size(destComm, &size); MPI_Comm_size(destComm, &size);
for (auto& dstRnk : destRank) for (auto& dstRnk : destRank)
{ {
if (dstRnk < 0 || dstRnk >= size) if (dstRnk == MPI_UNDEFINED) dstRnk = -1;
{ WALBERLA_CHECK_GREATER_EQUAL(dstRnk, -1);
WALBERLA_CHECK_EQUAL( dstRnk, MPI_UNDEFINED ); WALBERLA_CHECK_LESS(dstRnk, size);
dstRnk = -1;
}
} }
MPI_Group_free(&srcGroup); MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup); MPI_Group_free(&destGroup);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment