From d8fe0a2e97e2d5a1c09fdfd3d44ff3f6a820fbb7 Mon Sep 17 00:00:00 2001
From: Sebastian Eibl <sebastian.eibl@fau.de>
Date: Tue, 27 Feb 2018 13:56:32 +0100
Subject: [PATCH] [BUGFIX] prevented communication to non existent ranks
---
.../loadbalancing/DynamicParMetis.cpp | 20 +++++++++--
.../loadbalancing/DynamicParMetis.h | 6 ++++
src/core/mpi/MPIHelper.cpp | 33 +++++++++++++++++++
3 files changed, 56 insertions(+), 3 deletions(-)
diff --git a/src/blockforest/loadbalancing/DynamicParMetis.cpp b/src/blockforest/loadbalancing/DynamicParMetis.cpp
index a2cc8f1a9..9989aca6b 100644
--- a/src/blockforest/loadbalancing/DynamicParMetis.cpp
+++ b/src/blockforest/loadbalancing/DynamicParMetis.cpp
@@ -30,6 +30,7 @@
#include "core/mpi/MPIManager.h"
#include "core/mpi/Gather.h"
#include "core/mpi/Gatherv.h"
+#include "core/mpi/Reduce.h"
#include "core/timing/Timer.h"
@@ -42,6 +43,12 @@ namespace blockforest {
std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phantomForest, MPI_Comm comm )
{
const uint_t rank = uint_c(mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, MPIManager::instance()->rank()));
+ WALBERLA_DEBUG_SECTION()
+ {
+ int rankRaw;
+ MPI_Comm_rank(comm, &rankRaw);
+ WALBERLA_ASSERT_EQUAL(rank, rankRaw);
+ }
uint_t numLocalBlocks = phantomForest.getNumberOfBlocks();
@@ -68,7 +75,11 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
mpi::BufferSystem bs( comm );
for( auto it = neighborProcesses.begin(); it != neighborProcesses.end(); ++it )
- bs.sendBuffer( mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it)) ) << mapping;
+ {
+ auto destRank = mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it));
+ if (destRank != -1)
+ bs.sendBuffer( destRank ) << mapping;
+ }
bs.setReceiverInfoFromSendBufferState( false, true );
@@ -101,6 +112,8 @@ T * ptr( std::vector<T> & v )
typedef uint_t idx_t;
+
+
bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest,
@@ -177,14 +190,15 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
WALBERLA_ASSERT_EQUAL( vwgt.size(), phantomForest.getNumberOfBlocks() );
WALBERLA_ASSERT_EQUAL( vsize.size(), phantomForest.getNumberOfBlocks() );
WALBERLA_ASSERT_EQUAL( adjncy.size(), adjwgt.size() );
+ WALBERLA_ASSERT_EQUAL( adjwgt.size(), xadj.back() );
int64_t wgtflag = weightsToUse_;
int64_t numflag = 0; // C-style ordering
int64_t ncon = 1; // Number of constraints
int64_t ndims = 3; // Number of dimensions
- double ubvec[] = { real_t( 1.05 ) }; // imbalance tolerance
+ double ubvec[] = { real_t( 1.05 ) }; // imbalance tolerance
int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
- double ipc2redist = real_t( 1000000.0 ); // compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
+ auto ipc2redist = ipc2redist_;
MPI_Comm comm = subComm; //MPIManager::instance()->comm();
std::vector<double> tpwgts( uint_c(nparts * ncon), 1.0 / double_c( nparts ) ); // vertex weight fraction that is stored in a subdomain
int64_t options[] = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
diff --git a/src/blockforest/loadbalancing/DynamicParMetis.h b/src/blockforest/loadbalancing/DynamicParMetis.h
index 82bc4dd43..a17a2e974 100644
--- a/src/blockforest/loadbalancing/DynamicParMetis.h
+++ b/src/blockforest/loadbalancing/DynamicParMetis.h
@@ -54,6 +54,10 @@ public:
const PhantomBlockForest & phantomForest,
const uint_t iteration ) const;
+
+ void setipc2redist(double val) {ipc2redist_ = val;}
+ double getipc2redist() const {return ipc2redist_;}
+
bool edgeWeightsUsed() const { return ( weightsToUse_ == PARMETIS_EDGE_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexWeightsUsed() const { return ( weightsToUse_ == PARMETIS_VERTEX_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexSizeUsed() const { return algorithm_ == PARMETIS_ADAPTIVE_REPART; }
@@ -70,6 +74,8 @@ protected:
Algorithm algorithm_;
WeightsToUse weightsToUse_;
EdgeSource edgeSource_;
+
+ double ipc2redist_ = real_t( 1000.0 ); ///< compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
};
class DynamicParMetisBlockInfo
diff --git a/src/core/mpi/MPIHelper.cpp b/src/core/mpi/MPIHelper.cpp
index 8822c198d..0a667acd9 100644
--- a/src/core/mpi/MPIHelper.cpp
+++ b/src/core/mpi/MPIHelper.cpp
@@ -20,9 +20,18 @@
#include "MPIHelper.h"
+#include <core/debug/CheckFunctions.h>
+
namespace walberla {
namespace mpi {
+//!
+//! \brief This functions maps the rank in one communicator to the rank in another communicator.
+//! \param srcComm source communicator
+//! \param destComm destination communicator
+//! \param srcRank rank in the source communicator
+//! \return rank in the destination communicator or -1 if not available
+//!
int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int srcRank)
{
int destRank = -1;
@@ -30,11 +39,25 @@ int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int src
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, 1, const_cast<int*>(&srcRank), destGroup, &destRank);
+ int size;
+ MPI_Comm_size(destComm, &size);
+ if (destRank < 0 || destRank >= size)
+ {
+ WALBERLA_CHECK_EQUAL( destRank, MPI_UNDEFINED );
+ destRank = -1;
+ }
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
}
+//!
+//! \brief This functions converts a array of ranks in one communicator to an array of ranks in another communicator.
+//! \param srcComm source communicator
+//! \param destComm destination communicator
+//! \param srcRank source ranks
+//! \return converted ranks, -1 if not available
+//!
std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const std::vector<int>& srcRank)
{
std::vector<int> destRank(srcRank.size(), -1);
@@ -42,6 +65,16 @@ std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm,
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, int_c(srcRank.size()), const_cast<int*>(&srcRank[0]), destGroup, &destRank[0]);
+ int size;
+ MPI_Comm_size(destComm, &size);
+ for (auto& dstRnk : destRank)
+ {
+ if (dstRnk < 0 || dstRnk >= size)
+ {
+ WALBERLA_CHECK_EQUAL( dstRnk, MPI_UNDEFINED );
+ dstRnk = -1;
+ }
+ }
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
--
GitLab