Commit 51bdeacc authored by Nils Kohl's avatar Nils Kohl 🌝
Browse files

Implemented unknown sender communication.

See issue #121 and MR !297

Allows to set the receiver info in the GenericBufferSystem
to a number of sender ranks, without specifying the exact
ranks themselves.

Implementation optimized for minimal code duplication.
Functionality was implemented by extending IProbe-based
communication.
parent 13760c48
......@@ -233,12 +233,13 @@ protected:
internal::KnownSizeCommunication<RecvBuffer_T, SendBuffer_T> knownSizeComm_;
internal::UnknownSizeCommunication<RecvBuffer_T, SendBuffer_T> unknownSizeComm_;
internal::UnknownSizeCommunicationIProbe<RecvBuffer_T, SendBuffer_T> unknownSizeCommIProbe_;
internal::UnknownSizeCommunicationIProbe<RecvBuffer_T, SendBuffer_T> unknownSizeAndSenderCommIProbe_;
internal::NoMPICommunication<RecvBuffer_T, SendBuffer_T> noMPIComm_;
internal::AbstractCommunication<RecvBuffer_T, SendBuffer_T> * currentComm_; //< after receiver setup, this points to unknown- or knownSizeComm_
bool sizeChangesEverytime_; //< if set to true, the receiveSizeUnknown_ is set to true before communicating
bool communicationRunning_; //< indicates if a communication step is currently running
bool senderKnown_; //< if false, the sender ranks are unknown before receiving
/// Info about the message to be received from a certain rank:
/// information holds the buffer and, if known, the message size
......
......@@ -23,6 +23,7 @@
#include "core/mpi/MPIManager.h"
#include "core/debug/CheckFunctions.h"
#include <algorithm>
namespace walberla {
namespace mpi {
......@@ -128,10 +129,12 @@ GenericBufferSystem<Rb, Sb>::GenericBufferSystem( const MPI_Comm & communicator,
: knownSizeComm_ ( communicator, tag ),
unknownSizeComm_( communicator, tag ),
unknownSizeCommIProbe_( communicator, tag ),
unknownSizeAndSenderCommIProbe_( communicator, tag, false ),
noMPIComm_( communicator, tag ),
currentComm_ ( nullptr ),
sizeChangesEverytime_( true ),
communicationRunning_( false )
communicationRunning_( false ),
senderKnown_( true )
{
}
......@@ -140,10 +143,12 @@ GenericBufferSystem<Rb, Sb>::GenericBufferSystem( const GenericBufferSystem &oth
: knownSizeComm_ ( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
unknownSizeComm_( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
unknownSizeCommIProbe_( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
unknownSizeAndSenderCommIProbe_( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag(), false ),
noMPIComm_ ( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
currentComm_ ( nullptr ),
sizeChangesEverytime_( other.sizeChangesEverytime_ ),
communicationRunning_( other.communicationRunning_ ),
senderKnown_( true ),
recvInfos_( other.recvInfos_ ),
sendInfos_( other.sendInfos_ )
{
......@@ -154,6 +159,8 @@ GenericBufferSystem<Rb, Sb>::GenericBufferSystem( const GenericBufferSystem &oth
currentComm_ = &unknownSizeComm_;
else if ( other.currentComm_ == &other.unknownSizeCommIProbe_ )
currentComm_ = &unknownSizeCommIProbe_;
else if ( other.currentComm_ == &other.unknownSizeAndSenderCommIProbe_ )
currentComm_ = &unknownSizeAndSenderCommIProbe_;
else if ( other.currentComm_ == &other.noMPIComm_ )
currentComm_ = &noMPIComm_;
else
......@@ -167,6 +174,7 @@ GenericBufferSystem<Rb, Sb> & GenericBufferSystem<Rb, Sb>::operator=( const Gene
sizeChangesEverytime_ = other.sizeChangesEverytime_;
communicationRunning_ = other.communicationRunning_;
senderKnown_ = other.senderKnown_;
recvInfos_ = other.recvInfos_;
sendInfos_ = other.sendInfos_;
......@@ -176,6 +184,8 @@ GenericBufferSystem<Rb, Sb> & GenericBufferSystem<Rb, Sb>::operator=( const Gene
currentComm_ = &unknownSizeComm_;
else if ( other.currentComm_ == &other.unknownSizeCommIProbe_ )
currentComm_ = &unknownSizeCommIProbe_;
else if ( other.currentComm_ == &other.unknownSizeAndSenderCommIProbe_ )
currentComm_ = &unknownSizeAndSenderCommIProbe_;
else if ( other.currentComm_ == &other.noMPIComm_ )
currentComm_ = &noMPIComm_;
else
......@@ -228,7 +238,20 @@ void GenericBufferSystem<Rb, Sb>::setReceiverInfo( const std::set<MPIRank> & ran
template< typename Rb, typename Sb>
void GenericBufferSystem<Rb, Sb>::setReceiverInfo( const int numReceives )
{
WALBERLA_ABORT("NOT IMPLEMENTED!");
WALBERLA_ASSERT( ! communicationRunning_ );
recvInfos_.clear();
for ( MPIRank sender = 0; sender < numReceives; sender++ )
{
recvInfos_[ - 1 - sender ].size = INVALID_SIZE;
}
// "any sender"-communication is only supported via IProbe
useIProbe( true );
sizeChangesEverytime_ = true;
senderKnown_ = false;
setCommunicationType( false );
}
......@@ -526,13 +549,22 @@ void GenericBufferSystem<Rb, Sb>::setCommunicationType( const bool knownSize )
WALBERLA_MPI_SECTION()
{
if( knownSize )
if ( senderKnown_ )
{
if (knownSize)
currentComm_ = &knownSizeComm_;
else if ( useIProbe_ )
else if (useIProbe_)
currentComm_ = &unknownSizeCommIProbe_;
else
currentComm_ = &unknownSizeComm_;
}
else
{
WALBERLA_CHECK( useIProbe_, "Unknown sender communication is currently only supported with IProbe-based "
"communication." )
currentComm_ = &unknownSizeAndSenderCommIProbe_;
}
}
}
......
......@@ -167,8 +167,9 @@ namespace internal {
public:
using typename AbstractCommunication<RecvBuffer_T, SendBuffer_T>::ReceiveInfo;
UnknownSizeCommunicationIProbe( const MPI_Comm & communicator, int tag = 0 )
: AbstractCommunication<RecvBuffer_T, SendBuffer_T>( communicator, tag ), sending_(false), receiving_(false) {}
UnknownSizeCommunicationIProbe( const MPI_Comm & communicator, int tag = 0, bool senderKnown = true )
: AbstractCommunication<RecvBuffer_T, SendBuffer_T>( communicator, tag ), sending_(false), receiving_(false),
senderKnown_( senderKnown ) {}
virtual ~UnknownSizeCommunicationIProbe() {}
......@@ -184,6 +185,7 @@ namespace internal {
bool sending_;
bool receiving_;
int pendingReceives_;
bool senderKnown_;
std::vector<MPI_Request> sendRequests_;
};
......
......@@ -352,6 +352,7 @@ MPIRank UnknownSizeCommunication<Rb, Sb>::waitForNextReceive( std::map<MPIRank,
//======================================================================================================================
//
// Unknown Size Communication (IProbe method)
// Can also be used if there are unknown sender ranks.
//
//======================================================================================================================
......@@ -438,15 +439,29 @@ MPIRank UnknownSizeCommunicationIProbe<Rb, Sb>::waitForNextReceive( std::map<MPI
if (recvInfo.size != INVALID_SIZE) continue;
const MPIRank iprobeSender = senderKnown_ ? sender : MPI_ANY_SOURCE;
int probeFlag;
MPI_Status probeStatus;
MPI_Iprobe( sender,
MPI_Iprobe( iprobeSender,
this->tag_,
this->communicator_,
&probeFlag,
&probeStatus);
if (probeFlag)
{
const MPIRank actualSender = probeStatus.MPI_SOURCE;
if ( !senderKnown_ )
{
recvInfos.erase( sender );
recvInfo = recvInfos[actualSender];
}
else
{
WALBERLA_ASSERT_EQUAL( sender, actualSender );
}
int count = 0;
MPI_Get_count( &probeStatus, MPI_BYTE, &count );
//WALBERLA_LOG_DEVEL("received " << count << " from " << sender);
......@@ -457,14 +472,14 @@ MPIRank UnknownSizeCommunicationIProbe<Rb, Sb>::waitForNextReceive( std::map<MPI
MPI_Recv( recvInfo.buffer.ptr(), // where to store received size
count, // size of expected message
MPI_BYTE, // type
sender, // rank of sender process
actualSender, // rank of sender process
this->tag_, // message tag
this->communicator_, // communicator
&recvStatus // request, needed for wait
);
--pendingReceives_;
return sender;
return actualSender;
}
}
}
......@@ -539,6 +554,7 @@ MPIRank NoMPICommunication<Rb, Sb>::waitForNextReceive( std::map<MPIRank, Receiv
} // namespace internal
} // namespace mpi
} // namespace walberla
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment