diff --git a/src/blockforest/communication/UniformBufferedScheme.h b/src/blockforest/communication/UniformBufferedScheme.h
index 1abd5a95592d12247a944137679446bf788e9c43..c16a9146dd39bc683fc00cebf52be5dc8eeb0e1d 100644
--- a/src/blockforest/communication/UniformBufferedScheme.h
+++ b/src/blockforest/communication/UniformBufferedScheme.h
@@ -210,7 +210,7 @@ protected:
template< typename Stencil >
inline void UniformBufferedScheme<Stencil>::addPackInfo( const PackInfo & packInfo )
{
- WALBERLA_ASSERT( !communicationInProgress_ ); // You cannot add PackInfo's while the communication is in progress!
+ WALBERLA_ASSERT( !communicationInProgress_, "Cannot add pack info while communication is in progress");
packInfos_.push_back( packInfo );
setupBeforeNextCommunication_ = true;
diff --git a/src/core/mpi/BufferSystem.h b/src/core/mpi/BufferSystem.h
index 926a7181ffca5cab7ff418cef67057aef20c2066..2a04730f7e2cbade9c4bf2e8addc612786945234 100644
--- a/src/core/mpi/BufferSystem.h
+++ b/src/core/mpi/BufferSystem.h
@@ -35,7 +35,8 @@
namespace walberla {
namespace mpi {
-class OpenMPBufferSystem;
+template< typename RecvBuffer_T, typename SendBuffer_T>
+class GenericOpenMPBufferSystem;
//**********************************************************************************************************************
/*! Manages MPI Communication with a set of known communication partners.
@@ -106,7 +107,8 @@ class OpenMPBufferSystem;
*
*/
//**********************************************************************************************************************
-class BufferSystem
+template< typename RecvBuffer_T = RecvBuffer, typename SendBuffer_T = SendBuffer>
+class GenericBufferSystem
{
public:
class iterator;
@@ -114,10 +116,10 @@ public:
//**Construction and Destruction*************************************************************************************
/*!\name Constructors */
//@{
- explicit BufferSystem( const MPI_Comm & communicator, int tag = 0 );
- BufferSystem( const BufferSystem & other );
- BufferSystem & operator=( const BufferSystem & other );
- ~BufferSystem() {}
+ explicit GenericBufferSystem( const MPI_Comm & communicator, int tag = 0 );
+ GenericBufferSystem( const GenericBufferSystem & other );
+ GenericBufferSystem & operator=( const GenericBufferSystem & other );
+ ~GenericBufferSystem() {}
//@}
//*******************************************************************************************************************
@@ -127,7 +129,7 @@ public:
//@{
template<typename RankIter> void setReceiverInfo( RankIter begin, RankIter end, bool changingSize );
template<typename Range> void setReceiverInfo( const Range & range, bool changingSize );
- void setReceiverInfo( const std::set<MPIRank> & ranksToRecvFrom, bool changingSize );
+ void setReceiverInfo( const std::set<MPIRank> & ranksToRecvFrom, bool changingSize );
void setReceiverInfo( const std::map<MPIRank,MPISize> & ranksToRecvFrom );
void setReceiverInfoFromSendBufferState( bool useSizeFromSendBuffers, bool changingSize );
@@ -142,8 +144,8 @@ public:
//@{
void scheduleReceives() { startCommunication(); }
- SendBuffer & sendBuffer ( MPIRank rank );
- SendBuffer & sendBuffer ( uint_t rank ) { return sendBuffer( int_c( rank ) ); }
+ SendBuffer_T & sendBuffer ( MPIRank rank );
+ SendBuffer_T & sendBuffer ( uint_t rank ) { return sendBuffer( int_c( rank ) ); }
inline size_t size() const;
@@ -162,22 +164,22 @@ public:
class iterator
{
public:
- MPIRank rank() { return currentSenderRank_; }
- RecvBuffer & buffer() { return *currentRecvBuffer_; }
+ MPIRank rank() { return currentSenderRank_; }
+ RecvBuffer_T & buffer() { return *currentRecvBuffer_; }
void operator++();
bool operator==( const iterator & other );
bool operator!=( const iterator & other );
private:
- iterator( BufferSystem & bufferSystem, bool begin );
+ iterator( GenericBufferSystem & bufferSystem, bool begin );
- BufferSystem & bufferSystem_;
+ GenericBufferSystem & bufferSystem_;
- RecvBuffer * currentRecvBuffer_;
- MPIRank currentSenderRank_;
+ RecvBuffer_T * currentRecvBuffer_;
+ MPIRank currentSenderRank_;
- friend class BufferSystem;
+ friend class GenericBufferSystem;
};
friend class iterator;
//@}
@@ -219,17 +221,17 @@ public:
protected:
- friend class OpenMPBufferSystem;
+ friend class GenericOpenMPBufferSystem<RecvBuffer_T, SendBuffer_T>;
void startCommunication();
void endCommunication();
- RecvBuffer * waitForNext( MPIRank & fromRank );
+ RecvBuffer_T * waitForNext( MPIRank & fromRank );
void setCommunicationType( const bool knownSize );
- internal::KnownSizeCommunication knownSizeComm_;
- internal::UnknownSizeCommunication unknownSizeComm_;
- internal::NoMPICommunication noMPIComm_;
- internal::AbstractCommunication * currentComm_; //< after receiver setup, this points to unknown- or knownSizeComm_
+ internal::KnownSizeCommunication<RecvBuffer_T, SendBuffer_T> knownSizeComm_;
+ internal::UnknownSizeCommunication<RecvBuffer_T, SendBuffer_T> unknownSizeComm_;
+ internal::NoMPICommunication<RecvBuffer_T, SendBuffer_T> noMPIComm_;
+ internal::AbstractCommunication<RecvBuffer_T, SendBuffer_T> * currentComm_; //< after receiver setup, this points to unknown- or knownSizeComm_
bool sizeChangesEverytime_; //< if set to true, the receiveSizeUnknown_ is set to true before communicating
bool communicationRunning_; //< indicates if a communication step is currently running
@@ -237,12 +239,12 @@ protected:
/// Info about the message to be received from a certain rank:
/// information holds the buffer and, if known, the message size
- std::map<MPIRank, internal::AbstractCommunication::ReceiveInfo> recvInfos_;
+ std::map<MPIRank, typename internal::AbstractCommunication<RecvBuffer_T, SendBuffer_T>::ReceiveInfo> recvInfos_;
struct SendInfo {
SendInfo() : alreadySent(false) {}
- SendBuffer buffer;
+ SendBuffer_T buffer;
bool alreadySent;
};
std::map<MPIRank, SendInfo> sendInfos_;
@@ -258,55 +260,10 @@ protected:
int64_t numberOfReceives_ = 0; ///< number of communication partners during last receive
};
-
-
-
-
-
-
-//======================================================================================================================
-//
-// Template function definitions
-//
-//======================================================================================================================
-
-
-
-template<typename Range>
-void BufferSystem::setReceiverInfo( const Range & range, bool changingSize )
-{
- setReceiverInfo( range.begin(), range.end(), changingSize );
-}
-
-template<typename RankIter>
-void BufferSystem::setReceiverInfo( RankIter rankBegin, RankIter rankEnd, bool changingSize )
-{
- WALBERLA_ASSERT( ! communicationRunning_ );
-
- recvInfos_.clear();
- for ( auto it = rankBegin; it != rankEnd; ++it )
- {
- const MPIRank sender = *it;
- recvInfos_[ sender ].size = INVALID_SIZE;
- }
-
- sizeChangesEverytime_ = changingSize;
- setCommunicationType( false );
-}
-
-inline size_t BufferSystem::size() const
-{
- size_t sum = 0;
- for( auto iter = sendInfos_.begin(); iter != sendInfos_.end(); ++iter )
- {
- sum += iter->second.buffer.size();
- }
- return sum;
-}
-
-
+typedef GenericBufferSystem<RecvBuffer, SendBuffer> BufferSystem;
} // namespace mpi
} // namespace walberla
+#include "BufferSystem.impl.h"
diff --git a/src/core/mpi/BufferSystem.cpp b/src/core/mpi/BufferSystem.impl.h
similarity index 78%
rename from src/core/mpi/BufferSystem.cpp
rename to src/core/mpi/BufferSystem.impl.h
index a6e64b887be17fe5aaf3a91c160c61a2363b8467..852c7a1fac9a14f490917a4340d4fa180d3525c8 100644
--- a/src/core/mpi/BufferSystem.cpp
+++ b/src/core/mpi/BufferSystem.impl.h
@@ -13,13 +13,12 @@
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
-//! \file BufferSystem.cpp
+//! \file GenericBufferSystem.cpp
//! \ingroup core
//! \author Martin Bauer <martin.bauer@fau.de>
//
//======================================================================================================================
-#include "BufferSystem.h"
#include "core/logging/Logging.h"
#include "core/mpi/MPIManager.h"
#include "core/debug/CheckFunctions.h"
@@ -29,8 +28,9 @@ namespace walberla {
namespace mpi {
+template< typename Rb, typename Sb>
+std::set<int> GenericBufferSystem<Rb, Sb>::activeTags_;
-std::set<int> BufferSystem::activeTags_;
//======================================================================================================================
//
@@ -39,16 +39,16 @@ std::set<int> BufferSystem::activeTags_;
//======================================================================================================================
-
-BufferSystem::iterator::iterator( BufferSystem & bufferSystem, bool begin )
+template< typename Rb, typename Sb>
+GenericBufferSystem<Rb, Sb>::iterator::iterator( GenericBufferSystem<Rb, Sb> & bufferSystem, bool begin )
: bufferSystem_( bufferSystem), currentRecvBuffer_( nullptr ), currentSenderRank_( -1 )
{
if ( begin ) // init iterator
++(*this);
}
-
-void BufferSystem::iterator::operator++()
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::iterator::operator++()
{
currentRecvBuffer_ = bufferSystem_.waitForNext( currentSenderRank_ );
if ( ! currentRecvBuffer_ ) {
@@ -60,7 +60,8 @@ void BufferSystem::iterator::operator++()
}
}
-bool BufferSystem::iterator::operator==( const BufferSystem::iterator & other )
+template< typename Rb, typename Sb>
+bool GenericBufferSystem<Rb, Sb>::iterator::operator==( const typename GenericBufferSystem<Rb, Sb>::iterator & other )
{
// only equality checks with end iterators are allowed
WALBERLA_ASSERT( other.currentSenderRank_ == -1 || currentSenderRank_ == -1 );
@@ -68,7 +69,8 @@ bool BufferSystem::iterator::operator==( const BufferSystem::iterator & other )
return ( currentSenderRank_ == other.currentSenderRank_ );
}
-bool BufferSystem::iterator::operator!=( const BufferSystem::iterator & other )
+template< typename Rb, typename Sb>
+bool GenericBufferSystem<Rb, Sb>::iterator::operator!=( const typename GenericBufferSystem<Rb, Sb>::iterator & other )
{
// only equality checks with end iterators are allowed
WALBERLA_ASSERT( other.currentSenderRank_ == -1 || currentSenderRank_ == -1 );
@@ -77,6 +79,42 @@ bool BufferSystem::iterator::operator!=( const BufferSystem::iterator & other )
}
+template< typename Rb, typename Sb>
+template<typename Range>
+void GenericBufferSystem<Rb, Sb>::setReceiverInfo( const Range & range, bool changingSize )
+{
+ setReceiverInfo( range.begin(), range.end(), changingSize );
+}
+
+template< typename Rb, typename Sb>
+template<typename RankIter>
+void GenericBufferSystem<Rb, Sb>::setReceiverInfo( RankIter rankBegin, RankIter rankEnd, bool changingSize )
+{
+ WALBERLA_ASSERT( ! communicationRunning_ );
+
+ recvInfos_.clear();
+ for ( auto it = rankBegin; it != rankEnd; ++it )
+ {
+ const MPIRank sender = *it;
+ recvInfos_[ sender ].size = INVALID_SIZE;
+ }
+
+ sizeChangesEverytime_ = changingSize;
+ setCommunicationType( false );
+}
+
+template< typename Rb, typename Sb>
+inline size_t GenericBufferSystem<Rb, Sb>::size() const
+{
+ size_t sum = 0;
+ for( auto iter = sendInfos_.begin(); iter != sendInfos_.end(); ++iter )
+ {
+ sum += iter->second.buffer.size();
+ }
+ return sum;
+}
+
+
//======================================================================================================================
@@ -85,8 +123,8 @@ bool BufferSystem::iterator::operator!=( const BufferSystem::iterator & other )
//
//======================================================================================================================
-
-BufferSystem::BufferSystem( const MPI_Comm & communicator, int tag )
+template< typename Rb, typename Sb>
+GenericBufferSystem<Rb, Sb>::GenericBufferSystem( const MPI_Comm & communicator, int tag )
: knownSizeComm_ ( communicator, tag ),
unknownSizeComm_( communicator, tag ),
noMPIComm_( communicator, tag ),
@@ -96,8 +134,8 @@ BufferSystem::BufferSystem( const MPI_Comm & communicator, int tag )
{
}
-
-BufferSystem::BufferSystem( const BufferSystem &other )
+template< typename Rb, typename Sb>
+GenericBufferSystem<Rb, Sb>::GenericBufferSystem( const GenericBufferSystem &other )
: knownSizeComm_ ( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
unknownSizeComm_( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
noMPIComm_ ( other.knownSizeComm_.getCommunicator(), other.knownSizeComm_.getTag() ),
@@ -107,7 +145,7 @@ BufferSystem::BufferSystem( const BufferSystem &other )
recvInfos_( other.recvInfos_ ),
sendInfos_( other.sendInfos_ )
{
- WALBERLA_ASSERT( !communicationRunning_, "Can't copy BufferSystem while communication is running" );
+ WALBERLA_ASSERT( !communicationRunning_, "Can't copy GenericBufferSystem while communication is running" );
if( other.currentComm_ == &other.knownSizeComm_ )
currentComm_ = &knownSizeComm_;
else if ( other.currentComm_ == &other.unknownSizeComm_ )
@@ -118,10 +156,10 @@ BufferSystem::BufferSystem( const BufferSystem &other )
currentComm_ = nullptr; // receiver information not yet set
}
-
-BufferSystem & BufferSystem::operator=( const BufferSystem & other )
+template< typename Rb, typename Sb>
+GenericBufferSystem<Rb, Sb> & GenericBufferSystem<Rb, Sb>::operator=( const GenericBufferSystem<Rb, Sb> & other )
{
- WALBERLA_ASSERT( !communicationRunning_, "Can't copy BufferSystem while communication is running" );
+ WALBERLA_ASSERT( !communicationRunning_, "Can't copy GenericBufferSystem while communication is running" );
sizeChangesEverytime_ = other.sizeChangesEverytime_;
communicationRunning_ = other.communicationRunning_;
@@ -157,7 +195,8 @@ BufferSystem & BufferSystem::operator=( const BufferSystem & other )
* The behavior can be changed later one using setReceiverInfo() or sizeHasChanged().
*/
//**********************************************************************************************************************
-void BufferSystem::setReceiverInfo( const std::set<MPIRank> & ranksToRecvFrom, bool changingSize )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::setReceiverInfo( const std::set<MPIRank> & ranksToRecvFrom, bool changingSize )
{
WALBERLA_ASSERT( ! communicationRunning_ );
@@ -183,7 +222,8 @@ void BufferSystem::setReceiverInfo( const std::set<MPIRank> & ranksToRecvFrom, b
* behavior is changed with setReceiverInfo*() or sizeHasChanged()
*/
//**********************************************************************************************************************
-void BufferSystem::setReceiverInfo( const std::map<MPIRank,MPISize> & ranksToRecvFrom )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::setReceiverInfo( const std::map<MPIRank,MPISize> & ranksToRecvFrom )
{
WALBERLA_ASSERT( ! communicationRunning_ );
@@ -205,7 +245,7 @@ void BufferSystem::setReceiverInfo( const std::map<MPIRank,MPISize> & ranksToRec
//**********************************************************************************************************************
/*! Sets receiver information, using SendBuffers (symmetric communication)
*
-* Gives the BufferSystem the information that messages are received from the same processes that we
+* Gives the GenericBufferSystem the information that messages are received from the same processes that we
* send to (i.e. from all ranks where SendBuffers were already filled )
* sendBuffer() has to be called before, and corresponding SendBuffers have to be filled.
*
@@ -220,7 +260,8 @@ void BufferSystem::setReceiverInfo( const std::map<MPIRank,MPISize> & ranksToRec
* communicated in the first step but in all following steps.
*/
//**********************************************************************************************************************
-void BufferSystem::setReceiverInfoFromSendBufferState( bool useSizeFromSendBuffers, bool changingSize )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::setReceiverInfoFromSendBufferState( bool useSizeFromSendBuffers, bool changingSize )
{
WALBERLA_ASSERT( ! communicationRunning_ );
@@ -228,7 +269,7 @@ void BufferSystem::setReceiverInfoFromSendBufferState( bool useSizeFromSendBuffe
for ( auto it = sendInfos_.begin(); it != sendInfos_.end(); ++it )
{
const MPIRank sender = it->first;
- const SendBuffer & buffer = it->second.buffer;
+ const Sb & buffer = it->second.buffer;
if ( buffer.size() == 0 && useSizeFromSendBuffers )
continue;
@@ -244,16 +285,17 @@ void BufferSystem::setReceiverInfoFromSendBufferState( bool useSizeFromSendBuffe
//**********************************************************************************************************************
-/*! Notifies that BufferSystem that message sizes have changed ( and optionally are changing in all following steps)
+/*! Notifies that GenericBufferSystem that message sizes have changed ( and optionally are changing in all following steps)
*
-* Useful when setReceiverInfo was set such that BufferSystem assumes constant message sizes for all steps.
+* Useful when setReceiverInfo was set such that GenericBufferSystem assumes constant message sizes for all steps.
* Can only be called if no communication is currently running.
*
* \param alwaysChangingSize if true the message sizes is communicated in all following steps, if false
* only in the next step.
*/
//**********************************************************************************************************************
-void BufferSystem::sizeHasChanged( bool alwaysChangingSize )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::sizeHasChanged( bool alwaysChangingSize )
{
WALBERLA_ASSERT( ! communicationRunning_ );
@@ -277,7 +319,8 @@ void BufferSystem::sizeHasChanged( bool alwaysChangingSize )
* \param rank the rank where the buffer should be sent to
*/
//**********************************************************************************************************************
-SendBuffer & BufferSystem::sendBuffer( MPIRank rank )
+template< typename Rb, typename Sb>
+Sb & GenericBufferSystem<Rb, Sb>::sendBuffer( MPIRank rank )
{
return sendInfos_[rank].buffer;
}
@@ -295,7 +338,8 @@ SendBuffer & BufferSystem::sendBuffer( MPIRank rank )
* If communication was not started before, it is started with this function.
*/
//**********************************************************************************************************************
-void BufferSystem::sendAll()
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::sendAll()
{
WALBERLA_ASSERT_NOT_NULLPTR( currentComm_ ); // call setReceiverInfo first!
@@ -327,7 +371,8 @@ void BufferSystem::sendAll()
* If communication was not started before, it is started with this function.
*/
//**********************************************************************************************************************
-void BufferSystem::send( MPIRank rank )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::send( MPIRank rank )
{
WALBERLA_ASSERT_NOT_NULLPTR( currentComm_ ); // call setReceiverInfo first!
@@ -363,7 +408,8 @@ void BufferSystem::send( MPIRank rank )
* - schedules receives and reserves space for MPI_Request vectors in the currentComm_ member
*/
//**********************************************************************************************************************
-void BufferSystem::startCommunication()
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::startCommunication()
{
const auto tag = currentComm_->getTag();
WALBERLA_CHECK_EQUAL(activeTags_.find(tag), activeTags_.end(),
@@ -392,7 +438,8 @@ void BufferSystem::startCommunication()
* - manage sizeChangesEverytime
*/
//**********************************************************************************************************************
-void BufferSystem::endCommunication()
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::endCommunication()
{
WALBERLA_CHECK( communicationRunning_ );
currentComm_->waitForSends();
@@ -426,7 +473,8 @@ void BufferSystem::endCommunication()
* See documentation of AbstractCommunication::waitForNext()
*/
//**********************************************************************************************************************
-RecvBuffer * BufferSystem::waitForNext( MPIRank & fromRank )
+template< typename Rb, typename Sb>
+Rb * GenericBufferSystem<Rb, Sb>::waitForNext( MPIRank & fromRank )
{
WALBERLA_ASSERT( communicationRunning_ );
@@ -448,7 +496,8 @@ RecvBuffer * BufferSystem::waitForNext( MPIRank & fromRank )
/*! Sets the communication type to known size, unknown size or NoMPI comm
*/
//**********************************************************************************************************************
-void BufferSystem::setCommunicationType( const bool knownSize )
+template< typename Rb, typename Sb>
+void GenericBufferSystem<Rb, Sb>::setCommunicationType( const bool knownSize )
{
WALBERLA_NON_MPI_SECTION()
{
@@ -473,36 +522,36 @@ void BufferSystem::setCommunicationType( const bool knownSize )
// using boost::counting_range didn't work on all supported compilers
// so the range is created explicitly
-
-BufferSystem::RankRange BufferSystem::noRanks()
+template< typename Rb, typename Sb>
+typename GenericBufferSystem<Rb, Sb>::RankRange GenericBufferSystem<Rb,Sb>::noRanks()
{
return RankRange ( RankCountIter( 0 ),
RankCountIter( 0 ) );
}
-
-BufferSystem::RankRange BufferSystem::allRanks()
+template< typename Rb, typename Sb>
+typename GenericBufferSystem<Rb, Sb>::RankRange GenericBufferSystem<Rb,Sb>::allRanks()
{
int numProcesses = MPIManager::instance()->numProcesses();
return RankRange ( RankCountIter( 0 ),
RankCountIter( numProcesses ) );
}
-
-BufferSystem::RankRange BufferSystem::allRanksButRoot()
+template< typename Rb, typename Sb>
+typename GenericBufferSystem<Rb, Sb>::RankRange GenericBufferSystem<Rb,Sb>::allRanksButRoot()
{
int numProcesses = MPIManager::instance()->numProcesses();
return RankRange ( RankCountIter( 1 ),
RankCountIter( numProcesses ) );
}
-
-BufferSystem::RankRange BufferSystem::onlyRank( int includedRank )
+template< typename Rb, typename Sb>
+typename GenericBufferSystem<Rb, Sb>::RankRange GenericBufferSystem<Rb,Sb>::onlyRank( int includedRank )
{
WALBERLA_ASSERT_LESS( includedRank, MPIManager::instance()->numProcesses() );
return RankRange ( RankCountIter( includedRank ),
RankCountIter( includedRank+1 ) );
}
-
-BufferSystem::RankRange BufferSystem::onlyRoot()
+template< typename Rb, typename Sb>
+typename GenericBufferSystem<Rb, Sb>::RankRange GenericBufferSystem<Rb,Sb>::onlyRoot()
{
return RankRange ( RankCountIter( 0 ),
RankCountIter( 1 ) );
diff --git a/src/core/mpi/BufferSystemHelper.h b/src/core/mpi/BufferSystemHelper.h
index b8e7a29286eb786ad7d738bb3bbc72c5955e77eb..e143f6b10fba7ad99d4d687750a14b40f2123ce1 100644
--- a/src/core/mpi/BufferSystemHelper.h
+++ b/src/core/mpi/BufferSystemHelper.h
@@ -35,6 +35,7 @@ namespace walberla {
namespace mpi {
namespace internal {
+ template< typename RecvBuffer_T, typename SendBuffer_T>
class AbstractCommunication
{
public:
@@ -46,7 +47,7 @@ namespace internal {
struct ReceiveInfo {
- RecvBuffer buffer;
+ RecvBuffer_T buffer;
MPISize size;
};
@@ -54,7 +55,7 @@ namespace internal {
/*************************************************************************************************************//**
* Send buffer content to receiver using MPI_ISend, request is stored internally -> see waitForSends()
*****************************************************************************************************************/
- virtual void send( MPIRank rank, const SendBuffer & sendBuffer ) = 0;
+ virtual void send( MPIRank rank, const SendBuffer_T & sendBuffer ) = 0;
/*************************************************************************************************************//**
@@ -100,16 +101,18 @@ namespace internal {
};
-
- class KnownSizeCommunication : public AbstractCommunication
+ template< typename RecvBuffer_T, typename SendBuffer_T>
+ class KnownSizeCommunication : public AbstractCommunication<RecvBuffer_T, SendBuffer_T>
{
public:
+ using typename AbstractCommunication<RecvBuffer_T, SendBuffer_T>::ReceiveInfo;
+
KnownSizeCommunication( const MPI_Comm & communicator, int tag = 0 )
- : AbstractCommunication( communicator, tag ), sending_(false), receiving_(false) {}
+ : AbstractCommunication<RecvBuffer_T, SendBuffer_T>( communicator, tag ), sending_(false), receiving_(false) {}
virtual ~KnownSizeCommunication() {}
- virtual void send( MPIRank receiver, const SendBuffer & sendBuffer );
+ virtual void send( MPIRank receiver, const SendBuffer_T & sendBuffer );
virtual void waitForSends();
virtual void scheduleReceives ( std::map<MPIRank, ReceiveInfo> & recvInfos );
@@ -126,16 +129,18 @@ namespace internal {
};
-
- class UnknownSizeCommunication : public AbstractCommunication
+ template< typename RecvBuffer_T, typename SendBuffer_T>
+ class UnknownSizeCommunication : public AbstractCommunication<RecvBuffer_T, SendBuffer_T>
{
public:
+ using typename AbstractCommunication<RecvBuffer_T, SendBuffer_T>::ReceiveInfo;
+
UnknownSizeCommunication( const MPI_Comm & communicator, int tag = 0 )
- : AbstractCommunication( communicator, tag ), sending_(false), receiving_(false) {}
+ : AbstractCommunication<RecvBuffer_T, SendBuffer_T>( communicator, tag ), sending_(false), receiving_(false) {}
virtual ~UnknownSizeCommunication() {}
- virtual void send( MPIRank receiver, const SendBuffer & sendBuffer );
+ virtual void send( MPIRank receiver, const SendBuffer_T & sendBuffer );
virtual void waitForSends();
virtual void scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos );
@@ -156,16 +161,18 @@ namespace internal {
};
-
- class NoMPICommunication : public AbstractCommunication
+ template< typename RecvBuffer_T, typename SendBuffer_T>
+ class NoMPICommunication : public AbstractCommunication<RecvBuffer_T, SendBuffer_T>
{
public:
+ using typename AbstractCommunication<RecvBuffer_T, SendBuffer_T>::ReceiveInfo;
+
NoMPICommunication( const MPI_Comm & communicator, int tag = 0 )
- : AbstractCommunication( communicator, tag ), received_( false ) {}
+ : AbstractCommunication<RecvBuffer_T, SendBuffer_T>( communicator, tag ), received_( false ) {}
virtual ~NoMPICommunication() {}
- virtual void send( MPIRank receiver, const SendBuffer & sendBuffer );
+ virtual void send( MPIRank receiver, const SendBuffer_T & sendBuffer );
virtual void waitForSends();
virtual void scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos );
@@ -174,8 +181,8 @@ namespace internal {
virtual MPIRank waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos );
private:
- bool received_;
- RecvBuffer tmpBuffer_;
+ bool received_;
+ RecvBuffer_T tmpBuffer_;
};
@@ -184,4 +191,4 @@ namespace internal {
} // namespace mpi
} // namespace walberla
-
+#include "BufferSystemHelper.impl.h"
diff --git a/src/core/mpi/BufferSystemHelper.cpp b/src/core/mpi/BufferSystemHelper.impl.h
similarity index 83%
rename from src/core/mpi/BufferSystemHelper.cpp
rename to src/core/mpi/BufferSystemHelper.impl.h
index 8f8e510dcb5e85220b60c56101e27b0c897f35ca..f982aab6894e9aff58579f8875598caa11180cee 100644
--- a/src/core/mpi/BufferSystemHelper.cpp
+++ b/src/core/mpi/BufferSystemHelper.impl.h
@@ -37,8 +37,8 @@ namespace internal {
//
//======================================================================================================================
-
-void KnownSizeCommunication::send( MPIRank receiver, const SendBuffer & sendBuffer )
+template< typename Rb, typename Sb>
+void KnownSizeCommunication<Rb, Sb>::send( MPIRank receiver, const Sb & sendBuffer )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -51,15 +51,15 @@ void KnownSizeCommunication::send( MPIRank receiver, const SendBuffer & sendBuff
int_c(sendBuffer.size() ), // send one size
MPI_BYTE, // type
receiver, // receiver rank
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
&request // request needed for wait
);
}
-
-void KnownSizeCommunication::waitForSends( )
+template< typename Rb, typename Sb>
+void KnownSizeCommunication<Rb, Sb>::waitForSends( )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -76,8 +76,8 @@ void KnownSizeCommunication::waitForSends( )
}
-
-void KnownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+void KnownSizeCommunication<Rb, Sb>::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -101,8 +101,8 @@ void KnownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> &
recvInfo.size, // size of expected message
MPI_BYTE, // type
senderRank, // rank of sender process
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
& recvRequests_[recvCount] // request, needed for wait
);
@@ -115,7 +115,8 @@ void KnownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> &
}
-MPIRank KnownSizeCommunication::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+MPIRank KnownSizeCommunication<Rb, Sb>::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -170,8 +171,8 @@ MPIRank KnownSizeCommunication::waitForNextReceive( std::map<MPIRank, ReceiveInf
//
//======================================================================================================================
-
-void UnknownSizeCommunication::send( MPIRank receiver, const SendBuffer & sendBuffer )
+template< typename Rb, typename Sb>
+void UnknownSizeCommunication<Rb, Sb>::send( MPIRank receiver, const Sb & sendBuffer )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -190,8 +191,8 @@ void UnknownSizeCommunication::send( MPIRank receiver, const SendBuffer & sendBu
1, // one integer is sent
MPITrait<MPISize>::type(), // type
receiver, // receiver rank
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
& sizeMsgReq // request needed for wait
);
@@ -205,13 +206,14 @@ void UnknownSizeCommunication::send( MPIRank receiver, const SendBuffer & sendBu
int_c( sendBuffer.size() ), // send one size
MPI_BYTE, // type
receiver, // receiver rank
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
& contentMsgReq // request needed for wait
);
}
-void UnknownSizeCommunication::waitForSends()
+template< typename Rb, typename Sb>
+void UnknownSizeCommunication<Rb, Sb>::waitForSends()
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -228,7 +230,8 @@ void UnknownSizeCommunication::waitForSends()
outgoingBufferForSizes_.clear();
}
-void UnknownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+void UnknownSizeCommunication<Rb, Sb>::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -250,8 +253,8 @@ void UnknownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo>
1, // size of expected message
MPITrait<MPISize>::type(), // type
sender, // rank of sender process
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
& recvRequests_[recvCount] // request, needed for wait
);
@@ -261,7 +264,8 @@ void UnknownSizeCommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo>
}
-MPIRank UnknownSizeCommunication::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+MPIRank UnknownSizeCommunication<Rb, Sb>::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
WALBERLA_NON_MPI_SECTION() { WALBERLA_ASSERT( false ); }
@@ -322,8 +326,8 @@ MPIRank UnknownSizeCommunication::waitForNextReceive( std::map<MPIRank, ReceiveI
recvInfo.size, // size of expected message
MPI_BYTE, // type
senderRank, // rank of sender process
- tag_, // message tag
- communicator_, // communicator
+ this->tag_, // message tag
+ this->communicator_, // communicator
& recvRequests_[ uint_c( requestIndex ) ] // request, needed for wait
);
@@ -351,8 +355,8 @@ MPIRank UnknownSizeCommunication::waitForNextReceive( std::map<MPIRank, ReceiveI
//
//======================================================================================================================
-
-void NoMPICommunication::send( MPIRank receiver, const SendBuffer & sendBuffer )
+template< typename Rb, typename Sb>
+void NoMPICommunication<Rb, Sb>::send( MPIRank receiver, const Sb & sendBuffer )
{
WALBERLA_UNUSED( receiver);
@@ -361,12 +365,14 @@ void NoMPICommunication::send( MPIRank receiver, const SendBuffer & sendBuffer )
received_ = false;
}
-void NoMPICommunication::waitForSends()
+template< typename Rb, typename Sb>
+void NoMPICommunication<Rb, Sb>::waitForSends()
{
return;
}
-void NoMPICommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+void NoMPICommunication<Rb, Sb>::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
WALBERLA_DEBUG_SECTION()
{
@@ -380,7 +386,8 @@ void NoMPICommunication::scheduleReceives( std::map<MPIRank, ReceiveInfo> & recv
}
-MPIRank NoMPICommunication::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
+template< typename Rb, typename Sb>
+MPIRank NoMPICommunication<Rb, Sb>::waitForNextReceive( std::map<MPIRank, ReceiveInfo> & recvInfos )
{
if( recvInfos.empty() )
{
diff --git a/src/core/mpi/OpenMPBufferSystem.h b/src/core/mpi/OpenMPBufferSystem.h
index 3a41f36f0615a1aa2cd244f494438faee1d740a9..232f54f42a6dbe4037c4b2c8f2d7f02b98773bcb 100644
--- a/src/core/mpi/OpenMPBufferSystem.h
+++ b/src/core/mpi/OpenMPBufferSystem.h
@@ -44,12 +44,13 @@ namespace mpi {
* for the systems: the tag can be passed in the constructor.
*/
//**********************************************************************************************************************
-class OpenMPBufferSystem
+template< typename RecvBuffer_T, typename SendBuffer_T>
+class GenericOpenMPBufferSystem
{
public:
- OpenMPBufferSystem( const MPI_Comm & communicator, int tag = 0,
- bool _serialSends = false, bool _serialRecvs = false );
+ GenericOpenMPBufferSystem( const MPI_Comm & communicator, int tag = 0,
+ bool _serialSends = false, bool _serialRecvs = false );
void enforceSerialSends( bool val ) { serialSends_ = val; }
@@ -60,8 +61,8 @@ public:
bool isSizeCommunicatedInNextStep() const { return bs_.isSizeCommunicatedInNextStep(); }
- void addReceivingFunction( MPIRank rank, const std::function<void ( RecvBuffer & buf ) >& recvFunction );
- void addSendingFunction ( MPIRank rank, const std::function<void ( SendBuffer & buf ) >& sendFunction );
+ void addReceivingFunction( MPIRank rank, const std::function<void ( RecvBuffer_T & buf ) >& recvFunction );
+ void addSendingFunction ( MPIRank rank, const std::function<void ( SendBuffer_T & buf ) >& sendFunction );
void clearReceivingFunctions() { dirty_ = true; recvFunctions_.clear(); }
void clearSendingFunctions() { dirty_ = true; sendFunctions_.clear(); sendRanks_.clear(); }
@@ -75,7 +76,7 @@ public:
private:
- BufferSystem bs_;
+ GenericBufferSystem<RecvBuffer_T, SendBuffer_T> bs_;
bool dirty_;
@@ -86,11 +87,11 @@ private:
bool sizeChangesEverytime_;
- std::map<MPIRank, std::function<void ( RecvBuffer & )> > recvFunctions_;
+ std::map<MPIRank, std::function<void ( RecvBuffer_T & )> > recvFunctions_;
std::vector<MPIRank> sendRanks_;
- std::vector< std::function<void ( SendBuffer & )> > sendFunctions_;
+ std::vector< std::function<void ( SendBuffer_T & )> > sendFunctions_;
void startCommunicationOpenMP();
void startCommunicationSerial();
@@ -99,7 +100,10 @@ private:
void waitSerial();
};
+typedef GenericOpenMPBufferSystem<RecvBuffer, SendBuffer> OpenMPBufferSystem;
} // namespace mpi
} // namespace walberla
+
+#include "OpenMPBufferSystem.impl.h"
\ No newline at end of file
diff --git a/src/core/mpi/OpenMPBufferSystem.cpp b/src/core/mpi/OpenMPBufferSystem.impl.h
similarity index 81%
rename from src/core/mpi/OpenMPBufferSystem.cpp
rename to src/core/mpi/OpenMPBufferSystem.impl.h
index 45a614eb42b87451d40418ef59e46a7b79f1815e..c79b961ba0b0deaf0a8af506fc508e80715900cb 100644
--- a/src/core/mpi/OpenMPBufferSystem.cpp
+++ b/src/core/mpi/OpenMPBufferSystem.impl.h
@@ -19,8 +19,6 @@
//
//======================================================================================================================
-#include "OpenMPBufferSystem.h"
-
#include <boost/range/adaptor/map.hpp>
@@ -35,9 +33,9 @@ namespace mpi {
//
//======================================================================================================================
-
-OpenMPBufferSystem::OpenMPBufferSystem( const MPI_Comm & communicator, int tag,
- bool _serialSends, bool _serialRecvs )
+template<typename Rb, typename Sb>
+GenericOpenMPBufferSystem<Rb, Sb>::GenericOpenMPBufferSystem( const MPI_Comm & communicator, int tag,
+ bool _serialSends, bool _serialRecvs )
: bs_( communicator, tag),
dirty_( true ),
serialSends_( _serialSends ),
@@ -47,14 +45,16 @@ OpenMPBufferSystem::OpenMPBufferSystem( const MPI_Comm & communicator, int tag,
}
-void OpenMPBufferSystem::addReceivingFunction( MPIRank rank, const std::function<void ( RecvBuffer & buf ) >& recvFunction )
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::addReceivingFunction( MPIRank rank, const std::function<void ( Rb & buf ) >& recvFunction )
{
dirty_ = true;
recvFunctions_[rank] = recvFunction;
}
-void OpenMPBufferSystem::addSendingFunction ( MPIRank rank, const std::function<void ( SendBuffer & buf ) >& sendFunction )
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::addSendingFunction( MPIRank rank, const std::function<void ( Sb & buf ) >& sendFunction )
{
dirty_ = true;
sendRanks_.push_back( rank );
@@ -63,7 +63,8 @@ void OpenMPBufferSystem::addSendingFunction ( MPIRank rank, const std::function
}
-void OpenMPBufferSystem::setupBufferSystem()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::setupBufferSystem()
{
if ( ! dirty_ )
return;
@@ -85,7 +86,8 @@ void OpenMPBufferSystem::setupBufferSystem()
//======================================================================================================================
-void OpenMPBufferSystem::startCommunication()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::startCommunication()
{
setupBufferSystem();
if( serialSends_ )
@@ -94,7 +96,8 @@ void OpenMPBufferSystem::startCommunication()
startCommunicationOpenMP();
}
-void OpenMPBufferSystem::startCommunicationSerial()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::startCommunicationSerial()
{
bs_.scheduleReceives();
@@ -114,7 +117,8 @@ void OpenMPBufferSystem::startCommunicationSerial()
bs_.sendAll(); // for the case where sendFunctions_ is empty
}
-void OpenMPBufferSystem::startCommunicationOpenMP()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::startCommunicationOpenMP()
{
bs_.scheduleReceives();
@@ -150,8 +154,8 @@ void OpenMPBufferSystem::startCommunicationOpenMP()
//
//======================================================================================================================
-
-void OpenMPBufferSystem::wait()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::wait()
{
if ( serialRecvs_ )
waitSerial();
@@ -160,7 +164,8 @@ void OpenMPBufferSystem::wait()
}
-void OpenMPBufferSystem::waitSerial()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::waitSerial()
{
for( auto recvIt = bs_.begin(); recvIt != bs_.end(); ++recvIt )
{
@@ -174,7 +179,8 @@ void OpenMPBufferSystem::waitSerial()
}
-void OpenMPBufferSystem::waitOpenMP()
+template<typename Rb, typename Sb>
+void GenericOpenMPBufferSystem<Rb, Sb>::waitOpenMP()
{
const int numReceives = int_c( bs_.recvInfos_.size() );