From 78d563f185d38e98ac0081f12aeb169bdf9afef2 Mon Sep 17 00:00:00 2001
From: Sebastian Eibl <sebastian.eibl@fau.de>
Date: Mon, 20 Jul 2020 14:17:59 +0200
Subject: [PATCH] [BUGFIX] caching of buffers can lead to dead locks
If load balancing changes the neighborhood, caching of receive
buffers can lead to dead locks.
---
.../templates/mpi/BroadcastProperty.templ.h | 22 ++++++++++++++-----
.../templates/mpi/ReduceProperty.templ.h | 22 ++++++++++++++-----
.../templates/mpi/SyncGhostOwners.templ.cpp | 19 ++++++++++++++++
.../templates/mpi/SyncGhostOwners.templ.h | 16 ++++++++------
.../templates/mpi/SyncNextNeighbors.templ.cpp | 12 ++++++++--
.../templates/mpi/SyncNextNeighbors.templ.h | 18 +++++++++------
.../mpi/SyncNextNeighborsNoGhosts.templ.cpp | 12 ++++++++--
.../mpi/SyncNextNeighborsNoGhosts.templ.h | 18 +++++++++------
src/mesa_pd/common/ParticleFunctions.h | 6 +++++
src/mesa_pd/mpi/BroadcastProperty.h | 22 ++++++++++++++-----
src/mesa_pd/mpi/ReduceProperty.h | 22 ++++++++++++++-----
src/mesa_pd/mpi/SyncGhostOwners.cpp | 19 ++++++++++++++++
src/mesa_pd/mpi/SyncGhostOwners.h | 16 ++++++++------
src/mesa_pd/mpi/SyncNextNeighbors.cpp | 12 ++++++++--
src/mesa_pd/mpi/SyncNextNeighbors.h | 18 +++++++++------
.../mpi/SyncNextNeighborsBlockForest.cpp | 12 ++++++++--
.../mpi/SyncNextNeighborsBlockForest.h | 18 +++++++++------
src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.cpp | 12 ++++++++--
src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.h | 18 +++++++++------
19 files changed, 231 insertions(+), 83 deletions(-)
diff --git a/python/mesa_pd/templates/mpi/BroadcastProperty.templ.h b/python/mesa_pd/templates/mpi/BroadcastProperty.templ.h
index 2279df7d1..236c0a65c 100644
--- a/python/mesa_pd/templates/mpi/BroadcastProperty.templ.h
+++ b/python/mesa_pd/templates/mpi/BroadcastProperty.templ.h
@@ -59,15 +59,18 @@ public:
template <typename Notification>
void operator()(data::ParticleStorage& ps) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem(walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
template <typename Notification>
@@ -75,6 +78,8 @@ void BroadcastProperty::operator()(data::ParticleStorage& ps) const
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs(walberla::mpi::MPIManager::instance()->comm());
+
std::set<int> recvRanks; // potential message senders
WALBERLA_LOG_DETAIL( "Assembling of property reduction message starts...");
@@ -128,6 +133,11 @@ void BroadcastProperty::operator()(data::ParticleStorage& ps) const
}
}
WALBERLA_LOG_DETAIL( "Parsing of property broadcasting message ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
} // namespace mpi
diff --git a/python/mesa_pd/templates/mpi/ReduceProperty.templ.h b/python/mesa_pd/templates/mpi/ReduceProperty.templ.h
index 38f734085..60abb8985 100644
--- a/python/mesa_pd/templates/mpi/ReduceProperty.templ.h
+++ b/python/mesa_pd/templates/mpi/ReduceProperty.templ.h
@@ -68,15 +68,18 @@ public:
template <typename Notification>
void operator()(data::ParticleStorage& ps) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem(walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
template <typename Notification>
@@ -84,6 +87,8 @@ void ReduceProperty::operator()(data::ParticleStorage& ps) const
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs(walberla::mpi::MPIManager::instance()->comm());
+
std::set<int> recvRanks; // potential message senders
WALBERLA_LOG_DETAIL( "Assembling of property reduction message starts...");
@@ -139,6 +144,11 @@ void ReduceProperty::operator()(data::ParticleStorage& ps) const
}
}
WALBERLA_LOG_DETAIL( "Parsing of property reduction message ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
} // namespace mpi
diff --git a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp
index ce0475880..1dca2429d 100644
--- a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp
+++ b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp
@@ -39,6 +39,11 @@ void SyncGhostOwners::operator()( data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ bytesSent_ = 0;
+ bytesReceived_ = 0;
+ numberOfSends_ = 0;
+ numberOfReceives_ = 0;
+
//==========================================================
// STEP1: Update & Migrate
//==========================================================
@@ -59,6 +64,8 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
// STEP1: Update & Migrate
//==========================================================
+ walberla::mpi::BufferSystem bs1( walberla::mpi::MPIManager::instance()->comm(), 749861);
+
WALBERLA_CHECK(!bs1.isCommunicationRunning());
WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate starts..." );
@@ -192,6 +199,11 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate ended." );
+
+ bytesSent_ += bs1.getBytesSent();
+ bytesReceived_ += bs1.getBytesReceived();
+ numberOfSends_ += bs1.getNumberOfSends();
+ numberOfReceives_ += bs1.getNumberOfReceives();
}
void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
@@ -204,6 +216,8 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
// STEP2: Check&Resolve
//==========================================================
+ walberla::mpi::BufferSystem bs2( walberla::mpi::MPIManager::instance()->comm(), 255367);
+
WALBERLA_CHECK(!bs2.isCommunicationRunning());
//init buffers
@@ -350,6 +364,11 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve ended." );
+
+ bytesSent_ += bs2.getBytesSent();
+ bytesReceived_ += bs2.getBytesReceived();
+ numberOfSends_ += bs2.getNumberOfSends();
+ numberOfReceives_ += bs2.getNumberOfReceives();
}
} // namespace mpi
diff --git a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h
index 8ec931a68..0be135507 100644
--- a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h
+++ b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h
@@ -60,11 +60,11 @@ public:
const real_t dx = real_t(0),
const bool syncNonCommunicatingBodies = false ) const;
- int64_t getBytesSent() const { return bs1.getBytesSent() + bs2.getBytesSent(); }
- int64_t getBytesReceived() const { return bs1.getBytesReceived() + bs2.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs1.getNumberOfSends() + bs2.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs1.getNumberOfReceives() + bs2.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
void updateAndMigrate( data::ParticleStorage& ps,
const domain::IDomain& domain,
@@ -77,11 +77,13 @@ private:
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs1 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 749861);
- mutable walberla::mpi::BufferSystem bs2 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 255367);
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp
index dcf86e8be..f631bb0e7 100644
--- a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp
+++ b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp
@@ -38,6 +38,8 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs( walberla::mpi::MPIManager::instance()->comm() );
+
neighborRanks_ = domain.getNeighborProcesses();
for( uint_t nbProcessRank : neighborRanks_ )
{
@@ -47,7 +49,7 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
bs.sendBuffer(nbProcessRank) << walberla::uint8_c(0);
}
}
- generateSynchronizationMessages(ps, domain, dx);
+ generateSynchronizationMessages(bs, ps, domain, dx);
// size of buffer is unknown and changes with each send
bs.setReceiverInfoFromSendBufferState(false, true);
@@ -66,9 +68,15 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
-void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& ps,
+void SyncNextNeighbors::generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx) const
{
diff --git a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.h b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.h
index 5c9402d4c..7a9dc110b 100644
--- a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.h
+++ b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.h
@@ -57,21 +57,25 @@ public:
const domain::IDomain& domain,
const real_t dx = real_t(0)) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- void generateSynchronizationMessages(data::ParticleStorage& ps,
+ void generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx) const;
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.cpp b/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.cpp
index bab7acfcc..ce8c9100d 100644
--- a/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.cpp
+++ b/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.cpp
@@ -37,6 +37,8 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs( walberla::mpi::MPIManager::instance()->comm() );
+
neighborRanks_ = domain.getNeighborProcesses();
for( uint_t nbProcessRank : neighborRanks_ )
{
@@ -46,7 +48,7 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
bs.sendBuffer(nbProcessRank) << walberla::uint8_c(0);
}
}
- generateSynchronizationMessages(ps, domain);
+ generateSynchronizationMessages(bs, ps, domain);
// size of buffer is unknown and changes with each send
bs.setReceiverInfoFromSendBufferState(false, true);
@@ -65,9 +67,15 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
-void SyncNextNeighborsNoGhosts::generateSynchronizationMessages(data::ParticleStorage& ps,
+void SyncNextNeighborsNoGhosts::generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain) const
{
const uint_t ownRank = uint_c(rank_);
diff --git a/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.h b/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.h
index 7ffc5f2b7..8030c09e7 100644
--- a/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.h
+++ b/python/mesa_pd/templates/mpi/SyncNextNeighborsNoGhosts.templ.h
@@ -60,20 +60,24 @@ public:
void operator()(data::ParticleStorage& ps,
const domain::IDomain& domain) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- void generateSynchronizationMessages(data::ParticleStorage& ps,
+ void generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain) const;
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/src/mesa_pd/common/ParticleFunctions.h b/src/mesa_pd/common/ParticleFunctions.h
index 462e3f2e8..b546b36a7 100644
--- a/src/mesa_pd/common/ParticleFunctions.h
+++ b/src/mesa_pd/common/ParticleFunctions.h
@@ -18,6 +18,12 @@
//
//======================================================================================================================
+//======================================================================================================================
+//
+// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
+//
+//======================================================================================================================
+
#pragma once
#include <mesa_pd/data/DataTypes.h>
diff --git a/src/mesa_pd/mpi/BroadcastProperty.h b/src/mesa_pd/mpi/BroadcastProperty.h
index 856e030ec..27c4542cf 100644
--- a/src/mesa_pd/mpi/BroadcastProperty.h
+++ b/src/mesa_pd/mpi/BroadcastProperty.h
@@ -59,15 +59,18 @@ public:
template <typename Notification>
void operator()(data::ParticleStorage& ps) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem(walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
template <typename Notification>
@@ -75,6 +78,8 @@ void BroadcastProperty::operator()(data::ParticleStorage& ps) const
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs(walberla::mpi::MPIManager::instance()->comm());
+
std::set<int> recvRanks; // potential message senders
WALBERLA_LOG_DETAIL( "Assembling of property reduction message starts...");
@@ -128,6 +133,11 @@ void BroadcastProperty::operator()(data::ParticleStorage& ps) const
}
}
WALBERLA_LOG_DETAIL( "Parsing of property broadcasting message ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
} // namespace mpi
diff --git a/src/mesa_pd/mpi/ReduceProperty.h b/src/mesa_pd/mpi/ReduceProperty.h
index 7f55c3349..46736d9ed 100644
--- a/src/mesa_pd/mpi/ReduceProperty.h
+++ b/src/mesa_pd/mpi/ReduceProperty.h
@@ -68,15 +68,18 @@ public:
template <typename Notification>
void operator()(data::ParticleStorage& ps) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem(walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
template <typename Notification>
@@ -84,6 +87,8 @@ void ReduceProperty::operator()(data::ParticleStorage& ps) const
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs(walberla::mpi::MPIManager::instance()->comm());
+
std::set<int> recvRanks; // potential message senders
WALBERLA_LOG_DETAIL( "Assembling of property reduction message starts...");
@@ -139,6 +144,11 @@ void ReduceProperty::operator()(data::ParticleStorage& ps) const
}
}
WALBERLA_LOG_DETAIL( "Parsing of property reduction message ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
} // namespace mpi
diff --git a/src/mesa_pd/mpi/SyncGhostOwners.cpp b/src/mesa_pd/mpi/SyncGhostOwners.cpp
index 7f115fdb2..dd13a9fb2 100644
--- a/src/mesa_pd/mpi/SyncGhostOwners.cpp
+++ b/src/mesa_pd/mpi/SyncGhostOwners.cpp
@@ -39,6 +39,11 @@ void SyncGhostOwners::operator()( data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ bytesSent_ = 0;
+ bytesReceived_ = 0;
+ numberOfSends_ = 0;
+ numberOfReceives_ = 0;
+
//==========================================================
// STEP1: Update & Migrate
//==========================================================
@@ -59,6 +64,8 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
// STEP1: Update & Migrate
//==========================================================
+ walberla::mpi::BufferSystem bs1( walberla::mpi::MPIManager::instance()->comm(), 749861);
+
WALBERLA_CHECK(!bs1.isCommunicationRunning());
WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate starts..." );
@@ -192,6 +199,11 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate ended." );
+
+ bytesSent_ += bs1.getBytesSent();
+ bytesReceived_ += bs1.getBytesReceived();
+ numberOfSends_ += bs1.getNumberOfSends();
+ numberOfReceives_ += bs1.getNumberOfReceives();
}
void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
@@ -204,6 +216,8 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
// STEP2: Check&Resolve
//==========================================================
+ walberla::mpi::BufferSystem bs2( walberla::mpi::MPIManager::instance()->comm(), 255367);
+
WALBERLA_CHECK(!bs2.isCommunicationRunning());
//init buffers
@@ -350,6 +364,11 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve ended." );
+
+ bytesSent_ += bs2.getBytesSent();
+ bytesReceived_ += bs2.getBytesReceived();
+ numberOfSends_ += bs2.getNumberOfSends();
+ numberOfReceives_ += bs2.getNumberOfReceives();
}
} // namespace mpi
diff --git a/src/mesa_pd/mpi/SyncGhostOwners.h b/src/mesa_pd/mpi/SyncGhostOwners.h
index 0b9d1db82..02089e33c 100644
--- a/src/mesa_pd/mpi/SyncGhostOwners.h
+++ b/src/mesa_pd/mpi/SyncGhostOwners.h
@@ -60,11 +60,11 @@ public:
const real_t dx = real_t(0),
const bool syncNonCommunicatingBodies = false ) const;
- int64_t getBytesSent() const { return bs1.getBytesSent() + bs2.getBytesSent(); }
- int64_t getBytesReceived() const { return bs1.getBytesReceived() + bs2.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs1.getNumberOfSends() + bs2.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs1.getNumberOfReceives() + bs2.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
void updateAndMigrate( data::ParticleStorage& ps,
const domain::IDomain& domain,
@@ -77,11 +77,13 @@ private:
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs1 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 749861);
- mutable walberla::mpi::BufferSystem bs2 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 255367);
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/src/mesa_pd/mpi/SyncNextNeighbors.cpp b/src/mesa_pd/mpi/SyncNextNeighbors.cpp
index e36c352dc..52d922375 100644
--- a/src/mesa_pd/mpi/SyncNextNeighbors.cpp
+++ b/src/mesa_pd/mpi/SyncNextNeighbors.cpp
@@ -38,6 +38,8 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs( walberla::mpi::MPIManager::instance()->comm() );
+
neighborRanks_ = domain.getNeighborProcesses();
for( uint_t nbProcessRank : neighborRanks_ )
{
@@ -47,7 +49,7 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
bs.sendBuffer(nbProcessRank) << walberla::uint8_c(0);
}
}
- generateSynchronizationMessages(ps, domain, dx);
+ generateSynchronizationMessages(bs, ps, domain, dx);
// size of buffer is unknown and changes with each send
bs.setReceiverInfoFromSendBufferState(false, true);
@@ -66,9 +68,15 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
-void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& ps,
+void SyncNextNeighbors::generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx) const
{
diff --git a/src/mesa_pd/mpi/SyncNextNeighbors.h b/src/mesa_pd/mpi/SyncNextNeighbors.h
index 8d8364cbf..31e51f56b 100644
--- a/src/mesa_pd/mpi/SyncNextNeighbors.h
+++ b/src/mesa_pd/mpi/SyncNextNeighbors.h
@@ -57,21 +57,25 @@ public:
const domain::IDomain& domain,
const real_t dx = real_t(0)) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- void generateSynchronizationMessages(data::ParticleStorage& ps,
+ void generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx) const;
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.cpp b/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.cpp
index 417d1a86b..34778d0e2 100644
--- a/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.cpp
+++ b/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.cpp
@@ -51,6 +51,8 @@ void SyncNextNeighborsBlockForest::operator()(data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs( walberla::mpi::MPIManager::instance()->comm() );
+
WALBERLA_CHECK(!bs.isCommunicationRunning());
WALBERLA_CHECK_EQUAL(bs.size(), 0);
@@ -67,7 +69,7 @@ void SyncNextNeighborsBlockForest::operator()(data::ParticleStorage& ps,
}
}
- generateSynchronizationMessages(ps, bf, dx);
+ generateSynchronizationMessages(bs, ps, bf, dx);
// size of buffer is unknown and changes with each send
bs.setReceiverInfoFromSendBufferState(false, true);
@@ -86,9 +88,15 @@ void SyncNextNeighborsBlockForest::operator()(data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
-void SyncNextNeighborsBlockForest::generateSynchronizationMessages(data::ParticleStorage& ps,
+void SyncNextNeighborsBlockForest::generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const std::shared_ptr<blockforest::BlockForest>& bf,
const real_t dx) const
{
diff --git a/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.h b/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.h
index 2fa8dfc2d..8c04c3cee 100644
--- a/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.h
+++ b/src/mesa_pd/mpi/SyncNextNeighborsBlockForest.h
@@ -60,20 +60,24 @@ public:
const std::shared_ptr<domain::BlockForestDomain>& domain,
const real_t dx = real_t(0)) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- void generateSynchronizationMessages(data::ParticleStorage& ps,
+ void generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const std::shared_ptr<blockforest::BlockForest>& blockforest,
const real_t dx) const;
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
diff --git a/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.cpp b/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.cpp
index cfd313cef..145b95cd8 100644
--- a/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.cpp
+++ b/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.cpp
@@ -37,6 +37,8 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
{
if (numProcesses_ == 1) return;
+ walberla::mpi::BufferSystem bs( walberla::mpi::MPIManager::instance()->comm() );
+
neighborRanks_ = domain.getNeighborProcesses();
for( uint_t nbProcessRank : neighborRanks_ )
{
@@ -46,7 +48,7 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
bs.sendBuffer(nbProcessRank) << walberla::uint8_c(0);
}
}
- generateSynchronizationMessages(ps, domain);
+ generateSynchronizationMessages(bs, ps, domain);
// size of buffer is unknown and changes with each send
bs.setReceiverInfoFromSendBufferState(false, true);
@@ -65,9 +67,15 @@ void SyncNextNeighborsNoGhosts::operator()(data::ParticleStorage& ps,
}
}
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
+
+ bytesSent_ = bs.getBytesSent();
+ bytesReceived_ = bs.getBytesReceived();
+ numberOfSends_ = bs.getNumberOfSends();
+ numberOfReceives_ = bs.getNumberOfReceives();
}
-void SyncNextNeighborsNoGhosts::generateSynchronizationMessages(data::ParticleStorage& ps,
+void SyncNextNeighborsNoGhosts::generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain) const
{
const uint_t ownRank = uint_c(rank_);
diff --git a/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.h b/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.h
index a802fe4d6..a95bc567f 100644
--- a/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.h
+++ b/src/mesa_pd/mpi/SyncNextNeighborsNoGhosts.h
@@ -60,20 +60,24 @@ public:
void operator()(data::ParticleStorage& ps,
const domain::IDomain& domain) const;
- int64_t getBytesSent() const { return bs.getBytesSent(); }
- int64_t getBytesReceived() const { return bs.getBytesReceived(); }
+ int64_t getBytesSent() const { return bytesSent_; }
+ int64_t getBytesReceived() const { return bytesReceived_; }
- int64_t getNumberOfSends() const { return bs.getNumberOfSends(); }
- int64_t getNumberOfReceives() const { return bs.getNumberOfReceives(); }
+ int64_t getNumberOfSends() const { return numberOfSends_; }
+ int64_t getNumberOfReceives() const { return numberOfReceives_; }
private:
- void generateSynchronizationMessages(data::ParticleStorage& ps,
+ void generateSynchronizationMessages(walberla::mpi::BufferSystem& bs,
+ data::ParticleStorage& ps,
const domain::IDomain& domain) const;
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
- mutable walberla::mpi::BufferSystem bs = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm() );
-
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
+
+ mutable int64_t bytesSent_ = 0;
+ mutable int64_t bytesReceived_ = 0;
+ mutable int64_t numberOfSends_ = 0;
+ mutable int64_t numberOfReceives_ = 0;
};
} // namespace mpi
--
GitLab