From f4daad25fe56efabed88f9c09b473fae83876949 Mon Sep 17 00:00:00 2001
From: Florian Schornbaum <florian.schornbaum@fau.de>
Date: Wed, 19 Jul 2017 16:37:19 +0200
Subject: [PATCH] [BUGFIX] BufferSystem was incorrectly used in master-slave
 mode

---
 src/blockforest/BlockForest.cpp              |  6 ++++++
 src/blockforest/loadbalancing/DynamicCurve.h | 18 ++++++++++++++++++
 src/core/mpi/BufferSystem.cpp                | 18 +++++++-----------
 src/core/mpi/BufferSystem.h                  |  2 --
 4 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/src/blockforest/BlockForest.cpp b/src/blockforest/BlockForest.cpp
index a5af73a8e..dc67a03e3 100644
--- a/src/blockforest/BlockForest.cpp
+++ b/src/blockforest/BlockForest.cpp
@@ -2950,6 +2950,12 @@ void BlockForest::saveToFile( const std::string & filename, FileIOMode fileIOMod
 
          file.close();
       }
+      else
+      {
+         // begin()/end() must also be called on each slave process in order to
+         // properly finalize the communication
+         WALBERLA_CHECK( bufferSystem.begin() == bufferSystem.end() );
+      }
    }
    else
    {
diff --git a/src/blockforest/loadbalancing/DynamicCurve.h b/src/blockforest/loadbalancing/DynamicCurve.h
index 8af01a821..3cb88476c 100644
--- a/src/blockforest/loadbalancing/DynamicCurve.h
+++ b/src/blockforest/loadbalancing/DynamicCurve.h
@@ -364,6 +364,12 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterWeighted( std::vector<
       
       balanceWeighted( allBlocks, blocksPerLevel, targets, sender );
    }
+   else
+   {
+      // begin()/end() must also be called on each slave process in order to
+      // properly finalize the communication
+      WALBERLA_CHECK( bufferSystem.begin() == bufferSystem.end() );
+   }
    
    masterEnd( targets, sender, targetProcess, processesToRecvFrom );   
 }
@@ -431,6 +437,12 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterNoWeight( std::vector<
       
       balanceNoWeight( allBlocks, blocksPerLevel, targets, sender );
    }
+   else
+   {
+      // begin()/end() must also be called on each slave process in order to
+      // properly finalize the communication
+      WALBERLA_CHECK( bufferSystem.begin() == bufferSystem.end() );
+   }
    
    masterEnd( targets, sender, targetProcess, processesToRecvFrom );   
 }
@@ -868,6 +880,12 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterEnd( std::vector< std:
          recvIt.buffer() >> targets[0] >> sender[0];
       }
    }
+   else
+   {
+      // begin()/end() must also be called on each slave process in order to
+      // properly finalize the communication
+      WALBERLA_CHECK( resultsBufferSystem.begin() == resultsBufferSystem.end() );
+   }
    
    finalAssignment( uint_t(0), targets, sender, targetProcess, processesToRecvFrom );
 }
diff --git a/src/core/mpi/BufferSystem.cpp b/src/core/mpi/BufferSystem.cpp
index 5659a874a..1cabacc1d 100644
--- a/src/core/mpi/BufferSystem.cpp
+++ b/src/core/mpi/BufferSystem.cpp
@@ -22,15 +22,15 @@
 #include "BufferSystem.h"
 #include "core/logging/Logging.h"
 #include "core/mpi/MPIManager.h"
+#include "core/debug/CheckFunctions.h"
 
 
 namespace walberla {
 namespace mpi {
 
 
-#ifndef NDEBUG
+
 std::set<int> BufferSystem::activeTags_;
-#endif
 
 //======================================================================================================================
 //
@@ -353,14 +353,12 @@ void BufferSystem::send( MPIRank rank )
 //**********************************************************************************************************************
 void BufferSystem::startCommunication()
 {
-#ifndef NDEBUG
    const auto tag = currentComm_->getTag();
-   WALBERLA_ASSERT_EQUAL(activeTags_.find(tag), activeTags_.end(),
-                         "Another communication with the same MPI tag is currently in progress.");
-  activeTags_.insert(tag);
-#endif
+   WALBERLA_CHECK_EQUAL(activeTags_.find(tag), activeTags_.end(),
+                        "Another communication with the same MPI tag is currently in progress.");
+   activeTags_.insert(tag);
 
-   WALBERLA_ASSERT( ! communicationRunning_ );
+   WALBERLA_CHECK( ! communicationRunning_ );
 
    currentComm_->scheduleReceives( recvInfos_ );
    communicationRunning_ = true;
@@ -378,7 +376,7 @@ void BufferSystem::startCommunication()
 //**********************************************************************************************************************
 void BufferSystem::endCommunication()
 {
-   WALBERLA_ASSERT( communicationRunning_ );
+   WALBERLA_CHECK( communicationRunning_ );
    currentComm_->waitForSends();
 
    // Clear send buffers
@@ -399,9 +397,7 @@ void BufferSystem::endCommunication()
 
    communicationRunning_ = false;
 
-#ifndef NDEBUG
    activeTags_.erase( activeTags_.find( currentComm_->getTag() ) );
-#endif
 }
 
 
diff --git a/src/core/mpi/BufferSystem.h b/src/core/mpi/BufferSystem.h
index 34042e56f..a79322d1a 100644
--- a/src/core/mpi/BufferSystem.h
+++ b/src/core/mpi/BufferSystem.h
@@ -237,11 +237,9 @@ protected:
    };
    std::map<MPIRank, SendInfo> sendInfos_;
 
-   #ifndef NDEBUG
    //stores tags of running communications in debug mode to ensure that
    //each concurrently running communication uses different tags
    static std::set<int> activeTags_;
-   #endif
 };
 
 
-- 
GitLab