[BUGFIX] partial revert of aac96660, introduce new best practice

python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp

@@ -181,6 +181,7 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
    // Receiving the updates for the remote rigid bodies from the connected processes
    WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate starts..." );
    ParseMessage parseMessage;
+   parseMessage.allowMultipleGhostCopyNotifications(true);
    for( auto it = bs1.begin(); it != bs1.end(); ++it )
    {
       walberla::uint8_t tmp;
@@ -338,6 +339,7 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
    // Receiving the updates for the remote rigid bodies from the connected processes
    WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve starts..." );
    ParseMessage parseMessage;
+   parseMessage.allowMultipleGhostCopyNotifications(true);
    for( auto it = bs2.begin(); it != bs2.end(); ++it )
    {
       walberla::uint8_t tmp;

python/mesa_pd/templates/mpi/notifications/ParseMessage.templ.h

@@ -52,8 +52,11 @@ public:
                    walberla::mpi::RecvBuffer& rb,
                    data::ParticleStorage& ps,
                    const domain::IDomain& domain);
+   void allowMultipleGhostCopyNotifications(bool val) {allowMultipleGhostCopyNotifications_ = val;}
 private:
    int receiver_ = int_c( walberla::mpi::MPIManager::instance()->rank() );
+
+   bool allowMultipleGhostCopyNotifications_ = false;
 };
 
 inline
@@ -82,7 +85,14 @@ void ParseMessage::operator()(int sender,
          data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST);
       } else
       {
-         WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend.");
+         if (allowMultipleGhostCopyNotifications_)
+         {
+            //superfluous ghost creation messages might be send during ghost owner sync
+            WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend.");
+         } else
+         {
+            WALBERLA_ABORT("Ghost particle with id " << objparam.uid << " already existend.");
+         }
       }
 
       WALBERLA_LOG_DETAIL( "Processed PARTICLE_GHOST_COPY_NOTIFICATION for particle " << objparam.uid << "."  );

src/mesa_pd/domain/BlockForestDataHandling.cpp

@@ -97,7 +97,6 @@ void BlockForestDataHandling::serialize( IBlock * const block,
       //skip globals
       if (data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL)) continue;
 
-      pIt->getGhostOwnersRef().clear();
       buffer << ParticleCopyNotification( *pIt );
       ++numOfParticles;
    }
@@ -141,7 +140,6 @@ void BlockForestDataHandling::serializeCoarseToFine( Block * const block, const
 
       if( childAABB.contains( pIt->getPosition()) )
       {
-         pIt->getGhostOwnersRef().clear();
          buffer << ParticleCopyNotification( *pIt );
          ++numOfParticles;
       }
@@ -166,7 +164,6 @@ void BlockForestDataHandling::serializeFineToCoarse( Block * const block, const
       //skip globals
       if (data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL)) continue;
 
-      pIt->getGhostOwnersRef().clear();
       buffer << ParticleCopyNotification( *pIt );
       ++numOfParticles;
    }

src/mesa_pd/mpi/SyncGhostOwners.cpp

@@ -181,6 +181,7 @@ void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps,
    // Receiving the updates for the remote rigid bodies from the connected processes
    WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate starts..." );
    ParseMessage parseMessage;
+   parseMessage.allowMultipleGhostCopyNotifications(true);
    for( auto it = bs1.begin(); it != bs1.end(); ++it )
    {
       walberla::uint8_t tmp;
@@ -338,6 +339,7 @@ void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps,
    // Receiving the updates for the remote rigid bodies from the connected processes
    WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve starts..." );
    ParseMessage parseMessage;
+   parseMessage.allowMultipleGhostCopyNotifications(true);
    for( auto it = bs2.begin(); it != bs2.end(); ++it )
    {
       walberla::uint8_t tmp;

src/mesa_pd/mpi/notifications/ParseMessage.h

@@ -52,8 +52,11 @@ public:
                    walberla::mpi::RecvBuffer& rb,
                    data::ParticleStorage& ps,
                    const domain::IDomain& domain);
+   void allowMultipleGhostCopyNotifications(bool val) {allowMultipleGhostCopyNotifications_ = val;}
 private:
    int receiver_ = int_c( walberla::mpi::MPIManager::instance()->rank() );
+
+   bool allowMultipleGhostCopyNotifications_ = false;
 };
 
 inline
@@ -82,7 +85,14 @@ void ParseMessage::operator()(int sender,
          data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST);
       } else
       {
-         WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend.");
+         if (allowMultipleGhostCopyNotifications_)
+         {
+            //superfluous ghost creation messages might be send during ghost owner sync
+            WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend.");
+         } else
+         {
+            WALBERLA_ABORT("Ghost particle with id " << objparam.uid << " already existend.");
+         }
       }
 
       WALBERLA_LOG_DETAIL( "Processed PARTICLE_GHOST_COPY_NOTIFICATION for particle " << objparam.uid << "."  );

tests/mesa_pd/domain/SerializeDeserialize.cpp

@@ -21,8 +21,10 @@
 #include <mesa_pd/data/DataTypes.h>
 #include <mesa_pd/domain/BlockForestDataHandling.h>
 #include <mesa_pd/domain/BlockForestDomain.h>
+#include <mesa_pd/data/ParticleAccessor.h>
 #include <mesa_pd/data/ParticleStorage.h>
 #include <mesa_pd/data/ShapeStorage.h>
+#include <mesa_pd/mpi/ClearNextNeighborSync.h>
 #include <mesa_pd/mpi/SyncNextNeighbors.h>
 
 #include <blockforest/Initialization.h>
@@ -31,6 +33,7 @@
 #include <core/grid_generator/SCIterator.h>
 #include <core/mpi/Reduce.h>
 #include <core/logging/Logging.h>
+#include <mesa_pd/kernel/ParticleSelector.h>
 
 namespace walberla {
 namespace mesa_pd {
@@ -42,6 +45,7 @@ void createDump()
 
    WALBERLA_LOG_INFO_ON_ROOT("*** MESA_PD ***");
    auto ps = std::make_shared<data::ParticleStorage>(100);
+   auto ac = data::ParticleAccessor(ps);
    auto ss = std::make_shared<data::ShapeStorage>();
 
    auto smallSphere = ss->create<data::Sphere>(radius);
@@ -98,11 +102,14 @@ void createDump()
 
    WALBERLA_CHECK_EQUAL(ps->size(), 28);
 
+   mesa_pd::mpi::ClearNextNeighborSync CSNN;
    mesa_pd::mpi::SyncNextNeighbors SNN;
    SNN(*ps, domain);
 
+   CSNN(ac);
    WALBERLA_LOG_DEVEL_ON_ROOT("dumping simulation");
    forest->saveBlockData("SerializeDeserialize.dump", bfDataHandlingID);
+   SNN(*ps, domain);
 }
 
 void checkDump()