[BUGIFX] fixes inconsistencies in pe synchronization

src/domain_decomposition/MapPointToPeriodicDomain.cpp

@@ -40,7 +40,7 @@ void mapPointToPeriodicDomain( const std::array< bool, 3 > & periodic, const AAB
          shift = std::max( shift, real_t(0) );
          x = domain.xMax() - shift;
          if( isIdentical( x, domain.xMax() ) || x < domain.xMin() )
-            x = domain.xMin();
+            x = std::nextafter(domain.xMax(), domain.xMin());
       }
       else if( x >= domain.xMax() )
       {
@@ -61,7 +61,7 @@ void mapPointToPeriodicDomain( const std::array< bool, 3 > & periodic, const AAB
          shift = std::max( shift, real_t(0) );
          y = domain.yMax() - shift;
          if( isIdentical( y, domain.yMax() ) || y < domain.yMin() )
-            y = domain.yMin();
+            y = std::nextafter(domain.yMax(), domain.yMin());
       }
       else if( y >= domain.yMax() )
       {
@@ -82,7 +82,7 @@ void mapPointToPeriodicDomain( const std::array< bool, 3 > & periodic, const AAB
          shift = std::max( shift, real_t(0) );
          z = domain.zMax() - shift;
          if( isIdentical( z, domain.zMax() ) || z < domain.zMin() )
-            z = domain.zMin();
+            z = std::nextafter(domain.zMax(), domain.zMin());
       }
       else if( z >= domain.zMax() )
       {

src/pe/synchronization/SyncNextNeighbors.h

@@ -41,6 +41,7 @@
 #include "blockforest/BlockForest.h"
 #include "core/mpi/BufferSystem.h"
 #include "core/timing/TimingTree.h"
+#include "domain_decomposition/MapPointToPeriodicDomain.h"
 
 namespace walberla {
 namespace pe {
@@ -56,7 +57,16 @@ void generateSynchonizationMessages(mpi::BufferSystem& bs, const Block& block, B
    WALBERLA_LOG_DETAIL( "Assembling of body synchronization message starts..." );
 
    // position update
-   for( auto body = localStorage.begin(); body != localStorage.end(); ) {
+   for( auto body = localStorage.begin(); body != localStorage.end(); )
+   {
+      //correct position to make sure body is always inside the domain!
+      if (!body->isFixed())
+      {
+         auto pos = body->getPosition();
+         block.getBlockStorage().mapToPeriodicDomain(pos);
+         body->setPosition(pos);
+      }
+
       bool isInsideDomain = domain.contains( body->getAABB(), -dx );
 
       WALBERLA_ASSERT( !body->isRemote(), "Central body storage contains remote bodies." );
@@ -146,7 +156,7 @@ void generateSynchonizationMessages(mpi::BufferSystem& bs, const Block& block, B
       {
          // Body is no longer locally owned (body is about to be migrated).
          Owner owner( findContainingProcess( block, gpos ) );
-         WALBERLA_ASSERT_UNEQUAL( owner.blockID_, me.blockID_, "Position is " << gpos );
+         WALBERLA_ASSERT_UNEQUAL( owner.blockID_, me.blockID_, "Position is " << gpos << "\nlocal Block is: " << block.getAABB() );
 
          WALBERLA_LOG_DETAIL( "Local body " << b->getSystemID() << " is no longer on process " << body->MPITrait.getOwner() << " but on process " << owner );
 
@@ -189,6 +199,11 @@ void generateSynchonizationMessages(mpi::BufferSystem& bs, const Block& block, B
             b->setRemote( true );
 
             // Move body to shadow copy storage.
+            {
+               auto pos = b->getPosition();
+               correctBodyPosition(domain, block.getAABB().center(), pos);
+               b->setPosition(pos);
+            }
             shadowStorage.add( localStorage.release( body ) );
 
             // Note: We cannot determine here whether we require the body since we do not have up to date shadow copies.

src/pe/synchronization/SyncShadowOwners.h

@@ -81,6 +81,14 @@ void updateAndMigrate( BlockForest& forest, BlockDataID storageID, const bool sy
       {
          BodyID b (bodyIt.getBodyID());
 
+         //correct position to make sure body is always inside the domain!
+         if (!b->isFixed())
+         {
+            auto pos = b->getPosition();
+            block.getBlockStorage().mapToPeriodicDomain(pos);
+            b->setPosition(pos);
+         }
+
          if( !b->isCommunicating() && !syncNonCommunicatingBodies ) {
             ++bodyIt;
             continue;
@@ -131,6 +139,11 @@ void updateAndMigrate( BlockForest& forest, BlockDataID storageID, const bool sy
                b->setRemote( true );
 
                // Move body to shadow copy storage.
+               {
+                  auto pos = b->getPosition();
+                  correctBodyPosition(forest.getDomain(), block.getAABB().center(), pos);
+                  b->setPosition(pos);
+               }
                shadowStorage.add( localStorage.release( bodyIt ) );
 
                b->MPITrait.deregisterShadowOwner( owner );
@@ -215,7 +228,7 @@ void checkAndResolveOverlap( BlockForest& forest, BlockDataID storageID, const r
       BodyStorage& shadowStorage = (*storage)[1];
 
       const Owner me( int_c(block.getProcess()), block.getId().getID() );
-//      const math::AABB& blkAABB = block->getAABB();
+      //      const math::AABB& blkAABB = block->getAABB();
 
       for( auto bodyIt = localStorage.begin(); bodyIt != localStorage.end(); ++bodyIt)
       {
@@ -238,10 +251,10 @@ void checkAndResolveOverlap( BlockForest& forest, BlockDataID storageID, const r
 
             if (b->MPITrait.getOwner() == nbProcess) continue; // dont send to owner!!
             if (b->MPITrait.getBlockState( nbProcess.blockID_ )) continue; // only send to neighbor which do not know this body
-//            WALBERLA_LOG_DEVEL("neighobur aabb: " << block.getNeighborAABB(nb));
-//            WALBERLA_LOG_DEVEL("isInsideDomain: " << isInsideDomain);
-//            WALBERLA_LOG_DEVEL("body AABB: " << b->getAABB());
-//            WALBERLA_LOG_DEVEL("neighbour AABB: " << block.getNeighborAABB(nb));
+            //            WALBERLA_LOG_DEVEL("neighobur aabb: " << block.getNeighborAABB(nb));
+            //            WALBERLA_LOG_DEVEL("isInsideDomain: " << isInsideDomain);
+            //            WALBERLA_LOG_DEVEL("body AABB: " << b->getAABB());
+            //            WALBERLA_LOG_DEVEL("neighbour AABB: " << block.getNeighborAABB(nb));
 
             if( (isInsideDomain ? block.getNeighborAABB(nb).intersects( b->getAABB(), dx ) : block.getBlockStorage().periodicIntersect(block.getNeighborAABB(nb), b->getAABB(), dx)) )
             {

tests/pe/CMakeLists.txt

@@ -94,8 +94,9 @@ set_property( TEST PE_SERIALIZEDESERIALIZE04 PROPERTY DEPENDS PE_SERIALIZEDESERI
 set_property( TEST PE_SERIALIZEDESERIALIZE08 PROPERTY DEPENDS PE_SERIALIZEDESERIALIZE04 ) #serialize runs of tets to avoid i/o conflicts when running ctest with -jN
 endif()
 
-waLBerla_compile_test( NAME   PE_SHADOWCOPY FILES ShadowCopy.cpp DEPENDS core blockforest  )
-waLBerla_execute_test( NAME   PE_SHADOWCOPY )
+waLBerla_compile_test( NAME   PE_SHADOWCOPY FILES ShadowCopy.cpp DEPENDS core blockforest domain_decomposition  )
+waLBerla_execute_test( NAME   PE_SHADOWCOPY_NN COMMAND $<TARGET_FILE:PE_SHADOWCOPY> )
+waLBerla_execute_test( NAME   PE_SHADOWCOPY_SO COMMAND $<TARGET_FILE:PE_SHADOWCOPY> --syncShadowOwners )
 
 waLBerla_compile_test( NAME   PE_SIMPLECCD FILES SimpleCCD.cpp DEPENDS core  )
 waLBerla_execute_test( NAME   PE_SIMPLECCD )

tests/pe/ShadowCopy.cpp

@@ -42,6 +42,19 @@ int main( int argc, char** argv )
    walberla::debug::enterTestMode();
    walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
 
+   bool syncShadowOwners = false;
+   for( int i = 1; i < argc; ++i )
+   {
+      if( std::strcmp( argv[i], "--syncShadowOwners" ) == 0 ) syncShadowOwners = true;
+   }
+   if (syncShadowOwners)
+   {
+      WALBERLA_LOG_DEVEL("running with syncShadowOwners");
+   } else
+   {
+      WALBERLA_LOG_DEVEL("running with syncNextNeighbour");
+   }
+
    shared_ptr<BodyStorage> globalBodyStorage = make_shared<BodyStorage>();
 
    // create blocks
@@ -64,7 +77,7 @@ int main( int argc, char** argv )
 //   logging::Logging::instance()->setFileLogLevel(logging::Logging::DETAIL);
 //   logging::Logging::instance()->includeLoggingToFile("ShadowCopy");
 
-   bool syncShadowOwners = false;
+
    std::function<void(void)> syncCall;
    if (!syncShadowOwners)
    {
@@ -95,6 +108,23 @@ int main( int argc, char** argv )
    WALBERLA_CHECK_FLOAT_EQUAL( sp->getRadius(), real_t(1.2) );
    destroyBodyBySID( *globalBodyStorage, forest->getBlockStorage(), storageID, sid );
 
+   WALBERLA_LOG_PROGRESS_ON_ROOT( " *** SPHERE AT BLOCK EDGE *** ");
+   sp = pe::createSphere(
+            *globalBodyStorage,
+            forest->getBlockStorage(),
+            storageID,
+            999999999,
+            Vec3(0,2,2),
+            real_c(1.2));
+   sid = sp->getSystemID();
+   syncCall();
+   sp = static_cast<SphereID> (getBody( *globalBodyStorage, forest->getBlockStorage(), storageID, sid, StorageSelect::LOCAL ));
+   sp->setPosition(real_c(-1)*std::numeric_limits<real_t>::epsilon(),2,2);
+   syncCall();
+   sp = static_cast<SphereID> (getBody( *globalBodyStorage, forest->getBlockStorage(), storageID, sid, StorageSelect::LOCAL ));
+   WALBERLA_CHECK_NOT_NULLPTR(sp);
+   destroyBodyBySID( *globalBodyStorage, forest->getBlockStorage(), storageID, sid );
+
    WALBERLA_LOG_PROGRESS_ON_ROOT( " *** UNION *** ");
    UnionT* un   = createUnion< boost::tuple<Sphere> >( *globalBodyStorage, forest->getBlockStorage(), storageID, 0, Vec3(2,2,2) );
    auto sp1 = createSphere(un, 10, Vec3(real_t(4.9),2,2), real_t(1));