Merge branch 'checkpointing_bugfix' into 'master'

[BUGFIX] checkpointing did not reset ghost owner information

See merge request walberla/walberla!247

src/mesa_pd/domain/BlockForestDataHandling.cpp

@@ -97,6 +97,7 @@ void BlockForestDataHandling::serialize( IBlock * const block,
       //skip globals
       if (data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL)) continue;
 
+      pIt->getGhostOwnersRef().clear();
       buffer << ParticleCopyNotification( *pIt );
       ++numOfParticles;
    }
@@ -140,6 +141,7 @@ void BlockForestDataHandling::serializeCoarseToFine( Block * const block, const
 
       if( childAABB.contains( pIt->getPosition()) )
       {
+         pIt->getGhostOwnersRef().clear();
          buffer << ParticleCopyNotification( *pIt );
          ++numOfParticles;
       }
@@ -164,6 +166,7 @@ void BlockForestDataHandling::serializeFineToCoarse( Block * const block, const
       //skip globals
       if (data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL)) continue;
 
+      pIt->getGhostOwnersRef().clear();
       buffer << ParticleCopyNotification( *pIt );
       ++numOfParticles;
    }

tests/mesa_pd/domain/SerializeDeserialize.cpp

@@ -20,42 +20,45 @@
 
 #include <mesa_pd/data/DataTypes.h>
 #include <mesa_pd/domain/BlockForestDataHandling.h>
+#include <mesa_pd/domain/BlockForestDomain.h>
 #include <mesa_pd/data/ParticleStorage.h>
 #include <mesa_pd/data/ShapeStorage.h>
+#include <mesa_pd/mpi/SyncNextNeighbors.h>
 
 #include <blockforest/Initialization.h>
 #include <core/debug/TestSubsystem.h>
+#include <core/Environment.h>
 #include <core/grid_generator/SCIterator.h>
 #include <core/mpi/Reduce.h>
 #include <core/logging/Logging.h>
 
 namespace walberla {
-
-using namespace walberla::mesa_pd;
+namespace mesa_pd {
 
 void createDump()
 {
    const real_t spacing = real_c(1);
-   const real_t radius  = real_c(0.5);
+   const real_t radius = real_c(0.5);
 
    WALBERLA_LOG_INFO_ON_ROOT("*** MESA_PD ***");
    auto ps = std::make_shared<data::ParticleStorage>(100);
    auto ss = std::make_shared<data::ShapeStorage>();
 
-   auto  smallSphere = ss->create<data::Sphere>( radius );
+   auto smallSphere = ss->create<data::Sphere>(radius);
    ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707));
 
    WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***");
    // create forest
-   auto forest = blockforest::createBlockForest( math::AABB(real_t(0),
-                                                            real_t(0),
-                                                            real_t(0),
-                                                            real_t(6),
-                                                            real_t(6),
-                                                            real_t(6)),
-                                                 Vector3<uint_t>(2,2,2),
-                                                 Vector3<bool>(true, true, true) );
+   auto forest = blockforest::createBlockForest(math::AABB(real_t(0),
+                                                           real_t(0),
+                                                           real_t(0),
+                                                           real_t(6),
+                                                           real_t(6),
+                                                           real_t(6)),
+                                                Vector3<uint_t>(2, 2, 2),
+                                                Vector3<bool>(true, true, true));
    forest->saveToFile("SerializeDeserialize.sbf");
+   domain::BlockForestDomain domain(forest);
 
    auto bfDataHandlingID = forest->addBlockData(domain::createBlockForestDataHandling(ps), "BFDataHandling");
 
@@ -65,61 +68,68 @@ void createDump()
    //add one global particle
    {
       auto p = ps->create();
-      p->setPosition( Vec3(real_t(1)) );
-      data::particle_flags::set( p->getFlagsRef(), data::particle_flags::GLOBAL);
-      p->setInteractionRadius( radius );
-      p->setShapeID( smallSphere );
+      p->setPosition(Vec3(real_t(1)));
+      data::particle_flags::set(p->getFlagsRef(), data::particle_flags::GLOBAL);
+      data::particle_flags::set(p->getFlagsRef(), data::particle_flags::NON_COMMUNICATING);
+      p->setInteractionRadius(radius);
+      p->setShapeID(smallSphere);
+      p->setOwner(walberla::mpi::MPIManager::instance()->rank());
    }
 
-   for (auto& iBlk : *forest)
+   for (auto &iBlk : *forest)
    {
-      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing))
+      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.2),
+                                            spacing))
       {
          WALBERLA_CHECK(iBlk.getAABB().contains(pt));
 
-         auto p                       = ps->create();
-         p->setPosition( pt );
-         p->setInteractionRadius( radius );
-         p->setShapeID( smallSphere );
-         p->setOwner( mpi::MPIManager::instance()->rank() );
+         auto p = ps->create();
+         p->setPosition(pt);
+         p->setInteractionRadius(radius);
+         p->setShapeID(smallSphere);
+         p->setOwner(walberla::mpi::MPIManager::instance()->rank());
       }
    }
    int64_t numParticles = int64_c(ps->size());
-   mpi::reduceInplace(numParticles, mpi::SUM);
+   walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM);
    WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles);
 
    WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - END ***");
 
-   WALBERLA_CHECK_EQUAL( ps->size(), 28 );
+   WALBERLA_CHECK_EQUAL(ps->size(), 28);
+
+   mesa_pd::mpi::SyncNextNeighbors SNN;
+   SNN(*ps, domain);
 
    WALBERLA_LOG_DEVEL_ON_ROOT("dumping simulation");
    forest->saveBlockData("SerializeDeserialize.dump", bfDataHandlingID);
-
-   WALBERLA_CHECK_EQUAL( ps->size(), 28 );
 }
 
 void checkDump()
 {
    const real_t spacing = real_c(1);
-   const real_t radius  = real_c(0.5);
+   const real_t radius = real_c(0.5);
 
    WALBERLA_LOG_INFO_ON_ROOT("*** MESA_PD ***");
    auto ps = std::make_shared<data::ParticleStorage>(100);
    auto ss = std::make_shared<data::ShapeStorage>();
 
-   auto  smallSphere = ss->create<data::Sphere>( radius );
+   auto smallSphere = ss->create<data::Sphere>(radius);
    ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707));
 
    WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***");
-   auto forest = make_shared< BlockForest >( uint_c( MPIManager::instance()->rank() ), "SerializeDeserialize.sbf", true, false );
+   auto forest = make_shared<BlockForest>(uint_c(walberla::mpi::MPIManager::instance()->rank()),
+                                          "SerializeDeserialize.sbf", true, false);
+   domain::BlockForestDomain domain(forest);
 
    //add one global particle
    {
       auto p = ps->create();
-      p->setPosition( Vec3(real_t(1)) );
-      data::particle_flags::set( p->getFlagsRef(), data::particle_flags::GLOBAL);
-      p->setInteractionRadius( radius );
-      p->setShapeID( smallSphere );
+      p->setPosition(Vec3(real_t(1)));
+      data::particle_flags::set(p->getFlagsRef(), data::particle_flags::GLOBAL);
+      data::particle_flags::set(p->getFlagsRef(), data::particle_flags::NON_COMMUNICATING);
+      p->setInteractionRadius(radius);
+      p->setShapeID(smallSphere);
    }
 
    WALBERLA_CHECK_EQUAL(ps->size(), 1);
@@ -128,9 +138,10 @@ void checkDump()
 
    WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING ***");
    auto pIt = ++ps->begin();
-   for (auto& iBlk : *forest)
+   for (auto &iBlk : *forest)
    {
-      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing))
+      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.2),
+                                            spacing))
       {
          WALBERLA_CHECK(iBlk.getAABB().contains(pt));
          WALBERLA_CHECK_UNEQUAL(pIt, ps->end()); //still particles left
@@ -139,16 +150,16 @@ void checkDump()
       }
    }
    WALBERLA_CHECK_EQUAL(pIt, ps->end()); //all particles have been checked
+
+   mesa_pd::mpi::SyncNextNeighbors SNN;
+   SNN(*ps, domain);
 }
 
-int main( int argc, char ** argv )
+int main(int argc, char **argv)
 {
    walberla::debug::enterTestMode();
-
-   WALBERLA_MPI_SECTION()
-   {
-      walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
-   }
+   Environment env(argc, argv);
+   WALBERLA_UNUSED(env);
 
    WALBERLA_LOG_DEVEL_ON_ROOT("*** DUMPING ***");
    createDump();
@@ -161,9 +172,10 @@ int main( int argc, char ** argv )
 
    return EXIT_SUCCESS;
 }
+} //namespace mesa_pd
 } // namespace walberla
 
 int main( int argc, char* argv[] )
 {
-   return walberla::main( argc, argv );
+   return walberla::mesa_pd::main( argc, argv );
 }