added ClearGhostOwnerSync

python/mesa_pd.py

@@ -105,6 +105,7 @@ if __name__ == '__main__':
     mpd.add(kernel.VelocityVerletWithShape())
 
     mpd.add(mpi.BroadcastProperty())
+    mpd.add(mpi.ClearGhostOwnerSync())
     mpd.add(mpi.ClearNextNeighborSync())
     mpd.add(mpi.Notifications(ps))
     mpd.add(mpi.ReduceContactHistory())

python/mesa_pd/mpi/ClearGhostOwnerSync.py

+# -*- coding: utf-8 -*-
+
+from mesa_pd.accessor import create_access
+from mesa_pd.utility import generate_file
+
+
+class ClearGhostOwnerSync:
+    def __init__(self):
+        self.context = {'properties': [], 'interface': []}
+
+        self.context['interface'].append(
+            create_access("flags", 'walberla::mesa_pd::data::particle_flags::FlagT', access='g'))
+        self.context['interface'].append(create_access('ghostOwners', 'std::vector<int>', access='r'))
+        self.context['interface'].append(create_access('neighborState', 'std::unordered_set<walberla::mpi::MPIRank>',
+                                                       access='r'))
+
+    def generate(self, module):
+        ctx = {'module': module, **self.context}
+        generate_file(module['module_path'], 'mpi/ClearGhostOwnerSync.templ.h', ctx)

python/mesa_pd/mpi/__init__.py

 # -*- coding: utf-8 -*-
 
 from .BroadcastProperty import BroadcastProperty
+from .ClearGhostOwnerSync import ClearGhostOwnerSync
 from .ClearNextNeighborSync import ClearNextNeighborSync
 from .Notifications import Notifications
 from .ReduceContactHistory import ReduceContactHistory
@@ -11,6 +12,7 @@ from .SyncNextNeighbors import SyncNextNeighbors
 from .SyncNextNeighborsNoGhosts import SyncNextNeighborsNoGhosts
 
 __all__ = ['BroadcastProperty',
+           'ClearGhostOwnerSync',
            'ClearNextNeighborSync',
            'ReduceContactHistory',
            'ReduceProperty',

python/mesa_pd/templates/mpi/ClearGhostOwnerSync.templ.h

+//======================================================================================================================
+//
+//  This file is part of waLBerla. waLBerla is free software: you can
+//  redistribute it and/or modify it under the terms of the GNU General Public
+//  License as published by the Free Software Foundation, either version 3 of
+//  the License, or (at your option) any later version.
+//
+//  waLBerla is distributed in the hope that it will be useful, but WITHOUT
+//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+//  for more details.
+//
+//  You should have received a copy of the GNU General Public License along
+//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file ClearGhostOwnerSync.h
+//! \author Sebastian Eibl <sebastian.eibl@fau.de>
+//
+//======================================================================================================================
+
+//======================================================================================================================
+//
+//  THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
+//
+//======================================================================================================================
+
+#pragma once
+
+#include <mesa_pd/data/Flags.h>
+
+namespace walberla {
+namespace mesa_pd {
+namespace mpi {
+
+/**
+ * Clear all ghost particles and reset ghost owner information
+ *
+ * This kernel requires the following particle accessor interface
+ * \code
+   {%- for prop in interface %}
+   {%- if 'g' in prop.access %}
+ * const {{prop.type}}& get{{prop.name | capFirst}}(const size_t p_idx) const;
+   {%- endif %}
+   {%- if 's' in prop.access %}
+ * void set{{prop.name | capFirst}}(const size_t p_idx, const {{prop.type}}& v);
+   {%- endif %}
+   {%- if 'r' in prop.access %}
+ * {{prop.type}}& get{{prop.name | capFirst}}Ref(const size_t p_idx);
+   {%- endif %}
+ *
+   {%- endfor %}
+ * \endcode
+ *
+ * \post All ghost particles are deleted.
+ * \post All ghost owners are reset.
+ * \post All cached information is reset.
+ *
+ * \ingroup mesa_pd_mpi
+ */
+class ClearGhostOwnerSync
+{
+public:
+   template <typename Accessor>
+   void operator()(Accessor& ac) const;
+};
+
+template <typename Accessor>
+void ClearGhostOwnerSync::operator()(Accessor& ac) const
+{
+   for (size_t idx = 0; idx < ac.size(); )
+   {
+      if (data::particle_flags::isSet( ac.getFlags(idx), data::particle_flags::GHOST))
+      {
+         //ghost particle
+         idx = ac.erase(idx);
+         continue;
+      } else
+      {
+         //local particle
+         ac.getGhostOwnersRef(idx).clear();
+         ac.getNeighborStateRef(idx).clear();
+      }
+      ++idx;
+   }
+}
+
+}  // namespace mpi
+}  // namespace mesa_pd
+}  // namespace walberla

src/mesa_pd/mpi/ClearGhostOwnerSync.h

+//======================================================================================================================
+//
+//  This file is part of waLBerla. waLBerla is free software: you can
+//  redistribute it and/or modify it under the terms of the GNU General Public
+//  License as published by the Free Software Foundation, either version 3 of
+//  the License, or (at your option) any later version.
+//
+//  waLBerla is distributed in the hope that it will be useful, but WITHOUT
+//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+//  for more details.
+//
+//  You should have received a copy of the GNU General Public License along
+//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file ClearGhostOwnerSync.h
+//! \author Sebastian Eibl <sebastian.eibl@fau.de>
+//
+//======================================================================================================================
+
+//======================================================================================================================
+//
+//  THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
+//
+//======================================================================================================================
+
+#pragma once
+
+#include <mesa_pd/data/Flags.h>
+
+namespace walberla {
+namespace mesa_pd {
+namespace mpi {
+
+/**
+ * Clear all ghost particles and reset ghost owner information
+ *
+ * This kernel requires the following particle accessor interface
+ * \code
+ * const walberla::mesa_pd::data::particle_flags::FlagT& getFlags(const size_t p_idx) const;
+ *
+ * std::vector<int>& getGhostOwnersRef(const size_t p_idx);
+ *
+ * std::unordered_set<walberla::mpi::MPIRank>& getNeighborStateRef(const size_t p_idx);
+ *
+ * \endcode
+ *
+ * \post All ghost particles are deleted.
+ * \post All ghost owners are reset.
+ * \post All cached information is reset.
+ *
+ * \ingroup mesa_pd_mpi
+ */
+class ClearGhostOwnerSync
+{
+public:
+   template <typename Accessor>
+   void operator()(Accessor& ac) const;
+};
+
+template <typename Accessor>
+void ClearGhostOwnerSync::operator()(Accessor& ac) const
+{
+   for (size_t idx = 0; idx < ac.size(); )
+   {
+      if (data::particle_flags::isSet( ac.getFlags(idx), data::particle_flags::GHOST))
+      {
+         //ghost particle
+         idx = ac.erase(idx);
+         continue;
+      } else
+      {
+         //local particle
+         ac.getGhostOwnersRef(idx).clear();
+         ac.getNeighborStateRef(idx).clear();
+      }
+      ++idx;
+   }
+}
+
+}  // namespace mpi
+}  // namespace mesa_pd
+}  // namespace walberla
\ No newline at end of file

tests/mesa_pd/CMakeLists.txt

@@ -161,6 +161,12 @@ waLBerla_execute_test( NAME   MESA_PD_Kernel_VelocityVerletWithShape )
 waLBerla_compile_test( NAME   MESA_PD_MPI_BroadcastProperty FILES mpi/BroadcastProperty.cpp DEPENDS core )
 waLBerla_execute_test( NAME   MESA_PD_MPI_BroadcastProperty PROCESSES 8 )
 
+waLBerla_compile_test( NAME   MESA_PD_MPI_ClearGhostOwnerSync FILES mpi/ClearGhostOwnerSync.cpp DEPENDS blockforest core)
+waLBerla_execute_test( NAME   MESA_PD_MPI_ClearGhostOwnerSync PROCESSES 8 )
+
+waLBerla_compile_test( NAME   MESA_PD_MPI_ClearNextNeighborSync FILES mpi/ClearNextNeighborSync.cpp DEPENDS blockforest core )
+waLBerla_execute_test( NAME   MESA_PD_MPI_ClearNextNeighborSync PROCESSES 8 )
+
 waLBerla_compile_test( NAME   MESA_PD_MPI_Notifications FILES mpi/Notifications.cpp DEPENDS core )
 waLBerla_execute_test( NAME   MESA_PD_MPI_Notifications )
 

tests/mesa_pd/mpi/ClearGhostOwnerSync.cpp

+//======================================================================================================================
+//
+//  This file is part of waLBerla. waLBerla is free software: you can 
+//  redistribute it and/or modify it under the terms of the GNU General Public
+//  License as published by the Free Software Foundation, either version 3 of 
+//  the License, or (at your option) any later version.
+//  
+//  waLBerla is distributed in the hope that it will be useful, but WITHOUT 
+//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
+//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License 
+//  for more details.
+//  
+//  You should have received a copy of the GNU General Public License along
+//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file ClearGhostOwnerSync.cpp
+//! \author Sebastian Eibl <sebastian.eibl@fau.de>
+//
+//======================================================================================================================
+
+#include <mesa_pd/data/DataTypes.h>
+#include <mesa_pd/domain/BlockForestDomain.h>
+#include <mesa_pd/data/ParticleAccessor.h>
+#include <mesa_pd/data/ParticleStorage.h>
+#include <mesa_pd/data/ShapeStorage.h>
+#include <mesa_pd/mpi/ClearGhostOwnerSync.h>
+#include <mesa_pd/mpi/SyncGhostOwners.h>
+
+#include <blockforest/Initialization.h>
+#include <core/debug/TestSubsystem.h>
+#include <core/Environment.h>
+#include <core/grid_generator/SCIterator.h>
+#include <core/mpi/Reduce.h>
+#include <core/logging/Logging.h>
+
+namespace walberla {
+namespace mesa_pd {
+
+int main(int argc, char **argv)
+{
+   walberla::debug::enterTestMode();
+   Environment env(argc, argv);
+   WALBERLA_UNUSED(env);
+
+   const real_t spacing = real_c(1);
+   const real_t radius = real_c(0.5);
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** MESA_PD ***");
+   auto ps = std::make_shared<data::ParticleStorage>(100);
+   auto ac = data::ParticleAccessor(ps);
+   auto ss = std::make_shared<data::ShapeStorage>();
+
+   auto smallSphere = ss->create<data::Sphere>(radius);
+   ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707));
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***");
+   // create forest
+   auto forest = blockforest::createBlockForest(math::AABB(real_t(0),
+                                                           real_t(0),
+                                                           real_t(0),
+                                                           real_t(6),
+                                                           real_t(6),
+                                                           real_t(6)),
+                                                Vector3<uint_t>(2, 2, 2),
+                                                Vector3<bool>(true, true, true));
+   domain::BlockForestDomain domain(forest);
+
+   WALBERLA_CHECK_EQUAL(forest->size(), 1, "please run with 8 processes -> 1 process per block");
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - START ***");
+   for (auto &iBlk : *forest)
+   {
+      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.2),
+                                            spacing))
+      {
+         WALBERLA_CHECK(iBlk.getAABB().contains(pt));
+
+         auto p = ps->create();
+         p->setPosition(pt);
+         p->setInteractionRadius(radius);
+         p->setShapeID(smallSphere);
+         p->setOwner(walberla::mpi::MPIManager::instance()->rank());
+      }
+   }
+   int64_t numParticles = int64_c(ps->size());
+   walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM);
+   WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles);
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - END ***");
+
+   mesa_pd::mpi::ClearGhostOwnerSync CGO;
+   mesa_pd::mpi::SyncGhostOwners SGO;
+   SGO(*ps, domain);
+   SGO(*ps, domain);
+   SGO(*ps, domain);
+   CGO(ac);
+
+   for(auto p : *ps)
+   {
+      using namespace mesa_pd::data::particle_flags;
+      WALBERLA_CHECK(!isSet(p.getFlags(), GHOST));
+      WALBERLA_CHECK(p.getGhostOwners().empty());
+      WALBERLA_CHECK(p.getNeighborState().empty());
+   }
+
+   return EXIT_SUCCESS;
+}
+} //namespace mesa_pd
+} // namespace walberla
+
+int main( int argc, char* argv[] )
+{
+   return walberla::mesa_pd::main( argc, argv );
+}

tests/mesa_pd/mpi/ClearNextNeighborSync.cpp

+//======================================================================================================================
+//
+//  This file is part of waLBerla. waLBerla is free software: you can 
+//  redistribute it and/or modify it under the terms of the GNU General Public
+//  License as published by the Free Software Foundation, either version 3 of 
+//  the License, or (at your option) any later version.
+//  
+//  waLBerla is distributed in the hope that it will be useful, but WITHOUT 
+//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
+//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License 
+//  for more details.
+//  
+//  You should have received a copy of the GNU General Public License along
+//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file ClearNextNeighborSync.cpp
+//! \author Sebastian Eibl <sebastian.eibl@fau.de>
+//
+//======================================================================================================================
+
+#include <mesa_pd/data/DataTypes.h>
+#include <mesa_pd/domain/BlockForestDomain.h>
+#include <mesa_pd/data/ParticleAccessor.h>
+#include <mesa_pd/data/ParticleStorage.h>
+#include <mesa_pd/data/ShapeStorage.h>
+#include <mesa_pd/mpi/ClearNextNeighborSync.h>
+#include <mesa_pd/mpi/SyncNextNeighbors.h>
+
+#include <blockforest/Initialization.h>
+#include <core/debug/TestSubsystem.h>
+#include <core/Environment.h>
+#include <core/grid_generator/SCIterator.h>
+#include <core/mpi/Reduce.h>
+#include <core/logging/Logging.h>
+
+namespace walberla {
+namespace mesa_pd {
+
+int main(int argc, char **argv)
+{
+   walberla::debug::enterTestMode();
+   Environment env(argc, argv);
+   WALBERLA_UNUSED(env);
+
+   const real_t spacing = real_c(1);
+   const real_t radius = real_c(0.5);
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** MESA_PD ***");
+   auto ps = std::make_shared<data::ParticleStorage>(100);
+   auto ac = data::ParticleAccessor(ps);
+   auto ss = std::make_shared<data::ShapeStorage>();
+
+   auto smallSphere = ss->create<data::Sphere>(radius);
+   ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707));
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***");
+   // create forest
+   auto forest = blockforest::createBlockForest(math::AABB(real_t(0),
+                                                           real_t(0),
+                                                           real_t(0),
+                                                           real_t(6),
+                                                           real_t(6),
+                                                           real_t(6)),
+                                                Vector3<uint_t>(2, 2, 2),
+                                                Vector3<bool>(true, true, true));
+   domain::BlockForestDomain domain(forest);
+
+   WALBERLA_CHECK_EQUAL(forest->size(), 1, "please run with 8 processes -> 1 process per block");
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - START ***");
+   for (auto &iBlk : *forest)
+   {
+      for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.2),
+                                            spacing))
+      {
+         WALBERLA_CHECK(iBlk.getAABB().contains(pt));
+
+         auto p = ps->create();
+         p->setPosition(pt);
+         p->setInteractionRadius(radius);
+         p->setShapeID(smallSphere);
+         p->setOwner(walberla::mpi::MPIManager::instance()->rank());
+      }
+   }
+   int64_t numParticles = int64_c(ps->size());
+   walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM);
+   WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles);
+
+   WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - END ***");
+
+   mesa_pd::mpi::ClearNextNeighborSync CSNN;
+   mesa_pd::mpi::SyncNextNeighbors SNN;
+   SNN(*ps, domain);
+   SNN(*ps, domain);
+   SNN(*ps, domain);
+   CSNN(ac);
+
+   for(auto p : *ps)
+   {
+      using namespace mesa_pd::data::particle_flags;
+      WALBERLA_CHECK(!isSet(p.getFlags(), GHOST));
+      WALBERLA_CHECK(p.getGhostOwners().empty());
+   }
+
+   return EXIT_SUCCESS;
+}
+} //namespace mesa_pd
+} // namespace walberla
+
+int main( int argc, char* argv[] )
+{
+   return walberla::mesa_pd::main( argc, argv );
+}