diff --git a/python/mesa_pd.py b/python/mesa_pd.py index dd15b7a741c7d2a4f288d603ee3e464ef54f8b40..949463d66847775c975ed0425ec70f7b159ff493 100755 --- a/python/mesa_pd.py +++ b/python/mesa_pd.py @@ -111,6 +111,7 @@ if __name__ == '__main__': comm.append(mpi.ClearNextNeighborSync()) comm.append(mpi.ReduceContactHistory()) comm.append(mpi.ReduceProperty()) + comm.append(mpi.SyncGhostOwners(ps)) comm.append(mpi.SyncNextNeighbors(ps)) diff --git a/python/mesa_pd/data/ParticleStorage.py b/python/mesa_pd/data/ParticleStorage.py index e4efc99993b946fb2dcf7bdbdeb6c7dfb39beaa5..21c21c381bd657def2210cf79e63924dd25fa70c 100644 --- a/python/mesa_pd/data/ParticleStorage.py +++ b/python/mesa_pd/data/ParticleStorage.py @@ -10,12 +10,12 @@ class ParticleStorage(Container): self.addInclude("mesa_pd/data/Flags.h") - self.addProperty("uid", "walberla::id_t", defValue = "UniqueID<data::Particle>::invalidID()", syncMode="ALWAYS") + self.addProperty("uid", "walberla::id_t", defValue = "UniqueID<data::Particle>::invalidID()", syncMode="ALWAYS") self.addProperty("position", "walberla::mesa_pd::Vec3", defValue = "real_t(0)", syncMode="ALWAYS") - self.addProperty("interactionRadius", "walberla::real_t", defValue = "real_t(0)", syncMode="COPY") + self.addProperty("interactionRadius", "walberla::real_t", defValue = "real_t(0)", syncMode="COPY") self.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue = "", syncMode="COPY") - self.addProperty("owner", "int", defValue = "-1", syncMode="COPY") - self.addProperty("ghostOwners", "std::vector<int>", defValue = "", syncMode="MIGRATION") + self.addProperty("owner", "int", defValue = "-1", syncMode="COPY") + self.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue = "", syncMode="MIGRATION") def generate(self, path): self.unrollDimension() @@ -40,7 +40,9 @@ class ParticleStorage(Container): generateFile(path, 'mpi/notifications/HeatFluxNotification.templ.h', context) generateFile(path, 'mpi/notifications/ParseMessage.templ.h', context) generateFile(path, 'mpi/notifications/ParticleCopyNotification.templ.h', context) + generateFile(path, 'mpi/notifications/NewGhostParticleNotification.templ.h', context) generateFile(path, 'mpi/notifications/ParticleMigrationNotification.templ.h', context) generateFile(path, 'mpi/notifications/ParticleRemoteMigrationNotification.templ.h', context) + generateFile(path, 'mpi/notifications/ParticleRemovalInformationNotification.templ.h', context) generateFile(path, 'mpi/notifications/ParticleRemovalNotification.templ.h', context) generateFile(path, 'mpi/notifications/ParticleUpdateNotification.templ.h', context) diff --git a/python/mesa_pd/mpi/SyncGhostOwners.py b/python/mesa_pd/mpi/SyncGhostOwners.py new file mode 100644 index 0000000000000000000000000000000000000000..872f5146d3b1d5e36f6f9f74b332c34bc3f63251 --- /dev/null +++ b/python/mesa_pd/mpi/SyncGhostOwners.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from ..utility import generateFile + +class SyncGhostOwners: + def __init__(self, p): + p.addProperty("position", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS") + p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE") + p.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue="", syncMode="ONCE") + p.addProperty("owner", "int", defValue="-1", syncMode="ONCE") + p.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER") + p.addProperty("neighborState", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER") + + def generate(self, path): + generateFile(path, 'mpi/SyncGhostOwners.templ.h') + generateFile(path, 'mpi/SyncGhostOwners.templ.cpp') diff --git a/python/mesa_pd/mpi/SyncNextNeighbors.py b/python/mesa_pd/mpi/SyncNextNeighbors.py index 6fdc34f48955f86a4c7cad1c51b3330cfd791378..44d1851eb6218671e8740925d962d9629a044728 100644 --- a/python/mesa_pd/mpi/SyncNextNeighbors.py +++ b/python/mesa_pd/mpi/SyncNextNeighbors.py @@ -5,10 +5,10 @@ from ..utility import generateFile class SyncNextNeighbors: def __init__(self, p): p.addProperty("position", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS") - p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE") + p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE") p.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue="", syncMode="ONCE") - p.addProperty("owner", "int", defValue="-1", syncMode="ONCE") - p.addProperty("ghostOwners", "std::vector<int>", defValue="", syncMode="NEVER") + p.addProperty("owner", "int", defValue="-1", syncMode="ONCE") + p.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER") def generate(self, path): generateFile(path, 'mpi/SyncNextNeighbors.templ.h') diff --git a/python/mesa_pd/mpi/__init__.py b/python/mesa_pd/mpi/__init__.py index 56849e336aa95ba05b79e8c2470f4142bb51f062..7714d1098c0e44f3d40bbe34e3fd3366347abb1f 100644 --- a/python/mesa_pd/mpi/__init__.py +++ b/python/mesa_pd/mpi/__init__.py @@ -4,11 +4,13 @@ from .BroadcastProperty import BroadcastProperty from .ClearNextNeighborSync import ClearNextNeighborSync from .ReduceContactHistory import ReduceContactHistory from .ReduceProperty import ReduceProperty +from .SyncGhostOwners import SyncGhostOwners from .SyncNextNeighbors import SyncNextNeighbors __all__ = ['BroadcastProperty', 'ClearNextNeighborSync', 'ReduceContactHistory', 'ReduceProperty', + 'SyncGhostOwners', 'SyncNextNeighbors', ] diff --git a/python/mesa_pd/templates/data/ParticleStorage.templ.h b/python/mesa_pd/templates/data/ParticleStorage.templ.h index c6a0f2011832bd1161af3c305892a7c7ba67cc76..7d6b1fa4b21bb6cd70a01e88518b0b5b1a8a23be 100644 --- a/python/mesa_pd/templates/data/ParticleStorage.templ.h +++ b/python/mesa_pd/templates/data/ParticleStorage.templ.h @@ -31,6 +31,7 @@ #include <map> #include <type_traits> #include <unordered_map> +#include <unordered_set> #include <vector> #include <mesa_pd/data/ContactHistory.h> @@ -44,6 +45,7 @@ #include <core/Abort.h> #include <core/debug/Debug.h> #include <core/math/AABB.h> +#include <core/mpi/MPIWrapper.h> #include <core/OpenMP.h> #include <core/STLIO.h> #include <core/UniqueID.h> diff --git a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..da627c08addf2e50535274d581257ff67b11e7ea --- /dev/null +++ b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.cpp @@ -0,0 +1,355 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwners.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#include "SyncGhostOwners.h" + +#include <mesa_pd/mpi/RemoveAndNotify.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +void SyncGhostOwners::operator()( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const +{ + if (numProcesses_ == 1) return; + + //========================================================== + // STEP1: Update & Migrate + //========================================================== + updateAndMigrate( ps, domain, syncNonCommunicatingBodies ); + + //========================================================== + // STEP2: Check & Resolve + //========================================================== + checkAndResolveOverlap( ps, domain, dx, syncNonCommunicatingBodies ); +} + +void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps, + const domain::IDomain& domain, + const bool syncNonCommunicatingBodies ) const +{ + using namespace walberla::mesa_pd::data::particle_flags; + //========================================================== + // STEP1: Update & Migrate + //========================================================== + + WALBERLA_CHECK(!bs1.isCommunicationRunning()); + + WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate starts..." ); + std::set<walberla::mpi::MPIRank> recvRanks; // potential message senders + for( auto pIt = ps.begin(); pIt != ps.end(); ++pIt) + { + if (isSet( pIt->getFlags(), GHOST)) + { + if (!isSet( pIt->getFlags(), NON_COMMUNICATING) || syncNonCommunicatingBodies) + { + recvRanks.insert(pIt->getOwner()); + } + } + } + + for( auto pIt = ps.begin(); pIt != ps.end(); ) + { + if (isSet( pIt->getFlags(), GHOST)) + { + ++pIt; + continue; + } + + //================== + // LOCAL + + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) + { + ++pIt; + continue; + } + + //correct position to make sure particle is always inside the domain! + //everything is decided by the master particle therefore ghost particles are not touched + if (!data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::FIXED) && + !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST)) + { + domain.periodicallyMapToDomain( pIt->getPositionRef() ); + } + + // Update + for (auto ghostOwner : pIt->getGhostOwners()) + { + WALBERLA_LOG_DETAIL( "Sending update notification for body " << pIt->getUid() << " to process " << ghostOwner ); + walberla::mpi::SendBuffer& sb = bs1.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostOwner)); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleUpdateNotification( *pIt )); + } + + //particle has left subdomain? + const auto newOwner = domain.findContainingProcessRank( pIt->getPosition() ); + if( newOwner != int_c(rank_) ) + { + if ( newOwner < 0) + { + // No owner found: Outflow condition. + WALBERLA_LOG_DETAIL( "Sending deletion notifications for body " << pIt->getUid() << " due to outflow." ); + + //delete body + pIt = removeAndNotify( bs1, ps, pIt ); + + continue; + } + + // Set new owner and transform to shadow copy + pIt->setOwner( newOwner ); + set( pIt->getFlagsRef(), GHOST ); + + // currently position is mapped to periodically to global domain, + // this might not be the correct position for a ghost particle + domain.correctParticlePosition( pIt->getPositionRef() ); + + // Correct registration list (exclude new owner and us - the old owner) and + // notify registered processes (except for new owner) of (remote) migration since they possess a ghost particle. + auto ownerIt = std::find( pIt->getGhostOwners().begin(), pIt->getGhostOwners().end(), newOwner ); + WALBERLA_CHECK_UNEQUAL(ownerIt, pIt->getGhostOwners().end(), "New owner has to be former ghost owner!" ); + + pIt->getGhostOwnersRef().erase( ownerIt ); + + // Send remote migration notifications + for( auto ghostRank : pIt->getGhostOwners() ) + { + auto& buffer( bs1.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)) ); + if (buffer.isEmpty()) buffer << walberla::uint8_c(0); + + WALBERLA_LOG_DETAIL( "Sending remote migration notification for particle " << + pIt->getUid() << + " to process " << + ghostRank ); + + packNotification(buffer, ParticleRemoteMigrationNotification( *pIt, newOwner )); + } + + pIt->getGhostOwnersRef().insert( int_c(rank_) ); + + WALBERLA_LOG_DETAIL( "Sending migration notification for body " << + pIt->getUid() << + " to process " << + (newOwner) ); + + // Send migration notification to new owner + auto& sb( bs1.sendBuffer(newOwner) ); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleMigrationNotification( *pIt )); + + pIt->getGhostOwnersRef().clear(); + + continue; + } + ++pIt; + } + WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate ended." ); + + WALBERLA_LOG_DETAIL( "UM: number of recv " << recvRanks.size()); + bs1.setReceiverInfo(recvRanks, true); + bs1.sendAll(); + WALBERLA_LOG_DETAIL( "UM: number of sends " << bs1.getNumberOfSends()); + + // Receiving the updates for the remote rigid bodies from the connected processes + WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate starts..." ); + ParseMessage parseMessage; + for( auto it = bs1.begin(); it != bs1.end(); ++it ) + { + walberla::uint8_t tmp; + it.buffer() >> tmp; + while( !it.buffer().isEmpty() ) + { + parseMessage(it.rank(), it.buffer(), ps, domain); + } + } + WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate ended." ); +} + +void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const +{ + using namespace walberla::mesa_pd::data::particle_flags; + //========================================================== + // STEP2: Check&Resolve + //========================================================== + + WALBERLA_CHECK(!bs2.isCommunicationRunning()); + + //init buffers + neighborRanks_ = domain.getNeighborProcesses(); + for( uint_t nbProcessRank : neighborRanks_ ) + { + if (bs2.sendBuffer(nbProcessRank).isEmpty()) + { + // fill empty buffers with a dummy byte to force transmission + bs2.sendBuffer(nbProcessRank) << walberla::uint8_c(0); + } + } + bs2.sendBuffer(int_c(rank_)) << walberla::uint8_c(0); + + WALBERLA_LOG_DETAIL( "Assembling of Check&Resolve starts..." ); + + for( auto pIt = ps.begin(); pIt != ps.end(); ) + { + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) + { + ++pIt; + continue; + } + + if (!isSet( pIt->getFlags(), GHOST)) + { + //LOCAL + + walberla::mpi::SendBuffer& sbMaster = bs2.sendBuffer(pIt->getOwner()); + if (sbMaster.isEmpty()) sbMaster << walberla::uint8_c(0); + + // Update (nearest) neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + + // dont send to owner!! + if (pIt->getOwner() == int_c(nbProcessRank)) continue; + // only send to neighbor which do not know this body + if (pIt->getNeighborState().find( int_c(nbProcessRank) ) != pIt->getNeighborState().end()) continue; + + if( domain.intersectsWithProcessSubdomain( nbProcessRank, pIt->getPosition(), pIt->getInteractionRadius() + dx ) ) + { + // no ghost there -> create ghost + WALBERLA_LOG_DETAIL( "Sending copy notification for body " << pIt->getUid() << " to process " << (nbProcessRank) << "\n master: " << pIt->getOwner()); + packNotification(sb, ParticleCopyNotification( *pIt )); + packNotification(sbMaster, NewGhostParticleNotification( *pIt, int_c(nbProcessRank) )); + pIt->getNeighborStateRef().insert( int_c(nbProcessRank) ); + } + } + } else + { + //GHOST + + walberla::mpi::SendBuffer& sbMaster = bs2.sendBuffer(pIt->getOwner()); + if (sbMaster.isEmpty()) sbMaster << walberla::uint8_c(0); + + // Update (nearest) neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + + if (pIt->getOwner() == int_c(nbProcessRank)) continue; // dont send to owner!! + if (pIt->getNeighborState().find( int_c(nbProcessRank) ) != pIt->getNeighborState().end()) continue; // only send to neighbor which do not know this body + + if( domain.intersectsWithProcessSubdomain( nbProcessRank, pIt->getPosition(), pIt->getInteractionRadius() + dx ) ) + { + // no ghost there -> create ghost + WALBERLA_LOG_DETAIL( "Sending copy notification for body " << pIt->getUid() << " to process " << (nbProcessRank) << "\n master: " << pIt->getOwner()); + packNotification(sb, ParticleCopyNotification( *pIt )); + packNotification(sbMaster, NewGhostParticleNotification( *pIt, int_c(nbProcessRank) )); + pIt->getNeighborStateRef().insert( int_c(nbProcessRank) ); + } + } + + if ( !domain.intersectsWithProcessSubdomain(uint_c(rank_), pIt->getPosition(), pIt->getInteractionRadius() + dx) ) + { + // Delete + // inform nearest neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + WALBERLA_LOG_DETAIL( "Sending removal information notification for body " << pIt->getUid() << " to process " << (nbProcessRank) ); + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleRemovalInformationNotification( *pIt )); + } + + //notify owner + WALBERLA_LOG_DETAIL( "Sending removal information notification for body " << pIt->getUid() << " to process " << (pIt->getOwner()) ); + auto& sb = bs2.sendBuffer(pIt->getOwner()); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleRemovalInformationNotification( *pIt )); + + pIt = ps.erase( pIt ); + continue; + } + } + ++pIt; + } + + std::set<walberla::mpi::MPIRank> recvRanks; // potential message senders + // schedule receives + for( auto pIt = ps.begin(); pIt != ps.end(); ++pIt) + { + if (isSet( pIt->getFlags(), GHOST)) continue; + + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) continue; + + for( auto ghostRank : pIt->getGhostOwners() ) + { + recvRanks.insert(ghostRank); + } + } + + for( uint_t nbProcessRank : neighborRanks_ ) + { + recvRanks.insert(int_c(nbProcessRank)); + } + + recvRanks.insert( int_c(rank_) ); + WALBERLA_LOG_DETAIL( "Assembling of Check&Resolve ended." ); + + // size of buffer is unknown and changes with each send + WALBERLA_LOG_DETAIL( "CR: number of recv " << recvRanks.size()); + bs2.setReceiverInfo(recvRanks, true); + bs2.sendAll(); + WALBERLA_LOG_DETAIL( "CR: number of sends " << bs2.getNumberOfSends()); + + // Receiving the updates for the remote rigid bodies from the connected processes + WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve starts..." ); + ParseMessage parseMessage; + for( auto it = bs2.begin(); it != bs2.end(); ++it ) + { + walberla::uint8_t tmp; + it.buffer() >> tmp; + while( !it.buffer().isEmpty() ) + { + parseMessage(it.rank(), it.buffer(), ps, domain); + } + } + WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve ended." ); +} + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla diff --git a/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h new file mode 100644 index 0000000000000000000000000000000000000000..9ae5d924dbecad544239d744c1e43b2acd723c04 --- /dev/null +++ b/python/mesa_pd/templates/mpi/SyncGhostOwners.templ.h @@ -0,0 +1,89 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwners.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/Flags.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/domain/IDomain.h> +#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h> +#include <mesa_pd/mpi/notifications/PackNotification.h> +#include <mesa_pd/mpi/notifications/ParseMessage.h> +#include <mesa_pd/mpi/notifications/ParticleCopyNotification.h> +#include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h> +#include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h> + +#include <core/mpi/BufferSystem.h> +#include <core/logging/Logging.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +/** + * Kernel which updates all ghost particles. + * + * \ingroup mesa_pd_mpi + */ +class SyncGhostOwners +{ +public: + void operator()( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx = real_t(0), + const bool syncNonCommunicatingBodies = false ) const; + + int64_t getBytesSent() const { return bs1.getBytesSent() + bs2.getBytesSent(); } + int64_t getBytesReceived() const { return bs1.getBytesReceived() + bs2.getBytesReceived(); } + + int64_t getNumberOfSends() const { return bs1.getNumberOfSends() + bs2.getNumberOfSends(); } + int64_t getNumberOfReceives() const { return bs1.getNumberOfReceives() + bs2.getNumberOfReceives(); } +private: + void updateAndMigrate( data::ParticleStorage& ps, + const domain::IDomain& domain, + const bool syncNonCommunicatingBodies ) const; + + void checkAndResolveOverlap( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const; + + mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator() + + mutable walberla::mpi::BufferSystem bs1 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 749861); + mutable walberla::mpi::BufferSystem bs2 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 255367); + + int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses(); + int rank_ = walberla::mpi::MPIManager::instance()->rank(); +}; + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla diff --git a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp index 0f3350604f5b058561beb5e68f909d9e74592999..98a3fc09a96120fbf9e4e27bde8e2b4fcde1be7e 100644 --- a/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp +++ b/python/mesa_pd/templates/mpi/SyncNextNeighbors.templ.cpp @@ -26,6 +26,8 @@ #include "SyncNextNeighbors.h" +#include <mesa_pd/mpi/RemoveAndNotify.h> + namespace walberla { namespace mesa_pd { namespace mpi { @@ -66,38 +68,6 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps, WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." ); } -/** - * Removes a particle from the local storage and informs ghost particle holders. - * - * This function removes the particle from the particle storage and generates deletion notifications. - */ -inline -data::ParticleStorage::iterator removeAndNotify( walberla::mpi::BufferSystem& bs, - data::ParticleStorage& ps, - data::ParticleStorage::iterator& pIt ) -{ - WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST), - "Trying to remove ghost particle from the particle storage." ); - - WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL), - "Trying to remove a global particle from the particle storage." ); - - if( !pIt->getGhostOwners().empty() ) - { - // Notify registered processes (intersecting or interacting) of particle removal since they possess a shadow copy. - for( auto ghostRank : pIt->getGhostOwnersRef() ) - { - WALBERLA_LOG_DETAIL( "__Notify registered process " << ghostRank << " of deletion of particle " << pIt->getUid() ); - auto& sb = bs.sendBuffer(ghostRank); - if (sb.isEmpty()) sb << walberla::uint8_c(0); - packNotification(sb, ParticleRemovalNotification( *pIt )); - } - } - - pIt->getGhostOwnersRef().clear(); - return ps.erase( pIt ); -} - void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& ps, const domain::IDomain& domain, const real_t dx) const @@ -130,7 +100,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p for (const auto& ghostOwner : pIt->getGhostOwners() ) { - auto& buffer( bs.sendBuffer(ghostOwner) ); + auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostOwner)) ); WALBERLA_LOG_DETAIL( "Sending removal notification for particle " << pIt->getUid() << " to process " << ghostOwner ); @@ -174,7 +144,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p auto& buffer( bs.sendBuffer(nbProcessRank) ); WALBERLA_LOG_DETAIL( "Sending shadow copy notification for particle " << pIt->getUid() << " to process " << (nbProcessRank) ); packNotification(buffer, ParticleCopyNotification( *pIt )); - pIt->getGhostOwnersRef().emplace_back( int_c(nbProcessRank) ); + pIt->getGhostOwnersRef().insert( int_c(nbProcessRank) ); } } else @@ -232,7 +202,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p for( auto ghostRank : pIt->getGhostOwners() ) { - auto& buffer( bs.sendBuffer(ghostRank) ); + auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)) ); WALBERLA_LOG_DETAIL( "Sending remote migration notification for particle " << pIt->getUid() << " to process " << ghostRank ); @@ -240,7 +210,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p packNotification(buffer, ParticleRemoteMigrationNotification( *pIt, ownerRank )); } - pIt->getGhostOwnersRef().emplace_back( int_c(ownRank) ); + pIt->getGhostOwnersRef().insert( int_c(ownRank) ); // Send migration notification to new owner auto& buffer( bs.sendBuffer(ownerRank) ); diff --git a/python/mesa_pd/templates/mpi/notifications/NewGhostParticleNotification.templ.h b/python/mesa_pd/templates/mpi/notifications/NewGhostParticleNotification.templ.h new file mode 100644 index 0000000000000000000000000000000000000000..7d8a8ce43f68a178090f062a8966e695166832ca --- /dev/null +++ b/python/mesa_pd/templates/mpi/notifications/NewGhostParticleNotification.templ.h @@ -0,0 +1,100 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file NewGhostParticleNotification.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/mpi/notifications/NotificationType.h> + +#include <core/mpi/Datatype.h> +#include <core/mpi/MPIWrapper.h> +#include <core/mpi/RecvBuffer.h> +#include <core/mpi/SendBuffer.h> + +namespace walberla { +namespace mesa_pd { + +/** + * This notification is send to the owner of a particle + * to signal that a new ghost particle exists and the ghost particle list should be updated. + */ +class NewGhostParticleNotification +{ +public: + struct Parameters + { + id_t uid_; + walberla::mpi::MPIRank newOwner_; + }; + + inline explicit NewGhostParticleNotification( const data::Particle& particle, const walberla::mpi::MPIRank newOwner ) + : particle_(particle) + , newOwner_(newOwner) + {} + const data::Particle& particle_; + walberla::mpi::MPIRank newOwner_; +}; + +template<> +struct NotificationTrait<NewGhostParticleNotification> +{ + static const NotificationType id = NEW_GHOST_PARTICLE_NOTIFICATION; +}; + +} // namespace mesa_pd +} // namespace walberla + +//====================================================================================================================== +// +// Send/Recv Buffer Serialization Specialization +// +//====================================================================================================================== + +namespace walberla { +namespace mpi { + +template< typename T, // Element type of SendBuffer + typename G> // Growth policy of SendBuffer +mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::NewGhostParticleNotification& obj ) +{ + buf.addDebugMarker( "cn" ); + buf << obj.particle_.getUid(); + buf << obj.newOwner_; + return buf; +} + +template< typename T> // Element type of RecvBuffer +mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::NewGhostParticleNotification::Parameters& objparam ) +{ + buf.readDebugMarker( "cn" ); + buf >> objparam.uid_; + buf >> objparam.newOwner_; + return buf; +} + +} // mpi +} // walberla diff --git a/python/mesa_pd/templates/mpi/notifications/ParseMessage.templ.h b/python/mesa_pd/templates/mpi/notifications/ParseMessage.templ.h index 3bab14e36192a2788fef39902705731d73eb77cc..b144ddbdde45dfe7412152c864e82353515f7665 100644 --- a/python/mesa_pd/templates/mpi/notifications/ParseMessage.templ.h +++ b/python/mesa_pd/templates/mpi/notifications/ParseMessage.templ.h @@ -29,11 +29,13 @@ #include <mesa_pd/data/ParticleStorage.h> #include <mesa_pd/domain/IDomain.h> +#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h> #include <mesa_pd/mpi/notifications/NotificationType.h> #include <mesa_pd/mpi/notifications/ParticleCopyNotification.h> #include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h> #include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h> #include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h> #include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h> #include <core/debug/Debug.h> @@ -70,14 +72,18 @@ void ParseMessage::operator()(int sender, WALBERLA_LOG_DETAIL( "Received PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "from neighboring process with rank " << sender ); - WALBERLA_CHECK_EQUAL( ps.find(objparam.uid), ps.end(), "Ghost particle with id " << objparam.uid << " already existend."); + if ( ps.find(objparam.uid) == ps.end() ) + { + auto pIt = createNewParticle(ps, objparam); - auto pIt = createNewParticle(ps, objparam); + domain.correctParticlePosition(pIt->getPositionRef()); - domain.correctParticlePosition(pIt->getPositionRef()); - - WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)); - data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST); + //WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)); + data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST); + } else + { + WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend."); + } WALBERLA_LOG_DETAIL( "Processed PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "." ); @@ -185,8 +191,61 @@ void ParseMessage::operator()(int sender, break; } + case NEW_GHOST_PARTICLE_NOTIFICATION: { + NewGhostParticleNotification::Parameters objparam; + rb >> objparam; + + WALBERLA_LOG_DETAIL( "Received new ghost particle notification for particle " << + objparam.uid_ << + " from neighboring process with rank " << + sender << + "." ); + + auto pIt = ps.find( objparam.uid_ ); + WALBERLA_CHECK_UNEQUAL( pIt, ps.end() ); + + pIt->getGhostOwnersRef().insert( objparam.newOwner_ ); + + WALBERLA_LOG_DETAIL( "Processed new ghost particle notification" ); + + break; + } + case PARTICLE_REMOVAL_INFORMATION_NOTIFICATION: { + ParticleRemovalInformationNotification::Parameters objparam; + rb >> objparam; + + WALBERLA_LOG_DETAIL( "Received particle removal information notification for particle " << + objparam.uid_ << + " from neighboring process with rank " << + sender << + "." ); + + if (objparam.owner_ == receiver_) + { + using namespace walberla::mesa_pd::data::particle_flags; + auto pIt = ps.find( objparam.uid_ ); + WALBERLA_CHECK_UNEQUAL( pIt, ps.end() ); + WALBERLA_CHECK(!isSet( pIt->getFlags(), GHOST)); + + pIt->getGhostOwnersRef().erase( sender ); + pIt->getNeighborStateRef().erase( sender ); + } else + { + using namespace walberla::mesa_pd::data::particle_flags; + auto pIt = ps.find( objparam.uid_ ); + if (pIt != ps.end() ) + { + WALBERLA_CHECK(isSet( pIt->getFlags(), GHOST)); + pIt->getNeighborStateRef().erase( sender ); + } + } + + WALBERLA_LOG_DETAIL( "Processed rigid body removal information notification" ); + + break; + } default: - throw std::runtime_error( "Received invalid notification type." ); + WALBERLA_ABORT( "Received invalid notification type: " << notificationType << " from sender: " << sender ); } } diff --git a/python/mesa_pd/templates/mpi/notifications/ParticleRemovalInformationNotification.templ.h b/python/mesa_pd/templates/mpi/notifications/ParticleRemovalInformationNotification.templ.h new file mode 100644 index 0000000000000000000000000000000000000000..15f10d48cbb415941dc1685595509265d291852f --- /dev/null +++ b/python/mesa_pd/templates/mpi/notifications/ParticleRemovalInformationNotification.templ.h @@ -0,0 +1,97 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file ParticleRemovalInformationNotification.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/mpi/notifications/NotificationType.h> + +#include <core/mpi/Datatype.h> +#include <core/mpi/RecvBuffer.h> +#include <core/mpi/SendBuffer.h> + +namespace walberla { +namespace mesa_pd { + +/** + * The ParticleRemovalInformationNotification class is used to signal other processes that a + * shadow copy was destroyed. + */ +class ParticleRemovalInformationNotification +{ +public: + struct Parameters + { + id_t uid_; + walberla::mpi::MPIRank owner_; + }; + + inline explicit ParticleRemovalInformationNotification( const data::Particle& particle ) + : particle_(particle) + {} + const data::Particle& particle_; +}; + +template<> +struct NotificationTrait<ParticleRemovalInformationNotification> +{ + static const NotificationType id = PARTICLE_REMOVAL_INFORMATION_NOTIFICATION; +}; + +} // namespace mesa_pd +} // namespace walberla + +//====================================================================================================================== +// +// Send/Recv Buffer Serialization Specialization +// +//====================================================================================================================== + +namespace walberla { +namespace mpi { + +template< typename T, // Element type of SendBuffer + typename G> // Growth policy of SendBuffer +mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::ParticleRemovalInformationNotification& obj ) +{ + buf.addDebugMarker( "ri" ); + buf << obj.particle_.getUid(); + buf << static_cast<walberla::mpi::MPIRank>(obj.particle_.getOwner()); + return buf; +} + +template< typename T> // Element type of RecvBuffer +mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::ParticleRemovalInformationNotification::Parameters& objparam ) +{ + buf.readDebugMarker( "ri" ); + buf >> objparam.uid_; + buf >> objparam.owner_; + return buf; +} + +} // mpi +} // walberla diff --git a/src/core/mpi/BufferDataTypeExtensions.h b/src/core/mpi/BufferDataTypeExtensions.h index 955c153ec9d067e3ab1c48cebf6c3df52220f8fa..e1ec3755038f6e13c5df4ede2a738a9c17eb1977 100644 --- a/src/core/mpi/BufferDataTypeExtensions.h +++ b/src/core/mpi/BufferDataTypeExtensions.h @@ -37,6 +37,8 @@ #include <map> #include <set> #include <string> +#include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> @@ -411,6 +413,32 @@ GenericRecvBuffer<T>& operator>>( GenericRecvBuffer<T> & buf, std::set<CK, CC, C template<typename T, typename C, typename A> struct BufferSizeTrait< std::set<T,C,A> > { static const bool constantSize = false; }; +template< typename T, // Element type of SendBuffer + typename G, // Growth policy of SendBuffer + typename CK, // Key type + typename CC, // Comparison type + typename CA> // Allocator type +GenericSendBuffer<T,G>& operator<<( GenericSendBuffer<T,G> & buf, const std::unordered_set<CK, CC, CA> & c ) +{ + buf.addDebugMarker( "us" ); + sendAssocContainer(buf, c); + return buf; +} + +template< typename T, // Element type of RecvBuffer + typename CK, // Key type + typename CC, // Comparison type + typename CA> // Allocator type +GenericRecvBuffer<T>& operator>>( GenericRecvBuffer<T> & buf, std::unordered_set<CK, CC, CA> & c ) +{ + buf.readDebugMarker( "us" ); + recvAssocContainer(buf, c); + return buf; +} + +template<typename T, typename C, typename A> +struct BufferSizeTrait< std::unordered_set<T,C,A> > { static const bool constantSize = false; }; + @@ -473,6 +501,34 @@ GenericRecvBuffer<T>& operator>>( GenericRecvBuffer<T> & buf, std::map<CK, CT, C template<typename T, typename K, typename C, typename A> struct BufferSizeTrait< std::map<K,T,C,A> > { static const bool constantSize = false; }; +template< typename T, // Element type of SendBuffer + typename G, // Growth policy of SendBuffer + typename CK, // Key type + typename CT, // Element type + typename CC, // Comparison type + typename CA> // Allocator type +GenericSendBuffer<T,G>& operator<<( GenericSendBuffer<T,G> & buf, const std::unordered_map<CK, CT, CC, CA> & c ) +{ + buf.addDebugMarker( "um" ); + sendAssocContainer(buf, c); + return buf; +} + +template< typename T, // Element type of RecvBuffer + typename CK, // Key type + typename CT, // Element type + typename CC, // Comparison type + typename CA> // Allocator type +GenericRecvBuffer<T>& operator>>( GenericRecvBuffer<T> & buf, std::unordered_map<CK, CT, CC, CA> & c ) +{ + buf.readDebugMarker( "um" ); + recvMap(buf, c); + return buf; +} + +template<typename T, typename K, typename C, typename A> +struct BufferSizeTrait< std::unordered_map<K,T,C,A> > { static const bool constantSize = false; }; + diff --git a/src/core/mpi/BufferSystem.h b/src/core/mpi/BufferSystem.h index e4ba192e59876f68b3cf4d26d5da3fe6a10e9574..8be65ea4a355644a39381eb29944375010fbc4f5 100644 --- a/src/core/mpi/BufferSystem.h +++ b/src/core/mpi/BufferSystem.h @@ -189,7 +189,7 @@ public: /*! \name Status Queries */ //@{ bool isSizeCommunicatedInNextStep() const { return (currentComm_ == &unknownSizeComm_); } - bool isCommunciationRunning() const { return communicationRunning_; } + bool isCommunicationRunning() const { return communicationRunning_; } bool isReceiverInformationSet() const { return currentComm_ != NULL; } //@} //******************************************************************************************************************* diff --git a/src/core/mpi/OpenMPBufferSystem.impl.h b/src/core/mpi/OpenMPBufferSystem.impl.h index dc042e8f44d870ff02d513938006c0ad981579dd..dd0efe661cc0598d802627ebfda64952dbd4e640 100644 --- a/src/core/mpi/OpenMPBufferSystem.impl.h +++ b/src/core/mpi/OpenMPBufferSystem.impl.h @@ -210,7 +210,7 @@ void GenericOpenMPBufferSystem<Rb, Sb>::waitOpenMP() WALBERLA_ASSERT_NULLPTR( ret ); // call last time to finish communication WALBERLA_UNUSED( ret ); - WALBERLA_ASSERT( ! bs_.isCommunciationRunning() ); + WALBERLA_ASSERT( ! bs_.isCommunicationRunning() ); } diff --git a/src/mesa_pd/data/ParticleAccessor.h b/src/mesa_pd/data/ParticleAccessor.h index 38c316743621c81e628312c8d3ba76d67935f3c3..1f85d7348ae73806abf3529c4b65e2c8c203fceb 100644 --- a/src/mesa_pd/data/ParticleAccessor.h +++ b/src/mesa_pd/data/ParticleAccessor.h @@ -68,9 +68,9 @@ public: int& getOwnerRef(const size_t p_idx) {return ps_->getOwnerRef(p_idx);} void setOwner(const size_t p_idx, const int& v) { ps_->setOwner(p_idx, v);} - const std::vector<int>& getGhostOwners(const size_t p_idx) const {return ps_->getGhostOwners(p_idx);} - std::vector<int>& getGhostOwnersRef(const size_t p_idx) {return ps_->getGhostOwnersRef(p_idx);} - void setGhostOwners(const size_t p_idx, const std::vector<int>& v) { ps_->setGhostOwners(p_idx, v);} + const std::unordered_set<walberla::mpi::MPIRank>& getGhostOwners(const size_t p_idx) const {return ps_->getGhostOwners(p_idx);} + std::unordered_set<walberla::mpi::MPIRank>& getGhostOwnersRef(const size_t p_idx) {return ps_->getGhostOwnersRef(p_idx);} + void setGhostOwners(const size_t p_idx, const std::unordered_set<walberla::mpi::MPIRank>& v) { ps_->setGhostOwners(p_idx, v);} const size_t& getShapeID(const size_t p_idx) const {return ps_->getShapeID(p_idx);} size_t& getShapeIDRef(const size_t p_idx) {return ps_->getShapeIDRef(p_idx);} @@ -140,6 +140,10 @@ public: walberla::mesa_pd::Vec3& getDwRef(const size_t p_idx) {return ps_->getDwRef(p_idx);} void setDw(const size_t p_idx, const walberla::mesa_pd::Vec3& v) { ps_->setDw(p_idx, v);} + const std::unordered_set<walberla::mpi::MPIRank>& getNeighborState(const size_t p_idx) const {return ps_->getNeighborState(p_idx);} + std::unordered_set<walberla::mpi::MPIRank>& getNeighborStateRef(const size_t p_idx) {return ps_->getNeighborStateRef(p_idx);} + void setNeighborState(const size_t p_idx, const std::unordered_set<walberla::mpi::MPIRank>& v) { ps_->setNeighborState(p_idx, v);} + id_t getInvalidUid() const {return UniqueID<data::Particle>::invalidID();} size_t getInvalidIdx() const {return std::numeric_limits<size_t>::max();} @@ -205,9 +209,9 @@ public: void setOwner(const size_t /*p_idx*/, const int& v) { owner_ = v;} int& getOwnerRef(const size_t /*p_idx*/) {return owner_;} - const std::vector<int>& getGhostOwners(const size_t /*p_idx*/) const {return ghostOwners_;} - void setGhostOwners(const size_t /*p_idx*/, const std::vector<int>& v) { ghostOwners_ = v;} - std::vector<int>& getGhostOwnersRef(const size_t /*p_idx*/) {return ghostOwners_;} + const std::unordered_set<walberla::mpi::MPIRank>& getGhostOwners(const size_t /*p_idx*/) const {return ghostOwners_;} + void setGhostOwners(const size_t /*p_idx*/, const std::unordered_set<walberla::mpi::MPIRank>& v) { ghostOwners_ = v;} + std::unordered_set<walberla::mpi::MPIRank>& getGhostOwnersRef(const size_t /*p_idx*/) {return ghostOwners_;} const size_t& getShapeID(const size_t /*p_idx*/) const {return shapeID_;} void setShapeID(const size_t /*p_idx*/, const size_t& v) { shapeID_ = v;} @@ -277,6 +281,10 @@ public: void setDw(const size_t /*p_idx*/, const walberla::mesa_pd::Vec3& v) { dw_ = v;} walberla::mesa_pd::Vec3& getDwRef(const size_t /*p_idx*/) {return dw_;} + const std::unordered_set<walberla::mpi::MPIRank>& getNeighborState(const size_t /*p_idx*/) const {return neighborState_;} + void setNeighborState(const size_t /*p_idx*/, const std::unordered_set<walberla::mpi::MPIRank>& v) { neighborState_ = v;} + std::unordered_set<walberla::mpi::MPIRank>& getNeighborStateRef(const size_t /*p_idx*/) {return neighborState_;} + id_t getInvalidUid() const {return UniqueID<data::Particle>::invalidID();} size_t getInvalidIdx() const {return std::numeric_limits<size_t>::max();} @@ -293,7 +301,7 @@ private: walberla::real_t interactionRadius_; walberla::mesa_pd::data::particle_flags::FlagT flags_; int owner_; - std::vector<int> ghostOwners_; + std::unordered_set<walberla::mpi::MPIRank> ghostOwners_; size_t shapeID_; walberla::mesa_pd::Rot3 rotation_; walberla::mesa_pd::Vec3 angularVelocity_; @@ -311,6 +319,7 @@ private: walberla::real_t heatFlux_; walberla::mesa_pd::Vec3 dv_; walberla::mesa_pd::Vec3 dw_; + std::unordered_set<walberla::mpi::MPIRank> neighborState_; }; } //namespace data diff --git a/src/mesa_pd/data/ParticleStorage.h b/src/mesa_pd/data/ParticleStorage.h index 942b475ce7f569a8da16a71a5b4489f406159325..e4d766318a1b8de41ac884e1b8483e24a08b4a19 100644 --- a/src/mesa_pd/data/ParticleStorage.h +++ b/src/mesa_pd/data/ParticleStorage.h @@ -31,6 +31,7 @@ #include <map> #include <type_traits> #include <unordered_map> +#include <unordered_set> #include <vector> #include <mesa_pd/data/ContactHistory.h> @@ -42,6 +43,7 @@ #include <core/Abort.h> #include <core/debug/Debug.h> #include <core/math/AABB.h> +#include <core/mpi/MPIWrapper.h> #include <core/OpenMP.h> #include <core/STLIO.h> #include <core/UniqueID.h> @@ -88,9 +90,9 @@ public: int& getOwnerRef() {return storage_.getOwnerRef(i_);} void setOwner(const int& v) { storage_.setOwner(i_, v);} - const std::vector<int>& getGhostOwners() const {return storage_.getGhostOwners(i_);} - std::vector<int>& getGhostOwnersRef() {return storage_.getGhostOwnersRef(i_);} - void setGhostOwners(const std::vector<int>& v) { storage_.setGhostOwners(i_, v);} + const std::unordered_set<walberla::mpi::MPIRank>& getGhostOwners() const {return storage_.getGhostOwners(i_);} + std::unordered_set<walberla::mpi::MPIRank>& getGhostOwnersRef() {return storage_.getGhostOwnersRef(i_);} + void setGhostOwners(const std::unordered_set<walberla::mpi::MPIRank>& v) { storage_.setGhostOwners(i_, v);} const size_t& getShapeID() const {return storage_.getShapeID(i_);} size_t& getShapeIDRef() {return storage_.getShapeIDRef(i_);} @@ -160,6 +162,10 @@ public: walberla::mesa_pd::Vec3& getDwRef() {return storage_.getDwRef(i_);} void setDw(const walberla::mesa_pd::Vec3& v) { storage_.setDw(i_, v);} + const std::unordered_set<walberla::mpi::MPIRank>& getNeighborState() const {return storage_.getNeighborState(i_);} + std::unordered_set<walberla::mpi::MPIRank>& getNeighborStateRef() {return storage_.getNeighborStateRef(i_);} + void setNeighborState(const std::unordered_set<walberla::mpi::MPIRank>& v) { storage_.setNeighborState(i_, v);} + size_t getIdx() const {return i_;} public: @@ -240,9 +246,9 @@ public: int& getOwnerRef(const size_t idx) {return owner_[idx];} void setOwner(const size_t idx, const int& v) { owner_[idx] = v; } - const std::vector<int>& getGhostOwners(const size_t idx) const {return ghostOwners_[idx];} - std::vector<int>& getGhostOwnersRef(const size_t idx) {return ghostOwners_[idx];} - void setGhostOwners(const size_t idx, const std::vector<int>& v) { ghostOwners_[idx] = v; } + const std::unordered_set<walberla::mpi::MPIRank>& getGhostOwners(const size_t idx) const {return ghostOwners_[idx];} + std::unordered_set<walberla::mpi::MPIRank>& getGhostOwnersRef(const size_t idx) {return ghostOwners_[idx];} + void setGhostOwners(const size_t idx, const std::unordered_set<walberla::mpi::MPIRank>& v) { ghostOwners_[idx] = v; } const size_t& getShapeID(const size_t idx) const {return shapeID_[idx];} size_t& getShapeIDRef(const size_t idx) {return shapeID_[idx];} @@ -312,6 +318,10 @@ public: walberla::mesa_pd::Vec3& getDwRef(const size_t idx) {return dw_[idx];} void setDw(const size_t idx, const walberla::mesa_pd::Vec3& v) { dw_[idx] = v; } + const std::unordered_set<walberla::mpi::MPIRank>& getNeighborState(const size_t idx) const {return neighborState_[idx];} + std::unordered_set<walberla::mpi::MPIRank>& getNeighborStateRef(const size_t idx) {return neighborState_[idx];} + void setNeighborState(const size_t idx, const std::unordered_set<walberla::mpi::MPIRank>& v) { neighborState_[idx] = v; } + /** * @brief creates a new particle and returns an iterator pointing to it @@ -406,7 +416,7 @@ public: std::vector<walberla::real_t> interactionRadius_ {}; std::vector<walberla::mesa_pd::data::particle_flags::FlagT> flags_ {}; std::vector<int> owner_ {}; - std::vector<std::vector<int>> ghostOwners_ {}; + std::vector<std::unordered_set<walberla::mpi::MPIRank>> ghostOwners_ {}; std::vector<size_t> shapeID_ {}; std::vector<walberla::mesa_pd::Rot3> rotation_ {}; std::vector<walberla::mesa_pd::Vec3> angularVelocity_ {}; @@ -424,6 +434,7 @@ public: std::vector<walberla::real_t> heatFlux_ {}; std::vector<walberla::mesa_pd::Vec3> dv_ {}; std::vector<walberla::mesa_pd::Vec3> dw_ {}; + std::vector<std::unordered_set<walberla::mpi::MPIRank>> neighborState_ {}; std::unordered_map<id_t, size_t> uidToIdx_; static_assert(std::is_same<decltype(uid_)::value_type, id_t>::value, "Property uid of type id_t is missing. This property is required!"); @@ -456,6 +467,7 @@ ParticleStorage::Particle& ParticleStorage::Particle::operator=(const ParticleSt getHeatFluxRef() = rhs.getHeatFlux(); getDvRef() = rhs.getDv(); getDwRef() = rhs.getDw(); + getNeighborStateRef() = rhs.getNeighborState(); return *this; } @@ -485,6 +497,7 @@ ParticleStorage::Particle& ParticleStorage::Particle::operator=(ParticleStorage: getHeatFluxRef() = std::move(rhs.getHeatFluxRef()); getDvRef() = std::move(rhs.getDvRef()); getDwRef() = std::move(rhs.getDwRef()); + getNeighborStateRef() = std::move(rhs.getNeighborStateRef()); return *this; } @@ -516,6 +529,7 @@ std::ostream& operator<<( std::ostream& os, const ParticleStorage::Particle& p ) "heatFlux : " << p.getHeatFlux() << "\n" << "dv : " << p.getDv() << "\n" << "dw : " << p.getDw() << "\n" << + "neighborState : " << p.getNeighborState() << "\n" << "================================" << std::endl; return os; } @@ -616,6 +630,7 @@ inline ParticleStorage::iterator ParticleStorage::create(const id_t& uid) heatFlux_.emplace_back(real_t(0)); dv_.emplace_back(real_t(0)); dw_.emplace_back(real_t(0)); + neighborState_.emplace_back(); uid_.back() = uid; uidToIdx_[uid] = uid_.size() - 1; return iterator(this, size() - 1); @@ -671,6 +686,7 @@ inline ParticleStorage::iterator ParticleStorage::erase(iterator& it) heatFlux_.pop_back(); dv_.pop_back(); dw_.pop_back(); + neighborState_.pop_back(); return it; } @@ -713,6 +729,7 @@ inline void ParticleStorage::reserve(const size_t size) heatFlux_.reserve(size); dv_.reserve(size); dw_.reserve(size); + neighborState_.reserve(size); } inline void ParticleStorage::clear() @@ -740,6 +757,7 @@ inline void ParticleStorage::clear() heatFlux_.clear(); dv_.clear(); dw_.clear(); + neighborState_.clear(); uidToIdx_.clear(); } @@ -768,6 +786,7 @@ inline size_t ParticleStorage::size() const //WALBERLA_ASSERT_EQUAL( uid_.size(), heatFlux.size() ); //WALBERLA_ASSERT_EQUAL( uid_.size(), dv.size() ); //WALBERLA_ASSERT_EQUAL( uid_.size(), dw.size() ); + //WALBERLA_ASSERT_EQUAL( uid_.size(), neighborState.size() ); return uid_.size(); } template <typename Selector, typename Accessor, typename Func, typename... Args> @@ -959,10 +978,10 @@ public: class SelectParticleGhostOwners { public: - using return_type = std::vector<int>; - std::vector<int>& operator()(data::Particle& p) const {return p.getGhostOwnersRef();} - std::vector<int>& operator()(data::Particle&& p) const {return p.getGhostOwnersRef();} - const std::vector<int>& operator()(const data::Particle& p) const {return p.getGhostOwners();} + using return_type = std::unordered_set<walberla::mpi::MPIRank>; + std::unordered_set<walberla::mpi::MPIRank>& operator()(data::Particle& p) const {return p.getGhostOwnersRef();} + std::unordered_set<walberla::mpi::MPIRank>& operator()(data::Particle&& p) const {return p.getGhostOwnersRef();} + const std::unordered_set<walberla::mpi::MPIRank>& operator()(const data::Particle& p) const {return p.getGhostOwners();} }; ///Predicate that selects a certain property from a Particle class SelectParticleShapeID @@ -1117,6 +1136,15 @@ public: walberla::mesa_pd::Vec3& operator()(data::Particle&& p) const {return p.getDwRef();} const walberla::mesa_pd::Vec3& operator()(const data::Particle& p) const {return p.getDw();} }; +///Predicate that selects a certain property from a Particle +class SelectParticleNeighborState +{ +public: + using return_type = std::unordered_set<walberla::mpi::MPIRank>; + std::unordered_set<walberla::mpi::MPIRank>& operator()(data::Particle& p) const {return p.getNeighborStateRef();} + std::unordered_set<walberla::mpi::MPIRank>& operator()(data::Particle&& p) const {return p.getNeighborStateRef();} + const std::unordered_set<walberla::mpi::MPIRank>& operator()(const data::Particle& p) const {return p.getNeighborState();} +}; } //namespace data } //namespace mesa_pd diff --git a/src/mesa_pd/data/STLOverloads.h b/src/mesa_pd/data/STLOverloads.h index 28cf24885834109da472672f9e9c409f52682c4f..7a929e54c79fdc9af2adaae9a9aabeb8477cd8c1 100644 --- a/src/mesa_pd/data/STLOverloads.h +++ b/src/mesa_pd/data/STLOverloads.h @@ -23,6 +23,9 @@ #include <mesa_pd/data/DataTypes.h> #include <map> +#include <set> +#include <unordered_set> +#include <unordered_map> #include <vector> namespace walberla { @@ -55,6 +58,42 @@ std::ostream& operator<<( std::ostream& os, const std::map<Key, T, Compare, Allo return os; } +template< typename Key, typename Compare, typename Allocator > +std::ostream& operator<<( std::ostream& os, const std::set<Key, Compare, Allocator>& m ) +{ + os << "{"; + for (auto& v : m) + { + os << v << ", "; + } + os << "}"; + return os; +} + +template< typename Key, typename T, typename Compare, typename Allocator > +std::ostream& operator<<( std::ostream& os, const std::unordered_map<Key, T, Compare, Allocator>& m ) +{ + os << "{"; + for (auto& v : m) + { + os << v.first << ":" << v.second << ", "; + } + os << "}"; + return os; +} + +template< typename Key, typename Compare, typename Allocator > +std::ostream& operator<<( std::ostream& os, const std::unordered_set<Key, Compare, Allocator>& m ) +{ + os << "{"; + for (auto& v : m) + { + os << v << ", "; + } + os << "}"; + return os; +} + } //namespace data } //namespace mesa_pd } //namespace walberla diff --git a/src/mesa_pd/domain/BlockForestDomain.cpp b/src/mesa_pd/domain/BlockForestDomain.cpp index 483eaa9b8db983289a37e5da1b5713c01e845c9f..4339671d40ed67464635cd3957c2bd028970b6c7 100644 --- a/src/mesa_pd/domain/BlockForestDomain.cpp +++ b/src/mesa_pd/domain/BlockForestDomain.cpp @@ -175,43 +175,63 @@ bool BlockForestDomain::intersectsWithProcessSubdomain(const uint_t rank, const { if (blockForest_->empty()) return false; - WALBERLA_ASSERT(std::is_sorted(neighborSubdomains_.begin(), - neighborSubdomains_.end(), - [](const auto& lhs, const auto& rhs){ return lhs.rank < rhs.rank;})); - - WALBERLA_CHECK_UNEQUAL(uint_c(ownRank_), rank, "checking own domain is currently not implemented"); - - if (isInsideGlobalDomain(pt, radius)) + if (uint_c(ownRank_) == rank) { - size_t idx = 0; - WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); - while (neighborSubdomains_[idx].rank != int_c(rank)) + //===================== + // LOCAL DOMAIN + if (isInsideGlobalDomain(pt, radius)) { - ++idx; - WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); - } - while (neighborSubdomains_[idx].rank == int_c(rank)) + for (auto& aabb : localAABBs_) + { + if (sqDistancePointToAABB(pt, aabb) <= radius * radius) return true; + } + } else { - if (sqDistancePointToAABB(pt, neighborSubdomains_[idx].aabb) <= radius * radius) return true; - ++idx; - if (idx >= neighborSubdomains_.size()) break; - WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + for (auto& aabb : localAABBs_) + { + if (sqDistancePointToAABBPeriodic(pt, aabb, blockForest_->getDomain(), periodic_) <= radius * radius) return true; + } } } else { - size_t idx = 0; - WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); - while (neighborSubdomains_[idx].rank != int_c(rank)) + //===================== + // NEIGHBORING DOMAIN + WALBERLA_ASSERT(std::is_sorted(neighborSubdomains_.begin(), + neighborSubdomains_.end(), + [](const auto& lhs, const auto& rhs){ return lhs.rank < rhs.rank;})); + + if (isInsideGlobalDomain(pt, radius)) { - ++idx; + size_t idx = 0; WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); - } - while (neighborSubdomains_[idx].rank == int_c(rank)) + while (neighborSubdomains_[idx].rank != int_c(rank)) + { + ++idx; + WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + } + while (neighborSubdomains_[idx].rank == int_c(rank)) + { + if (sqDistancePointToAABB(pt, neighborSubdomains_[idx].aabb) <= radius * radius) return true; + ++idx; + if (idx >= neighborSubdomains_.size()) break; + WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + } + } else { - if (sqDistancePointToAABBPeriodic(pt, neighborSubdomains_[idx].aabb, blockForest_->getDomain(), periodic_) <= radius * radius) return true; - ++idx; - if (idx >= neighborSubdomains_.size()) break; + size_t idx = 0; WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + while (neighborSubdomains_[idx].rank != int_c(rank)) + { + ++idx; + WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + } + while (neighborSubdomains_[idx].rank == int_c(rank)) + { + if (sqDistancePointToAABBPeriodic(pt, neighborSubdomains_[idx].aabb, blockForest_->getDomain(), periodic_) <= radius * radius) return true; + ++idx; + if (idx >= neighborSubdomains_.size()) break; + WALBERLA_ASSERT_LESS(idx, neighborSubdomains_.size()); + } } } diff --git a/src/mesa_pd/mpi/RemoveAndNotify.cpp b/src/mesa_pd/mpi/RemoveAndNotify.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c04a40f949d3149dfaa51f16ebed7d1335fa1c7e --- /dev/null +++ b/src/mesa_pd/mpi/RemoveAndNotify.cpp @@ -0,0 +1,65 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file RemoveAndNotify.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include "RemoveAndNotify.h" + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/Flags.h> +#include <mesa_pd/mpi/notifications/PackNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +/** + * Removes a particle from the local storage and informs ghost particle holders. + * + * This function removes the particle from the particle storage and generates deletion notifications. + */ +data::ParticleStorage::iterator removeAndNotify( walberla::mpi::BufferSystem& bs, + data::ParticleStorage& ps, + data::ParticleStorage::iterator& pIt ) +{ + WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST), + "Trying to remove ghost particle from the particle storage." ); + + WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL), + "Trying to remove a global particle from the particle storage." ); + + if( !pIt->getGhostOwners().empty() ) + { + // Notify registered processes (intersecting or interacting) of particle removal since they possess a shadow copy. + for( auto ghostRank : pIt->getGhostOwnersRef() ) + { + WALBERLA_LOG_DETAIL( "__Notify registered process " << ghostRank << " of deletion of particle " << pIt->getUid() ); + auto& sb = bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleRemovalNotification( *pIt )); + } + } + + pIt->getGhostOwnersRef().clear(); + return ps.erase( pIt ); +} + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla diff --git a/src/mesa_pd/mpi/RemoveAndNotify.h b/src/mesa_pd/mpi/RemoveAndNotify.h new file mode 100644 index 0000000000000000000000000000000000000000..90cf8d3f1d24ce6e165ca06bfa48d1767bebbce3 --- /dev/null +++ b/src/mesa_pd/mpi/RemoveAndNotify.h @@ -0,0 +1,36 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file RemoveAndNotify.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/ParticleStorage.h> +#include <core/mpi/BufferSystem.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +data::ParticleStorage::iterator removeAndNotify( walberla::mpi::BufferSystem& bs, + data::ParticleStorage& ps, + data::ParticleStorage::iterator& pIt ); + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla diff --git a/src/mesa_pd/mpi/SyncGhostOwners.cpp b/src/mesa_pd/mpi/SyncGhostOwners.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bedc11c5f099fec92c9952f8bda1a557c97b4db6 --- /dev/null +++ b/src/mesa_pd/mpi/SyncGhostOwners.cpp @@ -0,0 +1,355 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwners.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#include "SyncGhostOwners.h" + +#include <mesa_pd/mpi/RemoveAndNotify.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +void SyncGhostOwners::operator()( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const +{ + if (numProcesses_ == 1) return; + + //========================================================== + // STEP1: Update & Migrate + //========================================================== + updateAndMigrate( ps, domain, syncNonCommunicatingBodies ); + + //========================================================== + // STEP2: Check & Resolve + //========================================================== + checkAndResolveOverlap( ps, domain, dx, syncNonCommunicatingBodies ); +} + +void SyncGhostOwners::updateAndMigrate( data::ParticleStorage& ps, + const domain::IDomain& domain, + const bool syncNonCommunicatingBodies ) const +{ + using namespace walberla::mesa_pd::data::particle_flags; + //========================================================== + // STEP1: Update & Migrate + //========================================================== + + WALBERLA_CHECK(!bs1.isCommunicationRunning()); + + WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate starts..." ); + std::set<walberla::mpi::MPIRank> recvRanks; // potential message senders + for( auto pIt = ps.begin(); pIt != ps.end(); ++pIt) + { + if (isSet( pIt->getFlags(), GHOST)) + { + if (!isSet( pIt->getFlags(), NON_COMMUNICATING) || syncNonCommunicatingBodies) + { + recvRanks.insert(pIt->getOwner()); + } + } + } + + for( auto pIt = ps.begin(); pIt != ps.end(); ) + { + if (isSet( pIt->getFlags(), GHOST)) + { + ++pIt; + continue; + } + + //================== + // LOCAL + + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) + { + ++pIt; + continue; + } + + //correct position to make sure particle is always inside the domain! + //everything is decided by the master particle therefore ghost particles are not touched + if (!data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::FIXED) && + !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST)) + { + domain.periodicallyMapToDomain( pIt->getPositionRef() ); + } + + // Update + for (auto ghostOwner : pIt->getGhostOwners()) + { + WALBERLA_LOG_DETAIL( "Sending update notification for body " << pIt->getUid() << " to process " << ghostOwner ); + walberla::mpi::SendBuffer& sb = bs1.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostOwner)); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleUpdateNotification( *pIt )); + } + + //particle has left subdomain? + const auto newOwner = domain.findContainingProcessRank( pIt->getPosition() ); + if( newOwner != int_c(rank_) ) + { + if ( newOwner < 0) + { + // No owner found: Outflow condition. + WALBERLA_LOG_DETAIL( "Sending deletion notifications for body " << pIt->getUid() << " due to outflow." ); + + //delete body + pIt = removeAndNotify( bs1, ps, pIt ); + + continue; + } + + // Set new owner and transform to shadow copy + pIt->setOwner( newOwner ); + set( pIt->getFlagsRef(), GHOST ); + + // currently position is mapped to periodically to global domain, + // this might not be the correct position for a ghost particle + domain.correctParticlePosition( pIt->getPositionRef() ); + + // Correct registration list (exclude new owner and us - the old owner) and + // notify registered processes (except for new owner) of (remote) migration since they possess a ghost particle. + auto ownerIt = std::find( pIt->getGhostOwners().begin(), pIt->getGhostOwners().end(), newOwner ); + WALBERLA_CHECK_UNEQUAL(ownerIt, pIt->getGhostOwners().end(), "New owner has to be former ghost owner!" ); + + pIt->getGhostOwnersRef().erase( ownerIt ); + + // Send remote migration notifications + for( auto ghostRank : pIt->getGhostOwners() ) + { + auto& buffer( bs1.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)) ); + if (buffer.isEmpty()) buffer << walberla::uint8_c(0); + + WALBERLA_LOG_DETAIL( "Sending remote migration notification for particle " << + pIt->getUid() << + " to process " << + ghostRank ); + + packNotification(buffer, ParticleRemoteMigrationNotification( *pIt, newOwner )); + } + + pIt->getGhostOwnersRef().insert( int_c(rank_) ); + + WALBERLA_LOG_DETAIL( "Sending migration notification for body " << + pIt->getUid() << + " to process " << + (newOwner) ); + + // Send migration notification to new owner + auto& sb( bs1.sendBuffer(newOwner) ); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleMigrationNotification( *pIt )); + + pIt->getGhostOwnersRef().clear(); + + continue; + } + ++pIt; + } + WALBERLA_LOG_DETAIL( "Assembling of Update&Migrate ended." ); + + WALBERLA_LOG_DETAIL( "UM: number of recv " << recvRanks.size()); + bs1.setReceiverInfo(recvRanks, true); + bs1.sendAll(); + WALBERLA_LOG_DETAIL( "UM: number of sends " << bs1.getNumberOfSends()); + + // Receiving the updates for the remote rigid bodies from the connected processes + WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate starts..." ); + ParseMessage parseMessage; + for( auto it = bs1.begin(); it != bs1.end(); ++it ) + { + walberla::uint8_t tmp; + it.buffer() >> tmp; + while( !it.buffer().isEmpty() ) + { + parseMessage(it.rank(), it.buffer(), ps, domain); + } + } + WALBERLA_LOG_DETAIL( "Parsing of Update&Migrate ended." ); +} + +void SyncGhostOwners::checkAndResolveOverlap( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const +{ + using namespace walberla::mesa_pd::data::particle_flags; + //========================================================== + // STEP2: Check&Resolve + //========================================================== + + WALBERLA_CHECK(!bs2.isCommunicationRunning()); + + //init buffers + neighborRanks_ = domain.getNeighborProcesses(); + for( uint_t nbProcessRank : neighborRanks_ ) + { + if (bs2.sendBuffer(nbProcessRank).isEmpty()) + { + // fill empty buffers with a dummy byte to force transmission + bs2.sendBuffer(nbProcessRank) << walberla::uint8_c(0); + } + } + bs2.sendBuffer(int_c(rank_)) << walberla::uint8_c(0); + + WALBERLA_LOG_DETAIL( "Assembling of Check&Resolve starts..." ); + + for( auto pIt = ps.begin(); pIt != ps.end(); ) + { + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) + { + ++pIt; + continue; + } + + if (!isSet( pIt->getFlags(), GHOST)) + { + //LOCAL + + walberla::mpi::SendBuffer& sbMaster = bs2.sendBuffer(pIt->getOwner()); + if (sbMaster.isEmpty()) sbMaster << walberla::uint8_c(0); + + // Update (nearest) neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + + // dont send to owner!! + if (pIt->getOwner() == int_c(nbProcessRank)) continue; + // only send to neighbor which do not know this body + if (pIt->getNeighborState().find( int_c(nbProcessRank) ) != pIt->getNeighborState().end()) continue; + + if( domain.intersectsWithProcessSubdomain( nbProcessRank, pIt->getPosition(), pIt->getInteractionRadius() + dx ) ) + { + // no ghost there -> create ghost + WALBERLA_LOG_DETAIL( "Sending copy notification for body " << pIt->getUid() << " to process " << (nbProcessRank) << "\n master: " << pIt->getOwner()); + packNotification(sb, ParticleCopyNotification( *pIt )); + packNotification(sbMaster, NewGhostParticleNotification( *pIt, int_c(nbProcessRank) )); + pIt->getNeighborStateRef().insert( int_c(nbProcessRank) ); + } + } + } else + { + //GHOST + + walberla::mpi::SendBuffer& sbMaster = bs2.sendBuffer(pIt->getOwner()); + if (sbMaster.isEmpty()) sbMaster << walberla::uint8_c(0); + + // Update (nearest) neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + + if (pIt->getOwner() == int_c(nbProcessRank)) continue; // dont send to owner!! + if (pIt->getNeighborState().find( int_c(nbProcessRank) ) != pIt->getNeighborState().end()) continue; // only send to neighbor which do not know this body + + if( domain.intersectsWithProcessSubdomain( nbProcessRank, pIt->getPosition(), pIt->getInteractionRadius() + dx ) ) + { + // no ghost there -> create ghost + WALBERLA_LOG_DETAIL( "Sending copy notification for body " << pIt->getUid() << " to process " << (nbProcessRank) << "\n master: " << pIt->getOwner()); + packNotification(sb, ParticleCopyNotification( *pIt )); + packNotification(sbMaster, NewGhostParticleNotification( *pIt, int_c(nbProcessRank) )); + pIt->getNeighborStateRef().insert( int_c(nbProcessRank) ); + } + } + + if ( !domain.intersectsWithProcessSubdomain(uint_c(rank_), pIt->getPosition(), pIt->getInteractionRadius() + dx) ) + { + // Delete + // inform nearest neighbor processes. + for( uint_t nbProcessRank : neighborRanks_ ) + { + WALBERLA_LOG_DETAIL( "Sending removal information notification for body " << pIt->getUid() << " to process " << (nbProcessRank) ); + auto& sb = bs2.sendBuffer(nbProcessRank); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleRemovalInformationNotification( *pIt )); + } + + //notify owner + WALBERLA_LOG_DETAIL( "Sending removal information notification for body " << pIt->getUid() << " to process " << (pIt->getOwner()) ); + auto& sb = bs2.sendBuffer(pIt->getOwner()); + if (sb.isEmpty()) sb << walberla::uint8_c(0); + packNotification(sb, ParticleRemovalInformationNotification( *pIt )); + + pIt = ps.erase( pIt ); + continue; + } + } + ++pIt; + } + + std::set<walberla::mpi::MPIRank> recvRanks; // potential message senders + // schedule receives + for( auto pIt = ps.begin(); pIt != ps.end(); ++pIt) + { + if (isSet( pIt->getFlags(), GHOST)) continue; + + //skip all particles that do not communicate (create ghost particles) on other processes + if (isSet( pIt->getFlags(), NON_COMMUNICATING) && !syncNonCommunicatingBodies) continue; + + for( auto ghostRank : pIt->getGhostOwners() ) + { + recvRanks.insert(ghostRank); + } + } + + for( uint_t nbProcessRank : neighborRanks_ ) + { + recvRanks.insert(int_c(nbProcessRank)); + } + + recvRanks.insert( int_c(rank_) ); + WALBERLA_LOG_DETAIL( "Assembling of Check&Resolve ended." ); + + // size of buffer is unknown and changes with each send + WALBERLA_LOG_DETAIL( "CR: number of recv " << recvRanks.size()); + bs2.setReceiverInfo(recvRanks, true); + bs2.sendAll(); + WALBERLA_LOG_DETAIL( "CR: number of sends " << bs2.getNumberOfSends()); + + // Receiving the updates for the remote rigid bodies from the connected processes + WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve starts..." ); + ParseMessage parseMessage; + for( auto it = bs2.begin(); it != bs2.end(); ++it ) + { + walberla::uint8_t tmp; + it.buffer() >> tmp; + while( !it.buffer().isEmpty() ) + { + parseMessage(it.rank(), it.buffer(), ps, domain); + } + } + WALBERLA_LOG_DETAIL( "Parsing of Check&Resolve ended." ); +} + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla \ No newline at end of file diff --git a/src/mesa_pd/mpi/SyncGhostOwners.h b/src/mesa_pd/mpi/SyncGhostOwners.h new file mode 100644 index 0000000000000000000000000000000000000000..17bcdde54912bdaaeb0a60bf584f67d72590018a --- /dev/null +++ b/src/mesa_pd/mpi/SyncGhostOwners.h @@ -0,0 +1,89 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwners.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/Flags.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/domain/IDomain.h> +#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h> +#include <mesa_pd/mpi/notifications/PackNotification.h> +#include <mesa_pd/mpi/notifications/ParseMessage.h> +#include <mesa_pd/mpi/notifications/ParticleCopyNotification.h> +#include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h> +#include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h> + +#include <core/mpi/BufferSystem.h> +#include <core/logging/Logging.h> + +namespace walberla { +namespace mesa_pd { +namespace mpi { + +/** + * Kernel which updates all ghost particles. + * + * \ingroup mesa_pd_mpi + */ +class SyncGhostOwners +{ +public: + void operator()( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx = real_t(0), + const bool syncNonCommunicatingBodies = false ) const; + + int64_t getBytesSent() const { return bs1.getBytesSent() + bs2.getBytesSent(); } + int64_t getBytesReceived() const { return bs1.getBytesReceived() + bs2.getBytesReceived(); } + + int64_t getNumberOfSends() const { return bs1.getNumberOfSends() + bs2.getNumberOfSends(); } + int64_t getNumberOfReceives() const { return bs1.getNumberOfReceives() + bs2.getNumberOfReceives(); } +private: + void updateAndMigrate( data::ParticleStorage& ps, + const domain::IDomain& domain, + const bool syncNonCommunicatingBodies ) const; + + void checkAndResolveOverlap( data::ParticleStorage& ps, + const domain::IDomain& domain, + const real_t dx, + const bool syncNonCommunicatingBodies ) const; + + mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator() + + mutable walberla::mpi::BufferSystem bs1 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 749861); + mutable walberla::mpi::BufferSystem bs2 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 255367); + + int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses(); + int rank_ = walberla::mpi::MPIManager::instance()->rank(); +}; + +} // namespace mpi +} // namespace mesa_pd +} // namespace walberla \ No newline at end of file diff --git a/src/mesa_pd/mpi/SyncNextNeighbors.cpp b/src/mesa_pd/mpi/SyncNextNeighbors.cpp index 4d5ef12cd2747e6aa6e7c853b24598369f7ddc5f..b139495464a60095a75913a3bd046cd7301b7d53 100644 --- a/src/mesa_pd/mpi/SyncNextNeighbors.cpp +++ b/src/mesa_pd/mpi/SyncNextNeighbors.cpp @@ -26,6 +26,8 @@ #include "SyncNextNeighbors.h" +#include <mesa_pd/mpi/RemoveAndNotify.h> + namespace walberla { namespace mesa_pd { namespace mpi { @@ -66,38 +68,6 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps, WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." ); } -/** - * Removes a particle from the local storage and informs ghost particle holders. - * - * This function removes the particle from the particle storage and generates deletion notifications. - */ -inline -data::ParticleStorage::iterator removeAndNotify( walberla::mpi::BufferSystem& bs, - data::ParticleStorage& ps, - data::ParticleStorage::iterator& pIt ) -{ - WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST), - "Trying to remove ghost particle from the particle storage." ); - - WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL), - "Trying to remove a global particle from the particle storage." ); - - if( !pIt->getGhostOwners().empty() ) - { - // Notify registered processes (intersecting or interacting) of particle removal since they possess a shadow copy. - for( auto ghostRank : pIt->getGhostOwnersRef() ) - { - WALBERLA_LOG_DETAIL( "__Notify registered process " << ghostRank << " of deletion of particle " << pIt->getUid() ); - auto& sb = bs.sendBuffer(ghostRank); - if (sb.isEmpty()) sb << walberla::uint8_c(0); - packNotification(sb, ParticleRemovalNotification( *pIt )); - } - } - - pIt->getGhostOwnersRef().clear(); - return ps.erase( pIt ); -} - void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& ps, const domain::IDomain& domain, const real_t dx) const @@ -130,7 +100,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p for (const auto& ghostOwner : pIt->getGhostOwners() ) { - auto& buffer( bs.sendBuffer(ghostOwner) ); + auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostOwner)) ); WALBERLA_LOG_DETAIL( "Sending removal notification for particle " << pIt->getUid() << " to process " << ghostOwner ); @@ -174,7 +144,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p auto& buffer( bs.sendBuffer(nbProcessRank) ); WALBERLA_LOG_DETAIL( "Sending shadow copy notification for particle " << pIt->getUid() << " to process " << (nbProcessRank) ); packNotification(buffer, ParticleCopyNotification( *pIt )); - pIt->getGhostOwnersRef().emplace_back( int_c(nbProcessRank) ); + pIt->getGhostOwnersRef().insert( int_c(nbProcessRank) ); } } else @@ -232,7 +202,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p for( auto ghostRank : pIt->getGhostOwners() ) { - auto& buffer( bs.sendBuffer(ghostRank) ); + auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)) ); WALBERLA_LOG_DETAIL( "Sending remote migration notification for particle " << pIt->getUid() << " to process " << ghostRank ); @@ -240,7 +210,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p packNotification(buffer, ParticleRemoteMigrationNotification( *pIt, ownerRank )); } - pIt->getGhostOwnersRef().emplace_back( int_c(ownRank) ); + pIt->getGhostOwnersRef().insert( int_c(ownRank) ); // Send migration notification to new owner auto& buffer( bs.sendBuffer(ownerRank) ); diff --git a/src/mesa_pd/mpi/notifications/NewGhostParticleNotification.h b/src/mesa_pd/mpi/notifications/NewGhostParticleNotification.h new file mode 100644 index 0000000000000000000000000000000000000000..3c929cee446dca9770755b6071d72142ae6de2c6 --- /dev/null +++ b/src/mesa_pd/mpi/notifications/NewGhostParticleNotification.h @@ -0,0 +1,100 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file NewGhostParticleNotification.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/mpi/notifications/NotificationType.h> + +#include <core/mpi/Datatype.h> +#include <core/mpi/MPIWrapper.h> +#include <core/mpi/RecvBuffer.h> +#include <core/mpi/SendBuffer.h> + +namespace walberla { +namespace mesa_pd { + +/** + * This notification is send to the owner of a particle + * to signal that a new ghost particle exists and the ghost particle list should be updated. + */ +class NewGhostParticleNotification +{ +public: + struct Parameters + { + id_t uid_; + walberla::mpi::MPIRank newOwner_; + }; + + inline explicit NewGhostParticleNotification( const data::Particle& particle, const walberla::mpi::MPIRank newOwner ) + : particle_(particle) + , newOwner_(newOwner) + {} + const data::Particle& particle_; + walberla::mpi::MPIRank newOwner_; +}; + +template<> +struct NotificationTrait<NewGhostParticleNotification> +{ + static const NotificationType id = NEW_GHOST_PARTICLE_NOTIFICATION; +}; + +} // namespace mesa_pd +} // namespace walberla + +//====================================================================================================================== +// +// Send/Recv Buffer Serialization Specialization +// +//====================================================================================================================== + +namespace walberla { +namespace mpi { + +template< typename T, // Element type of SendBuffer + typename G> // Growth policy of SendBuffer +mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::NewGhostParticleNotification& obj ) +{ + buf.addDebugMarker( "cn" ); + buf << obj.particle_.getUid(); + buf << obj.newOwner_; + return buf; +} + +template< typename T> // Element type of RecvBuffer +mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::NewGhostParticleNotification::Parameters& objparam ) +{ + buf.readDebugMarker( "cn" ); + buf >> objparam.uid_; + buf >> objparam.newOwner_; + return buf; +} + +} // mpi +} // walberla \ No newline at end of file diff --git a/src/mesa_pd/mpi/notifications/ParseMessage.h b/src/mesa_pd/mpi/notifications/ParseMessage.h index 6baade76d7bfd360f187b3ba3b5d22256e2297ca..60f80dedb89e935daa9122ede7d3d7acffb8a3ba 100644 --- a/src/mesa_pd/mpi/notifications/ParseMessage.h +++ b/src/mesa_pd/mpi/notifications/ParseMessage.h @@ -29,11 +29,13 @@ #include <mesa_pd/data/ParticleStorage.h> #include <mesa_pd/domain/IDomain.h> +#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h> #include <mesa_pd/mpi/notifications/NotificationType.h> #include <mesa_pd/mpi/notifications/ParticleCopyNotification.h> #include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h> #include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h> #include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h> +#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h> #include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h> #include <core/debug/Debug.h> @@ -70,14 +72,18 @@ void ParseMessage::operator()(int sender, WALBERLA_LOG_DETAIL( "Received PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "from neighboring process with rank " << sender ); - WALBERLA_CHECK_EQUAL( ps.find(objparam.uid), ps.end(), "Ghost particle with id " << objparam.uid << " already existend."); + if ( ps.find(objparam.uid) == ps.end() ) + { + auto pIt = createNewParticle(ps, objparam); - auto pIt = createNewParticle(ps, objparam); + domain.correctParticlePosition(pIt->getPositionRef()); - domain.correctParticlePosition(pIt->getPositionRef()); - - WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)); - data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST); + //WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)); + data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST); + } else + { + WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend."); + } WALBERLA_LOG_DETAIL( "Processed PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "." ); @@ -184,8 +190,61 @@ void ParseMessage::operator()(int sender, break; } + case NEW_GHOST_PARTICLE_NOTIFICATION: { + NewGhostParticleNotification::Parameters objparam; + rb >> objparam; + + WALBERLA_LOG_DETAIL( "Received new ghost particle notification for particle " << + objparam.uid_ << + " from neighboring process with rank " << + sender << + "." ); + + auto pIt = ps.find( objparam.uid_ ); + WALBERLA_CHECK_UNEQUAL( pIt, ps.end() ); + + pIt->getGhostOwnersRef().insert( objparam.newOwner_ ); + + WALBERLA_LOG_DETAIL( "Processed new ghost particle notification" ); + + break; + } + case PARTICLE_REMOVAL_INFORMATION_NOTIFICATION: { + ParticleRemovalInformationNotification::Parameters objparam; + rb >> objparam; + + WALBERLA_LOG_DETAIL( "Received particle removal information notification for particle " << + objparam.uid_ << + " from neighboring process with rank " << + sender << + "." ); + + if (objparam.owner_ == receiver_) + { + using namespace walberla::mesa_pd::data::particle_flags; + auto pIt = ps.find( objparam.uid_ ); + WALBERLA_CHECK_UNEQUAL( pIt, ps.end() ); + WALBERLA_CHECK(!isSet( pIt->getFlags(), GHOST)); + + pIt->getGhostOwnersRef().erase( sender ); + pIt->getNeighborStateRef().erase( sender ); + } else + { + using namespace walberla::mesa_pd::data::particle_flags; + auto pIt = ps.find( objparam.uid_ ); + if (pIt != ps.end() ) + { + WALBERLA_CHECK(isSet( pIt->getFlags(), GHOST)); + pIt->getNeighborStateRef().erase( sender ); + } + } + + WALBERLA_LOG_DETAIL( "Processed rigid body removal information notification" ); + + break; + } default: - throw std::runtime_error( "Received invalid notification type." ); + WALBERLA_ABORT( "Received invalid notification type: " << notificationType << " from sender: " << sender ); } } diff --git a/src/mesa_pd/mpi/notifications/ParticleMigrationNotification.h b/src/mesa_pd/mpi/notifications/ParticleMigrationNotification.h index 095f6201928e96e0fc3be0df22497d2f6eb14434..78246ac932580a03d5530c1a9e69f044f3d73662 100644 --- a/src/mesa_pd/mpi/notifications/ParticleMigrationNotification.h +++ b/src/mesa_pd/mpi/notifications/ParticleMigrationNotification.h @@ -45,7 +45,7 @@ class ParticleMigrationNotification { public: struct Parameters { id_t uid_; - std::vector<int> ghostOwners_ {}; + std::unordered_set<walberla::mpi::MPIRank> ghostOwners_ {}; walberla::mesa_pd::Vec3 oldForce_ {real_t(0)}; walberla::mesa_pd::Vec3 oldTorque_ {real_t(0)}; }; diff --git a/src/mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h b/src/mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h new file mode 100644 index 0000000000000000000000000000000000000000..aec2ccdfb7904b512f912fa9aa4fa383668c9a09 --- /dev/null +++ b/src/mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h @@ -0,0 +1,97 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file ParticleRemovalInformationNotification.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/mpi/notifications/NotificationType.h> + +#include <core/mpi/Datatype.h> +#include <core/mpi/RecvBuffer.h> +#include <core/mpi/SendBuffer.h> + +namespace walberla { +namespace mesa_pd { + +/** + * The ParticleRemovalInformationNotification class is used to signal other processes that a + * shadow copy was destroyed. + */ +class ParticleRemovalInformationNotification +{ +public: + struct Parameters + { + id_t uid_; + walberla::mpi::MPIRank owner_; + }; + + inline explicit ParticleRemovalInformationNotification( const data::Particle& particle ) + : particle_(particle) + {} + const data::Particle& particle_; +}; + +template<> +struct NotificationTrait<ParticleRemovalInformationNotification> +{ + static const NotificationType id = PARTICLE_REMOVAL_INFORMATION_NOTIFICATION; +}; + +} // namespace mesa_pd +} // namespace walberla + +//====================================================================================================================== +// +// Send/Recv Buffer Serialization Specialization +// +//====================================================================================================================== + +namespace walberla { +namespace mpi { + +template< typename T, // Element type of SendBuffer + typename G> // Growth policy of SendBuffer +mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::ParticleRemovalInformationNotification& obj ) +{ + buf.addDebugMarker( "ri" ); + buf << obj.particle_.getUid(); + buf << static_cast<walberla::mpi::MPIRank>(obj.particle_.getOwner()); + return buf; +} + +template< typename T> // Element type of RecvBuffer +mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::ParticleRemovalInformationNotification::Parameters& objparam ) +{ + buf.readDebugMarker( "ri" ); + buf >> objparam.uid_; + buf >> objparam.owner_; + return buf; +} + +} // mpi +} // walberla \ No newline at end of file diff --git a/tests/core/mpi/BufferSystemTest.cpp b/tests/core/mpi/BufferSystemTest.cpp index 6de0e0ce4f9de043fae0a5dcfd5884ea06bb4d04..a2e95dc63686763c3bfe28a329c0712ab4aa1acf 100644 --- a/tests/core/mpi/BufferSystemTest.cpp +++ b/tests/core/mpi/BufferSystemTest.cpp @@ -219,7 +219,7 @@ void timeVaryingCommunication(const bool useIProbe) bs.send( rightNeighbor ); - WALBERLA_CHECK( bs.isCommunciationRunning() ); + WALBERLA_CHECK( bs.isCommunicationRunning() ); for( auto it = bs.begin(); it != bs.end(); ++it ) { @@ -245,7 +245,7 @@ void timeVaryingCommunication(const bool useIProbe) WALBERLA_CHECK( it.buffer().isEmpty() ); } - WALBERLA_CHECK( ! bs.isCommunciationRunning() ); + WALBERLA_CHECK( ! bs.isCommunicationRunning() ); } } diff --git a/tests/core/mpi/BufferTest.cpp b/tests/core/mpi/BufferTest.cpp index b5d43f478ea9f03da32a12627cc09d1ab7044351..887927a47179bf4cf39bc58ee49984160a38c296 100644 --- a/tests/core/mpi/BufferTest.cpp +++ b/tests/core/mpi/BufferTest.cpp @@ -143,15 +143,17 @@ void bufferTest() initCellContainer(cellVector); initCellContainer(cellSet); - std::vector <bool> boolStdVec, boolStdVecEmpty; - std::vector <unsigned int> stdVec, stdVecEmpty; - std::deque <unsigned int> stdDeque, stdDequeEmpty; - std::list <unsigned int> stdList, stdListEmpty; - std::set <unsigned int> stdSet, stdSetEmpty; - std::multiset<unsigned int> stdMultiSet, stdMultiSetEmpty; - - std::map <unsigned int, walberla::int64_t> stdMap, stdMapEmpty; - std::multimap<unsigned int, walberla::int64_t> stdMultiMap, stdMultiMapEmpty; + std::vector <bool> boolStdVec, boolStdVecEmpty; + std::vector <unsigned int> stdVec, stdVecEmpty; + std::deque <unsigned int> stdDeque, stdDequeEmpty; + std::list <unsigned int> stdList, stdListEmpty; + std::set <unsigned int> stdSet, stdSetEmpty; + std::multiset <unsigned int> stdMultiSet, stdMultiSetEmpty; + std::unordered_set<unsigned int> stdUnorderedSet, stdUnorderedSetEmpty; + + std::map <unsigned int, walberla::int64_t> stdMap, stdMapEmpty; + std::multimap <unsigned int, walberla::int64_t> stdMultiMap, stdMultiMapEmpty; + std::unordered_map<unsigned int, walberla::int64_t> stdUnorderedMap, stdUnorderedMapEmpty; std::array < unsigned int, 19 > stdArray; @@ -161,24 +163,28 @@ void bufferTest() initIntegerContainer(stdList); initIntegerAssocContainer(stdSet); initIntegerAssocContainer(stdMultiSet); + initIntegerAssocContainer(stdUnorderedSet); initIntegerMap(stdMap); initIntegerMap(stdMultiMap); + initIntegerMap(stdUnorderedMap); initStdArray(stdArray); // Create send buffer and put two values in it GenericSendBuffer<T> sb; - sb << testDouble << testInt; - sb << vec << mat; - sb << cell << cellInterval; - sb << cellVector << cellSet; - sb << boolStdVec << boolStdVecEmpty; - sb << stdVec << stdVecEmpty; - sb << stdDeque << stdDequeEmpty; - sb << stdList << stdListEmpty; - sb << stdSet << stdSetEmpty; - sb << stdMultiSet << stdMultiSetEmpty; - sb << stdMap << stdMapEmpty; - sb << stdMultiMap << stdMultiMapEmpty; + sb << testDouble << testInt; + sb << vec << mat; + sb << cell << cellInterval; + sb << cellVector << cellSet; + sb << boolStdVec << boolStdVecEmpty; + sb << stdVec << stdVecEmpty; + sb << stdDeque << stdDequeEmpty; + sb << stdList << stdListEmpty; + sb << stdSet << stdSetEmpty; + sb << stdMultiSet << stdMultiSetEmpty; + sb << stdUnorderedSet << stdUnorderedSetEmpty; + sb << stdMap << stdMapEmpty; + sb << stdMultiMap << stdMultiMapEmpty; + sb << stdUnorderedMap << stdUnorderedMapEmpty; sb << stdArray; // Copying @@ -199,30 +205,34 @@ void bufferTest() CellVector recvCellVector; CellSet recvCellSet; - std::vector <bool> recvBoolStdVec, recvBoolStdVecEmpty; - std::vector <unsigned int> recvStdVec, recvStdVecEmpty; - std::deque <unsigned int> recvStdDeque, recvStdDequeEmpty; - std::list <unsigned int> recvStdList, recvStdListEmpty; - std::set <unsigned int> recvStdSet, recvStdSetEmpty; - std::multiset<unsigned int> recvStdMultiSet, recvStdMultiSetEmpty; + std::vector <bool> recvBoolStdVec, recvBoolStdVecEmpty; + std::vector <unsigned int> recvStdVec, recvStdVecEmpty; + std::deque <unsigned int> recvStdDeque, recvStdDequeEmpty; + std::list <unsigned int> recvStdList, recvStdListEmpty; + std::set <unsigned int> recvStdSet, recvStdSetEmpty; + std::multiset <unsigned int> recvStdMultiSet, recvStdMultiSetEmpty; + std::unordered_set <unsigned int> recvStdUnorderedSet, recvStdUnorderedSetEmpty; - std::map <unsigned int, walberla::int64_t> recvStdMap, recvStdMapEmpty; - std::multimap<unsigned int, walberla::int64_t> recvStdMultiMap, recvStdMultiMapEmpty; + std::map <unsigned int, walberla::int64_t> recvStdMap, recvStdMapEmpty; + std::multimap <unsigned int, walberla::int64_t> recvStdMultiMap, recvStdMultiMapEmpty; + std::unordered_map <unsigned int, walberla::int64_t> recvStdUnorderedMap, recvStdUnorderedMapEmpty; std::array <unsigned int, 19> recvStdArray; - rb >> recvD >> recvI; - rb >> recvVec >> recvMat; - rb >> recvCell >> recvCellInterval; - rb >> recvCellVector >> recvCellSet; - rb >> recvBoolStdVec >> recvBoolStdVecEmpty; - rb >> recvStdVec >> recvStdVecEmpty; - rb >> recvStdDeque >> recvStdDequeEmpty; - rb >> recvStdList >> recvStdListEmpty; - rb >> recvStdSet >> recvStdSetEmpty; - rb >> recvStdMultiSet >> recvStdMultiSetEmpty; - rb >> recvStdMap >> recvStdMapEmpty; - rb >> recvStdMultiMap >> recvStdMultiMapEmpty; + rb >> recvD >> recvI; + rb >> recvVec >> recvMat; + rb >> recvCell >> recvCellInterval; + rb >> recvCellVector >> recvCellSet; + rb >> recvBoolStdVec >> recvBoolStdVecEmpty; + rb >> recvStdVec >> recvStdVecEmpty; + rb >> recvStdDeque >> recvStdDequeEmpty; + rb >> recvStdList >> recvStdListEmpty; + rb >> recvStdSet >> recvStdSetEmpty; + rb >> recvStdMultiSet >> recvStdMultiSetEmpty; + rb >> recvStdUnorderedSet >> recvStdUnorderedSetEmpty; + rb >> recvStdMap >> recvStdMapEmpty; + rb >> recvStdMultiMap >> recvStdMultiMapEmpty; + rb >> recvStdUnorderedMap >> recvStdUnorderedMapEmpty; rb >> recvStdArray; // Validate @@ -245,14 +255,19 @@ void bufferTest() WALBERLA_CHECK_EQUAL(recvStdList, stdList); WALBERLA_CHECK_EQUAL(recvStdListEmpty, stdListEmpty); - WALBERLA_CHECK_EQUAL(recvStdSet, stdSet); - WALBERLA_CHECK_EQUAL(recvStdSetEmpty, stdSetEmpty); - WALBERLA_CHECK_EQUAL(recvStdMultiSet, stdMultiSet); - WALBERLA_CHECK_EQUAL(recvStdMultiSetEmpty, stdMultiSetEmpty); - WALBERLA_CHECK_EQUAL(recvStdMap, stdMap); - WALBERLA_CHECK_EQUAL(recvStdMapEmpty, stdMapEmpty); - WALBERLA_CHECK_EQUAL(recvStdMultiMap, stdMultiMap); - WALBERLA_CHECK_EQUAL(recvStdMultiMapEmpty, stdMultiMapEmpty); + WALBERLA_CHECK_EQUAL(recvStdSet, stdSet); + WALBERLA_CHECK_EQUAL(recvStdSetEmpty, stdSetEmpty); + WALBERLA_CHECK_EQUAL(recvStdMultiSet, stdMultiSet); + WALBERLA_CHECK_EQUAL(recvStdMultiSetEmpty, stdMultiSetEmpty); + WALBERLA_CHECK_EQUAL(recvStdUnorderedSet, stdUnorderedSet); + WALBERLA_CHECK_EQUAL(recvStdUnorderedSetEmpty, stdUnorderedSetEmpty); + + WALBERLA_CHECK_EQUAL(recvStdMap, stdMap); + WALBERLA_CHECK_EQUAL(recvStdMapEmpty, stdMapEmpty); + WALBERLA_CHECK_EQUAL(recvStdMultiMap, stdMultiMap); + WALBERLA_CHECK_EQUAL(recvStdMultiMapEmpty, stdMultiMapEmpty); + WALBERLA_CHECK_EQUAL(recvStdUnorderedMap, stdUnorderedMap); + WALBERLA_CHECK_EQUAL(recvStdUnorderedMapEmpty, stdUnorderedMapEmpty); WALBERLA_CHECK_EQUAL(recvStdArray, stdArray); } diff --git a/tests/mesa_pd/CMakeLists.txt b/tests/mesa_pd/CMakeLists.txt index 5cfc0f96ff816f5daa33dcc057c2a4c1763d964c..ae8501523febf9a160ccd85681ab8cc52f07cb99 100644 --- a/tests/mesa_pd/CMakeLists.txt +++ b/tests/mesa_pd/CMakeLists.txt @@ -98,6 +98,12 @@ waLBerla_execute_test( NAME MESA_PD_Kernel_SingleCast ) waLBerla_compile_test( NAME MESA_PD_Kernel_SpringDashpot FILES kernel/SpringDashpot.cpp DEPENDS core ) waLBerla_execute_test( NAME MESA_PD_Kernel_SpringDashpot ) +waLBerla_compile_test( NAME MESA_PD_Kernel_SyncGhostOwners FILES kernel/SyncGhostOwners.cpp DEPENDS core ) +waLBerla_execute_test( NAME MESA_PD_Kernel_SyncGhostOwners PROCESSES 27 ) + +waLBerla_compile_test( NAME MESA_PD_Kernel_SyncGhostOwnersLarge FILES kernel/SyncGhostOwnersLarge.cpp DEPENDS core ) +waLBerla_execute_test( NAME MESA_PD_Kernel_SyncGhostOwnersLarge PROCESSES 27 ) + waLBerla_compile_test( NAME MESA_PD_Kernel_SyncNextNeighbors FILES kernel/SyncNextNeighbors.cpp DEPENDS core ) waLBerla_execute_test( NAME MESA_PD_Kernel_SyncNextNeighbors PROCESSES 27 ) diff --git a/tests/mesa_pd/kernel/SyncGhostOwners.cpp b/tests/mesa_pd/kernel/SyncGhostOwners.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cf16223ee801d8202d34b27c2bf2b062e221f831 --- /dev/null +++ b/tests/mesa_pd/kernel/SyncGhostOwners.cpp @@ -0,0 +1,147 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwners.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/domain/BlockForestDomain.h> +#include <mesa_pd/mpi/SyncGhostOwners.h> + +#include <blockforest/BlockForest.h> +#include <blockforest/Initialization.h> +#include <core/Environment.h> +#include <core/logging/Logging.h> +#include <core/mpi/Reduce.h> + +#include <iostream> +#include <memory> + +namespace walberla { +namespace mesa_pd { + +const real_t radius = real_t(1); + +walberla::id_t createSphere(data::ParticleStorage& ps, domain::IDomain& domain) +{ + walberla::id_t uid = 0; + auto owned = domain.isContainedInProcessSubdomain( uint_c(walberla::mpi::MPIManager::instance()->rank()), Vec3(0,0,0) ); + if (owned) + { + data::Particle&& p = *ps.create(); + p.getPositionRef() = Vec3(0,0,0); + p.getInteractionRadiusRef() = radius; + p.getRotationRef() = Rot3(Quat()); + p.getLinearVelocityRef() = Vec3(1,2,3); + p.getAngularVelocityRef() = Vec3(4,5,6); + p.getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); + uid = p.getUid(); + WALBERLA_LOG_DETAIL("SPHERE CREATED"); + } + + walberla::mpi::allReduceInplace(uid, walberla::mpi::SUM); + return uid; +} + +int main( int argc, char ** argv ) +{ + Environment env(argc, argv); + WALBERLA_UNUSED(env); + walberla::mpi::MPIManager::instance()->useWorldComm(); + + //logging::Logging::instance()->setStreamLogLevel(logging::Logging::DETAIL); +// logging::Logging::instance()->includeLoggingToFile("MESA_PD_Kernel_SyncGhostOwners"); +// logging::Logging::instance()->setFileLogLevel(logging::Logging::DETAIL); + + //init domain partitioning + auto forest = blockforest::createBlockForest( AABB(-15,-15,-15,15,15,15), // simulation domain + Vector3<uint_t>(3,3,3), // blocks in each direction + Vector3<bool>(true, true, true) // periodicity + ); + domain::BlockForestDomain domain(forest); + std::array< bool, 3 > periodic; + periodic[0] = forest->isPeriodic(0); + periodic[1] = forest->isPeriodic(1); + periodic[2] = forest->isPeriodic(2); + + //init data structures + data::ParticleStorage ps(100); + + //initialize particle + auto uid = createSphere(ps, domain); + WALBERLA_LOG_DEVEL_ON_ROOT("uid: " << uid); + + //init kernels + mpi::SyncGhostOwners SNN; + + std::vector<real_t> deltas { real_t(0), + real_t(4.9), + real_t(5.1), + real_t(10), + real_t(14.9), + real_t(-14.9), + real_t(-10), + real_t(-5.1), + real_t(-4.9), + real_t(0)}; + + for (auto delta : deltas) + { + WALBERLA_LOG_DEVEL(delta); + auto pos = Vec3(1,-1,1) * delta; + WALBERLA_LOG_DETAIL("checking position: " << pos); + // owner moves particle to new position + auto pIt = ps.find(uid); + if (pIt != ps.end()) + { + if (!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)) + { + pIt->setPosition(pos); + } + } + + //sync + SNN(ps, domain); + + //check + if (sqDistancePointToAABBPeriodic(pos, forest->begin()->getAABB(), forest->getDomain(), periodic) <= radius * radius) + { + WALBERLA_CHECK_EQUAL(ps.size(), 1); + if (forest->begin()->getAABB().contains(pos)) + { + WALBERLA_CHECK(!data::particle_flags::isSet(ps.begin()->getFlags(), data::particle_flags::GHOST)); + } else + { + WALBERLA_CHECK(data::particle_flags::isSet(ps.begin()->getFlags(), data::particle_flags::GHOST)); + } + } else + { + WALBERLA_CHECK_EQUAL(ps.size(), 0); + } + } + + + return EXIT_SUCCESS; +} + +} //namespace mesa_pd +} //namespace walberla + +int main( int argc, char ** argv ) +{ + return walberla::mesa_pd::main(argc, argv); +} diff --git a/tests/mesa_pd/kernel/SyncGhostOwnersLarge.cpp b/tests/mesa_pd/kernel/SyncGhostOwnersLarge.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b04dd3c602f073d804f77c7d9e2e0d21e203ade2 --- /dev/null +++ b/tests/mesa_pd/kernel/SyncGhostOwnersLarge.cpp @@ -0,0 +1,151 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SyncGhostOwnersLarge.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/domain/BlockForestDomain.h> +#include <mesa_pd/mpi/SyncGhostOwners.h> + +#include <blockforest/BlockForest.h> +#include <blockforest/Initialization.h> +#include <core/Environment.h> +#include <core/logging/Logging.h> +#include <core/mpi/Reduce.h> + +#include <iostream> +#include <memory> + +namespace walberla { +namespace mesa_pd { + +const real_t radius = real_t(20); + +walberla::id_t createSphere(data::ParticleStorage& ps, domain::IDomain& domain) +{ + walberla::id_t uid = 0; + auto owned = domain.isContainedInProcessSubdomain( uint_c(walberla::mpi::MPIManager::instance()->rank()), Vec3(0,0,0) ); + if (owned) + { + data::Particle&& p = *ps.create(); + p.getPositionRef() = Vec3(0,0,0); + p.getInteractionRadiusRef() = radius; + p.getRotationRef() = Rot3(Quat()); + p.getLinearVelocityRef() = Vec3(1,2,3); + p.getAngularVelocityRef() = Vec3(4,5,6); + p.getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); + uid = p.getUid(); + WALBERLA_LOG_DETAIL("SPHERE CREATED"); + } + + walberla::mpi::allReduceInplace(uid, walberla::mpi::SUM); + return uid; +} + +int main( int argc, char ** argv ) +{ + Environment env(argc, argv); + WALBERLA_UNUSED(env); + walberla::mpi::MPIManager::instance()->useWorldComm(); + + //logging::Logging::instance()->setStreamLogLevel(logging::Logging::DETAIL); + //logging::Logging::instance()->includeLoggingToFile("MESA_PD_Kernel_SyncGhostOwnersLarge"); + //logging::Logging::instance()->setFileLogLevel(logging::Logging::DETAIL); + + //init domain partitioning + auto forest = blockforest::createBlockForest( AABB(-5,-5,-5,25,25,25), // simulation domain + Vector3<uint_t>(3,3,3), // blocks in each direction + Vector3<bool>(false, false, false) // periodicity + ); + domain::BlockForestDomain domain(forest); + std::array< bool, 3 > periodic; + periodic[0] = forest->isPeriodic(0); + periodic[1] = forest->isPeriodic(1); + periodic[2] = forest->isPeriodic(2); + + //init data structures + data::ParticleStorage ps(100); + + //initialize particle + auto uid = createSphere(ps, domain); + WALBERLA_LOG_DEVEL_ON_ROOT("uid: " << uid); + + //init kernels + mpi::SyncGhostOwners SNN; + + SNN(ps, domain); + SNN(ps, domain); + SNN(ps, domain); + SNN(ps, domain); + + std::vector<real_t> deltas { + real_t(0), + real_t(4.9), + real_t(5.1), + real_t(10), + real_t(14.9), + real_t(15.1), + real_t(20), + real_t(24.9)}; + + for (auto delta : deltas) + { + WALBERLA_LOG_DEVEL(delta); + auto pos = Vec3(1,0,0) * delta; + WALBERLA_LOG_DETAIL("checking position: " << pos); + // owner moves particle to new position + auto pIt = ps.find(uid); + if (pIt != ps.end()) + { + if (!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST)) + { + pIt->setPosition(pos); + } + } + + //sync + SNN(ps, domain); + + //check + if (sqDistancePointToAABB(pos, forest->begin()->getAABB()) <= radius * radius) + { + WALBERLA_CHECK_EQUAL(ps.size(), 1); + if (forest->begin()->getAABB().contains(pos)) + { + WALBERLA_CHECK(!data::particle_flags::isSet(ps.begin()->getFlags(), data::particle_flags::GHOST)); + } else + { + WALBERLA_CHECK(data::particle_flags::isSet(ps.begin()->getFlags(), data::particle_flags::GHOST)); + } + } else + { + WALBERLA_CHECK_EQUAL(ps.size(), 0); + } + } + + + return EXIT_SUCCESS; +} + +} //namespace mesa_pd +} //namespace walberla + +int main( int argc, char ** argv ) +{ + return walberla::mesa_pd::main(argc, argv); +} diff --git a/tests/mesa_pd/kernel/SyncNextNeighbors.cpp b/tests/mesa_pd/kernel/SyncNextNeighbors.cpp index 373dffbc353320457b5b87a4ad6c3448828b719e..93a7d6768ed2fb294c556cae0a6697d17c802d33 100644 --- a/tests/mesa_pd/kernel/SyncNextNeighbors.cpp +++ b/tests/mesa_pd/kernel/SyncNextNeighbors.cpp @@ -50,6 +50,7 @@ walberla::id_t createSphere(data::ParticleStorage& ps, domain::IDomain& domain) p.getAngularVelocityRef() = Vec3(4,5,6); p.getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); uid = p.getUid(); + WALBERLA_LOG_DETAIL("SPHERE CREATED"); } walberla::mpi::allReduceInplace(uid, walberla::mpi::SUM); @@ -62,7 +63,7 @@ int main( int argc, char ** argv ) WALBERLA_UNUSED(env); walberla::mpi::MPIManager::instance()->useWorldComm(); -// logging::Logging::instance()->setStreamLogLevel(logging::Logging::DETAIL); + //logging::Logging::instance()->setStreamLogLevel(logging::Logging::DETAIL); // logging::Logging::instance()->includeLoggingToFile("MESA_PD_Kernel_SyncNextNeighbor"); // logging::Logging::instance()->setFileLogLevel(logging::Logging::DETAIL); @@ -100,6 +101,7 @@ int main( int argc, char ** argv ) for (auto delta : deltas) { + WALBERLA_LOG_DEVEL(delta); auto pos = Vec3(1,-1,1) * delta; WALBERLA_LOG_DETAIL("checking position: " << pos); // owner moves particle to new position