Due to a key rollover of our WebSSO provider there might be a short downtime of this Gitlab server on Friday, November 2.

Commit b234daf1 authored by Sebastian Eibl's avatar Sebastian Eibl

Merge branch 'SyncGhostOwner' into 'master'

Sync Ghost Owner

See merge request !214
parents 7932bf65 4204afe7
Pipeline #16437 failed with stages
in 392 minutes and 33 seconds
......@@ -111,6 +111,7 @@ if __name__ == '__main__':
comm.append(mpi.ClearNextNeighborSync())
comm.append(mpi.ReduceContactHistory())
comm.append(mpi.ReduceProperty())
comm.append(mpi.SyncGhostOwners(ps))
comm.append(mpi.SyncNextNeighbors(ps))
......
......@@ -10,12 +10,12 @@ class ParticleStorage(Container):
self.addInclude("mesa_pd/data/Flags.h")
self.addProperty("uid", "walberla::id_t", defValue = "UniqueID<data::Particle>::invalidID()", syncMode="ALWAYS")
self.addProperty("uid", "walberla::id_t", defValue = "UniqueID<data::Particle>::invalidID()", syncMode="ALWAYS")
self.addProperty("position", "walberla::mesa_pd::Vec3", defValue = "real_t(0)", syncMode="ALWAYS")
self.addProperty("interactionRadius", "walberla::real_t", defValue = "real_t(0)", syncMode="COPY")
self.addProperty("interactionRadius", "walberla::real_t", defValue = "real_t(0)", syncMode="COPY")
self.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue = "", syncMode="COPY")
self.addProperty("owner", "int", defValue = "-1", syncMode="COPY")
self.addProperty("ghostOwners", "std::vector<int>", defValue = "", syncMode="MIGRATION")
self.addProperty("owner", "int", defValue = "-1", syncMode="COPY")
self.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue = "", syncMode="MIGRATION")
def generate(self, path):
self.unrollDimension()
......@@ -40,7 +40,9 @@ class ParticleStorage(Container):
generateFile(path, 'mpi/notifications/HeatFluxNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParseMessage.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleCopyNotification.templ.h', context)
generateFile(path, 'mpi/notifications/NewGhostParticleNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleMigrationNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleRemoteMigrationNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleRemovalInformationNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleRemovalNotification.templ.h', context)
generateFile(path, 'mpi/notifications/ParticleUpdateNotification.templ.h', context)
# -*- coding: utf-8 -*-
from ..utility import generateFile
class SyncGhostOwners:
def __init__(self, p):
p.addProperty("position", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS")
p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE")
p.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue="", syncMode="ONCE")
p.addProperty("owner", "int", defValue="-1", syncMode="ONCE")
p.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER")
p.addProperty("neighborState", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER")
def generate(self, path):
generateFile(path, 'mpi/SyncGhostOwners.templ.h')
generateFile(path, 'mpi/SyncGhostOwners.templ.cpp')
......@@ -5,10 +5,10 @@ from ..utility import generateFile
class SyncNextNeighbors:
def __init__(self, p):
p.addProperty("position", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS")
p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE")
p.addProperty("interactionRadius", "walberla::real_t", defValue="real_t(0)", syncMode="ONCE")
p.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue="", syncMode="ONCE")
p.addProperty("owner", "int", defValue="-1", syncMode="ONCE")
p.addProperty("ghostOwners", "std::vector<int>", defValue="", syncMode="NEVER")
p.addProperty("owner", "int", defValue="-1", syncMode="ONCE")
p.addProperty("ghostOwners", "std::unordered_set<walberla::mpi::MPIRank>", defValue="", syncMode="NEVER")
def generate(self, path):
generateFile(path, 'mpi/SyncNextNeighbors.templ.h')
......
......@@ -4,11 +4,13 @@ from .BroadcastProperty import BroadcastProperty
from .ClearNextNeighborSync import ClearNextNeighborSync
from .ReduceContactHistory import ReduceContactHistory
from .ReduceProperty import ReduceProperty
from .SyncGhostOwners import SyncGhostOwners
from .SyncNextNeighbors import SyncNextNeighbors
__all__ = ['BroadcastProperty',
'ClearNextNeighborSync',
'ReduceContactHistory',
'ReduceProperty',
'SyncGhostOwners',
'SyncNextNeighbors',
]
......@@ -31,6 +31,7 @@
#include <map>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include <mesa_pd/data/ContactHistory.h>
......@@ -44,6 +45,7 @@
#include <core/Abort.h>
#include <core/debug/Debug.h>
#include <core/math/AABB.h>
#include <core/mpi/MPIWrapper.h>
#include <core/OpenMP.h>
#include <core/STLIO.h>
#include <core/UniqueID.h>
......
This diff is collapsed.
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file SyncGhostOwners.h
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//
//======================================================================================================================
//======================================================================================================================
//
// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
//
//======================================================================================================================
#pragma once
#include <mesa_pd/data/DataTypes.h>
#include <mesa_pd/data/Flags.h>
#include <mesa_pd/data/ParticleStorage.h>
#include <mesa_pd/domain/IDomain.h>
#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h>
#include <mesa_pd/mpi/notifications/PackNotification.h>
#include <mesa_pd/mpi/notifications/ParseMessage.h>
#include <mesa_pd/mpi/notifications/ParticleCopyNotification.h>
#include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h>
#include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h>
#include <core/mpi/BufferSystem.h>
#include <core/logging/Logging.h>
namespace walberla {
namespace mesa_pd {
namespace mpi {
/**
* Kernel which updates all ghost particles.
*
* \ingroup mesa_pd_mpi
*/
class SyncGhostOwners
{
public:
void operator()( data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx = real_t(0),
const bool syncNonCommunicatingBodies = false ) const;
int64_t getBytesSent() const { return bs1.getBytesSent() + bs2.getBytesSent(); }
int64_t getBytesReceived() const { return bs1.getBytesReceived() + bs2.getBytesReceived(); }
int64_t getNumberOfSends() const { return bs1.getNumberOfSends() + bs2.getNumberOfSends(); }
int64_t getNumberOfReceives() const { return bs1.getNumberOfReceives() + bs2.getNumberOfReceives(); }
private:
void updateAndMigrate( data::ParticleStorage& ps,
const domain::IDomain& domain,
const bool syncNonCommunicatingBodies ) const;
void checkAndResolveOverlap( data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx,
const bool syncNonCommunicatingBodies ) const;
mutable std::vector<uint_t> neighborRanks_; ///cache for neighbor ranks -> will be updated in operator()
mutable walberla::mpi::BufferSystem bs1 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 749861);
mutable walberla::mpi::BufferSystem bs2 = walberla::mpi::BufferSystem( walberla::mpi::MPIManager::instance()->comm(), 255367);
int numProcesses_ = walberla::mpi::MPIManager::instance()->numProcesses();
int rank_ = walberla::mpi::MPIManager::instance()->rank();
};
} // namespace mpi
} // namespace mesa_pd
} // namespace walberla
......@@ -26,6 +26,8 @@
#include "SyncNextNeighbors.h"
#include <mesa_pd/mpi/RemoveAndNotify.h>
namespace walberla {
namespace mesa_pd {
namespace mpi {
......@@ -66,38 +68,6 @@ void SyncNextNeighbors::operator()(data::ParticleStorage& ps,
WALBERLA_LOG_DETAIL( "Parsing of particle synchronization response ended." );
}
/**
* Removes a particle from the local storage and informs ghost particle holders.
*
* This function removes the particle from the particle storage and generates deletion notifications.
*/
inline
data::ParticleStorage::iterator removeAndNotify( walberla::mpi::BufferSystem& bs,
data::ParticleStorage& ps,
data::ParticleStorage::iterator& pIt )
{
WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GHOST),
"Trying to remove ghost particle from the particle storage." );
WALBERLA_ASSERT( !data::particle_flags::isSet( pIt->getFlags(), data::particle_flags::GLOBAL),
"Trying to remove a global particle from the particle storage." );
if( !pIt->getGhostOwners().empty() )
{
// Notify registered processes (intersecting or interacting) of particle removal since they possess a shadow copy.
for( auto ghostRank : pIt->getGhostOwnersRef() )
{
WALBERLA_LOG_DETAIL( "__Notify registered process " << ghostRank << " of deletion of particle " << pIt->getUid() );
auto& sb = bs.sendBuffer(ghostRank);
if (sb.isEmpty()) sb << walberla::uint8_c(0);
packNotification(sb, ParticleRemovalNotification( *pIt ));
}
}
pIt->getGhostOwnersRef().clear();
return ps.erase( pIt );
}
void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& ps,
const domain::IDomain& domain,
const real_t dx) const
......@@ -130,7 +100,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p
for (const auto& ghostOwner : pIt->getGhostOwners() )
{
auto& buffer( bs.sendBuffer(ghostOwner) );
auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostOwner)) );
WALBERLA_LOG_DETAIL( "Sending removal notification for particle " << pIt->getUid() << " to process " << ghostOwner );
......@@ -174,7 +144,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p
auto& buffer( bs.sendBuffer(nbProcessRank) );
WALBERLA_LOG_DETAIL( "Sending shadow copy notification for particle " << pIt->getUid() << " to process " << (nbProcessRank) );
packNotification(buffer, ParticleCopyNotification( *pIt ));
pIt->getGhostOwnersRef().emplace_back( int_c(nbProcessRank) );
pIt->getGhostOwnersRef().insert( int_c(nbProcessRank) );
}
}
else
......@@ -232,7 +202,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p
for( auto ghostRank : pIt->getGhostOwners() )
{
auto& buffer( bs.sendBuffer(ghostRank) );
auto& buffer( bs.sendBuffer(static_cast<walberla::mpi::MPIRank>(ghostRank)) );
WALBERLA_LOG_DETAIL( "Sending remote migration notification for particle " << pIt->getUid() <<
" to process " << ghostRank );
......@@ -240,7 +210,7 @@ void SyncNextNeighbors::generateSynchronizationMessages(data::ParticleStorage& p
packNotification(buffer, ParticleRemoteMigrationNotification( *pIt, ownerRank ));
}
pIt->getGhostOwnersRef().emplace_back( int_c(ownRank) );
pIt->getGhostOwnersRef().insert( int_c(ownRank) );
// Send migration notification to new owner
auto& buffer( bs.sendBuffer(ownerRank) );
......
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file NewGhostParticleNotification.h
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//
//======================================================================================================================
//======================================================================================================================
//
// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
//
//======================================================================================================================
#pragma once
#include <mesa_pd/data/DataTypes.h>
#include <mesa_pd/data/ParticleStorage.h>
#include <mesa_pd/mpi/notifications/NotificationType.h>
#include <core/mpi/Datatype.h>
#include <core/mpi/MPIWrapper.h>
#include <core/mpi/RecvBuffer.h>
#include <core/mpi/SendBuffer.h>
namespace walberla {
namespace mesa_pd {
/**
* This notification is send to the owner of a particle
* to signal that a new ghost particle exists and the ghost particle list should be updated.
*/
class NewGhostParticleNotification
{
public:
struct Parameters
{
id_t uid_;
walberla::mpi::MPIRank newOwner_;
};
inline explicit NewGhostParticleNotification( const data::Particle& particle, const walberla::mpi::MPIRank newOwner )
: particle_(particle)
, newOwner_(newOwner)
{}
const data::Particle& particle_;
walberla::mpi::MPIRank newOwner_;
};
template<>
struct NotificationTrait<NewGhostParticleNotification>
{
static const NotificationType id = NEW_GHOST_PARTICLE_NOTIFICATION;
};
} // namespace mesa_pd
} // namespace walberla
//======================================================================================================================
//
// Send/Recv Buffer Serialization Specialization
//
//======================================================================================================================
namespace walberla {
namespace mpi {
template< typename T, // Element type of SendBuffer
typename G> // Growth policy of SendBuffer
mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::NewGhostParticleNotification& obj )
{
buf.addDebugMarker( "cn" );
buf << obj.particle_.getUid();
buf << obj.newOwner_;
return buf;
}
template< typename T> // Element type of RecvBuffer
mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::NewGhostParticleNotification::Parameters& objparam )
{
buf.readDebugMarker( "cn" );
buf >> objparam.uid_;
buf >> objparam.newOwner_;
return buf;
}
} // mpi
} // walberla
......@@ -29,11 +29,13 @@
#include <mesa_pd/data/ParticleStorage.h>
#include <mesa_pd/domain/IDomain.h>
#include <mesa_pd/mpi/notifications/NewGhostParticleNotification.h>
#include <mesa_pd/mpi/notifications/NotificationType.h>
#include <mesa_pd/mpi/notifications/ParticleCopyNotification.h>
#include <mesa_pd/mpi/notifications/ParticleMigrationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemoteMigrationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemovalNotification.h>
#include <mesa_pd/mpi/notifications/ParticleRemovalInformationNotification.h>
#include <mesa_pd/mpi/notifications/ParticleUpdateNotification.h>
#include <core/debug/Debug.h>
......@@ -70,14 +72,18 @@ void ParseMessage::operator()(int sender,
WALBERLA_LOG_DETAIL( "Received PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "from neighboring process with rank " << sender );
WALBERLA_CHECK_EQUAL( ps.find(objparam.uid), ps.end(), "Ghost particle with id " << objparam.uid << " already existend.");
if ( ps.find(objparam.uid) == ps.end() )
{
auto pIt = createNewParticle(ps, objparam);
auto pIt = createNewParticle(ps, objparam);
domain.correctParticlePosition(pIt->getPositionRef());
domain.correctParticlePosition(pIt->getPositionRef());
WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST));
data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST);
//WALBERLA_CHECK(!data::particle_flags::isSet(pIt->getFlags(), data::particle_flags::GHOST));
data::particle_flags::set(pIt->getFlagsRef(), data::particle_flags::GHOST);
} else
{
WALBERLA_LOG_DETAIL("Ghost particle with id " << objparam.uid << " already existend.");
}
WALBERLA_LOG_DETAIL( "Processed PARTICLE_COPY_NOTIFICATION for particle " << objparam.uid << "." );
......@@ -185,8 +191,61 @@ void ParseMessage::operator()(int sender,
break;
}
case NEW_GHOST_PARTICLE_NOTIFICATION: {
NewGhostParticleNotification::Parameters objparam;
rb >> objparam;
WALBERLA_LOG_DETAIL( "Received new ghost particle notification for particle " <<
objparam.uid_ <<
" from neighboring process with rank " <<
sender <<
"." );
auto pIt = ps.find( objparam.uid_ );
WALBERLA_CHECK_UNEQUAL( pIt, ps.end() );
pIt->getGhostOwnersRef().insert( objparam.newOwner_ );
WALBERLA_LOG_DETAIL( "Processed new ghost particle notification" );
break;
}
case PARTICLE_REMOVAL_INFORMATION_NOTIFICATION: {
ParticleRemovalInformationNotification::Parameters objparam;
rb >> objparam;
WALBERLA_LOG_DETAIL( "Received particle removal information notification for particle " <<
objparam.uid_ <<
" from neighboring process with rank " <<
sender <<
"." );
if (objparam.owner_ == receiver_)
{
using namespace walberla::mesa_pd::data::particle_flags;
auto pIt = ps.find( objparam.uid_ );
WALBERLA_CHECK_UNEQUAL( pIt, ps.end() );
WALBERLA_CHECK(!isSet( pIt->getFlags(), GHOST));
pIt->getGhostOwnersRef().erase( sender );
pIt->getNeighborStateRef().erase( sender );
} else
{
using namespace walberla::mesa_pd::data::particle_flags;
auto pIt = ps.find( objparam.uid_ );
if (pIt != ps.end() )
{
WALBERLA_CHECK(isSet( pIt->getFlags(), GHOST));
pIt->getNeighborStateRef().erase( sender );
}
}
WALBERLA_LOG_DETAIL( "Processed rigid body removal information notification" );
break;
}
default:
throw std::runtime_error( "Received invalid notification type." );
WALBERLA_ABORT( "Received invalid notification type: " << notificationType << " from sender: " << sender );
}
}
......
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ParticleRemovalInformationNotification.h
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//
//======================================================================================================================
//======================================================================================================================
//
// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!!
//
//======================================================================================================================
#pragma once
#include <mesa_pd/data/DataTypes.h>
#include <mesa_pd/data/ParticleStorage.h>
#include <mesa_pd/mpi/notifications/NotificationType.h>
#include <core/mpi/Datatype.h>
#include <core/mpi/RecvBuffer.h>
#include <core/mpi/SendBuffer.h>
namespace walberla {
namespace mesa_pd {
/**
* The ParticleRemovalInformationNotification class is used to signal other processes that a
* shadow copy was destroyed.
*/
class ParticleRemovalInformationNotification
{
public:
struct Parameters
{
id_t uid_;
walberla::mpi::MPIRank owner_;
};
inline explicit ParticleRemovalInformationNotification( const data::Particle& particle )
: particle_(particle)
{}
const data::Particle& particle_;
};
template<>
struct NotificationTrait<ParticleRemovalInformationNotification>
{
static const NotificationType id = PARTICLE_REMOVAL_INFORMATION_NOTIFICATION;
};
} // namespace mesa_pd
} // namespace walberla
//======================================================================================================================
//
// Send/Recv Buffer Serialization Specialization
//
//======================================================================================================================
namespace walberla {
namespace mpi {
template< typename T, // Element type of SendBuffer
typename G> // Growth policy of SendBuffer
mpi::GenericSendBuffer<T,G>& operator<<( mpi::GenericSendBuffer<T,G> & buf, const mesa_pd::ParticleRemovalInformationNotification& obj )
{
buf.addDebugMarker( "ri" );
buf << obj.particle_.getUid();
buf << static_cast<walberla::mpi::MPIRank>(obj.particle_.getOwner());
return buf;
}
template< typename T> // Element type of RecvBuffer
mpi::GenericRecvBuffer<T>& operator>>( mpi::GenericRecvBuffer<T> & buf, mesa_pd::ParticleRemovalInformationNotification::Parameters& objparam )
{
buf.readDebugMarker( "ri" );
buf >> objparam.uid_;
buf >> objparam.owner_;
return buf;
}
} // mpi
} // walberla
......@@ -37,6 +37,8 @@
#include <map>
#include <set>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
......@@ -411,6 +413,32 @@ GenericRecvBuffer<T>& operator>>( GenericRecvBuffer<T> & buf, std::set<CK, CC, C
template<typename T, typename C, typename A>
struct BufferSizeTrait< std::set<T,C,A> > { static const bool constantSize = false; };
template< typename T, // Element type of SendBuffer
typename G, // Growth policy of SendBuffer
typename CK, // Key type
typename CC, // Comparison type
typename CA> // Allocator type
GenericSendBuffer<T,G>& operator<<( GenericSendBuffer<T,G> & buf, const std::unordered_set<CK, CC, CA> & c )
{
buf.addDebugMarker( "us" );
sendAssocContainer(buf, c);
return buf;
}
template< typename T, // Element type of RecvBuffer
typename CK, // Key type