diff --git a/src/field/communication/UniformPullReductionPackInfo.h b/src/field/communication/UniformPullReductionPackInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..71ceb7e8e6fba381d5623f80e3ca6f9bb2b8ae47 --- /dev/null +++ b/src/field/communication/UniformPullReductionPackInfo.h @@ -0,0 +1,153 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file UniformPullReductionPackInfo.h +//! \ingroup field +//! \author Tobias Schruff <schruff@iww.rwth-aachen.de> +//! \author Christoph Rettinger <christoph.rettinger@fau.de> +// +//====================================================================================================================== + +#pragma once + +#include "communication/UniformPackInfo.h" + +#include "core/debug/Debug.h" + +#include "field/GhostLayerField.h" + +#include "stencil/Directions.h" + + +namespace walberla { +namespace field { +namespace communication { + +/** + * Data packing/unpacking for ghost layer based communication of a single walberla::field::Field + * + * \ingroup comm + * + * template ReduceOperation is e.g. std::plus + * + * This pack info is used to apply a given reduce operation (e.g. +) to the values in the interior of a ghost layer field + * together with the values coming from the sender's ghost layer. + */ +template< template<typename> class ReduceOperation, typename GhostLayerField_T > +class UniformPullReductionPackInfo : public walberla::communication::UniformPackInfo +{ +public: + typedef typename GhostLayerField_T::value_type T; + + UniformPullReductionPackInfo( const BlockDataID & bdID ) : bdID_( bdID ), communicateAllGhostLayers_( true ), + numberOfGhostLayers_( 0 ) {} + + UniformPullReductionPackInfo( const BlockDataID & bdID, const uint_t numberOfGhostLayers ) : bdID_( bdID ), + communicateAllGhostLayers_( false ), numberOfGhostLayers_( numberOfGhostLayers ) {} + + virtual ~UniformPullReductionPackInfo() {} + + bool constantDataExchange() const { return mpi::BufferSizeTrait<T>::constantSize; } + bool threadsafeReceiving() const { return true; } + + void unpackData(IBlock * receiver, stencil::Direction dir, mpi::RecvBuffer & buffer); + + void communicateLocal(const IBlock * sender, IBlock * receiver, stencil::Direction dir); + +protected: + void packDataImpl(const IBlock * sender, stencil::Direction dir, mpi::SendBuffer & outBuffer) const; + + uint_t numberOfGhostLayersToCommunicate( const GhostLayerField_T * const field ) const; + + const BlockDataID bdID_; + bool communicateAllGhostLayers_; + uint_t numberOfGhostLayers_; + ReduceOperation<T> op_; +}; + + + +template< template<typename> class ReduceOperation, typename GhostLayerField_T > +void UniformPullReductionPackInfo<ReduceOperation, GhostLayerField_T>::unpackData( IBlock * receiver, stencil::Direction dir, mpi::RecvBuffer & buffer ) +{ + GhostLayerField_T * f = receiver->getData< GhostLayerField_T >( bdID_ ); + WALBERLA_ASSERT_NOT_NULLPTR(f); + + cell_idx_t nrOfGhostLayers = cell_idx_c( numberOfGhostLayersToCommunicate( f ) ); + + T buf( 0 ); + for( auto i = f->beginSliceBeforeGhostLayer( dir, nrOfGhostLayers ); i != f->end(); ++i ) { + buffer >> buf; + *i = op_( *i, buf ); + } +} + + + +template< template<typename> class ReduceOperation, typename GhostLayerField_T> +void UniformPullReductionPackInfo<ReduceOperation, GhostLayerField_T>::communicateLocal( const IBlock * sender, IBlock * receiver, stencil::Direction dir ) +{ + const GhostLayerField_T * sf = sender ->getData< GhostLayerField_T >( bdID_ ); + GhostLayerField_T * rf = receiver->getData< GhostLayerField_T >( bdID_ ); + + WALBERLA_ASSERT_EQUAL(sf->xSize(), rf->xSize()); + WALBERLA_ASSERT_EQUAL(sf->ySize(), rf->ySize()); + WALBERLA_ASSERT_EQUAL(sf->zSize(), rf->zSize()); + + uint_t nrOfGhostLayers = numberOfGhostLayersToCommunicate( sf ); + auto srcIter = sf->beginGhostLayerOnly( nrOfGhostLayers, dir ); + auto dstIter = rf->beginSliceBeforeGhostLayer( stencil::inverseDir[dir], cell_idx_c( nrOfGhostLayers ) ); + + while( srcIter != sf->end() ) { + *dstIter = op_( *srcIter, *dstIter ); + ++srcIter; + ++dstIter; + } + + WALBERLA_ASSERT(srcIter == sf->end() && dstIter == rf->end()); +} + + + +template< template<typename> class ReduceOperation, typename GhostLayerField_T> +void UniformPullReductionPackInfo<ReduceOperation, GhostLayerField_T>::packDataImpl( const IBlock * sender, stencil::Direction dir, mpi::SendBuffer & outBuffer ) const +{ + const GhostLayerField_T * f = sender->getData< GhostLayerField_T >( bdID_ ); + WALBERLA_ASSERT_NOT_NULLPTR(f); + + uint_t nrOfGhostLayers = numberOfGhostLayersToCommunicate( f ); + for( auto i = f->beginGhostLayerOnly( nrOfGhostLayers, dir ); i != f->end(); ++i ) + outBuffer << *i; +} + + + +template< template<typename> class ReduceOperation, typename GhostLayerField_T> +uint_t UniformPullReductionPackInfo<ReduceOperation, GhostLayerField_T>::numberOfGhostLayersToCommunicate( const GhostLayerField_T * const field ) const +{ + if( communicateAllGhostLayers_ ) + { + return field->nrOfGhostLayers(); + } + else + { + WALBERLA_ASSERT_LESS_EQUAL( numberOfGhostLayers_, field->nrOfGhostLayers() ); + return numberOfGhostLayers_; + } +} + +} // namespace communication +} // namespace field +} // namespace walberla diff --git a/src/field/communication/all.h b/src/field/communication/all.h index 55782d6e21d8503f473b12a58e10f74ac1d67a8b..a98de888790240fbf77b0454adaf5266ede9ec6b 100644 --- a/src/field/communication/all.h +++ b/src/field/communication/all.h @@ -27,3 +27,4 @@ #include "MPIDatatypes.h" #include "ReducePackInfo.h" #include "UniformMPIDatatypeInfo.h" +#include "UniformPullReductionPackInfo.h" diff --git a/tests/field/communication/FieldPackInfoTest.cpp b/tests/field/communication/FieldPackInfoTest.cpp index 45f9b00e51e56ada7bf184b095d49c6b88fbe161..d7b94789b8699ffd1b659b9cfdc192ac3fdac4c0 100644 --- a/tests/field/communication/FieldPackInfoTest.cpp +++ b/tests/field/communication/FieldPackInfoTest.cpp @@ -16,6 +16,7 @@ //! \file FieldPackInfoTest.cpp //! \ingroup field //! \author Martin Bauer <martin.bauer@fau.de> +//! \author Christoph Rettinger <christoph.rettinger@fau.de> //! \brief Tests if a Field is correctly packed into buffers // //====================================================================================================================== @@ -23,6 +24,7 @@ #include "field/AddToStorage.h" #include "field/GhostLayerField.h" #include "field/communication/PackInfo.h" +#include "field/communication/UniformPullReductionPackInfo.h" #include "blockforest/Initialization.h" @@ -85,9 +87,60 @@ void testScalarField( IBlock * block, BlockDataID fieldId ) WALBERLA_CHECK_EQUAL ( field(1,1,+2), 4 ); } +void testScalarFieldPullReduction( IBlock * block, BlockDataID fieldId ) +{ + GhostLayerField<int,1> & field = *(block->getData<GhostLayerField<int,1> > (fieldId)); + field.setWithGhostLayer( 0 ); + WALBERLA_CHECK_EQUAL(field.xSize(), 2); + WALBERLA_CHECK_EQUAL(field.ySize(), 2); + WALBERLA_CHECK_EQUAL(field.zSize(), 2); + // initialize the bottom ghost layer cells + field(0,0,-1) = 1; + field(0,1,-1) = 2; + field(1,0,-1) = 3; + field(1,1,-1) = 4; + + // initialize the top interior cells + field(0,0,1) = 1; + field(0,1,1) = 1; + field(1,0,1) = 1; + field(1,1,1) = 1; + + // communicate periodic from bottom to top with uniform pull scheme + field::communication::UniformPullReductionPackInfo<std::plus, GhostLayerField<int,1> > pi1 (fieldId); + pi1.communicateLocal( block, block, stencil::B ); + + // check values in top ghost layer + WALBERLA_CHECK_EQUAL ( field(0,0,2), 0 ); + WALBERLA_CHECK_EQUAL ( field(0,1,2), 0 ); + WALBERLA_CHECK_EQUAL ( field(1,0,2), 0 ); + WALBERLA_CHECK_EQUAL ( field(1,1,2), 0 ); + + // check values in top interior cells + WALBERLA_CHECK_EQUAL ( field(0,0,1), 2 ); + WALBERLA_CHECK_EQUAL ( field(0,1,1), 3 ); + WALBERLA_CHECK_EQUAL ( field(1,0,1), 4 ); + WALBERLA_CHECK_EQUAL ( field(1,1,1), 5 ); + + // communicate periodic from top to bottom with standard form to sync ghost layers + field::communication::PackInfo< GhostLayerField<int,1> > pi2 (fieldId); + pi2.communicateLocal( block, block, stencil::T ); + + // check values in bottom ghost layer + WALBERLA_CHECK_EQUAL ( field(0,0,-1), 2 ); + WALBERLA_CHECK_EQUAL ( field(0,1,-1), 3 ); + WALBERLA_CHECK_EQUAL ( field(1,0,-1), 4 ); + WALBERLA_CHECK_EQUAL ( field(1,1,-1), 5 ); + + // check values in top interior cells + WALBERLA_CHECK_EQUAL ( field(0,0,1), 2 ); + WALBERLA_CHECK_EQUAL ( field(0,1,1), 3 ); + WALBERLA_CHECK_EQUAL ( field(1,0,1), 4 ); + WALBERLA_CHECK_EQUAL ( field(1,1,1), 5 ); +} int main(int argc, char **argv) { @@ -127,7 +180,8 @@ int main(int argc, char **argv) for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt ) // block loop testScalarField( &(*blockIt), scalarFieldId ); - + for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt ) // block loop + testScalarFieldPullReduction( &(*blockIt), scalarFieldId ); return 0; }