Commit 8783776c by Sebastian Eibl

Merge branch 'ParMetisFix' into 'master'

Fix calling of parmetis See merge request !88
parents 252cc28b 2c44004b
Pipeline #7916 failed with stages
in 421 minutes 3 seconds
......@@ -23,10 +23,12 @@
#include "blockforest/PhantomBlockForest.h"
#include "core/debug/Debug.h"
#include "core/DataTypes.h"
#include "core/math/Vector3.h"
#include "core/mpi/MPIWrapper.h"
#include <cmath>
#include <map>
namespace walberla {
......@@ -40,7 +42,7 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
class DynamicParMetis
{
public:
enum Algorithm { PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum Algorithm { PARMETIS_PART_GEOM, PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 };
enum EdgeSource { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS };
......@@ -54,6 +56,10 @@ public:
const PhantomBlockForest & phantomForest,
const uint_t iteration ) const;
void setipc2redist(double val) {ipc2redist_ = val;}
double getipc2redist() const {return ipc2redist_;}
bool edgeWeightsUsed() const { return ( weightsToUse_ == PARMETIS_EDGE_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexWeightsUsed() const { return ( weightsToUse_ == PARMETIS_VERTEX_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexSizeUsed() const { return algorithm_ == PARMETIS_ADAPTIVE_REPART; }
......@@ -62,10 +68,16 @@ public:
static WeightsToUse stringToWeightsToUse( std::string s );
static EdgeSource stringToEdgeSource( std::string s );
std::string algorithmToString() const;
std::string weightsToUseToString() const;
std::string edgeSourceToString() const;
protected:
Algorithm algorithm_;
WeightsToUse weightsToUse_;
EdgeSource edgeSource_;
double ipc2redist_ = real_t( 1000.0 ); ///< compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
};
class DynamicParMetisBlockInfo
......@@ -90,7 +102,7 @@ public:
vsize_t getVertexSize() const { return vertexSize_; }
void setVertexSize( const vsize_t size ) { vertexSize_ = size; }
const Vector3<double> & getVertexCoords() const { return vertexCoords_; }
const Vector3<double> & getVertexCoords() const { WALBERLA_ASSERT( !std::isnan(vertexCoords_[0]) && !std::isnan(vertexCoords_[1]) && !std::isnan(vertexCoords_[2]) ); return vertexCoords_; }
void setVertexCoords( const Vector3<double> & p ) { vertexCoords_ = p; }
void setEdgeWeight( const blockforest::BlockID & blockId, const weight_t edgeWeight ) { edgeWeights_[blockId] = edgeWeight; }
......@@ -107,7 +119,7 @@ private:
weight_t vertexWeight_; /// Defines the weight of a block
vsize_t vertexSize_; /// Defines the cost of rebalancing a block
Vector3<double> vertexCoords_; /// Defines where the block is located in space. Needed by some ParMetis algorithms
Vector3<double> vertexCoords_ = Vector3<double>(std::numeric_limits<double>::signaling_NaN()); /// Defines where the block is located in space. Needed by some ParMetis algorithms
std::map< blockforest::BlockID, weight_t > edgeWeights_; /// Defines the cost of communication with other blocks
};
......
......@@ -20,28 +20,67 @@
#include "MPIHelper.h"
#include <core/debug/CheckFunctions.h>
namespace walberla {
namespace mpi {
//!
//! \brief This functions maps the rank in one communicator to the rank in another communicator.
//! \param srcComm source communicator
//! \param destComm destination communicator
//! \param srcRank rank in the source communicator
//! \return rank in the destination communicator or -1 if not available
//!
int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int srcRank)
{
if (srcComm == destComm)
{
return srcRank;
}
int destRank = -1;
MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, 1, const_cast<int*>(&srcRank), destGroup, &destRank);
int size;
MPI_Comm_size(destComm, &size);
if (destRank == MPI_UNDEFINED) destRank = -1;
WALBERLA_CHECK_GREATER_EQUAL(destRank, -1);
WALBERLA_CHECK_LESS(destRank, size);
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
}
//!
//! \brief This functions converts a array of ranks in one communicator to an array of ranks in another communicator.
//! \param srcComm source communicator
//! \param destComm destination communicator
//! \param srcRank source ranks
//! \return converted ranks, -1 if not available
//!
std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const std::vector<int>& srcRank)
{
if (srcComm == destComm)
{
return srcRank;
}
std::vector<int> destRank(srcRank.size(), -1);
MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, int_c(srcRank.size()), const_cast<int*>(&srcRank[0]), destGroup, &destRank[0]);
int size;
MPI_Comm_size(destComm, &size);
for (auto& dstRnk : destRank)
{
if (dstRnk == MPI_UNDEFINED) dstRnk = -1;
WALBERLA_CHECK_GREATER_EQUAL(dstRnk, -1);
WALBERLA_CHECK_LESS(dstRnk, size);
}
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
......
......@@ -35,26 +35,38 @@ public:
typedef blockforest::DynamicParMetisBlockInfo PhantomBlockWeight;
typedef blockforest::DynamicParMetisBlockInfoPackUnpack PhantomBlockWeightPackUnpackFunctor;
MetisAssignmentFunctor( const shared_ptr<InfoCollection>& ic ) : ic_( ic )
{}
MetisAssignmentFunctor( shared_ptr<InfoCollection>& ic, const real_t baseWeight = real_t(10.0) ) : ic_(ic), baseWeight_(baseWeight) {}
void operator()( std::vector< std::pair< const PhantomBlock *, walberla::any > > & blockData, const PhantomBlockForest & )
{
for( auto it = blockData.begin(); it != blockData.end(); ++it )
{
const uint_t& weight = ic_->find( it->first->getId() )->second.numberOfLocalBodies;
blockforest::DynamicParMetisBlockInfo info( int64_c(weight) );
info.setVertexSize(int64_c( weight ));
const double weight = double_c( ic_->find( it->first->getId() )->second.numberOfLocalBodies ) + baseWeight_;
//const double commWeight = double_c( edgeWeightFactor * (double_c(ic_->find( it->first->getId() )->second.numberOfShadowBodies) + baseWeight_)) + 1;
blockforest::DynamicParMetisBlockInfo info( 0 );
info.setVertexWeight( int64_c(weight) );
info.setVertexSize( int64_c( weight ) );
info.setVertexCoords( it->first->getAABB().center() );
for( uint_t nb = uint_t(0); nb < it->first->getNeighborhoodSize(); ++nb )
{
info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(weight) );
info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(edgeWeight_) );
}
it->second = info;
}
}
inline void setBaseWeight( const double weight) { baseWeight_ = weight;}
inline double getBaseWeight() const { return baseWeight_; }
inline void setEdgeWeight( const double weight) { edgeWeight_ = weight;}
inline double getEdgeWeight() const { return edgeWeight_; }
private:
shared_ptr< InfoCollection > ic_;
///Base weight due to allocated data structures. A weight of zero for blocks is dangerous as empty blocks might accumulate on one process!
double baseWeight_ = 10.0;
double edgeWeight_ = 1.0;
};
}
......
......@@ -69,6 +69,11 @@ waLBerla_execute_test( NAME PE_OVERLAP )
waLBerla_compile_test( NAME PE_PARALLELEQUIVALENCE FILES ParallelEquivalence.cpp DEPENDS core blockforest )
waLBerla_execute_test( NAME PE_PARALLELEQUIVALENCE PROCESSES 4 )
if( WALBERLA_BUILD_WITH_PARMETIS )
waLBerla_compile_test( NAME PE_PARMETIS FILES ParMetis.cpp DEPENDS core blockforest )
waLBerla_execute_test( NAME PE_PARMETIS PROCESSES 64 )
endif()
waLBerla_compile_test( NAME PE_PARSEMESSAGE FILES ParseMessage.cpp DEPENDS core )
waLBerla_execute_test( NAME PE_PARSEMESSAGE )
......
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ParMetis.cpp
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//
//======================================================================================================================
#include "pe/utility/CreateWorld.h"
#include <blockforest/loadbalancing/DynamicParMetis.h>
#include <core/Environment.h>
#include <core/logging/Logging.h>
#include <core/math/Sample.h>
using namespace walberla;
using namespace walberla::pe;
class ReGrid
{
public:
void operator()( std::vector< std::pair< const Block *, uint_t > > & minTargetLevels,
std::vector< const Block * > &, const BlockForest & /*forest*/ )
{
std::for_each( minTargetLevels.begin(),
minTargetLevels.end(),
[](auto& pair){pair.second = pair.first->getLevel() + 1;} );
}
};
class MetisAssignmentFunctor
{
public:
typedef blockforest::DynamicParMetisBlockInfo PhantomBlockWeight;
typedef blockforest::DynamicParMetisBlockInfoPackUnpack PhantomBlockWeightPackUnpackFunctor;
void operator()( std::vector< std::pair< const PhantomBlock *, walberla::any > > & blockData, const PhantomBlockForest & )
{
for( auto it = blockData.begin(); it != blockData.end(); ++it )
{
const auto& corner = it->first->getAABB().maxCorner();
const int weight = int_c( corner[0] + corner[1] + corner[2] );
blockforest::DynamicParMetisBlockInfo info(0);
info.setVertexWeight( weight );
info.setVertexSize( weight );
info.setVertexCoords( it->first->getAABB().center() );
for( uint_t nb = uint_t(0); nb < it->first->getNeighborhoodSize(); ++nb )
{
info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(weight) );
}
it->second = info;
}
}
};
int parmetisTest(const std::string& algorithm,
const std::string& weightsToUse,
const std::string& edgeSource)
{
walberla::MPIManager::instance()->resetMPI();
walberla::MPIManager::instance()->useWorldComm();
WALBERLA_LOG_INFO_ON_ROOT("****** " << algorithm << " | " << weightsToUse << " | " << edgeSource);
// create forest
shared_ptr< BlockForest > forest = createBlockForest( math::AABB(0,0,0,40,40,40),
Vector3<uint_t>(4, 4, 4),
Vector3<bool>(false, false, false),
64,
0);
if (!forest)
{
WALBERLA_LOG_INFO_ON_ROOT( "No BlockForest created ... exiting!");
return EXIT_SUCCESS;
}
forest->setRefreshMinTargetLevelDeterminationFunction( ReGrid() );
auto assFunc = MetisAssignmentFunctor();
forest->setRefreshPhantomBlockDataAssignmentFunction( assFunc );
forest->setRefreshPhantomBlockDataPackFunction( MetisAssignmentFunctor::PhantomBlockWeightPackUnpackFunctor() );
forest->setRefreshPhantomBlockDataUnpackFunction( MetisAssignmentFunctor::PhantomBlockWeightPackUnpackFunctor() );
auto alg = blockforest::DynamicParMetis::stringToAlgorithm( algorithm );
auto vWeight = blockforest::DynamicParMetis::stringToWeightsToUse( weightsToUse );
auto eWeight = blockforest::DynamicParMetis::stringToEdgeSource( edgeSource );
auto prepFunc = blockforest::DynamicParMetis( alg, vWeight, eWeight );
forest->setRefreshPhantomBlockMigrationPreparationFunction( prepFunc );
forest->refresh();
math::Sample numBlocks;
numBlocks.castToRealAndInsert( forest->size() );
numBlocks.mpiGatherRoot();
WALBERLA_LOG_INFO_ON_ROOT("#blocks: " << numBlocks.format() );
int weight = 0;
for (const auto& block : *forest)
{
const auto& corner = block.getAABB().maxCorner();
weight += int_c( corner[0] + corner[1] + corner[2] );
}
math::Sample weightSample;
weightSample.castToRealAndInsert( weight );
weightSample.mpiGatherRoot();
WALBERLA_LOG_INFO_ON_ROOT("#weights: " << weightSample.format() );
return EXIT_SUCCESS;
}
int main( int argc, char ** argv )
{
walberla::debug::enterTestMode();
walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
walberla::MPIManager::instance()->useWorldComm();
std::vector<std::string> algs = {"PART_GEOM", "PART_GEOM_KWAY", "PART_KWAY", "PART_ADAPTIVE_REPART", "REFINE_KWAY"};
std::vector<std::string> wtu = {"NO_WEIGHTS", "EDGE_WEIGHTS", "VERTEX_WEIGHTS", "BOTH_WEIGHTS"};
std::vector<std::string> es = {"EDGES_FROM_FOREST", "EDGES_FROM_EDGE_WEIGHTS"};
for (const auto& a : algs)
for (const auto& w : wtu)
for (const auto& e : es)
{
parmetisTest(a,w,e);
}
return EXIT_SUCCESS;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment