From 5c456a60f04e879750d3eead07c4f07652732aaf Mon Sep 17 00:00:00 2001
From: Sebastian Eibl <sebastian.eibl@fau.de>
Date: Fri, 8 Dec 2017 17:19:47 +0100
Subject: [PATCH] made DynamicCurve balance levelwise optional
---
src/blockforest/loadbalancing/DynamicCurve.h | 91 ++++++++++++--------
1 file changed, 56 insertions(+), 35 deletions(-)
diff --git a/src/blockforest/loadbalancing/DynamicCurve.h b/src/blockforest/loadbalancing/DynamicCurve.h
index 3cb88476c..14a08f559 100644
--- a/src/blockforest/loadbalancing/DynamicCurve.h
+++ b/src/blockforest/loadbalancing/DynamicCurve.h
@@ -22,6 +22,7 @@
#pragma once
#include "NoPhantomData.h"
+#include "blockforest/BlockForest.h"
#include "blockforest/HilbertCurveConstruction.h"
#include "blockforest/PhantomBlockForest.h"
@@ -88,9 +89,14 @@ struct Node
} // namespace internal
-
+/**
+ * This class implements Hilber and Morton space filling curves for load balancing.
+ *
+ * All algorithms are implemented to work levelwise. Load balancing with levels ignored is possible
+ * by specifying levelwise = false in the constructor.
+**/
template< typename PhantomData_T >
-class DynamicLevelwiseCurveBalance
+class DynamicCurveBalance
{
public:
@@ -99,8 +105,8 @@ public:
typedef uint16_t idx_t; // limits the maximum number of blocks per process to 65536
typedef internal::Node< pid_t, idx_t > Node;
- DynamicLevelwiseCurveBalance( const bool hilbert = true, const bool allGather = true ) :
- hilbert_( hilbert ), allGather_( allGather )
+ DynamicCurveBalance( const bool hilbert = true, const bool allGather = true, const bool levelwise = true ) :
+ hilbert_( hilbert ), allGather_( allGather ), levelwise_(levelwise)
{}
bool operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
@@ -180,12 +186,17 @@ private:
bool hilbert_;
bool allGather_;
+ /// All gets for levels are wrapped like
+ /// \code levelwise_ ? getCorrectLevel() : 0
+ ///
+ /// This allows to use the same algorithm for levelwise balancing as well as for balancing without levels.
+ bool levelwise_;
};
template< typename PhantomData_T >
-bool DynamicLevelwiseCurveBalance< PhantomData_T >::operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
+bool DynamicCurveBalance< PhantomData_T >::operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest, const uint_t ) const
{
@@ -214,7 +225,7 @@ bool DynamicLevelwiseCurveBalance< PhantomData_T >::operator()( std::vector< std
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherWeighted( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
+void DynamicCurveBalance< PhantomData_T >::allGatherWeighted( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest ) const
{
@@ -241,7 +252,8 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherWeighted( std::vect
recvBuffer >> allBlocks[p];
recvBuffer.reset();
- std::vector< std::vector< std::pair< pid_t, idx_t > > > blocksPerLevel( phantomForest.getNumberOfLevels() ); // for every level one vector of pair(source process ID, index in 'allBlocks')
+ const uint_t numLevels = levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1);
+ std::vector< std::vector< std::pair< pid_t, idx_t > > > blocksPerLevel( numLevels ); // for every level one vector of pair(source process ID, index in 'allBlocks')
if( hilbert_ )
hilbertOrderWeighted( allBlocks, blocksPerLevel, phantomForest );
@@ -259,7 +271,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherWeighted( std::vect
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherNoWeight( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
+void DynamicCurveBalance< PhantomData_T >::allGatherNoWeight( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest ) const
{
@@ -283,7 +295,8 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherNoWeight( std::vect
recvBuffer >> allBlocks[p];
recvBuffer.reset();
- std::vector< std::vector< std::pair< pid_t, idx_t > > > blocksPerLevel( phantomForest.getNumberOfLevels() ); // for every level one vector of pair(source process ID, index in 'allBlocks')
+ const uint_t numLevels = levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1);
+ std::vector< std::vector< std::pair< pid_t, idx_t > > > blocksPerLevel( numLevels ); // for every level one vector of pair(source process ID, index in 'allBlocks')
if( hilbert_ )
hilbertOrderNoWeight( allBlocks, blocksPerLevel, phantomForest );
@@ -301,7 +314,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::allGatherNoWeight( std::vect
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::masterWeighted( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
+void DynamicCurveBalance< PhantomData_T >::masterWeighted( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest ) const
{
@@ -352,7 +365,8 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterWeighted( std::vector<
WALBERLA_ASSERT( allBlocks[0].empty() );
allBlocks[0] = localBlocks;
- blocksPerLevel.resize( phantomForest.getNumberOfLevels() );
+ const uint_t numLevels = levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1);
+ blocksPerLevel.resize( numLevels );
if( hilbert_ )
hilbertOrderWeighted( allBlocks, blocksPerLevel, phantomForest );
@@ -377,7 +391,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterWeighted( std::vector<
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::masterNoWeight( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
+void DynamicCurveBalance< PhantomData_T >::masterNoWeight( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest ) const
{
@@ -425,7 +439,8 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterNoWeight( std::vector<
WALBERLA_ASSERT( allBlocks[0].empty() );
allBlocks[0] = localBlocks;
- blocksPerLevel.resize( phantomForest.getNumberOfLevels() );
+ const uint_t numLevels = levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1);
+ blocksPerLevel.resize( numLevels );
if( hilbert_ )
hilbertOrderNoWeight( allBlocks, blocksPerLevel, phantomForest );
@@ -450,9 +465,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterNoWeight( std::vector<
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::hilbertOrderWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
- const PhantomBlockForest & phantomForest ) const
+ const PhantomBlockForest & phantomForest) const
{
// construct forest of octrees
@@ -519,8 +534,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderWeighted( const
else
{
auto & index = node->index_;
- WALBERLA_ASSERT_LESS( blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ].first ), phantomForest.getNumberOfLevels() );
- blocksPerLevel[ blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ].first ) ].push_back( index );
+ const uint_t level = levelwise_ ? blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ].first ) : uint_t(0);
+ WALBERLA_ASSERT_LESS( level, levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1) );
+ blocksPerLevel[ level ].push_back( index );
}
}
}
@@ -539,9 +555,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderWeighted( const
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::hilbertOrderNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
- const PhantomBlockForest & phantomForest ) const
+ const PhantomBlockForest & phantomForest) const
{
// construct forest of octrees
@@ -608,8 +624,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderNoWeight( const
else
{
auto & index = node->index_;
- WALBERLA_ASSERT_LESS( blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ] ), phantomForest.getNumberOfLevels() );
- blocksPerLevel[ blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ] ) ].push_back( index );
+ const uint_t level = levelwise_ ? blockforest.getLevelFromBlockId( allBlocks[ uint_c(index.first) ][ index.second ] ) : uint_t(0);
+ WALBERLA_ASSERT_LESS( level , levelwise_ ? phantomForest.getNumberOfLevels() : uint_t(1) );
+ blocksPerLevel[ level ].push_back( index );
}
}
}
@@ -628,7 +645,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::hilbertOrderNoWeight( const
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::addBlockToForest( std::vector< shared_ptr< Node > > & forest,
+void DynamicCurveBalance< PhantomData_T >::addBlockToForest( std::vector< shared_ptr< Node > > & forest,
const std::pair< pid_t, idx_t > & index, BlockID & id, const uint_t level ) const
{
std::stack< uint_t > path;
@@ -681,9 +698,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::addBlockToForest( std::vecto
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::mortonOrderWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
- const PhantomBlockForest & phantomForest ) const
+ const PhantomBlockForest & phantomForest) const
{
const uint_t processes = uint_c( mpi::MPIManager::instance()->numProcesses() );
@@ -691,8 +708,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderWeighted( const s
{
for( uint_t i = uint_t(0); i != allBlocks[p].size(); ++i )
{
- WALBERLA_ASSERT_LESS( phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i].first ), blocksPerLevel.size() );
- blocksPerLevel[ phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i].first ) ].push_back( std::make_pair( pid_c(p), idx_c(i) ) );
+ uint_t level = levelwise_ ? phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i].first ) : uint_t(0);
+ WALBERLA_ASSERT_LESS( level, blocksPerLevel.size() );
+ blocksPerLevel[ level ].push_back( std::make_pair( pid_c(p), idx_c(i) ) );
}
}
@@ -710,9 +728,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderWeighted( const s
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::mortonOrderNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
- const PhantomBlockForest & phantomForest ) const
+ const PhantomBlockForest & phantomForest) const
{
const uint_t processes = uint_c( mpi::MPIManager::instance()->numProcesses() );
@@ -720,8 +738,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderNoWeight( const s
{
for( uint_t i = uint_t(0); i != allBlocks[p].size(); ++i )
{
- WALBERLA_ASSERT_LESS( phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i] ), blocksPerLevel.size() );
- blocksPerLevel[ phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i] ) ].push_back( std::make_pair( pid_c(p), idx_c(i) ) );
+ uint_t level = levelwise_ ? phantomForest.getBlockForest().getLevelFromBlockId( allBlocks[p][i] ) : uint_t(0);
+ WALBERLA_ASSERT_LESS( level, blocksPerLevel.size() );
+ blocksPerLevel[ level ].push_back( std::make_pair( pid_c(p), idx_c(i) ) );
}
}
@@ -739,7 +758,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::mortonOrderNoWeight( const s
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::balanceWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::balanceWeighted( const std::vector< std::vector< std::pair< BlockID, typename PhantomData_T::weight_t > > > & allBlocks,
const std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
std::vector< std::vector<pid_t> > & targets,
std::vector< std::set<pid_t> > & sender ) const
@@ -790,7 +809,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::balanceWeighted( const std::
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::balanceNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
+void DynamicCurveBalance< PhantomData_T >::balanceNoWeight( const std::vector< std::vector< BlockID > > & allBlocks,
const std::vector< std::vector< std::pair< pid_t, idx_t > > > & blocksPerLevel,
std::vector< std::vector<pid_t> > & targets,
std::vector< std::set<pid_t> > & sender ) const
@@ -825,7 +844,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::balanceNoWeight( const std::
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::masterEnd( std::vector< std::vector<pid_t> > & targets,
+void DynamicCurveBalance< PhantomData_T >::masterEnd( std::vector< std::vector<pid_t> > & targets,
std::vector< std::set<pid_t> > & sender,
std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom ) const
@@ -893,7 +912,7 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::masterEnd( std::vector< std:
template< typename PhantomData_T >
-void DynamicLevelwiseCurveBalance< PhantomData_T >::finalAssignment( const uint_t index, const std::vector< std::vector<pid_t> > & targets,
+void DynamicCurveBalance< PhantomData_T >::finalAssignment( const uint_t index, const std::vector< std::vector<pid_t> > & targets,
const std::vector< std::set<pid_t> > & sender,
std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom ) const
@@ -907,7 +926,9 @@ void DynamicLevelwiseCurveBalance< PhantomData_T >::finalAssignment( const uint_
processesToRecvFrom.insert( uint_c(*s) ) ;
}
-
+///This class is deprecated use DynamicCurveBalance instead.
+template< typename PhantomData_T >
+using DynamicLevelwiseCurveBalance = DynamicCurveBalance<PhantomData_T> ;
} // namespace blockforest
} // namespace walberla
--
GitLab