DynamicParMetis.h 7.07 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
//======================================================================================================================
//
//  This file is part of waLBerla. waLBerla is free software: you can
//  redistribute it and/or modify it under the terms of the GNU General Public
//  License as published by the Free Software Foundation, either version 3 of
//  the License, or (at your option) any later version.
//
//  waLBerla is distributed in the hope that it will be useful, but WITHOUT
//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
//  for more details.
//
//  You should have received a copy of the GNU General Public License along
//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file DynamicParMetis.h
//! \ingroup blockforest
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
//
//======================================================================================================================

Sebastian Eibl's avatar
Sebastian Eibl committed
22
23
#pragma once

24
25
#include "blockforest/PhantomBlockForest.h"

26
#include "core/debug/Debug.h"
27
28
29
30
#include "core/DataTypes.h"
#include "core/math/Vector3.h"
#include "core/mpi/MPIWrapper.h"

31
#include <cmath>
32
#include <map>
33
#include <vector>
34
35
36
37
38
39
40
41
42
43
44
45

namespace walberla {
namespace blockforest {

std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phantomForest, MPI_Comm comm );

std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest & phantomForest, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm );


class DynamicParMetis
{
public:
46
   enum Algorithm    { PARMETIS_PART_GEOM, PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
47
48
49
50
51
   enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 };
   enum EdgeSource   { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS };

   DynamicParMetis( const Algorithm algorithm = PARMETIS_PART_GEOM_KWAY,
                    const WeightsToUse weightsToUse = PARMETIS_BOTH_WEIGHTS,
52
53
54
                    const EdgeSource edgeSource = PARMETIS_EDGES_FROM_EDGE_WEIGHTS,
                    const uint_t ncon = uint_t(1))
      : algorithm_( algorithm ), weightsToUse_( weightsToUse ), edgeSource_( edgeSource ), ncon_( ncon ), ubvec_( ncon, real_t(1.05) ) {}
55
56
57
58
59
60

   bool operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
                    std::set< uint_t > & processesToRecvFrom,
                    const PhantomBlockForest & phantomForest,
                    const uint_t iteration ) const;

61
62
63
64

   void   setipc2redist(double val) {ipc2redist_ = val;}
   double getipc2redist() const {return ipc2redist_;}

65
66
67
68
   void setImbalanceTolerances(const std::vector<double> & tolerances){ WALBERLA_ASSERT_EQUAL(tolerances.size(), ncon_-uint_t(1)); ubvec_ = tolerances; }
   void setImbalanceTolerance(double tolerance, const uint_t con = 0){ WALBERLA_ASSERT_LESS(con, ncon_); ubvec_[con] = tolerance; }
   double getImbalanceTolerance(const uint_t con = 0) const { WALBERLA_ASSERT_LESS(con, ncon_); return ubvec_[con]; }

69
70
71
72
73
74
75
76
   bool edgeWeightsUsed()   const { return ( weightsToUse_ == PARMETIS_EDGE_WEIGHTS   ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
   bool vertexWeightsUsed() const { return ( weightsToUse_ == PARMETIS_VERTEX_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
   bool vertexSizeUsed()    const { return algorithm_ == PARMETIS_ADAPTIVE_REPART; }

   static Algorithm    stringToAlgorithm( std::string s );
   static WeightsToUse stringToWeightsToUse( std::string s );
   static EdgeSource   stringToEdgeSource( std::string s );

77
78
79
80
   std::string algorithmToString() const;
   std::string weightsToUseToString() const;
   std::string edgeSourceToString() const;

81
82
83
84
protected:
   Algorithm algorithm_;
   WeightsToUse weightsToUse_;
   EdgeSource edgeSource_;
85

86
87
   uint_t ncon_ = uint_t(1); ///< number of constraints
   std::vector<double> ubvec_; ///< imbalance tolerance for each constraint, value between 1 (perfect balance) and number of subdomains (= number of processes, perfect imbalance)
88
   double ipc2redist_ = real_t( 1000.0 ); ///< compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
89
90
91
92
93
94
95
96
97
};

class DynamicParMetisBlockInfo
{
public:

   typedef int64_t weight_t;
   typedef int64_t vsize_t;

98
99
   DynamicParMetisBlockInfo( const weight_t vertexWeight, const uint_t ncon = 1 )
      : vertexWeight_(ncon, vertexWeight), vertexSize_(1)
100
101
102
103
104
105
106
   { }

   DynamicParMetisBlockInfo( mpi::RecvBuffer & buffer )
   {
      buffer >> vertexWeight_ >> vertexSize_ >> vertexCoords_ >> edgeWeights_;
   }

107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
   DynamicParMetisBlockInfo( const std::vector<weight_t> & vertexWeight )
      : vertexWeight_(vertexWeight), vertexSize_(1)
   { }

   // get number of constraints ( = number of vertex weights), default: 1, > 1 for multi physics problems
   uint_t getNcon() const
   {
      return vertexWeight_.size();
   }

   weight_t getVertexWeight( const uint_t con = 0 ) const
   {
      WALBERLA_ASSERT_LESS(con, getNcon());
      return vertexWeight_[con];
   }
   void setVertexWeight( const weight_t vertexWeight, const uint_t con = 0 )
   {
      WALBERLA_ASSERT_LESS(con, getNcon());
      vertexWeight_[con] = vertexWeight;
   }
127

128
129
   vsize_t getVertexSize( ) const { return vertexSize_; }
   void setVertexSize( const vsize_t size) { vertexSize_ = size; }
130

131
132
133
134
135
   const Vector3<double> & getVertexCoords() const
   {
      WALBERLA_ASSERT( !std::isnan(vertexCoords_[0]) && !std::isnan(vertexCoords_[1]) && !std::isnan(vertexCoords_[2]) );
      return vertexCoords_;
   }
136
137
138
139
140
141
142
143
144
145
146
147
148
149
   void setVertexCoords( const Vector3<double> & p ) { vertexCoords_ = p; }

   void setEdgeWeight( const blockforest::BlockID & blockId, const weight_t edgeWeight ) { edgeWeights_[blockId] = edgeWeight; }
   void setEdgeWeights( const std::map< blockforest::BlockID, weight_t > & edgeWeights ) { edgeWeights_ = edgeWeights; }

   const std::map< blockforest::BlockID, weight_t > & getEdgeWeights() const { return edgeWeights_; }

   void toBuffer( mpi::SendBuffer & buffer )
   {
      buffer << vertexWeight_ << vertexSize_ << vertexCoords_ << edgeWeights_;
   }

private:

150
151
   std::vector<weight_t> vertexWeight_; /// Defines the ncon weights of a block
   vsize_t vertexSize_;    /// Defines the cost of rebalancing a block. Needed by some ParMetis algorithms
152
   Vector3<double> vertexCoords_ = Vector3<double>(std::numeric_limits<double>::signaling_NaN()); /// Defines where the block is located in space. Needed by some ParMetis algorithms
153
154
155
156
157
158
159
160
161
162
163
   std::map< blockforest::BlockID, weight_t > edgeWeights_; /// Defines the cost of communication with other blocks
};


struct DynamicParMetisBlockInfoPackUnpack
{
   void operator()( mpi::SendBuffer & buffer, const PhantomBlock & block )
   {
      block.getData< DynamicParMetisBlockInfo >().toBuffer( buffer );
   }

164
   void operator()( mpi::RecvBuffer & buffer, const PhantomBlock &, walberla::any & data )
165
166
167
168
169
170
171
172
173
174
175
176
177
   {
      data = DynamicParMetisBlockInfo( buffer );
   }
};







} // namespace blockforest
} // namespace walberla