Commit 6d7530fd authored by Marcel Koch's avatar Marcel Koch
Browse files

constrain globalInfo to active process subset

parent 7fe55a7f
Pipeline #34754 failed with stages
in 12 minutes and 20 seconds
...@@ -1613,7 +1613,7 @@ void PrimitiveStorage::initializeAndDeserializeAllPrimitiveData( walberla::mpi:: ...@@ -1613,7 +1613,7 @@ void PrimitiveStorage::initializeAndDeserializeAllPrimitiveData( walberla::mpi::
} }
} }
std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly, MPI_Comm comm ) const
{ {
uint_t globalNumberOfVertices; uint_t globalNumberOfVertices;
uint_t globalNumberOfEdges; uint_t globalNumberOfEdges;
...@@ -1635,46 +1635,47 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const ...@@ -1635,46 +1635,47 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const
if ( onRootOnly ) if ( onRootOnly )
{ {
globalNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::SUM ); globalNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::SUM, 0, comm );
globalNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::SUM ); globalNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::SUM, 0, comm );
globalNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::SUM ); globalNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::SUM, 0, comm );
globalNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::SUM ); globalNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::SUM, 0, comm );
globalNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM ); globalNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM, 0, comm );
globalMaxNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MAX ); globalMaxNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MAX ); globalMaxNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MAX ); globalMaxNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MAX ); globalMaxNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX ); globalMaxNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX, 0, comm );
globalMinNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MIN ); globalMinNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MIN ); globalMinNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MIN ); globalMinNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MIN ); globalMinNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN ); globalMinNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN, 0, comm );
} }
else else
{ {
globalNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::SUM ); globalNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::SUM, comm );
globalNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::SUM ); globalNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::SUM, comm );
globalNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::SUM ); globalNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::SUM, comm );
globalNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::SUM ); globalNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::SUM, comm );
globalNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM ); globalNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM, comm );
globalMaxNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MAX ); globalMaxNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MAX, comm );
globalMaxNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MAX ); globalMaxNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MAX, comm );
globalMaxNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MAX ); globalMaxNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MAX, comm );
globalMaxNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MAX ); globalMaxNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MAX, comm );
globalMaxNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX ); globalMaxNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX, comm );
globalMinNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MIN ); globalMinNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MIN, comm );
globalMinNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MIN ); globalMinNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MIN, comm );
globalMinNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MIN ); globalMinNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MIN, comm );
globalMinNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MIN ); globalMinNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MIN, comm );
globalMinNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN ); globalMinNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN, comm );
} }
const uint_t numberOfProcesses = uint_c( walberla::mpi::MPIManager::instance()->numProcesses() ); int numberOfProcesses;
MPI_Comm_size(comm, &numberOfProcesses);
const double globalAvgNumberOfVertices = (double) globalNumberOfVertices / (double) numberOfProcesses; const double globalAvgNumberOfVertices = (double) globalNumberOfVertices / (double) numberOfProcesses;
const double globalAvgNumberOfEdges = (double) globalNumberOfEdges / (double) numberOfProcesses; const double globalAvgNumberOfEdges = (double) globalNumberOfEdges / (double) numberOfProcesses;
...@@ -1682,8 +1683,8 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const ...@@ -1682,8 +1683,8 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const
const double globalAvgNumberOfCells = (double) globalNumberOfCells / (double) numberOfProcesses; const double globalAvgNumberOfCells = (double) globalNumberOfCells / (double) numberOfProcesses;
const double globalAvgNumberOfPrimitives = (double) globalNumberOfPrimitives / (double) numberOfProcesses; const double globalAvgNumberOfPrimitives = (double) globalNumberOfPrimitives / (double) numberOfProcesses;
walberla::math::DistributedSample neighborhoodSample; walberla::math::DistributedSample neighborhoodSample(comm);
walberla::math::DistributedSample neighborhoodVolumeSample; walberla::math::DistributedSample neighborhoodVolumeSample(comm);
const auto numNeighborProcesses = getNeighboringRanks().size(); const auto numNeighborProcesses = getNeighboringRanks().size();
const auto numNeighborVolumeProcesses = additionalHaloDepth_ > 0 ? getNeighboringVolumeRanksOfAllVolumes().size() : 0; const auto numNeighborVolumeProcesses = additionalHaloDepth_ > 0 ? getNeighboringVolumeRanksOfAllVolumes().size() : 0;
neighborhoodSample.castToRealAndInsert( numNeighborProcesses ); neighborhoodSample.castToRealAndInsert( numNeighborProcesses );
......
...@@ -441,7 +441,7 @@ class PrimitiveStorage : private walberla::NonCopyable ...@@ -441,7 +441,7 @@ class PrimitiveStorage : private walberla::NonCopyable
/// Returns a formatted string that contains global information about the storage. /// Returns a formatted string that contains global information about the storage.
/// Must be called by all processes! /// Must be called by all processes!
/// Involves global communication and should therefore not be called in performance critical code. /// Involves global communication and should therefore not be called in performance critical code.
std::string getGlobalInfo( bool onRootOnly = false ) const; std::string getGlobalInfo( bool onRootOnly = false, MPI_Comm comm = MPI_COMM_WORLD ) const;
/// \brief Returns the global number of primitives. /// \brief Returns the global number of primitives.
/// ///
......
diff --git a/src/core/math/DistributedSample.cpp b/src/core/math/DistributedSample.cpp
index 965ab816..6ed38d70 100644
--- a/src/core/math/DistributedSample.cpp
+++ b/src/core/math/DistributedSample.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -54,10 +54,10 @@ void DistributedSample::mpiAllGather()
WALBERLA_MPI_SECTION()
{
- mpi::allReduceInplace( sum_, mpi::SUM );
- mpi::allReduceInplace( min_, mpi::MIN );
- mpi::allReduceInplace( max_, mpi::MAX );
- mpi::allReduceInplace( size_, mpi::SUM );
+ mpi::allReduceInplace( sum_, mpi::SUM, comm_ );
+ mpi::allReduceInplace( min_, mpi::MIN, comm_ );
+ mpi::allReduceInplace( max_, mpi::MAX, comm_ );
+ mpi::allReduceInplace( size_, mpi::SUM, comm_ );
}
mean_ = sum_ / real_c(size_);
@@ -71,11 +71,11 @@ void DistributedSample::mpiAllGather()
WALBERLA_MPI_SECTION()
{
- mpi::allReduceInplace( variance_, mpi::SUM );
+ mpi::allReduceInplace( variance_, mpi::SUM, comm_ );
}
variance_ /= real_c(size_);
-
+
if( size_ == uint_t(0) )
{
min_ = real_t(0);
@@ -113,10 +113,10 @@ void DistributedSample::mpiGather( int rank )
WALBERLA_MPI_SECTION()
{
- mpi::reduceInplace( sum_, mpi::SUM, rank );
- mpi::reduceInplace( min_, mpi::MIN, rank );
- mpi::reduceInplace( max_, mpi::MAX, rank );
- mpi::reduceInplace( size_, mpi::SUM, rank );
+ mpi::reduceInplace( sum_, mpi::SUM, rank, comm_ );
+ mpi::reduceInplace( min_, mpi::MIN, rank, comm_ );
+ mpi::reduceInplace( max_, mpi::MAX, rank, comm_ );
+ mpi::reduceInplace( size_, mpi::SUM, rank, comm_ );
}
mean_ = sum_ / real_c(size_);
@@ -130,18 +130,18 @@ void DistributedSample::mpiGather( int rank )
WALBERLA_MPI_SECTION()
{
- mpi::reduceInplace( variance_, mpi::SUM, rank );
+ mpi::reduceInplace( variance_, mpi::SUM, rank, comm_ );
}
variance_ /= real_c(size_);
-
+
if( size_ == uint_t(0) )
{
min_ = real_t(0);
max_ = real_t(0);
mean_ = real_t(0);
variance_ = real_t(0);
- }
+ }
}
/*******************************************************************************************************************//**
diff --git a/src/core/math/DistributedSample.h b/src/core/math/DistributedSample.h
index e8c817d9..65638e7d 100644
--- a/src/core/math/DistributedSample.h
+++ b/src/core/math/DistributedSample.h
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -27,6 +27,7 @@
#include <type_traits>
#include <iterator>
#include <vector>
+#include <mpi.h>
namespace walberla {
@@ -39,8 +40,8 @@ class DistributedSample
{
public:
- DistributedSample() :
- sum_( real_t(0) ), min_( real_t(0) ), max_( real_t(0) ), size_( uint_t(0) ), mean_( real_t(0) ), variance_( real_t(0) ) {}
+ explicit DistributedSample(MPI_Comm comm = MPI_COMM_WORLD) :
+ comm_(comm), sum_( real_t(0) ), min_( real_t(0) ), max_( real_t(0) ), size_( uint_t(0) ), mean_( real_t(0) ), variance_( real_t(0) ) {}
// insert
@@ -82,7 +83,8 @@ public:
std::string format( const std::string & formatString = DEFAULT_FORMAT_STRING ) const;
private:
-
+ MPI_Comm comm_;
+
std::vector< real_t > data_;
real_t sum_;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment