Skip to content

GitLab

Commit 6d7530fd authored by Marcel Koch's avatar Marcel Koch
Browse files

constrain globalInfo to active process subset

parent 7fe55a7f
Pipeline #34754 failed with stages
in 12 minutes and 20 seconds
Hide whitespace changes
Inline Side-by-side
src/hyteg/primitivestorage/PrimitiveStorage.cpp
View file @ 6d7530fd
......@@ -1613,7 +1613,7 @@ void PrimitiveStorage::initializeAndDeserializeAllPrimitiveData( walberla::mpi::
}
}
std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const
std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly, MPI_Comm comm ) const
{
uint_t globalNumberOfVertices;
uint_t globalNumberOfEdges;
......@@ -1635,46 +1635,47 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const
if ( onRootOnly )
{
globalNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::SUM );
globalNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::SUM );
globalNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::SUM );
globalNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::SUM );
globalNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM );
globalNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::SUM, 0, comm );
globalNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::SUM, 0, comm );
globalNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::SUM, 0, comm );
globalNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::SUM, 0, comm );
globalNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM, 0, comm );
globalMaxNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MAX );
globalMaxNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MAX );
globalMaxNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MAX );
globalMaxNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MAX );
globalMaxNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX );
globalMaxNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MAX, 0, comm );
globalMaxNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX, 0, comm );
globalMinNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MIN );
globalMinNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MIN );
globalMinNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MIN );
globalMinNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MIN );
globalMinNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN );
globalMinNumberOfVertices = walberla::mpi::reduce( getNumberOfLocalVertices(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfEdges = walberla::mpi::reduce( getNumberOfLocalEdges(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfFaces = walberla::mpi::reduce( getNumberOfLocalFaces(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfCells = walberla::mpi::reduce( getNumberOfLocalCells(), walberla::mpi::MIN, 0, comm );
globalMinNumberOfPrimitives = walberla::mpi::reduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN, 0, comm );
}
else
{
globalNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::SUM );
globalNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::SUM );
globalNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::SUM );
globalNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::SUM );
globalNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM );
globalNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::SUM, comm );
globalNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::SUM, comm );
globalNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::SUM, comm );
globalNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::SUM, comm );
globalNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::SUM, comm );
globalMaxNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MAX );
globalMaxNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MAX );
globalMaxNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MAX );
globalMaxNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MAX );
globalMaxNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX );
globalMaxNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MAX, comm );
globalMaxNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MAX, comm );
globalMaxNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MAX, comm );
globalMaxNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MAX, comm );
globalMaxNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MAX, comm );
globalMinNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MIN );
globalMinNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MIN );
globalMinNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MIN );
globalMinNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MIN );
globalMinNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN );
globalMinNumberOfVertices = walberla::mpi::allReduce( getNumberOfLocalVertices(), walberla::mpi::MIN, comm );
globalMinNumberOfEdges = walberla::mpi::allReduce( getNumberOfLocalEdges(), walberla::mpi::MIN, comm );
globalMinNumberOfFaces = walberla::mpi::allReduce( getNumberOfLocalFaces(), walberla::mpi::MIN, comm );
globalMinNumberOfCells = walberla::mpi::allReduce( getNumberOfLocalCells(), walberla::mpi::MIN, comm );
globalMinNumberOfPrimitives = walberla::mpi::allReduce( getNumberOfLocalPrimitives(), walberla::mpi::MIN, comm );
}
const uint_t numberOfProcesses = uint_c( walberla::mpi::MPIManager::instance()->numProcesses() );
int numberOfProcesses;
MPI_Comm_size(comm, &numberOfProcesses);
const double globalAvgNumberOfVertices = (double) globalNumberOfVertices / (double) numberOfProcesses;
const double globalAvgNumberOfEdges = (double) globalNumberOfEdges / (double) numberOfProcesses;
......@@ -1682,8 +1683,8 @@ std::string PrimitiveStorage::getGlobalInfo( bool onRootOnly ) const
const double globalAvgNumberOfCells = (double) globalNumberOfCells / (double) numberOfProcesses;
const double globalAvgNumberOfPrimitives = (double) globalNumberOfPrimitives / (double) numberOfProcesses;
walberla::math::DistributedSample neighborhoodSample;
walberla::math::DistributedSample neighborhoodVolumeSample;
walberla::math::DistributedSample neighborhoodSample(comm);
walberla::math::DistributedSample neighborhoodVolumeSample(comm);
const auto numNeighborProcesses = getNeighboringRanks().size();
const auto numNeighborVolumeProcesses = additionalHaloDepth_ > 0 ? getNeighboringVolumeRanksOfAllVolumes().size() : 0;
neighborhoodSample.castToRealAndInsert( numNeighborProcesses );
......
src/hyteg/primitivestorage/PrimitiveStorage.hpp
View file @ 6d7530fd
......@@ -441,7 +441,7 @@ class PrimitiveStorage : private walberla::NonCopyable
/// Returns a formatted string that contains global information about the storage.
/// Must be called by all processes!
/// Involves global communication and should therefore not be called in performance critical code.
std::string getGlobalInfo( bool onRootOnly = false ) const;
std::string getGlobalInfo( bool onRootOnly = false, MPI_Comm comm = MPI_COMM_WORLD ) const;
/// \brief Returns the global number of primitives.
///
......
walberla.patch 0 → 100644
View file @ 6d7530fd
diff --git a/src/core/math/DistributedSample.cpp b/src/core/math/DistributedSample.cpp
index 965ab816..6ed38d70 100644
--- a/src/core/math/DistributedSample.cpp
+++ b/src/core/math/DistributedSample.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -54,10 +54,10 @@ void DistributedSample::mpiAllGather()
WALBERLA_MPI_SECTION()
{
- mpi::allReduceInplace( sum_, mpi::SUM );
- mpi::allReduceInplace( min_, mpi::MIN );
- mpi::allReduceInplace( max_, mpi::MAX );
- mpi::allReduceInplace( size_, mpi::SUM );
+ mpi::allReduceInplace( sum_, mpi::SUM, comm_ );
+ mpi::allReduceInplace( min_, mpi::MIN, comm_ );
+ mpi::allReduceInplace( max_, mpi::MAX, comm_ );
+ mpi::allReduceInplace( size_, mpi::SUM, comm_ );
}
mean_ = sum_ / real_c(size_);
@@ -71,11 +71,11 @@ void DistributedSample::mpiAllGather()
WALBERLA_MPI_SECTION()
{
- mpi::allReduceInplace( variance_, mpi::SUM );
+ mpi::allReduceInplace( variance_, mpi::SUM, comm_ );
}
variance_ /= real_c(size_);
-
+
if( size_ == uint_t(0) )
{
min_ = real_t(0);
@@ -113,10 +113,10 @@ void DistributedSample::mpiGather( int rank )
WALBERLA_MPI_SECTION()
{
- mpi::reduceInplace( sum_, mpi::SUM, rank );
- mpi::reduceInplace( min_, mpi::MIN, rank );
- mpi::reduceInplace( max_, mpi::MAX, rank );
- mpi::reduceInplace( size_, mpi::SUM, rank );
+ mpi::reduceInplace( sum_, mpi::SUM, rank, comm_ );
+ mpi::reduceInplace( min_, mpi::MIN, rank, comm_ );
+ mpi::reduceInplace( max_, mpi::MAX, rank, comm_ );
+ mpi::reduceInplace( size_, mpi::SUM, rank, comm_ );
}
mean_ = sum_ / real_c(size_);
@@ -130,18 +130,18 @@ void DistributedSample::mpiGather( int rank )
WALBERLA_MPI_SECTION()
{
- mpi::reduceInplace( variance_, mpi::SUM, rank );
+ mpi::reduceInplace( variance_, mpi::SUM, rank, comm_ );
}
variance_ /= real_c(size_);
-
+
if( size_ == uint_t(0) )
{
min_ = real_t(0);
max_ = real_t(0);
mean_ = real_t(0);
variance_ = real_t(0);
- }
+ }
}
/*******************************************************************************************************************//**
diff --git a/src/core/math/DistributedSample.h b/src/core/math/DistributedSample.h
index e8c817d9..65638e7d 100644
--- a/src/core/math/DistributedSample.h
+++ b/src/core/math/DistributedSample.h
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -27,6 +27,7 @@
#include <type_traits>
#include <iterator>
#include <vector>
+#include <mpi.h>
namespace walberla {
@@ -39,8 +40,8 @@ class DistributedSample
{
public:
- DistributedSample() :
- sum_( real_t(0) ), min_( real_t(0) ), max_( real_t(0) ), size_( uint_t(0) ), mean_( real_t(0) ), variance_( real_t(0) ) {}
+ explicit DistributedSample(MPI_Comm comm = MPI_COMM_WORLD) :
+ comm_(comm), sum_( real_t(0) ), min_( real_t(0) ), max_( real_t(0) ), size_( uint_t(0) ), mean_( real_t(0) ), variance_( real_t(0) ) {}
// insert
@@ -82,7 +83,8 @@ public:
std::string format( const std::string & formatString = DEFAULT_FORMAT_STRING ) const;
private:
-
+ MPI_Comm comm_;
+
std::vector< real_t > data_;
real_t sum_;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment