diff --git a/.gitignore b/.gitignore
index a0d82191669d3d3fbb30e64fdf028852e0c990f9..91448c9a7457db41013e441f403c70755ddc358d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,7 +39,7 @@ CMakeLists.txt.user.*
/bin/
/lib/
*.a
-/build
+/build*
# Logfiles
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1a58571600fb6788aa0d0b6b458e62ea5f5073c5..0387e0998a753afeba11bcc668d4bd31110f29e2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ stages:
- test
- deploy
-
+
###############################################################################
## ##
## Build templates ##
@@ -1657,8 +1657,8 @@ coverage:
## Windows Builds ##
## ##
###############################################################################
-
-
+
+
.win_build_template: &win_build_definition
tags:
- win
@@ -1687,7 +1687,7 @@ msvc-14.1_Hybrid_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_Hybrid_SP_Dbg:
<<: *win_build_definition
variables:
@@ -1700,7 +1700,7 @@ msvc-14.1_Hybrid_SP_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_Hybrid:
<<: *win_build_definition
variables:
@@ -1713,7 +1713,7 @@ msvc-14.1_Hybrid:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_Serial_Dbg:
<<: *win_build_definition
variables:
@@ -1726,7 +1726,7 @@ msvc-14.1_Serial_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_Serial:
<<: *win_build_definition
variables:
@@ -1739,7 +1739,7 @@ msvc-14.1_Serial:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_MpiOnly_Dbg:
<<: *win_build_definition
variables:
@@ -1752,7 +1752,7 @@ msvc-14.1_MpiOnly_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.1_MpiOnly:
<<: *win_build_definition
variables:
@@ -1778,7 +1778,7 @@ msvc-14.2_Hybrid_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
-
+
msvc-14.2_Hybrid_SP_Dbg:
<<: *win_build_definition
variables:
@@ -1791,7 +1791,7 @@ msvc-14.2_Hybrid_SP_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
-
+
msvc-14.2_Hybrid:
<<: *win_build_definition
variables:
@@ -1804,7 +1804,7 @@ msvc-14.2_Hybrid:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
-
+
msvc-14.2_Serial_Dbg:
<<: *win_build_definition
variables:
@@ -1817,7 +1817,7 @@ msvc-14.2_Serial_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
-
+
msvc-14.2_Serial:
<<: *win_build_definition
variables:
@@ -1830,7 +1830,7 @@ msvc-14.2_Serial:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
-
+
msvc-14.2_MpiOnly_Dbg:
<<: *win_build_definition
variables:
@@ -1843,7 +1843,7 @@ msvc-14.2_MpiOnly_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
-
+
msvc-14.2_MpiOnly:
<<: *win_build_definition
variables:
@@ -1949,7 +1949,7 @@ mac_MpiOnly:
dependencies: []
when: manual
only:
- - master@walberla/walberla
+ - master@walberla/walberla
- tags@walberla/walberla
conda-py36-win:
@@ -1958,7 +1958,7 @@ conda-py36-win:
- win
script:
- conda build --python=3.6 --user=lssfau utilities\\conda\\walberla
-
+
conda-py37-win:
<<: *conda_deploy_definition
tags:
@@ -1975,7 +1975,7 @@ conda-py37-linux:
- apt-get update
- apt-get install -y build-essential
- conda build --python=3.7 --user=lssfau utilities/conda/walberla
-
+
conda-py36-linux:
<<: *conda_deploy_definition
tags:
diff --git a/apps/benchmarks/UniformGrid/UniformGrid.cpp b/apps/benchmarks/UniformGrid/UniformGrid.cpp
index a55ca158317e82e716b19773224c1d8673656d17..d5d355e491812c639ac1b4fca5a982622c19af5e 100644
--- a/apps/benchmarks/UniformGrid/UniformGrid.cpp
+++ b/apps/benchmarks/UniformGrid/UniformGrid.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -269,31 +269,20 @@ void createSetupBlockForest( blockforest::SetupBlockForest & sforest, const Conf
WALBERLA_MPI_SECTION()
{
- if ( MPIManager::instance()->isCartesianCommValid() )
- {
- MPIManager::instance()->createCartesianComm(numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, false, false, false);
+ MPIManager::instance()->createCartesianComm(numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, false, false, false);
- processIdMap = make_shared<std::vector<uint_t> >(numberOfXProcesses * numberOfYProcesses * numberOfZProcesses);
+ processIdMap = make_shared<std::vector<uint_t> >(numberOfXProcesses * numberOfYProcesses * numberOfZProcesses);
- for (uint_t z = 0; z != numberOfZProcesses; ++z) {
- for (uint_t y = 0; y != numberOfYProcesses; ++y) {
- for (uint_t x = 0; x != numberOfXProcesses; ++x)
- {
- (*processIdMap)[z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x] =
- uint_c(MPIManager::instance()->cartesianRank(x, y, z));
- }
+ for (uint_t z = 0; z != numberOfZProcesses; ++z) {
+ for (uint_t y = 0; y != numberOfYProcesses; ++y) {
+ for (uint_t x = 0; x != numberOfXProcesses; ++x)
+ {
+ (*processIdMap)[z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x] =
+ uint_c(MPIManager::instance()->cartesianRank(x, y, z));
}
}
}
- else {
- WALBERLA_LOG_WARNING_ON_ROOT( "Your version of OpenMPI contains a bug. See waLBerla issue #73 for more "
- "information. As a workaround, MPI_COMM_WORLD instead of a "
- "Cartesian MPI communicator is used." );
- MPIManager::instance()->useWorldComm();
- }
}
- else
- MPIManager::instance()->useWorldComm();
sforest.balanceLoad( blockforest::CartesianDistribution( numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, processIdMap.get() ),
numberOfXProcesses * numberOfYProcesses * numberOfZProcesses );
@@ -615,7 +604,7 @@ struct AddLB< LatticeModel_T, typename std::enable_if< std::is_same< typename La
std::function< void () > commFunction;
if( directComm )
{
-
+
if( fullComm )
{
blockforest::communication::UniformDirectScheme< stencil::D3Q27 > comm( blocks );
diff --git a/src/blockforest/BlockForest.cpp b/src/blockforest/BlockForest.cpp
index a156b7e3f0a77352d82ab2bea02851038fbade58..cfe71d653f3835014ba7db5f28591dd31772742d 100644
--- a/src/blockforest/BlockForest.cpp
+++ b/src/blockforest/BlockForest.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -672,7 +672,7 @@ void BlockForest::getAABB( AABB& aabb, const IBlockID& id ) const {
}
else {
const Block* const block = getBlock( id );
- if( block == nullptr )
+ if( block == nullptr )
{
const BlockID& bid = *static_cast< const BlockID* >( &id );
aabb = getAABBFromBlockId( bid );
@@ -906,7 +906,7 @@ void BlockForest::refresh()
WALBERLA_ASSERT_NOT_NULLPTR( block->second );
block->second->setTargetLevel( block->second->getLevel() );
}
-
+
if( recalculateBlockLevelsInRefresh_ )
{
WALBERLA_LOG_PROGRESS( "BlockForest refresh: determining new block levels" );
@@ -929,22 +929,22 @@ void BlockForest::refresh()
if( alwaysRebalanceInRefresh_ && refreshPhantomBlockMigrationPreparationFunction_ )
rebalanceAndRedistribute = true;
-
+
while( rebalanceAndRedistribute )
{
// create phantom block forest
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: creating phantom forest/blocks" );
refreshTiming_[ "phantom forest creation" ].start();
PhantomBlockForest phantomForest( *this );
phantomForest.initialize( refreshBlockStateDeterminationFunction_, allowChangingDepth_ );
-
+
refreshTiming_[ "phantom forest creation" ].end();
// move phantom blocks between processes (= dynamic load balancing)
-
+
WALBERLA_MPI_SECTION()
{
refreshTiming_[ "phantom block redistribution (= load balancing)" ].start();
@@ -966,15 +966,15 @@ void BlockForest::refresh()
++iteration;
}
phantomBlockMigrationIterations_ = iteration;
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: phantom block redistribution/load balancing finished after " << phantomBlockMigrationIterations_ << " iterations" );
}
refreshTiming_[ "phantom block redistribution (= load balancing)" ].end();
}
-
+
// update block forest: transfer block data (includes migrating, deleting and creating blocks), adapt depth, adapt process neighborhood, etc.
-
+
bool performUpdate( true );
if( checkForEarlyOutAfterLoadBalancing_ )
{
@@ -1010,23 +1010,23 @@ void BlockForest::refresh()
{
WALBERLA_ASSERT( recalculateBlockLevelsInRefresh_ );
WALBERLA_ASSERT( allowMultipleRefreshCycles_ );
-
+
refreshTiming_[ "block level determination" ].start();
-
+
for( auto block = blocks_.begin(); block != blocks_.end(); ++block )
{
WALBERLA_ASSERT_NOT_NULLPTR( block->second );
block->second->setTargetLevel( block->second->getLevel() );
}
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: determining new block levels (again -> more refresh cycles required!)" );
-
+
bool rerun( true );
while( rerun )
{
rebalanceAndRedistribute = determineBlockTargetLevels( additionalRefreshCycleRequired, rerun );
}
-
+
if( !rebalanceAndRedistribute )
WALBERLA_LOG_PROGRESS( "BlockForest refresh: block levels do not change" );
@@ -1035,7 +1035,7 @@ void BlockForest::refresh()
else
rebalanceAndRedistribute = false;
}
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: finished!" );
}
@@ -1267,7 +1267,7 @@ void BlockForest::restoreSnapshot( const SnapshotRestorenFunction & processMappi
++iteration;
}
phantomBlockMigrationIterations_ = iteration;
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: phantom block redistribution/load balancing finished after " << phantomBlockMigrationIterations_ << " iterations" );
}
@@ -1651,7 +1651,7 @@ bool BlockForest::determineBlockTargetLevels( bool & additionalRefreshCycleRequi
const uint_t level = block->getLevel();
uint_t minTargetLevel = minTargetLevels[id];
-
+
if( allowMultipleRefreshCycles_ )
{
if( minTargetLevel > ( level + uint_t(1) ) )
@@ -1683,7 +1683,7 @@ bool BlockForest::determineBlockTargetLevels( bool & additionalRefreshCycleRequi
}
}
}
-
+
if( allowMultipleRefreshCycles_ )
mpi::allReduceInplace( additionalRefreshCycleRequired, mpi::LOGICAL_OR );
@@ -2037,7 +2037,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
//////////////////////////////////////
std::map< uint_t, uint_t > numberOfBlocksToRecv; // does not include local transfers
-
+
const auto & phantomBlocks = phantomForest.getBlockMap();
for( auto phantom = phantomBlocks.begin(); phantom != phantomBlocks.end(); ++phantom )
{
@@ -2061,15 +2061,15 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
else { WALBERLA_ASSERT( blocks_.find( pBlock->getId() ) != blocks_.end() ); }
#endif
}
-
+
std::map< uint_t, std::vector< uint_t > > recvBufferSizes; // does not include local transfers
-
+
for( auto it = numberOfBlocksToRecv.begin(); it != numberOfBlocksToRecv.end(); ++it )
{
WALBERLA_ASSERT_GREATER( it->second, uint_t(0) );
recvBufferSizes[ it->first ].resize( it->second );
}
-
+
std::vector< MPI_Request > recvBufferSizesRequests( numberOfBlocksToRecv.size() ); // do not resize this vector!
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - schedule receives for buffer sizes" );
@@ -2081,15 +2081,15 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
int_c( it->first ), 0, MPIManager::instance()->comm(), &(recvBufferSizesRequests[i]) );
++i;
}
-
+
///////////////////////////////
// FETCH TARGET BLOCK STATES //
///////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - fetch target block states" );
-
+
std::set< mpi::MPIRank > ranksToRecvFrom;
-
+
for( auto it = blocks_.begin(); it != blocks_.end(); ++it )
{
auto & block = it->second;
@@ -2102,12 +2102,12 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
ranksToRecvFrom.insert( numeric_cast< mpi::MPIRank >(*p) );
}
}
-
+
mpi::BufferSystem bufferSystem( MPIManager::instance()->comm(), 1184 ); // blockforest = 98 108 111 99 107 102 111 114 101 115 116 + 2
- bufferSystem.setReceiverInfo( ranksToRecvFrom, true ); // ATTENTION: true = the size of a message from A to B varies
-
+ bufferSystem.setReceiverInfo( ranksToRecvFrom, true ); // ATTENTION: true = the size of a message from A to B varies
+
std::map< uint_t, std::map< BlockID, Set<SUID> > > blockStates;
-
+
for( auto phantom = phantomBlocks.begin(); phantom != phantomBlocks.end(); ++phantom )
{
auto & pBlock = phantom->second;
@@ -2126,9 +2126,9 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
}
bufferSystem.sendAll();
-
+
std::map< BlockID, Set<SUID> > targetBlockStates;
-
+
for( auto recvIt = bufferSystem.begin(); recvIt != bufferSystem.end(); ++recvIt )
{
WALBERLA_ASSERT_UNEQUAL( uint_c( recvIt.rank() ), process_ );
@@ -2147,9 +2147,9 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
{
WALBERLA_ASSERT( targetBlockStates.find( state->first ) == targetBlockStates.end() );
targetBlockStates[ state->first ] = state->second;
- }
+ }
}
-
+
///////////////
// PACK DATA //
///////////////
@@ -2187,7 +2187,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
auto & id = block->getId();
auto & targetProcesses = block->getTargetProcess();
-
+
std::vector< Set<SUID> > targetState;
// header = sender block ID + receiver block ID
@@ -2204,7 +2204,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[0] ].push_back( &(buffers[0]) );
else
localBlocks.push_back( &(buffers[0]) );
-
+
WALBERLA_ASSERT( targetBlockStates.find(id) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[id] );
}
@@ -2221,7 +2221,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[0] ].push_back( &(buffers[0]) );
else
localBlocks.push_back( &(buffers[0]) );
-
+
WALBERLA_ASSERT( targetBlockStates.find(fid) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[fid] );
}
@@ -2241,7 +2241,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[c] ].push_back( &(buffers[c]) );
else
localBlocks.push_back( &(buffers[c]) );
-
+
WALBERLA_ASSERT( targetBlockStates.find(cid) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[cid] );
}
@@ -2298,7 +2298,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
WALBERLA_ASSERT_EQUAL( buffers.size(), uint_t(8) );
WALBERLA_ASSERT_EQUAL( targetProcesses.size(), uint_t(8) );
WALBERLA_ASSERT_EQUAL( targetState.size(), uint_t(8) );
-
+
for( auto dataItem = blockDataItem_.begin(); dataItem != blockDataItem_.end(); ++dataItem )
{
for( uint_t c = uint_t(0); c != uint_t(8); ++c )
@@ -2344,12 +2344,12 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
for( auto it = processesToSendTo.begin(); it != processesToSendTo.end(); ++it )
{
WALBERLA_ASSERT( sendBufferSizes.find( it->first ) == sendBufferSizes.end() );
-
+
auto & sizes = sendBufferSizes[ it->first ];
WALBERLA_ASSERT( sizes.empty() );
for( auto buffer = it->second.begin(); buffer != it->second.end(); ++buffer )
sizes.push_back( (*buffer)->size() );
-
+
blockDataSendRequests[ it->first ].resize( it->second.size() ); // do not resize this vector after this point!
}
@@ -2378,7 +2378,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
++i;
}
-
+
///////////////////
// CREATE BLOCKS //
///////////////////
@@ -2399,7 +2399,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
blocks_[ pBlock->getId() ]->resetNeighborhood( *pBlock );
}
}
-
+
// adapt depth
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - adapting block structure (use current state of phantom forest as reference)" );
@@ -2421,21 +2421,21 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
//////////////////////////////////////
// WAIT FOR RECV's FOR BUFFER SIZES //
//////////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for buffer sizes" );
if( ! recvBufferSizesRequests.empty() )
MPI_Waitall( int_c( recvBufferSizesRequests.size() ), &(recvBufferSizesRequests[0]), MPI_STATUSES_IGNORE );
-
+
////////////////////////////////////
// SCHEDULE RECV's FOR BLOCK DATA //
////////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - schedule block data receive operations" );
std::map< uint_t, std::vector< mpi::RecvBuffer > > recvBlockData;
std::map< uint_t, std::vector< MPI_Request > > blockDataRecvRequests;
-
+
for( auto it = recvBufferSizes.begin(); it != recvBufferSizes.end(); ++it )
{
WALBERLA_ASSERT_UNEQUAL( it->first, process_ );
@@ -2446,7 +2446,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
requests.resize( sizes.size() );
}
auto & recvLocalBlocks = recvBlockData[ process_ ];
-
+
for( auto it = recvBufferSizes.begin(); it != recvBufferSizes.end(); ++it )
{
WALBERLA_ASSERT_UNEQUAL( it->first, process_ );
@@ -2462,11 +2462,11 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
int_c( it->first ), int_c(i)+1, MPIManager::instance()->comm(), &(requests[i]) );
}
}
-
+
///////////////////////////
// COPY LOCAL BLOCK DATA //
///////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - perform local data transfer" );
for( auto buffer = localBlocks.begin(); buffer != localBlocks.end(); ++buffer )
@@ -2475,7 +2475,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////////
// WAIT FOR RECV's FOR BLOCK DATA //
////////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for block data to be received" );
for( auto it = blockDataRecvRequests.begin(); it != blockDataRecvRequests.end(); ++it )
@@ -2483,16 +2483,16 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
auto & requests = it->second;
MPI_Waitall( int_c( requests.size() ), &(requests[0]), MPI_STATUSES_IGNORE );
}
-
+
//////////////////////////////////
// WAIT FOR ALL SENDS TO FINISH //
//////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for block data sends to complete" );
if( ! sendBufferSizesRequests.empty() )
MPI_Waitall( int_c( sendBufferSizesRequests.size() ), &(sendBufferSizesRequests[0]), MPI_STATUSES_IGNORE );
-
+
for( auto it = blockDataSendRequests.begin(); it != blockDataSendRequests.end(); ++it )
{
auto & requests = it->second;
@@ -2502,7 +2502,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////////////
// CLEAR SEND BUFFERS (= FREE MEMORY) //
////////////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - clear send buffers (= free memory)" );
for( auto it = blocksToPack.begin(); it != blocksToPack.end(); ++it )
@@ -2515,11 +2515,11 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////
// PREPARE DATA FOR UNPACKING //
////////////////////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - prepare received data for unpacking" );
std::map< Block *, std::vector< std::pair< Set<SUID>, mpi::RecvBuffer * > > > blocksToUnpack; // includes data that is NOT transfered via MPI but copied locally
-
+
for( auto it = recvBlockData.begin(); it != recvBlockData.end(); ++it )
{
auto & buffers = it->second;
@@ -2536,7 +2536,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
WALBERLA_ASSERT( phantomBlocks.find( rId ) != phantomBlocks.end() );
Block * block = blocks_[ rId ].get();
const auto & phantom = phantomBlocks.find(rId)->second;
-
+
if( phantom->sourceBlockHasTheSameSize() || phantom->sourceBlockIsLarger() )
{
WALBERLA_ASSERT( blocksToUnpack.find( block ) == blocksToUnpack.end() );
@@ -2552,7 +2552,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
}
}
}
-
+
#ifndef NDEBUG
for( auto it = phantomBlocks.begin(); it != phantomBlocks.end(); ++it )
{
@@ -2571,7 +2571,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
//////////////////////////////
// GLOBAL BLOCK INFORMATION //
//////////////////////////////
-
+
// adapt global block information stored on every process (this is only done if the block forest is set-up to store global information!)
if( containsGlobalBlockInformation() )
@@ -2579,7 +2579,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - update global block information that is stored locally on every process" );
constructBlockInformation();
}
-
+
++modificationStamp_;
//////////////
@@ -2595,7 +2595,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
/////////////////
// UNPACK DATA //
/////////////////
-
+
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - unpacking block data from buffers" );
std::vector< std::pair< Block *, std::vector< std::pair< Set<SUID>, mpi::RecvBuffer * > > > > dataToUnpack;
@@ -2870,6 +2870,13 @@ void BlockForest::saveToFile( const std::string & filename, FileIOMode fileIOMod
fileIOMode = MASTER_SLAVE;
}
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D Cartesian MPI
+ // communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
+ {
+ fileIOMode = MASTER_SLAVE;
+ }
+
if( fileIOMode == MPI_PARALLEL )
{
MPI_File mpiFile = MPI_FILE_NULL;
diff --git a/src/blockforest/Initialization.cpp b/src/blockforest/Initialization.cpp
index 8771996fbe2715a09e1a8f39876ad85403435848..11d5243e219e826559d0a9e7c9f9ac0cc1ea74fb 100644
--- a/src/blockforest/Initialization.cpp
+++ b/src/blockforest/Initialization.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -230,27 +230,19 @@ createBlockForest( const AABB& domainAABB,
//create cartesian communicator only if not yet a cartesian communicator (or other communicator was created)
if ( ! mpiManager->rankValid() )
{
- if ( mpiManager->isCartesianCommValid() ) {
- mpiManager->createCartesianComm( numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, xPeriodic, yPeriodic, zPeriodic );
+ mpiManager->createCartesianComm( numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, xPeriodic, yPeriodic, zPeriodic );
- processIdMap.resize( numberOfProcesses );
+ processIdMap.resize( numberOfProcesses );
- for( uint_t z = 0; z != numberOfZProcesses; ++z ) {
- for( uint_t y = 0; y != numberOfYProcesses; ++y ) {
- for( uint_t x = 0; x != numberOfXProcesses; ++x ) {
+ for( uint_t z = 0; z != numberOfZProcesses; ++z ) {
+ for( uint_t y = 0; y != numberOfYProcesses; ++y ) {
+ for( uint_t x = 0; x != numberOfXProcesses; ++x ) {
- processIdMap[ z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x ] =
- uint_c( MPIManager::instance()->cartesianRank(x,y,z) );
- }
+ processIdMap[ z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x ] =
+ uint_c( MPIManager::instance()->cartesianRank(x,y,z) );
}
}
}
- else {
- WALBERLA_LOG_WARNING_ON_ROOT( "Your version of OpenMPI contains a bug. See waLBerla issue #73 for more "
- "information. As a workaround, MPI_COMM_WORLD instead of a "
- "Cartesian MPI communicator is used." );
- mpiManager->useWorldComm();
- }
}
}
diff --git a/src/core/mpi/MPIIO.cpp b/src/core/mpi/MPIIO.cpp
index b8d85cd6db0324064345ab36735a73b420e42f42..b8eb53a7b761ceeba39a8270686ba654b2caf60f 100644
--- a/src/core/mpi/MPIIO.cpp
+++ b/src/core/mpi/MPIIO.cpp
@@ -17,180 +17,303 @@
//! \ingroup core
//! \author Florian Schornbaum <florian.schornbaum@fau.de>
//! \author Martin Bauer <martin.bauer@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
#include "core/DataTypes.h"
+#include "core/logging/Logging.h"
+#include "core/mpi/MPIManager.h"
#include "core/mpi/RecvBuffer.h"
#include "core/mpi/SendBuffer.h"
-#include "core/mpi/MPIManager.h"
-#include "core/mpi/Reduce.h"
-#include <fstream>
+#include <fstream>
-namespace walberla {
-namespace mpi {
+namespace walberla
+{
+namespace mpi
+{
+void writeMPIIO(const std::string& file, SendBuffer& buffer)
+{
+ uint_t dataSize = sizeof(mpi::SendBuffer::ElementType) * buffer.size();
- void writeMPIIO(const std::string & file, SendBuffer & buffer)
+ WALBERLA_NON_MPI_SECTION()
{
- uint_t dataSize = sizeof( mpi::SendBuffer::ElementType ) * buffer.size();
+ std::ofstream ofile(file.c_str(), std::ofstream::binary);
+ if (ofile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for writing."); }
- WALBERLA_NON_MPI_SECTION()
+ ofile.write(reinterpret_cast< const char* >(buffer.ptr()), numeric_cast< std::streamsize >(dataSize));
+ if (ofile.fail()) { WALBERLA_ABORT("Error while writing to file \"" << file << "\"."); }
+
+ ofile.close();
+ if (ofile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for reading."); }
+ }
+
+ WALBERLA_MPI_SECTION()
+ {
+ // get the size of the data type to be written in the file header
+ int uintSize;
+ MPI_Type_size(MPITrait< uint_t >::type(), &uintSize);
+
+ // get the position at which a process writes its data (offset from beginning of the file)
+ uint_t exscanResult;
+ MPI_Exscan(&dataSize, &exscanResult, 1, MPITrait< uint_t >::type(), MPI_SUM, MPIManager::instance()->comm());
+ if (MPIManager::instance()->rank() == 0) exscanResult = uint_t(0);
+ const uint_t offset = uint_c(mpi::MPIManager::instance()->numProcesses() * 2 * uintSize) + exscanResult;
+
+ // header of the file contains each process' offset and buffer size
+ uint_t offsetData[2];
+ offsetData[0] = offset; // position at which a process writes its data
+ offsetData[1] = uint_c(buffer.size());
+
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D
+ // Cartesian MPI communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
{
- std::ofstream ofile( file.c_str(), std::ofstream::binary );
- ofile.write( reinterpret_cast< const char* >( buffer.ptr() ), numeric_cast< std::streamsize >( dataSize ) );
- ofile.close();
+ std::ofstream ofile;
+
+ // write each process' offset and buffer size to file
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ if (MPIManager::instance()->rank() == 0) { ofile.open(file.c_str(), std::ofstream::binary); }
+ else
+ {
+ ofile.open(file.c_str(), std::ofstream::binary | std::ofstream::app);
+ }
+ if (ofile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for writing."); }
+
+ ofile.write(reinterpret_cast< const char* >(offsetData), numeric_cast< std::streamsize >(2 * uintSize));
+ if (ofile.fail()) { WALBERLA_ABORT("Error while writing to file \"" << file << "\"."); }
+
+ ofile.close();
+ if (ofile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for writing."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
+
+ // write each process' data to file (starting at offset)
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ ofile.open(file.c_str(), std::ofstream::binary | std::ofstream::app);
+ if (ofile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for writing."); }
+
+ ofile.write(reinterpret_cast< const char* >(buffer.ptr()), numeric_cast< std::streamsize >(dataSize));
+ if (ofile.fail()) { WALBERLA_ABORT("Error while writing to file \"" << file << "\"."); }
+
+ ofile.close();
+ if (ofile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for writing."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
}
-
- WALBERLA_MPI_SECTION()
+ else // use MPI-IO
{
MPI_File mpiFile = MPI_FILE_NULL;
- int result = MPI_SUCCESS;
- result = MPI_File_open( MPIManager::instance()->comm(), const_cast<char*>( file.c_str() ), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpiFile );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while opening file \"" << file << "\" for writing. MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- int uintSize;
- MPI_Type_size( MPITrait< uint_t >::type(), &uintSize );
-
- const MPI_Offset filesize = numeric_cast<MPI_Offset>( mpi::MPIManager::instance()->numProcesses() * 2 * uintSize ) +
- numeric_cast<MPI_Offset>( mpi::allReduce( dataSize, mpi::SUM, MPIManager::instance()->comm() ) );
- MPI_File_set_size( mpiFile, filesize );
+ int result = MPI_SUCCESS;
+ result = MPI_File_open(MPIManager::instance()->comm(), const_cast< char* >(file.c_str()),
+ MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpiFile);
- uint_t exscanResult;
- MPI_Exscan( &dataSize, &exscanResult, 1, MPITrait<uint_t>::type(), MPI_SUM, MPIManager::instance()->comm() );
- if( MPIManager::instance()->rank() == 0 )
- exscanResult = uint_t( 0 );
- const uint_t offset = uint_c( mpi::MPIManager::instance()->numProcesses() * 2 * uintSize ) + exscanResult;
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while opening file \"" << file << "\" for writing. MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
MPI_Datatype offsettype;
- MPI_Type_contiguous( 2, MPITrait< uint_t >::type(), &offsettype );
- MPI_Type_commit( &offsettype );
+ MPI_Type_contiguous(2, MPITrait< uint_t >::type(), &offsettype);
+ MPI_Type_commit(&offsettype);
MPI_Datatype arraytype;
- MPI_Type_contiguous( int_c( buffer.size() ), MPITrait< mpi::SendBuffer::ElementType >::type(), &arraytype );
- MPI_Type_commit( &arraytype );
+ MPI_Type_contiguous(int_c(buffer.size()), MPITrait< mpi::SendBuffer::ElementType >::type(), &arraytype);
+ MPI_Type_commit(&arraytype);
- // offsets to processes data (+ buffer size)
+ // write each process' offset and buffer size to file
+ result =
+ MPI_File_set_view(mpiFile, numeric_cast< MPI_Offset >(mpi::MPIManager::instance()->rank() * 2 * uintSize),
+ MPITrait< uint_t >::type(), offsettype, const_cast< char* >("native"), MPI_INFO_NULL);
- result = MPI_File_set_view( mpiFile, numeric_cast<MPI_Offset>( mpi::MPIManager::instance()->rank() * 2 * uintSize ),
- MPITrait< uint_t >::type(), offsettype, const_cast<char*>( "native" ), MPI_INFO_NULL );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ result = MPI_File_write_all(mpiFile, reinterpret_cast< char* >(offsetData), 2, MPITrait< uint_t >::type(),
+ MPI_STATUS_IGNORE);
- uint_t offsetData[2];
- offsetData[0] = offset;
- offsetData[1] = uint_c( buffer.size() );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while writing to file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- result = MPI_File_write_all( mpiFile, reinterpret_cast<char*>( offsetData ), 2, MPITrait< uint_t >::type(), MPI_STATUS_IGNORE );
+ // write each process' data to file (starting at offset)
+ result = MPI_File_set_view(mpiFile, numeric_cast< MPI_Offset >(offset),
+ MPITrait< mpi::SendBuffer::ElementType >::type(), arraytype,
+ const_cast< char* >("native"), MPI_INFO_NULL);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- // the data
+ result = MPI_File_write_all(mpiFile, reinterpret_cast< char* >(buffer.ptr()), int_c(buffer.size()),
+ MPITrait< mpi::SendBuffer::ElementType >::type(), MPI_STATUS_IGNORE);
- result = MPI_File_set_view( mpiFile, numeric_cast<MPI_Offset>( offset ), MPITrait< mpi::SendBuffer::ElementType >::type(), arraytype,
- const_cast<char*>( "native" ), MPI_INFO_NULL );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while writing to file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ result = MPI_File_close(&mpiFile);
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while closing file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
+
+ MPI_Type_free(&arraytype);
+ MPI_Type_free(&offsettype);
+ }
+ }
+}
- result = MPI_File_write_all( mpiFile, reinterpret_cast<char*>( buffer.ptr() ), int_c( buffer.size() ), MPITrait< mpi::SendBuffer::ElementType >::type(), MPI_STATUS_IGNORE );
+void readMPIIO(const std::string& file, RecvBuffer& buffer)
+{
+ WALBERLA_NON_MPI_SECTION()
+ {
+ std::ifstream ifile(file.c_str(), std::ifstream::binary);
+ if (ifile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for reading."); }
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ ifile.seekg(0, std::ios::end);
+ const uint_t length = uint_c(static_cast< std::streamoff >(ifile.tellg()));
+ ifile.seekg(0, std::ios::beg);
- result = MPI_File_close( &mpiFile );
+ buffer.resize(length / sizeof(mpi::RecvBuffer::ElementType));
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while closing file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ ifile.read(reinterpret_cast< char* >(buffer.ptr()), numeric_cast< std::streamsize >(length));
+ if (ifile.fail()) { WALBERLA_ABORT("Error while reading from file \"" << file << "\"."); }
- MPI_Type_free( &arraytype );
- MPI_Type_free( &offsettype );
- }
+ ifile.close();
+ if (ifile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for reading."); }
}
-
- void readMPIIO(const std::string & file, RecvBuffer & buffer)
+ WALBERLA_MPI_SECTION()
{
- WALBERLA_NON_MPI_SECTION()
- {
- std::ifstream ifile( file.c_str(), std::ifstream::binary );
-
- ifile.seekg( 0, std::ios::end );
- const uint_t length = uint_c( static_cast< std::streamoff >( ifile.tellg() ) );
- ifile.seekg( 0, std::ios::beg );
+ // get the size of the data type to be read in the file header
+ int uintSize;
+ MPI_Type_size(MPITrait< uint_t >::type(), &uintSize);
- buffer.resize( length / sizeof( mpi::RecvBuffer::ElementType ) );
+ // header of the file contains each process' offset and buffer size
+ uint_t offsetData[2];
- ifile.read( reinterpret_cast< char* >( buffer.ptr() ), numeric_cast< std::streamsize >( length ) );
- ifile.close();
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D
+ // Cartesian MPI communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
+ {
+ std::ifstream ifile;
+
+ // read each process' offset and buffer size from file
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ ifile.open(file.c_str(), std::ofstream::binary);
+ if (ifile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for reading."); }
+
+ ifile.seekg(numeric_cast< std::streamoff >(mpi::MPIManager::instance()->rank() * 2 * uintSize));
+ ifile.read(reinterpret_cast< char* >(offsetData), numeric_cast< std::streamsize >(2 * uintSize));
+ if (ifile.fail()) { WALBERLA_ABORT("Error while reading from file \"" << file << "\"."); }
+
+ ifile.close();
+ if (ifile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for reading."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
+
+ buffer.resize(offsetData[1] / sizeof(mpi::RecvBuffer::ElementType));
+
+ // read each process' data from file (starting at offset)
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ ifile.open(file.c_str(), std::ofstream::binary);
+ if (ifile.fail()) { WALBERLA_ABORT("Error while opening file \"" << file << "\" for reading."); }
+
+ ifile.seekg(numeric_cast< std::streamoff >(offsetData[0]));
+ ifile.read(reinterpret_cast< char* >(buffer.ptr()), numeric_cast< std::streamsize >(offsetData[1]));
+ if (ifile.fail()) { WALBERLA_ABORT("Error while reading from file \"" << file << "\"."); }
+
+ ifile.close();
+ if (ifile.fail()) { WALBERLA_ABORT("Error while closing file \"" << file << "\" for reading."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
}
-
- WALBERLA_MPI_SECTION()
+ else // use MPI-IO
{
MPI_File mpiFile = MPI_FILE_NULL;
- int result = MPI_SUCCESS;
- result = MPI_File_open( MPIManager::instance()->comm(), const_cast<char*>( file.c_str() ), MPI_MODE_RDONLY, MPI_INFO_NULL, &mpiFile );
+ int result = MPI_SUCCESS;
+ result = MPI_File_open(MPIManager::instance()->comm(), const_cast< char* >(file.c_str()), MPI_MODE_RDONLY,
+ MPI_INFO_NULL, &mpiFile);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while opening file \"" << file << "\" for writing. MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- int uintSize;
- MPI_Type_size( MPITrait< uint_t >::type(), &uintSize );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while opening file \"" << file << "\" for reading. MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
MPI_Datatype offsettype;
- MPI_Type_contiguous( 2, MPITrait< uint_t >::type(), &offsettype );
- MPI_Type_commit( &offsettype );
-
- // offsets to processes data (+ buffer size)
-
- result = MPI_File_set_view( mpiFile, numeric_cast<MPI_Offset>( mpi::MPIManager::instance()->rank() * 2 * uintSize ),
- MPITrait< uint_t >::type(), offsettype, const_cast<char*>( "native" ), MPI_INFO_NULL );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ MPI_Type_contiguous(2, MPITrait< uint_t >::type(), &offsettype);
+ MPI_Type_commit(&offsettype);
- uint_t offsetData[2];
+ // read each process' offset and buffer size from file
+ result =
+ MPI_File_set_view(mpiFile, numeric_cast< MPI_Offset >(mpi::MPIManager::instance()->rank() * 2 * uintSize),
+ MPITrait< uint_t >::type(), offsettype, const_cast< char* >("native"), MPI_INFO_NULL);
- result = MPI_File_read_all( mpiFile, reinterpret_cast<char*>( offsetData ), 2, MPITrait< uint_t >::type(), MPI_STATUS_IGNORE );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ result = MPI_File_read_all(mpiFile, reinterpret_cast< char* >(offsetData), 2, MPITrait< uint_t >::type(),
+ MPI_STATUS_IGNORE);
- // the data
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while reading from file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
+ // read each process' data from file (starting at offset)
MPI_Datatype arraytype;
- MPI_Type_contiguous( int_c( offsetData[1] ), MPITrait< mpi::RecvBuffer::ElementType >::type(), &arraytype );
- MPI_Type_commit( &arraytype );
+ MPI_Type_contiguous(int_c(offsetData[1]), MPITrait< mpi::RecvBuffer::ElementType >::type(), &arraytype);
+ MPI_Type_commit(&arraytype);
- result = MPI_File_set_view( mpiFile, numeric_cast<MPI_Offset>( offsetData[0] ), MPITrait< mpi::RecvBuffer::ElementType >::type(), arraytype,
- const_cast<char*>( "native" ), MPI_INFO_NULL );
+ result = MPI_File_set_view(mpiFile, numeric_cast< MPI_Offset >(offsetData[0]),
+ MPITrait< mpi::RecvBuffer::ElementType >::type(), arraytype,
+ const_cast< char* >("native"), MPI_INFO_NULL);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- buffer.resize( offsetData[1] );
+ buffer.resize(offsetData[1]);
- result = MPI_File_read_all( mpiFile, reinterpret_cast<char*>( buffer.ptr() ), int_c( buffer.size() ), MPITrait< mpi::SendBuffer::ElementType >::type(), MPI_STATUS_IGNORE );
+ result = MPI_File_read_all(mpiFile, reinterpret_cast< char* >(buffer.ptr()), int_c(buffer.size()),
+ MPITrait< mpi::SendBuffer::ElementType >::type(), MPI_STATUS_IGNORE);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while reading from file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- result = MPI_File_close( &mpiFile );
+ result = MPI_File_close(&mpiFile);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while closing file \"" << file << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while closing file \"" << file << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
- MPI_Type_free( &arraytype );
- MPI_Type_free( &offsettype );
+ MPI_Type_free(&arraytype);
+ MPI_Type_free(&offsettype);
}
}
-
+}
} // namespace mpi
} // namespace walberla
-
-
diff --git a/src/core/mpi/MPIManager.cpp b/src/core/mpi/MPIManager.cpp
index f8944ab781e8c7c7a587e1a7a484eca30cb1950f..2a2ac24824c86659a405508cb098e6419f44b772 100644
--- a/src/core/mpi/MPIManager.cpp
+++ b/src/core/mpi/MPIManager.cpp
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -18,23 +18,25 @@
//! \author Martin Bauer <martin.bauer@fau.de>
//! \author Florian Schornbaum <florian.schornbaum@fau.de>
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
#include "MPIManager.h"
+
#include "core/logging/Logging.h"
#include <exception>
#include <iostream>
+#include <mutex>
#include <stdexcept>
-#include <vector>
#include <string>
+#include <vector>
-
-namespace walberla{
-namespace mpi {
-
-
+namespace walberla
+{
+namespace mpi
+{
/**
* Terminate Handler that calls MPI_Abort instead of std::abort
*
@@ -55,220 +57,209 @@ static void customTerminateHandler()
// in linux environments. If this pointer is null,
// i.e. in cases when this hack does not work, we just cannot print the message
// otherwise we re-throw the exception to get the type, and print exception.what()
- try {
- if(std::current_exception() )
- std::rethrow_exception(std::current_exception());
- } catch (std::exception const & exc) {
+ try
+ {
+ if (std::current_exception()) std::rethrow_exception(std::current_exception());
+ } catch (std::exception const& exc)
+ {
std::cerr << exc.what() << std::endl;
}
- WALBERLA_MPI_SECTION() {
- MPIManager::instance()->abort();
- }
- else {
- std::abort();
- }
+ WALBERLA_MPI_SECTION() { MPIManager::instance()->abort(); }
+ else { std::abort(); }
}
-
-
-MPIManager::~MPIManager()
-{
- finalizeMPI();
-}
+MPIManager::~MPIManager() { finalizeMPI(); }
void MPIManager::abort()
{
currentlyAborting_ = true;
WALBERLA_MPI_SECTION()
{
- if( MPIManager::instance()->isMPIInitialized() )
- MPI_Abort( MPI_COMM_WORLD, EXIT_FAILURE );
+ if (MPIManager::instance()->isMPIInitialized()) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
}
- std::exit( EXIT_FAILURE );
+ std::exit(EXIT_FAILURE);
}
-
-void MPIManager::initializeMPI( int* argc, char*** argv, bool abortOnException )
+void MPIManager::initializeMPI(int* argc, char*** argv, bool abortOnException)
{
WALBERLA_MPI_SECTION()
{
- WALBERLA_ASSERT( !isMPIInitialized_ );
+ WALBERLA_ASSERT(!isMPIInitialized_);
// Check first that MPI was not initialized before
// f.e. when using Python, MPI could have been initialized by
// a different MPI module like mpi4py
- int mpiAlreadyInitialized=0;
- MPI_Initialized( &mpiAlreadyInitialized );
- if ( ! mpiAlreadyInitialized ) {
- MPI_Init( argc, argv );
+ int mpiAlreadyInitialized = 0;
+ MPI_Initialized(&mpiAlreadyInitialized);
+ if (!mpiAlreadyInitialized)
+ {
+ MPI_Init(argc, argv);
finalizeOnDestruction_ = true;
}
isMPIInitialized_ = true;
- MPI_Comm_size( MPI_COMM_WORLD, &numProcesses_ );
- MPI_Comm_rank( MPI_COMM_WORLD, &worldRank_ );
+ MPI_Comm_size(MPI_COMM_WORLD, &numProcesses_);
+ MPI_Comm_rank(MPI_COMM_WORLD, &worldRank_);
- if( abortOnException )
- std::set_terminate( customTerminateHandler );
+ if (abortOnException) std::set_terminate(customTerminateHandler);
}
}
-
-
void MPIManager::finalizeMPI()
{
- WALBERLA_MPI_SECTION() {
- if( isMPIInitialized_ && !currentlyAborting_ )
+ WALBERLA_MPI_SECTION()
+ {
+ if (isMPIInitialized_ && !currentlyAborting_)
{
isMPIInitialized_ = false;
- if (finalizeOnDestruction_)
- {
- MPI_Finalize();
- }
+ if (finalizeOnDestruction_) { MPI_Finalize(); }
}
}
}
-
-
void MPIManager::resetMPI()
{
- WALBERLA_MPI_SECTION() {
- WALBERLA_ASSERT( isMPIInitialized_ );
- if( rank_ != -1 )
+ WALBERLA_MPI_SECTION()
+ {
+ WALBERLA_ASSERT(isMPIInitialized_);
+ if (rank_ != -1)
{
- if( comm_ == MPI_COMM_WORLD )
+ if (comm_ == MPI_COMM_WORLD)
comm_ = MPI_COMM_NULL;
else
- MPI_Comm_free( &comm_ );
+ MPI_Comm_free(&comm_);
rank_ = -1;
}
cartesianSetup_ = false;
- WALBERLA_ASSERT_EQUAL( comm_, MPI_COMM_NULL );
- WALBERLA_ASSERT_EQUAL( rank_, -1 );
+ WALBERLA_ASSERT_EQUAL(comm_, MPI_COMM_NULL);
+ WALBERLA_ASSERT_EQUAL(rank_, -1);
}
}
-
-
-void MPIManager::createCartesianComm( int dims[3], int periodicity[3] )
+void MPIManager::createCartesianComm(int dims[3], int periodicity[3])
{
- WALBERLA_ASSERT( isMPIInitialized_ );
- WALBERLA_ASSERT_EQUAL( rank_, -1 );
- WALBERLA_ASSERT( !cartesianSetup_ );
- WALBERLA_ASSERT_GREATER( dims[0], 0 );
- WALBERLA_ASSERT_GREATER( dims[1], 0 );
- WALBERLA_ASSERT_GREATER( dims[2], 0 );
-
- if ( ! isCartesianCommValid() ) {
- WALBERLA_LOG_WARNING_ON_ROOT( "Your version of OpenMPI contains a bug which might lead to a segmentation fault "
- "when generating vtk output. Since the bug only occurs with a 3D Cartesian MPI "
- "communicator, try to use MPI_COMM_WORLD instead. See waLBerla issue #73 for "
- "additional information." );
- }
-
- MPI_Cart_create( MPI_COMM_WORLD, 3, dims, periodicity, true, &comm_ );
- MPI_Comm_rank( comm_, &rank_ );
+ WALBERLA_ASSERT(isMPIInitialized_);
+ WALBERLA_ASSERT_EQUAL(rank_, -1);
+ WALBERLA_ASSERT(!cartesianSetup_);
+ WALBERLA_ASSERT_GREATER(dims[0], 0);
+ WALBERLA_ASSERT_GREATER(dims[1], 0);
+ WALBERLA_ASSERT_GREATER(dims[2], 0);
+
+ MPI_Cart_create(MPI_COMM_WORLD, 3, dims, periodicity, true, &comm_);
+ MPI_Comm_rank(comm_, &rank_);
cartesianSetup_ = true;
WALBERLA_ASSERT_UNEQUAL(comm_, MPI_COMM_NULL);
}
-
-
-void MPIManager::createCartesianComm( const uint_t xProcesses, const uint_t yProcesses, const uint_t zProcesses,
- const bool xPeriodic, const bool yPeriodic, const bool zPeriodic )
+void MPIManager::createCartesianComm(const uint_t xProcesses, const uint_t yProcesses, const uint_t zProcesses,
+ const bool xPeriodic, const bool yPeriodic, const bool zPeriodic)
{
int dims[3];
- dims[0] = numeric_cast<int>( xProcesses );
- dims[1] = numeric_cast<int>( yProcesses );
- dims[2] = numeric_cast<int>( zProcesses );
+ dims[0] = numeric_cast< int >(xProcesses);
+ dims[1] = numeric_cast< int >(yProcesses);
+ dims[2] = numeric_cast< int >(zProcesses);
int periodicity[3];
periodicity[0] = xPeriodic ? 1 : 0;
periodicity[1] = yPeriodic ? 1 : 0;
periodicity[2] = zPeriodic ? 1 : 0;
- createCartesianComm( dims, periodicity );
+ createCartesianComm(dims, periodicity);
}
+void MPIManager::cartesianCoord(int coordOut[3]) const { cartesianCoord(rank_, coordOut); }
-
-void MPIManager::cartesianCoord( int coordOut[3] ) const
+void MPIManager::cartesianCoord(int rankIn, int coordOut[3]) const
{
- cartesianCoord( rank_, coordOut );
-}
-
-
-
-void MPIManager::cartesianCoord( int rankIn, int coordOut[3] ) const
-{
- WALBERLA_ASSERT( isMPIInitialized_ );
- WALBERLA_ASSERT( cartesianSetup_ );
- WALBERLA_ASSERT_UNEQUAL( comm_, MPI_COMM_NULL );
+ WALBERLA_ASSERT(isMPIInitialized_);
+ WALBERLA_ASSERT(cartesianSetup_);
+ WALBERLA_ASSERT_UNEQUAL(comm_, MPI_COMM_NULL);
- MPI_Cart_coords( comm_, rankIn, 3, coordOut );
+ MPI_Cart_coords(comm_, rankIn, 3, coordOut);
}
-
-
-int MPIManager::cartesianRank( int coords[3] ) const
+int MPIManager::cartesianRank(int coords[3]) const
{
- WALBERLA_ASSERT( isMPIInitialized_ );
- WALBERLA_ASSERT( cartesianSetup_ );
- WALBERLA_ASSERT_UNEQUAL( comm_, MPI_COMM_NULL );
+ WALBERLA_ASSERT(isMPIInitialized_);
+ WALBERLA_ASSERT(cartesianSetup_);
+ WALBERLA_ASSERT_UNEQUAL(comm_, MPI_COMM_NULL);
int r;
- MPI_Cart_rank( comm_, coords, &r );
+ MPI_Cart_rank(comm_, coords, &r);
return r;
}
-
-
-int MPIManager::cartesianRank( const uint_t x, const uint_t y, const uint_t z ) const
+int MPIManager::cartesianRank(const uint_t x, const uint_t y, const uint_t z) const
{
int coords[3];
- coords[0] = numeric_cast<int>(x);
- coords[1] = numeric_cast<int>(y);
- coords[2] = numeric_cast<int>(z);
+ coords[0] = numeric_cast< int >(x);
+ coords[1] = numeric_cast< int >(y);
+ coords[2] = numeric_cast< int >(z);
- return cartesianRank( coords );
+ return cartesianRank(coords);
}
-bool MPIManager::isCartesianCommValid() const
+bool MPIManager::isCommMPIIOValid() const
{
- // Avoid using the Cartesian MPI-communicator with certain (buggy) versions of OpenMPI (see waLBerla issue #73)
- #if defined(OMPI_MAJOR_VERSION) && defined(OMPI_MINOR_VERSION) && defined(OMPI_RELEASE_VERSION)
- std::string ompi_ver = std::to_string(OMPI_MAJOR_VERSION) + "." + std::to_string(OMPI_MINOR_VERSION) + "." +
- std::to_string(OMPI_RELEASE_VERSION);
-
- if (ompi_ver == "2.0.0" || ompi_ver == "2.0.1" || ompi_ver == "2.0.2" || ompi_ver == "2.0.3" ||
- ompi_ver == "2.1.0" || ompi_ver == "2.1.1") {
- return false;
- }
- else {
- return true;
- }
- #else
- return true;
- #endif
+ // certain versions of OpenMPI produce segmentation faults when using MPI-IO with a 3D Cartesian MPI communicator
+ // (see waLBerla issue #73)
+
+ if (!hasCartesianSetup()) { return true; }
+
+#if defined(OMPI_MAJOR_VERSION) && defined(OMPI_MINOR_VERSION) && defined(OMPI_RELEASE_VERSION)
+
+ static std::once_flag printWarningOnce;
+
+ std::string ompi_ver = std::to_string(OMPI_MAJOR_VERSION) + "." + std::to_string(OMPI_MINOR_VERSION) + "." +
+ std::to_string(OMPI_RELEASE_VERSION);
+
+ if (ompi_ver == "2.0.0" || ompi_ver == "2.0.1" || ompi_ver == "2.0.2" || ompi_ver == "2.0.3" ||
+ ompi_ver == "2.1.0" || ompi_ver == "2.1.1")
+ {
+ std::call_once(printWarningOnce, [](){
+ WALBERLA_LOG_WARNING_ON_ROOT(
+ "Your version of OpenMPI is known to produce segmentation faults when using MPI-IO with a 3D Cartesian MPI "
+ "communicator (see waLBerla issue #73). Please try to use a different version of OpenMPI. As a workaround, "
+ "serial I/O is used now. This might lead to a decrease in performance.");
+ });
+
+ return false;
+ }
+
+ if (ompi_ver == "4.0.0")
+ {
+ std::call_once(printWarningOnce, [](){
+ WALBERLA_LOG_WARNING_ON_ROOT(
+ "Several users of waLBerla with your version of OpenMPI have experienced issues with corrupt VTK output "
+ "files. While the VTK files are fine at first, the MPI-IO seems to sporadically produce erroneous output from "
+ "a certain point on. This has primarily been noticed when writing large files from more than a few hundreds "
+ "of processes. We are unaware whether this is a general problem with OpenMPI 4.0.0 or if this problem only "
+ "occurs on the specific machine that these users have used. If you experience similar problems, please do not "
+ "hesitate to inform us.");
+ });
+ }
+#endif
+
+ return true;
}
std::string MPIManager::getMPIErrorString(int errorCode)
{
WALBERLA_NON_MPI_SECTION()
{
- throw std::logic_error("Trying to use function 'MPIManager::getMPIErrorString' but waLBerla is compiled without MPI-support!");
+ throw std::logic_error(
+ "Trying to use function 'MPIManager::getMPIErrorString' but waLBerla is compiled without MPI-support!");
}
WALBERLA_ASSERT_GREATER(MPI_MAX_ERROR_STRING, 0);
- std::vector<char> errorString(MPI_MAX_ERROR_STRING);
+ std::vector< char > errorString(MPI_MAX_ERROR_STRING);
int resultLen;
MPI_Error_string(errorCode, &errorString[0], &resultLen);
WALBERLA_ASSERT_GREATER_EQUAL(resultLen, 0);
- WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast<int>(errorString.size()));
+ WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast< int >(errorString.size()));
return std::string(errorString.begin(), errorString.begin() + resultLen);
}
@@ -276,16 +267,17 @@ std::string MPIManager::getMPICommName(MPI_Comm comm)
{
WALBERLA_NON_MPI_SECTION()
{
- throw std::logic_error("Trying to use function 'MPIManager::getMPICommName' but waLBerla is compiled without MPI-support!");
+ throw std::logic_error(
+ "Trying to use function 'MPIManager::getMPICommName' but waLBerla is compiled without MPI-support!");
}
WALBERLA_ASSERT_GREATER(MPI_MAX_OBJECT_NAME, 0);
- std::vector<char> commName(MPI_MAX_OBJECT_NAME);
+ std::vector< char > commName(MPI_MAX_OBJECT_NAME);
int resultLen;
MPI_Comm_get_name(comm, &commName[0], &resultLen);
WALBERLA_ASSERT_GREATER_EQUAL(resultLen, 0);
- WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast<int>(commName.size()));
+ WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast< int >(commName.size()));
return std::string(commName.begin(), commName.begin() + resultLen);
}
diff --git a/src/core/mpi/MPIManager.h b/src/core/mpi/MPIManager.h
index 620d69a70898520db946a8f3110e67c5b541a76a..cef90cd62eeffa5e898f007a88fbf829280992cb 100644
--- a/src/core/mpi/MPIManager.h
+++ b/src/core/mpi/MPIManager.h
@@ -1,15 +1,15 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
@@ -123,8 +123,9 @@ public:
/// Rank is valid after calling createCartesianComm() or useWorldComm()
bool rankValid() const { return rank_ >= 0; }
- /// Using a Cartesian MPI communicator is not valid for certain versions of OpenMPI (see waLBerla issue #73)
- bool isCartesianCommValid() const;
+ /// Indicates whether MPI-IO can be used with the current MPI communicator; certain versions of OpenMPI produce
+ /// segmentation faults when using MPI-IO with a 3D Cartesian MPI communicator (see waLBerla issue #73)
+ bool isCommMPIIOValid() const;
//@}
//*******************************************************************************************************************
diff --git a/src/core/mpi/MPITextFile.cpp b/src/core/mpi/MPITextFile.cpp
index ac9a40cea0bba20bc75c6a55d3abbe7e4c8df7f9..2d78b1acd8e812f02f76e5341105d04f6fcda483 100644
--- a/src/core/mpi/MPITextFile.cpp
+++ b/src/core/mpi/MPITextFile.cpp
@@ -1,34 +1,36 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file MPITextFile.cpp
//! \ingroup core
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
#include "MPITextFile.h"
+#include "core/logging/Logging.h"
#include "core/mpi/Reduce.h"
#include <fstream>
-namespace walberla {
-namespace mpi {
-
-
+namespace walberla
+{
+namespace mpi
+{
//======================================================================================================================
/*!
* \brief Writes file using MPI IO with each process providing a part of it
@@ -40,12 +42,12 @@ namespace mpi {
* \param processLocalPart The part of the file belonging to the calling process (size may differ among processes)
*/
//======================================================================================================================
-void writeMPITextFile( const std::string & filename, const std::string & processLocalPart,
- const MPI_Comm comm /*= MPI_COMM_WORLD*/ )
+void writeMPITextFile(const std::string& filename, const std::string& processLocalPart,
+ const MPI_Comm comm /*= MPI_COMM_WORLD*/)
{
WALBERLA_NON_MPI_SECTION()
{
- std::ofstream ofs( filename.c_str() );
+ std::ofstream ofs(filename.c_str());
ofs << processLocalPart;
ofs.close();
}
@@ -54,53 +56,82 @@ void writeMPITextFile( const std::string & filename, const std::string & process
{
int rank;
int numProcesses;
- MPI_Comm_rank( comm, &rank );
- MPI_Comm_size( comm, &numProcesses );
-
- if( processLocalPart.size() > numeric_cast<std::string::size_type>( std::numeric_limits<int>::max() ) )
- WALBERLA_ABORT( "writeMPITextFile does not support more than " << std::numeric_limits<int>::max() << " characters per process!" );
-
- MPI_File mpiFile;
- int result = MPI_SUCCESS;
- result = MPI_File_open( comm, const_cast<char*>( filename.c_str() ), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL,
- &mpiFile );
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while opening file \"" << filename << "\" for writing. MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- const MPI_Offset filesize = numeric_cast<MPI_Offset>( allReduce( processLocalPart.size(), mpi::SUM, comm ) );
-
- size_t exscanResult;
- size_t input = processLocalPart.size();
- MPI_Exscan( &input, &exscanResult, 1, MPITrait<size_t>::type(), MPI_SUM, comm );
- if( rank == 0 )
- exscanResult = size_t( 0 );
-
- const MPI_Offset offset = numeric_cast<MPI_Offset>( exscanResult );
-
- MPI_File_set_size( mpiFile, filesize );
-
- MPI_Datatype arraytype;
- MPI_Type_contiguous( int_c( processLocalPart.size() ), MPITrait<char>::type(), &arraytype);
- MPI_Type_commit( &arraytype );
-
- result = MPI_File_set_view( mpiFile, offset, MPITrait<char>::type(), arraytype, const_cast<char*>( "native" ),
- MPI_INFO_NULL );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- result = MPI_File_write_all( mpiFile, const_cast<char*>( processLocalPart.c_str() ), int_c( processLocalPart.size() ),
- MPITrait<char>::type(), MPI_STATUS_IGNORE );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << filename << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- result = MPI_File_close( &mpiFile );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while closing file \"" << filename << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
-
- MPI_Type_free( &arraytype );
+ MPI_Comm_rank(comm, &rank);
+ MPI_Comm_size(comm, &numProcesses);
+
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D Cartesian
+ // MPI communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
+ {
+ std::ofstream ofs;
+
+ for (int i = 0; i != numProcesses; ++i)
+ {
+ if (i == rank)
+ {
+ if (rank == 0) { ofs.open(filename.c_str()); }
+ else
+ {
+ ofs.open(filename.c_str(), std::ofstream::app);
+ }
+ ofs << processLocalPart;
+ ofs.close();
+ }
+ WALBERLA_MPI_BARRIER();
+ }
+ }
+ else
+ {
+ if (processLocalPart.size() > numeric_cast< std::string::size_type >(std::numeric_limits< int >::max()))
+ WALBERLA_ABORT("writeMPITextFile does not support more than " << std::numeric_limits< int >::max()
+ << " characters per process!");
+
+ MPI_File mpiFile;
+ int result = MPI_SUCCESS;
+ result = MPI_File_open(comm, const_cast< char* >(filename.c_str()), MPI_MODE_WRONLY | MPI_MODE_CREATE,
+ MPI_INFO_NULL, &mpiFile);
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while opening file \"" << filename << "\" for writing. MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
+
+ const MPI_Offset filesize = numeric_cast< MPI_Offset >(allReduce(processLocalPart.size(), mpi::SUM, comm));
+
+ size_t exscanResult;
+ size_t input = processLocalPart.size();
+ MPI_Exscan(&input, &exscanResult, 1, MPITrait< size_t >::type(), MPI_SUM, comm);
+ if (rank == 0) exscanResult = size_t(0);
+
+ const MPI_Offset offset = numeric_cast< MPI_Offset >(exscanResult);
+
+ MPI_File_set_size(mpiFile, filesize);
+
+ MPI_Datatype arraytype;
+ MPI_Type_contiguous(int_c(processLocalPart.size()), MPITrait< char >::type(), &arraytype);
+ MPI_Type_commit(&arraytype);
+
+ result = MPI_File_set_view(mpiFile, offset, MPITrait< char >::type(), arraytype, const_cast< char* >("native"),
+ MPI_INFO_NULL);
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
+
+ result = MPI_File_write_all(mpiFile, const_cast< char* >(processLocalPart.c_str()),
+ int_c(processLocalPart.size()), MPITrait< char >::type(), MPI_STATUS_IGNORE);
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while writing to file \"" << filename << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
+
+ result = MPI_File_close(&mpiFile);
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while closing file \"" << filename << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
+
+ MPI_Type_free(&arraytype);
+ }
}
}
diff --git a/src/field/FileIO.impl.h b/src/field/FileIO.impl.h
index 910ff2f880fdc6642dada18f77aafb90ea2e049c..eae64c6b39b1281c9f2d0c01b5c27e2770d18ffe 100644
--- a/src/field/FileIO.impl.h
+++ b/src/field/FileIO.impl.h
@@ -16,325 +16,423 @@
//! \file FileIO.impl.h
//! \ingroup field
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
#pragma once
-namespace walberla {
-namespace field {
-
-namespace internal {
-
-
+namespace walberla
+{
+namespace field
+{
+namespace internal
+{
template< typename FieldT >
class FieldWriter
{
-public:
- FieldWriter( const std::string & filename, const BlockDataID & fieldID,
- const Set<SUID> & requiredSelectors = Set<SUID>::emptySet(), const Set<SUID> & incompatibleSelectors = Set<SUID>::emptySet() )
- : filename_( filename ), fieldID_( fieldID ), requiredSelectors_( requiredSelectors ), incompatibleSelectors_( incompatibleSelectors )
- {
- }
+ public:
+ FieldWriter(const std::string& filename, const BlockDataID& fieldID,
+ const Set< SUID >& requiredSelectors = Set< SUID >::emptySet(),
+ const Set< SUID >& incompatibleSelectors = Set< SUID >::emptySet())
+ : filename_(filename), fieldID_(fieldID), requiredSelectors_(requiredSelectors),
+ incompatibleSelectors_(incompatibleSelectors)
+ {}
- void writeToFile( const BlockStorage & blockStorage ) const;
- void readFromFile( BlockStorage & blockStorage ) const;
-private:
+ void writeToFile(const BlockStorage& blockStorage) const;
+ void readFromFile(BlockStorage& blockStorage) const;
- std::vector< IBlock * > getBlocks( BlockStorage & blockStorage ) const;
- std::vector< const IBlock * > getBlocks( const BlockStorage & blockStorage ) const;
+ private:
+ std::vector< IBlock* > getBlocks(BlockStorage& blockStorage) const;
+ std::vector< const IBlock* > getBlocks(const BlockStorage& blockStorage) const;
- void writeToFileNonMPI( const std::vector< const IBlock * > & blocks ) const;
- void readFromFileNonMPI( const std::vector< IBlock * > & blocks ) const;
+ void writeToFileNonMPI(const std::vector< const IBlock* >& blocks) const;
+ void readFromFileNonMPI(const std::vector< IBlock* >& blocks) const;
- std::vector< uint_t > computeBlockOffsets( const std::vector< const IBlock * > & blocks ) const;
- std::vector< uint_t > computeBlockOffsets( const std::vector< IBlock * > & blocks ) const;
-
- MPI_Offset computeProcessByteOffset( uint_t processNumElements ) const;
+ std::vector< uint_t > computeBlockOffsets(const std::vector< const IBlock* >& blocks) const;
+ std::vector< uint_t > computeBlockOffsets(const std::vector< IBlock* >& blocks) const;
+ MPI_Offset computeProcessByteOffset(uint_t processNumElements) const;
std::string filename_;
BlockDataID fieldID_;
-
- Set<SUID> requiredSelectors_;
- Set<SUID> incompatibleSelectors_;
-};
-
+ Set< SUID > requiredSelectors_;
+ Set< SUID > incompatibleSelectors_;
+};
template< typename FieldT >
-void FieldWriter<FieldT>::writeToFile( const BlockStorage & blockStorage ) const
+void FieldWriter< FieldT >::writeToFile(const BlockStorage& blockStorage) const
{
- std::vector< const IBlock * > blocks = getBlocks( blockStorage );
-
+ std::vector< const IBlock* > blocks = getBlocks(blockStorage);
+
WALBERLA_NON_MPI_SECTION()
{
- writeToFileNonMPI( blocks );
+ writeToFileNonMPI(blocks);
return;
}
- std::vector< uint_t > blockOffsets = computeBlockOffsets( blocks );
+ std::vector< uint_t > blockOffsets = computeBlockOffsets(blocks);
+ const MPI_Offset offset = computeProcessByteOffset(blockOffsets.back());
- if( blockOffsets.back() > uint_c( std::numeric_limits<int>::max() ) )
- WALBERLA_ABORT( "writeToFile does not support writing more than " << std::numeric_limits<int>::max() << " field elements per process!" );
+ if (blockOffsets.back() > uint_c(std::numeric_limits< int >::max()))
+ WALBERLA_ABORT("writeToFile does not support writing more than " << std::numeric_limits< int >::max()
+ << " field elements per process!");
- const MPI_Offset filesize = numeric_cast<MPI_Offset>( mpi::allReduce( blockOffsets.back(), mpi::SUM, MPIManager::instance()->comm() ) * sizeof( typename FieldT::value_type ) );
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D Cartesian MPI
+ // communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
+ {
+ std::ofstream ofs;
- MPI_Datatype arraytype;
- MPI_Type_contiguous( int_c( blockOffsets.back() ), MPITrait< typename FieldT::value_type >::type(), &arraytype );
- MPI_Type_commit( &arraytype );
+ // clear an existing file's content
+ WALBERLA_ROOT_SECTION()
+ {
+ ofs.open(filename_.c_str(), std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
+ if (ofs.fail()) { WALBERLA_ABORT("Error while opening file \"" << filename_ << "\" for writing."); }
- const MPI_Offset offset = computeProcessByteOffset( blockOffsets.back() );
+ ofs.close();
+ if (ofs.fail()) { WALBERLA_ABORT("Error while closing file \"" << filename_ << "\" for writing."); }
+ }
- MPI_File mpiFile = MPI_FILE_NULL;
- int result = MPI_SUCCESS;
- result = MPI_File_open( MPIManager::instance()->comm(), const_cast<char*>( filename_.c_str() ), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpiFile );
+ // write every block's field to the file
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ ofs.open(filename_.c_str(), std::ofstream::out | std::ofstream::binary | std::ofstream::app);
+ if (ofs.fail()) { WALBERLA_ABORT("Error while opening file \"" << filename_ << "\" for writing."); }
+
+ size_t blockIdx = 0;
+ for (auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx)
+ {
+ const FieldT* field = (*block)->template getData< FieldT >(fieldID_);
+
+ std::vector< typename FieldT::value_type > data(field->xSize() * field->ySize() * field->zSize() *
+ field->fSize());
+
+ auto dataIt = data.begin();
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt)
+ {
+ WALBERLA_ASSERT_UNEQUAL(dataIt, data.end());
+ *dataIt = *fieldIt;
+ }
+ WALBERLA_ASSERT_EQUAL(dataIt, data.end());
+
+ ofs.seekp(numeric_cast< std::streamoff >(numeric_cast< uint_t >(offset) +
+ blockOffsets[blockIdx] * sizeof(typename FieldT::value_type)));
+ ofs.write(reinterpret_cast< const char* >(&data[0]),
+ numeric_cast< std::streamsize >(data.size() * sizeof(typename FieldT::value_type)));
+ if (ofs.fail()) { WALBERLA_ABORT("Error while writing to file \"" << filename_ << "\"."); }
+ }
+ ofs.close();
+ if (ofs.fail()) { WALBERLA_ABORT("Error while closing file \"" << filename_ << "\" for writing."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
+ }
+ else // use MPI-IO
+ {
+ const MPI_Offset filesize =
+ numeric_cast< MPI_Offset >(mpi::allReduce(blockOffsets.back(), mpi::SUM, MPIManager::instance()->comm()) *
+ sizeof(typename FieldT::value_type));
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while opening file \"" << filename_ << "\" for writing. MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ MPI_Datatype arraytype;
+ MPI_Type_contiguous(int_c(blockOffsets.back()), MPITrait< typename FieldT::value_type >::type(), &arraytype);
+ MPI_Type_commit(&arraytype);
- MPI_File_set_size( mpiFile, filesize );
+ MPI_File mpiFile = MPI_FILE_NULL;
+ int result = MPI_SUCCESS;
+ result = MPI_File_open(MPIManager::instance()->comm(), const_cast< char* >(filename_.c_str()),
+ MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpiFile);
- result = MPI_File_set_view( mpiFile, offset, MPITrait< typename FieldT::value_type >::type(), arraytype, const_cast<char*>( "native" ), MPI_INFO_NULL );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while opening file \"" << filename_ << "\" for writing. MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ MPI_File_set_size(mpiFile, filesize);
- size_t blockIdx = 0;
- for( auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx )
- {
- const FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ result = MPI_File_set_view(mpiFile, offset, MPITrait< typename FieldT::value_type >::type(), arraytype,
+ const_cast< char* >("native"), MPI_INFO_NULL);
- std::vector< typename FieldT::value_type > data( field->xSize() * field->ySize() * field->zSize() * field->fSize() );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- auto dataIt = data.begin();
- for( auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt )
+ size_t blockIdx = 0;
+ for (auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx)
{
- WALBERLA_ASSERT( dataIt != data.end() );
- *dataIt = *fieldIt;
+ const FieldT* field = (*block)->template getData< FieldT >(fieldID_);
+
+ std::vector< typename FieldT::value_type > data(field->xSize() * field->ySize() * field->zSize() *
+ field->fSize());
+
+ auto dataIt = data.begin();
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt)
+ {
+ WALBERLA_ASSERT_UNEQUAL(dataIt, data.end());
+ *dataIt = *fieldIt;
+ }
+ WALBERLA_ASSERT_EQUAL(dataIt, data.end());
+
+ result =
+ MPI_File_write_at(mpiFile, numeric_cast< MPI_Offset >(blockOffsets[blockIdx]), &data[0], int_c(data.size()),
+ MPITrait< typename FieldT::value_type >::type(), MPI_STATUS_IGNORE);
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while writing to file \"" << filename_ << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
}
- WALBERLA_ASSERT_EQUAL( dataIt, data.end() );
-
- result = MPI_File_write_at( mpiFile, numeric_cast<MPI_Offset>( blockOffsets[blockIdx] ), &data[0], int_c( data.size() ),
- MPITrait<typename FieldT::value_type>::type(), MPI_STATUS_IGNORE );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while writing to file \"" << filename_ << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
- }
- result = MPI_File_close( &mpiFile );
+ result = MPI_File_close(&mpiFile);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while closing file \"" << filename_ << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while closing file \"" << filename_ << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
- MPI_Type_free( &arraytype );
+ MPI_Type_free(&arraytype);
+ }
}
-
-
template< typename FieldT >
-void FieldWriter<FieldT>::readFromFile( BlockStorage & blockStorage ) const
+void FieldWriter< FieldT >::readFromFile(BlockStorage& blockStorage) const
{
- std::vector< IBlock * > blocks = getBlocks( blockStorage );
-
+ std::vector< IBlock* > blocks = getBlocks(blockStorage);
+
WALBERLA_NON_MPI_SECTION()
{
- readFromFileNonMPI( blocks );
+ readFromFileNonMPI(blocks);
return;
}
- std::vector< uint_t > blockOffsets = computeBlockOffsets( blocks );
-
- if( blockOffsets.back() > uint_c( std::numeric_limits<int>::max() ) )
- WALBERLA_ABORT( "readFromFile does not support reading more than " << std::numeric_limits<int>::max() << " field elements per process!" );
-
- MPI_Datatype arraytype;
- MPI_Type_contiguous( int_c( blockOffsets.back() ), MPITrait< typename FieldT::value_type >::type(), &arraytype );
- MPI_Type_commit( &arraytype );
-
- const MPI_Offset offset = computeProcessByteOffset( blockOffsets.back() );
+ std::vector< uint_t > blockOffsets = computeBlockOffsets(blocks);
+ const MPI_Offset offset = computeProcessByteOffset(blockOffsets.back());
- MPI_File mpiFile;
- int result = MPI_SUCCESS;
- result = MPI_File_open( MPIManager::instance()->comm(), const_cast<char*>( filename_.c_str() ), MPI_MODE_RDONLY, MPI_INFO_NULL, &mpiFile );
+ if (blockOffsets.back() > uint_c(std::numeric_limits< int >::max()))
+ WALBERLA_ABORT("readFromFile does not support reading more than " << std::numeric_limits< int >::max()
+ << " field elements per process!");
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while opening file \"" << filename_ << "\" for reading. MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ // use serial I/O for versions of OpenMPI that produce segmentation faults when using MPI-IO with a 3D Cartesian MPI
+ // communicator (see waLBerla issue #73)
+ if (!MPIManager::instance()->isCommMPIIOValid())
+ {
+ std::ifstream ifs;
- result = MPI_File_set_view( mpiFile, offset, MPITrait< typename FieldT::value_type >::type(), arraytype, const_cast<char*>( "native" ), MPI_INFO_NULL );
+ // read every block's field from file
+ for (int i = 0; i != MPIManager::instance()->numProcesses(); ++i)
+ {
+ if (i == MPIManager::instance()->rank())
+ {
+ ifs.open(filename_.c_str(), std::ifstream::in | std::ifstream::binary);
+ if (ifs.fail()) { WALBERLA_ABORT("Error while opening file \"" << filename_ << "\" for reading."); }
+
+ size_t blockIdx = 0;
+ for (auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx)
+ {
+ FieldT* field = (*block)->template getData< FieldT >(fieldID_);
+
+ std::vector< typename FieldT::value_type > data(field->xSize() * field->ySize() * field->zSize() *
+ field->fSize());
+
+ ifs.seekg(numeric_cast< std::streamoff >(numeric_cast< uint_t >(offset) +
+ blockOffsets[blockIdx] * sizeof(typename FieldT::value_type)));
+ ifs.read(reinterpret_cast< char* >(&data[0]),
+ numeric_cast< std::streamsize >(data.size() * sizeof(typename FieldT::value_type)));
+ if (ifs.fail()) { WALBERLA_ABORT("Error while reading from file \"" << filename_ << "\"."); }
+
+ auto dataIt = data.begin();
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt)
+ {
+ WALBERLA_ASSERT_UNEQUAL(dataIt, data.end());
+ *fieldIt = *dataIt;
+ }
+ WALBERLA_ASSERT_EQUAL(dataIt, data.end());
+ }
+ ifs.close();
+ if (ifs.fail()) { WALBERLA_ABORT("Error while closing file \"" << filename_ << "\" for reading."); }
+ }
+ WALBERLA_MPI_BARRIER();
+ }
+ }
+ else // use MPI-IO
+ {
+ MPI_Datatype arraytype;
+ MPI_Type_contiguous(int_c(blockOffsets.back()), MPITrait< typename FieldT::value_type >::type(), &arraytype);
+ MPI_Type_commit(&arraytype);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ MPI_File mpiFile;
+ int result = MPI_SUCCESS;
+ result = MPI_File_open(MPIManager::instance()->comm(), const_cast< char* >(filename_.c_str()), MPI_MODE_RDONLY,
+ MPI_INFO_NULL, &mpiFile);
- size_t blockIdx = 0;
- for( auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx )
- {
- FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while opening file \"" << filename_ << "\" for reading. MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
- std::vector< typename FieldT::value_type > data( field->xSize() * field->ySize() * field->zSize() * field->fSize() );
+ result = MPI_File_set_view(mpiFile, offset, MPITrait< typename FieldT::value_type >::type(), arraytype,
+ const_cast< char* >("native"), MPI_INFO_NULL);
- result = MPI_File_read_at( mpiFile, numeric_cast<MPI_Offset>( blockOffsets[blockIdx] ), &data[0], int_c( data.size() ),
- MPITrait<typename FieldT::value_type>::type(), MPI_STATUS_IGNORE );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Internal MPI-IO error! MPI Error is \"" << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
- auto dataIt = data.begin();
- for( auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt )
+ size_t blockIdx = 0;
+ for (auto block = blocks.begin(); block != blocks.end(); ++block, ++blockIdx)
{
- WALBERLA_ASSERT_UNEQUAL( dataIt, data.end() );
- *fieldIt = *dataIt;
+ FieldT* field = (*block)->template getData< FieldT >(fieldID_);
+
+ std::vector< typename FieldT::value_type > data(field->xSize() * field->ySize() * field->zSize() *
+ field->fSize());
+
+ result =
+ MPI_File_read_at(mpiFile, numeric_cast< MPI_Offset >(blockOffsets[blockIdx]), &data[0], int_c(data.size()),
+ MPITrait< typename FieldT::value_type >::type(), MPI_STATUS_IGNORE);
+
+ auto dataIt = data.begin();
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt, ++dataIt)
+ {
+ WALBERLA_ASSERT_UNEQUAL(dataIt, data.end());
+ *fieldIt = *dataIt;
+ }
+ WALBERLA_ASSERT_EQUAL(dataIt, data.end());
+
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while reading from file \"" << filename_ << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result)
+ << "\"");
}
- WALBERLA_ASSERT_EQUAL( dataIt, data.end() );
-
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while reading from file \"" << filename_ << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
- }
- result = MPI_File_close( &mpiFile );
+ result = MPI_File_close(&mpiFile);
- if( result != MPI_SUCCESS )
- WALBERLA_ABORT( "Error while closing file \"" << filename_ << "\". MPI Error is \"" << MPIManager::instance()->getMPIErrorString( result ) << "\"" );
+ if (result != MPI_SUCCESS)
+ WALBERLA_ABORT("Error while closing file \"" << filename_ << "\". MPI Error is \""
+ << MPIManager::instance()->getMPIErrorString(result) << "\"");
- MPI_Type_free( &arraytype );
+ MPI_Type_free(&arraytype);
+ }
}
-
-
-inline bool sortBlocksByID( IBlock * lhs, IBlock * rhs ) { return lhs->getId() < rhs->getId(); }
+inline bool sortBlocksByID(IBlock* lhs, IBlock* rhs) { return lhs->getId() < rhs->getId(); }
template< typename FieldT >
-std::vector< IBlock * > FieldWriter<FieldT>::getBlocks( BlockStorage & blockStorage ) const
+std::vector< IBlock* > FieldWriter< FieldT >::getBlocks(BlockStorage& blockStorage) const
{
- std::vector< IBlock * > blocks;
- for( auto it = blockStorage.begin( requiredSelectors_, incompatibleSelectors_ ); it != blockStorage.end(); ++it )
- blocks.push_back( it.get() );
- std::sort( blocks.begin(), blocks.end(), sortBlocksByID );
+ std::vector< IBlock* > blocks;
+ for (auto it = blockStorage.begin(requiredSelectors_, incompatibleSelectors_); it != blockStorage.end(); ++it)
+ blocks.push_back(it.get());
+ std::sort(blocks.begin(), blocks.end(), sortBlocksByID);
return blocks;
}
-
-
-inline bool sortConstBlocksByID( const IBlock * lhs, const IBlock * rhs ) { return lhs->getId() < rhs->getId(); }
+inline bool sortConstBlocksByID(const IBlock* lhs, const IBlock* rhs) { return lhs->getId() < rhs->getId(); }
template< typename FieldT >
-std::vector< const IBlock * > FieldWriter<FieldT>::getBlocks( const BlockStorage & blockStorage ) const
+std::vector< const IBlock* > FieldWriter< FieldT >::getBlocks(const BlockStorage& blockStorage) const
{
- std::vector< const IBlock * > blocks;
- for( auto it = blockStorage.begin( requiredSelectors_, incompatibleSelectors_ ); it != blockStorage.end(); ++it )
- blocks.push_back( it.get() );
- std::sort( blocks.begin(), blocks.end(), sortConstBlocksByID );
+ std::vector< const IBlock* > blocks;
+ for (auto it = blockStorage.begin(requiredSelectors_, incompatibleSelectors_); it != blockStorage.end(); ++it)
+ blocks.push_back(it.get());
+ std::sort(blocks.begin(), blocks.end(), sortConstBlocksByID);
return blocks;
}
-
-
template< typename FieldT >
-void FieldWriter<FieldT>::writeToFileNonMPI( const std::vector< const IBlock * > & blocks ) const
+void FieldWriter< FieldT >::writeToFileNonMPI(const std::vector< const IBlock* >& blocks) const
{
- std::ofstream ofs( filename_.c_str(), std::ofstream::out | std::ofstream::binary );
+ std::ofstream ofs(filename_.c_str(), std::ofstream::out | std::ofstream::binary);
- for( auto block = blocks.begin(); block != blocks.end(); ++block )
+ for (auto block = blocks.begin(); block != blocks.end(); ++block)
{
- const FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ const FieldT* field = (*block)->template getData< FieldT >(fieldID_);
- for( auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt )
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt)
{
- ofs.write( reinterpret_cast<const char*>( &( *fieldIt ) ), sizeof( typename FieldT::value_type ) );
+ ofs.write(reinterpret_cast< const char* >(&(*fieldIt)), sizeof(typename FieldT::value_type));
}
}
ofs.close();
}
-
-
template< typename FieldT >
-void FieldWriter<FieldT>::readFromFileNonMPI( const std::vector< IBlock * > & blocks ) const
+void FieldWriter< FieldT >::readFromFileNonMPI(const std::vector< IBlock* >& blocks) const
{
- std::ifstream ifs( filename_.c_str(), std::ifstream::in | std::ifstream::binary );
+ std::ifstream ifs(filename_.c_str(), std::ifstream::in | std::ifstream::binary);
- for( auto block = blocks.begin(); block != blocks.end(); ++block )
+ for (auto block = blocks.begin(); block != blocks.end(); ++block)
{
- FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ FieldT* field = (*block)->template getData< FieldT >(fieldID_);
- for( auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt )
+ for (auto fieldIt = field->begin(); fieldIt != field->end(); ++fieldIt)
{
- ifs.read( reinterpret_cast<char*>( &( *fieldIt ) ), sizeof( typename FieldT::value_type ) );
+ ifs.read(reinterpret_cast< char* >(&(*fieldIt)), sizeof(typename FieldT::value_type));
}
}
ifs.close();
}
-
-
template< typename FieldT >
-std::vector< uint_t > FieldWriter<FieldT>::computeBlockOffsets( const std::vector< const IBlock * > & blocks ) const
+std::vector< uint_t > FieldWriter< FieldT >::computeBlockOffsets(const std::vector< const IBlock* >& blocks) const
{
std::vector< uint_t > blockOffsets;
- blockOffsets.push_back( 0 );
+ blockOffsets.push_back(0);
- for( auto block = blocks.begin(); block != blocks.end(); ++block )
+ for (auto block = blocks.begin(); block != blocks.end(); ++block)
{
- const FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ const FieldT* field = (*block)->template getData< FieldT >(fieldID_);
const uint_t offset = blockOffsets.back() + field->xSize() * field->ySize() * field->zSize() * field->fSize();
- blockOffsets.push_back( offset );
+ blockOffsets.push_back(offset);
}
return blockOffsets;
}
-
-
template< typename FieldT >
-std::vector< uint_t > FieldWriter<FieldT>::computeBlockOffsets( const std::vector< IBlock * > & blocks ) const
+std::vector< uint_t > FieldWriter< FieldT >::computeBlockOffsets(const std::vector< IBlock* >& blocks) const
{
std::vector< uint_t > blockOffsets;
- blockOffsets.push_back( 0 );
+ blockOffsets.push_back(0);
- for( auto block = blocks.begin(); block != blocks.end(); ++block )
+ for (auto block = blocks.begin(); block != blocks.end(); ++block)
{
- const FieldT * field = (*block)->template getData<FieldT>( fieldID_ );
+ const FieldT* field = (*block)->template getData< FieldT >(fieldID_);
const uint_t offset = blockOffsets.back() + field->xSize() * field->ySize() * field->zSize() * field->fSize();
- blockOffsets.push_back( offset );
+ blockOffsets.push_back(offset);
}
return blockOffsets;
}
-
-
template< typename FieldT >
-MPI_Offset FieldWriter<FieldT>::computeProcessByteOffset( uint_t processNumElements ) const
+MPI_Offset FieldWriter< FieldT >::computeProcessByteOffset(uint_t processNumElements) const
{
uint_t exscanResult;
- MPI_Exscan( &processNumElements, &exscanResult, 1, MPITrait<uint_t>::type(), MPI_SUM, MPIManager::instance()->comm() );
- if( MPIManager::instance()->rank() == 0 )
- exscanResult = uint_t( 0 );
+ MPI_Exscan(&processNumElements, &exscanResult, 1, MPITrait< uint_t >::type(), MPI_SUM,
+ MPIManager::instance()->comm());
+ if (MPIManager::instance()->rank() == 0) exscanResult = uint_t(0);
- return numeric_cast<MPI_Offset>( exscanResult * sizeof( typename FieldT::value_type ) );
+ return numeric_cast< MPI_Offset >(exscanResult * sizeof(typename FieldT::value_type));
}
-
} // namespace internal
-
-
template< typename FieldT >
-void writeToFile( const std::string & filename, const BlockStorage & blockStorage, const BlockDataID & fieldID,
- const Set<SUID> & requiredSelectors, const Set<SUID> & incompatibleSelectors )
+void writeToFile(const std::string& filename, const BlockStorage& blockStorage, const BlockDataID& fieldID,
+ const Set< SUID >& requiredSelectors, const Set< SUID >& incompatibleSelectors)
{
- internal::FieldWriter<FieldT> writer( filename, fieldID, requiredSelectors, incompatibleSelectors );
- writer.writeToFile( blockStorage );
+ internal::FieldWriter< FieldT > writer(filename, fieldID, requiredSelectors, incompatibleSelectors);
+ writer.writeToFile(blockStorage);
}
-
-
template< typename FieldT >
-void readFromFile( const std::string & filename, BlockStorage & blockStorage, const BlockDataID & fieldID,
- const Set<SUID> & requiredSelectors, const Set<SUID> & incompatibleSelectors )
+void readFromFile(const std::string& filename, BlockStorage& blockStorage, const BlockDataID& fieldID,
+ const Set< SUID >& requiredSelectors, const Set< SUID >& incompatibleSelectors)
{
- internal::FieldWriter<FieldT> writer( filename, fieldID, requiredSelectors, incompatibleSelectors );
- writer.readFromFile( blockStorage );
+ internal::FieldWriter< FieldT > writer(filename, fieldID, requiredSelectors, incompatibleSelectors);
+ writer.readFromFile(blockStorage);
}
-} // namespace walberla
} // namespace field
+} // namespace walberla
diff --git a/tests/blockforest/BlockDataIOTest.cpp b/tests/blockforest/BlockDataIOTest.cpp
index 291b3322a6e1b1324bf74f2ce27cbaed5ec0423e..7f1507bc1ca508cca10dd070d5feb7f549d5ffe9 100644
--- a/tests/blockforest/BlockDataIOTest.cpp
+++ b/tests/blockforest/BlockDataIOTest.cpp
@@ -1,21 +1,22 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file BlockDataIOTest.cpp
//! \ingroup field
//! \author Florian Schornbaum <florian.schornbaum@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
@@ -27,102 +28,110 @@
#include "core/debug/TestSubsystem.h"
#include "core/math/Random.h"
#include "core/mpi/Environment.h"
-#include "core/timing/Timer.h"
#include "field/AddToStorage.h"
#include "field/Field.h"
-
-namespace block_data_io_test {
-
+namespace block_data_io_test
+{
using namespace walberla;
using walberla::uint8_t;
-const SUID Empty( "empty" );
-const Set<SUID> None( Set<SUID>::emptySet() );
+const SUID Empty("empty");
+const Set< SUID > None(Set< SUID >::emptySet());
-static void refinementSelectionFunction( SetupBlockForest& forest )
+static void refinementSelectionFunction(SetupBlockForest& forest)
{
- for( auto block = forest.begin(); block != forest.end(); ++block )
- if( block->getAABB().contains( Vector3<real_t>( real_t(75) ) ) )
- if( !block->hasFather() )
- block->setMarker( true );
+ for (auto block = forest.begin(); block != forest.end(); ++block)
+ if (block->getAABB().contains(Vector3< real_t >(real_t(75))))
+ if (!block->hasFather()) block->setMarker(true);
}
-static void workloadMemorySUIDAssignmentFunction( SetupBlockForest& forest )
+static void workloadMemorySUIDAssignmentFunction(SetupBlockForest& forest)
{
- for( auto block = forest.begin(); block != forest.end(); ++block )
+ for (auto block = forest.begin(); block != forest.end(); ++block)
{
- block->setMemory( memory_t(1) );
- block->setWorkload( workload_t(1) );
- if( block->getAABB().contains( Vector3<real_t>( real_t(25) ) ) )
- block->addState( Empty );
+ block->setMemory(memory_t(1));
+ block->setWorkload(workload_t(1));
+ if (block->getAABB().contains(Vector3< real_t >(real_t(25)))) block->addState(Empty);
}
}
-int main( int argc, char* argv[] )
+void test()
{
- typedef field::GhostLayerField<double, 2> FieldType;
+ typedef field::GhostLayerField< double, 2 > FieldType;
- debug::enterTestMode();
+ SetupBlockForest sforest;
- mpi::Environment mpiEnv( argc, argv );
+ sforest.addRefinementSelectionFunction(refinementSelectionFunction);
+ sforest.addWorkloadMemorySUIDAssignmentFunction(workloadMemorySUIDAssignmentFunction);
- MPIManager::instance()->useWorldComm();
+ AABB domain(0, 0, 0, 100, 100, 100);
- SetupBlockForest sforest;
+ sforest.init(domain, uint_t(2), uint_t(2), uint_t(2), true, false, false);
- sforest.addRefinementSelectionFunction( refinementSelectionFunction );
- sforest.addWorkloadMemorySUIDAssignmentFunction( workloadMemorySUIDAssignmentFunction );
+ sforest.balanceLoad(blockforest::StaticLevelwiseCurveBalance(true), uint_c(MPIManager::instance()->numProcesses()));
- AABB domain( 0, 0, 0, 100, 100, 100 );
-
- sforest.init( domain, uint_t(2), uint_t(2), uint_t(2), true, false, false );
+ auto sbf = make_shared< StructuredBlockForest >(
+ make_shared< BlockForest >(uint_c(MPIManager::instance()->rank()), sforest, true), uint_t(10), uint_t(8),
+ uint_t(14));
- sforest.balanceLoad( blockforest::StaticLevelwiseCurveBalance(true), uint_c( MPIManager::instance()->numProcesses() ) );
-
- auto sbf = make_shared<StructuredBlockForest>( make_shared< BlockForest >( uint_c( MPIManager::instance()->rank() ), sforest, true ),
- uint_t(10), uint_t(8), uint_t(14) );
+ blockforest::BlockForestEvaluation evaluation(sbf->getBlockForest());
+ WALBERLA_LOG_INFO_ON_ROOT("BlockForest:\n" << evaluation.toString());
- blockforest::BlockForestEvaluation evaluation( sbf->getBlockForest() );
- WALBERLA_LOG_INFO_ON_ROOT( "BlockForest:\n" << evaluation.toString() );
+ // auto originalFieldId = field::addToStorage< FieldType >( sbf, "OriginalField", 0.0, field::zyxf, uint_t(3), false,
+ // None, Empty );
+ auto dataHandling = make_shared< field::DefaultBlockDataHandling< FieldType > >(sbf, uint_t(3), 0.0, field::zyxf);
+ auto originalFieldId = sbf->addBlockData(dataHandling, "OriginalField", None, Empty);
- //auto originalFieldId = field::addToStorage< FieldType >( sbf, "OriginalField", 0.0, field::zyxf, uint_t(3), false, None, Empty );
- auto dataHandling = make_shared< field::DefaultBlockDataHandling< FieldType > >( sbf, uint_t(3), 0.0, field::zyxf );
- auto originalFieldId = sbf->addBlockData( dataHandling, "OriginalField", None, Empty );
-
- math::seedRandomGenerator( numeric_cast<std::mt19937::result_type>( MPIManager::instance()->rank() ) );
+ math::seedRandomGenerator(numeric_cast< std::mt19937::result_type >(MPIManager::instance()->rank()));
- for( auto it = sbf->begin( None, Empty ); it != sbf->end(); ++it )
+ for (auto it = sbf->begin(None, Empty); it != sbf->end(); ++it)
{
- auto field = it->getData< FieldType >( originalFieldId );
+ auto field = it->getData< FieldType >(originalFieldId);
- for( auto dataIt = field->begin(); dataIt != field->end(); ++dataIt )
+ for (auto dataIt = field->begin(); dataIt != field->end(); ++dataIt)
*dataIt = math::realRandom< FieldType::value_type >();
}
-
- sbf->saveBlockData( "block.data", originalFieldId );
-
+
+ sbf->saveBlockData("block.data", originalFieldId);
+
WALBERLA_MPI_BARRIER()
-
- auto readFieldId = sbf->loadBlockData( "block.data", dataHandling, "ReadField", None, Empty );
-
- for( auto it = sbf->begin( None, Empty ); it != sbf->end(); ++it )
+
+ auto readFieldId = sbf->loadBlockData("block.data", dataHandling, "ReadField", None, Empty);
+
+ for (auto it = sbf->begin(None, Empty); it != sbf->end(); ++it)
{
- auto originalField = it->getData< FieldType >( originalFieldId );
- auto readField = it->getData< FieldType >( readFieldId );
+ auto originalField = it->getData< FieldType >(originalFieldId);
+ auto readField = it->getData< FieldType >(readFieldId);
auto readIt = readField->begin();
- for( auto origIt = originalField->begin(); origIt != originalField->end(); ++origIt, ++readIt )
- WALBERLA_CHECK_IDENTICAL( *origIt, *readIt );
+ for (auto origIt = originalField->begin(); origIt != originalField->end(); ++origIt, ++readIt)
+ WALBERLA_CHECK_IDENTICAL(*origIt, *readIt);
}
-
- return EXIT_SUCCESS;
}
-}
+} // namespace block_data_io_test
-int main( int argc, char* argv[] )
+int main(int argc, char* argv[])
{
- return block_data_io_test::main( argc, argv );
+ walberla::debug::enterTestMode();
+
+ walberla::mpi::Environment mpiEnv(argc, argv);
+
+ // test with MPI_WORLD_COMM
+ walberla::MPIManager::instance()->useWorldComm();
+ block_data_io_test::test();
+
+ // test with Cartesian MPI communicator
+ // this is tested additionally because some versions of OpenMPI are known to produce segmentation faults when using
+ // MPI-IO with a 3D Cartesian MPI communicator; for those OpenMPI versions, serial I/O is used instead
+ if (walberla::MPIManager::instance()->numProcesses() == 8)
+ {
+ walberla::MPIManager::instance()->resetMPI();
+
+ walberla::MPIManager::instance()->createCartesianComm(walberla::uint_c(2), walberla::uint_c(2),
+ walberla::uint_c(2), false, false, false);
+ block_data_io_test::test();
+ }
}
diff --git a/tests/core/mpi/MPITextFileTest.cpp b/tests/core/mpi/MPITextFileTest.cpp
index f3e17dbd188ad3f0a091fafa801cfdb58303e252..34eeba5099cb4b46b9965fdd2126e89a258dce99 100644
--- a/tests/core/mpi/MPITextFileTest.cpp
+++ b/tests/core/mpi/MPITextFileTest.cpp
@@ -1,81 +1,75 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file MPITextFileText.cpp
//! \ingroup core
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
+#include "core/mpi/MPITextFile.h"
+
#include "core/Abort.h"
#include "core/DataTypes.h"
-
+#include "core/Filesystem.h"
#include "core/debug/TestSubsystem.h"
-
#include "core/mpi/MPIManager.h"
-#include "core/mpi/MPITextFile.h"
-
#include "core/stringToNum.h"
-#include "core/Filesystem.h"
-
-#include <vector>
-#include <sstream>
#include <fstream>
+#include <sstream>
+#include <vector>
-
-void testSameSizeFile( const std::string & filename, const size_t chunkSize )
+void testSameSizeFile(const std::string& filename, const size_t chunkSize)
{
using namespace walberla;
- WALBERLA_CHECK_GREATER( chunkSize, 0 );
+ WALBERLA_CHECK_GREATER(chunkSize, 0);
const int rank = MPIManager::instance()->rank();
std::ostringstream oss;
oss << rank;
- std::string chunk( chunkSize, char( 'A' + static_cast<char>( rank % 26 ) ) );
- chunk[ chunk.size() - size_t(1) ] = '\n';
-
- mpi::writeMPITextFile( filename, chunk );
+ std::string chunk(chunkSize, char('A' + static_cast< char >(rank % 26)));
+ chunk[chunk.size() - size_t(1)] = '\n';
- std::ifstream ifs( filename.c_str() );
- ifs.seekg( numeric_cast< std::ifstream::off_type >( uint_c(rank) * chunkSize ), std::ios_base::beg );
- std::vector<char> buffer( chunkSize );
- ifs.read( &(buffer[0]), numeric_cast< std::streamsize >( chunkSize ) );
+ mpi::writeMPITextFile(filename, chunk);
+
+ std::ifstream ifs(filename.c_str());
+ ifs.seekg(numeric_cast< std::ifstream::off_type >(uint_c(rank) * chunkSize), std::ios_base::beg);
+ std::vector< char > buffer(chunkSize);
+ ifs.read(&(buffer[0]), numeric_cast< std::streamsize >(chunkSize));
ifs.close();
- std::string referenceChunk( buffer.begin(), buffer.end() );
+ std::string referenceChunk(buffer.begin(), buffer.end());
- WALBERLA_CHECK_EQUAL( chunk, referenceChunk );
+ WALBERLA_CHECK_EQUAL(chunk, referenceChunk);
WALBERLA_MPI_BARRIER();
WALBERLA_ROOT_SECTION()
{
- if( filesystem::exists( filename ) )
- filesystem::remove( filename );
+ if (filesystem::exists(filename)) filesystem::remove(filename);
}
WALBERLA_MPI_BARRIER();
}
-
-
-void testDifferentSizeFile( const std::string & filename, const size_t minChunkSize )
+void testDifferentSizeFile(const std::string& filename, const size_t minChunkSize)
{
using namespace walberla;
- WALBERLA_CHECK_GREATER( minChunkSize, 0 );
+ WALBERLA_CHECK_GREATER(minChunkSize, 0);
const int rank = MPIManager::instance()->rank();
std::ostringstream oss;
@@ -83,53 +77,64 @@ void testDifferentSizeFile( const std::string & filename, const size_t minChunkS
const size_t chunkSize = minChunkSize * uint_c(rank + 1);
- std::string chunk(chunkSize , char( 'A' + static_cast<char>( rank % 26 ) ) );
- chunk[ chunk.size() - size_t(1) ] = '\n';
+ std::string chunk(chunkSize, char('A' + static_cast< char >(rank % 26)));
+ chunk[chunk.size() - size_t(1)] = '\n';
- mpi::writeMPITextFile( filename, chunk );
+ mpi::writeMPITextFile(filename, chunk);
- std::ifstream ifs( filename.c_str() );
- ifs.seekg( numeric_cast< std::ifstream::off_type >( uint_c( ( rank * rank + rank ) / 2 ) * minChunkSize ), std::ios_base::beg );
- std::vector<char> buffer( chunkSize );
- ifs.read( &(buffer[0]), numeric_cast< std::streamsize >( chunkSize ) );
+ std::ifstream ifs(filename.c_str());
+ ifs.seekg(numeric_cast< std::ifstream::off_type >(uint_c((rank * rank + rank) / 2) * minChunkSize),
+ std::ios_base::beg);
+ std::vector< char > buffer(chunkSize);
+ ifs.read(&(buffer[0]), numeric_cast< std::streamsize >(chunkSize));
ifs.close();
- std::string referenceChunk( buffer.begin(), buffer.end() );
+ std::string referenceChunk(buffer.begin(), buffer.end());
- WALBERLA_CHECK_EQUAL( chunk, referenceChunk );
+ WALBERLA_CHECK_EQUAL(chunk, referenceChunk);
WALBERLA_MPI_BARRIER();
WALBERLA_ROOT_SECTION()
{
- if( filesystem::exists( filename ) )
- filesystem::remove( filename );
+ if (filesystem::exists(filename)) filesystem::remove(filename);
}
WALBERLA_MPI_BARRIER();
}
-
-
-int main( int argc, char * argv[] )
+int main(int argc, char* argv[])
{
- walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
-
- walberla::debug::enterTestMode();
+ walberla::MPIManager::instance()->initializeMPI(&argc, &argv);
- walberla::MPIManager::instance()->useWorldComm();
+ walberla::debug::enterTestMode();
- std::vector<std::string> args( argv, argv + argc );
+ std::vector< std::string > args(argv, argv + argc);
- size_t chunkSize;
+ size_t chunkSize;
std::string filename;
try
{
- chunkSize = walberla::stringToNum<size_t>( args.at(2) );
- filename = args.at( 1 );
- }
- catch( ... )
+ chunkSize = walberla::stringToNum< size_t >(args.at(2));
+ filename = args.at(1);
+ } catch (...)
+
{
- WALBERLA_ABORT_NO_DEBUG_INFO( "Usage:\n" << args[0] << " FILENAME CHUNK_SIZE" );
+ WALBERLA_ABORT_NO_DEBUG_INFO("Usage:\n" << args[0] << " FILENAME CHUNK_SIZE");
}
- testSameSizeFile( filename, chunkSize );
- testDifferentSizeFile( filename, chunkSize );
+ // test with MPI_WORLD_COMM
+ walberla::MPIManager::instance()->useWorldComm();
+ testSameSizeFile(filename, chunkSize);
+ testDifferentSizeFile(filename, chunkSize);
+
+ // test with Cartesian MPI communicator
+ // this is tested additionally since some versions of OpenMPI are known to produce segmentation faults when using
+ // MPI-IO with a 3D Cartesian MPI communicator; for those OpenMPI versions, serial I/O is used instead
+ if (walberla::MPIManager::instance()->numProcesses() == 8)
+ {
+ walberla::MPIManager::instance()->resetMPI();
+ walberla::MPIManager::instance()->createCartesianComm(walberla::uint_c(2), walberla::uint_c(2),
+ walberla::uint_c(2), false, false, false);
+
+ testSameSizeFile(filename, chunkSize);
+ testDifferentSizeFile(filename, chunkSize);
+ }
}
diff --git a/tests/field/FieldFileIOTest.cpp b/tests/field/FieldFileIOTest.cpp
index 53d1c106166cab0265eff8f5d96510dd3c6ce960..8559deacfd9f9a0eb567f5c276e241b0df451fd7 100644
--- a/tests/field/FieldFileIOTest.cpp
+++ b/tests/field/FieldFileIOTest.cpp
@@ -1,21 +1,22 @@
//======================================================================================================================
//
-// This file is part of waLBerla. waLBerla is free software: you can
+// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
-// License as published by the Free Software Foundation, either version 3 of
+// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
-//
-// waLBerla is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
-//
+//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file FieldFileIOTest.cpp
//! \ingroup field
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
+//! \author Christoph Schwarzmeier <christoph.schwarzmeier@fau.de>
//
//======================================================================================================================
@@ -24,102 +25,95 @@
#include "blockforest/loadbalancing/StaticCurve.h"
#include "core/debug/TestSubsystem.h"
+#include "core/math/IntegerFactorization.h"
#include "core/math/Random.h"
#include "core/mpi/Environment.h"
-#include "core/timing/Timer.h"
-#include "core/math/IntegerFactorization.h"
#include "core/stringToNum.h"
+#include "core/timing/Timer.h"
#include "field/AddToStorage.h"
#include "field/Field.h"
#include "field/FileIO.h"
-
-namespace mpi_file_io_test {
-
+namespace mpi_file_io_test
+{
using namespace walberla;
using walberla::uint8_t;
-static void refinementSelectionFunction( SetupBlockForest& forest )
+static void refinementSelectionFunction(SetupBlockForest& forest)
{
const uint_t numRootBlocks = forest.getNumberOfRootBlocks();
- for( uint_t i = 0; i < numRootBlocks; i += 8 )
+ for (uint_t i = 0; i < numRootBlocks; i += 8)
{
- SetupBlock* block = forest.getRootBlock( i );
+ SetupBlock* block = forest.getRootBlock(i);
- if( !block->hasChildren() )
- block->setMarker( true );
+ if (!block->hasChildren()) block->setMarker(true);
}
}
-
-
-static void workloadMemorySUIDAssignmentFunction( SetupBlockForest& forest )
+static void workloadMemorySUIDAssignmentFunction(SetupBlockForest& forest)
{
std::vector< SetupBlock* > blocks;
- forest.getBlocks( blocks );
+ forest.getBlocks(blocks);
- for( uint_t i = 0; i != blocks.size(); ++i ) {
- blocks[i]->setMemory( 1.0 );
- blocks[i]->setWorkload( 1.0 );
+ for (uint_t i = 0; i != blocks.size(); ++i)
+ {
+ blocks[i]->setMemory(1.0);
+ blocks[i]->setWorkload(1.0);
}
}
-
-int main( int argc, char* argv[] )
+void test(int argc, char* argv[])
{
- typedef field::GhostLayerField<double, 3> FieldType;
+ typedef field::GhostLayerField< double, 3 > FieldType;
- debug::enterTestMode();
+ std::vector< std::string > args(argv, argv + argc);
- mpi::Environment mpiEnv( argc, argv );
-
- MPIManager::instance()->useWorldComm();
-
- std::vector<std::string> args( argv, argv + argc );
-
- uint_t numBlocks = 8;
+ uint_t numBlocks = 8;
uint_t xBlockSize = 3;
uint_t yBlockSize = 5;
uint_t zBlockSize = 7;
-
- if( args.size() == 5 )
+
+ if (args.size() == 5)
{
- numBlocks = stringToNum<uint_t>( args[1] );
- xBlockSize = stringToNum<uint_t>( args[2] );
- yBlockSize = stringToNum<uint_t>( args[3] );
- zBlockSize = stringToNum<uint_t>( args[4] );
+ numBlocks = stringToNum< uint_t >(args[1]);
+ xBlockSize = stringToNum< uint_t >(args[2]);
+ yBlockSize = stringToNum< uint_t >(args[3]);
+ zBlockSize = stringToNum< uint_t >(args[4]);
}
- else if( args.size() > 5 )
+ else if (args.size() > 5)
{
- WALBERLA_ABORT( "USAGE:\n\n" << args[0] << " <NUMBER_OF_COARSE_BLOCKS> <X_BLOCK_SIZE> <Y_BLOCK_SIZE> <Z_BLOCK_SIZE>" );
+ WALBERLA_ABORT("USAGE:\n\n"
+ << args[0] << " <NUMBER_OF_COARSE_BLOCKS> <X_BLOCK_SIZE> <Y_BLOCK_SIZE> <Z_BLOCK_SIZE>");
}
SetupBlockForest sforest;
- sforest.addRefinementSelectionFunction( refinementSelectionFunction );
- sforest.addWorkloadMemorySUIDAssignmentFunction( workloadMemorySUIDAssignmentFunction );
+ sforest.addRefinementSelectionFunction(refinementSelectionFunction);
+ sforest.addWorkloadMemorySUIDAssignmentFunction(workloadMemorySUIDAssignmentFunction);
- AABB domain( 0, 0, 0, 100, 100, 100 );
+ AABB domain(0, 0, 0, 100, 100, 100);
- auto factors = math::getFactors3D( numBlocks );
+ auto factors = math::getFactors3D(numBlocks);
- sforest.init( domain, factors[0], factors[1], factors[2], true, false, false );
+ sforest.init(domain, factors[0], factors[1], factors[2], true, false, false);
- sforest.balanceLoad( blockforest::StaticLevelwiseCurveBalance(true), uint_c( MPIManager::instance()->numProcesses() ) );
+ sforest.balanceLoad(blockforest::StaticLevelwiseCurveBalance(true), uint_c(MPIManager::instance()->numProcesses()));
- auto sbf = make_shared<StructuredBlockForest>( make_shared< BlockForest >( uint_c( MPIManager::instance()->rank() ), sforest, true ), xBlockSize, yBlockSize, zBlockSize );
+ auto sbf = make_shared< StructuredBlockForest >(
+ make_shared< BlockForest >(uint_c(MPIManager::instance()->rank()), sforest, true), xBlockSize, yBlockSize,
+ zBlockSize);
- auto originalFieldId = field::addToStorage< FieldType >( sbf, "OriginalField" );
- auto readFieldId = field::addToStorage< FieldType >( sbf, "ReadField" );
+ auto originalFieldId = field::addToStorage< FieldType >(sbf, "OriginalField");
+ auto readFieldId = field::addToStorage< FieldType >(sbf, "ReadField");
- math::seedRandomGenerator( numeric_cast<std::mt19937::result_type>( MPIManager::instance()->rank() ) );
+ math::seedRandomGenerator(numeric_cast< std::mt19937::result_type >(MPIManager::instance()->rank()));
- for( auto it = sbf->begin(); it != sbf->end(); ++it )
+ for (auto it = sbf->begin(); it != sbf->end(); ++it)
{
- auto field = it->getData< FieldType >( originalFieldId );
+ auto field = it->getData< FieldType >(originalFieldId);
- for( auto dataIt = field->begin(); dataIt != field->end(); ++dataIt )
+ for (auto dataIt = field->begin(); dataIt != field->end(); ++dataIt)
*dataIt = math::realRandom< FieldType::value_type >();
}
@@ -127,34 +121,50 @@ int main( int argc, char* argv[] )
WALBERLA_MPI_BARRIER();
timer.start();
- field::writeToFile<FieldType>( "mpiFile.wlb", sbf->getBlockStorage(), originalFieldId );
+ field::writeToFile< FieldType >("mpiFile.wlb", sbf->getBlockStorage(), originalFieldId);
WALBERLA_MPI_BARRIER();
timer.end();
- WALBERLA_LOG_INFO_ON_ROOT( "Writing took " << timer.last() << "s" );
+ WALBERLA_LOG_INFO_ON_ROOT("Writing took " << timer.last() << "s");
WALBERLA_MPI_BARRIER();
timer.start();
- field::readFromFile<FieldType>( "mpiFile.wlb", sbf->getBlockStorage(), readFieldId );
+ field::readFromFile< FieldType >("mpiFile.wlb", sbf->getBlockStorage(), readFieldId);
WALBERLA_MPI_BARRIER();
timer.end();
- WALBERLA_LOG_INFO_ON_ROOT( "Reading took " << timer.last() << "s" );
+ WALBERLA_LOG_INFO_ON_ROOT("Reading took " << timer.last() << "s");
- for( auto it = sbf->begin(); it != sbf->end(); ++it )
+ for (auto it = sbf->begin(); it != sbf->end(); ++it)
{
- auto originalField = it->getData< FieldType >( originalFieldId );
- auto readField = it->getData< FieldType >( readFieldId );
+ auto originalField = it->getData< FieldType >(originalFieldId);
+ auto readField = it->getData< FieldType >(readFieldId);
auto readIt = readField->begin();
- for( auto origIt = originalField->begin(); origIt != originalField->end(); ++origIt, ++readIt )
- WALBERLA_CHECK_IDENTICAL( *origIt, *readIt );
+ for (auto origIt = originalField->begin(); origIt != originalField->end(); ++origIt, ++readIt)
+ WALBERLA_CHECK_IDENTICAL(*origIt, *readIt);
}
-
- return EXIT_SUCCESS;
}
-}
+} // namespace mpi_file_io_test
-int main( int argc, char* argv[] )
+int main(int argc, char* argv[])
{
- return mpi_file_io_test::main( argc, argv );
+ walberla::debug::enterTestMode();
+
+ walberla::mpi::Environment mpiEnv(argc, argv);
+
+ // test with MPI_WORLD_COMM
+ walberla::MPIManager::instance()->useWorldComm();
+ mpi_file_io_test::test(argc, argv);
+
+ // test with Cartesian MPI communicator
+ // this is tested additionally because some versions of OpenMPI are known to produce segmentation faults when using
+ // MPI-IO with a 3D Cartesian MPI communicator; for those OpenMPI versions, serial I/O is used instead
+ if (walberla::MPIManager::instance()->numProcesses() == 16)
+ {
+ walberla::MPIManager::instance()->resetMPI();
+
+ walberla::MPIManager::instance()->createCartesianComm(walberla::uint_c(4), walberla::uint_c(2),
+ walberla::uint_c(2), false, false, false);
+ mpi_file_io_test::test(argc, argv);
+ }
}