Commit 547ec376 authored by Christoph Schwarzmeier's avatar Christoph Schwarzmeier Committed by Michael Kuron
Browse files

Use serial I/O instead of MPI-IO for certain versions of OpenMPI

parent f15fa5e4
......@@ -39,7 +39,7 @@ CMakeLists.txt.user.*
/bin/
/lib/
*.a
/build
/build*
# Logfiles
......
......@@ -9,7 +9,7 @@ stages:
- test
- deploy
###############################################################################
## ##
## Build templates ##
......@@ -1657,8 +1657,8 @@ coverage:
## Windows Builds ##
## ##
###############################################################################
.win_build_template: &win_build_definition
tags:
- win
......@@ -1687,7 +1687,7 @@ msvc-14.1_Hybrid_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_Hybrid_SP_Dbg:
<<: *win_build_definition
variables:
......@@ -1700,7 +1700,7 @@ msvc-14.1_Hybrid_SP_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_Hybrid:
<<: *win_build_definition
variables:
......@@ -1713,7 +1713,7 @@ msvc-14.1_Hybrid:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_Serial_Dbg:
<<: *win_build_definition
variables:
......@@ -1726,7 +1726,7 @@ msvc-14.1_Serial_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_Serial:
<<: *win_build_definition
variables:
......@@ -1739,7 +1739,7 @@ msvc-14.1_Serial:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_MpiOnly_Dbg:
<<: *win_build_definition
variables:
......@@ -1752,7 +1752,7 @@ msvc-14.1_MpiOnly_Dbg:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.1_MpiOnly:
<<: *win_build_definition
variables:
......@@ -1778,7 +1778,7 @@ msvc-14.2_Hybrid_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
msvc-14.2_Hybrid_SP_Dbg:
<<: *win_build_definition
variables:
......@@ -1791,7 +1791,7 @@ msvc-14.2_Hybrid_SP_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
msvc-14.2_Hybrid:
<<: *win_build_definition
variables:
......@@ -1804,7 +1804,7 @@ msvc-14.2_Hybrid:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
msvc-14.2_Serial_Dbg:
<<: *win_build_definition
variables:
......@@ -1817,7 +1817,7 @@ msvc-14.2_Serial_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
msvc-14.2_Serial:
<<: *win_build_definition
variables:
......@@ -1830,7 +1830,7 @@ msvc-14.2_Serial:
only:
variables:
- $ENABLE_NIGHTLY_BUILDS
msvc-14.2_MpiOnly_Dbg:
<<: *win_build_definition
variables:
......@@ -1843,7 +1843,7 @@ msvc-14.2_MpiOnly_Dbg:
except:
variables:
- $DISABLE_PER_COMMIT_BUILDS
msvc-14.2_MpiOnly:
<<: *win_build_definition
variables:
......@@ -1949,7 +1949,7 @@ mac_MpiOnly:
dependencies: []
when: manual
only:
- master@walberla/walberla
- master@walberla/walberla
- tags@walberla/walberla
conda-py36-win:
......@@ -1958,7 +1958,7 @@ conda-py36-win:
- win
script:
- conda build --python=3.6 --user=lssfau utilities\\conda\\walberla
conda-py37-win:
<<: *conda_deploy_definition
tags:
......@@ -1975,7 +1975,7 @@ conda-py37-linux:
- apt-get update
- apt-get install -y build-essential
- conda build --python=3.7 --user=lssfau utilities/conda/walberla
conda-py36-linux:
<<: *conda_deploy_definition
tags:
......
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
......@@ -269,31 +269,20 @@ void createSetupBlockForest( blockforest::SetupBlockForest & sforest, const Conf
WALBERLA_MPI_SECTION()
{
if ( MPIManager::instance()->isCartesianCommValid() )
{
MPIManager::instance()->createCartesianComm(numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, false, false, false);
MPIManager::instance()->createCartesianComm(numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, false, false, false);
processIdMap = make_shared<std::vector<uint_t> >(numberOfXProcesses * numberOfYProcesses * numberOfZProcesses);
processIdMap = make_shared<std::vector<uint_t> >(numberOfXProcesses * numberOfYProcesses * numberOfZProcesses);
for (uint_t z = 0; z != numberOfZProcesses; ++z) {
for (uint_t y = 0; y != numberOfYProcesses; ++y) {
for (uint_t x = 0; x != numberOfXProcesses; ++x)
{
(*processIdMap)[z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x] =
uint_c(MPIManager::instance()->cartesianRank(x, y, z));
}
for (uint_t z = 0; z != numberOfZProcesses; ++z) {
for (uint_t y = 0; y != numberOfYProcesses; ++y) {
for (uint_t x = 0; x != numberOfXProcesses; ++x)
{
(*processIdMap)[z * numberOfXProcesses * numberOfYProcesses + y * numberOfXProcesses + x] =
uint_c(MPIManager::instance()->cartesianRank(x, y, z));
}
}
}
else {
WALBERLA_LOG_WARNING_ON_ROOT( "Your version of OpenMPI contains a bug. See waLBerla issue #73 for more "
"information. As a workaround, MPI_COMM_WORLD instead of a "
"Cartesian MPI communicator is used." );
MPIManager::instance()->useWorldComm();
}
}
else
MPIManager::instance()->useWorldComm();
sforest.balanceLoad( blockforest::CartesianDistribution( numberOfXProcesses, numberOfYProcesses, numberOfZProcesses, processIdMap.get() ),
numberOfXProcesses * numberOfYProcesses * numberOfZProcesses );
......@@ -615,7 +604,7 @@ struct AddLB< LatticeModel_T, typename std::enable_if< std::is_same< typename La
std::function< void () > commFunction;
if( directComm )
{
if( fullComm )
{
blockforest::communication::UniformDirectScheme< stencil::D3Q27 > comm( blocks );
......
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
......@@ -672,7 +672,7 @@ void BlockForest::getAABB( AABB& aabb, const IBlockID& id ) const {
}
else {
const Block* const block = getBlock( id );
if( block == nullptr )
if( block == nullptr )
{
const BlockID& bid = *static_cast< const BlockID* >( &id );
aabb = getAABBFromBlockId( bid );
......@@ -906,7 +906,7 @@ void BlockForest::refresh()
WALBERLA_ASSERT_NOT_NULLPTR( block->second );
block->second->setTargetLevel( block->second->getLevel() );
}
if( recalculateBlockLevelsInRefresh_ )
{
WALBERLA_LOG_PROGRESS( "BlockForest refresh: determining new block levels" );
......@@ -929,22 +929,22 @@ void BlockForest::refresh()
if( alwaysRebalanceInRefresh_ && refreshPhantomBlockMigrationPreparationFunction_ )
rebalanceAndRedistribute = true;
while( rebalanceAndRedistribute )
{
// create phantom block forest
WALBERLA_LOG_PROGRESS( "BlockForest refresh: creating phantom forest/blocks" );
refreshTiming_[ "phantom forest creation" ].start();
PhantomBlockForest phantomForest( *this );
phantomForest.initialize( refreshBlockStateDeterminationFunction_, allowChangingDepth_ );
refreshTiming_[ "phantom forest creation" ].end();
// move phantom blocks between processes (= dynamic load balancing)
WALBERLA_MPI_SECTION()
{
refreshTiming_[ "phantom block redistribution (= load balancing)" ].start();
......@@ -966,15 +966,15 @@ void BlockForest::refresh()
++iteration;
}
phantomBlockMigrationIterations_ = iteration;
WALBERLA_LOG_PROGRESS( "BlockForest refresh: phantom block redistribution/load balancing finished after " << phantomBlockMigrationIterations_ << " iterations" );
}
refreshTiming_[ "phantom block redistribution (= load balancing)" ].end();
}
// update block forest: transfer block data (includes migrating, deleting and creating blocks), adapt depth, adapt process neighborhood, etc.
bool performUpdate( true );
if( checkForEarlyOutAfterLoadBalancing_ )
{
......@@ -1010,23 +1010,23 @@ void BlockForest::refresh()
{
WALBERLA_ASSERT( recalculateBlockLevelsInRefresh_ );
WALBERLA_ASSERT( allowMultipleRefreshCycles_ );
refreshTiming_[ "block level determination" ].start();
for( auto block = blocks_.begin(); block != blocks_.end(); ++block )
{
WALBERLA_ASSERT_NOT_NULLPTR( block->second );
block->second->setTargetLevel( block->second->getLevel() );
}
WALBERLA_LOG_PROGRESS( "BlockForest refresh: determining new block levels (again -> more refresh cycles required!)" );
bool rerun( true );
while( rerun )
{
rebalanceAndRedistribute = determineBlockTargetLevels( additionalRefreshCycleRequired, rerun );
}
if( !rebalanceAndRedistribute )
WALBERLA_LOG_PROGRESS( "BlockForest refresh: block levels do not change" );
......@@ -1035,7 +1035,7 @@ void BlockForest::refresh()
else
rebalanceAndRedistribute = false;
}
WALBERLA_LOG_PROGRESS( "BlockForest refresh: finished!" );
}
......@@ -1267,7 +1267,7 @@ void BlockForest::restoreSnapshot( const SnapshotRestorenFunction & processMappi
++iteration;
}
phantomBlockMigrationIterations_ = iteration;
WALBERLA_LOG_PROGRESS( "BlockForest refresh: phantom block redistribution/load balancing finished after " << phantomBlockMigrationIterations_ << " iterations" );
}
......@@ -1651,7 +1651,7 @@ bool BlockForest::determineBlockTargetLevels( bool & additionalRefreshCycleRequi
const uint_t level = block->getLevel();
uint_t minTargetLevel = minTargetLevels[id];
if( allowMultipleRefreshCycles_ )
{
if( minTargetLevel > ( level + uint_t(1) ) )
......@@ -1683,7 +1683,7 @@ bool BlockForest::determineBlockTargetLevels( bool & additionalRefreshCycleRequi
}
}
}
if( allowMultipleRefreshCycles_ )
mpi::allReduceInplace( additionalRefreshCycleRequired, mpi::LOGICAL_OR );
......@@ -2037,7 +2037,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
//////////////////////////////////////
std::map< uint_t, uint_t > numberOfBlocksToRecv; // does not include local transfers
const auto & phantomBlocks = phantomForest.getBlockMap();
for( auto phantom = phantomBlocks.begin(); phantom != phantomBlocks.end(); ++phantom )
{
......@@ -2061,15 +2061,15 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
else { WALBERLA_ASSERT( blocks_.find( pBlock->getId() ) != blocks_.end() ); }
#endif
}
std::map< uint_t, std::vector< uint_t > > recvBufferSizes; // does not include local transfers
for( auto it = numberOfBlocksToRecv.begin(); it != numberOfBlocksToRecv.end(); ++it )
{
WALBERLA_ASSERT_GREATER( it->second, uint_t(0) );
recvBufferSizes[ it->first ].resize( it->second );
}
std::vector< MPI_Request > recvBufferSizesRequests( numberOfBlocksToRecv.size() ); // do not resize this vector!
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - schedule receives for buffer sizes" );
......@@ -2081,15 +2081,15 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
int_c( it->first ), 0, MPIManager::instance()->comm(), &(recvBufferSizesRequests[i]) );
++i;
}
///////////////////////////////
// FETCH TARGET BLOCK STATES //
///////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - fetch target block states" );
std::set< mpi::MPIRank > ranksToRecvFrom;
for( auto it = blocks_.begin(); it != blocks_.end(); ++it )
{
auto & block = it->second;
......@@ -2102,12 +2102,12 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
ranksToRecvFrom.insert( numeric_cast< mpi::MPIRank >(*p) );
}
}
mpi::BufferSystem bufferSystem( MPIManager::instance()->comm(), 1184 ); // blockforest = 98 108 111 99 107 102 111 114 101 115 116 + 2
bufferSystem.setReceiverInfo( ranksToRecvFrom, true ); // ATTENTION: true = the size of a message from A to B varies
bufferSystem.setReceiverInfo( ranksToRecvFrom, true ); // ATTENTION: true = the size of a message from A to B varies
std::map< uint_t, std::map< BlockID, Set<SUID> > > blockStates;
for( auto phantom = phantomBlocks.begin(); phantom != phantomBlocks.end(); ++phantom )
{
auto & pBlock = phantom->second;
......@@ -2126,9 +2126,9 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
}
bufferSystem.sendAll();
std::map< BlockID, Set<SUID> > targetBlockStates;
for( auto recvIt = bufferSystem.begin(); recvIt != bufferSystem.end(); ++recvIt )
{
WALBERLA_ASSERT_UNEQUAL( uint_c( recvIt.rank() ), process_ );
......@@ -2147,9 +2147,9 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
{
WALBERLA_ASSERT( targetBlockStates.find( state->first ) == targetBlockStates.end() );
targetBlockStates[ state->first ] = state->second;
}
}
}
///////////////
// PACK DATA //
///////////////
......@@ -2187,7 +2187,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
auto & id = block->getId();
auto & targetProcesses = block->getTargetProcess();
std::vector< Set<SUID> > targetState;
// header = sender block ID + receiver block ID
......@@ -2204,7 +2204,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[0] ].push_back( &(buffers[0]) );
else
localBlocks.push_back( &(buffers[0]) );
WALBERLA_ASSERT( targetBlockStates.find(id) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[id] );
}
......@@ -2221,7 +2221,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[0] ].push_back( &(buffers[0]) );
else
localBlocks.push_back( &(buffers[0]) );
WALBERLA_ASSERT( targetBlockStates.find(fid) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[fid] );
}
......@@ -2241,7 +2241,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
processesToSendTo[ targetProcesses[c] ].push_back( &(buffers[c]) );
else
localBlocks.push_back( &(buffers[c]) );
WALBERLA_ASSERT( targetBlockStates.find(cid) != targetBlockStates.end() );
targetState.push_back( targetBlockStates[cid] );
}
......@@ -2298,7 +2298,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
WALBERLA_ASSERT_EQUAL( buffers.size(), uint_t(8) );
WALBERLA_ASSERT_EQUAL( targetProcesses.size(), uint_t(8) );
WALBERLA_ASSERT_EQUAL( targetState.size(), uint_t(8) );
for( auto dataItem = blockDataItem_.begin(); dataItem != blockDataItem_.end(); ++dataItem )
{
for( uint_t c = uint_t(0); c != uint_t(8); ++c )
......@@ -2344,12 +2344,12 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
for( auto it = processesToSendTo.begin(); it != processesToSendTo.end(); ++it )
{
WALBERLA_ASSERT( sendBufferSizes.find( it->first ) == sendBufferSizes.end() );
auto & sizes = sendBufferSizes[ it->first ];
WALBERLA_ASSERT( sizes.empty() );
for( auto buffer = it->second.begin(); buffer != it->second.end(); ++buffer )
sizes.push_back( (*buffer)->size() );
blockDataSendRequests[ it->first ].resize( it->second.size() ); // do not resize this vector after this point!
}
......@@ -2378,7 +2378,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
++i;
}
///////////////////
// CREATE BLOCKS //
///////////////////
......@@ -2399,7 +2399,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
blocks_[ pBlock->getId() ]->resetNeighborhood( *pBlock );
}
}
// adapt depth
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - adapting block structure (use current state of phantom forest as reference)" );
......@@ -2421,21 +2421,21 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
//////////////////////////////////////
// WAIT FOR RECV's FOR BUFFER SIZES //
//////////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for buffer sizes" );
if( ! recvBufferSizesRequests.empty() )
MPI_Waitall( int_c( recvBufferSizesRequests.size() ), &(recvBufferSizesRequests[0]), MPI_STATUSES_IGNORE );
////////////////////////////////////
// SCHEDULE RECV's FOR BLOCK DATA //
////////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - schedule block data receive operations" );
std::map< uint_t, std::vector< mpi::RecvBuffer > > recvBlockData;
std::map< uint_t, std::vector< MPI_Request > > blockDataRecvRequests;
for( auto it = recvBufferSizes.begin(); it != recvBufferSizes.end(); ++it )
{
WALBERLA_ASSERT_UNEQUAL( it->first, process_ );
......@@ -2446,7 +2446,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
requests.resize( sizes.size() );
}
auto & recvLocalBlocks = recvBlockData[ process_ ];
for( auto it = recvBufferSizes.begin(); it != recvBufferSizes.end(); ++it )
{
WALBERLA_ASSERT_UNEQUAL( it->first, process_ );
......@@ -2462,11 +2462,11 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
int_c( it->first ), int_c(i)+1, MPIManager::instance()->comm(), &(requests[i]) );
}
}
///////////////////////////
// COPY LOCAL BLOCK DATA //
///////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - perform local data transfer" );
for( auto buffer = localBlocks.begin(); buffer != localBlocks.end(); ++buffer )
......@@ -2475,7 +2475,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////////
// WAIT FOR RECV's FOR BLOCK DATA //
////////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for block data to be received" );
for( auto it = blockDataRecvRequests.begin(); it != blockDataRecvRequests.end(); ++it )
......@@ -2483,16 +2483,16 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
auto & requests = it->second;
MPI_Waitall( int_c( requests.size() ), &(requests[0]), MPI_STATUSES_IGNORE );
}
//////////////////////////////////
// WAIT FOR ALL SENDS TO FINISH //
//////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - wait for block data sends to complete" );
if( ! sendBufferSizesRequests.empty() )
MPI_Waitall( int_c( sendBufferSizesRequests.size() ), &(sendBufferSizesRequests[0]), MPI_STATUSES_IGNORE );
for( auto it = blockDataSendRequests.begin(); it != blockDataSendRequests.end(); ++it )
{
auto & requests = it->second;
......@@ -2502,7 +2502,7 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////////////
// CLEAR SEND BUFFERS (= FREE MEMORY) //
////////////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - clear send buffers (= free memory)" );
for( auto it = blocksToPack.begin(); it != blocksToPack.end(); ++it )
......@@ -2515,11 +2515,11 @@ void BlockForest::update( PhantomBlockForest & phantomForest )
////////////////////////////////
// PREPARE DATA FOR UNPACKING //
////////////////////////////////
WALBERLA_LOG_PROGRESS( "BlockForest refresh: - prepare received data for unpacking" );
std::map< Block *, std::vector< std::pair< Set<SUID>, mpi::RecvBuffer * > > > blocksToUnpack; // includes data that is NOT transfered via MPI but copied locally
for( auto it = recvBlockData.begin(); it != recvBlockData.end(); ++it )
{