Skip to content

GitLab

Commit 2c27ccff authored by Sebastian Eibl's avatar Sebastian Eibl
Browse files

added extensive benchmark for IProbe communication

parent 630fce6d
Hide whitespace changes
Inline Side-by-side
apps/benchmarks/CMakeLists.txt
View file @ 2c27ccff
......@@ -8,6 +8,7 @@ add_subdirectory( NonUniformGrid )
add_subdirectory( MotionSingleHeavySphere )
add_subdirectory( PeriodicGranularGas )
add_subdirectory( PoiseuilleChannel )
add_subdirectory( ProbeVsExtraMessage )
add_subdirectory( SchaeferTurek )
add_subdirectory( UniformGrid )
add_subdirectory( UniformGridGPU )
apps/benchmarks/ProbeVsExtraMessage/CMakeLists.txt 0 → 100644
View file @ 2c27ccff
waLBerla_add_executable ( NAME ProbeVsExtraMessage
DEPENDS core postprocessing stencil )
apps/benchmarks/ProbeVsExtraMessage/ProbeVsExtraMessage.cpp 0 → 100644
View file @ 2c27ccff
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ProbeVsExtraMessage.h
//! \author Martin Bauer <martin.bauer@fau.de>
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//! \brief Micro Benchmark, measuring time for different variable sized communications
//
//======================================================================================================================
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/math/Vector3.h"
#include "core/mpi/BufferSystem.h"
#include "core/mpi/MPIManager.h"
#include "core/timing/TimingPool.h"
#include "postprocessing/sqlite/SQLite.h"
#include "stencil/D3Q27.h"
#include "stencil/D3Q19.h"
#include "stencil/D3Q7.h"
#include <array>
#include <iostream>
#include <sstream>
namespace walberla {
class MPIInfo
{
public:
MPIInfo( const Vector3<uint_t>& procs, const Vector3<bool>& periodicity );
int getNeighborRank(const stencil::Direction& dir);
private:
Vector3<uint_t> procs_;
Vector3<bool> periodicity_;
Vector3<int> pos_;
};
MPIInfo::MPIInfo( const Vector3<uint_t>& procs, const Vector3<bool>& periodicity )
: procs_(procs)
, periodicity_(periodicity)
{
mpi::MPIManager::instance()->createCartesianComm(procs[0], procs[1], procs[2], periodicity[0], periodicity[1], periodicity[2]);
mpi::MPIManager::instance()->cartesianCoord(pos_.data());
}
int MPIInfo::getNeighborRank( const stencil::Direction& dir )
{
auto neighborCoord = pos_;
neighborCoord[0] += stencil::cx[dir];
neighborCoord[1] += stencil::cy[dir];
neighborCoord[2] += stencil::cz[dir];
if (neighborCoord[0] < 0) return -1;
if (neighborCoord[1] < 0) return -1;
if (neighborCoord[2] < 0) return -1;
if (neighborCoord[0] >= int_c(procs_[0])) return -1;
if (neighborCoord[1] >= int_c(procs_[1])) return -1;
if (neighborCoord[2] >= int_c(procs_[2])) return -1;
return mpi::MPIManager::instance()->cartesianRank(uint_c(neighborCoord[0]), uint_c(neighborCoord[1]), uint_c(neighborCoord[2]));
}
template <typename Stencil>
void communicate( MPIInfo& mpiInfo,
const uint_t iterations,
const uint_t messageSize,
const bool iProbe,
WcTimingPool& tp)
{
std::vector<char> sendBuf(messageSize);
std::vector<char> recvBuf(messageSize);
WcTimer& timer = tp[iProbe ? "IProbe" : "twoMessage"];
mpi::BufferSystem bs( mpi::MPIManager::instance()->comm() );
bs.useIProbe(iProbe);
for( uint_t i =0; i < iterations; ++i )
{
timer.start();
for (auto dirIt = Stencil::beginNoCenter(); dirIt != Stencil::end(); ++dirIt)
{
auto recvRank = mpiInfo.getNeighborRank( *dirIt );
if (recvRank == -1) continue;
bs.sendBuffer(recvRank) << sendBuf;
WALBERLA_ASSERT_EQUAL(bs.sendBuffer(recvRank).size(), messageSize + sizeof(size_t));
}
bs.setReceiverInfoFromSendBufferState(false, true);
bs.sendAll();
for( auto it = bs.begin(); it != bs.end(); ++it )
{
WALBERLA_ASSERT_EQUAL(it.buffer().size(), messageSize + sizeof(size_t));
it.buffer() >> recvBuf;
WALBERLA_ASSERT_EQUAL(recvBuf.size(), messageSize);
WALBERLA_ASSERT(it.buffer().isEmpty());
}
timer.end();
}
}
int main( int argc, char ** argv )
{
mpi::Environment mpiEnv( argc, argv );
if ( argc != 10 )
{
WALBERLA_ROOT_SECTION()
{
std::cout << "Usage ./probeVsExtraMessage x y z px py pz iterations messageSize stencil " << std::endl;
std::cout << std::endl;
std::cout << "x: number of processes in x direction" << std::endl;
std::cout << "y: number of processes in y direction" << std::endl;
std::cout << "z: number of processes in z direction" << std::endl;
std::cout << "px: periodic in x direction?" << std::endl;
std::cout << "py: periodic in y direction?" << std::endl;
std::cout << "pz: periodic in z direction?" << std::endl;
std::cout << "iterations: number of communications per case" << std::endl;
std::cout << "messageSize: size of the SendBuffer in bytes" << std::endl;
std::cout << "stencil: communication stencil (D3Q27, D3Q19, D3Q7)" << std::endl;
}
return EXIT_FAILURE;
}
Vector3<uint_t> procs;
procs[0] = std::stoul(argv[1]);
procs[1] = std::stoul(argv[2]);
procs[2] = std::stoul(argv[3]);
WALBERLA_CHECK_EQUAL(procs[0]*procs[1]*procs[2], mpi::MPIManager::instance()->numProcesses());
Vector3<bool> periodicity;
periodicity[0] = std::stoul(argv[4]);
periodicity[1] = std::stoul(argv[5]);
periodicity[2] = std::stoul(argv[6]);
uint_t iterations = std::stoul(argv[7]);
uint_t messageSize = std::stoul(argv[8]);
std::string stencil = argv[9];
WALBERLA_CHECK_EQUAL(stencil, "D3Q27", "only D3Q27 is supported!");
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(procs);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(periodicity);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(iterations);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(messageSize);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(stencil);
MPIInfo mpiInfo(procs, periodicity);
WcTimingPool tp;
WALBERLA_MPI_BARRIER();
if (stencil == "D3Q27")
{
communicate<stencil::D3Q27>(mpiInfo, iterations, messageSize, false, tp);
communicate<stencil::D3Q27>(mpiInfo, iterations, messageSize, true, tp);
} else if (stencil == "D3Q19")
{
communicate<stencil::D3Q19>(mpiInfo, iterations, messageSize, false, tp);
communicate<stencil::D3Q19>(mpiInfo, iterations, messageSize, true, tp);
} else if (stencil == "D3Q7")
{
communicate<stencil::D3Q7>(mpiInfo, iterations, messageSize, false, tp);
communicate<stencil::D3Q7>(mpiInfo, iterations, messageSize, true, tp);
} else
{
WALBERLA_ABORT("stencil not supported: " << stencil);
}
WALBERLA_LOG_INFO_ON_ROOT(tp);
WALBERLA_ROOT_SECTION()
{
std::map< std::string, walberla::int64_t > integerProperties;
std::map< std::string, double > realProperties;
std::map< std::string, std::string > stringProperties;
integerProperties["procs_x"] = int64_c(procs[0]);
integerProperties["procs_y"] = int64_c(procs[1]);
integerProperties["procs_z"] = int64_c(procs[2]);
integerProperties["priodicity_x"] = int64_c(periodicity[0]);
integerProperties["priodicity_y"] = int64_c(periodicity[1]);
integerProperties["priodicity_z"] = int64_c(periodicity[2]);
integerProperties["iterations"] = int64_c(iterations);
integerProperties["messageSize"] = int64_c(messageSize);
stringProperties["stencil"] = stencil;
auto runId = postprocessing::storeRunInSqliteDB( "ProbeVsTwoMessages.sqlite", integerProperties, stringProperties, realProperties );
postprocessing::storeTimingPoolInSqliteDB( "ProbeVsTwoMessages.sqlite", runId, tp, "Timings" );
}
return 0;
}
} // namespace walberla
int main( int argc, char* argv[] )
{
return walberla::main( argc, argv );
}
tests/core/CMakeLists.txt
View file @ 2c27ccff
......@@ -140,9 +140,6 @@ waLBerla_execute_test( NAME SetReductionTest27 COMMAND $<TARGET_FILE:SetReductio
waLBerla_compile_test( FILES mpi/ProbeVsExtraMessage.cpp DEPENDS postprocessing)
##############
# selectable #
##############
......
tests/core/mpi/ProbeVsExtraMessage.cpp deleted 100644 → 0
View file @ 630fce6d
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ProbeVsExtraMessage.h
//! \author Martin Bauer <martin.bauer@fau.de>
//! \brief Micro Benchmark, measuring time for different variable sized communications
//
//======================================================================================================================
#include "core/debug/TestSubsystem.h"
#include "core/Environment.h"
#include "core/mpi/MPIManager.h"
#include "core/math/Random.h"
#include "core/timing/TimingPool.h"
#include "postprocessing/sqlite/SQLite.h"
#include <iostream>
#include <sstream>
namespace walberla {
int getProcToReceiveFrom()
{
auto mpi = MPIManager::instance();
int res = mpi->worldRank() -1;
if ( res < 0 )
res = mpi->numProcesses() -1;
return res;
}
int getProcToSendTo()
{
auto mpi = MPIManager::instance();
int res = mpi->worldRank() +1;
if ( res == mpi->numProcesses() )
res = 0;
return res;
}
int getRandomMessageSize( uint_t maxMessageSize )
{
return int_c( math::intRandom( maxMessageSize / 3, maxMessageSize-1 ) );
}
void iprobeVersion( uint_t iterations, uint_t maxMessageSize, WcTimingPool & timingPool )
{
char * sendBuf = new char[ maxMessageSize ];
char * recvBuf = new char[ maxMessageSize ];
auto & timer = timingPool["iprobe"];
for( uint_t i =0; i < iterations; ++i )
{
int messageSize = getRandomMessageSize( maxMessageSize );
timer.start();
MPI_Request sendRequest;
MPI_Isend( sendBuf, messageSize, MPI_BYTE, getProcToSendTo(), 0, MPI_COMM_WORLD, &sendRequest );
MPI_Status probeStatus;
MPI_Probe( getProcToReceiveFrom(), 0, MPI_COMM_WORLD, &probeStatus );
int count = 0;
MPI_Get_count( &probeStatus, MPI_BYTE, &count );
MPI_Recv( recvBuf, count, MPI_BYTE, getProcToReceiveFrom(), 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
MPI_Waitall( 1, &sendRequest, MPI_STATUSES_IGNORE );
timer.end();
}
delete [] sendBuf;
delete [] recvBuf;
}
void twoMessageVersion( uint_t iterations, uint_t maxMessageSize, WcTimingPool & timingPool )
{
char * sendBuf = new char[ maxMessageSize ];
char * recvBuf = new char[ maxMessageSize ];
auto & timer = timingPool["twoMessages"];
int recvSize;
MPI_Request sendRequests[2];
const int TAG_SIZE_MSG = 1;
const int TAG_CONTENT_MSG = 0;
for( uint_t i =0; i < iterations; ++i )
{
int sendSize = getRandomMessageSize( maxMessageSize );
timer.start();
MPI_Request recvSizeMsgRequest;
MPI_Irecv( &recvSize, 1 , MPI_INT, getProcToReceiveFrom(), TAG_SIZE_MSG, MPI_COMM_WORLD, &recvSizeMsgRequest );
MPI_Isend( &sendSize, 1 , MPI_INT, getProcToSendTo(), TAG_SIZE_MSG, MPI_COMM_WORLD, &sendRequests[0] );
MPI_Isend( sendBuf , sendSize, MPI_BYTE, getProcToSendTo(), TAG_CONTENT_MSG, MPI_COMM_WORLD, &sendRequests[1] );
// wait for size message to arrive
MPI_Waitall( 1, &recvSizeMsgRequest, MPI_STATUSES_IGNORE );
// receive content
MPI_Recv( recvBuf, recvSize, MPI_BYTE, getProcToReceiveFrom(), TAG_CONTENT_MSG, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
// wait for sends to finish
MPI_Waitall( 2, sendRequests, MPI_STATUSES_IGNORE );
timer.end();
}
delete [] sendBuf;
delete [] recvBuf;
}
void maxMessageSizeVersion( uint_t iterations, uint_t maxMessageSize, WcTimingPool & timingPool )
{
char * sendBuf = new char[ maxMessageSize ];
char * recvBuf = new char[ maxMessageSize ];
auto & timer = timingPool["maxMessageSizeKnown"];
for( uint_t i =0; i < iterations; ++i )
{
int messageSize = getRandomMessageSize( maxMessageSize );
timer.start();
MPI_Request sendRequest;
MPI_Request recvRequest;
MPI_Irecv( recvBuf, int_c(maxMessageSize), MPI_BYTE, getProcToReceiveFrom(), 0, MPI_COMM_WORLD, &recvRequest );
MPI_Isend( sendBuf, messageSize, MPI_BYTE, getProcToSendTo(), 0, MPI_COMM_WORLD, &sendRequest );
MPI_Status status;
MPI_Waitall( 1, &recvRequest, &status );
int count = 0;
MPI_Get_count( &status, MPI_BYTE, &count );
MPI_Waitall( 1, &sendRequest, MPI_STATUSES_IGNORE );
timer.end();
}
delete [] sendBuf;
delete [] recvBuf;
}
int main( int argc, char ** argv )
{
debug::enterTestMode();
mpi::Environment mpiEnv( argc, argv );
MPIManager::instance()->useWorldComm();
using namespace std;
WALBERLA_ROOT_SECTION() {
if ( argc != 3 && argc != 4 ) {
cerr << "Wrong number of arguments " << endl;
cerr << "Usage ./probeVsExtraMessage <iterations> <messageSize> " << endl;
}
}
uint_t iterations;
uint_t messageSize;
std::string arg1( argv[1] );
std::string arg2( argv[2] );
std::stringstream streamIterations ( arg1 );
std::stringstream streamMessageSize ( arg2 );
streamIterations >> iterations;
streamMessageSize >> messageSize;
WcTimingPool tp;
iprobeVersion ( iterations, messageSize, tp );
twoMessageVersion ( iterations, messageSize, tp );
maxMessageSizeVersion ( iterations, messageSize, tp );
WALBERLA_ROOT_SECTION() {
cout << tp << endl;
}
if( argc == 4) {
const auto reducedTimeloopTiming = tp.getReduced();
WALBERLA_ROOT_SECTION() {
std::string dbFile( argv[3] );
std::map<std::string, walberla::int64_t> integerProperties;
integerProperties["iterations"] = int64_c(iterations);
integerProperties["messageSize"] = int64_c(messageSize);
integerProperties["processes"] = int64_c(mpi::MPIManager::instance()->numProcesses());
postprocessing::SQLiteDB db( dbFile );
auto runid = db.storeRun( integerProperties, std::map<std::string, std::string>(), std::map<string, double>());
db.storeTimingPool( runid, *reducedTimeloopTiming, "timings" );
}
}
return 0;
}
} // namespace walberla
int main( int argc, char* argv[] )
{
return walberla::main( argc, argv );
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment