Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Markus Holzer
waLBerla
Commits
2c27ccff
Commit
2c27ccff
authored
Apr 11, 2019
by
Sebastian Eibl
Browse files
added extensive benchmark for IProbe communication
parent
630fce6d
Changes
5
Hide whitespace changes
Inline
Side-by-side
apps/benchmarks/CMakeLists.txt
View file @
2c27ccff
...
...
@@ -8,6 +8,7 @@ add_subdirectory( NonUniformGrid )
add_subdirectory
(
MotionSingleHeavySphere
)
add_subdirectory
(
PeriodicGranularGas
)
add_subdirectory
(
PoiseuilleChannel
)
add_subdirectory
(
ProbeVsExtraMessage
)
add_subdirectory
(
SchaeferTurek
)
add_subdirectory
(
UniformGrid
)
add_subdirectory
(
UniformGridGPU
)
apps/benchmarks/ProbeVsExtraMessage/CMakeLists.txt
0 → 100644
View file @
2c27ccff
waLBerla_add_executable
(
NAME ProbeVsExtraMessage
DEPENDS core postprocessing stencil
)
apps/benchmarks/ProbeVsExtraMessage/ProbeVsExtraMessage.cpp
0 → 100644
View file @
2c27ccff
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ProbeVsExtraMessage.h
//! \author Martin Bauer <martin.bauer@fau.de>
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//! \brief Micro Benchmark, measuring time for different variable sized communications
//
//======================================================================================================================
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/math/Vector3.h"
#include "core/mpi/BufferSystem.h"
#include "core/mpi/MPIManager.h"
#include "core/timing/TimingPool.h"
#include "postprocessing/sqlite/SQLite.h"
#include "stencil/D3Q27.h"
#include "stencil/D3Q19.h"
#include "stencil/D3Q7.h"
#include <array>
#include <iostream>
#include <sstream>
namespace
walberla
{
class
MPIInfo
{
public:
MPIInfo
(
const
Vector3
<
uint_t
>&
procs
,
const
Vector3
<
bool
>&
periodicity
);
int
getNeighborRank
(
const
stencil
::
Direction
&
dir
);
private:
Vector3
<
uint_t
>
procs_
;
Vector3
<
bool
>
periodicity_
;
Vector3
<
int
>
pos_
;
};
MPIInfo
::
MPIInfo
(
const
Vector3
<
uint_t
>&
procs
,
const
Vector3
<
bool
>&
periodicity
)
:
procs_
(
procs
)
,
periodicity_
(
periodicity
)
{
mpi
::
MPIManager
::
instance
()
->
createCartesianComm
(
procs
[
0
],
procs
[
1
],
procs
[
2
],
periodicity
[
0
],
periodicity
[
1
],
periodicity
[
2
]);
mpi
::
MPIManager
::
instance
()
->
cartesianCoord
(
pos_
.
data
());
}
int
MPIInfo
::
getNeighborRank
(
const
stencil
::
Direction
&
dir
)
{
auto
neighborCoord
=
pos_
;
neighborCoord
[
0
]
+=
stencil
::
cx
[
dir
];
neighborCoord
[
1
]
+=
stencil
::
cy
[
dir
];
neighborCoord
[
2
]
+=
stencil
::
cz
[
dir
];
if
(
neighborCoord
[
0
]
<
0
)
return
-
1
;
if
(
neighborCoord
[
1
]
<
0
)
return
-
1
;
if
(
neighborCoord
[
2
]
<
0
)
return
-
1
;
if
(
neighborCoord
[
0
]
>=
int_c
(
procs_
[
0
]))
return
-
1
;
if
(
neighborCoord
[
1
]
>=
int_c
(
procs_
[
1
]))
return
-
1
;
if
(
neighborCoord
[
2
]
>=
int_c
(
procs_
[
2
]))
return
-
1
;
return
mpi
::
MPIManager
::
instance
()
->
cartesianRank
(
uint_c
(
neighborCoord
[
0
]),
uint_c
(
neighborCoord
[
1
]),
uint_c
(
neighborCoord
[
2
]));
}
template
<
typename
Stencil
>
void
communicate
(
MPIInfo
&
mpiInfo
,
const
uint_t
iterations
,
const
uint_t
messageSize
,
const
bool
iProbe
,
WcTimingPool
&
tp
)
{
std
::
vector
<
char
>
sendBuf
(
messageSize
);
std
::
vector
<
char
>
recvBuf
(
messageSize
);
WcTimer
&
timer
=
tp
[
iProbe
?
"IProbe"
:
"twoMessage"
];
mpi
::
BufferSystem
bs
(
mpi
::
MPIManager
::
instance
()
->
comm
()
);
bs
.
useIProbe
(
iProbe
);
for
(
uint_t
i
=
0
;
i
<
iterations
;
++
i
)
{
timer
.
start
();
for
(
auto
dirIt
=
Stencil
::
beginNoCenter
();
dirIt
!=
Stencil
::
end
();
++
dirIt
)
{
auto
recvRank
=
mpiInfo
.
getNeighborRank
(
*
dirIt
);
if
(
recvRank
==
-
1
)
continue
;
bs
.
sendBuffer
(
recvRank
)
<<
sendBuf
;
WALBERLA_ASSERT_EQUAL
(
bs
.
sendBuffer
(
recvRank
).
size
(),
messageSize
+
sizeof
(
size_t
));
}
bs
.
setReceiverInfoFromSendBufferState
(
false
,
true
);
bs
.
sendAll
();
for
(
auto
it
=
bs
.
begin
();
it
!=
bs
.
end
();
++
it
)
{
WALBERLA_ASSERT_EQUAL
(
it
.
buffer
().
size
(),
messageSize
+
sizeof
(
size_t
));
it
.
buffer
()
>>
recvBuf
;
WALBERLA_ASSERT_EQUAL
(
recvBuf
.
size
(),
messageSize
);
WALBERLA_ASSERT
(
it
.
buffer
().
isEmpty
());
}
timer
.
end
();
}
}
int
main
(
int
argc
,
char
**
argv
)
{
mpi
::
Environment
mpiEnv
(
argc
,
argv
);
if
(
argc
!=
10
)
{
WALBERLA_ROOT_SECTION
()
{
std
::
cout
<<
"Usage ./probeVsExtraMessage x y z px py pz iterations messageSize stencil "
<<
std
::
endl
;
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
"x: number of processes in x direction"
<<
std
::
endl
;
std
::
cout
<<
"y: number of processes in y direction"
<<
std
::
endl
;
std
::
cout
<<
"z: number of processes in z direction"
<<
std
::
endl
;
std
::
cout
<<
"px: periodic in x direction?"
<<
std
::
endl
;
std
::
cout
<<
"py: periodic in y direction?"
<<
std
::
endl
;
std
::
cout
<<
"pz: periodic in z direction?"
<<
std
::
endl
;
std
::
cout
<<
"iterations: number of communications per case"
<<
std
::
endl
;
std
::
cout
<<
"messageSize: size of the SendBuffer in bytes"
<<
std
::
endl
;
std
::
cout
<<
"stencil: communication stencil (D3Q27, D3Q19, D3Q7)"
<<
std
::
endl
;
}
return
EXIT_FAILURE
;
}
Vector3
<
uint_t
>
procs
;
procs
[
0
]
=
std
::
stoul
(
argv
[
1
]);
procs
[
1
]
=
std
::
stoul
(
argv
[
2
]);
procs
[
2
]
=
std
::
stoul
(
argv
[
3
]);
WALBERLA_CHECK_EQUAL
(
procs
[
0
]
*
procs
[
1
]
*
procs
[
2
],
mpi
::
MPIManager
::
instance
()
->
numProcesses
());
Vector3
<
bool
>
periodicity
;
periodicity
[
0
]
=
std
::
stoul
(
argv
[
4
]);
periodicity
[
1
]
=
std
::
stoul
(
argv
[
5
]);
periodicity
[
2
]
=
std
::
stoul
(
argv
[
6
]);
uint_t
iterations
=
std
::
stoul
(
argv
[
7
]);
uint_t
messageSize
=
std
::
stoul
(
argv
[
8
]);
std
::
string
stencil
=
argv
[
9
];
WALBERLA_CHECK_EQUAL
(
stencil
,
"D3Q27"
,
"only D3Q27 is supported!"
);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT
(
procs
);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT
(
periodicity
);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT
(
iterations
);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT
(
messageSize
);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT
(
stencil
);
MPIInfo
mpiInfo
(
procs
,
periodicity
);
WcTimingPool
tp
;
WALBERLA_MPI_BARRIER
();
if
(
stencil
==
"D3Q27"
)
{
communicate
<
stencil
::
D3Q27
>
(
mpiInfo
,
iterations
,
messageSize
,
false
,
tp
);
communicate
<
stencil
::
D3Q27
>
(
mpiInfo
,
iterations
,
messageSize
,
true
,
tp
);
}
else
if
(
stencil
==
"D3Q19"
)
{
communicate
<
stencil
::
D3Q19
>
(
mpiInfo
,
iterations
,
messageSize
,
false
,
tp
);
communicate
<
stencil
::
D3Q19
>
(
mpiInfo
,
iterations
,
messageSize
,
true
,
tp
);
}
else
if
(
stencil
==
"D3Q7"
)
{
communicate
<
stencil
::
D3Q7
>
(
mpiInfo
,
iterations
,
messageSize
,
false
,
tp
);
communicate
<
stencil
::
D3Q7
>
(
mpiInfo
,
iterations
,
messageSize
,
true
,
tp
);
}
else
{
WALBERLA_ABORT
(
"stencil not supported: "
<<
stencil
);
}
WALBERLA_LOG_INFO_ON_ROOT
(
tp
);
WALBERLA_ROOT_SECTION
()
{
std
::
map
<
std
::
string
,
walberla
::
int64_t
>
integerProperties
;
std
::
map
<
std
::
string
,
double
>
realProperties
;
std
::
map
<
std
::
string
,
std
::
string
>
stringProperties
;
integerProperties
[
"procs_x"
]
=
int64_c
(
procs
[
0
]);
integerProperties
[
"procs_y"
]
=
int64_c
(
procs
[
1
]);
integerProperties
[
"procs_z"
]
=
int64_c
(
procs
[
2
]);
integerProperties
[
"priodicity_x"
]
=
int64_c
(
periodicity
[
0
]);
integerProperties
[
"priodicity_y"
]
=
int64_c
(
periodicity
[
1
]);
integerProperties
[
"priodicity_z"
]
=
int64_c
(
periodicity
[
2
]);
integerProperties
[
"iterations"
]
=
int64_c
(
iterations
);
integerProperties
[
"messageSize"
]
=
int64_c
(
messageSize
);
stringProperties
[
"stencil"
]
=
stencil
;
auto
runId
=
postprocessing
::
storeRunInSqliteDB
(
"ProbeVsTwoMessages.sqlite"
,
integerProperties
,
stringProperties
,
realProperties
);
postprocessing
::
storeTimingPoolInSqliteDB
(
"ProbeVsTwoMessages.sqlite"
,
runId
,
tp
,
"Timings"
);
}
return
0
;
}
}
// namespace walberla
int
main
(
int
argc
,
char
*
argv
[]
)
{
return
walberla
::
main
(
argc
,
argv
);
}
tests/core/CMakeLists.txt
View file @
2c27ccff
...
...
@@ -140,9 +140,6 @@ waLBerla_execute_test( NAME SetReductionTest27 COMMAND $<TARGET_FILE:SetReductio
waLBerla_compile_test
(
FILES mpi/ProbeVsExtraMessage.cpp DEPENDS postprocessing
)
##############
# selectable #
##############
...
...
tests/core/mpi/ProbeVsExtraMessage.cpp
deleted
100644 → 0
View file @
630fce6d
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ProbeVsExtraMessage.h
//! \author Martin Bauer <martin.bauer@fau.de>
//! \brief Micro Benchmark, measuring time for different variable sized communications
//
//======================================================================================================================
#include "core/debug/TestSubsystem.h"
#include "core/Environment.h"
#include "core/mpi/MPIManager.h"
#include "core/math/Random.h"
#include "core/timing/TimingPool.h"
#include "postprocessing/sqlite/SQLite.h"
#include <iostream>
#include <sstream>
namespace
walberla
{
int
getProcToReceiveFrom
()
{
auto
mpi
=
MPIManager
::
instance
();
int
res
=
mpi
->
worldRank
()
-
1
;
if
(
res
<
0
)
res
=
mpi
->
numProcesses
()
-
1
;
return
res
;
}
int
getProcToSendTo
()
{
auto
mpi
=
MPIManager
::
instance
();
int
res
=
mpi
->
worldRank
()
+
1
;
if
(
res
==
mpi
->
numProcesses
()
)
res
=
0
;
return
res
;
}
int
getRandomMessageSize
(
uint_t
maxMessageSize
)
{
return
int_c
(
math
::
intRandom
(
maxMessageSize
/
3
,
maxMessageSize
-
1
)
);
}
void
iprobeVersion
(
uint_t
iterations
,
uint_t
maxMessageSize
,
WcTimingPool
&
timingPool
)
{
char
*
sendBuf
=
new
char
[
maxMessageSize
];
char
*
recvBuf
=
new
char
[
maxMessageSize
];
auto
&
timer
=
timingPool
[
"iprobe"
];
for
(
uint_t
i
=
0
;
i
<
iterations
;
++
i
)
{
int
messageSize
=
getRandomMessageSize
(
maxMessageSize
);
timer
.
start
();
MPI_Request
sendRequest
;
MPI_Isend
(
sendBuf
,
messageSize
,
MPI_BYTE
,
getProcToSendTo
(),
0
,
MPI_COMM_WORLD
,
&
sendRequest
);
MPI_Status
probeStatus
;
MPI_Probe
(
getProcToReceiveFrom
(),
0
,
MPI_COMM_WORLD
,
&
probeStatus
);
int
count
=
0
;
MPI_Get_count
(
&
probeStatus
,
MPI_BYTE
,
&
count
);
MPI_Recv
(
recvBuf
,
count
,
MPI_BYTE
,
getProcToReceiveFrom
(),
0
,
MPI_COMM_WORLD
,
MPI_STATUS_IGNORE
);
MPI_Waitall
(
1
,
&
sendRequest
,
MPI_STATUSES_IGNORE
);
timer
.
end
();
}
delete
[]
sendBuf
;
delete
[]
recvBuf
;
}
void
twoMessageVersion
(
uint_t
iterations
,
uint_t
maxMessageSize
,
WcTimingPool
&
timingPool
)
{
char
*
sendBuf
=
new
char
[
maxMessageSize
];
char
*
recvBuf
=
new
char
[
maxMessageSize
];
auto
&
timer
=
timingPool
[
"twoMessages"
];
int
recvSize
;
MPI_Request
sendRequests
[
2
];
const
int
TAG_SIZE_MSG
=
1
;
const
int
TAG_CONTENT_MSG
=
0
;
for
(
uint_t
i
=
0
;
i
<
iterations
;
++
i
)
{
int
sendSize
=
getRandomMessageSize
(
maxMessageSize
);
timer
.
start
();
MPI_Request
recvSizeMsgRequest
;
MPI_Irecv
(
&
recvSize
,
1
,
MPI_INT
,
getProcToReceiveFrom
(),
TAG_SIZE_MSG
,
MPI_COMM_WORLD
,
&
recvSizeMsgRequest
);
MPI_Isend
(
&
sendSize
,
1
,
MPI_INT
,
getProcToSendTo
(),
TAG_SIZE_MSG
,
MPI_COMM_WORLD
,
&
sendRequests
[
0
]
);
MPI_Isend
(
sendBuf
,
sendSize
,
MPI_BYTE
,
getProcToSendTo
(),
TAG_CONTENT_MSG
,
MPI_COMM_WORLD
,
&
sendRequests
[
1
]
);
// wait for size message to arrive
MPI_Waitall
(
1
,
&
recvSizeMsgRequest
,
MPI_STATUSES_IGNORE
);
// receive content
MPI_Recv
(
recvBuf
,
recvSize
,
MPI_BYTE
,
getProcToReceiveFrom
(),
TAG_CONTENT_MSG
,
MPI_COMM_WORLD
,
MPI_STATUS_IGNORE
);
// wait for sends to finish
MPI_Waitall
(
2
,
sendRequests
,
MPI_STATUSES_IGNORE
);
timer
.
end
();
}
delete
[]
sendBuf
;
delete
[]
recvBuf
;
}
void
maxMessageSizeVersion
(
uint_t
iterations
,
uint_t
maxMessageSize
,
WcTimingPool
&
timingPool
)
{
char
*
sendBuf
=
new
char
[
maxMessageSize
];
char
*
recvBuf
=
new
char
[
maxMessageSize
];
auto
&
timer
=
timingPool
[
"maxMessageSizeKnown"
];
for
(
uint_t
i
=
0
;
i
<
iterations
;
++
i
)
{
int
messageSize
=
getRandomMessageSize
(
maxMessageSize
);
timer
.
start
();
MPI_Request
sendRequest
;
MPI_Request
recvRequest
;
MPI_Irecv
(
recvBuf
,
int_c
(
maxMessageSize
),
MPI_BYTE
,
getProcToReceiveFrom
(),
0
,
MPI_COMM_WORLD
,
&
recvRequest
);
MPI_Isend
(
sendBuf
,
messageSize
,
MPI_BYTE
,
getProcToSendTo
(),
0
,
MPI_COMM_WORLD
,
&
sendRequest
);
MPI_Status
status
;
MPI_Waitall
(
1
,
&
recvRequest
,
&
status
);
int
count
=
0
;
MPI_Get_count
(
&
status
,
MPI_BYTE
,
&
count
);
MPI_Waitall
(
1
,
&
sendRequest
,
MPI_STATUSES_IGNORE
);
timer
.
end
();
}
delete
[]
sendBuf
;
delete
[]
recvBuf
;
}
int
main
(
int
argc
,
char
**
argv
)
{
debug
::
enterTestMode
();
mpi
::
Environment
mpiEnv
(
argc
,
argv
);
MPIManager
::
instance
()
->
useWorldComm
();
using
namespace
std
;
WALBERLA_ROOT_SECTION
()
{
if
(
argc
!=
3
&&
argc
!=
4
)
{
cerr
<<
"Wrong number of arguments "
<<
endl
;
cerr
<<
"Usage ./probeVsExtraMessage <iterations> <messageSize> "
<<
endl
;
}
}
uint_t
iterations
;
uint_t
messageSize
;
std
::
string
arg1
(
argv
[
1
]
);
std
::
string
arg2
(
argv
[
2
]
);
std
::
stringstream
streamIterations
(
arg1
);
std
::
stringstream
streamMessageSize
(
arg2
);
streamIterations
>>
iterations
;
streamMessageSize
>>
messageSize
;
WcTimingPool
tp
;
iprobeVersion
(
iterations
,
messageSize
,
tp
);
twoMessageVersion
(
iterations
,
messageSize
,
tp
);
maxMessageSizeVersion
(
iterations
,
messageSize
,
tp
);
WALBERLA_ROOT_SECTION
()
{
cout
<<
tp
<<
endl
;
}
if
(
argc
==
4
)
{
const
auto
reducedTimeloopTiming
=
tp
.
getReduced
();
WALBERLA_ROOT_SECTION
()
{
std
::
string
dbFile
(
argv
[
3
]
);
std
::
map
<
std
::
string
,
walberla
::
int64_t
>
integerProperties
;
integerProperties
[
"iterations"
]
=
int64_c
(
iterations
);
integerProperties
[
"messageSize"
]
=
int64_c
(
messageSize
);
integerProperties
[
"processes"
]
=
int64_c
(
mpi
::
MPIManager
::
instance
()
->
numProcesses
());
postprocessing
::
SQLiteDB
db
(
dbFile
);
auto
runid
=
db
.
storeRun
(
integerProperties
,
std
::
map
<
std
::
string
,
std
::
string
>
(),
std
::
map
<
string
,
double
>
());
db
.
storeTimingPool
(
runid
,
*
reducedTimeloopTiming
,
"timings"
);
}
}
return
0
;
}
}
// namespace walberla
int
main
(
int
argc
,
char
*
argv
[]
)
{
return
walberla
::
main
(
argc
,
argv
);
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment