From 2a9e49bf91ef3064b40a49cd0c69bd11fa3897b8 Mon Sep 17 00:00:00 2001
From: Samuel Kemmler <samuel.kemmler@fau.de>
Date: Thu, 22 Jun 2023 12:47:51 +0200
Subject: [PATCH] Check for cuda aware mpi
---
src/gpu/communication/NonUniformGPUScheme.h | 12 ++++++++++++
src/gpu/communication/UniformGPUScheme.impl.h | 18 ++++++++++++++++--
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/src/gpu/communication/NonUniformGPUScheme.h b/src/gpu/communication/NonUniformGPUScheme.h
index 28a8fd0d0..4b9576434 100644
--- a/src/gpu/communication/NonUniformGPUScheme.h
+++ b/src/gpu/communication/NonUniformGPUScheme.h
@@ -134,6 +134,12 @@ NonUniformGPUScheme< Stencil >::NonUniformGPUScheme(const weak_ptr< StructuredBl
: blockForest_(bf), sendFromGPU_(sendDirectlyFromGPU), baseTag_(tag), parallelSectionManager_(-1),
requiredBlockSelectors_(Set< SUID >::emptySet()), incompatibleBlockSelectors_(Set< SUID >::emptySet())
{
+ WALBERLA_MPI_SECTION()
+ {
+#if !(defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT)
+ WALBERLA_CHECK(!sendDirectlyFromGPU)
+#endif
+ }
init();
}
@@ -146,6 +152,12 @@ NonUniformGPUScheme< Stencil >::NonUniformGPUScheme(const weak_ptr< StructuredBl
incompatibleBlockSelectors_(incompatibleBlockSelectors), sendFromGPU_(sendDirectlyFromGPU), baseTag_(tag),
parallelSectionManager_(-1)
{
+ WALBERLA_MPI_SECTION()
+ {
+#if !(defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT)
+ WALBERLA_CHECK(!sendDirectlyFromGPU)
+#endif
+ }
init();
}
diff --git a/src/gpu/communication/UniformGPUScheme.impl.h b/src/gpu/communication/UniformGPUScheme.impl.h
index a12017cf7..28033d146 100644
--- a/src/gpu/communication/UniformGPUScheme.impl.h
+++ b/src/gpu/communication/UniformGPUScheme.impl.h
@@ -42,7 +42,14 @@ namespace communication {
parallelSectionManager_( -1 ),
requiredBlockSelectors_( Set<SUID>::emptySet() ),
incompatibleBlockSelectors_( Set<SUID>::emptySet() )
- {}
+ {
+ WALBERLA_MPI_SECTION()
+ {
+#if !(defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT)
+ WALBERLA_CHECK(!sendDirectlyFromGPU)
+#endif
+ }
+ }
template<typename Stencil>
UniformGPUScheme<Stencil>::UniformGPUScheme( weak_ptr <StructuredBlockForest> bf,
@@ -61,7 +68,14 @@ namespace communication {
parallelSectionManager_( -1 ),
requiredBlockSelectors_( requiredBlockSelectors ),
incompatibleBlockSelectors_( incompatibleBlockSelectors )
- {}
+ {
+ WALBERLA_MPI_SECTION()
+ {
+#if !(defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT)
+ WALBERLA_CHECK(!sendDirectlyFromGPU)
+#endif
+ }
+ }
template<typename Stencil>
--
GitLab