Commit b7d9cc47 authored by Tobias Schruff's avatar Tobias Schruff
Browse files

Merge remote-tracking branch 'upstream'

parents d09f67dc cd7d2c52
Pipeline #8174 failed with stage
in 59 minutes and 11 seconds
###############################################################################
## ##
## Genral settings ##
## General settings ##
## ##
###############################################################################
......@@ -970,6 +970,71 @@ msvc-14.1_MpiOnly:
- triggers
###############################################################################
## ##
## macOS Builds ##
## ##
###############################################################################
.mac_build_template: &mac_build_definition
script:
- export NUM_CORES=$(system_profiler SPHardwareDataType | grep 'Total Number of Cores' | awk '{print $5}')
- export MAX_BUILD_CORES=$(( $(system_profiler SPHardwareDataType | grep 'Memory' | awk '{print $2}') / 4 ))
- "[[ $MAX_BUILD_CORES -lt $NUM_CORES ]] && export NUM_BUILD_CORES=$MAX_BUILD_CORES || export NUM_BUILD_CORES=$NUM_CORES"
- c++ --version
- cmake --version
- mpirun --version
- mkdir build
- cd build
- cmake .. -DWALBERLA_BUILD_TESTS=ON -DWALBERLA_BUILD_BENCHMARKS=ON -DWALBERLA_BUILD_TUTORIALS=ON -DWALBERLA_BUILD_TOOLS=ON -DWALBERLA_BUILD_WITH_MPI=$WALBERLA_BUILD_WITH_MPI -DWALBERLA_BUILD_WITH_PYTHON=$WALBERLA_BUILD_WITH_PYTHON -DWALBERLA_BUILD_WITH_OPENMP=$WALBERLA_BUILD_WITH_OPENMP -DWALBERLA_BUILD_WITH_CUDA=$WALBERLA_BUILD_WITH_CUDA -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DWARNING_ERROR=ON
- cmake . -LAH
- make -j $NUM_BUILD_CORES -l $NUM_CORES
- ctest -LE "$CTEST_EXCLUDE_LABELS|cuda" -C $CMAKE_BUILD_TYPE --output-on-failure -j $NUM_CORES
tags:
- mac
mac_Serial_Dbg:
<<: *mac_build_definition
variables:
CMAKE_BUILD_TYPE: "DebugOptimized"
CTEST_EXCLUDE_LABELS: "longrun"
WALBERLA_BUILD_WITH_MPI: "OFF"
WALBERLA_BUILD_WITH_OPENMP: "OFF"
WALBERLA_BUILD_WITH_PYTHON: "ON"
WALBERLA_BUILD_WITH_CUDA: "ON"
mac_Serial:
<<: *mac_build_definition
variables:
CMAKE_BUILD_TYPE: "Release"
CTEST_EXCLUDE_LABELS: "longrun"
WALBERLA_BUILD_WITH_MPI: "OFF"
WALBERLA_BUILD_WITH_OPENMP: "OFF"
WALBERLA_BUILD_WITH_PYTHON: "ON"
WALBERLA_BUILD_WITH_CUDA: "ON"
mac_MpiOnly_Dbg:
<<: *mac_build_definition
variables:
CMAKE_BUILD_TYPE: "DebugOptimized"
CTEST_EXCLUDE_LABELS: "longrun"
WALBERLA_BUILD_WITH_MPI: "ON"
WALBERLA_BUILD_WITH_OPENMP: "OFF"
WALBERLA_BUILD_WITH_PYTHON: "ON"
WALBERLA_BUILD_WITH_CUDA: "ON"
mac_MpiOnly:
<<: *mac_build_definition
variables:
CMAKE_BUILD_TYPE: "Release"
CTEST_EXCLUDE_LABELS: "longrun"
WALBERLA_BUILD_WITH_MPI: "ON"
WALBERLA_BUILD_WITH_OPENMP: "OFF"
WALBERLA_BUILD_WITH_PYTHON: "ON"
WALBERLA_BUILD_WITH_CUDA: "ON"
###############################################################################
## ##
## Deploy jobs ##
......
......@@ -360,17 +360,6 @@ if( NOT WARNING_DEPRECATED)
endif()
endif()
# Treat warnings as errors
if ( WARNING_ERROR )
if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
add_flag ( CMAKE_CXX_FLAGS "-pedantic-errors -Werror" )
elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
add_flag ( CMAKE_CXX_FLAGS "/WX" )
elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
add_flag ( CMAKE_CXX_FLAGS "-h error_on_warning" )
endif()
endif ( )
if ( WALBERLA_CXX_COMPILER_IS_CLANG )
add_flag ( CMAKE_CXX_FLAGS "-Wall -Wconversion -Wshadow -Wno-c++11-extensions -Qunused-arguments" )
......@@ -784,18 +773,16 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
if ( WIN32 )
message ( STATUS "Enter Workaround Routine for Windows and OpenMPI: PRESS CONFIGURE ONE MORE TIME!" )
string ( REGEX REPLACE "(.*)/bin/.*" "\\1" MPI_PATH ${MPI_CXX_COMPILER} )
find_path ( MPI_INCLUDE_PATH mpi.h
find_path ( MPI_C_INCLUDE_PATH mpi.h
HINTS ${MPI_PATH}
PATH_SUFFIXES include Inc)
set ( MPI_CXX_INCLUDE_PATH ${MPI_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
set ( MPI_C_INCLUDE_PATH ${MPI_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
set ( MPI_CXX_INCLUDE_PATH ${MPI_C_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
set ( MPI_CXX_LIBRARIES "MPI_CXX_LIBRARIES-NOTFOUND" CACHE FILEPATH "Cleared" FORCE )
find_library ( MPI_CXX_LIBRARIES
NAMES mpi++ mpicxx cxx mpi_cxx libmpi++ libmpicxx libcxx libmpi_cxx
HINTS ${MPI_PATH}
PATH_SUFFIXES lib )
set ( MPI_LIBRARY "MPI_CXX_LIBRARIES" CACHE FILEPATH "" FORCE )
if ( NOT MPI_CXX_LIBRARIES STREQUAL "MPI_CXX_LIBRARIES-NOTFOUND" )
set ( MPI_CXX_FOUND ON FORCE )
......@@ -806,7 +793,6 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
NAMES mpi mpich mpich2 msmpi libmpi libmpich libmpich2 libmsmpi
HINTS ${MPI_PATH}
PATH_SUFFIXES lib )
set ( MPI_EXTRA_LIBRARY "MPI_C_LIBRARIES" CACHE FILEPATH "" FORCE )
if ( NOT MPI_C_LIBRARIES STREQUAL "MPI_C_LIBRARIES-NOTFOUND" )
set ( MPI_C_FOUND ON FORCE )
......@@ -823,23 +809,14 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
endif ( )
if ( MPI_FOUND )
if ( MPI_CXX_FOUND )
include_directories ( SYSTEM ${MPI_CXX_INCLUDE_PATH} ${MPI_C_INCLUDE_PATH} )
foreach( LIB ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES} )
if ( LIB )
list ( APPEND SERVICE_LIBS ${LIB} )
endif ( )
endforeach ( )
add_flag ( CMAKE_CXX_FLAGS "${MPI_CXX_COMPILE_FLAGS}" )
add_flag ( CMAKE_C_FLAGS "${MPI_C_COMPILE_FLAGS}" )
else ( ) # For older CMake versions
include_directories ( SYSTEM ${MPI_INCLUDE_PATH} )
list ( APPEND SERVICE_LIBS ${MPI_LIBRARY} )
if ( MPI_EXTRA_LIBRARY )
list ( APPEND SERVICE_LIBS ${MPI_EXTRA_LIBRARY} )
include_directories ( SYSTEM ${MPI_CXX_INCLUDE_PATH} ${MPI_C_INCLUDE_PATH} )
foreach( LIB ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES} )
if ( LIB )
list ( APPEND SERVICE_LIBS ${LIB} )
endif ( )
add_flag ( CMAKE_C_FLAGS "${MPI_COMPILE_FLAGS}" )
endif ( )
endforeach ( )
add_flag ( CMAKE_CXX_FLAGS "${MPI_CXX_COMPILE_FLAGS}" )
add_flag ( CMAKE_C_FLAGS "${MPI_C_COMPILE_FLAGS}" )
add_flag ( CMAKE_MODULE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
add_flag ( CMAKE_EXE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
......@@ -847,7 +824,7 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
# When using Intel MPI, mpi.h has to be included before including the standard library
# therefore we use the -include flag to enforce this.
if ( MPI_INCLUDE_PATH MATCHES "intel" )
if ( MPI_C_INCLUDE_PATH MATCHES "intel" )
message (STATUS "Activating IntelMPI include workaround for mpi.h" )
add_flag ( CMAKE_CXX_FLAGS "-include mpi.h" )
add_flag ( CMAKE_C_FLAGS "-include mpi.h" )
......@@ -1233,7 +1210,23 @@ if ( WALBERLA_BUILD_WITH_LTO )
endif( )
endif ( )
############################################################################################################################
##
## Some more compiler flags that need to happen after any try_compile (e.g. inside FindMPI)
##
############################################################################################################################
# Treat warnings as errors
if ( WARNING_ERROR )
if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
add_flag ( CMAKE_CXX_FLAGS "-pedantic-errors -Werror" )
elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
add_flag ( CMAKE_CXX_FLAGS "/WX" )
elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
add_flag ( CMAKE_CXX_FLAGS "-h error_on_warning" )
endif()
endif ( )
############################################################################################################################
##
......
......@@ -156,6 +156,11 @@ function ( waLBerla_export )
# Export link paths
set ( WALBERLA_LINK_DIRS ${LINK_DIRS} CACHE INTERNAL "waLBerla link directories" )
set( WALBERLA_CXX_STANDARD ${CMAKE_CXX_STANDARD} CACHE INTERNAL "CXX standard")
set( WALBERLA_CXX_STANDARD_REQUIRED ${CMAKE_CXX_STANDARD_REQUIRED} CACHE INTERNAL "CXX Standard Required")
set( WALBERLA_CXX_EXTENSIONS ${CMAKE_CXX_EXTENSIONS} CACHE INTERNAL "CXX Extensions")
endfunction( waLBerla_export)
#######################################################################################################################
......@@ -210,7 +215,11 @@ function ( waLBerla_import )
endif()
set( SERVICE_LIBS ${WALBERLA_SERVICE_LIBS} PARENT_SCOPE )
set( CMAKE_CXX_STANDARD ${WALBERLA_CXX_STANDARD} PARENT_SCOPE)
set( CMAKE_CXX_STANDARD_REQUIRED ${WALBERLA_STANDARD_REQUIRED} PARENT_SCOPE)
set( CMAKE_CXX_EXTENSIONS ${WALBERLA_EXTENSIONS} PARENT_SCOPE)
link_directories( ${WALBERLA_LINK_DIRS} )
endfunction( waLBerla_import)
#######################################################################################################################
......
......@@ -30,20 +30,28 @@
#include "core/mpi/MPIManager.h"
#include "core/mpi/Gather.h"
#include "core/mpi/Gatherv.h"
#include "core/mpi/Reduce.h"
#include "core/timing/Timer.h"
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <array>
#include <vector>
namespace walberla {
namespace blockforest {
std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phantomForest, MPI_Comm comm )
std::pair<uint_t, uint_t> getBlockSequenceRange( uint_t numLocalBlocks, MPI_Comm comm )
{
const uint_t rank = uint_c(mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, MPIManager::instance()->rank()));
uint_t numLocalBlocks = phantomForest.getNumberOfBlocks();
WALBERLA_DEBUG_SECTION()
{
int rankRaw;
MPI_Comm_rank(comm, &rankRaw);
WALBERLA_ASSERT_EQUAL(rank, rankRaw);
}
size_t sequenceStartOnProcess = 0;
MPI_Exscan( &numLocalBlocks, &sequenceStartOnProcess, 1, MPITrait<uint_t>::type(), MPI_SUM, comm );
......@@ -53,14 +61,13 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan
return std::make_pair( sequenceStartOnProcess, sequenceStartOnProcess + numLocalBlocks );
}
std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest & phantomForest, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm )
std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest& phantomForest, const std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm )
{
std::map< blockforest::BlockID, uint_t > mapping;
const auto & blockMap = phantomForest.getBlockMap();
uint_t sequenceId = blockSequenceRange.first;
for( auto it = blockMap.begin(); it != blockMap.end(); ++it )
mapping.insert( std::make_pair( it->first, sequenceId++ ) );
for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it )
mapping.insert( std::make_pair( it->first->getId(), sequenceId++ ) );
WALBERLA_ASSERT_EQUAL( sequenceId, blockSequenceRange.second );
const std::vector<uint_t> neighborProcesses = phantomForest.getNeighboringProcesses();
......@@ -68,7 +75,11 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
mpi::BufferSystem bs( comm );
for( auto it = neighborProcesses.begin(); it != neighborProcesses.end(); ++it )
bs.sendBuffer( mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it)) ) << mapping;
{
auto destRank = mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it));
if (destRank != -1)
bs.sendBuffer( destRank ) << mapping;
}
bs.setReceiverInfoFromSendBufferState( false, true );
......@@ -101,6 +112,8 @@ T * ptr( std::vector<T> & v )
typedef uint_t idx_t;
bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest,
......@@ -111,11 +124,11 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
globalTimer.start();
//create new communicator which excludes processes which do not have blocks
MPI_Comm subComm;
MPI_Comm subComm = MPI_COMM_NULL;
MPI_Group allGroup, subGroup;
MPI_Comm_group( MPIManager::instance()->comm(), &allGroup );
std::vector<int> ranks;
if (phantomForest.getNumberOfBlocks() > 0)
if (targetProcess.size() > 0)
ranks.push_back( MPIManager::instance()->rank() );
ranks = mpi::allGatherv( ranks, MPIManager::instance()->comm() );
auto numSubProcesses = ranks.size();
......@@ -123,22 +136,51 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
MPI_Group_incl(allGroup, int_c(ranks.size()), &ranks[0], &subGroup);
MPI_Comm_create( MPIManager::instance()->comm(), subGroup, &subComm);
if ( targetProcess.size() != 0)
{
int subRank;
int subSize;
MPI_Comm_rank(subComm, &subRank);
MPI_Comm_size(subComm, &subSize);
WALBERLA_CHECK_GREATER_EQUAL(subRank, 0);
WALBERLA_CHECK_LESS(subRank, subSize);
} else
{
int subRank;
MPI_Comm_rank(subComm, &subRank);
WALBERLA_CHECK_EQUAL(subRank, MPI_UNDEFINED);
}
int64_t edgecut = 0;
std::vector<int64_t> part( phantomForest.getNumberOfBlocks(), int64_c( MPIManager::instance()->rank() ) );
WALBERLA_CHECK_EQUAL( phantomForest.getNumberOfBlocks(), targetProcess.size() );
std::vector<int64_t> part( targetProcess.size(), int64_c( MPIManager::instance()->rank() ) );
if (subComm != MPI_COMM_NULL)
{
const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( phantomForest, subComm );
const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, blockSequenceRange, subComm );
int subRank;
int subSize;
MPI_Comm_rank(subComm, &subRank);
MPI_Comm_size(subComm, &subSize);
WALBERLA_CHECK_UNEQUAL(targetProcess.size(), 0);
const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( targetProcess.size(), subComm );
const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, targetProcess, blockSequenceRange, subComm ); //blockid to vertex id
std::vector<int64_t> vtxdist = mpi::allGather( int64_c( blockSequenceRange.second ), subComm );
vtxdist.insert( vtxdist.begin(), uint_t( 0 ) );
WALBERLA_CHECK_EQUAL( vtxdist.size(), subSize + 1 );
for (size_t i = 1; i < vtxdist.size(); ++i)
{
WALBERLA_ASSERT_LESS( vtxdist[i-1], vtxdist[i] );
}
std::vector<int64_t> adjncy, xadj, vsize, vwgt, adjwgt;
std::vector<double> xyz;
for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it )
uint_t blockIndex = 0;
for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it, ++blockIndex )
{
WALBERLA_CHECK_EQUAL(blockIndex, mapping.find(it->first->getId())->second - blockSequenceRange.first);
xadj.push_back( int64_c( adjncy.size() ) );
const PhantomBlock & block = *( it->first );
auto bi = block.getData< DynamicParMetisBlockInfo >();
......@@ -150,9 +192,12 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{
auto mapIt = mapping.find( nit->getId() );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) );
auto edgeWeightIt = bi.getEdgeWeights().find( nit->getId() );
adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 0 ) : edgeWeightIt->second );
//WALBERLA_CHECK_UNEQUAL( edgeWeightIt->second, 0 ); // perhaps WARNING
adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 1 ) : edgeWeightIt->second );
}
break;
case PARMETIS_EDGES_FROM_EDGE_WEIGHTS:
......@@ -160,10 +205,15 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{
auto mapIt = mapping.find( edgeIt->first );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) );
//WALBERLA_CHECK_UNEQUAL( edgeIt->second, 0 ); // perhaps WARNING
adjwgt.push_back( edgeIt->second );
}
break;
}
WALBERLA_CHECK_UNEQUAL( bi.getVertexWeight(), 0 );
vwgt.push_back( bi.getVertexWeight() );
vsize.push_back( bi.getVertexSize() );
xyz.push_back( bi.getVertexCoords()[0] );
......@@ -172,45 +222,58 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
}
xadj.push_back( int64_c( adjncy.size() ) );
WALBERLA_ASSERT_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) );
WALBERLA_ASSERT_EQUAL( xadj.size(), phantomForest.getNumberOfBlocks() + 1 );
WALBERLA_ASSERT_EQUAL( vwgt.size(), phantomForest.getNumberOfBlocks() );
WALBERLA_ASSERT_EQUAL( vsize.size(), phantomForest.getNumberOfBlocks() );
WALBERLA_ASSERT_EQUAL( adjncy.size(), adjwgt.size() );
int64_t wgtflag = weightsToUse_;
int64_t numflag = 0; // C-style ordering
int64_t ncon = 1; // Number of constraints
int64_t ndims = 3; // Number of dimensions
double ubvec[] = { real_t( 1.05 ) }; // imbalance tolerance
int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
double ipc2redist = real_t( 1000000.0 ); // compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
MPI_Comm comm = subComm; //MPIManager::instance()->comm();
WALBERLA_CHECK_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) );
WALBERLA_CHECK_EQUAL( xadj.size(), targetProcess.size() + 1 );
WALBERLA_CHECK_EQUAL( xadj.front(), 0);
WALBERLA_CHECK_EQUAL( xadj.back(), adjncy.size() );
for (size_t i = 1; i < xadj.size(); ++i)
{
WALBERLA_ASSERT_LESS( xadj[i-1], xadj[i] );
}
WALBERLA_CHECK_EQUAL( vwgt.size(), targetProcess.size() );
WALBERLA_CHECK_EQUAL( vsize.size(), targetProcess.size() );
WALBERLA_CHECK_EQUAL( xyz.size(), targetProcess.size() * 3 );
WALBERLA_CHECK_EQUAL( adjncy.size(), adjwgt.size() );
WALBERLA_CHECK_EQUAL( adjwgt.size(), xadj.back() );
int64_t wgtflag = weightsToUse_;
int64_t numflag = 0; // C-style ordering
int64_t ncon = 1; // Number of constraints
int64_t ndims = 3; // Number of dimensions
std::vector<double> ubvec( uint_c(ncon), double_c( 1.05 ) ); // imbalance tolerance
int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
double ipc2redist = double_c(ipc2redist_);
MPI_Comm comm = subComm; //MPIManager::instance()->comm();
std::vector<double> tpwgts( uint_c(nparts * ncon), 1.0 / double_c( nparts ) ); // vertex weight fraction that is stored in a subdomain
int64_t options[] = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
std::vector<int64_t> options = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
int metisResult = core::METIS_OK;
switch( algorithm_ )
{
case PARMETIS_PART_GEOM:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartGeom( ptr( vtxdist ), &ndims, ptr( xyz ), ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_PART_GEOM_KWAY:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ptr( ubvec ), ptr( options ), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_PART_KWAY:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_ADAPTIVE_REPART:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, &ipc2redist, options, &edgecut, ptr( part ), &comm );
metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), &ipc2redist, ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_REFINE_KWAY:
parmetisTimer.start();
metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
}
......@@ -265,6 +328,8 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s )
if( s == "PART_GEOM_KWAY" )
return PARMETIS_PART_GEOM_KWAY;
else if( s == "PART_GEOM" )
return PARMETIS_PART_GEOM;
else if( s == "PART_KWAY" )
return PARMETIS_PART_KWAY;
else if( s == "PART_ADAPTIVE_REPART" )
......@@ -272,7 +337,7 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s )
else if( s == "REFINE_KWAY" )
return PARMETIS_REFINE_KWAY;
else
WALBERLA_ABORT( "Illegal ParMetis algorithm specified! Valid choices are: \"PART_GEOM_KWAY\", \"PART_KWAY\", \"PART_ADAPTIVE_REPART\", or \"REFINE_KWAY\"." );
WALBERLA_ABORT( "Illegal ParMetis algorithm specified (" << s << ")! Valid choices are: \"PART_GEOM_KWAY\", \"PART_KWAY\", \"PART_ADAPTIVE_REPART\", or \"REFINE_KWAY\"." );
}
......@@ -290,7 +355,7 @@ DynamicParMetis::WeightsToUse DynamicParMetis::stringToWeightsToUse( std::string
else if( s == "BOTH_WEIGHTS" )
return PARMETIS_BOTH_WEIGHTS;
else
WALBERLA_ABORT( "Illegal ParMetis weights usage specified! Valid choices are: \"NO_WEIGHTS\", \"EDGE_WEIGHTS\", \"VERTEX_WEIGHTS\", or \"BOTH_WEIGHTS\"." );
WALBERLA_ABORT( "Illegal ParMetis weights usage specified (" << s << ")! Valid choices are: \"NO_WEIGHTS\", \"EDGE_WEIGHTS\", \"VERTEX_WEIGHTS\", or \"BOTH_WEIGHTS\"." );
}
......@@ -304,7 +369,56 @@ DynamicParMetis::EdgeSource DynamicParMetis::stringToEdgeSource( std::string s )
else if( s == "EDGES_FROM_EDGE_WEIGHTS" )
return PARMETIS_EDGES_FROM_EDGE_WEIGHTS;
else
WALBERLA_ABORT( "Illegal ParMetis weights usage specified! Valid choices are: \"EDGES_FROM_FOREST\" or \"EDGES_FROM_EDGE_WEIGHTS\"" );
WALBERLA_ABORT( "Illegal ParMetis weights usage specified (" << s << ")! Valid choices are: \"EDGES_FROM_FOREST\" or \"EDGES_FROM_EDGE_WEIGHTS\"" );
}
std::string DynamicParMetis::algorithmToString( ) const
{
switch (algorithm_)
{
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM_KWAY:
return "PART_GEOM_KWAY";
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM:
return "PART_GEOM";
case walberla::blockforest::DynamicParMetis::PARMETIS_PART_KWAY:
return "PART_KWAY";
case walberla::blockforest::DynamicParMetis::PARMETIS_ADAPTIVE_REPART:
return "PART_ADAPTIVE_REPART";
case walberla::blockforest::DynamicParMetis::PARMETIS_REFINE_KWAY:
return "PARMETIS_REFINE_KWAY";
}
return "Unknown";
}
std::string DynamicParMetis::weightsToUseToString( ) const
{
switch (weightsToUse_)
{
case walberla::blockforest::DynamicParMetis::PARMETIS_NO_WEIGHTS:
return "NO_WEIGHTS";
case walberla::blockforest::DynamicParMetis::PARMETIS_EDGE_WEIGHTS:
return "EDGE_WEIGHTS";
case walberla::blockforest::DynamicParMetis::PARMETIS_VERTEX_WEIGHTS:
return "VERTEX_WEIGHTS";
case walberla::blockforest::DynamicParMetis::PARMETIS_BOTH_WEIGHTS:
return "BOTH_WEIGHTS";
}
return "Unknown";
}
std::string DynamicParMetis::edgeSourceToString( ) const
{
switch (edgeSource_)
{
case walberla::blockforest::DynamicParMetis::PARMETIS_EDGES_FROM_FOREST:
return "EDGES_FROM_FOREST";
case walberla::blockforest::DynamicParMetis::PARMETIS_EDGES_FROM_EDGE_WEIGHTS:
return "EDGES_FROM_EDGE_WEIGHTS";
}
return "Unknown";
}
......
......@@ -23,10 +23,12 @@
#include "blockforest/PhantomBlockForest.h"
#include "core/debug/Debug.h"
#include "core/DataTypes.h"
#include "core/math/Vector3.h"
#include "core/mpi/MPIWrapper.h"
#include <cmath>
#include <map>
namespace walberla {
......@@ -40,7 +42,7 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
class DynamicParMetis
{
public:
enum Algorithm { PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum Algorithm { PARMETIS_PART_GEOM, PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 };
enum EdgeSource { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS };
......@@ -54,6 +56,10 @@ public:
const PhantomBlockForest & phantomForest,
const uint_t iteration ) const;
void setipc2redist(double val) {ipc2redist_ = val;}
double getipc2redist() const {return ipc2redist_;}
bool edgeWeightsUsed() const { return ( weightsToUse_ == PARMETIS_EDGE_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexWeightsUsed() const { return ( weightsToUse_ == PARMETIS_VERTEX_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexSizeUsed() const { return algorithm_ == PARMETIS_ADAPTIVE_REPART; }
......@@ -62,10 +68,16 @@ public:
static WeightsToUse stringToWeightsToUse( std::string s );
static EdgeSource stringToEdgeSource( std::string s );
std::string algorithmToString() const;
std::string weightsToUseToString() const;
std::string edgeSourceToString() const;
protected:
Algorithm algorithm_;
WeightsToUse weightsToUse_;