MPIManager.cpp

//======================================================================================================================
//
//  This file is part of waLBerla. waLBerla is free software: you can 
//  redistribute it and/or modify it under the terms of the GNU General Public
//  License as published by the Free Software Foundation, either version 3 of 
//  the License, or (at your option) any later version.
//  
//  waLBerla is distributed in the hope that it will be useful, but WITHOUT 
//  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
//  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License 
//  for more details.
//  
//  You should have received a copy of the GNU General Public License along
//  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file MPIManager.cpp
//! \ingroup core
//! \author Martin Bauer <martin.bauer@fau.de>
//! \author Florian Schornbaum <florian.schornbaum@fau.de>
//! \author Christian Godenschwager <christian.godenschwager@fau.de>
//
//======================================================================================================================

#include "MPIManager.h"
#include "core/logging/Logging.h"

#include <boost/exception_ptr.hpp>
#include <exception>
#include <iostream>
#include <stdexcept>
#include <vector>
#include <string>


namespace walberla{
namespace mpi {


/**
 * Terminate Handler that calls MPI_Abort instead of std::abort
 *
 * Terminate handler is called when an exception is not caught
 * and the program has to be aborted
 * The standard terminate handler prints exception.what() and
 * calls std::abort(). We overwrite the terminate handler when MPI was
 * initialized, to call MPI_Abort in this case.
 */
static void customTerminateHandler()
{
   std::cerr << "Execution failed: Uncaught Exception." << std::endl;

   // The standard terminate handler prints the exception text,
   // Here we want to do the same. To get the exception we use a hack
   // that only works on linux environments: We retrieve the current_exception() pointer
   // which strictly is only allowed in catch() blocks, however works here as well, if we are
   // in linux environments. If this pointer is null,
   // i.e. in cases when this hack does not work, we just cannot print the message
   // otherwise we re-throw the exception to get the type, and print exception.what()
   try {
      if(boost::current_exception() )
         boost::rethrow_exception(boost::current_exception());
   } catch (std::exception const & exc) {
      std::cerr << exc.what() << std::endl;
   }

   WALBERLA_MPI_SECTION() {
      MPIManager::instance()->abort();
   }
   else {
      std::abort();
   }
}



MPIManager::~MPIManager()
{
   finalizeMPI();
}

void MPIManager::abort()
{
   currentlyAborting_ = true;
   WALBERLA_MPI_SECTION()
   {
      if( MPIManager::instance()->isMPIInitialized() )
         MPI_Abort( MPI_COMM_WORLD, EXIT_FAILURE );
   }
   std::exit( EXIT_FAILURE );
}


void MPIManager::initializeMPI( int* argc, char*** argv, bool abortOnException )
{
   WALBERLA_MPI_SECTION()
   {
      WALBERLA_ASSERT( !isMPIInitialized_ );

      // Check first that MPI was not initialized before
      // f.e. when using Python, MPI could have been initialized by
      // a different MPI module like mpi4py
      int mpiAlreadyInitialized=0;
      MPI_Initialized( &mpiAlreadyInitialized );
      if ( ! mpiAlreadyInitialized ) {
         MPI_Init( argc, argv );
      }

      isMPIInitialized_ = true;
      MPI_Comm_size( MPI_COMM_WORLD, &numProcesses_ );
      MPI_Comm_rank( MPI_COMM_WORLD, &worldRank_ );

      if( abortOnException )
         std::set_terminate( customTerminateHandler );
   }
}



void MPIManager::finalizeMPI()
{
   WALBERLA_MPI_SECTION() {
      if( isMPIInitialized_ && !currentlyAborting_ )
      {
         isMPIInitialized_ = false;
         MPI_Finalize();
      }
   }
}



void MPIManager::resetMPI()
{
   WALBERLA_MPI_SECTION() {
      WALBERLA_ASSERT( isMPIInitialized_ );
      if( rank_ != -1 )
      {
         if( comm_ == MPI_COMM_WORLD )
            comm_ = MPI_COMM_NULL;
         else
            MPI_Comm_free( &comm_ );
         rank_ = -1;
      }
      cartesianSetup_ = false;
      WALBERLA_ASSERT_EQUAL( comm_, MPI_COMM_NULL );
      WALBERLA_ASSERT_EQUAL( rank_, -1 );
   }
}



void MPIManager::createCartesianComm( int dims[3], int periodicity[3] )
{
   WALBERLA_ASSERT( isMPIInitialized_ );
   WALBERLA_ASSERT_EQUAL( rank_, -1 );
   WALBERLA_ASSERT( !cartesianSetup_ );
   WALBERLA_ASSERT_GREATER( dims[0], 0 );
   WALBERLA_ASSERT_GREATER( dims[1], 0 );
   WALBERLA_ASSERT_GREATER( dims[2], 0 );

   if ( ! isCartesianCommValid() ) {
      WALBERLA_LOG_WARNING_ON_ROOT( "Your version of OpenMPI contains a bug which might lead to a segmentation fault "
                                    "when generating vtk output. Since the bug only occurs with a 3D Cartesian MPI "
                                    "communicator, try to use MPI_COMM_WORLD instead. See waLBerla issue #73 for "
                                    "additional information." );
   }

   MPI_Cart_create( MPI_COMM_WORLD, 3, dims, periodicity, true, &comm_ );
   MPI_Comm_rank( comm_, &rank_ );
   cartesianSetup_ = true;

   WALBERLA_ASSERT_UNEQUAL(comm_, MPI_COMM_NULL);
}



void MPIManager::createCartesianComm( const uint_t xProcesses, const uint_t  yProcesses, const uint_t zProcesses,
                                      const bool   xPeriodic,  const bool    yPeriodic,  const bool   zPeriodic )
{
   int dims[3];
   dims[0] = numeric_cast<int>( xProcesses );
   dims[1] = numeric_cast<int>( yProcesses );
   dims[2] = numeric_cast<int>( zProcesses );

   int periodicity[3];
   periodicity[0] = xPeriodic ? 1 : 0;
   periodicity[1] = yPeriodic ? 1 : 0;
   periodicity[2] = zPeriodic ? 1 : 0;

   createCartesianComm( dims, periodicity );
}



void MPIManager::cartesianCoord( int coordOut[3] ) const
{
   cartesianCoord( rank_, coordOut );
}



void MPIManager::cartesianCoord( int rankIn, int coordOut[3] ) const
{
   WALBERLA_ASSERT( isMPIInitialized_ );
   WALBERLA_ASSERT( cartesianSetup_ );
   WALBERLA_ASSERT_UNEQUAL( comm_, MPI_COMM_NULL );

   MPI_Cart_coords( comm_, rankIn, 3, coordOut );
}



int MPIManager::cartesianRank( int coords[3] ) const
{
   WALBERLA_ASSERT( isMPIInitialized_ );
   WALBERLA_ASSERT( cartesianSetup_ );
   WALBERLA_ASSERT_UNEQUAL( comm_, MPI_COMM_NULL );

   int r;
   MPI_Cart_rank( comm_, coords, &r );
   return r;
}



int MPIManager::cartesianRank( const uint_t x, const uint_t y, const uint_t z ) const
{
   int coords[3];
   coords[0] = numeric_cast<int>(x);
   coords[1] = numeric_cast<int>(y);
   coords[2] = numeric_cast<int>(z);

   return cartesianRank( coords );
}

bool MPIManager::isCartesianCommValid() const
{
   // Avoid using the Cartesian MPI-communicator with certain (buggy) versions of OpenMPI (see waLBerla issue #73)
   #if defined(OMPI_MAJOR_VERSION) && defined(OMPI_MINOR_VERSION) && defined(OMPI_RELEASE_VERSION)
      std::string ompi_ver = std::to_string(OMPI_MAJOR_VERSION) + "." + std::to_string(OMPI_MINOR_VERSION) + "." +
            std::to_string(OMPI_RELEASE_VERSION);

      if (ompi_ver == "2.0.0" || ompi_ver == "2.0.1" || ompi_ver == "2.0.2" || ompi_ver == "2.0.3" ||
          ompi_ver == "2.1.0" || ompi_ver == "2.1.1") {
         return false;
      }
      else {
         return true;
      }
   #else
      return true;
   #endif
}

std::string MPIManager::getMPIErrorString(int errorCode)
{
   WALBERLA_NON_MPI_SECTION()
   {
      throw std::logic_error("Trying to use function 'MPIManager::getMPIErrorString' but waLBerla is compiled without MPI-support!");
   }
   WALBERLA_ASSERT_GREATER(MPI_MAX_ERROR_STRING, 0);
   std::vector<char> errorString(MPI_MAX_ERROR_STRING);
   int resultLen;

   MPI_Error_string(errorCode, &errorString[0], &resultLen);

   WALBERLA_ASSERT_GREATER_EQUAL(resultLen, 0);
   WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast<int>(errorString.size()));
   return std::string(errorString.begin(), errorString.begin() + resultLen);
}

std::string MPIManager::getMPICommName(MPI_Comm comm)
{
   WALBERLA_NON_MPI_SECTION()
   {
      throw std::logic_error("Trying to use function 'MPIManager::getMPICommName' but waLBerla is compiled without MPI-support!");
   }
   WALBERLA_ASSERT_GREATER(MPI_MAX_OBJECT_NAME, 0);
   std::vector<char> commName(MPI_MAX_OBJECT_NAME);
   int resultLen;

   MPI_Comm_get_name(comm, &commName[0], &resultLen);

   WALBERLA_ASSERT_GREATER_EQUAL(resultLen, 0);
   WALBERLA_ASSERT_LESS_EQUAL(resultLen, numeric_cast<int>(commName.size()));
   return std::string(commName.begin(), commName.begin() + resultLen);
}

} // namespace mpi
} // namespace walberla