Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
No results found
Show changes
Hide whitespace changes
Inline Side-by-side
Showing
with 7416 additions and 237 deletions
apps/benchmarks/ComplexGeometry/ComplexGeometry.cpp
View file @ 7a6d3b57
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
//! \file ComplexGeometry.cpp //! \file ComplexGeometry.cpp
//! \ingroup mesh //! \ingroup mesh
//! \author Christian Godenschwager <christian.godenschwager@fau.de> //! \author Christian Godenschwager <christian.godenschwager@fau.de>
//! \author Markus Holzer <markus.holzer@fau.de>
// //
//====================================================================================================================== //======================================================================================================================
...@@ -24,282 +25,268 @@ ...@@ -24,282 +25,268 @@
#include "blockforest/communication/UniformBufferedScheme.h" #include "blockforest/communication/UniformBufferedScheme.h"
#include "blockforest/loadbalancing/StaticParMetis.h" #include "blockforest/loadbalancing/StaticParMetis.h"
#include "core/SharedFunctor.h"
#include "core/Environment.h" #include "core/Environment.h"
#include "core/SharedFunctor.h"
#include "core/logging/Logging.h" #include "core/logging/Logging.h"
#include "core/math/IntegerFactorization.h" #include "core/math/IntegerFactorization.h"
#include "core/timing/RemainingTimeLogger.h"
#include "domain_decomposition/SharedSweep.h" #include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h" #include "field/AddToStorage.h"
#include "field/StabilityChecker.h" #include "field/StabilityChecker.h"
#include "geometry/InitBoundaryHandling.h"
#include "geometry/mesh/TriangleMesh.h"
#include "geometry/mesh/TriangleMeshIO.h"
#include "lbm/BlockForestEvaluation.h"
#include "lbm/PerformanceEvaluation.h"
#include "lbm/PerformanceLogger.h"
#include "lbm/boundary/factories/DefaultBoundaryHandling.h" #include "lbm/boundary/factories/DefaultBoundaryHandling.h"
#include "lbm/communication/PdfFieldPackInfo.h" #include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/communication/SparsePdfFieldPackInfo.h" #include "lbm/communication/SparsePdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h" #include "lbm/field/AddToStorage.h"
#include "lbm/lattice_model/CollisionModel.h"
#include "lbm/lattice_model/D3Q19.h" #include "lbm/lattice_model/D3Q19.h"
#include "lbm/lattice_model/D3Q27.h" #include "lbm/lattice_model/D3Q27.h"
#include "lbm/lattice_model/CollisionModel.h"
#include "lbm/lattice_model/ForceModel.h" #include "lbm/lattice_model/ForceModel.h"
#include "lbm/refinement/TimeStep.h" #include "lbm/refinement/TimeStep.h"
#include "lbm/sweeps/CellwiseSweep.h" #include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SplitPureSweep.h" #include "lbm/sweeps/SplitPureSweep.h"
#include "lbm/vtk/VTKOutput.h" #include "lbm/vtk/VTKOutput.h"
#include "lbm/BlockForestEvaluation.h"
#include "lbm/PerformanceEvaluation.h"
#include "lbm/PerformanceLogger.h"
#include "geometry/mesh/TriangleMesh.h" #include "mesh/blockforest/BlockExclusion.h"
#include "geometry/mesh/TriangleMeshIO.h"
#include "geometry/InitBoundaryHandling.h"
#include "mesh/TriangleMeshes.h"
#include "mesh/MeshOperations.h"
#include "mesh/DistanceComputations.h"
#include "mesh/MeshIO.h"
#include "mesh/MatrixVectorOperations.h"
#include "mesh/blockforest/BlockForestInitialization.h" #include "mesh/blockforest/BlockForestInitialization.h"
#include "mesh/blockforest/BlockWorkloadMemory.h" #include "mesh/blockforest/BlockWorkloadMemory.h"
#include "mesh/blockforest/BlockExclusion.h"
#include "mesh/blockforest/RefinementSelection.h" #include "mesh/blockforest/RefinementSelection.h"
#include "mesh/distance_octree/DistanceOctree.h"
#include "mesh/boundary/BoundarySetup.h"
#include "mesh/boundary/BoundaryInfo.h" #include "mesh/boundary/BoundaryInfo.h"
#include "mesh/boundary/BoundaryLocation.h" #include "mesh/boundary/BoundaryLocation.h"
#include "mesh/boundary/BoundaryLocationFunction.h"
#include "mesh/boundary/BoundarySetup.h"
#include "mesh/boundary/BoundaryUIDFaceDataSource.h" #include "mesh/boundary/BoundaryUIDFaceDataSource.h"
#include "mesh/boundary/ColorToBoundaryMapper.h" #include "mesh/boundary/ColorToBoundaryMapper.h"
#include "mesh/vtk/VTKMeshWriter.h"
#include "mesh/vtk/CommonDataSources.h"
#include "stencil/D3Q19.h" #include "stencil/D3Q19.h"
#include "timeloop/SweepTimeloop.h" #include "timeloop/SweepTimeloop.h"
#include "core/timing/RemainingTimeLogger.h"
#include <boost/lexical_cast.hpp>
#include <cmath> #include <cmath>
#include <vector>
#include <string> #include <string>
#include <vector>
namespace walberla { #include "mesh_common/DistanceComputations.h"
#include "mesh_common/DistanceFunction.h"
#include "mesh_common/MatrixVectorOperations.h"
template< typename MeshDistanceType > #include "mesh_common/MeshIO.h"
struct MeshDistanceFunction #include "mesh_common/MeshOperations.h"
{ #include "mesh_common/TriangleMeshes.h"
MeshDistanceFunction( const shared_ptr< MeshDistanceType > & meshDistanceObject ) : meshDistanceObject_( meshDistanceObject ) { } #include "mesh_common/distance_octree/DistanceOctree.h"
#include "mesh_common/vtk/CommonDataSources.h"
inline real_t operator()( const Vector3< real_t > & p ) const { return real_c( meshDistanceObject_->sqSignedDistance( mesh::toOpenMesh( p ) ) ); } #include "mesh_common/vtk/VTKMeshWriter.h"
shared_ptr< MeshDistanceType > meshDistanceObject_; namespace walberla
};
template< typename MeshDistanceType >
inline MeshDistanceFunction< MeshDistanceType > makeMeshDistanceFunction( const shared_ptr< MeshDistanceType > & meshDistanceObject )
{
return MeshDistanceFunction< MeshDistanceType >( meshDistanceObject );
}
template< typename MeshDistanceType, typename MeshType >
struct BoundaryLocationFunction
{
BoundaryLocationFunction( const shared_ptr< MeshDistanceType > & meshDistanceObject, const shared_ptr< mesh::BoundaryLocation< MeshType > > & boundaryLocation )
: meshDistanceObject_( meshDistanceObject ), boundaryLocation_( boundaryLocation ) { }
inline const mesh::BoundaryInfo & operator()( const Vector3< real_t > & p ) const
{
typename MeshType::FaceHandle fh;
meshDistanceObject_->sqSignedDistance( mesh::toOpenMesh( p ), fh );
return (*boundaryLocation_)[ fh ];
}
shared_ptr< MeshDistanceType > meshDistanceObject_;
shared_ptr< mesh::BoundaryLocation< MeshType > > boundaryLocation_;
};
template< typename MeshDistanceType, typename MeshType >
inline BoundaryLocationFunction< MeshDistanceType, MeshType > makeBoundaryLocationFunction( const shared_ptr< MeshDistanceType > & meshDistanceObject, const shared_ptr< mesh::BoundaryLocation< MeshType > > & boundaryLocation )
{ {
return BoundaryLocationFunction< MeshDistanceType, MeshType >( meshDistanceObject, boundaryLocation );
}
template< typename MeshType > template< typename MeshType >
void vertexToFaceColor( MeshType & mesh, const typename MeshType::Color & defaultColor ) void vertexToFaceColor(MeshType& mesh, const typename MeshType::Color& defaultColor)
{ {
WALBERLA_CHECK( mesh.has_vertex_colors() ); WALBERLA_CHECK(mesh.has_vertex_colors())
mesh.request_face_colors(); mesh.request_face_colors();
for( auto faceIt = mesh.faces_begin(); faceIt != mesh.faces_end(); ++faceIt ) for (auto faceIt = mesh.faces_begin(); faceIt != mesh.faces_end(); ++faceIt)
{ {
typename MeshType::Color vertexColor; typename MeshType::Color vertexColor;
bool useVertexColor = true; bool useVertexColor = true;
auto vertexIt = mesh.fv_iter( *faceIt ); auto vertexIt = mesh.fv_iter(*faceIt);
WALBERLA_ASSERT( vertexIt.is_valid() ); WALBERLA_ASSERT(vertexIt.is_valid())
vertexColor = mesh.color( *vertexIt ); vertexColor = mesh.color(*vertexIt);
++vertexIt; ++vertexIt;
while( vertexIt.is_valid() && useVertexColor ) while (vertexIt.is_valid() && useVertexColor)
{ {
if( vertexColor != mesh.color( *vertexIt ) ) if (vertexColor != mesh.color(*vertexIt)) useVertexColor = false;
useVertexColor = false;
++vertexIt; ++vertexIt;
} }
mesh.set_color( *faceIt, useVertexColor ? vertexColor : defaultColor ); mesh.set_color(*faceIt, useVertexColor ? vertexColor : defaultColor);
} }
} }
int main(int argc, char* argv[])
int main( int argc, char * argv[] )
{ {
Environment env( argc, argv ); Environment env(argc, argv);
if( !env.config() ) if (!env.config()) { WALBERLA_ABORT_NO_DEBUG_INFO("USAGE: " << argv[0] << " INPUT_FILE") }
{
WALBERLA_ABORT_NO_DEBUG_INFO( "USAGE: " << argv[0] << " INPUT_FILE" );
}
mpi::MPIManager::instance()->useWorldComm(); mpi::MPIManager::instance()->useWorldComm();
const auto & config = *( env.config() ); const auto& config = *(env.config());
Config::BlockHandle configBlock = config.getOneBlock( "ComplexGeometry" ); Config::BlockHandle configBlock = config.getOneBlock("ComplexGeometry");
const std::string meshFile = configBlock.getParameter< std::string >( "meshFile" ); const std::string meshFile = configBlock.getParameter< std::string >("meshFile");
const real_t dx = configBlock.getParameter< real_t >( "coarseDx" ); const real_t dx = configBlock.getParameter< real_t >("coarseDx");
const real_t omega = configBlock.getParameter< real_t >( "coarseOmega" ); const real_t omega = configBlock.getParameter< real_t >("coarseOmega");
//const uint_t blockPerProcess = configBlock.getParameter< uint_t >( "blocksPerProcess", uint_t(6) );
const uint_t timeSteps = configBlock.getParameter< uint_t >( "coarseTimeSteps" );
const Vector3<real_t> bodyForce = configBlock.getParameter< Vector3<real_t> >( "bodyForce" );
//const bool sparseCommunication = configBlock.getParameter< bool >( "sparseCommunication", true );
const Vector3<real_t> domainBlowUp = configBlock.getParameter< Vector3<real_t> >( "domainBlowUp", Vector3<real_t>(6) );
const Vector3<uint_t> blockSize = configBlock.getParameter< Vector3<uint_t> >( "blockSize", Vector3<uint_t>(16) );
uint_t numLevels = configBlock.getParameter< uint_t >( "numLevels", uint_t(2) );
numLevels = std::max( numLevels, uint_t(1) ); if (configBlock.getParameter< bool >("logLevelDetail"))
walberla::logging::Logging::instance()->setLogLevel(walberla::logging::Logging::DETAIL);
//uint_t numProcesses = uint_c( MPIManager::instance()->numProcesses() ); const uint_t timeSteps = configBlock.getParameter< uint_t >("coarseTimeSteps");
const Vector3< real_t > bodyForce = configBlock.getParameter< Vector3< real_t > >("bodyForce");
const Vector3< real_t > domainBlowUp =
configBlock.getParameter< Vector3< real_t > >("domainBlowUp", Vector3< real_t >(6));
const Vector3< uint_t > blockSize =
configBlock.getParameter< Vector3< uint_t > >("blockSize", Vector3< uint_t >(16));
uint_t numLevels = configBlock.getParameter< uint_t >("numLevels", uint_t(2));
const bool WriteDistanceOctree = configBlock.getParameter< bool >("WriteDistanceOctree", false);
const bool WriteSetupForestAndReturn = configBlock.getParameter< bool >("WriteSetupForestAndReturn", false);
WALBERLA_LOG_DEVEL_VAR_ON_ROOT( meshFile ); numLevels = std::max(numLevels, uint_t(1));
const real_t fineDX = dx / real_c(std::pow(2, numLevels));
// uint_t numProcesses = uint_c( MPIManager::instance()->numProcesses() );
WALBERLA_LOG_DEVEL_VAR_ON_ROOT(meshFile)
auto mesh = make_shared< mesh::TriangleMesh >(); auto mesh = make_shared< mesh::TriangleMesh >();
mesh->request_vertex_colors(); mesh->request_vertex_colors();
WALBERLA_LOG_DEVEL_ON_ROOT( "Loading mesh" ); WALBERLA_LOG_DEVEL_ON_ROOT("Loading mesh")
mesh::readAndBroadcast( meshFile, *mesh); mesh::readAndBroadcast(meshFile, *mesh);
vertexToFaceColor( *mesh, mesh::TriangleMesh::Color(255,255,255) ); vertexToFaceColor(*mesh, mesh::TriangleMesh::Color(255, 255, 255));
WALBERLA_LOG_DEVEL_ON_ROOT("Adding distance info to mesh")
auto triDist = make_shared< mesh::TriangleDistance< mesh::TriangleMesh > >(mesh);
WALBERLA_LOG_DEVEL_ON_ROOT("Building distance octree")
auto distanceOctree = make_shared< mesh::DistanceOctree< mesh::TriangleMesh > >(triDist);
WALBERLA_LOG_DEVEL_ON_ROOT("done. Octree has height " << distanceOctree->height())
// write distance octree to file
if (WriteDistanceOctree) {
distanceOctree->writeVTKOutput("distanceOctree");
}
WALBERLA_LOG_DEVEL_ON_ROOT( "Adding distance info to mesh" ); auto aabb = computeAABB(*mesh);
auto triDist = make_shared< mesh::TriangleDistance<mesh::TriangleMesh> >( mesh ); aabb.scale(domainBlowUp);
WALBERLA_LOG_DEVEL_ON_ROOT( "Building distance octree" );
auto distanceOctree = make_shared< mesh::DistanceOctree<mesh::TriangleMesh> >( triDist );
WALBERLA_LOG_DEVEL_ON_ROOT( "done. Octree has height " << distanceOctree->height() );
distanceOctree->writeVTKOutput("distanceOctree"); mesh::ComplexGeometryStructuredBlockforestCreator bfc(aabb, Vector3< real_t >(dx));
auto aabb = computeAABB( *mesh ); bfc.setRootBlockExclusionFunction(mesh::makeExcludeMeshInterior(distanceOctree, dx));
aabb.scale( domainBlowUp ); bfc.setBlockExclusionFunction(mesh::makeExcludeMeshInteriorRefinement(distanceOctree, fineDX));
mesh::ComplexGeometryStructuredBlockforestCreator bfc( aabb, Vector3<real_t>( dx ), mesh::makeExcludeMeshInterior( distanceOctree, dx ) ); auto meshWorkloadMemory = mesh::makeMeshWorkloadMemory(distanceOctree, dx);
auto meshWorkloadMemory = mesh::makeMeshWorkloadMemory( distanceOctree, dx );
meshWorkloadMemory.setInsideCellWorkload(1); meshWorkloadMemory.setInsideCellWorkload(1);
meshWorkloadMemory.setOutsideCellWorkload(1); meshWorkloadMemory.setOutsideCellWorkload(1);
bfc.setWorkloadMemorySUIDAssignmentFunction( meshWorkloadMemory ); bfc.setWorkloadMemorySUIDAssignmentFunction(meshWorkloadMemory);
bfc.setPeriodicity( Vector3<bool>(true) ); bfc.setPeriodicity(Vector3< bool >(true));
bfc.setRefinementSelectionFunction( makeRefinementSelection( distanceOctree, numLevels - uint_t(1), dx, dx * real_t(1) ) ); bfc.setRefinementSelectionFunction(
makeRefinementSelection(distanceOctree, numLevels - uint_t(1), dx, dx * real_t(1)));
auto structuredBlockforest = bfc.createStructuredBlockForest( blockSize ); if (WriteSetupForestAndReturn)
{
WALBERLA_LOG_INFO_ON_ROOT("Setting up SetupBlockForest")
auto setupForest = bfc.createSetupBlockForest(blockSize);
WALBERLA_LOG_INFO_ON_ROOT("Writing SetupBlockForest to VTK file")
WALBERLA_ROOT_SECTION()
{
setupForest->writeVTKOutput("SetupBlockForest");
}
WALBERLA_LOG_INFO_ON_ROOT("Stopping program")
return EXIT_SUCCESS;
}
auto structuredBlockforest = bfc.createStructuredBlockForest(blockSize);
typedef lbm::D3Q19<lbm::collision_model::SRT, false, lbm::force_model::SimpleConstant> LatticeModel_T; typedef lbm::D3Q19< lbm::collision_model::SRT, false, lbm::force_model::SimpleConstant > LatticeModel_T;
using flag_t = walberla::uint8_t; using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>; using FlagField_T = FlagField< flag_t >;
using PdfField_T = lbm::PdfField<LatticeModel_T>; using PdfField_T = lbm::PdfField< LatticeModel_T >;
LatticeModel_T latticeModel{ lbm::collision_model::SRT( omega ), lbm::force_model::SimpleConstant( bodyForce ) }; LatticeModel_T latticeModel{ lbm::collision_model::SRT(omega), lbm::force_model::SimpleConstant(bodyForce) };
static const uint_t NUM_GHOSTLAYERS = 4; static const uint_t NUM_GHOSTLAYERS = 4;
BlockDataID pdfFieldId = lbm::addPdfFieldToStorage( structuredBlockforest, "pdf field", latticeModel, Vector3<real_t>(0), real_t(1), NUM_GHOSTLAYERS, field::fzyx ); BlockDataID pdfFieldId = lbm::addPdfFieldToStorage(structuredBlockforest, "pdf field", latticeModel,
BlockDataID flagFieldId = field::addFlagFieldToStorage< FlagField_T >( structuredBlockforest, "flag field", NUM_GHOSTLAYERS ); Vector3< real_t >(0), real_t(1), NUM_GHOSTLAYERS, field::fzyx);
BlockDataID flagFieldId =
field::addFlagFieldToStorage< FlagField_T >(structuredBlockforest, "flag field", NUM_GHOSTLAYERS);
const FlagUID fluidFlagUID( "Fluid" ); const FlagUID fluidFlagUID("Fluid");
typedef lbm::DefaultBoundaryHandlingFactory< LatticeModel_T, FlagField_T > BHFactory; typedef lbm::DefaultBoundaryHandlingFactory< LatticeModel_T, FlagField_T > BHFactory;
auto boundariesConfig = configBlock.getOneBlock( "Boundaries" ); auto boundariesConfig = configBlock.getOneBlock("Boundaries");
BlockDataID boundaryHandlingId = BHFactory::addBoundaryHandlingToStorage( structuredBlockforest, "boundary handling", flagFieldId, pdfFieldId, fluidFlagUID, BlockDataID boundaryHandlingId = BHFactory::addBoundaryHandlingToStorage(
boundariesConfig.getParameter< Vector3<real_t> >( "velocity0", Vector3<real_t>() ), structuredBlockforest, "boundary handling", flagFieldId, pdfFieldId, fluidFlagUID,
boundariesConfig.getParameter< Vector3<real_t> >( "velocity1", Vector3<real_t>() ), boundariesConfig.getParameter< Vector3< real_t > >("velocity0", Vector3< real_t >()),
boundariesConfig.getParameter< real_t > ( "pressure0", real_c( 1.0 ) ), boundariesConfig.getParameter< Vector3< real_t > >("velocity1", Vector3< real_t >()),
boundariesConfig.getParameter< real_t > ( "pressure1", real_c( 1.001 ) ) ); boundariesConfig.getParameter< real_t >("pressure0", real_c(1.0)),
boundariesConfig.getParameter< real_t >("pressure1", real_c(1.001)));
mesh::ColorToBoundaryMapper<mesh::TriangleMesh> colorToBoundryMapper(( mesh::BoundaryInfo( BHFactory::getNoSlipBoundaryUID() ) )); mesh::ColorToBoundaryMapper< mesh::TriangleMesh > colorToBoundryMapper(
(mesh::BoundaryInfo(BHFactory::getNoSlipBoundaryUID())));
// colorToBoundryMapper.set( mesh::TriangleMesh::Color(255,0,0), mesh::BoundaryInfo( BHFactory::getPressure0BoundaryUID() ) ); // colorToBoundryMapper.set( mesh::TriangleMesh::Color(255,0,0), mesh::BoundaryInfo(
// colorToBoundryMapper.set( mesh::TriangleMesh::Color(0,0,255), mesh::BoundaryInfo( BHFactory::getPressure1BoundaryUID() ) ); // BHFactory::getPressure0BoundaryUID() ) ); colorToBoundryMapper.set( mesh::TriangleMesh::Color(0,0,255),
// colorToBoundryMapper.set( mesh::TriangleMesh::Color(255,255,255), mesh::BoundaryInfo( BHFactory::getNoSlipBoundaryUID() ) ); // mesh::BoundaryInfo( BHFactory::getPressure1BoundaryUID() ) ); colorToBoundryMapper.set(
// mesh::TriangleMesh::Color(255,255,255), mesh::BoundaryInfo( BHFactory::getNoSlipBoundaryUID() ) );
auto boundaryLocations = colorToBoundryMapper.addBoundaryInfoToMesh( *mesh ); auto boundaryLocations = colorToBoundryMapper.addBoundaryInfoToMesh(*mesh);
mesh::VTKMeshWriter< mesh::TriangleMesh > meshWriter( mesh, "meshBoundaries", 1 ); mesh::VTKMeshWriter< mesh::TriangleMesh > meshWriter(mesh, "meshBoundaries", 1);
meshWriter.addDataSource( make_shared< mesh::BoundaryUIDFaceDataSource< mesh::TriangleMesh > >( boundaryLocations ) ); meshWriter.addDataSource(make_shared< mesh::BoundaryUIDFaceDataSource< mesh::TriangleMesh > >(boundaryLocations));
meshWriter.addDataSource( make_shared< mesh::ColorFaceDataSource< mesh::TriangleMesh > >() ); meshWriter.addDataSource(make_shared< mesh::ColorFaceDataSource< mesh::TriangleMesh > >());
meshWriter.addDataSource( make_shared< mesh::ColorVertexDataSource< mesh::TriangleMesh > >() ); meshWriter.addDataSource(make_shared< mesh::ColorVertexDataSource< mesh::TriangleMesh > >());
meshWriter(); meshWriter();
WALBERLA_LOG_DEVEL_ON_ROOT( "Voxelizing mesh" ); WALBERLA_LOG_DEVEL_ON_ROOT("Voxelizing mesh")
mesh::BoundarySetup boundarySetup( structuredBlockforest, makeMeshDistanceFunction( distanceOctree ), NUM_GHOSTLAYERS ); mesh::BoundarySetup boundarySetup(structuredBlockforest, makeMeshDistanceFunction(distanceOctree), NUM_GHOSTLAYERS);
//WALBERLA_LOG_DEVEL( "Writing Voxelisation" ); // WALBERLA_LOG_DEVEL( "Writing Voxelisation" );
//boundarySetup.writeVTKVoxelfile(); // boundarySetup.writeVTKVoxelfile();
WALBERLA_LOG_DEVEL_ON_ROOT( "Setting up fluid cells" ); WALBERLA_LOG_DEVEL_ON_ROOT("Setting up fluid cells")
boundarySetup.setDomainCells<BHFactory::BoundaryHandling>( boundaryHandlingId, mesh::BoundarySetup::OUTSIDE ); boundarySetup.setDomainCells< BHFactory::BoundaryHandling >(boundaryHandlingId, mesh::BoundarySetup::OUTSIDE);
WALBERLA_LOG_DEVEL_ON_ROOT( "Setting up boundaries" ); WALBERLA_LOG_DEVEL_ON_ROOT("Setting up boundaries")
boundarySetup.setBoundaries<BHFactory::BoundaryHandling>( boundaryHandlingId, makeBoundaryLocationFunction( distanceOctree, boundaryLocations ), mesh::BoundarySetup::INSIDE ); boundarySetup.setBoundaries< BHFactory::BoundaryHandling >(
WALBERLA_LOG_DEVEL_ON_ROOT( "done" ); boundaryHandlingId, makeBoundaryLocationFunction(distanceOctree, boundaryLocations), mesh::BoundarySetup::INSIDE);
WALBERLA_LOG_DEVEL_ON_ROOT("done")
lbm::BlockForestEvaluation<FlagField_T>( structuredBlockforest, flagFieldId, fluidFlagUID ).logInfoOnRoot(); lbm::BlockForestEvaluation< FlagField_T >(structuredBlockforest, flagFieldId, fluidFlagUID).logInfoOnRoot();
lbm::PerformanceLogger<FlagField_T> perfLogger( structuredBlockforest, flagFieldId, fluidFlagUID, 100 ); lbm::PerformanceLogger< FlagField_T > perfLogger(structuredBlockforest, flagFieldId, fluidFlagUID, 100);
SweepTimeloop timeloop( structuredBlockforest->getBlockStorage(), timeSteps ); SweepTimeloop timeloop(structuredBlockforest->getBlockStorage(), timeSteps);
auto sweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldId, flagFieldId, fluidFlagUID ); auto sweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >(pdfFieldId, flagFieldId, fluidFlagUID);
auto refinementTimeStep = lbm::refinement::makeTimeStep<LatticeModel_T, BHFactory::BoundaryHandling > ( structuredBlockforest, sweep, pdfFieldId, boundaryHandlingId ); auto refinementTimeStep = lbm::refinement::makeTimeStep< LatticeModel_T, BHFactory::BoundaryHandling >(
timeloop.addFuncBeforeTimeStep( makeSharedFunctor( refinementTimeStep ), "Refinement time step" ); structuredBlockforest, sweep, pdfFieldId, boundaryHandlingId);
timeloop.addFuncBeforeTimeStep(makeSharedFunctor(refinementTimeStep), "Refinement time step");
// log remaining time // log remaining time
timeloop.addFuncAfterTimeStep( timing::RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "remaining time logger" ); timeloop.addFuncAfterTimeStep(timing::RemainingTimeLogger(timeloop.getNrOfTimeSteps()), "remaining time logger");
// LBM stability check // LBM stability check
timeloop.addFuncAfterTimeStep( makeSharedFunctor( field::makeStabilityChecker< PdfField_T, FlagField_T >( env.config(), structuredBlockforest, pdfFieldId, timeloop.addFuncAfterTimeStep(makeSharedFunctor(field::makeStabilityChecker< PdfField_T, FlagField_T >(
flagFieldId, fluidFlagUID ) ), env.config(), structuredBlockforest, pdfFieldId, flagFieldId, fluidFlagUID)),
"LBM stability check" ); "LBM stability check");
timeloop.addFuncAfterTimeStep( perfLogger, "PerformanceLogger" );
timeloop.addFuncAfterTimeStep(perfLogger, "Evaluator: performance logging");
// add VTK output to time loop // add VTK output to time loop
lbm::VTKOutput< LatticeModel_T, FlagField_T >::addToTimeloop( timeloop, structuredBlockforest, env.config(), pdfFieldId, flagFieldId, fluidFlagUID ); lbm::VTKOutput< LatticeModel_T, FlagField_T >::addToTimeloop(timeloop, structuredBlockforest, env.config(),
pdfFieldId, flagFieldId, fluidFlagUID);
WcTimingPool timingPool; WcTimingPool timingPool;
WALBERLA_LOG_INFO_ON_ROOT( "Starting timeloop" ); WALBERLA_LOG_INFO_ON_ROOT("Starting timeloop")
timeloop.run( timingPool ); timeloop.run(timingPool);
WALBERLA_LOG_INFO_ON_ROOT( "Timeloop done" ); WALBERLA_LOG_INFO_ON_ROOT("Timeloop done")
timingPool.unifyRegisteredTimersAcrossProcesses(); timingPool.unifyRegisteredTimersAcrossProcesses();
timingPool.logResultOnRoot( WcTimingPool::REDUCE_TOTAL, true ); timingPool.logResultOnRoot(timing::REDUCE_TOTAL, true);
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
} // namespace walberla } // namespace walberla
int main( int argc, char * argv[] ) int main(int argc, char* argv[]) { return walberla::main(argc, argv); }
{
return walberla::main( argc, argv );
}
\ No newline at end of file
apps/benchmarks/ComplexGeometry/test.conf
View file @ 7a6d3b57
ComplexGeometry ComplexGeometry
{ {
meshFile cube.obj; meshFile bunny.obj;
coarseDx 0.1; coarseDx 4;
coarseOmega 1.6; coarseOmega 1.6;
coarseTimeSteps 1; coarseTimeSteps 1;
numLevels 2; numLevels 4;
bodyForce <0.0001, 0, 0>; bodyForce <0.0001, 0, 0>;
blockSize <16,16,16>; blockSize <16,16,16>;
domainBlowUp <5,5,5>; // simulation domain is blow up factor times mesh size per dimension domainBlowUp <5,5,5>; // simulation domain is blow up factor times mesh size per dimension
WriteDistanceOctree false;
WriteSetupForestAndReturn true;
logLevelDetail false;
Boundaries { Boundaries {
} }
......
apps/benchmarks/CouetteFlow/CMakeLists.txt
View file @ 7a6d3b57
waLBerla_link_files_to_builddir( "*.dat" ) waLBerla_link_files_to_builddir( "*.dat" )
waLBerla_add_executable( NAME CouetteFlow DEPENDS blockforest boundary core field lbm postprocessing stencil timeloop vtk ) waLBerla_add_executable( NAME CouetteFlow DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::field walberla::lbm walberla::postprocessing walberla::stencil walberla::timeloop walberla::vtk walberla::sqlite )
############## ##############
# Some tests # # Some tests #
############## ##############
waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTestNoCheckRelease COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/TestNoCheck.dat --trt --linear-exp PROCESSES 4 CONFIGURATIONS Release RelWithDbgInfo ) waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTestNoCheckRelease COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/TestNoCheck.dat --trt --linear-exp PROCESSES 4 CONFIGURATIONS Release RelWithDbgInfo DEPENDS_ON_TARGETS CouetteFlow )
waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTestNoCheckDebug COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/TestNoCheck.dat --trt --linear-exp PROCESSES 4 LABELS longrun CONFIGURATIONS Debug DebugOptimized ) waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTestNoCheckDebug COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/TestNoCheck.dat --trt --linear-exp PROCESSES 4 LABELS longrun CONFIGURATIONS Debug DebugOptimized DEPENDS_ON_TARGETS CouetteFlow )
waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTest0 COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/Test0.dat --trt --linear-exp LABELS longrun CONFIGURATIONS Release RelWithDbgInfo ) waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTest0 COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/Test0.dat --trt --linear-exp LABELS longrun CONFIGURATIONS Release RelWithDbgInfo DEPENDS_ON_TARGETS CouetteFlow )
waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTest2 COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/Test2.dat --trt --linear-exp LABELS longrun verylongrun PROCESSES 4 CONFIGURATIONS Release RelWithDbgInfo ) waLBerla_execute_test( NO_MODULE_LABEL NAME CouetteFlowTest2 COMMAND $<TARGET_FILE:CouetteFlow> ${CMAKE_CURRENT_SOURCE_DIR}/Test2.dat --trt --linear-exp LABELS longrun verylongrun PROCESSES 4 CONFIGURATIONS Release RelWithDbgInfo DEPENDS_ON_TARGETS CouetteFlow )
apps/benchmarks/CouetteFlow/CouetteFlow.cpp
View file @ 7a6d3b57
...@@ -79,7 +79,7 @@ ...@@ -79,7 +79,7 @@
#include "lbm/vtk/NonEquilibrium.h" #include "lbm/vtk/NonEquilibrium.h"
#include "lbm/vtk/Velocity.h" #include "lbm/vtk/Velocity.h"
#include "postprocessing/sqlite/SQLite.h" #include "sqlite/SQLite.h"
#include "stencil/D3Q15.h" #include "stencil/D3Q15.h"
#include "stencil/D3Q19.h" #include "stencil/D3Q19.h"
...@@ -91,9 +91,6 @@ ...@@ -91,9 +91,6 @@
#include "vtk/Initialization.h" #include "vtk/Initialization.h"
#include "vtk/VTKOutput.h" #include "vtk/VTKOutput.h"
#include <boost/mpl/or.hpp>
#include <boost/type_traits/is_same.hpp>
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <cstdlib> #include <cstdlib>
...@@ -101,6 +98,7 @@ ...@@ -101,6 +98,7 @@
#include <functional> #include <functional>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <type_traits>
#include <utility> #include <utility>
#include <vector> #include <vector>
...@@ -120,21 +118,21 @@ using walberla::real_t; ...@@ -120,21 +118,21 @@ using walberla::real_t;
// TYPEDEFS // // TYPEDEFS //
////////////// //////////////
typedef lbm::D3Q15< lbm::collision_model::SRT, false > D3Q15_SRT_INCOMP; using D3Q15_SRT_INCOMP = lbm::D3Q15<lbm::collision_model::SRT, false>;
typedef lbm::D3Q15< lbm::collision_model::SRT, true > D3Q15_SRT_COMP; using D3Q15_SRT_COMP = lbm::D3Q15<lbm::collision_model::SRT, true>;
typedef lbm::D3Q15< lbm::collision_model::TRT, false > D3Q15_TRT_INCOMP; using D3Q15_TRT_INCOMP = lbm::D3Q15<lbm::collision_model::TRT, false>;
typedef lbm::D3Q15< lbm::collision_model::TRT, true > D3Q15_TRT_COMP; using D3Q15_TRT_COMP = lbm::D3Q15<lbm::collision_model::TRT, true>;
typedef lbm::D3Q19< lbm::collision_model::SRT, false > D3Q19_SRT_INCOMP; using D3Q19_SRT_INCOMP = lbm::D3Q19<lbm::collision_model::SRT, false>;
typedef lbm::D3Q19< lbm::collision_model::SRT, true > D3Q19_SRT_COMP; using D3Q19_SRT_COMP = lbm::D3Q19<lbm::collision_model::SRT, true>;
typedef lbm::D3Q19< lbm::collision_model::TRT, false > D3Q19_TRT_INCOMP; using D3Q19_TRT_INCOMP = lbm::D3Q19<lbm::collision_model::TRT, false>;
typedef lbm::D3Q19< lbm::collision_model::TRT, true > D3Q19_TRT_COMP; using D3Q19_TRT_COMP = lbm::D3Q19<lbm::collision_model::TRT, true>;
typedef lbm::D3Q19< lbm::collision_model::D3Q19MRT, false > D3Q19_MRT_INCOMP; using D3Q19_MRT_INCOMP = lbm::D3Q19<lbm::collision_model::D3Q19MRT, false>;
typedef lbm::D3Q27< lbm::collision_model::SRT, false > D3Q27_SRT_INCOMP; using D3Q27_SRT_INCOMP = lbm::D3Q27<lbm::collision_model::SRT, false>;
typedef lbm::D3Q27< lbm::collision_model::SRT, true > D3Q27_SRT_COMP; using D3Q27_SRT_COMP = lbm::D3Q27<lbm::collision_model::SRT, true>;
typedef lbm::D3Q27< lbm::collision_model::TRT, false > D3Q27_TRT_INCOMP; using D3Q27_TRT_INCOMP = lbm::D3Q27<lbm::collision_model::TRT, false>;
typedef lbm::D3Q27< lbm::collision_model::TRT, true > D3Q27_TRT_COMP; using D3Q27_TRT_COMP = lbm::D3Q27<lbm::collision_model::TRT, true>;
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct Types struct Types
...@@ -164,19 +162,19 @@ template< typename LatticeModel_T, class Enable = void > ...@@ -164,19 +162,19 @@ template< typename LatticeModel_T, class Enable = void >
struct StencilString; struct StencilString;
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct StencilString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::Stencil, stencil::D3Q15 >::value >::type > struct StencilString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::Stencil, stencil::D3Q15 > > >
{ {
static const char * str() { return "D3Q15"; } static const char * str() { return "D3Q15"; }
}; };
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct StencilString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::Stencil, stencil::D3Q19 >::value >::type > struct StencilString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::Stencil, stencil::D3Q19 > > >
{ {
static const char * str() { return "D3Q19"; } static const char * str() { return "D3Q19"; }
}; };
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct StencilString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::Stencil, stencil::D3Q27 >::value >::type > struct StencilString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::Stencil, stencil::D3Q27 > > >
{ {
static const char * str() { return "D3Q27"; } static const char * str() { return "D3Q27"; }
}; };
...@@ -186,22 +184,22 @@ template< typename LatticeModel_T, class Enable = void > ...@@ -186,22 +184,22 @@ template< typename LatticeModel_T, class Enable = void >
struct CollisionModelString; struct CollisionModelString;
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct CollisionModelString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::CollisionModel::tag, struct CollisionModelString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::CollisionModel::tag,
lbm::collision_model::SRT_tag >::value >::type > lbm::collision_model::SRT_tag > > >
{ {
static const char * str() { return "SRT"; } static const char * str() { return "SRT"; }
}; };
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct CollisionModelString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::CollisionModel::tag, struct CollisionModelString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::CollisionModel::tag,
lbm::collision_model::TRT_tag >::value >::type > lbm::collision_model::TRT_tag > > >
{ {
static const char * str() { return "TRT"; } static const char * str() { return "TRT"; }
}; };
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct CollisionModelString< LatticeModel_T, typename boost::enable_if_c< boost::is_same< typename LatticeModel_T::CollisionModel::tag, struct CollisionModelString< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::CollisionModel::tag,
lbm::collision_model::MRT_tag >::value >::type > lbm::collision_model::MRT_tag > > >
{ {
static const char * str() { return "MRT"; } static const char * str() { return "MRT"; }
}; };
...@@ -273,10 +271,10 @@ private: ...@@ -273,10 +271,10 @@ private:
static void workloadAndMemoryAssignment( SetupBlockForest& forest, const memory_t memoryPerBlock ) static void workloadAndMemoryAssignment( SetupBlockForest& forest, const memory_t memoryPerBlock )
{ {
for( auto block = forest.begin(); block != forest.end(); ++block ) for(auto & block : forest)
{ {
block->setWorkload( numeric_cast< workload_t >( uint_t(1) << block->getLevel() ) ); block.setWorkload( numeric_cast< workload_t >( uint_t(1) << block.getLevel() ) );
block->setMemory( memoryPerBlock ); block.setMemory( memoryPerBlock );
} }
} }
...@@ -295,7 +293,7 @@ static shared_ptr< SetupBlockForest > createSetupBlockForest( const blockforest: ...@@ -295,7 +293,7 @@ static shared_ptr< SetupBlockForest > createSetupBlockForest( const blockforest:
( setup.zCells + uint_t(2) * FieldGhostLayers ) ) * memoryPerCell; ( setup.zCells + uint_t(2) * FieldGhostLayers ) ) * memoryPerCell;
forest->addRefinementSelectionFunction( refinementSelectionFunctions ); forest->addRefinementSelectionFunction( refinementSelectionFunctions );
forest->addWorkloadMemorySUIDAssignmentFunction( std::bind( workloadAndMemoryAssignment, std::placeholders::_1, memoryPerBlock ) ); forest->addWorkloadMemorySUIDAssignmentFunction([memoryPerBlock](auto & sbf) { workloadAndMemoryAssignment(sbf, memoryPerBlock); });
forest->init( AABB( real_c(0), real_c(0), real_c(0), real_c( setup.xBlocks * setup.xCells ), forest->init( AABB( real_c(0), real_c(0), real_c(0), real_c( setup.xBlocks * setup.xCells ),
real_c( setup.yBlocks * setup.yCells ), real_c( setup.yBlocks * setup.yCells ),
...@@ -372,12 +370,10 @@ class MyBoundaryHandling ...@@ -372,12 +370,10 @@ class MyBoundaryHandling
{ {
public: public:
typedef lbm::NoSlip< LatticeModel_T, flag_t > NoSlip_T; using NoSlip_T = lbm::NoSlip<LatticeModel_T, flag_t>;
typedef lbm::SimpleUBB< LatticeModel_T, flag_t > UBB_T; using UBB_T = lbm::SimpleUBB<LatticeModel_T, flag_t>;
typedef boost::tuples::tuple< NoSlip_T, UBB_T > BoundaryConditions_T;
typedef BoundaryHandling< FlagField_T, typename Types<LatticeModel_T>::Stencil_T, BoundaryConditions_T > BoundaryHandling_T; using BoundaryHandling_T = BoundaryHandling<FlagField_T, typename Types<LatticeModel_T>::Stencil_T, NoSlip_T, UBB_T>;
...@@ -407,8 +403,8 @@ MyBoundaryHandling<LatticeModel_T>::operator()( IBlock * const block ) const ...@@ -407,8 +403,8 @@ MyBoundaryHandling<LatticeModel_T>::operator()( IBlock * const block ) const
const flag_t fluid = flagField->registerFlag( Fluid_Flag ); const flag_t fluid = flagField->registerFlag( Fluid_Flag );
return new BoundaryHandling_T( "boundary handling", flagField, fluid, return new BoundaryHandling_T( "boundary handling", flagField, fluid,
boost::tuples::make_tuple( NoSlip_T( "no slip", NoSlip_Flag, pdfField ), NoSlip_T( "no slip", NoSlip_Flag, pdfField ),
UBB_T( "velocity bounce back", UBB_Flag, pdfField, topVelocity_, real_t(0), real_t(0) ) ) ); UBB_T( "velocity bounce back", UBB_Flag, pdfField, topVelocity_, real_t(0), real_t(0) ) );
} }
...@@ -620,7 +616,7 @@ struct AddRefinementTimeStep ...@@ -620,7 +616,7 @@ struct AddRefinementTimeStep
} }
else else
{ {
typedef lbm::SplitSweep< LatticeModel_T, FlagField_T > Sweep_T; using Sweep_T = lbm::SplitSweep<LatticeModel_T, FlagField_T>;
auto mySweep = make_shared< Sweep_T >( pdfFieldId, flagFieldId, Fluid_Flag ); auto mySweep = make_shared< Sweep_T >( pdfFieldId, flagFieldId, Fluid_Flag );
addRefinementTimeStep< LatticeModel_T, Sweep_T >( timeloop, blocks, pdfFieldId, boundaryHandlingId, timingPool, levelwiseTimingPool, addRefinementTimeStep< LatticeModel_T, Sweep_T >( timeloop, blocks, pdfFieldId, boundaryHandlingId, timingPool, levelwiseTimingPool,
...@@ -638,10 +634,10 @@ struct AddRefinementTimeStep ...@@ -638,10 +634,10 @@ struct AddRefinementTimeStep
}; };
template< typename LatticeModel_T > template< typename LatticeModel_T >
struct AddRefinementTimeStep< LatticeModel_T, typename boost::enable_if< boost::mpl::or_< boost::is_same< typename LatticeModel_T::CollisionModel::tag, struct AddRefinementTimeStep< LatticeModel_T, std::enable_if_t< std::is_same_v< typename LatticeModel_T::CollisionModel::tag, lbm::collision_model::MRT_tag > ||
lbm::collision_model::MRT_tag >, std::is_same_v< typename LatticeModel_T::Stencil, stencil::D3Q15 > ||
boost::is_same< typename LatticeModel_T::Stencil, stencil::D3Q15 >, std::is_same_v< typename LatticeModel_T::Stencil, stencil::D3Q27 >
boost::is_same< typename LatticeModel_T::Stencil, stencil::D3Q27 > > >::type > > >
{ {
static void add( SweepTimeloop & timeloop, shared_ptr< blockforest::StructuredBlockForest > & blocks, static void add( SweepTimeloop & timeloop, shared_ptr< blockforest::StructuredBlockForest > & blocks,
const BlockDataID & pdfFieldId, const BlockDataID & flagFieldId, const BlockDataID & boundaryHandlingId, const BlockDataID & pdfFieldId, const BlockDataID & flagFieldId, const BlockDataID & boundaryHandlingId,
...@@ -722,9 +718,9 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod ...@@ -722,9 +718,9 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod
if( vtkBeforeTimeStep ) if( vtkBeforeTimeStep )
{ {
for( auto output = vtkOutputFunctions.begin(); output != vtkOutputFunctions.end(); ++output ) for(auto & output : vtkOutputFunctions)
timeloop.addFuncBeforeTimeStep( output->second.outputFunction, std::string("VTK: ") + output->first, timeloop.addFuncBeforeTimeStep( output.second.outputFunction, std::string("VTK: ") + output.first,
output->second.requiredGlobalStates, output->second.incompatibleGlobalStates ); output.second.requiredGlobalStates, output.second.incompatibleGlobalStates );
} }
// add 'refinement' LB time step to time loop // add 'refinement' LB time step to time loop
...@@ -737,11 +733,11 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod ...@@ -737,11 +733,11 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod
// evaluation // evaluation
const auto exactSolutionFunction = std::bind( exactVelocity, std::placeholders::_1, blocks->getDomain(), setup.maxVelocity_L ); const auto exactSolutionFunction = [domain = blocks->getDomain(), maxVel = setup.maxVelocity_L](auto & p) { return exactVelocity(p, domain, maxVel); };
auto volumetricFlowRate = field::makeVolumetricFlowRateEvaluation< VelocityAdaptor_T, FlagField_T >( configBlock, blocks, velocityAdaptorId, auto volumetricFlowRate = field::makeVolumetricFlowRateEvaluation< VelocityAdaptor_T, FlagField_T >( configBlock, blocks, velocityAdaptorId,
flagFieldId, Fluid_Flag, flagFieldId, Fluid_Flag,
std::bind( exactFlowRate, setup.flowRate_L ), [flowRate = setup.flowRate_L] { return exactFlowRate(flowRate); },
exactSolutionFunction ); exactSolutionFunction );
volumetricFlowRate->setNormalizationFactor( real_t(1) / setup.maxVelocity_L ); volumetricFlowRate->setNormalizationFactor( real_t(1) / setup.maxVelocity_L );
volumetricFlowRate->setDomainNormalization( Vector3<real_t>( real_t(1) ) ); volumetricFlowRate->setDomainNormalization( Vector3<real_t>( real_t(1) ) );
...@@ -770,14 +766,14 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod ...@@ -770,14 +766,14 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod
if( !vtkBeforeTimeStep ) if( !vtkBeforeTimeStep )
{ {
for( auto output = vtkOutputFunctions.begin(); output != vtkOutputFunctions.end(); ++output ) for(auto & output : vtkOutputFunctions)
timeloop.addFuncAfterTimeStep( output->second.outputFunction, std::string("VTK: ") + output->first, timeloop.addFuncAfterTimeStep( output.second.outputFunction, std::string("VTK: ") + output.first,
output->second.requiredGlobalStates, output->second.incompatibleGlobalStates ); output.second.requiredGlobalStates, output.second.incompatibleGlobalStates );
} }
// remaining time logger // remaining time logger
const double remainingTimeLoggerFrequency = configBlock.getParameter< double >( "remainingTimeLoggerFrequency", 3.0 ); const real_t remainingTimeLoggerFrequency = configBlock.getParameter< real_t >( "remainingTimeLoggerFrequency", real_c(3.0) );
timeloop.addFuncAfterTimeStep( timing::RemainingTimeLogger( timeloop.getNrOfTimeSteps(), remainingTimeLoggerFrequency ), "Remaining time logger" ); timeloop.addFuncAfterTimeStep( timing::RemainingTimeLogger( timeloop.getNrOfTimeSteps(), remainingTimeLoggerFrequency ), "Remaining time logger" );
// logging right before the simulation starts // logging right before the simulation starts
...@@ -875,10 +871,10 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod ...@@ -875,10 +871,10 @@ void run( const shared_ptr< Config > & config, const LatticeModel_T & latticeMod
realProperties[ "simulationProgress" ] = double_c( ( outerRun + uint_t(1) ) * innerTimeSteps ) / double_c( outerTimeSteps * innerTimeSteps ); realProperties[ "simulationProgress" ] = double_c( ( outerRun + uint_t(1) ) * innerTimeSteps ) / double_c( outerTimeSteps * innerTimeSteps );
auto runId = postprocessing::storeRunInSqliteDB( sqlFile, integerProperties, stringProperties, realProperties ); auto runId = sqlite::storeRunInSqliteDB( sqlFile, integerProperties, stringProperties, realProperties );
postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedTimeloopTiming, "Timeloop" ); sqlite::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedTimeloopTiming, "Timeloop" );
postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedRTSTiming, "RefinementTimeStep" ); sqlite::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedRTSTiming, "RefinementTimeStep" );
postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedRTSLTiming, "RefinementTimeStepLevelwise" ); sqlite::storeTimingPoolInSqliteDB( sqlFile, runId, *reducedRTSLTiming, "RefinementTimeStepLevelwise" );
} }
} }
} }
......
apps/benchmarks/DEM/CMakeLists.txt
View file @ 7a6d3b57
waLBerla_add_executable( NAME DEM FILES DEM.cpp DEPENDS blockforest core pe ) waLBerla_add_executable( NAME DEM FILES DEM.cpp DEPENDS walberla::blockforest walberla::core walberla::pe )
apps/benchmarks/DEM/DEM.cpp
View file @ 7a6d3b57
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#include <core/DataTypes.h> #include <core/DataTypes.h>
#include <string> #include <string>
#include <tuple>
namespace walberla { namespace walberla {
...@@ -32,13 +33,13 @@ namespace dem { ...@@ -32,13 +33,13 @@ namespace dem {
real_t calcCoefficientOfRestitution(const real_t k, const real_t gamma, const real_t meff) real_t calcCoefficientOfRestitution(const real_t k, const real_t gamma, const real_t meff)
{ {
auto a = real_t(0.5) * gamma / meff; auto a = real_t(0.5) * gamma / meff;
return std::exp(-a * math::PI / std::sqrt(k / meff - a*a)); return std::exp(-a * math::pi / std::sqrt(k / meff - a*a));
} }
real_t calcCollisionTime(const real_t k, const real_t gamma, const real_t meff) real_t calcCollisionTime(const real_t k, const real_t gamma, const real_t meff)
{ {
auto a = real_t(0.5) * gamma / meff; auto a = real_t(0.5) * gamma / meff;
return math::PI / std::sqrt( k/meff - a*a); return math::pi / std::sqrt( k/meff - a*a);
} }
} }
...@@ -47,7 +48,7 @@ int main( int argc, char** argv ) ...@@ -47,7 +48,7 @@ int main( int argc, char** argv )
using namespace walberla; using namespace walberla;
using namespace walberla::pe; using namespace walberla::pe;
typedef boost::tuple<Sphere, Plane> BodyTuple ; using BodyTuple = std::tuple<Sphere, Plane> ;
walberla::MPIManager::instance()->initializeMPI( &argc, &argv ); walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
......
apps/benchmarks/FieldCommunication/CMakeLists.txt 0 → 100644
View file @ 7a6d3b57
waLBerla_link_files_to_builddir( "*.dat" )
waLBerla_link_files_to_builddir( "*.py" )
waLBerla_add_executable ( NAME FieldCommunication
DEPENDS walberla::blockforest walberla::core walberla::domain_decomposition walberla::field walberla::postprocessing walberla::sqlite walberla::python_coupling )
apps/benchmarks/FieldCommunication/FieldCommunication.cpp 0 → 100644
View file @ 7a6d3b57
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include "blockforest/communication/UniformDirectScheme.h"
#include "core/mpi/MPIManager.h"
#include "core/Environment.h"
#include "core/OpenMP.h"
#include "core/mpi/Broadcast.h"
#include "core/math/IntegerFactorization.h"
#include "core/timing/TimingPool.h"
#include "core/waLBerlaBuildInfo.h"
#include "field/communication/StencilRestrictedMPIDatatypeInfo.h"
#include "field/AddToStorage.h"
#include "field/GhostLayerField.h"
#include "field/communication/PackInfo.h"
#include "field/communication/StencilRestrictedPackInfo.h"
#include "field/communication/UniformMPIDatatypeInfo.h"
#include "sqlite/SQLite.h"
#include "python_coupling/CreateConfig.h"
#include "stencil/D3Q7.h"
#include "stencil/D3Q19.h"
#include "stencil/D3Q27.h"
#include <functional>
using namespace walberla;
using blockforest::communication::UniformDirectScheme;
using blockforest::communication::UniformBufferedScheme;
using field::communication::UniformMPIDatatypeInfo;
using field::communication::PackInfo;
using field::communication::StencilRestrictedPackInfo;
using field::communication::StencilRestrictedMPIDatatypeInfo;
template<typename Stencil_T>
class SingleMessageBufferedScheme
{
public:
using Stencil = Stencil_T;
SingleMessageBufferedScheme( const weak_ptr< StructuredBlockForest > & bf, const int tag = 17953 )
: blockForest_( bf ), tag_( tag ) {}
inline void addDataToCommunicate( const shared_ptr< communication::UniformPackInfo > &packInfo )
{
tag_ += 1;
auto newScheme = make_shared< UniformBufferedScheme< Stencil > >( blockForest_, tag_++ );
newScheme->addDataToCommunicate( packInfo );
schemes_.push_back( newScheme );
}
inline void setLocalMode( const blockforest::LocalCommunicationMode &mode )
{
for ( auto &s : schemes_ )
s->setLocalMOde( mode );
}
inline void startCommunication()
{
for ( auto &s : schemes_ )
s->startCommunication();
}
inline void wait()
{
for ( auto &s : schemes_ )
s->wait();
}
private:
std::vector< shared_ptr< UniformBufferedScheme< Stencil>> > schemes_;
weak_ptr< StructuredBlockForest > blockForest_;
int tag_;
};
template<typename FieldType, typename Stencil>
void addDataToCommunicate( const shared_ptr< UniformDirectScheme< Stencil > > &scheme, BlockDataID id, uint_t ghostLayers )
{
scheme->addDataToCommunicate( make_shared< UniformMPIDatatypeInfo< FieldType > >( id, ghostLayers ));
}
template<typename FieldType, typename Scheme_T>
void addDataToCommunicate( const shared_ptr< Scheme_T > &scheme, BlockDataID id, uint_t ghostLayers )
{
scheme->addDataToCommunicate( make_shared< PackInfo< FieldType > >( id, ghostLayers ));
}
template<typename FieldType, typename Scheme_T>
void addDataToCommunicate( const shared_ptr< Scheme_T > &scheme, BlockDataID id, uint_t ghostLayers, bool )
{
if ( ghostLayers != 1 )
scheme->addDataToCommunicate( make_shared< PackInfo< FieldType > >( id, ghostLayers ));
else
scheme->addDataToCommunicate( make_shared< StencilRestrictedPackInfo< FieldType, typename Scheme_T::Stencil > >( id ));
}
template<typename FieldType, typename Stencil_T>
void addDataToCommunicate( const shared_ptr< UniformDirectScheme< Stencil_T > > &scheme, BlockDataID id, uint_t ghostLayers, bool )
{
if ( ghostLayers != 1 )
scheme->addDataToCommunicate( make_shared< UniformMPIDatatypeInfo< FieldType > >( id, ghostLayers ));
else
{
scheme->addDataToCommunicate( make_shared< StencilRestrictedMPIDatatypeInfo< FieldType, Stencil_T > >( id ));
}
}
template<typename Scheme1, typename Scheme2>
void addData( const shared_ptr< StructuredBlockForest > &blocks, const config::Config::BlockHandle &configBlock,
const shared_ptr< Scheme1 > &scheme1, const shared_ptr< Scheme2 > &scheme2,
uint_t ghostLayers, field::Layout layout )
{
auto numPdfFields = configBlock.getParameter< uint_t >( "pdf" );
for ( uint_t i = 0; i < numPdfFields; ++i )
{
typedef field::GhostLayerField< real_t, Scheme1::Stencil::Q > Field_T;
BlockDataID bdId = field::addToStorage< Field_T >( blocks, "pdf", 0.0, layout, ghostLayers );
addDataToCommunicate< Field_T >( scheme1, bdId, ghostLayers );
addDataToCommunicate< Field_T >( scheme2, bdId, ghostLayers );
}
auto numPdfOptFields = configBlock.getParameter< uint_t >( "pdfOpt" );
for ( uint_t i = 0; i < numPdfOptFields; ++i )
{
typedef field::GhostLayerField< real_t, Scheme1::Stencil::Q > Field_T;
BlockDataID bdId = field::addToStorage< Field_T >( blocks, "pdfopt", 0.0, layout, ghostLayers );
addDataToCommunicate< Field_T >( scheme1, bdId, ghostLayers, true );
addDataToCommunicate< Field_T >( scheme2, bdId, ghostLayers, true );
}
auto numVectorFields = configBlock.getParameter< uint_t >( "vector" );
for ( uint_t i = 0; i < numVectorFields; ++i )
{
typedef field::GhostLayerField< real_t, 3 > Field_T;
BlockDataID bdId = field::addToStorage< Field_T >( blocks, "vector", 0.0, layout, ghostLayers );
addDataToCommunicate< Field_T >( scheme1, bdId, ghostLayers );
addDataToCommunicate< Field_T >( scheme2, bdId, ghostLayers );
}
auto numScalarFields = configBlock.getParameter< uint_t >( "scalar" );
for ( uint_t i = 0; i < numScalarFields; ++i )
{
typedef field::GhostLayerField< real_t, 1 > Field_T;
BlockDataID bdId = field::addToStorage< Field_T >( blocks, "scalar", 0.0, layout, ghostLayers );
addDataToCommunicate< Field_T >( scheme1, bdId, ghostLayers );
addDataToCommunicate< Field_T >( scheme2, bdId, ghostLayers );
}
}
template<typename Stencil>
void createCommunication( const shared_ptr< StructuredBlockForest > &blocks,
bool buffered, const config::Config::BlockHandle &fieldCfg, uint_t ghostLayers, field::Layout layout,
blockforest::LocalCommunicationMode localCommunicationMode, bool singleMessage,
std::function< void() > &commStart, std::function< void() > &commWait )
{
auto directScheme = make_shared< UniformDirectScheme< Stencil > >( blocks, shared_ptr< communication::UniformMPIDatatypeInfo >(), 42 );
auto bufferedScheme = make_shared< UniformBufferedScheme< Stencil > >( blocks, 4242 );
auto bufferedSchemeSingle = make_shared< SingleMessageBufferedScheme< Stencil > >( blocks , 24242);
bufferedScheme->setLocalMode( localCommunicationMode );
if ( buffered )
{
if ( !singleMessage )
{
addData( blocks, fieldCfg, directScheme, bufferedScheme, ghostLayers, layout );
commStart = [=]() { bufferedScheme->startCommunication(); };
commWait = [=]() { bufferedScheme->wait(); };
}
else
{
addData( blocks, fieldCfg, directScheme, bufferedSchemeSingle, ghostLayers, layout );
commStart = [=]() { bufferedSchemeSingle->startCommunication(); };
commWait = [=]() { bufferedSchemeSingle->wait(); };
}
}
else
{
addData( blocks, fieldCfg, directScheme, bufferedScheme, ghostLayers, layout );
commStart = [=]() { directScheme->startCommunication(); };
commWait = [=]() { directScheme->wait(); };
}
}
std::string fromEnv( const char *envVar )
{
auto env = std::getenv( envVar );
return env != nullptr ? std::string( env ) : "";
}
int main( int argc, char **argv )
{
mpi::Environment env( argc, argv );
int scenarioNr = 0;
auto mpiManager = mpi::MPIManager::instance();
for ( auto cfg = python_coupling::configBegin( argc, argv ); cfg != python_coupling::configEnd(); ++cfg )
{
if ( mpiManager->isMPIInitialized())
mpiManager->resetMPI();
auto config = *cfg;
auto commCfg = config->getOneBlock( "Communication" );
auto domainCfg = config->getOneBlock( "Domain" );
bool cartesianCommunicator = commCfg.getParameter< bool >( "cartesianCommunicator", true );
if ( !cartesianCommunicator )
mpiManager->useWorldComm();
scenarioNr += 1;
WALBERLA_LOG_INFO_ON_ROOT( "Simulating scenario " << scenarioNr );
WALBERLA_LOG_INFO_ON_ROOT( *config );
// ---- Domain Setup ----
const Vector3< uint_t > cellsPerBlock = domainCfg.getParameter< Vector3< uint_t > >( "cellsPerBlock" );
const Vector3< real_t > domainWeights = domainCfg.getParameter< Vector3< real_t > >( "domainWeights", Vector3< real_t >( 1.0, 1.0, 1.0 ));
uint_t blocksPerProcess = domainCfg.getParameter< uint_t >( "blocksPerProcess", 1 );
auto numProcesses = mpiManager->numProcesses();
auto processes = math::getFactors3D( uint_c( numProcesses ), domainWeights );
auto blockDecomposition = math::getFactors3D( uint_c( numProcesses ) * blocksPerProcess, domainWeights );
auto aabb = AABB( real_t( 0 ), real_t( 0 ), real_t( 0 ),
real_c( cellsPerBlock[0] * processes[0] * blocksPerProcess ),
real_c( cellsPerBlock[1] * processes[1] * blocksPerProcess ),
real_c( cellsPerBlock[2] * processes[2] * blocksPerProcess ));
auto blocks = blockforest::createUniformBlockGrid( aabb,
blockDecomposition[0], blockDecomposition[1], blockDecomposition[2],
cellsPerBlock[0], cellsPerBlock[1], cellsPerBlock[2],
processes[0], processes[1], processes[2],
true, true, true, //periodicity
false // keepGlobalBlockInformation
);
// ---- Communication Setup ----
auto fieldCfg = commCfg.getOneBlock( "Fields" );
const bool buffered = commCfg.getParameter< bool >( "buffered", true );
const std::string stencil = commCfg.getParameter< std::string >( "stencil", "D3Q19" );
const uint_t ghostLayers = commCfg.getParameter< uint_t >( "ghostLayers", 1 );
const std::string layoutStr = commCfg.getParameter< std::string >( "layout", "fzyx" );
const std::string localCommModeStr = commCfg.getParameter< std::string >( "localCommunicationMode", "start" );
const bool singleMessage = commCfg.getParameter< bool >( "singleMessage", false );
blockforest::LocalCommunicationMode localCommunicationMode;
if ( localCommModeStr == "start" )
localCommunicationMode = blockforest::START;
else if ( localCommModeStr == "wait" )
localCommunicationMode = blockforest::WAIT;
else if ( localCommModeStr == "buffer" )
localCommunicationMode = blockforest::BUFFER;
else if ( localCommModeStr == "noOptimization" )
localCommunicationMode = blockforest::NO_OPTIMIZATION;
else
{
WALBERLA_ABORT_NO_DEBUG_INFO( "Unknown localCommunicationMode " << layoutStr << ". Valid values are start, wait, buffer and noOptimization" )
}
field::Layout layout;
if ( layoutStr == "fzyx" )
layout = field::fzyx;
else if ( layoutStr == "zyxf" )
layout = field::zyxf;
else
{
WALBERLA_ABORT_NO_DEBUG_INFO( "Unknown layout string " << layoutStr << ". Valid values are fzyx and zyxf." )
}
std::function< void() > commStart;
std::function< void() > commWait;
if ( stencil == "D3Q19" )
createCommunication< stencil::D3Q19 >( blocks, buffered, fieldCfg, ghostLayers, layout, localCommunicationMode, singleMessage, commStart,
commWait );
else if ( stencil == "D3Q27" )
createCommunication< stencil::D3Q27 >( blocks, buffered, fieldCfg, ghostLayers, layout, localCommunicationMode, singleMessage, commStart,
commWait );
else if ( stencil == "D3Q7" )
createCommunication< stencil::D3Q7 >( blocks, buffered, fieldCfg, ghostLayers, layout, localCommunicationMode, singleMessage, commStart, commWait );
else
{
WALBERLA_ABORT_NO_DEBUG_INFO( "Unknown stencil " << stencil << ". Has to be one of D3Q7, D3Q19, D3Q27." )
}
// ---- Timing ----
auto runCfg = config->getOneBlock( "Run" );
const uint_t warmupIterations = runCfg.getParameter< uint_t >( "warmupIterations", 2 );
uint_t iterations = runCfg.getParameter< uint_t >( "iterations", 10 );
const uint_t minIterations = runCfg.getParameter< uint_t >( "minIterations", 2 );
const uint_t maxIterations = runCfg.getParameter< uint_t >( "maxIterations", 100 );
const real_t timeForBenchmark = runCfg.getParameter< real_t >( "timeForBenchmark", real_t(-1.0) );
const uint_t outerIterations = runCfg.getParameter< uint_t >( "outerIterations", 2 );
const std::string databaseFile = runCfg.getParameter< std::string >( "databaseFile", "FieldCommunication.sqlite" );
commStart();
commWait();
WcTimer warmupTimer;
warmupTimer.start();
for ( uint_t warmupCounter = 0; warmupCounter < warmupIterations; ++warmupCounter )
{
commStart();
commWait();
}
warmupTimer.end();
auto estimatedTimePerIteration = warmupTimer.last() / real_c(warmupIterations);
if( timeForBenchmark > 0 ) {
iterations = uint_c( timeForBenchmark / estimatedTimePerIteration );
if( iterations < minIterations )
iterations = minIterations;
if( iterations > maxIterations)
iterations = maxIterations;
}
mpi::broadcastObject(iterations);
WcTimingPool timingPool;
WALBERLA_MPI_BARRIER();
WALBERLA_LOG_INFO_ON_ROOT("Running " << outerIterations << " outer iterations of size " << iterations );
for ( uint_t outerCtr = 0; outerCtr < outerIterations; ++outerCtr )
{
timingPool["totalTime"].start();
for ( uint_t ctr = 0; ctr < iterations; ++ctr )
{
timingPool["commStart"].start();
commStart();
timingPool["commStart"].end();
timingPool["commWait"].start();
commWait();
timingPool["commWait"].end();
}
timingPool["totalTime"].end();
}
auto numThreads = omp_get_max_threads();
auto reducedTimingPool = timingPool.getReduced( timing::REDUCE_TOTAL, 0 );
WALBERLA_ROOT_SECTION()
{
WALBERLA_LOG_RESULT( *reducedTimingPool );
std::map< std::string, walberla::int64_t > integerProperties;
std::map< std::string, double > realProperties;
std::map< std::string, std::string > stringProperties;
auto databaseBlock = config->getBlock( "Database" );
if ( databaseBlock )
{
for (const auto & it : databaseBlock)
stringProperties[it.first] = it.second;
}
realProperties["total_min"] = real_c( timingPool["totalTime"].min()) / real_c( iterations );
realProperties["total_avg"] = real_c( timingPool["totalTime"].average() / real_c( iterations ));
realProperties["total_max"] = real_c( timingPool["totalTime"].max() / real_c( iterations ));
integerProperties["cellsPerBlock0"] = int64_c( cellsPerBlock[0] );
integerProperties["cellsPerBlock1"] = int64_c( cellsPerBlock[1] );
integerProperties["cellsPerBlock2"] = int64_c( cellsPerBlock[2] );
integerProperties["processes0"] = int64_c( processes[0] );
integerProperties["processes1"] = int64_c( processes[1] );
integerProperties["processes2"] = int64_c( processes[2] );
integerProperties["blocks0"] = int64_c( blockDecomposition[0] );
integerProperties["blocks1"] = int64_c( blockDecomposition[1] );
integerProperties["blocks2"] = int64_c( blockDecomposition[2] );
integerProperties["blocksPerProcess"] = int64_c( blocksPerProcess );
integerProperties["ghostLayers"] = int64_c( ghostLayers );
integerProperties["fieldsPdf"] = fieldCfg.getParameter< int64_t >( "pdf" );
integerProperties["fieldsPdfOpt"] = fieldCfg.getParameter< int64_t >( "pdfOpt" );
integerProperties["fieldsVector"] = fieldCfg.getParameter< int64_t >( "vector" );
integerProperties["fieldsScalar"] = fieldCfg.getParameter< int64_t >( "scalar" );
integerProperties["numThreads"] = int64_c( numThreads );
integerProperties["cartesianCommunicator"] = mpiManager->hasCartesianSetup();
integerProperties["warmupIterations"] = int64_c( warmupIterations );
integerProperties["iterations"] = int64_c( iterations );
integerProperties["outerIterations"] = int64_c( outerIterations );
integerProperties["buffered"] = int64_c( buffered );
integerProperties["singleMessage"] = int64_c( singleMessage );
stringProperties["stencil"] = stencil;
stringProperties["layout"] = layoutStr;
stringProperties["localCommunicationMode"] = localCommModeStr;
stringProperties["SLURM_CLUSTER_NAME"] = fromEnv( "SLURM_CLUSTER_NAME" );
stringProperties["SLURM_CPUS_ON_NODE"] = fromEnv( "SLURM_CPUS_ON_NODE" );
stringProperties["SLURM_CPUS_PER_TASK"] = fromEnv( "SLURM_CPUS_PER_TASK" );
stringProperties["SLURM_JOB_ACCOUNT"] = fromEnv( "SLURM_JOB_ACCOUNT" );
stringProperties["SLURM_JOB_ID"] = fromEnv( "SLURM_JOB_ID" );
stringProperties["SLURM_JOB_CPUS_PER_NODE"] = fromEnv( "SLURM_JOB_CPUS_PER_NODE" );
stringProperties["SLURM_JOB_NAME"] = fromEnv( "SLURM_JOB_NAME" );
stringProperties["SLURM_JOB_NUM_NODES"] = fromEnv( "SLURM_JOB_NUM_NODES" );
stringProperties["SLURM_NTASKS"] = fromEnv( "SLURM_NTASKS" );
stringProperties["SLURM_NTASKS_PER_CORE"] = fromEnv( "SLURM_NTASKS_PER_CORE" );
stringProperties["SLURM_NTASKS_PER_NODE"] = fromEnv( "SLURM_NTASKS_PER_NODE" );
stringProperties["SLURM_NTASKS_PER_SOCKET"] = fromEnv( "SLURM_NTASKS_PER_SOCKET" );
stringProperties["SLURM_TASKS_PER_NODE"] = fromEnv( "SLURM_TASKS_PER_NODE" );
stringProperties["buildMachine"] = std::string( WALBERLA_BUILD_MACHINE );
stringProperties["gitVersion"] = std::string( WALBERLA_GIT_SHA1 );
stringProperties["buildType"] = std::string( WALBERLA_BUILD_TYPE );
stringProperties["compilerFlags"] = std::string( WALBERLA_COMPILER_FLAGS );
auto runId = sqlite::storeRunInSqliteDB( databaseFile, integerProperties, stringProperties, realProperties );
sqlite::storeTimingPoolInSqliteDB( databaseFile, runId, timingPool, "TimingRoot" );
sqlite::storeTimingPoolInSqliteDB( databaseFile, runId, *reducedTimingPool, "TimingReduced" );
}
}
return 0;
}
\ No newline at end of file
apps/benchmarks/FieldCommunication/config.dat 0 → 100644
View file @ 7a6d3b57
Domain
{
cellsPerBlock < 60, 60, 60 >;
domainWeights < 1, 1, 1 >;
blocksPerProcess 1;
}
Communication
{
stencil D3Q19;
ghostLayers 1;
buffered 1;
singleMessage 1;
Fields {
pdf 1;
pdfOpt 1;
vector 0;
scalar 0;
}
localCommunicationMode start;
layout fzyx;
}
Run
{
warmupIterations 2;
iterations 50;
databaseFile FieldCommunication.sqlite;
}
\ No newline at end of file
apps/benchmarks/FieldCommunication/config.py 0 → 100644
View file @ 7a6d3b57
import waLBerla
import subprocess
import re
from collections import defaultdict
import sys
from datetime import timedelta
base = (32, 16, 2, 64)
BlOCK_SIZES_SQ = [(i, i, i) for i in base]
BLOCK_SIZES_RECT = [(i, i, i // 2) for i in base] + [(i, i // 2, i // 2) for i in base]
time_for_benchmark = 0.25
outer_iterations = 2
db_file = 'FieldCommunication.sqlite'
def supermuc_network_spread():
try:
node_list = subprocess.check_output("scontrol show hostname $SLURM_JOB_NODELIST", shell=True, encoding='utf8')
except subprocess.CalledProcessError:
return defaultdict(lambda: 0)
spread = defaultdict(set)
for s in node_list.split("\n"):
m = re.search("i(\d\d)r(\d\d)c(\d\d)s(\d\d)", s)
if m:
for name, idx in zip(['i', 'r', 'c', 's'], range(1, 5)):
spread[name].add(m.group(idx))
return {k: len(v) for k, v in spread.items()}
sng_network = supermuc_network_spread()
class AlreadySimulated:
def __init__(self, db_file, properties=('processes0*processes1*processes2', 'layout', 'ghostLayers',
'cartesianCommunicator', 'stencil',
'cellsPerBlock0', 'cellsPerBlock1', 'cellsPerBlock2',
'blocksPerProcess', 'localCommunicationMode', 'singleMessage',
'fieldsPdf', 'fieldsPdfOpt', 'fieldsVector', 'fieldsScalar',
'buffered')):
self.properties = properties
import sqlite3
conn = sqlite3.connect(db_file)
self.data = set()
try:
for row in conn.execute("SELECT {} FROM runs;".format(",".join(self.properties))):
self.data.add(row)
except sqlite3.OperationalError:
pass
waLBerla.log_info_on_root("Loaded {} scenarios".format(len(self.data)))
def in_db(self, args):
return args in self.data
@waLBerla.callback("config")
def config(processes=None):
simulated_db = AlreadySimulated(db_file)
isWalberlaRun = processes is None
skipped = 0
simulated = 0
if isWalberlaRun:
processes = waLBerla.mpi.numProcesses()
for layout in ('fzyx', 'zyxf'):
for ghost_layers in (1, 2):
for cartesian_comm in (False, True):
for stencil in ('D3Q19', 'D3Q27', 'D3Q7'):
for cells in BlOCK_SIZES_SQ + BLOCK_SIZES_RECT:
for blocksPerProcess in (1, 2, 4, 8, 16):
for local_comm in ('start', 'noOptimization', 'buffer'):
for single_message in (True, False):
for pdf, pdf_opt, vector, scalar in ([1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1],
[0, 0, 0, 3], [0, 0, 0, 19], [2, 0, 0, 0], [0, 2, 0, 0]):
for buffered in (0, 1):
if blocksPerProcess >= 8 and cells[0] >= 64 and cells[1] >= 64 and cells[2] >= 64:
continue
data = (processes, layout, ghost_layers, int(cartesian_comm), stencil, *cells, blocksPerProcess, local_comm,
int(single_message), pdf, pdf_opt, vector, scalar, buffered)
if simulated_db.in_db(data):
skipped += 1
if skipped % 100 == 0 and isWalberlaRun:
waLBerla.log_info_on_root("Skipped {} scenarios".format(skipped))
continue
else:
simulated += 1
if not isWalberlaRun:
continue
cfg = {
'Domain': {
'cellsPerBlock': cells,
'domainWeights': (1, 1, 1),
'blocksPerProcess': blocksPerProcess,
},
'Communication': {
'buffered': buffered,
'stencil': stencil,
'ghostLayers': ghost_layers,
'cartesianCommunicator': cartesian_comm,
'singleMessage': single_message,
'Fields': {
'pdf': pdf,
'pdfOpt': pdf_opt,
'vector': vector,
'scalar': scalar,
},
'layout': layout,
'localCommunicationMode': local_comm,
},
'Run': {
'warmupIterations': 3,
'iterations': 100,
'outerIterations': outer_iterations,
'databaseFile': db_file,
'timeForBenchmark': time_for_benchmark,
'minIterations': 2,
'maxIterations': 10000,
},
'Database': {
'sngNetworkIslands': sng_network['i'],
'sngNetworkRows': sng_network['r'],
'sngNetworkCabinets': sng_network['c'],
'sngNetworkSlots': sng_network['s'],
}
}
yield cfg
if not isWalberlaRun:
print("Skipped", skipped, "to simulate", simulated)
estimated_seconds = simulated * time_for_benchmark * outer_iterations
print("Estimated time ", timedelta(seconds=estimated_seconds))
if __name__ == '__main__':
for _ in config(int(sys.argv[1])):
pass
apps/benchmarks/FluidParticleCoupling/CMakeLists.txt 0 → 100644
View file @ 7a6d3b57
waLBerla_link_files_to_builddir( "*.dat" )
if( WALBERLA_BUILD_WITH_CODEGEN )
waLBerla_generate_target_from_python(NAME FluidParticleCouplingGeneratedLBM FILE GeneratedLBM.py
OUT_FILES GeneratedLBM.cpp GeneratedLBM.h
)
waLBerla_generate_target_from_python(NAME FluidParticleCouplingGeneratedLBMWithForce FILE GeneratedLBMWithForce.py
OUT_FILES GeneratedLBMWithForce.cpp GeneratedLBMWithForce.h
)
waLBerla_add_executable(NAME SphereWallCollision FILES SphereWallCollision.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling
walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME SettlingSphereInBox FILES SettlingSphereInBox.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling
walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME SphereMovingWithPrescribedVelocity FILES SphereMovingWithPrescribedVelocity.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::mesa_pd walberla::lbm_mesapd_coupling
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME LubricationForceEvaluation FILES LubricationForceEvaluation.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME DragForceSphere FILES DragForceSphere.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBMWithForce )
waLBerla_add_executable(NAME ForcesOnSphereNearPlane FILES ForcesOnSphereNearPlane.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME ObliqueWetCollision FILES ObliqueWetCollision.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
waLBerla_add_executable(NAME MotionSettlingSphere FILES MotionSettlingSphere.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd
walberla::postprocessing walberla::timeloop walberla::vtk FluidParticleCouplingGeneratedLBM )
else()
waLBerla_add_executable ( NAME SphereWallCollision FILES SphereWallCollision.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME SettlingSphereInBox FILES SettlingSphereInBox.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME SphereMovingWithPrescribedVelocity FILES SphereMovingWithPrescribedVelocity.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::mesa_pd walberla::lbm_mesapd_coupling walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME LubricationForceEvaluation FILES LubricationForceEvaluation.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME DragForceSphere FILES DragForceSphere.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME ForcesOnSphereNearPlane FILES ForcesOnSphereNearPlane.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
waLBerla_add_executable ( NAME ObliqueWetCollision FILES ObliqueWetCollision.cpp
DEPENDS walberla::blockforest walberla::boundary walberla::core walberla::domain_decomposition walberla::field walberla::lbm walberla::lbm_mesapd_coupling walberla::mesa_pd walberla::postprocessing walberla::timeloop walberla::vtk )
endif()
waLBerla_add_executable ( NAME ObliqueDryCollision FILES ObliqueDryCollision.cpp
DEPENDS walberla::blockforest walberla::core walberla::mesa_pd walberla::postprocessing )
apps/benchmarks/FluidParticleCoupling/DragForceSphere.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file DragForceSphere.cpp
//! \ingroup lbm_mesapd_coupling
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include "boundary/all.h"
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/debug/Debug.h"
#include "core/debug/TestSubsystem.h"
#include "core/logging/Logging.h"
#include "core/math/all.h"
#include "core/SharedFunctor.h"
#include "core/timing/RemainingTimeLogger.h"
#include "core/mpi/MPIManager.h"
#include "core/mpi/Reduce.h"
#include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h"
#include "field/communication/PackInfo.h"
#include "lbm/boundary/all.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/MacroscopicValueCalculation.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/lattice_model/ForceModel.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/SimpleBB.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "stencil/D3Q27.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/all.h"
#include "field/vtk/all.h"
#include "lbm/vtk/all.h"
#include <iomanip>
#include <iostream>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBMWithForce.h"
#define USE_TRTLIKE
//#define USE_D3Q27TRTLIKE
//#define USE_CUMULANTTRT
//#define USE_CUMULANT
#endif
namespace drag_force_sphere_mem
{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBMWithForce;
#else
using ForceModel_T = lbm::force_model::LuoConstant;
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT, false, ForceModel_T>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag ( "fluid" );
const FlagUID MO_BB_Flag ( "moving obstacle BB" );
const FlagUID MO_CLI_Flag ( "moving obstacle CLI" );
////////////////
// PARAMETERS //
////////////////
struct Setup{
uint_t checkFrequency;
real_t visc;
real_t tau;
real_t radius;
uint_t length;
real_t chi;
real_t extForce;
real_t analyticalDrag;
};
enum MEMVariant { BB, CLI };
MEMVariant to_MEMVariant( const std::string& s )
{
if( s == "BB" ) return MEMVariant::BB;
if( s == "CLI" ) return MEMVariant::CLI;
throw std::runtime_error("invalid conversion from text to MEMVariant");
}
std::string MEMVariant_to_string ( const MEMVariant& m )
{
if( m == MEMVariant::BB ) return "BB";
if( m == MEMVariant::CLI ) return "CLI";
throw std::runtime_error("invalid conversion from MEMVariant to string");
}
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using SBB_T = lbm_mesapd_coupling::SimpleBB< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using CLI_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, SBB_T, CLI_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ), ac_( ac ) {}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
auto * flagField = block->getData< FlagField_T >( flagFieldID_ );
auto * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
SBB_T("SBB_BB", MO_BB_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ),
CLI_T("CLI_BB", MO_CLI_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ) );
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
};
template< typename ParticleAccessor_T>
class DragForceEvaluator
{
public:
DragForceEvaluator( SweepTimeloop* timeloop, Setup* setup, const shared_ptr< StructuredBlockStorage > & blocks,
const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const shared_ptr<ParticleAccessor_T>& ac, walberla::id_t sphereID )
: timeloop_( timeloop ), setup_( setup ), blocks_( blocks ),
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ),
ac_( ac ), sphereID_( sphereID ),
normalizedDragOld_( 0.0 ), normalizedDragNew_( 0.0 )
{
// calculate the analytical drag force value based on the series expansion of chi
// see also Sangani - Slow flow through a periodic array of spheres, IJMF 1982. Eq. 60 and Table 1
real_t analyticalDrag = real_c(0);
real_t tempChiPowS = real_c(1);
// coefficients to calculate the drag in a series expansion
real_t dragCoefficients[31] = { real_c(1.000000), real_c(1.418649), real_c(2.012564), real_c(2.331523), real_c(2.564809), real_c(2.584787),
real_c(2.873609), real_c(3.340163), real_c(3.536763), real_c(3.504092), real_c(3.253622), real_c(2.689757),
real_c(2.037769), real_c(1.809341), real_c(1.877347), real_c(1.534685), real_c(0.9034708), real_c(0.2857896),
real_c(-0.5512626), real_c(-1.278724), real_c(1.013350), real_c(5.492491), real_c(4.615388), real_c(-0.5736023),
real_c(-2.865924), real_c(-4.709215), real_c(-6.870076), real_c(0.1455304), real_c(12.51891), real_c(9.742811),
real_c(-5.566269)};
for(uint_t s = 0; s <= uint_t(30); ++s){
analyticalDrag += dragCoefficients[s] * tempChiPowS;
tempChiPowS *= setup->chi;
}
setup_->analyticalDrag = analyticalDrag;
}
// evaluate the acting drag force
void operator()()
{
const uint_t timestep (timeloop_->getCurrentTimeStep()+1);
if( timestep % setup_->checkFrequency != 0) return;
// get force in x-direction acting on the sphere
real_t forceX = computeDragForce();
// get average volumetric flowrate in the domain
real_t uBar = computeAverageVel();
averageVel_ = uBar;
// f_total = f_drag + f_buoyancy
real_t totalForce = forceX + real_c(4.0/3.0) * math::pi * setup_->radius * setup_->radius * setup_->radius * setup_->extForce ;
real_t normalizedDragForce = totalForce / real_c( 6.0 * math::pi * setup_->visc * setup_->radius * uBar );
// update drag force values
normalizedDragOld_ = normalizedDragNew_;
normalizedDragNew_ = normalizedDragForce;
}
// return the relative temporal change in the normalized drag
real_t getDragForceDiff() const
{
return std::fabs( ( normalizedDragNew_ - normalizedDragOld_ ) / normalizedDragNew_ );
}
// return the drag force
real_t getDragForce() const
{
return normalizedDragNew_;
}
real_t getAverageVel()
{
return averageVel_;
}
void logResultToFile( const std::string & filename ) const
{
// write to file if desired
// format: length tau viscosity simulatedDrag analyticalDrag\n
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( filename.c_str(), std::ofstream::app );
file.precision(8);
file << setup_->length << " " << setup_->tau << " " << setup_->visc << " " << normalizedDragNew_ << " " << setup_->analyticalDrag << "\n";
file.close();
}
}
private:
// obtain the drag force acting on the sphere by summing up all the process local parts of fX
real_t computeDragForce()
{
size_t idx = ac_->uidToIdx(sphereID_);
real_t force = real_t(0);
if( idx!= ac_->getInvalidIdx())
{
force = ac_->getHydrodynamicForce(idx)[0];
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( force, mpi::SUM );
}
return force;
}
// calculate the average velocity in forcing direction (here: x) inside the domain (assuming dx=1)
real_t computeAverageVel()
{
auto velocity_sum = real_t(0);
// iterate all blocks stored locally on this process
for( auto blockIt = blocks_->begin(); blockIt != blocks_->end(); ++blockIt )
{
// retrieve the pdf field and the flag field from the block
PdfField_T * pdfField = blockIt->getData< PdfField_T >( pdfFieldID_ );
FlagField_T * flagField = blockIt->getData< FlagField_T >( flagFieldID_ );
// get the flag that marks a cell as being fluid
auto fluid = flagField->getFlag( Fluid_Flag );
auto xyzField = pdfField->xyzSize();
for (auto cell : xyzField) {
if( flagField->isFlagSet( cell.x(), cell.y(), cell.z(), fluid ) )
{
velocity_sum += pdfField->getVelocity(cell)[0];
}
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( velocity_sum, mpi::SUM );
}
return velocity_sum / real_c( setup_->length * setup_->length * setup_->length );
}
SweepTimeloop* timeloop_;
Setup* setup_;
shared_ptr< StructuredBlockStorage > blocks_;
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
const walberla::id_t sphereID_;
// drag coefficient
real_t normalizedDragOld_;
real_t normalizedDragNew_;
real_t averageVel_;
};
//////////
// MAIN //
//////////
//*******************************************************************************************************************
/*!\brief Testcase that checks the drag force acting on a fixed sphere in the center of a cubic domain in Stokes flow
*
* The drag force for this problem (often denoted as Simple Cubic setup) is given by a semi-analytical series expansion.
* The cubic domain is periodic in all directions, making it a physically infinite periodic array of spheres.
* _______________
* ->| |->
* ->| ___ |->
* W ->| / \ |-> E
* E ->| | x | |-> A
* S ->| \___/ |-> S
* T ->| |-> T
* ->|_______________|->
*
* The collision model used for the LBM is TRT with a relaxation parameter tau=1.5 and the magic parameter 3/16.
* The Stokes approximation of the equilibrium PDFs is used.
* The flow is driven by a constant body force of 1e-5.
* The domain is length x length x length, and the sphere has a diameter of chi * length cells
* The simulation is run until the relative change in the dragforce between 100 time steps is less than 1e-5.
* The RPD is not used since the sphere is kept fixed and the force is explicitly reset after each time step.
* To avoid periodicity constrain problems, the sphere is set as global.
*
*/
//*******************************************************************************************************************
int main( int argc, char **argv )
{
debug::enterTestMode();
mpi::Environment env( argc, argv );
auto processes = MPIManager::instance()->numProcesses();
if( processes != 1 && processes != 2 && processes != 4 && processes != 8)
{
std::cerr << "Number of processes must be equal to either 1, 2, 4, or 8!" << std::endl;
return EXIT_FAILURE;
}
///////////////////
// Customization //
///////////////////
bool shortrun = false;
bool funcTest = false;
bool logging = true;
uint_t vtkIOFreq = 0;
std::string baseFolder = "vtk_out_DragForceSphere";
real_t tau = real_c( 1.5 );
real_t externalForcing = real_t(1e-8);
uint_t length = uint_c( 40 );
MEMVariant method = MEMVariant::CLI;
real_t bulkViscRateFactor = real_t(1); // ratio between bulk and shear viscosity related relaxation factors, 1 = TRT, > 1 for stability with MRT
// see Khirevich - Coarse- and fine-grid numerical behavior of MRT/TRT lattice-Boltzmann schemes in regular and random sphere packings
bool useSRT = false;
real_t magicNumber = real_t(3) / real_t(16);
bool useOmegaBulkAdaption = false;
real_t adaptionLayerSize = real_t(2);
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--shortrun" ) == 0 ) { shortrun = true; continue; }
if( std::strcmp( argv[i], "--funcTest" ) == 0 ) { funcTest = true; continue; }
if( std::strcmp( argv[i], "--noLogging" ) == 0 ) { logging = false; continue; }
if( std::strcmp( argv[i], "--vtkIOFreq" ) == 0 ) { vtkIOFreq = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--MEMVariant" ) == 0 ) { method = to_MEMVariant( argv[++i] ); continue; }
if( std::strcmp( argv[i], "--tau" ) == 0 ) { tau = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--extForce" ) == 0 ) { externalForcing = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--length" ) == 0 ) { length = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--bulkViscRateFactor" ) == 0 ) { bulkViscRateFactor = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--useSRT" ) == 0 ) { useSRT = true; continue; }
if( std::strcmp( argv[i], "--magicNumber" ) == 0 ) { magicNumber = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--useOmegaBulkAdaption" ) == 0 ) { useOmegaBulkAdaption = true; continue; }
if( std::strcmp( argv[i], "--adaptionLayerSize" ) == 0 ) { adaptionLayerSize = real_c(std::atof( argv[++i] )); continue; }
WALBERLA_ABORT("Unrecognized command line argument found: " << argv[i]);
}
///////////////////////////
// SIMULATION PROPERTIES //
///////////////////////////
Setup setup;
setup.length = length; // length of the cubic domain in lattice cells
setup.chi = real_c( 0.5 ); // porosity parameter: diameter / length
setup.tau = tau; // relaxation time
setup.extForce = externalForcing; // constant body force in lattice units
setup.checkFrequency = uint_t( 100 ); // evaluate the drag force only every checkFrequency time steps
setup.radius = real_c(0.5) * setup.chi * real_c( setup.length ); // sphere radius
setup.visc = ( setup.tau - real_c(0.5) ) / real_c(3); // viscosity in lattice units
const real_t omega = real_c(1) / setup.tau; // relaxation rate
const real_t dx = real_c(1); // lattice dx
const real_t convergenceLimit = real_t(0.1) * setup.extForce; // tolerance for relative change in drag force
const uint_t timesteps = funcTest ? 1 : ( shortrun ? uint_c(150) : uint_c( 100000 ) ); // maximum number of time steps for the whole simulation
WALBERLA_LOG_INFO_ON_ROOT("tau = " << tau);
WALBERLA_LOG_INFO_ON_ROOT("diameter = " << real_t(2) * setup.radius);
WALBERLA_LOG_INFO_ON_ROOT("viscosity = " << setup.visc);
WALBERLA_LOG_INFO_ON_ROOT("MEM variant = " << MEMVariant_to_string(method));
WALBERLA_LOG_INFO_ON_ROOT("external forcing = " << setup.extForce);
///////////////////////////
// BLOCK STRUCTURE SETUP //
///////////////////////////
const uint_t XBlocks = (processes >= 2) ? uint_t( 2 ) : uint_t( 1 );
const uint_t YBlocks = (processes >= 4) ? uint_t( 2 ) : uint_t( 1 );
const uint_t ZBlocks = (processes == 8) ? uint_t( 2 ) : uint_t( 1 );
const uint_t XCells = setup.length / XBlocks;
const uint_t YCells = setup.length / YBlocks;
const uint_t ZCells = setup.length / ZBlocks;
// create fully periodic domain
auto blocks = blockforest::createUniformBlockGrid( XBlocks, YBlocks, ZBlocks, XCells, YCells, ZCells, dx, true,
true, true, true );
/////////
// RPD //
/////////
mesa_pd::domain::BlockForestDomain domain(blocks->getBlockForestPointer());
//init data structures
auto ps = std::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = std::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = make_shared<ParticleAccessor_T >(ps, ss);
auto sphereShape = ss->create<mesa_pd::data::Sphere>( setup.radius );
//////////////////
// RPD COUPLING //
//////////////////
// connect to pe
const real_t overlap = real_t( 1.5 ) * dx;
if( setup.radius > real_c( setup.length ) * real_t(0.5) - overlap )
{
std::cerr << "Periodic sphere is too large and would lead to incorrect mapping!" << std::endl;
// solution: create the periodic copies explicitly
return EXIT_FAILURE;
}
// create the sphere in the middle of the domain
// it is global and thus present on all processes
Vector3<real_t> position (real_c(setup.length) * real_c(0.5));
walberla::id_t sphereID;
{
mesa_pd::data::Particle&& p = *ps->create(true);
p.setPosition(position);
p.setInteractionRadius(setup.radius);
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
sphereID = p.getUid();
}
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = (useSRT) ? lambda_e : lbm::collision_model::TRT::lambda_d( omega, magicNumber );
real_t omegaBulk = (useSRT) ? lambda_e : lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#if defined(USE_TRTLIKE) || defined(USE_D3Q27TRTLIKE)
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
WALBERLA_LOG_INFO_ON_ROOT(" - magic number " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - omegaBulk = " << omegaBulk << ", bulk visc. = " << lbm_mesapd_coupling::bulkViscosityFromOmegaBulk(omegaBulk) << " (bvrf " << bulkViscRateFactor << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - lambda_e " << lambda_e << ", lambda_d " << lambda_d << ", omegaBulk " << omegaBulk );
WALBERLA_LOG_INFO_ON_ROOT(" - use omega bulk adaption = " << useOmegaBulkAdaption << " (adaption layer size = " << adaptionLayerSize << ")");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, setup.extForce, lambda_d, lambda_e);
#elif defined(USE_CUMULANTTRT)
WALBERLA_LOG_INFO_ON_ROOT("Using generated cumulant TRT lattice model!");
WALBERLA_LOG_INFO_ON_ROOT(" - magic number " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - lambda_e " << lambda_e << ", lambda_d " << lambda_d );
LatticeModel_T latticeModel = LatticeModel_T(setup.extForce, lambda_d, lambda_e);
#elif defined(USE_CUMULANT)
LatticeModel_T latticeModel = LatticeModel_T(setup.extForce, omega);
#endif
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in MRT lattice model and ignoring omega bulk field since not supported!");
WALBERLA_LOG_INFO_ON_ROOT(" - magic number " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - omegaBulk = " << omegaBulk << ", bulk visc. = " << lbm_mesapd_coupling::bulkViscosityFromOmegaBulk(omegaBulk) << " (bvrf " << bulkViscRateFactor << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - lambda_e " << lambda_e << ", lambda_d " << lambda_d << ", omegaBulk " << omegaBulk );
LatticeModel_T latticeModel = LatticeModel_T(lbm::collision_model::D3Q19MRT( omegaBulk, omegaBulk, lambda_d, lambda_e, lambda_e, lambda_d ), ForceModel_T( Vector3<real_t> ( setup.extForce, 0, 0 ) ));
#endif
// add PDF field
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field (fzyx)", latticeModel,
Vector3< real_t >( real_t(0) ), real_t(1),
uint_t(1), field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage<FlagField_T>( blocks, "flag field" );
// add particle field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor), "boundary handling" );
///////////////
// TIME LOOP //
///////////////
// create the timeloop
SweepTimeloop timeloop( blocks->getBlockStorage(), timesteps );
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
//blockforest::communication::UniformBufferedScheme< stencil::D3Q27 > optimizedPDFCommunicationScheme( blocks );
//optimizedPDFCommunicationScheme.addPackInfo( make_shared< field::communication::PackInfo< PdfField_T > >( pdfFieldID ) );
// initially map particles into the LBM simulation
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
if( method == MEMVariant::CLI )
{
// uses a higher order boundary condition (CLI)
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, movingParticleMappingKernel, *accessor, MO_CLI_Flag);
}else{
// uses standard bounce back boundary conditions
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, movingParticleMappingKernel, *accessor, MO_BB_Flag);
}
// update bulk omega in all cells to adapt to changed particle position
if(useOmegaBulkAdaption)
{
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, mesa_pd::kernel::SelectAll>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, mesa_pd::kernel::SelectAll());
for (auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt) {
(*omegaBulkAdapter)(blockIt.get());
}
}
if( vtkIOFreq != uint_t(0) )
{
// spheres
auto particleVtkOutput = make_shared<mesa_pd::vtk::ParticleVtkOutput>(ps);
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleLinearVelocity>("velocity");
auto particleVtkWriter = vtk::createVTKOutput_PointData(particleVtkOutput, "Particles", vtkIOFreq, baseFolder, "simulation_step");
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( particleVtkWriter ), "VTK (sphere data)" );
// pdf field
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", vtkIOFreq, 0, false, baseFolder );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
pdfFieldVTK->addCellInclusionFilter( fluidFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
// omega bulk field
timeloop.addFuncBeforeTimeStep( field::createVTKOutput<ScalarField_T, float>( omegaBulkFieldID, *blocks, "omega_bulk_field", vtkIOFreq, uint_t(0), false, baseFolder ), "VTK (omega bulk field)" );
}
// since external forcing is applied, the evaluation of the velocity has to be carried out directly after the streaming step
// however, the default sweep is a stream - collide step, i.e. after the sweep, the velocity evaluation is not correct
// solution: split the sweep explicitly into collide and stream
using DragForceEval_T = DragForceEvaluator<ParticleAccessor_T>;
auto forceEval = make_shared< DragForceEval_T >( &timeloop, &setup, blocks, flagFieldID, pdfFieldID, accessor, sphereID );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
// splitting stream and collide is currently buggy in lbmpy with vectorization -> build without optimize for local host and vectorization
// collide LBM step
timeloop.add() << Sweep([&lbmSweep](IBlock * const block){lbmSweep.collide(block);}, "collide LB sweep" );
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep( BoundaryHandling_T::getBlockSweep( boundaryHandlingID ), "Boundary Handling" );
// stream LBM step
timeloop.add() << Sweep([&lbmSweep](IBlock * const block){lbmSweep.stream(block);}, "stream LB sweep" )
<< AfterFunction( SharedFunctor< DragForceEval_T >( forceEval ), "drag force evaluation" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
// collision sweep
timeloop.add() << Sweep( lbm::makeCollideSweep( lbmSweep ), "cell-wise LB sweep (collide)" );
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep( BoundaryHandling_T::getBlockSweep( boundaryHandlingID ), "Boundary Handling" );
// streaming & force evaluation
timeloop.add() << Sweep( lbm::makeStreamSweep( lbmSweep ), "cell-wise LB sweep (stream)" )
<< AfterFunction( SharedFunctor< DragForceEval_T >( forceEval ), "drag force evaluation" );
#endif
// resetting force
timeloop.addFuncAfterTimeStep( [ps,accessor](){ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel(),*accessor);}, "reset force on sphere");
timeloop.addFuncAfterTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "Remaining Time Logger" );
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
// time loop
for (uint_t i = 0; i < timesteps; ++i )
{
// perform a single simulation step
timeloop.singleStep( timeloopTiming );
// check if the relative change in the normalized drag force is below the specified convergence criterion
if ( i > setup.checkFrequency && forceEval->getDragForceDiff() < convergenceLimit )
{
// if simulation has converged, terminate simulation
break;
}
if( std::isnan(forceEval->getDragForce()) ) WALBERLA_ABORT("Nan found!");
if(i%1000 == 0)
{
WALBERLA_LOG_INFO_ON_ROOT("Current drag force: " << forceEval->getDragForce());
}
}
WALBERLA_LOG_INFO_ON_ROOT("Final drag force: " << forceEval->getDragForce());
WALBERLA_LOG_INFO_ON_ROOT("Re = " << forceEval->getAverageVel() * setup.radius * real_t(2) / setup.visc);
timeloopTiming.logResultOnRoot();
if ( !funcTest && !shortrun ){
// check the result
real_t relErr = std::fabs( ( setup.analyticalDrag - forceEval->getDragForce() ) / setup.analyticalDrag );
if ( logging )
{
WALBERLA_ROOT_SECTION()
{
std::cout << "Analytical drag: " << setup.analyticalDrag << "\n"
<< "Simulated drag: " << forceEval->getDragForce() << "\n"
<< "Relative error: " << relErr << "\n";
}
std::string fileName( MEMVariant_to_string(method) );
fileName += "_bvrf" + std::to_string(uint_c(bulkViscRateFactor));
fileName += "_mn" + std::to_string(float(magicNumber));
if(useOmegaBulkAdaption ) fileName += "_uOBA" + std::to_string(uint_c(adaptionLayerSize));
if(useSRT) fileName += "_SRT";
forceEval->logResultToFile( "log_DragForceSphereMEM_Generated_"+fileName+".txt" );
}
}
return 0;
}
} //namespace drag_force_sphere_mem
int main( int argc, char **argv ){
drag_force_sphere_mem::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/ForcesOnSphereNearPlane.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ForcesOnSphereNearPlane.cpp
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include <blockforest/loadbalancing/StaticCurve.h>
#include <blockforest/SetupBlockForest.h>
#include "boundary/all.h"
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/SharedFunctor.h"
#include "core/debug/Debug.h"
#include "core/debug/TestSubsystem.h"
#include "core/math/all.h"
#include "core/timing/RemainingTimeLogger.h"
#include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h"
#include "field/StabilityChecker.h"
#include "field/communication/PackInfo.h"
#include "lbm/boundary/FreeSlip.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm_mesapd_coupling/mapping/ParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/SimpleBB.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/utility/AddForceOnParticlesKernel.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/utility/AverageHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/InitializeHydrodynamicForceTorqueForAveragingKernel.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/shape/HalfSpace.h"
#include "mesa_pd/data/shape/Sphere.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/mpi/SyncNextNeighbors.h"
#include "mesa_pd/mpi/ReduceProperty.h"
#include "mesa_pd/mpi/notifications/HydrodynamicForceTorqueNotification.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/all.h"
#include "field/vtk/all.h"
#include "lbm/vtk/all.h"
#include <functional>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBM.h"
#endif
namespace forces_on_sphere_near_plane
{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
//////////////
// TYPEDEFS //
//////////////
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBM;
#else
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag( "fluid" );
const FlagUID CLI_Flag( "cli boundary" );
const FlagUID SBB_Flag( "sbb boundary" );
/////////////////////
// BLOCK STRUCTURE //
/////////////////////
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using CLI_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using SBB_T = lbm_mesapd_coupling::SimpleBB< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, CLI_T, SBB_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ), ac_( ac ) {}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
auto * flagField = block->getData< FlagField_T >( flagFieldID_ );
auto * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
CLI_T( "MO CLI", CLI_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ),
SBB_T( "MO SBB", SBB_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ) );
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
};
//*******************************************************************************************************************
template< typename ParticleAccessor_T>
class SpherePropertyLogger
{
public:
SpherePropertyLogger( const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid,
const std::string & fileName, bool fileIO,
real_t dragNormalizationFactor, real_t liftNormalizationFactor, real_t physicalTimeScale ) :
ac_( ac ), sphereUid_( sphereUid ),
fileName_( fileName ), fileIO_(fileIO),
dragNormalizationFactor_( dragNormalizationFactor ), liftNormalizationFactor_( liftNormalizationFactor ),
physicalTimeScale_( physicalTimeScale )
{
if ( fileIO_ )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str() );
file << "#\t t\t Cd\t Cl\t fX\t fY\t fZ\t tX\t tY\t tZ\n";
file.close();
}
}
}
void operator()(const uint_t timestep)
{
Vector3<real_t> force(real_t(0));
Vector3<real_t> torque(real_t(0));
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != ac_->getInvalidIdx())
{
if(!mesa_pd::data::particle_flags::isSet(ac_->getFlags(idx),mesa_pd::data::particle_flags::GHOST))
{
force = ac_->getHydrodynamicForce(idx);
torque = ac_->getHydrodynamicTorque(idx);
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( force, mpi::SUM );
mpi::allReduceInplace( torque, mpi::SUM );
}
if( fileIO_ )
writeToFile( timestep, force, torque);
dragForce_ = force[0];
liftForce_ = force[2];
}
real_t getDragForce()
{
return dragForce_;
}
real_t getLiftForce()
{
return liftForce_;
}
real_t getDragCoefficient()
{
return dragForce_ / dragNormalizationFactor_;
}
real_t getLiftCoefficient()
{
return liftForce_ / liftNormalizationFactor_;
}
private:
void writeToFile( uint_t timestep, const Vector3<real_t> & force, const Vector3<real_t> & torque )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str(), std::ofstream::app );
file << timestep << "\t" << real_c(timestep) / physicalTimeScale_
<< "\t" << force[0] / dragNormalizationFactor_ << "\t" << force[2] / liftNormalizationFactor_
<< "\t" << force[0] << "\t" << force[1] << "\t" << force[2]
<< "\t" << torque[0] << "\t" << torque[1] << "\t" << torque[2]
<< "\n";
file.close();
}
}
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
std::string fileName_;
bool fileIO_;
real_t dragForce_, liftForce_;
real_t dragNormalizationFactor_, liftNormalizationFactor_;
real_t physicalTimeScale_;
};
template <typename BoundaryHandling_T>
void initializeCouetteProfile( const shared_ptr< StructuredBlockStorage > & blocks, const BlockDataID & pdfFieldID, const BlockDataID & boundaryHandlingID,
const real_t & domainHeight, const real_t wallVelocity )
{
const real_t rho = real_c(1);
for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt )
{
auto pdfField = blockIt->getData< PdfField_T > ( pdfFieldID );
auto boundaryHandling = blockIt->getData< BoundaryHandling_T >( boundaryHandlingID );
WALBERLA_FOR_ALL_CELLS_XYZ(pdfField,
if( !boundaryHandling->isDomain(x,y,z) ) continue;
const Vector3< real_t > coord = blocks->getBlockLocalCellCenter( *blockIt, Cell(x,y,z) );
Vector3< real_t > velocity( real_c(0) );
velocity[0] = wallVelocity * coord[2] / domainHeight;
pdfField->setToEquilibrium( x, y, z, velocity, rho );
)
}
}
void createPlaneSetup(const shared_ptr<mesa_pd::data::ParticleStorage> & ps, const shared_ptr<mesa_pd::data::ShapeStorage> & ss, const math::AABB & simulationDomain, const real_t wallVelocity )
{
// create bounding planes
mesa_pd::data::Particle p0 = *ps->create(true);
p0.setPosition(simulationDomain.minCorner());
p0.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p0.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,1) ));
p0.setOwner(mpi::MPIManager::instance()->rank());
p0.setType(0);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p1 = *ps->create(true);
p1.setPosition(simulationDomain.maxCorner());
p1.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p1.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,-1) ));
p1.setOwner(mpi::MPIManager::instance()->rank());
p1.setType(0);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
p1.setLinearVelocity(Vector3<real_t>(wallVelocity, real_t(0), real_t(0))); //moving wall
}
void logFinalResult( const std::string & fileName, real_t Re, real_t wallDistance, real_t diameter,
uint_t domainLength, uint_t domainWidth, uint_t domainHeight,
real_t dragCoeff, real_t dragCoeffRef,
real_t liftCoeff, real_t liftCoeffRef,
uint_t timestep, real_t nonDimTimestep )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName.c_str(), std::ofstream::app );
file << Re << "\t " << wallDistance << "\t " << diameter << "\t "
<< domainLength << "\t " << domainWidth << "\t " << domainHeight << "\t "
<< dragCoeff << "\t " << dragCoeffRef << "\t "<< std::abs(dragCoeff-dragCoeffRef) / dragCoeffRef << "\t "
<< liftCoeff << "\t " << liftCoeffRef << "\t "<< std::abs(liftCoeff-liftCoeffRef) / liftCoeffRef << "\t "
<< timestep << "\t " << nonDimTimestep
<< "\n";
file.close();
}
}
//////////
// MAIN //
//////////
//*******************************************************************************************************************
/*!\brief Testcase that evaluates the drag and lift force on a sphere that is close to the bottom plane in shear flow
*
* see overview paper:
* Zeng, Najjar, Balachandar, Fischer - "Forces on a finite-sized particle located close to a wall in a linear shear flow", 2009
*
* contains references to:
* Leighton, Acrivos - "The lift on a small sphere touching a plane in the presence of a simple shear flow", 1985
* Zeng, Balachandar, Fischer - "Wall-induced forces on a rigid sphere at finite Reynolds number", 2005
*
* CFD-IBM simulations in:
* Lee, Balachandar - "Drag and lift forces on a spherical particle moving on a wall in a shear flow at finite Re", 2010
*
* Description:
* - Domain size [x, y, z] = [48 x 16 x 8 ] * diameter = [L(ength), W(idth), H(eight)]
* - horizontally periodic, bounded by two planes in z-direction
* - top plane is moving with constant wall velocity -> shear flow
* - sphere is placed in the vicinity of the bottom plane at [ L/2 + xOffset, W/2 + yOffset, wallDistance * diameter]
* - distance of sphere center to the bottom plane is crucial parameter
* - viscosity is adjusted to match specified Reynolds number = shearRate * diameter * wallDistance / viscosity
* - dimensionless drag and lift forces are evaluated and written to logging file
*/
//*******************************************************************************************************************
int main( int argc, char **argv )
{
debug::enterTestMode();
mpi::Environment env( argc, argv );
///////////////////
// Customization //
///////////////////
// simulation control
bool fileIO = true;
uint_t vtkIOFreq = 0;
std::string baseFolderVTK = "vtk_out_ForcesNearPlane";
std::string baseFolderLogging = ".";
// physical setup
real_t diameter = real_t(20); // cells per diameter -> determines overall resolution
real_t normalizedWallDistance = real_t(1); // distance of the sphere center to the bottom wall, normalized by the diameter
real_t ReynoldsNumberShear = real_t(1); // = shearRate * wallDistance * diameter / viscosity
//numerical parameters
real_t maximumNonDimTimesteps = real_t(100); // maximum number of non-dimensional time steps
real_t xOffsetOfSpherePosition = real_t(0); // offset in x-direction of sphere position
real_t yOffsetOfSpherePosition = real_t(0); // offset in y-direction of sphere position
real_t bulkViscRateFactor = real_t(1);
real_t magicNumber = real_t(3)/real_t(16);
real_t wallVelocity = real_t(0.1);
bool initializeVelocityProfile = false;
bool useOmegaBulkAdaption = false;
real_t adaptionLayerSize = real_t(2);
std::string boundaryCondition = "CLI"; // SBB, CLI
real_t relativeChangeConvergenceEps = real_t(1e-5);
real_t physicalCheckingFrequency = real_t(0.1);
// command line arguments
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--noLogging" ) == 0 ) { fileIO = false; continue; }
if( std::strcmp( argv[i], "--vtkIOFreq" ) == 0 ) { vtkIOFreq = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--diameter" ) == 0 ) { diameter = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--timesteps" ) == 0 ) { maximumNonDimTimesteps = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--wallDistance" ) == 0 ) { normalizedWallDistance = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--Re" ) == 0 ) { ReynoldsNumberShear = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--boundaryCondition" ) == 0 ) { boundaryCondition = argv[++i]; continue; }
if( std::strcmp( argv[i], "--velocity" ) == 0 ) { wallVelocity = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--xOffset" ) == 0 ) { xOffsetOfSpherePosition = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--yOffset" ) == 0 ) { yOffsetOfSpherePosition = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--bvrf" ) == 0 ) { bulkViscRateFactor = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--baseFolderVTK" ) == 0 ) { baseFolderVTK = argv[++i]; continue; }
if( std::strcmp( argv[i], "--baseFolderLogging" ) == 0 ) { baseFolderLogging = argv[++i]; continue; }
if( std::strcmp( argv[i], "--initializeVelocityProfile" ) == 0 ) { initializeVelocityProfile = true; continue; }
if( std::strcmp( argv[i], "--magicNumber" ) == 0 ) { magicNumber = real_c(std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--useOmegaBulkAdaption" ) == 0 ) { useOmegaBulkAdaption = true; continue; }
if( std::strcmp( argv[i], "--adaptionLayerSize" ) == 0 ) { adaptionLayerSize = real_c(std::atof( argv[++i] )); continue; }
if( std::strcmp( argv[i], "--convergenceLimit" ) == 0 ) { relativeChangeConvergenceEps = real_c(std::atof( argv[++i] )); continue; }
if( std::strcmp( argv[i], "--checkingFrequency" ) == 0 ) { physicalCheckingFrequency = real_c(std::atof( argv[++i] )); continue; }
WALBERLA_ABORT("Unrecognized command line argument found: " << argv[i]);
}
WALBERLA_CHECK_GREATER_EQUAL(normalizedWallDistance, real_t(0.5));
WALBERLA_CHECK_GREATER_EQUAL(ReynoldsNumberShear, real_t(0));
WALBERLA_CHECK_GREATER_EQUAL(diameter, real_t(0));
WALBERLA_CHECK(boundaryCondition == "SBB" || boundaryCondition == "CLI");
//////////////////////////
// NUMERICAL PARAMETERS //
//////////////////////////
const real_t domainLength = real_t(48) * diameter; //x
const real_t domainWidth = real_t(16) * diameter; //y
const real_t domainHeight = real_t( 8) * diameter; //z
Vector3<uint_t> domainSize( uint_c( std::ceil(domainLength)), uint_c( std::ceil(domainWidth)), uint_c( std::ceil(domainHeight)) );
const real_t wallDistance = diameter * normalizedWallDistance;
const real_t shearRate = wallVelocity / domainHeight;
const real_t velAtSpherePosition = shearRate * wallDistance;
const real_t viscosity = velAtSpherePosition * diameter / ReynoldsNumberShear;
const real_t relaxationTime = real_t(1) / lbm::collision_model::omegaFromViscosity(viscosity);
const real_t densityFluid = real_t(1);
const real_t dx = real_t(1);
const real_t physicalTimeScale = diameter / velAtSpherePosition;
const uint_t timesteps = uint_c(maximumNonDimTimesteps * physicalTimeScale);
const real_t omega = real_t(1) / relaxationTime;
const real_t omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
Vector3<real_t> initialPosition( domainLength * real_t(0.5) + xOffsetOfSpherePosition,
domainWidth * real_t(0.5) + yOffsetOfSpherePosition,
wallDistance );
WALBERLA_LOG_INFO_ON_ROOT("Setup:");
WALBERLA_LOG_INFO_ON_ROOT(" - domain size = " << domainSize );
WALBERLA_LOG_INFO_ON_ROOT(" - normalized wall distance = " << normalizedWallDistance );
WALBERLA_LOG_INFO_ON_ROOT(" - shear rate = " << shearRate );
WALBERLA_LOG_INFO_ON_ROOT(" - wall velocity = " << wallVelocity );
WALBERLA_LOG_INFO_ON_ROOT(" - Reynolds number (shear rate based) = " << ReynoldsNumberShear << ", vel at sphere pos = " << velAtSpherePosition);
WALBERLA_LOG_INFO_ON_ROOT(" - density = " << densityFluid );
WALBERLA_LOG_INFO_ON_ROOT(" - viscosity = " << viscosity << " -> omega = " << omega << " , tau = " << relaxationTime);
WALBERLA_LOG_INFO_ON_ROOT(" - magic number = " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - omegaBulk = " << omegaBulk << ", bulk visc. = " << lbm_mesapd_coupling::bulkViscosityFromOmegaBulk(omegaBulk) << " (bvrf " << bulkViscRateFactor << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - use omega bulk adaption = " << useOmegaBulkAdaption << " (adaption layer size = " << adaptionLayerSize << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - sphere diameter = " << diameter << ", position = " << initialPosition << " ( xOffset = " << xOffsetOfSpherePosition << ", yOffset = " << yOffsetOfSpherePosition << " )");
WALBERLA_LOG_INFO_ON_ROOT(" - base folder VTK = " << baseFolderVTK << ", base folder logging = " << baseFolderLogging );
WALBERLA_LOG_INFO_ON_ROOT(" - boundary condition = " << boundaryCondition );
///////////////////////////
// BLOCK STRUCTURE SETUP //
///////////////////////////
Vector3<uint_t> blocksPerDirection( 24, 5, 2 );
//Vector3<uint_t> blocksPerDirection( 3, 3, 1 );
WALBERLA_CHECK( domainSize[0] % blocksPerDirection[0] == 0 &&
domainSize[1] % blocksPerDirection[1] == 0 &&
domainSize[2] % blocksPerDirection[2] == 0 );
Vector3<uint_t> blockSizeInCells(domainSize[0] / ( blocksPerDirection[0] ),
domainSize[1] / ( blocksPerDirection[1] ),
domainSize[2] / ( blocksPerDirection[2] ) );
AABB simulationDomain( real_t(0), real_t(0), real_t(0), real_c(domainSize[0]), real_c(domainSize[1]), real_c(domainSize[2]) );
auto blocks = blockforest::createUniformBlockGrid( blocksPerDirection[0], blocksPerDirection[1], blocksPerDirection[2],
blockSizeInCells[0], blockSizeInCells[1], blockSizeInCells[2], dx,
0, false, false,
true, true, false, //periodicity
false );
WALBERLA_LOG_INFO_ON_ROOT(" - blocks = " << blocksPerDirection << ", block size = " << blockSizeInCells );
//write domain decomposition to file
if( vtkIOFreq > 0 )
{
vtk::writeDomainDecomposition( blocks, "initial_domain_decomposition", baseFolderVTK );
}
/////////////////
// PE COUPLING //
/////////////////
auto rpdDomain = std::make_shared<mesa_pd::domain::BlockForestDomain>(blocks->getBlockForestPointer());
//init data structures
auto ps = walberla::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = walberla::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
// bounding planes
createPlaneSetup(ps,ss,blocks->getDomain(), wallVelocity);
// create sphere and store Uid
auto sphereShape = ss->create<mesa_pd::data::Sphere>( diameter * real_t(0.5) );
walberla::id_t sphereUid = 0;
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), initialPosition ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(initialPosition);
p.setInteractionRadius(diameter * real_t(0.5));
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
sphereUid = p.getUid();
}
mpi::allReduceInplace(sphereUid, mpi::SUM);
// set up RPD functionality
std::function<void(void)> syncCall = [ps,rpdDomain](){
const real_t overlap = real_t( 1.5 );
mesa_pd::mpi::SyncNextNeighbors syncNextNeighborFunc;
syncNextNeighborFunc(*ps, *rpdDomain, overlap);
};
syncCall();
mesa_pd::mpi::ReduceProperty reduceProperty;
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumber );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, lambda_d, lambda_e);
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in MRT lattice model and ignoring omega bulk field since not supported!");
LatticeModel_T latticeModel = LatticeModel_T(lbm::collision_model::D3Q19MRT( omegaBulk, omegaBulk, lambda_d, lambda_e, lambda_e, lambda_d ));
#endif
// add PDF field
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field (fzyx)", latticeModel,
Vector3< real_t >( real_t(0) ), real_t(1),
FieldGhostLayers, field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage<FlagField_T>( blocks, "flag field", FieldGhostLayers );
// add particle field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor), "boundary handling" );
// set up coupling functionality
lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel resetHydrodynamicForceTorque;
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
lbm_mesapd_coupling::AverageHydrodynamicForceTorqueKernel averageHydrodynamicForceTorque;
if(boundaryCondition == "CLI")
{
// map sphere into the LBM simulation
ps->forEachParticle(false, lbm_mesapd_coupling::RegularParticlesSelector(), *accessor, movingParticleMappingKernel, *accessor, CLI_Flag);
// map planes
ps->forEachParticle(false, lbm_mesapd_coupling::GlobalParticlesSelector(), *accessor, movingParticleMappingKernel, *accessor, CLI_Flag);
}
else
{
// map sphere into the LBM simulation
ps->forEachParticle(false, lbm_mesapd_coupling::RegularParticlesSelector(), *accessor, movingParticleMappingKernel, *accessor, SBB_Flag);
// map planes
ps->forEachParticle(false, lbm_mesapd_coupling::GlobalParticlesSelector(), *accessor, movingParticleMappingKernel, *accessor, SBB_Flag);
}
// initialize Couette velocity profile in whole domain
if(initializeVelocityProfile)
{
WALBERLA_LOG_INFO_ON_ROOT("Initializing Couette velocity profile.");
initializeCouetteProfile<BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, domainHeight, wallVelocity);
}
// update bulk omega in all cells to adapt to changed particle position
if(useOmegaBulkAdaption)
{
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, lbm_mesapd_coupling::RegularParticlesSelector>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, lbm_mesapd_coupling::RegularParticlesSelector());
for (auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt) {
(*omegaBulkAdapter)(blockIt.get());
}
}
///////////////
// TIME LOOP //
///////////////
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
// create the timeloop
SweepTimeloop timeloop( blocks->getBlockStorage(), timesteps );
timeloop.addFuncBeforeTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "Remaining Time Logger" );
if( vtkIOFreq != uint_t(0) )
{
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", vtkIOFreq, uint_t(0), false, baseFolderVTK );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
pdfFieldVTK->addCellInclusionFilter( fluidFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
}
auto bhSweep = BoundaryHandling_T::getBlockSweep( boundaryHandlingID );
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep(bhSweep, "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
timeloop.add() << Sweep( lbmSweep, "LB sweep" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
timeloop.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
// add force evaluation and logging
real_t normalizationFactor = math::pi / real_t(8) * densityFluid * shearRate * shearRate * wallDistance * wallDistance * diameter * diameter ;
std::string loggingFileName( baseFolderLogging + "/LoggingForcesNearPlane");
loggingFileName += "_D" + std::to_string(uint_c(diameter));
loggingFileName += "_Re" + std::to_string(uint_c(ReynoldsNumberShear));
loggingFileName += "_WD" + std::to_string(uint_c(normalizedWallDistance * real_t(1000)));
loggingFileName += "_" + boundaryCondition;
loggingFileName += "_bvrf" + std::to_string(uint_c(bulkViscRateFactor));
if(useOmegaBulkAdaption) loggingFileName += "_uOBA" + std::to_string(uint_c(adaptionLayerSize));
loggingFileName += "_mn" + std::to_string(magicNumber);
loggingFileName += ".txt";
WALBERLA_LOG_INFO_ON_ROOT(" - writing logging file " << loggingFileName);
SpherePropertyLogger<ParticleAccessor_T >logger( accessor, sphereUid, loggingFileName, fileIO, normalizationFactor, normalizationFactor, physicalTimeScale);
// compute reference values from literature
const real_t normalizedGapSize = normalizedWallDistance - real_t(0.5);
// drag correlation for the drag coefficient
const real_t standardDragCorrelation = real_t(24) / ReynoldsNumberShear * (real_t(1) + real_t(0.15) * std::pow(ReynoldsNumberShear, real_t(0.687))); // Schiller-Naumann correlation
const real_t dragCorrelationWithGapSizeStokes = real_t(24) / ReynoldsNumberShear * (real_t(1) + real_t(0.138) * std::exp(real_t(-2) * normalizedGapSize) + real_t(9)/( real_t(16) * (real_t(1) + real_t(2) * normalizedGapSize) ) ); // Goldman et al. (1967)
const real_t alphaDragS = real_t(0.15) - real_t(0.046) * ( real_t(1) - real_t(0.16) * normalizedGapSize * normalizedGapSize ) * std::exp( -real_t(0.7) * normalizedGapSize);
const real_t betaDragS = real_t(0.687) + real_t(0.066)*(real_t(1) - real_t(0.76) * normalizedGapSize * normalizedGapSize) * std::exp( - std::pow( normalizedGapSize, real_t(0.9) ) );
const real_t dragCorrelationZeng = dragCorrelationWithGapSizeStokes * ( real_t(1) + alphaDragS * std::pow( ReynoldsNumberShear, betaDragS ) ); // Zeng et al. (2009) - Eqs. (13) and (14)
// lift correlations for the lift coefficient
const real_t liftCorrelationZeroGapStokes = real_t(5.87); // Leighton, Acrivos (1985)
const real_t liftCorrelationZeroGap = real_t(3.663) / std::pow( ReynoldsNumberShear * ReynoldsNumberShear + real_t(0.1173), real_t(0.22) ); // Zeng et al. (2009) - Eq. (19)
const real_t alphaLiftS = - std::exp( -real_t(0.3) + real_t(0.025) * ReynoldsNumberShear);
const real_t betaLiftS = real_t(0.8) + real_t(0.01) * ReynoldsNumberShear;
const real_t lambdaLiftS = ( real_t(1) - std::exp(-normalizedGapSize)) * std::pow( ReynoldsNumberShear / real_t(250), real_t(5) / real_t(2) );
const real_t liftCorrelationZeng = liftCorrelationZeroGap * std::exp( - real_t(0.5) * normalizedGapSize * std::pow( ReynoldsNumberShear / real_t(250), real_t(4)/real_t(3))) *
( std::exp( alphaLiftS * std::pow( normalizedGapSize, betaLiftS ) ) - lambdaLiftS ); // Zeng et al. (2009) - Eqs. (28) and (29)
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
const uint_t checkingFrequency = uint_c( physicalCheckingFrequency * physicalTimeScale );
WALBERLA_LOG_INFO_ON_ROOT("Starting simulation with at maximum of " << timesteps << " time steps");
WALBERLA_LOG_INFO_ON_ROOT("Convergence checking frequency = " << checkingFrequency << " (physically = " << physicalCheckingFrequency << ") until relative difference of " << relativeChangeConvergenceEps << " is reached.");
real_t maxDragCurrentCheckingPeriod = -math::Limits<real_t>::inf();
real_t minDragCurrentCheckingPeriod = math::Limits<real_t>::inf();
real_t maxLiftCurrentCheckingPeriod = -math::Limits<real_t>::inf();
real_t minLiftCurrentCheckingPeriod = math::Limits<real_t>::inf();
uint_t timestep = 0;
// time loop
while( true )
{
// perform a single simulation step
timeloop.singleStep( timeloopTiming );
// sync hydrodynamic force to local particle
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
// average force
if( timestep == 0 )
{
lbm_mesapd_coupling::InitializeHydrodynamicForceTorqueForAveragingKernel initializeHydrodynamicForceTorqueForAveragingKernel;
ps->forEachParticle(false, mesa_pd::kernel::SelectLocal(), *accessor, initializeHydrodynamicForceTorqueForAveragingKernel, *accessor );
}
ps->forEachParticle(false, mesa_pd::kernel::SelectLocal(), *accessor, averageHydrodynamicForceTorque, *accessor );
// evaluation
timeloopTiming["Logging"].start();
logger(timestep);
timeloopTiming["Logging"].end();
// reset after logging here
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
// check for termination
real_t curDrag = logger.getDragCoefficient();
real_t curLift = logger.getLiftCoefficient();
if(std::isinf(curDrag) || std::isnan(curDrag)) WALBERLA_ABORT("Found invalid drag value " << curDrag << " in time step " << timestep);
maxDragCurrentCheckingPeriod = std::max( maxDragCurrentCheckingPeriod, curDrag);
minDragCurrentCheckingPeriod = std::min( minDragCurrentCheckingPeriod, curDrag);
maxLiftCurrentCheckingPeriod = std::max( maxLiftCurrentCheckingPeriod, curLift);
minLiftCurrentCheckingPeriod = std::min( minLiftCurrentCheckingPeriod, curLift);
real_t dragDiffCurrentCheckingPeriod = std::fabs(maxDragCurrentCheckingPeriod - minDragCurrentCheckingPeriod)/std::fabs(maxDragCurrentCheckingPeriod);
real_t liftDiffCurrentCheckingPeriod = std::fabs(maxLiftCurrentCheckingPeriod - minLiftCurrentCheckingPeriod)/std::fabs(maxLiftCurrentCheckingPeriod);
// continuous output during simulation
if( timestep % (checkingFrequency * uint_t(10)) == 0)
{
WALBERLA_LOG_INFO_ON_ROOT("Drag: current C_D = " << curDrag );
WALBERLA_LOG_INFO_ON_ROOT(" - standard C_D = " << standardDragCorrelation );
WALBERLA_LOG_INFO_ON_ROOT(" - C_D ( Stokes fit ) = " << dragCorrelationWithGapSizeStokes );
WALBERLA_LOG_INFO_ON_ROOT(" - C_D ( Zeng ) = " << dragCorrelationZeng );
WALBERLA_LOG_INFO_ON_ROOT("Lift: current C_L = " << curLift );
WALBERLA_LOG_INFO_ON_ROOT(" - C_L ( Stokes, zero gap ) = " << liftCorrelationZeroGapStokes);
WALBERLA_LOG_INFO_ON_ROOT(" - C_L ( zero gap ) = " << liftCorrelationZeroGap);
WALBERLA_LOG_INFO_ON_ROOT(" - C_L ( Zeng ) = " << liftCorrelationZeng);
WALBERLA_LOG_INFO_ON_ROOT( "Drag difference [(max-min)/max] = " << dragDiffCurrentCheckingPeriod << ", lift difference = " << liftDiffCurrentCheckingPeriod);
}
// check for convergence ( = difference between min and max values in current checking period is below limit)
if( timestep % checkingFrequency == 0 && timestep > 0 )
{
if( dragDiffCurrentCheckingPeriod < relativeChangeConvergenceEps &&
liftDiffCurrentCheckingPeriod < relativeChangeConvergenceEps )
{
WALBERLA_LOG_INFO_ON_ROOT("Forces converged with an eps of " << relativeChangeConvergenceEps );
WALBERLA_LOG_INFO_ON_ROOT(" - drag min = " << minDragCurrentCheckingPeriod << " , max = " << maxDragCurrentCheckingPeriod);
WALBERLA_LOG_INFO_ON_ROOT(" - lift min = " << minLiftCurrentCheckingPeriod << " , max = " << maxLiftCurrentCheckingPeriod);
break;
}
//reset min and max values for new checking period
maxDragCurrentCheckingPeriod = -math::Limits<real_t>::inf();
minDragCurrentCheckingPeriod = math::Limits<real_t>::inf();
maxLiftCurrentCheckingPeriod = -math::Limits<real_t>::inf();
minLiftCurrentCheckingPeriod = math::Limits<real_t>::inf();
}
++timestep;
}
timeloopTiming.logResultOnRoot();
std::string resultFileName( baseFolderLogging + "/ResultForcesNearPlane");
resultFileName += "_D" + std::to_string(uint_c(diameter));
resultFileName += "_Re" + std::to_string(uint_c(ReynoldsNumberShear));
resultFileName += "_WD" + std::to_string(uint_c(normalizedWallDistance * real_t(1000)));
resultFileName += "_" + boundaryCondition;
resultFileName += "_bvrf" + std::to_string(uint_c(bulkViscRateFactor));
if(useOmegaBulkAdaption) resultFileName += "_uOBA" + std::to_string(uint_c(adaptionLayerSize));
resultFileName += "_mn" + std::to_string(magicNumber);
resultFileName += ".txt";
WALBERLA_LOG_INFO_ON_ROOT(" - writing final result to file " << resultFileName);
logFinalResult(resultFileName, ReynoldsNumberShear, normalizedWallDistance, diameter, domainSize[0], domainSize[1], domainSize[2],
logger.getDragCoefficient(), dragCorrelationZeng, logger.getLiftCoefficient(), liftCorrelationZeng, timestep, real_c(timestep) / physicalTimeScale );
return EXIT_SUCCESS;
}
} // namespace forces_on_sphere_near_plane
int main( int argc, char **argv ){
forces_on_sphere_near_plane::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/GeneratedLBM.py 0 → 100644
View file @ 7a6d3b57
import sympy as sp
from sympy.core.cache import clear_cache
import pystencils as ps
from lbmpy.creationfunctions import LBMConfig, LBMOptimisation, Method, Stencil, create_lb_method
from lbmpy_walberla import generate_lattice_model
from pystencils_walberla import CodeGeneration
from pystencils_walberla import get_vectorize_instruction_set
from lbmpy.creationfunctions import create_lb_collision_rule
from lbmpy.moments import is_even, get_order, MOMENT_SYMBOLS
from lbmpy.stencils import LBStencil
from collections import OrderedDict
with CodeGeneration() as ctx:
generatedMethod = 'TRTlike'
# generatedMethod = 'D3Q27TRTlike'
# generatedMethod = 'cumulant'
clear_cache()
dtype_string = 'float64' if ctx.double_accuracy else 'float32'
cpu_vectorize_info = {'instruction_set': get_vectorize_instruction_set(ctx)}
if generatedMethod == 'TRTlike':
omegaVisc = sp.Symbol('omega_visc')
omegaBulk = ps.fields(f'omega_bulk: {dtype_string}[3D]', layout='fzyx')
omegaMagic = sp.Symbol('omega_magic')
stencil = LBStencil(Stencil.D3Q19)
x, y, z = MOMENT_SYMBOLS
one = sp.Rational(1, 1)
sq = x ** 2 + y ** 2 + z ** 2
moments = [
[one, x, y, z], # [0, 3, 5, 7]
[sq - 1], # [1]
[3 * sq ** 2 - 6 * sq + 1], # [2]
[(3 * sq - 5) * x, (3 * sq - 5) * y, (3 * sq - 5) * z], # [4, 6, 8]
[3 * x ** 2 - sq, y ** 2 - z ** 2, x * y, y * z, x * z], # [9, 11, 13, 14, 15]
[(2 * sq - 3) * (3 * x ** 2 - sq), (2 * sq - 3) * (y ** 2 - z ** 2)], # [10, 12]
[(y ** 2 - z ** 2) * x, (z ** 2 - x ** 2) * y, (x ** 2 - y ** 2) * z] # [16, 17, 18]
]
# relaxation rate for first group of moments (1,x,y,z) is set to zero internally
relaxation_rates = [omegaBulk.center, omegaBulk.center,
omegaMagic, omegaVisc, omegaVisc, omegaMagic]
lbm_config = LBMConfig(stencil=stencil, method=Method.MRT, continuous_equilibrium=False,
zero_centered=False, delta_equilibrium=False,
nested_moments=moments, relaxation_rates=relaxation_rates)
lbm_opt = LBMOptimisation(cse_global=True)
methodWithForce = create_lb_method(lbm_config=lbm_config)
# print(methodWithForce.relaxation_rates)
# print(methodWithForce.moment_matrix)
collision_rule = create_lb_collision_rule(lbm_config=lbm_config, lbm_optimisation=lbm_opt)
generate_lattice_model(ctx, 'GeneratedLBM', collision_rule, field_layout='fzyx', cpu_vectorize_info=cpu_vectorize_info)
if generatedMethod == 'D3Q27TRTlike':
omegaVisc = sp.Symbol('omega_visc')
omegaBulk = ps.fields(f'omega_bulk: {dtype_string}[3D]', layout='fzyx')
omegaMagic = sp.Symbol('omega_magic')
stencil = LBStencil(Stencil.D3Q27)
relaxation_rates = [omegaVisc, omegaBulk.center_vector, omegaMagic, omegaVisc, omegaMagic, omegaVisc]
lbm_config = LBMConfig(stencil=stencil, method=Method.MRT, maxwellian_moments=False, weighted=True,
compressible=False, relaxation_rates=relaxation_rates)
lbm_opt = LBMOptimisation(cse_global=True)
methodWithForce = create_lb_method(lbm_config=lbm_config)
collision_rule = create_lb_collision_rule(lbm_config=lbm_config, lbm_optimisation=lbm_opt)
generate_lattice_model(ctx, 'GeneratedLBM', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
# using create_with_continuous_maxwellian_eq_moments won't work probably
if generatedMethod == 'cumulant':
x, y, z = MOMENT_SYMBOLS
cumulants = [0] * 27
cumulants[0] = sp.sympify(1) # 000
cumulants[1] = x # 100
cumulants[2] = y # 010
cumulants[3] = z # 001
cumulants[4] = x*y # 110
cumulants[5] = x*z # 101
cumulants[6] = y*z # 011
cumulants[7] = x**2 - y**2 # 200 - 020
cumulants[8] = x**2 - z**2 # 200 - 002
cumulants[9] = x**2 + y**2 + z**2 # 200 + 020 + 002
cumulants[10] = x*y**2 + x*z**2 # 120 + 102
cumulants[11] = x**2*y + y*z**2 # 210 + 012
cumulants[12] = x**2*z + y**2*z # 201 + 021
cumulants[13] = x*y**2 - x*z**2 # 120 - 102
cumulants[14] = x**2*y - y*z**2 # 210 - 012
cumulants[15] = x**2*z - y**2*z # 201 - 021
cumulants[16] = x*y*z # 111
cumulants[17] = x**2*y**2 - 2*x**2*z**2 + y**2*z**2 # 220- 2*202 +022
cumulants[18] = x**2*y**2 + x**2*z**2 - 2*y**2*z**2 # 220 + 202 - 2*002
cumulants[19] = x**2*y**2 + x**2*z**2 + y**2*z**2 # 220 + 202 + 022
cumulants[20] = x**2*y*z # 211
cumulants[21] = x*y**2*z # 121
cumulants[22] = x*y*z**2 # 112
cumulants[23] = x**2*y**2*z # 221
cumulants[24] = x**2*y*z**2 # 212
cumulants[25] = x*y**2*z**2 # 122
cumulants[26] = x**2*y**2*z**2 # 222
def get_relaxation_rate(cumulant, omega):
if get_order(cumulant) <= 1:
return 0
elif get_order(cumulant) == 2 and cumulant != x**2 + y**2 + z**2:
return omega
else:
return 1
stencil = LBStencil(Stencil.D3Q27)
omega = sp.Symbol('omega')
rr_dict = OrderedDict((c, get_relaxation_rate(c, omega))
for c in cumulants)
from lbmpy.methods import create_with_continuous_maxwellian_eq_moments
my_method = create_with_continuous_maxwellian_eq_moments(stencil, rr_dict, cumulant=True, compressible=True,)
collision_rule = create_lb_collision_rule(lb_method=my_method,
optimization={'cse_global': True,
'cse_pdfs': False})
print(my_method.relaxation_rates)
generate_lattice_model(ctx, 'GeneratedLBM', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
apps/benchmarks/FluidParticleCoupling/GeneratedLBMWithForce.py 0 → 100644
View file @ 7a6d3b57
import sympy as sp
from sympy.core.cache import clear_cache
import pystencils as ps
from lbmpy.creationfunctions import LBMConfig, LBMOptimisation, ForceModel, Method, Stencil, create_lb_method
from lbmpy_walberla import generate_lattice_model
from pystencils_walberla import CodeGeneration
from pystencils_walberla import get_vectorize_instruction_set
from lbmpy.creationfunctions import create_lb_collision_rule
from lbmpy.moments import MOMENT_SYMBOLS, is_even, get_order
from lbmpy.stencils import LBStencil
from collections import OrderedDict
with CodeGeneration() as ctx:
forcing = (sp.symbols('fx'), 0, 0)
generatedMethod = 'TRTlike'
# generatedMethod = 'D3Q27TRTlike'
# generatedMethod = 'cumulant'
# generatedMethod = 'cumulantTRT'
print('Generating ' + generatedMethod + ' LBM method')
clear_cache()
dtype_string = 'float64' if ctx.double_accuracy else 'float32'
cpu_vectorize_info = {'instruction_set': get_vectorize_instruction_set(ctx)}
if generatedMethod == 'TRTlike':
omegaVisc = sp.Symbol('omega_visc')
omegaBulk = ps.fields(f'omega_bulk: {dtype_string}[3D]', layout='fzyx')
omegaMagic = sp.Symbol('omega_magic')
stencil = LBStencil(Stencil.D3Q19)
x, y, z = MOMENT_SYMBOLS
one = sp.Rational(1, 1)
sq = x ** 2 + y ** 2 + z ** 2
moments = [
[one, x, y, z], # [0, 3, 5, 7]
[sq - 1], # [1]
[3 * sq ** 2 - 6 * sq + 1], # [2]
[(3 * sq - 5) * x, (3 * sq - 5) * y, (3 * sq - 5) * z], # [4, 6, 8]
[3 * x ** 2 - sq, y ** 2 - z ** 2, x * y, y * z, x * z], # [9, 11, 13, 14, 15]
[(2 * sq - 3) * (3 * x ** 2 - sq), (2 * sq - 3) * (y ** 2 - z ** 2)], # [10, 12]
[(y ** 2 - z ** 2) * x, (z ** 2 - x ** 2) * y, (x ** 2 - y ** 2) * z] # [16, 17, 18]
]
# relaxation rate for first group of moments (1,x,y,z) is set to zero internally
relaxation_rates = [omegaBulk.center, omegaBulk.center,
omegaMagic, omegaVisc, omegaVisc, omegaMagic]
lbm_config = LBMConfig(stencil=stencil, method=Method.MRT, continuous_equilibrium=False,
zero_centered=False, delta_equilibrium=False,
force=forcing, force_model=ForceModel.LUO,
nested_moments=moments, relaxation_rates=relaxation_rates)
lbm_opt = LBMOptimisation(cse_global=True)
methodWithForce = create_lb_method(lbm_config=lbm_config)
# print(methodWithForce.relaxation_rates)
# print(methodWithForce.moment_matrix)
collision_rule = create_lb_collision_rule(lbm_config=lbm_config, lbm_optimisation=lbm_opt)
generate_lattice_model(ctx, 'GeneratedLBMWithForce', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
if generatedMethod == 'D3Q27TRTlike':
omegaVisc = sp.Symbol('omega_visc')
omegaBulk = ps.fields(f'omega_bulk: {dtype_string}[3D]', layout='fzyx')
omegaMagic = sp.Symbol('omega_magic')
stencil = LBStencil(Stencil.D3Q27)
relaxation_rates = [omegaVisc, omegaBulk.center_vector, omegaMagic, omegaVisc, omegaMagic, omegaVisc]
lbm_config = LBMConfig(stencil=stencil, method=Method.MRT, maxwellian_moments=False, weighted=True,
force=forcing, force_model=ForceModel.LUO,
compressible=False, relaxation_rates=relaxation_rates)
lbm_opt = LBMOptimisation(cse_global=True)
methodWithForce = create_lb_method(lbm_config=lbm_config)
collision_rule = create_lb_collision_rule(lbm_config=lbm_config, lbm_optimisation=lbm_opt)
generate_lattice_model(ctx, 'GeneratedLBMWithForce', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
if generatedMethod == 'cumulant':
x, y, z = MOMENT_SYMBOLS
cumulants = [0] * 27
cumulants[0] = sp.sympify(1) # 000
cumulants[1] = x # 100
cumulants[2] = y # 010
cumulants[3] = z # 001
cumulants[4] = x*y # 110
cumulants[5] = x*z # 101
cumulants[6] = y*z # 011
cumulants[7] = x**2 - y**2 # 200 - 020
cumulants[8] = x**2 - z**2 # 200 - 002
cumulants[9] = x**2 + y**2 + z**2 # 200 + 020 + 002
cumulants[10] = x*y**2 + x*z**2 # 120 + 102
cumulants[11] = x**2*y + y*z**2 # 210 + 012
cumulants[12] = x**2*z + y**2*z # 201 + 021
cumulants[13] = x*y**2 - x*z**2 # 120 - 102
cumulants[14] = x**2*y - y*z**2 # 210 - 012
cumulants[15] = x**2*z - y**2*z # 201 - 021
cumulants[16] = x*y*z # 111
cumulants[17] = x**2*y**2 - 2*x**2*z**2 + y**2*z**2 # 220- 2*202 +022
cumulants[18] = x**2*y**2 + x**2*z**2 - 2*y**2*z**2 # 220 + 202 - 2*002
cumulants[19] = x**2*y**2 + x**2*z**2 + y**2*z**2 # 220 + 202 + 022
cumulants[20] = x**2*y*z # 211
cumulants[21] = x*y**2*z # 121
cumulants[22] = x*y*z**2 # 112
cumulants[23] = x**2*y**2*z # 221
cumulants[24] = x**2*y*z**2 # 212
cumulants[25] = x*y**2*z**2 # 122
cumulants[26] = x**2*y**2*z**2 # 222
def get_relaxation_rate(cumulant, omega):
if get_order(cumulant) <= 1:
return 0
elif get_order(cumulant) == 2 and cumulant != x**2 + y**2 + z**2:
return omega
else:
return 1
stencil = LBStencil(Stencil.D3Q27)
omega = sp.Symbol('omega')
rr_dict = OrderedDict((c, get_relaxation_rate(c, omega))
for c in cumulants)
from lbmpy.methods import create_with_continuous_maxwellian_eq_moments
my_method = create_with_continuous_maxwellian_eq_moments(stencil, rr_dict, cumulant=True, compressible=True,
force_model=forcemodel)
collision_rule = create_lb_collision_rule(lb_method=my_method,
optimization={'cse_global': True,
'cse_pdfs': False})
print(my_method.relaxation_rates)
generate_lattice_model(ctx, 'GeneratedLBMWithForce', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
if generatedMethod == 'cumulantTRT':
x, y, z = MOMENT_SYMBOLS
cumulants = [0] * 27
cumulants[0] = sp.sympify(1) # 000
cumulants[1] = x # 100
cumulants[2] = y # 010
cumulants[3] = z # 001
cumulants[4] = x*y # 110
cumulants[5] = x*z # 101
cumulants[6] = y*z # 011
cumulants[7] = x**2 - y**2 # 200 - 020
cumulants[8] = x**2 - z**2 # 200 - 002
cumulants[9] = x**2 + y**2 + z**2 # 200 + 020 + 002
cumulants[10] = x*y**2 + x*z**2 # 120 + 102
cumulants[11] = x**2*y + y*z**2 # 210 + 012
cumulants[12] = x**2*z + y**2*z # 201 + 021
cumulants[13] = x*y**2 - x*z**2 # 120 - 102
cumulants[14] = x**2*y - y*z**2 # 210 - 012
cumulants[15] = x**2*z - y**2*z # 201 - 021
cumulants[16] = x*y*z # 111
cumulants[17] = x**2*y**2 - 2*x**2*z**2 + y**2*z**2 # 220- 2*202 +022
cumulants[18] = x**2*y**2 + x**2*z**2 - 2*y**2*z**2 # 220 + 202 - 2*002
cumulants[19] = x**2*y**2 + x**2*z**2 + y**2*z**2 # 220 + 202 + 022
cumulants[20] = x**2*y*z # 211
cumulants[21] = x*y**2*z # 121
cumulants[22] = x*y*z**2 # 112
cumulants[23] = x**2*y**2*z # 221
cumulants[24] = x**2*y*z**2 # 212
cumulants[25] = x*y**2*z**2 # 122
cumulants[26] = x**2*y**2*z**2 # 222
def get_relaxation_rate(cumulant, omegaVisc, omegaMagic):
if get_order(cumulant) <= 1:
return 0
elif is_even(cumulant):
return omegaVisc
else:
return omegaMagic
stencil = LBStencil(Stencil.D3Q27)
omegaVisc = sp.Symbol('omega_visc')
omegaMagic = sp.Symbol('omega_magic')
rr_dict = OrderedDict((c, get_relaxation_rate(c, omegaVisc, omegaMagic))
for c in cumulants)
from lbmpy.methods import create_with_continuous_maxwellian_eq_moments
my_method = create_with_continuous_maxwellian_eq_moments(stencil, rr_dict, cumulant=True, compressible=True,
force_model=forcemodel)
collision_rule = create_lb_collision_rule(lb_method=my_method,
optimization={'cse_global': True,
'cse_pdfs': False})
print(my_method.relaxation_rates)
generate_lattice_model(ctx, 'GeneratedLBMWithForce', collision_rule, field_layout='fzyx',
cpu_vectorize_info=cpu_vectorize_info)
apps/benchmarks/FluidParticleCoupling/LubricationForceEvaluation.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file LubricationForceEvaluation.cpp
//! \ingroup lbm_mesapd_coupling
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include "boundary/all.h"
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/SharedFunctor.h"
#include "core/debug/Debug.h"
#include "core/debug/TestSubsystem.h"
#include "core/math/all.h"
#include "core/mpi/Broadcast.h"
#include "core/timing/RemainingTimeLogger.h"
#include "core/logging/all.h"
#include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h"
#include "field/StabilityChecker.h"
#include "field/communication/PackInfo.h"
#include "lbm/boundary/all.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/mapping/ParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/SimpleBB.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/LubricationCorrectionKernel.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "mesa_pd/collision_detection/AnalyticContactDetection.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/shape/HalfSpace.h"
#include "mesa_pd/data/shape/Sphere.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/mpi/SyncNextNeighbors.h"
#include "mesa_pd/mpi/ContactFilter.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/all.h"
#include "field/vtk/all.h"
#include "lbm/vtk/all.h"
#include <functional>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBM.h"
#endif
namespace lubrication_force_evaluation
{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBM;
#else
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag( "fluid" );
const FlagUID FreeSlip_Flag( "free slip" );
const FlagUID MO_SBB_Flag( "moving obstacle sbb" );
const FlagUID MO_CLI_Flag( "moving obstacle cli" );
const FlagUID FormerMO_Flag( "former moving obstacle" );
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using FreeSlip_T = lbm::FreeSlip< LatticeModel_T, FlagField_T>;
using MO_SBB_T = lbm_mesapd_coupling::SimpleBB< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using MO_CLI_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, FreeSlip_T, MO_SBB_T, MO_CLI_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ), ac_( ac ) {}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
auto * flagField = block->getData< FlagField_T >( flagFieldID_ );
auto * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
FreeSlip_T( "FreeSlip", FreeSlip_Flag, pdfField, flagField, fluid ),
MO_SBB_T( "SBB", MO_SBB_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ),
MO_CLI_T( "CLI", MO_CLI_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ) );
const auto freeslip = flagField->getFlag( FreeSlip_Flag );
CellInterval domainBB = storage->getDomainCellBB();
domainBB.xMin() -= cell_idx_c( FieldGhostLayers );
domainBB.xMax() += cell_idx_c( FieldGhostLayers );
domainBB.yMin() -= cell_idx_c( FieldGhostLayers );
domainBB.yMax() += cell_idx_c( FieldGhostLayers );
// SOUTH
CellInterval south( domainBB.xMin(), domainBB.yMin(), domainBB.zMin(), domainBB.xMax(), domainBB.yMin(), domainBB.zMax() );
storage->transformGlobalToBlockLocalCellInterval( south, *block );
handling->forceBoundary( freeslip, south );
// NORTH
CellInterval north( domainBB.xMin(), domainBB.yMax(), domainBB.zMin(), domainBB.xMax(), domainBB.yMax(), domainBB.zMax() );
storage->transformGlobalToBlockLocalCellInterval( north, *block );
handling->forceBoundary( freeslip, north );
domainBB.zMin() -= cell_idx_c( FieldGhostLayers );
domainBB.zMax() += cell_idx_c( FieldGhostLayers );
// BOTTOM
CellInterval bottom( domainBB.xMin(), domainBB.yMin(), domainBB.zMin(), domainBB.xMax(), domainBB.yMax(), domainBB.zMin() );
storage->transformGlobalToBlockLocalCellInterval( bottom, *block );
handling->forceBoundary( freeslip, bottom );
// TOP
CellInterval top( domainBB.xMin(), domainBB.yMin(), domainBB.zMax(), domainBB.xMax(), domainBB.yMax(), domainBB.zMax() );
storage->transformGlobalToBlockLocalCellInterval( top, *block );
handling->forceBoundary( freeslip, top );
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
};
//*******************************************************************************************************************
//////////
// MAIN //
//////////
//*******************************************************************************************************************
/*!\brief Evaluates the hydrodynamic force for the lubrication case for sphere-sphere and sphere-wall case.
*
* 4 different setups are available that change the relative velocity to investigate the different components individually.
* All particles are fixed but have a small velocity which is a valid assumption in Stokes flow.
* The simulations are run until steady state is reached.
*
* The positions are always shifted by a random offset to account for the dependence of the resulting force/torque
* on the exact location w.r.t. the underlying grid.
* This can be eliminated by averaging over several realizations of the same physical setup.
*
* see also Rettinger, Ruede 2020 for the details
*/
//*******************************************************************************************************************
int main( int argc, char **argv )
{
debug::enterTestMode();
mpi::Environment env( argc, argv );
bool sphSphTest = true;
bool fileIO = true;
uint_t vtkIOFreq = 0;
std::string fileNameEnding = "";
std::string baseFolder = "vtk_out_Lubrication";
real_t radius = real_t(5);
real_t ReynoldsNumber = real_t(1e-2);
real_t tau = real_t(1);
real_t gapSize = real_t(0);
real_t bulkViscRateFactor = real_t(1);
real_t magicNumber = real_t(3)/real_t(16);
bool useOmegaBulkAdaption = false;
real_t adaptionLayerSize = real_t(2);
bool useSBB = false;
// 1: translation in normal direction -> normal Lubrication force
// 2: translation in tangential direction -> tangential Lubrication force and torque
// 3: rotation around tangential direction -> force & torque
// 4: rotation around normal direction -> torque
uint_t setup = 1;
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--sphWallTest" ) == 0 ) { sphSphTest = false; continue; }
if( std::strcmp( argv[i], "--noLogging" ) == 0 ) { fileIO = false; continue; }
if( std::strcmp( argv[i], "--vtkIOFreq" ) == 0 ) { vtkIOFreq = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--setup" ) == 0 ) { setup = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--baseFolder" ) == 0 ) { baseFolder = argv[++i]; continue; }
if( std::strcmp( argv[i], "--diameter" ) == 0 ) { radius = real_t(0.5)*real_c(std::atof(argv[++i])); continue; }
if( std::strcmp( argv[i], "--gapSize" ) == 0 ) { gapSize = real_c(std::atof(argv[++i])); continue; }
if( std::strcmp( argv[i], "--bulkViscRateFactor" ) == 0 ) { bulkViscRateFactor = real_c(std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--tau" ) == 0 ) { tau = real_c(std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--fileName" ) == 0 ) { fileNameEnding = argv[++i]; continue; }
if( std::strcmp( argv[i], "--magicNumber" ) == 0 ) { magicNumber = real_c(std::atof(argv[++i])); continue; }
if( std::strcmp( argv[i], "--useOmegaBulkAdaption" ) == 0 ) { useOmegaBulkAdaption = true; continue; }
if( std::strcmp( argv[i], "--adaptionLayerSize" ) == 0 ) { adaptionLayerSize = real_c(std::atof( argv[++i] )); continue; }
if( std::strcmp( argv[i], "--useSBB" ) == 0 ) { useSBB = true; continue; }
WALBERLA_ABORT("Unrecognized command line argument found: " << argv[i]);
}
///////////////////////////
// SIMULATION PROPERTIES //
///////////////////////////
uint_t xSize = uint_c(real_t(24) * radius);
uint_t ySize = uint_c(real_t(24) * radius);
uint_t zSize = uint_c(real_t(24) * radius);
uint_t xBlocks = uint_c(4); // number of blocks in x-direction
uint_t yBlocks = uint_c(1); // number of blocks in y-direction
uint_t zBlocks = uint_c(1); // number of blocks in z-direction
uint_t xCells = xSize / xBlocks; // number of cells in x-direction on each block
uint_t yCells = ySize / yBlocks; // number of cells in y-direction on each block
uint_t zCells = zSize / zBlocks; // number of cells in z-direction on each block
// Perform missing variable calculations
real_t omega = real_t(1) / tau;
real_t nu = walberla::lbm::collision_model::viscosityFromOmega(omega);
real_t velocity = ReynoldsNumber * nu / (real_t(2) * radius);
real_t omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
uint_t timesteps = uint_c( 10000 );
real_t fStokes = real_t(6) * math::pi * nu * radius * velocity;
real_t tStokes = real_t(8) * math::pi * nu * radius * radius * velocity;
WALBERLA_LOG_INFO_ON_ROOT_SECTION()
{
std::stringstream ss;
if ( sphSphTest )
{
ss << "-------------------------------------------------------\n"
<< " Parameters for the sphere-sphere lubrication test \n"
<< "-------------------------------------------------------\n";
}
else
{
ss << "-------------------------------------------------------\n"
<< " Parameters for the sphere-wall lubrication test \n"
<< "-------------------------------------------------------\n";
}
ss << " omega = " << omega << "\n"
<< " omegaBulk = " << omegaBulk << " (bvrf " << bulkViscRateFactor << ")\n"
<< " use omega bulk adaption = " << useOmegaBulkAdaption << " (adaption layer size = " << adaptionLayerSize << ")\n"
<< " radius = " << radius << "\n"
<< " velocity = " << velocity << "\n"
<< " Re = " << ReynoldsNumber << "\n"
<< " gap size = " << gapSize << "\n"
<< " time steps = " << timesteps << "\n"
<< " fStokes = " << fStokes << "\n"
<< " setup = " << setup << "\n"
<< " useSBB = " << useSBB << "\n"
<< "-------------------------------------------------------\n"
<< " domainSize = " << xSize << " x " << ySize << " x " << zSize << "\n"
<< " blocks = " << xBlocks << " x " << yBlocks << " x " << zBlocks << "\n"
<< " blockSize = " << xCells << " x " << yCells << " x " << zCells << "\n"
<< "-------------------------------------------------------\n";
WALBERLA_LOG_INFO( ss.str() );
}
auto blocks = blockforest::createUniformBlockGrid( xBlocks, yBlocks, zBlocks, xCells, yCells, zCells, real_t(1),
0, false, false,
sphSphTest, true, true, //periodicity
false );
//////////////////
// RPD COUPLING //
//////////////////
auto rpdDomain = std::make_shared<mesa_pd::domain::BlockForestDomain>(blocks->getBlockForestPointer());
//init data structures
auto ps = walberla::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = walberla::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
auto sphereShape = ss->create<mesa_pd::data::Sphere>( radius );
uint_t id1 (0);
uint_t id2 (0);
uint_t randomSeed = uint_c(std::chrono::system_clock::now().time_since_epoch().count());
mpi::broadcastObject(randomSeed); // root process chooses seed and broadcasts it
std::mt19937 randomNumberGenerator(static_cast<unsigned int>(randomSeed));
Vector3<real_t> domainCenter( real_c(xSize) * real_t(0.5), real_c(ySize) * real_t(0.5), real_c(zSize) * real_t(0.5) );
Vector3<real_t> offsetVector(math::realRandom<real_t>(real_t(0), real_t(1), randomNumberGenerator),
math::realRandom<real_t>(real_t(0), real_t(1), randomNumberGenerator),
math::realRandom<real_t>(real_t(0), real_t(1), randomNumberGenerator));
if ( sphSphTest )
{
Vector3<real_t> pos1 = domainCenter + offsetVector;
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), pos1 ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(pos1);
p.setInteractionRadius(radius);
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
if(setup == 1)
{
// only normal translational vel
p.setLinearVelocity(Vector3<real_t>( velocity, real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 2)
{
// only tangential translational velocity
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), velocity));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 3)
{
// only rotation around axis perpendicular to center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), velocity / radius, real_t(0)));
} else if (setup == 4)
{
// only rotation around center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( velocity / radius, real_t(0) , real_t(0)));
}
id1 = p.getUid();
}
Vector3<real_t> pos2 = pos1 + Vector3<real_t>(real_t(2) * radius + gapSize, real_t(0), real_t(0));
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), pos2 ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(pos2);
p.setInteractionRadius(radius);
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
if(setup == 1)
{
// only normal translational vel
p.setLinearVelocity(Vector3<real_t>( -velocity, real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 2)
{
// only tangential translational velocity
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), -velocity));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 3)
{
// only rotation around axis perpendicular to center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), velocity / radius, real_t(0)));
} else if (setup == 4)
{
// only rotation around center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( -velocity / radius, real_t(0) , real_t(0)));
}
id2 = p.getUid();
}
mpi::allReduceInplace(id1, mpi::SUM);
mpi::allReduceInplace(id2, mpi::SUM);
WALBERLA_LOG_INFO_ON_ROOT("pos sphere 1 = " << pos1);
WALBERLA_LOG_INFO_ON_ROOT("pos sphere 2 = " << pos2);
} else
{
// sphere-wall test
Vector3<real_t> referenceVector(offsetVector[0], domainCenter[1], domainCenter[2]);
// create two planes
mesa_pd::data::Particle&& p0 = *ps->create(true);
p0.setPosition(referenceVector);
p0.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p0.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(1,0,0) ));
p0.setOwner(mpi::MPIManager::instance()->rank());
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
id2 = p0.getUid();
mesa_pd::data::Particle&& p1 = *ps->create(true);
p1.setPosition(Vector3<real_t>(real_c(xSize),0,0));
p1.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p1.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(-1,0,0) ));
p1.setOwner(mpi::MPIManager::instance()->rank());
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
Vector3<real_t> pos1 = referenceVector + Vector3<real_t>(radius + gapSize, real_t(0), real_t(0));
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), pos1 ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(pos1);
p.setInteractionRadius(radius);
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
if(setup == 1)
{
// only normal translational vel
p.setLinearVelocity(Vector3<real_t>( -velocity, real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 2)
{
// only tangential translational velocity
p.setLinearVelocity(Vector3<real_t>( real_t(0), velocity, real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
} else if (setup == 3)
{
// only rotation around axis perpendicular to center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( real_t(0), velocity / radius, real_t(0)));
} else if (setup == 4)
{
// only rotation around center line
p.setLinearVelocity(Vector3<real_t>( real_t(0), real_t(0), real_t(0)));
p.setAngularVelocity(Vector3<real_t>( velocity / radius, real_t(0) , real_t(0)));
}
id1 = p.getUid();
}
mpi::allReduceInplace(id1, mpi::SUM);
// id2 is globally known
WALBERLA_LOG_INFO_ON_ROOT("pos plane = " << referenceVector);
WALBERLA_LOG_INFO_ON_ROOT("pos sphere = " << pos1);
}
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumber );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, lambda_d, lambda_e);
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in MRT lattice model and ignoring omega bulk field since not supported!");
LatticeModel_T latticeModel = LatticeModel_T(lbm::collision_model::D3Q19MRT( omegaBulk, omegaBulk, lambda_d, lambda_e, lambda_e, lambda_d ));
#endif
// add PDF field
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field", latticeModel,
Vector3< real_t >( real_t(0) ), real_t(1),
uint_t(1), field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage<FlagField_T>( blocks, "flag field" );
// add body field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor), "boundary handling" );
// set up RPD functionality
std::function<void(void)> syncCall = [&ps,&rpdDomain](){
const real_t overlap = real_t( 1.5 );
mesa_pd::mpi::SyncNextNeighbors syncNextNeighborFunc;
syncNextNeighborFunc(*ps, *rpdDomain, overlap);
};
syncCall();
lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel resetHydrodynamicForceTorque;
real_t lubricationCutOffDistanceNormal = real_t(2) / real_t(3);
real_t lubricationCutOffDistanceTangentialTranslational = real_t(0.5);
real_t lubricationCutOffDistanceTangentialRotational = real_t(0.5);
lbm_mesapd_coupling::LubricationCorrectionKernel lubricationCorrectionKernel(nu, [](real_t){return real_t(0);}, lubricationCutOffDistanceNormal, lubricationCutOffDistanceTangentialTranslational, lubricationCutOffDistanceTangentialRotational );
real_t maximalCutOffDistance = std::max(lubricationCutOffDistanceNormal, std::max(lubricationCutOffDistanceTangentialTranslational,lubricationCutOffDistanceTangentialRotational ));
lbm_mesapd_coupling::RegularParticlesSelector sphereSelector;
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
///////////////
// TIME LOOP //
///////////////
// map particles into the LBM simulation
if(useSBB)
{
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, movingParticleMappingKernel, *accessor, MO_SBB_Flag);
} else
{
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, movingParticleMappingKernel, *accessor, MO_CLI_Flag);
}
// create the timeloop
SweepTimeloop timeloop( blocks->getBlockStorage(), timesteps );
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
blockforest::communication::UniformBufferedScheme< Stencil_T > fullPDFCommunicationScheme( blocks );
fullPDFCommunicationScheme.addPackInfo( make_shared< field::communication::PackInfo< PdfField_T > >( pdfFieldID ) ); // full sync
timeloop.addFuncBeforeTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "Remaining Time Logger" );
if( vtkIOFreq != uint_t(0) )
{
// spheres
auto particleVtkOutput = make_shared<mesa_pd::vtk::ParticleVtkOutput>(ps);
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleOwner>("owner");
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleLinearVelocity>("velocity");
auto particleVtkWriter = vtk::createVTKOutput_PointData(particleVtkOutput, "Particles", vtkIOFreq, baseFolder, "simulation_step");
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( particleVtkWriter ), "VTK (sphere data)" );
// flag field (written only once in the first time step, ghost layers are also written)
auto flagFieldVTK = vtk::createVTKOutput_BlockData( blocks, "flag_field", vtkIOFreq, FieldGhostLayers, false, baseFolder );
flagFieldVTK->addCellDataWriter( make_shared< field::VTKWriter< FlagField_T > >( flagFieldID, "FlagField" ) );
vtk::writeFiles( flagFieldVTK )();
// pdf field
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", vtkIOFreq, 0, false, baseFolder );
pdfFieldVTK->addBeforeFunction( fullPDFCommunicationScheme );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
pdfFieldVTK->addCellInclusionFilter( fluidFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
// omega bulk field
timeloop.addFuncBeforeTimeStep( field::createVTKOutput<ScalarField_T, float>( omegaBulkFieldID, *blocks, "omega_bulk_field", vtkIOFreq, uint_t(0), false, baseFolder ), "VTK (omega bulk field)" );
}
// update bulk omega in all cells to adapt to changed particle position
if(useOmegaBulkAdaption)
{
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, decltype(sphereSelector)>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, sphereSelector);
for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt)
{
(*omegaBulkAdapter)(&(*blockIt));
}
}
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep( BoundaryHandling_T::getBlockSweep( boundaryHandlingID ), "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
timeloop.add() << Sweep( lbmSweep, "LB sweep" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
timeloop.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
Vector3<real_t> hydForce(0.);
Vector3<real_t> lubForce(0.);
Vector3<real_t> hydTorque(0.);
Vector3<real_t> lubTorque(0.);
real_t curForceNorm = real_t(0);
real_t oldForceNorm = real_t(0);
real_t curTorqueNorm = real_t(0);
real_t oldTorqueNorm = real_t(0);
real_t convergenceLimit = real_t(1e-5);
// time loop
for (uint_t i = 1; i <= timesteps; ++i )
{
// perform a single simulation step -> this contains LBM and setting of the hydrodynamic interactions
timeloop.singleStep( timeloopTiming );
// lubrication correction
mesa_pd::collision_detection::AnalyticContactDetection acd;
acd.getContactThreshold() = maximalCutOffDistance;
{
auto idx1 = accessor->uidToIdx(id1);
if( idx1 != accessor->getInvalidIdx() )
{
auto idx2 = accessor->uidToIdx(id2);
if( idx2 != accessor->getInvalidIdx() )
{
mesa_pd::kernel::DoubleCast double_cast;
mesa_pd::mpi::ContactFilter contact_filter;
if (double_cast(idx1, idx2, *accessor, acd, *accessor ))
{
if (contact_filter(acd.getIdx1(), acd.getIdx2(), *accessor, acd.getContactPoint(), *rpdDomain))
{
double_cast(acd.getIdx1(), acd.getIdx2(), *accessor, lubricationCorrectionKernel, *accessor, acd.getContactNormal(), acd.getPenetrationDepth());
}
}
}
}
}
if( i% 100 == 0 && i > 1 )
{
oldForceNorm = curForceNorm;
oldTorqueNorm = curTorqueNorm;
hydForce.reset();
lubForce.reset();
hydTorque.reset();
lubTorque.reset();
auto idx1 = accessor->uidToIdx(id1);
if( idx1 != accessor->getInvalidIdx() )
{
hydForce = accessor->getHydrodynamicForce(idx1);
lubForce = accessor->getForce(idx1);
hydTorque = accessor->getHydrodynamicTorque(idx1);
lubTorque= accessor->getTorque(idx1);
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( hydForce, mpi::SUM );
mpi::reduceInplace( lubForce, mpi::SUM );
mpi::allReduceInplace( hydTorque, mpi::SUM );
mpi::reduceInplace( lubTorque, mpi::SUM );
}
curForceNorm = hydForce.length();
curTorqueNorm = hydTorque.length();
real_t forceDiff = std::fabs((curForceNorm - oldForceNorm) / oldForceNorm);
real_t torqueDiff = std::fabs((curTorqueNorm - oldTorqueNorm) / oldTorqueNorm);
WALBERLA_LOG_INFO_ON_ROOT( "F/Fs = " << hydForce/fStokes << " ( " << forceDiff << " ), T/Ts = " << hydTorque/tStokes << " ( " << torqueDiff << " )");
if( i == 100 ) {
WALBERLA_LOG_INFO_ON_ROOT("Flub = " << lubForce << ", Tlub = " << lubTorque);
WALBERLA_LOG_INFO_ON_ROOT("Flub/Fs = " << lubForce/fStokes << ", Tlub/Ts = " << lubTorque/tStokes);
}
if( forceDiff < convergenceLimit && torqueDiff < convergenceLimit )
{
WALBERLA_LOG_INFO_ON_ROOT("Force and torque norms converged - terminating simulation");
break;
}
}
// reset forces
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor,
[](const size_t idx, auto& ac){
ac.getForceRef(idx) = Vector3<real_t>(real_t(0));
ac.getTorqueRef(idx) = Vector3<real_t>(real_t(0));
}, *accessor );
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
}
if( fileIO )
{
std::string loggingFileName( baseFolder + "/Logging" );
if( sphSphTest ) loggingFileName += "_SphSph";
else loggingFileName += "_SphPla";
loggingFileName += "_Setup" + std::to_string(setup);
loggingFileName += "_gapSize" + std::to_string(uint_c(gapSize*real_t(100)));
loggingFileName += "_radius" + std::to_string(uint_c(radius));
loggingFileName += "_bvrf" + std::to_string(uint_c(bulkViscRateFactor));
loggingFileName += "_mn" + std::to_string(float(magicNumber));
if( useOmegaBulkAdaption ) loggingFileName += "_uOBA" + std::to_string(uint_c(adaptionLayerSize));
if( useSBB ) loggingFileName += "_SBB";
if( !fileNameEnding.empty()) loggingFileName += "_" + fileNameEnding;
loggingFileName += ".txt";
WALBERLA_ROOT_SECTION()
{
std::ofstream file1;
file1.open( loggingFileName.c_str(), std::ofstream::app );
file1.setf( std::ios::unitbuf );
file1.precision(15);
file1 << radius << " " << gapSize << " " << fStokes << " "
<< hydForce[0] << " " << hydForce[1] << " " << hydForce[2] << " "
<< lubForce[0] << " " << lubForce[1] << " " << lubForce[2] << " "
<< hydTorque[0] << " " << hydTorque[1] << " " << hydTorque[2] << " "
<< lubTorque[0] << " " << lubTorque[1] << " " << lubTorque[2] << std::endl;
file1.close();
}
}
timeloopTiming.logResultOnRoot();
return EXIT_SUCCESS;
}
} // namespace lubrication_force_evaluation
int main( int argc, char **argv ){
lubrication_force_evaluation::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/MotionSettlingSphere.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file MotionSettlingSphere.cpp
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "boundary/all.h"
#include "blockforest/all.h"
#include "core/all.h"
#include "domain_decomposition/all.h"
#include "field/AddToStorage.h"
#include "field/communication/PackInfo.h"
#include "field/vtk/all.h"
#include "lbm/boundary/all.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm/vtk/all.h"
#include "lbm_mesapd_coupling/mapping/ParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/SimpleBB.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/Reconstructor.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/ExtrapolationDirectionFinder.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/PdfReconstructionManager.h"
#include "lbm_mesapd_coupling/utility/AddForceOnParticlesKernel.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/utility/AverageHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/AddHydrodynamicInteractionKernel.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "lbm_mesapd_coupling/utility/InspectionProbe.h"
#include "lbm_mesapd_coupling/utility/InitializeHydrodynamicForceTorqueForAveragingKernel.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/shape/Sphere.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/domain/BlockForestDataHandling.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/kernel/VelocityVerlet.h"
#include "mesa_pd/kernel/ExplicitEuler.h"
#include "mesa_pd/mpi/SyncNextNeighbors.h"
#include "mesa_pd/mpi/SyncGhostOwners.h"
#include "mesa_pd/mpi/ReduceProperty.h"
#include "mesa_pd/mpi/notifications/ForceTorqueNotification.h"
#include "mesa_pd/mpi/notifications/HydrodynamicForceTorqueNotification.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "stencil/D3Q27.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/BlockCellDataWriter.h"
#include "vtk/Initialization.h"
#include "vtk/VTKOutput.h"
#include <functional>
#include <memory>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBM.h"
#endif
namespace motion_settling_sphere{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
//////////////
// TYPEDEFS //
//////////////
using namespace walberla;
using walberla::uint_t;
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBM;
#else
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag ( "fluid" );
const FlagUID UBB_Flag ( "velocity bounce back" );
const FlagUID Outlet_Flag ( "outlet" );
const FlagUID MEM_BB_Flag ( "moving obstacle BB" );
const FlagUID MEM_CLI_Flag ( "moving obstacle CLI" );
const FlagUID FormerMEM_Flag( "former moving obstacle" );
//////////////////////////////
// Coupling algorithm enums //
//////////////////////////////
enum MEMVariant { BB, CLI, MR };
MEMVariant to_MEMVariant( const std::string& s )
{
if( s == "BB" ) return MEMVariant::BB;
if( s == "CLI" ) return MEMVariant::CLI;
throw std::runtime_error("invalid conversion from text to MEMVariant");
}
std::string MEMVariant_to_string ( const MEMVariant& m )
{
if( m == MEMVariant::BB ) return "BB";
if( m == MEMVariant::CLI ) return "CLI";
throw std::runtime_error("invalid conversion from MEMVariant to string");
}
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using UBB_T = lbm::SimpleUBB< LatticeModel_T, flag_t >;
using Outlet_T = lbm::SimplePressure< LatticeModel_T, flag_t >;
using MO_SBB_T = lbm_mesapd_coupling::SimpleBB< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using MO_CLI_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, UBB_T, Outlet_T, MO_SBB_T, MO_CLI_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac,
Vector3<real_t> uInfty) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ),
ac_( ac ), velocity_( uInfty )
{}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
FlagField_T * flagField = block->getData< FlagField_T >( flagFieldID_ );
PdfField_T * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
UBB_T( "UBB", UBB_Flag, pdfField, velocity_),
Outlet_T( "Outlet", Outlet_Flag, pdfField, real_t(1) ),
MO_SBB_T ( "MEM_BB", MEM_BB_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ),
MO_CLI_T( "MEM_CLI", MEM_CLI_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ) );
const auto ubb = flagField->getFlag( UBB_Flag );
const auto outlet = flagField->getFlag( Outlet_Flag );
CellInterval domainBB = storage->getDomainCellBB();
domainBB.xMin() -= cell_idx_c( FieldGhostLayers ); // -1
domainBB.xMax() += cell_idx_c( FieldGhostLayers ); // cellsX
domainBB.yMin() -= cell_idx_c( FieldGhostLayers ); // 0
domainBB.yMax() += cell_idx_c( FieldGhostLayers ); // cellsY+1
domainBB.zMin() -= cell_idx_c( FieldGhostLayers ); // 0
domainBB.zMax() += cell_idx_c( FieldGhostLayers ); // cellsZ+1
// BOTTOM
// -1........-1........-1
// cellsX+1..cellsY+1..-1
CellInterval bottom( domainBB.xMin(), domainBB.yMin(), domainBB.zMin(), domainBB.xMax(), domainBB.yMax(), domainBB.zMin() );
storage->transformGlobalToBlockLocalCellInterval( bottom, *block );
handling->forceBoundary( ubb, bottom );
// TOP
// -1........-1........cellsZ+1
// cellsX+1..cellsY+1..cellsZ+1
CellInterval top( domainBB.xMin(), domainBB.yMin(), domainBB.zMax(), domainBB.xMax(), domainBB.yMax(), domainBB.zMax() );
storage->transformGlobalToBlockLocalCellInterval( top, *block );
handling->forceBoundary( outlet, top );
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
const Vector3<real_t> velocity_;
}; // class MyBoundaryHandling
/*
* Functionality for continuous logging of sphere properties during initial (resting) simulation
*/
template< typename ParticleAccessor_T>
class RestingSphereForceEvaluator
{
public:
RestingSphereForceEvaluator( const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid,
uint_t averageFrequency, const std::string & basefolder ) :
ac_( ac ), sphereUid_( sphereUid ), averageFrequency_( averageFrequency )
{
WALBERLA_ROOT_SECTION() {
// initially write header in file
std::ofstream file;
filename_ = basefolder;
filename_ += "/log_init.txt";
file.open(filename_.c_str());
file << "# f_z_current f_z_average f_x f_y\n";
file.close();
}
}
// evaluate the acting drag force on a fixed sphere
void operator()(const uint_t timestep)
{
// average the force over averageFrequency consecutive timesteps since there are small fluctuations in the force
auto currentForce = getForce();
currentAverage_ += currentForce[2];
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( filename_.c_str(), std::ofstream::app );
file.precision(8);
file << timestep << "\t" << currentForce[2] << "\t" << dragForceNew_
<< "\t" << currentForce[0] << "\t" << currentForce[1] << std::endl;
file.close();
}
if( timestep % averageFrequency_ != 0) return;
dragForceOld_ = dragForceNew_;
dragForceNew_ = currentAverage_ / real_c( averageFrequency_ );
currentAverage_ = real_t(0);
}
// Return the relative temporal change in the drag force
real_t getForceDiff() const
{
return std::fabs( ( dragForceNew_ - dragForceOld_ ) / dragForceNew_ );
}
real_t getDragForce() const
{
return dragForceNew_;
}
private:
// Obtain the drag force acting on the sphere
Vector3<real_t> getForce()
{
Vector3<real_t> force(real_t( 0 ));
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != ac_->getInvalidIdx())
{
if(!mesa_pd::data::particle_flags::isSet( ac_->getFlags(idx), mesa_pd::data::particle_flags::GHOST))
{
force = ac_->getHydrodynamicForce(idx);
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( force, mpi::SUM );
}
return force;
}
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
real_t currentAverage_ = real_t(0);
uint_t averageFrequency_;
std::string filename_;
real_t dragForceOld_ = real_t(0);
real_t dragForceNew_ = real_t(0);
};
/*
* Functionality for continuous logging of sphere properties during moving simulation
*/
template< typename ParticleAccessor_T>
class MovingSpherePropertyEvaluator
{
public:
MovingSpherePropertyEvaluator( const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid, const std::string & basefolder,
const Vector3<real_t> & u_infty, const real_t Galileo, const real_t GalileoSim,
const real_t gravity, const real_t viscosity, const real_t diameter,
const real_t densityRatio ) :
ac_( ac ), sphereUid_( sphereUid ),
u_infty_( u_infty ), gravity_( gravity ), viscosity_( viscosity ),
diameter_( diameter ), densityRatio_( densityRatio )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream fileSetup;
std::string filenameSetup = basefolder;
filenameSetup += "/setup.txt";
fileSetup.open( filenameSetup.c_str() );
fileSetup << "# setup parameters: \n";
fileSetup << "processes = " << MPIManager::instance()->numProcesses() << "\n";
fileSetup << "Galileo number (targeted) = " << Galileo << "\n";
fileSetup << "Galileo number (simulated) = " << GalileoSim << "\n";
fileSetup << "gravity = " << gravity << "\n";
fileSetup << "viscosity = " << viscosity << "\n";
fileSetup << "diameter = " << diameter << "\n";
fileSetup << "uInfty = " << u_infty_[2] << "\n";
real_t u_ref = std::sqrt( std::fabs(densityRatio - real_t(1)) * gravity * diameter );
fileSetup << "u_{ref} = " << u_ref << "\n";
fileSetup.close();
}
filenameRes_ = basefolder;
filenameRes_ += "/log_results.txt";
filenameAdd_ = basefolder;
filenameAdd_ += "/log_additional.txt";
WALBERLA_ROOT_SECTION()
{
// initially write header in file
std::ofstream fileRes;
fileRes.open( filenameRes_.c_str() );
// raw data
fileRes << "#\t x_p_x\t x_p_y\t x_p_z\t u_p_x\t u_p_y\t u_p_z\t omega_p_x\t omega_p_y\t omega_p_z\t u_{pr}_x\t u_{pr}_y\t u_{pr}_z\t ";
// processed data, 14 - 18
fileRes << "Re\t u_{pV}\t u_{pH}\t omega_{pV}\t omega_{pH}\t alpha\n";
fileRes.close();
std::ofstream fileAdd;
fileAdd.open( filenameAdd_.c_str() );
fileAdd << "# forceX forceY forceZ torqueX torqueY torqueZ\n";
fileAdd.close();
}
}
// evaluate and write the sphere properties
void operator()(const uint_t timestep)
{
Vector3<real_t> particleTransVel( real_t(0) );
Vector3<real_t> particleAngularVel( real_t(0) );
Vector3<real_t> particlePos( real_t(0) );
Vector3<real_t> particleForce( real_t(0) );
Vector3<real_t> particleTorque( real_t(0) );
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != ac_->getInvalidIdx())
{
if(!mesa_pd::data::particle_flags::isSet( ac_->getFlags(idx), mesa_pd::data::particle_flags::GHOST))
{
particlePos = ac_->getPosition(idx);
particleTransVel = ac_->getLinearVelocity(idx);
particleAngularVel = ac_->getAngularVelocity(idx);
particleForce = ac_->getHydrodynamicForce(idx);
particleTorque = ac_->getHydrodynamicTorque(idx);
}
}
// reduce to root
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( particlePos, mpi::SUM );
mpi::reduceInplace( particleTransVel, mpi::SUM );
mpi::reduceInplace( particleAngularVel, mpi::SUM );
mpi::reduceInplace( particleForce, mpi::SUM );
mpi::reduceInplace( particleTorque, mpi::SUM );
}
particleHeight_ = particlePos[2];
//only root process has all the data
WALBERLA_ROOT_SECTION()
{
Vector3<real_t> u_p_r( particleTransVel[0] - u_infty_[0], particleTransVel[1] - u_infty_[1], particleTransVel[2] - u_infty_[2]);
real_t u_p_H = std::sqrt( u_p_r[0] * u_p_r[0] + u_p_r[1] * u_p_r[1]);
real_t u_p_V = u_p_r[2];
real_t omega_p_H = std::sqrt( particleAngularVel[0] * particleAngularVel[0] + particleAngularVel[1] * particleAngularVel[1] );
real_t omega_p_V = particleAngularVel[2];
real_t u_ref = std::sqrt( std::fabs(densityRatio_ - real_t(1)) * gravity_ * diameter_ );
real_t Reynolds = u_p_r.length() * diameter_ / viscosity_;
// results
std::ofstream fileRes;
fileRes.open( filenameRes_.c_str(), std::ofstream::app );
fileRes.precision(8);
fileRes << timestep
<< "\t" << particlePos[0] << "\t" << particlePos[1] << "\t" << particlePos[2]
<< "\t" << particleTransVel[0] << "\t" << particleTransVel[1] << "\t" << particleTransVel[2]
<< "\t" << particleAngularVel[0] << "\t" << particleAngularVel[1] << "\t" << particleAngularVel[2]
<< "\t" << u_p_r[0] << "\t" << u_p_r[1] << "\t" << u_p_r[2]
<< "\t" << Reynolds << "\t" << u_p_V/u_ref << "\t" << u_p_H/u_ref
<< "\t" << omega_p_V*(diameter_/u_ref) << "\t" << omega_p_H*(diameter_/u_ref)
<< "\t" << std::atan( u_p_H/std::fabs(u_p_V) )
<< std::endl;
fileRes.close();
// additional
std::ofstream fileAdd;
fileAdd.open( filenameAdd_.c_str(), std::ofstream::app );
fileAdd.precision(8);
fileAdd << timestep
<< "\t" << particleForce[0] << "\t" << particleForce[1] << "\t" << particleForce[2]
<< "\t" << particleTorque[0] << "\t" << particleTorque[1] << "\t" << particleTorque[2]
<< std::endl;
fileAdd.close();
}
}
real_t getParticleHeight() const
{
return particleHeight_;
}
private:
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
std::string filenameRes_;
std::string filenameAdd_;
Vector3<real_t> u_infty_;
real_t gravity_;
real_t viscosity_;
real_t diameter_;
real_t densityRatio_;
real_t particleHeight_;
};
/*
* Result evaluation for the written VTK files
*
* input: vel (fluid velocity), u_p (particle velocity), u_infty (inflow velocity)
*
* needed data:
* relative flow velocity: vel_r = vel - u_p
* relative particle velocity: u_p_r = u_p - u_infty
*
* magnitude of relative particle velocity in horizontal plane: u_p_H = sqrt( (u_p_r)_x^2 + (u_p_r)_y^2 )
* unit vector of particle motion in horizontal plane: e_p_H = ( (u_p_r)_x, (u_p_r)_y, 0) / u_p_H
* unit vector perpendicular to e_p_H and e_z: e_p_Hz_perp = ( -(u_p_r)_y, (u_p_r)_x, 0) / u_p_H
* unit vector of particle motion: e_p_parallel = u_p_r / ||u_p_r||
* unit vector perpendicular to e_p_parallel and e_p_Hz_perp: e_p_perp = e_p_Hz_perp x e_p_parallel
* projected fluid velocities: vel_r_parallel = vel_r * (-e_p_parallel)
* vel_r_perp = vel_r * e_p_perp
* vel_r_Hz_perp = vel_r * e_p_Hz_perp
*/
template< typename ParticleAccessor_T>
class VTKInfoLogger
{
public:
VTKInfoLogger( SweepTimeloop* timeloop, const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid,
const std::string & baseFolder, const Vector3<real_t>& u_infty ) :
timeloop_( timeloop ), ac_( ac ), sphereUid_( sphereUid ), baseFolder_( baseFolder ), u_infty_( u_infty )
{ }
void operator()()
{
Vector3<real_t> u_p( real_t(0) );
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != ac_->getInvalidIdx())
{
if(!mesa_pd::data::particle_flags::isSet( ac_->getFlags(idx), mesa_pd::data::particle_flags::GHOST))
{
u_p = ac_->getLinearVelocity(idx);
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( u_p, mpi::SUM );
}
Vector3<real_t> u_p_r = u_p - u_infty_;
real_t u_p_r_sqr_mag = u_p_r.sqrLength();
real_t u_p_H = std::sqrt( u_p_r[0] * u_p_r[0] + u_p_r[1] * u_p_r[1]);
Vector3<real_t> e_p_H (u_p_r[0], u_p_r[1], real_t(0));
e_p_H /= u_p_H;
Vector3<real_t> e_p_Hz_perp (-u_p_r[1], u_p_r[0], real_t(0));
e_p_Hz_perp /= u_p_H;
Vector3<real_t> e_p_parallel = u_p_r / u_p_r.length();
Vector3<real_t> e_p_perp = e_p_Hz_perp%e_p_parallel;
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
std::string filename(baseFolder_);
filename += "/log_vtk/log_vtk_";
filename += std::to_string( timeloop_->getCurrentTimeStep() );
filename += ".txt";
file.open( filename.c_str() );
file.precision(8);
file << "u_p = " << u_p << "\n";
file << "u_p_r_2 = " << u_p_r_sqr_mag << "\n\n";
file << "e_{pH} = " << e_p_H << "\n";
file << "e_{pparallel} = " << e_p_parallel << "\n";
file << "e_{pperp} = " << e_p_perp << "\n";
file << "e_{pHzperp} = " << e_p_Hz_perp << "\n";
file.close();
}
}
private:
SweepTimeloop* timeloop_;
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
std::string baseFolder_;
Vector3<real_t> u_infty_;
};
/*
* This extensive, physical test case simulates a single, heavy sphere in ambient fluid flow.
* It is based on the benchmark proposed in
* Uhlmann, Dusek - "The motion of a single heavy sphere in ambient fluid: A benchmark for interface-resolved
* particulate flow simulations with significant relative velocities" IJMF (2014),
* doi: 10.1016/j.ijmultiphaseflow.2013.10.010
* Results for LBM done with waLBerla are published in
* Rettinger, Ruede - "A comparative study of fluid-particle coupling methods for fully resolved
* lattice Boltzmann simulations" CaF (2017),
* doi: 10.1016/j.compfluid.2017.05.033
* The setup and the benchmark itself are explained in detail in these two papers.
* Several logging files are written to evaluate the benchmark quantities.
*
* This case is the same as can be found in apps/benchmarks/MotionSingleHeavySphere
* but using the lbm_mesapd_coupling, instead of the pe_coupling.
* Compared to this previous work, significantly different outcomes can be observed when using the CLI boundary condition.
* This can be traced back to the distinct way of force averaging.
* The force averaging applied in the previous study (two LBM steps, then average) introduces a error in the Galilean invariance.
* Surprisingly, this error can result in better agreement to the reference solutions.
*
*/
int main( int argc, char **argv )
{
debug::enterTestMode();
mpi::Environment env( argc, argv );
bool noViscosityIterations = false;
bool vtkIOInit = false;
bool longDom = false;
bool broadDom = false;
bool useOmegaBulkAdaption = false;
real_t bulkViscRateFactor = real_t(1);
uint_t averageForceTorqueOverTwoTimeStepsMode = 1; // 0: no, 1: running, 2: 2LBM
bool conserveMomentum = false;
bool useDiffusiveScaling = true;
bool useVelocityVerlet = true;
bool initFluidVelocity = true;
bool vtkOutputGhostlayers = false;
MEMVariant memVariant = MEMVariant::BB;
std::string reconstructorType = "Grad"; // Eq, EAN, Ext, Grad
real_t diameter = real_t(18);
uint_t XBlocks = uint_t(4);
uint_t YBlocks = uint_t(4);
real_t viscosity = real_t(0.01); // used in diffusive scaling
real_t uIn = real_t(0.02); // used for acoustic scaling
/*
* 0: custom
* 1: Ga = 144
* 2: Ga = 178.46
* 3: Ga = 190
* 4: Ga = 250
*/
uint_t simulationCase = 1;
real_t Galileo = real_t(1);
real_t Re_target = real_t(1);
real_t timestepsNonDim = real_t(1);
real_t vtkWriteSpacingNonDim = real_t(3); // write every 3 non-dim timesteps
uint_t vtkNumOutputFiles = 10; // write only 10 vtk files: the 10 final ones
std::string folderEnding = "";
////////////////////////////
// COMMAND LINE ARGUMENTS //
////////////////////////////
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--case" ) == 0 ) {simulationCase = uint_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--Ga" ) == 0 ){Galileo = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--Re" ) == 0 ){Re_target = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--timestepsNonDim" ) == 0 ){timestepsNonDim = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--diameter" ) == 0 ){diameter = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--noViscosityIterations" ) == 0 ){noViscosityIterations = true; continue;}
if( std::strcmp( argv[i], "--viscosity" ) == 0 ){viscosity = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--uIn" ) == 0 ){uIn = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--vtkIOInit" ) == 0 ){vtkIOInit = true; continue;}
if( std::strcmp( argv[i], "--longDom" ) == 0 ){longDom = true; continue;}
if( std::strcmp( argv[i], "--broadDom" ) == 0 ){broadDom = true; continue;}
if( std::strcmp( argv[i], "--variant" ) == 0 ){memVariant = to_MEMVariant( argv[++i] ); continue;}
if( std::strcmp( argv[i], "--useOmegaBulkAdaption" ) == 0 ){useOmegaBulkAdaption = true; continue;}
if( std::strcmp( argv[i], "--bvrf" ) == 0 ){bulkViscRateFactor = real_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--forceTorqueAverageMode" ) == 0 ){averageForceTorqueOverTwoTimeStepsMode = uint_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--conserveMomentum" ) == 0 ){conserveMomentum = true; continue;}
if( std::strcmp( argv[i], "--useDiffusiveScaling" ) == 0 ){useDiffusiveScaling = true; continue;}
if( std::strcmp( argv[i], "--XBlocks" ) == 0 ){XBlocks = uint_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--YBlocks" ) == 0 ){YBlocks = uint_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--folderEnding" ) == 0 ){folderEnding = argv[++i]; continue;}
if( std::strcmp( argv[i], "--reconstructorType" ) == 0 ) {reconstructorType = argv[++i]; continue;}
if( std::strcmp( argv[i], "--useEuler" ) == 0 ) {useVelocityVerlet = false; continue;}
if( std::strcmp( argv[i], "--noFluidVelocityInit" ) == 0 ) {initFluidVelocity = false; continue;}
if( std::strcmp( argv[i], "--vtkOutputGhostlayers" ) == 0 ) {vtkOutputGhostlayers = true; continue;}
if( std::strcmp( argv[i], "--vtkNumOutputFiles" ) == 0 ){vtkNumOutputFiles = uint_c( std::atof( argv[++i] ) ); continue;}
if( std::strcmp( argv[i], "--vtkWriteSpacingNonDim" ) == 0 ){vtkWriteSpacingNonDim = real_c( std::atof( argv[++i] ) ); continue;}
WALBERLA_ABORT("command line argument unknown: " << argv[i] );
}
///////////////////////////
// SIMULATION PROPERTIES //
///////////////////////////
const real_t radius = real_t(0.5) * diameter;
uint_t xlength = uint_c( diameter * real_t(5.34) );
uint_t ylength = xlength;
uint_t zlength = uint_c( diameter * real_t(16) );
if(broadDom)
{
xlength *= uint_t(2);
ylength *= uint_t(2);
}
if(longDom)
{
zlength *= uint_t(3);
}
// estimate Reynolds number (i.e. inflow velocity) based on Galileo number
// values are taken from the original simulation of Uhlmann, Dusek
switch( simulationCase )
{
case 0:
WALBERLA_LOG_INFO_ON_ROOT("Using custom simulation case -> you have to provide Ga, Re, and time steps via command line arguments!");
break;
case 1:
Galileo = real_t(144);
Re_target = real_t(185.08);
timestepsNonDim = real_t(100);
break;
case 2:
Galileo = real_t(178.46);
Re_target = real_t(243.01);
timestepsNonDim = real_t(250);
break;
case 3:
Galileo = real_t(190);
Re_target = real_t(262.71);
timestepsNonDim = real_t(250);
break;
case 4:
Galileo = real_t(250);
Re_target = real_t(365.10);
timestepsNonDim = real_t(510);
break;
default:
WALBERLA_ABORT("Simulation case is not supported!");
}
if( useDiffusiveScaling)
{
// estimate fitting inflow velocity (diffusive scaling, viscosity is fixed)
// attention: might lead to large velocities for small diameters
uIn = Re_target * viscosity / diameter;
} else {
// estimate fitting viscosity (acoustic scaling: velocity is fixed)
// note that this is different to the approach taking in the paper, where an diffusive scaling with visc = 0.01 is taken
// attention: might lead to small viscosities for small diameters
viscosity = uIn * diameter / Re_target;
}
real_t omega = lbm::collision_model::omegaFromViscosity(viscosity);
real_t omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
Vector3<real_t> uInfty = Vector3<real_t>( real_t(0), real_t(0), uIn );
const real_t densityRatio = real_t(1.5);
const uint_t averageFrequency = uint_c( ( ( uint_c(Galileo) >= 200) ? real_t(500) : real_t(2) ) * diameter / uIn ); // for initial simulation
const real_t convergenceLimit = real_t(1e-4);
const real_t convergenceLimitGalileo = real_t(1e-4);
const real_t dx = real_t(1);
const real_t magicNumberTRT = lbm::collision_model::TRT::threeSixteenth;
const uint_t timestepsInit = uint_c( ( ( uint_c(Galileo) >= 200) ? real_t(3000) : real_t(100) ) * diameter / uIn ); // maximum number of time steps for the initial simulation
const uint_t writeFrequencyInit = uint_t(1000); // vtk write frequency init
///////////////////////////
// BLOCK STRUCTURE SETUP //
///////////////////////////
const int numProcs = MPIManager::instance()->numProcesses();
uint_t ZBlocks = uint_c(numProcs) / ( XBlocks * YBlocks );
const uint_t XCells = xlength / XBlocks;
const uint_t YCells = ylength / YBlocks;
const uint_t ZCells = zlength / ZBlocks;
if( (xlength != XCells * XBlocks) || (ylength != YCells * YBlocks) || (zlength != ZCells * ZBlocks) )
{
WALBERLA_ABORT("Domain partitioning does not fit to total domain size!");
}
WALBERLA_LOG_INFO_ON_ROOT("Motion settling sphere simulation of case " << simulationCase << " -> Ga = " << Galileo);
WALBERLA_LOG_INFO_ON_ROOT("Diameter: " << diameter);
WALBERLA_LOG_INFO_ON_ROOT("Domain: " << xlength << " x " << ylength << " x " << zlength);
WALBERLA_LOG_INFO_ON_ROOT("Processes: " << XBlocks << " x " << YBlocks << " x " << ZBlocks);
WALBERLA_LOG_INFO_ON_ROOT("Subdomains: " << XCells << " x " << YCells << " x " << ZCells);
WALBERLA_LOG_INFO_ON_ROOT("Tau: " << real_t(1)/omega);
WALBERLA_LOG_INFO_ON_ROOT("Viscosity: " << viscosity);
WALBERLA_LOG_INFO_ON_ROOT("uIn: " << uIn);
WALBERLA_LOG_INFO_ON_ROOT("Re_infty: " << uIn * diameter / viscosity);
auto blocks = blockforest::createUniformBlockGrid( XBlocks, YBlocks, ZBlocks, XCells, YCells, ZCells, dx, true,
true, true, false );
//////////////////
// RPD COUPLING //
//////////////////
auto rpdDomain = std::make_shared<mesa_pd::domain::BlockForestDomain>(blocks->getBlockForestPointer());
//init data structures
auto ps = walberla::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = walberla::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
real_t xParticle = real_t(0);
real_t yParticle = real_t(0);
real_t zParticle = real_t(0);
// root determines particle position, then broadcasts it
WALBERLA_ROOT_SECTION()
{
if( simulationCase == 1 )
{
xParticle = real_c( xlength ) * real_t(0.5);
yParticle = real_c( ylength ) * real_t(0.5);
}
else if( simulationCase == 4 )
{
// add random perturbance for chaotic regime
walberla::math::seedRandomGenerator( std::mt19937::result_type(std::time(nullptr)) );
xParticle = real_c( xlength ) * real_t(0.5) + walberla::math::realRandom( real_t(-0.5), real_t(0.5) );
yParticle = real_c( ylength ) * real_t(0.5) + walberla::math::realRandom( real_t(-0.5), real_t(0.5) );
}
else
{
//add small perturbance to sphere position to break stability due to numerical symmetry
real_t perturbance = real_t(0.35);
xParticle = real_c( xlength ) * real_t(0.5) + perturbance;
yParticle = real_c( ylength ) * real_t(0.5);
}
zParticle = (longDom) ? ( diameter * real_t(16) + real_c( xlength ) ) : real_c( xlength );
}
// broadcast to other ranks
WALBERLA_MPI_SECTION()
{
mpi::broadcastObject( xParticle );
mpi::broadcastObject( yParticle );
mpi::broadcastObject( zParticle );
}
// add sphere
Vector3<real_t> position( xParticle, yParticle, zParticle );
auto sphereShape = ss->create<mesa_pd::data::Sphere>( radius );
ss->shapes[sphereShape]->updateMassAndInertia(densityRatio);
walberla::id_t sphereUid = 0;
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), position ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(position);
p.setInteractionRadius(radius);
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
sphereUid = p.getUid();
}
mpi::allReduceInplace(sphereUid, mpi::SUM);
// set up synchronization procedure
const real_t overlap = real_t( 1.5 ) * dx;
std::function<void(void)> syncCall;
if( XBlocks <= uint_t(4) )
{
WALBERLA_LOG_INFO_ON_ROOT("Using next neighbor sync!")
syncCall = [&ps,&rpdDomain,overlap](){
mesa_pd::mpi::SyncNextNeighbors syncNextNeighborFunc;
syncNextNeighborFunc(*ps, *rpdDomain, overlap);
};
syncCall();
}
else
{
WALBERLA_LOG_INFO_ON_ROOT("Using ghost owner sync!")
syncCall = [&ps,&rpdDomain,overlap](){
mesa_pd::mpi::SyncGhostOwners syncGhostOwnersFunc;
syncGhostOwnersFunc(*ps, *rpdDomain, overlap);
};
for(uint_t i = 0; i < uint_c(XBlocks/2); ++i) syncCall();
}
WALBERLA_LOG_INFO_ON_ROOT("Initial sphere position: " << position);
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumberTRT );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, lambda_d, lambda_e);
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in TRT lattice model and ignoring omega bulk!");
LatticeModel_T latticeModel = LatticeModel_T( lbm::collision_model::TRT::constructWithMagicNumber( omega, magicNumberTRT ) );
#endif
// add PDF field
// initial velocity in domain = inflow velocity
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field", latticeModel, initFluidVelocity ? uInfty : Vector3<real_t>(0), real_t(1), uint_t(1), field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage< FlagField_T >( blocks, "flag field" );
// add particle field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling & initialize outer domain boundaries
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor, uInfty), "boundary handling" );
// kernels
mesa_pd::mpi::ReduceProperty reduceProperty;
lbm_mesapd_coupling::AddHydrodynamicInteractionKernel addHydrodynamicInteraction;
lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel resetHydrodynamicForceTorque;
lbm_mesapd_coupling::AverageHydrodynamicForceTorqueKernel averageHydrodynamicForceTorque;
lbm_mesapd_coupling::InitializeHydrodynamicForceTorqueForAveragingKernel initializeHydrodynamicForceTorqueForAveragingKernel;
lbm_mesapd_coupling::RegularParticlesSelector sphereSelector;
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
// mapping of sphere required by MEM variants
// sets the correct flags
if( memVariant == MEMVariant::CLI )
{
ps->forEachParticle(false, sphereSelector, *accessor, movingParticleMappingKernel, *accessor, MEM_CLI_Flag);
}else {
ps->forEachParticle(false, sphereSelector, *accessor, movingParticleMappingKernel, *accessor, MEM_BB_Flag);
}
// base folder to store all logs and vtk output
std::string basefolder ("MSHS_");
basefolder += std::to_string( uint_c( Galileo ) );
basefolder += "_";
basefolder += std::to_string( uint_c( diameter ) );
basefolder += "_MEM_";
basefolder += MEMVariant_to_string( memVariant );
basefolder += "_bvrf";
basefolder += std::to_string(int(bulkViscRateFactor));
if( useOmegaBulkAdaption ) basefolder += "Adapt";
basefolder += "_" + reconstructorType;
if( longDom ) basefolder += "_longDom";
if( broadDom ) basefolder += "_broadDom";
if( conserveMomentum ) basefolder += "_conserveMomentum";
if( !folderEnding.empty() ) basefolder += "_" + folderEnding;
WALBERLA_LOG_INFO_ON_ROOT("Basefolder for simulation results: " << basefolder);
// create base directory if it does not yet exist
filesystem::path tpath( basefolder );
if( !filesystem::exists( tpath ) )
filesystem::create_directory( tpath );
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
blockforest::communication::UniformBufferedScheme< Stencil_T > fullPDFCommunicationScheme( blocks );
fullPDFCommunicationScheme.addPackInfo( make_shared< field::communication::PackInfo< PdfField_T > >( pdfFieldID ) ); // full sync
// omega bulk adaption
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, decltype(sphereSelector)>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
real_t adaptionLayerSize = real_t(2);
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, sphereSelector);
for (auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt) {
(*omegaBulkAdapter)(blockIt.get());
}
//////////////////////////////////////
// TIME LOOP FOR INITIAL SIMULATION //
//////////////////////////////////////
// Initialization simulation: fixed sphere and simulate until convergence (of drag force)
// create the timeloop
SweepTimeloop timeloopInit( blocks->getBlockStorage(), timestepsInit );
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
auto bhSweep = BoundaryHandling_T::getBlockSweep( boundaryHandlingID );
timeloopInit.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep(bhSweep, "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
timeloopInit.add() << Sweep( lbmSweep, "LB sweep" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
timeloopInit.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
if( vtkIOInit )
{
auto pdfFieldVTKInit = vtk::createVTKOutput_BlockData( blocks, "fluid_field_init", writeFrequencyInit, 0, false, basefolder );
field::FlagFieldCellFilter< FlagField_T > fluidFilterInit( flagFieldID );
fluidFilterInit.addFlag( Fluid_Flag );
pdfFieldVTKInit->addCellInclusionFilter( fluidFilterInit );
pdfFieldVTKInit->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTKInit->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloopInit.addFuncAfterTimeStep( vtk::writeFiles( pdfFieldVTKInit ), "VTK (fluid field data)" );
}
timeloopInit.addFuncAfterTimeStep( RemainingTimeLogger( timeloopInit.getNrOfTimeSteps(), real_t(30) ), "Remaining Time Logger" );
////////////////////////////////
// EXECUTE INITIAL SIMULATION //
////////////////////////////////
real_t gravity = real_t(1);
real_t GalileoSim = real_t(1);
real_t ReynoldsSim = real_t(1);
real_t u_ref = real_t(1);
WALBERLA_LOG_INFO_ON_ROOT("Starting initialization phase (sphere is kept fixed).");
WALBERLA_LOG_INFO_ON_ROOT("Iterating, and adapting the viscosity, until the targeted Galileo number is set.");
RestingSphereForceEvaluator<ParticleAccessor_T> forceEval( accessor, sphereUid, averageFrequency, basefolder );
while (true) {
WcTimingPool timeloopInitTiming;
WALBERLA_LOG_INFO_ON_ROOT("(Re-)Starting initial simulation.");
for( uint_t i = 1; i < timestepsInit; ++i ){
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
timeloopInit.singleStep( timeloopInitTiming );
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
forceEval(timeloopInit.getCurrentTimeStep()+1);
// check if the relative change in the average drag force is below the specified convergence criterion
if (forceEval.getForceDiff() < convergenceLimit && i > 2 * std::max(averageFrequency, zlength) )
{
// conditions to break:
// - force diff sufficiently small
// - AND more time steps than twice the domain size in z direction to ensure new information due
// to possible viscosity change has traveled twice through domain,
// or twice the average frequency to have converged averages
WALBERLA_LOG_INFO_ON_ROOT("Drag force converged at time " << timeloopInit.getCurrentTimeStep());
break;
}
// if simulation gets unstable
if( std::isnan(forceEval.getDragForce()) )
{
WALBERLA_ABORT("NAN value detected in drag force during initial simulation, exiting....");
}
}
WALBERLA_LOG_INFO_ON_ROOT("Initial simulation has ended.")
//evaluate the gravitational force necessary to keep the sphere at a approximately fixed position
gravity = forceEval.getDragForce() / ( (densityRatio - real_t(1) ) * diameter * diameter * diameter * math::pi / real_t(6) );
GalileoSim = std::sqrt( ( densityRatio - real_t(1) ) * gravity * diameter * diameter * diameter ) / viscosity;
ReynoldsSim = uIn * diameter / viscosity;
u_ref = std::sqrt( std::fabs(densityRatio - real_t(1)) * gravity * diameter );
WALBERLA_LOG_INFO_ON_ROOT("Acting gravity = " << gravity );
WALBERLA_LOG_INFO_ON_ROOT("Simulated Galileo number = " << GalileoSim );
WALBERLA_LOG_INFO_ON_ROOT("Targeted Galileo number = " << Galileo );
WALBERLA_LOG_INFO_ON_ROOT("Reynolds number infty = " << ReynoldsSim );
if ( noViscosityIterations )
{
timeloopInitTiming.logResultOnRoot();
WALBERLA_LOG_INFO_ON_ROOT("Terminate iterations since viscosity should not be changed, flag \"--noViscosityIterations\"");
break;
}
// if simulated Galileo number is close enough to targeted Galileo number, stop the initial simulation
if( std::abs( GalileoSim - Galileo ) / Galileo < convergenceLimitGalileo )
{
timeloopInitTiming.logResultOnRoot();
WALBERLA_LOG_INFO_ON_ROOT("Iterations converged, simulated Galileo number is close enough to targeted one");
break;
}
// else update the simulation parameter accordingly and resume simulation
real_t diff = GalileoSim/Galileo;
viscosity = diff * viscosity;
omega = lbm::collision_model::omegaFromViscosity( viscosity );
lambda_e = lbm::collision_model::TRT::lambda_e( omega );
lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumberTRT );
// also adapt omega bulk
omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
omegaBulkAdapter->setDefaultOmegaBulk(defaultOmegaBulk);
omegaBulkAdapter->setAdaptedOmegaBulk(omegaBulk);
// iterate all blocks with an iterator 'block' and change the collision model
for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt )
{
// get the field data out of the block
auto pdf = blockIt->getData< PdfField_T > ( pdfFieldID );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
pdf->latticeModel().omega_magic_ = lambda_d;
pdf->latticeModel().omega_visc_ = lambda_e;
#else
pdf->latticeModel().collisionModel().resetWithMagicNumber( omega, magicNumberTRT );
#endif
(*omegaBulkAdapter)(blockIt.get());
}
WALBERLA_LOG_INFO_ON_ROOT("==> Adapting viscosity:");
WALBERLA_LOG_INFO_ON_ROOT("New viscosity = " << viscosity );
WALBERLA_LOG_INFO_ON_ROOT("New omega = " << omega );
WALBERLA_LOG_INFO_ON_ROOT("New omega bulk = " << omegaBulk );
}
if( averageForceTorqueOverTwoTimeStepsMode == 1 )
{
// maintain a good initial guess for averaging
ps->forEachParticle(false, mesa_pd::kernel::SelectLocal(), *accessor, initializeHydrodynamicForceTorqueForAveragingKernel, *accessor );
}
ps->forEachParticle(false, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
///////////////
// TIME LOOP //
///////////////
// actual simulation: freely moving sphere with acting gravity
// calculate the number of timesteps
real_t dtSim = (averageForceTorqueOverTwoTimeStepsMode == 2) ? real_t(2) : real_t(1);
const real_t t_ref = ( diameter / u_ref );
const uint_t timesteps = uint_c( timestepsNonDim * t_ref / dtSim );
// set vtk write frequency accordingly
const uint_t writeFrequency = uint_c( vtkWriteSpacingNonDim * t_ref ); // vtk write frequency
const uint_t initWriteCallsToSkip = uint_c(std::max( 0, int( timesteps - ( vtkNumOutputFiles - uint_t(1) ) * writeFrequency - uint_t(1) ) ) ); // write calls to be skipped
WALBERLA_LOG_INFO_ON_ROOT("Starting simulation timeloop with " << timesteps << " timesteps!");
WALBERLA_LOG_INFO_ON_ROOT("Sphere is allowed to move freely under action of gravity");
SweepTimeloop timeloop( blocks->getBlockStorage(), uint_c(dtSim) * timesteps );
timeloop.addFuncBeforeTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps(), real_t(30) ), "Remaining Time Logger" );
// vtk output
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", writeFrequency, (vtkOutputGhostlayers) ? FieldGhostLayers : 0, false, basefolder );
pdfFieldVTK->setInitialWriteCallsToSkip( initWriteCallsToSkip );
pdfFieldVTK->addBeforeFunction( fullPDFCommunicationScheme );
// function to output plane infos for vtk output
pdfFieldVTK->addBeforeFunction(VTKInfoLogger<ParticleAccessor_T>( &timeloop, accessor, sphereUid, basefolder, uInfty ));
// create folder for log_vtk files to not pollute the basefolder
filesystem::path tpath2( basefolder+"/log_vtk" );
if( !filesystem::exists( tpath2 ) )
filesystem::create_directory( tpath2 );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
pdfFieldVTK->addCellInclusionFilter( fluidFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep(bhSweep, "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
timeloop.add() << Sweep( lbmSweep, "LB sweep" );
#else
timeloop.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
SweepTimeloop timeloopAfterParticles( blocks->getBlockStorage(), timesteps );
// sweep for updating the pe body mapping into the LBM simulation
if( memVariant == MEMVariant::CLI )
timeloopAfterParticles.add() << Sweep( lbm_mesapd_coupling::makeMovingParticleMapping<PdfField_T, BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, MEM_CLI_Flag, FormerMEM_Flag, sphereSelector, conserveMomentum), "Particle Mapping" );
else
timeloopAfterParticles.add() << Sweep( lbm_mesapd_coupling::makeMovingParticleMapping<PdfField_T, BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, MEM_BB_Flag, FormerMEM_Flag, sphereSelector, conserveMomentum), "Particle Mapping" );
// sweep for restoring PDFs in cells previously occupied by particles
if( reconstructorType == "EAN")
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
auto equilibriumAndNonEquilibriumSphereNormalReconstructor = lbm_mesapd_coupling::makeEquilibriumAndNonEquilibriumReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
auto reconstructionManager = lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMEM_Flag, Fluid_Flag, equilibriumAndNonEquilibriumSphereNormalReconstructor, conserveMomentum);
timeloopAfterParticles.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(reconstructionManager), "PDF Restore" );
} else if( reconstructorType == "Ext" )
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
auto extrapolationSphereNormalReconstructor = lbm_mesapd_coupling::makeExtrapolationReconstructor<BoundaryHandling_T,lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder,true>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
timeloopAfterParticles.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMEM_Flag, Fluid_Flag, extrapolationSphereNormalReconstructor, conserveMomentum) ), "PDF Restore" );
} else if( reconstructorType == "Grad")
{
auto gradReconstructor = lbm_mesapd_coupling::makeGradsMomentApproximationReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, omega, false, true, true);
timeloopAfterParticles.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMEM_Flag, Fluid_Flag, gradReconstructor, conserveMomentum) ), "PDF Restore" );
} else if( reconstructorType == "Eq")
{
timeloopAfterParticles.add() << Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMEM_Flag, Fluid_Flag, conserveMomentum) ), "PDF Restore" );
} else {
WALBERLA_ABORT("Unknown reconstructor type " << reconstructorType);
}
// update bulk omega in all cells to adapt to changed particle position
if( useOmegaBulkAdaption )
{
timeloopAfterParticles.add() << Sweep( makeSharedSweep(omegaBulkAdapter), "Omega Bulk Adapter");
}
Vector3<real_t> extForcesOnSphere( real_t(0), real_t(0), - gravity * ( densityRatio - real_t(1) ) * diameter * diameter * diameter * math::pi / real_t(6));
lbm_mesapd_coupling::AddForceOnParticlesKernel addGravitationalForce(extForcesOnSphere);
mesa_pd::kernel::VelocityVerletPreForceUpdate vvIntegratorPreForce(dtSim);
mesa_pd::kernel::VelocityVerletPostForceUpdate vvIntegratorPostForce(dtSim);
mesa_pd::kernel::ExplicitEuler explicitEulerIntegrator(dtSim);
MovingSpherePropertyEvaluator<ParticleAccessor_T> movingSpherePropertyEvaluator( accessor, sphereUid, basefolder, uInfty, Galileo, GalileoSim, gravity, viscosity, diameter, densityRatio );
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
const bool useOpenMP = false;
// time loop
for (uint_t i = 0; i < timesteps; ++i )
{
// perform a single simulation step -> this contains LBM and setting of the hydrodynamic interactions
timeloop.singleStep( timeloopTiming );
if( averageForceTorqueOverTwoTimeStepsMode == 2)
{
// store current force and torque in fOld and tOld
ps->forEachParticle(useOpenMP, sphereSelector, *accessor, initializeHydrodynamicForceTorqueForAveragingKernel, *accessor );
// reset f and t
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
timeloop.singleStep( timeloopTiming );
// f = (f+fOld)/2
ps->forEachParticle(useOpenMP, sphereSelector, *accessor, averageHydrodynamicForceTorque, *accessor );
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
}
else
{
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
if( averageForceTorqueOverTwoTimeStepsMode == 1 )
{
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, averageHydrodynamicForceTorque, *accessor );
}
}
timeloopTiming["RPD"].start();
if( useVelocityVerlet)
{
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPreForce, *accessor);
syncCall();
}
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addHydrodynamicInteraction, *accessor );
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addGravitationalForce, *accessor );
reduceProperty.operator()<mesa_pd::ForceTorqueNotification>(*ps);
if( useVelocityVerlet ) {
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPostForce, *accessor);
}
else
{
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, explicitEulerIntegrator, *accessor);
}
syncCall();
timeloopTiming["RPD"].end();
// evaluation
timeloopTiming["Logging"].start();
movingSpherePropertyEvaluator((i+1)*uint_c(dtSim));
timeloopTiming["Logging"].end();
// reset after logging here
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
timeloopAfterParticles.singleStep( timeloopTiming );
if(movingSpherePropertyEvaluator.getParticleHeight() > real_c(zlength) - real_t(3) * diameter)
{
pdfFieldVTK->forceWrite(timeloop.getCurrentTimeStep());
WALBERLA_LOG_WARNING_ON_ROOT("Sphere is too close to outflow, stopping simulation");
break;
}
}
timeloopTiming.logResultOnRoot();
return EXIT_SUCCESS;
}
} // namespace motion_settling_sphere
int main( int argc, char **argv ){
motion_settling_sphere::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/ObliqueDryCollision.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ObliqueDryCollision.cpp
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "mesa_pd/collision_detection/AnalyticContactDetection.h"
#include "mesa_pd/common/ParticleFunctions.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ExplicitEuler.h"
#include "mesa_pd/kernel/VelocityVerlet.h"
#include "mesa_pd/kernel/LinearSpringDashpot.h"
#include "mesa_pd/mpi/ReduceContactHistory.h"
#include "core/Environment.h"
#include "core/logging/Logging.h"
#include <iostream>
namespace walberla {
namespace mesa_pd {
/*
* Simulates oblique sphere-wall collision and checks rebound angle, i.e. the tangential part of the collision model.
*
*/
int main( int argc, char ** argv )
{
walberla::mpi::Environment env(argc, argv);
WALBERLA_UNUSED(env);
walberla::mpi::MPIManager::instance()->useWorldComm();
real_t uNin_SI = real_t(1.7); // m/s
real_t diameter_SI = real_t(0.00318); // m
//real_t density_SI = real_t(2500); // kg/m**3, not used
real_t uNin = real_t(0.02);
real_t diameter = real_t(20);
real_t radius = real_t(0.5) * diameter;
real_t density = real_c(2.5);
// these values have actually no influence here and are just computed for completeness
real_t dx_SI = diameter_SI / diameter;
real_t dt_SI = uNin / uNin_SI * dx_SI;
real_t impactAngle = real_t(0);
real_t dt = real_t(0.1); // = (1 / #sub steps)
real_t frictionCoeff_s = real_t(0.8); // no influence
real_t frictionCoeff_d = real_t(0.12); // paper: 0.125+-0.007
std::string filename = "TangentialCollision.txt";
real_t collisionTime = real_t(80);
real_t nu = real_t(0.22); //Poissons ratio
bool useVelocityVerlet = false;
real_t restitutionCoeff = real_t(0.83);
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--impactAngle" ) == 0 ) { impactAngle = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--dt" ) == 0 ) { dt = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--staticFriction" ) == 0 ) { frictionCoeff_s = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--dynamicFriction" ) == 0 ) { frictionCoeff_d = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--collisionTime" ) == 0 ) { collisionTime = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--nu" ) == 0 ) { nu = real_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--filename" ) == 0 ) { filename = argv[++i]; continue; }
if( std::strcmp( argv[i], "--useVV" ) == 0 ) { useVelocityVerlet = true; continue; }
if( std::strcmp( argv[i], "--coefficientOfRestitution" ) == 0 ) { restitutionCoeff = real_c( std::atof( argv[++i] ) ); continue; }
}
WALBERLA_LOG_INFO_ON_ROOT("******************************************************");
WALBERLA_LOG_INFO_ON_ROOT("** NEW SIMULATION **");
WALBERLA_LOG_INFO_ON_ROOT("******************************************************");
WALBERLA_LOG_INFO_ON_ROOT("dt_SI = " << dt_SI);
WALBERLA_LOG_INFO_ON_ROOT("dx_SI = " << dx_SI);
WALBERLA_LOG_INFO_ON_ROOT("impactAngle = " << impactAngle);
WALBERLA_LOG_INFO_ON_ROOT("dt = " << dt);
WALBERLA_LOG_INFO_ON_ROOT("frictionCoeff_s = " << frictionCoeff_s);
WALBERLA_LOG_INFO_ON_ROOT("frictionCoeff_d = " << frictionCoeff_d);
WALBERLA_LOG_INFO_ON_ROOT("collisionTime = " << collisionTime);
WALBERLA_LOG_INFO_ON_ROOT("nu = " << nu);
WALBERLA_LOG_INFO_ON_ROOT("integrator = " << (useVelocityVerlet ? "Velocity Verlet" : "Explicit Euler"));
WALBERLA_LOG_INFO_ON_ROOT("restitutionCoeff = " << restitutionCoeff);
//init data structures
auto ps = walberla::make_shared<data::ParticleStorage>(2);
auto ss = walberla::make_shared<data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
auto sphereShape = ss->create<data::Sphere>( radius );
ss->shapes[sphereShape]->updateMassAndInertia(density);
const real_t particleMass = real_t(1) / ss->shapes[sphereShape]->getInvMass();
const real_t Mij = particleMass; // * particleMass / ( real_t(2) * particleMass ); // Mij = M for sphere-wall collision
const real_t kappa = real_t(2) * ( real_t(1) - nu ) / ( real_t(2) - nu ) ; // from Thornton et al
real_t uTin = uNin * impactAngle;
// create sphere
data::Particle&& p = *ps->create();
p.setPosition(Vec3(0,0,2*radius));
p.setLinearVelocity(Vec3(uTin, 0., -uNin));
p.setInteractionRadius(radius);
p.setType(0);
// create plane
data::Particle&& p0 = *ps->create(true);
p0.setPosition(Vec3(0,0,0));
p0.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p0.setShapeID(ss->create<data::HalfSpace>(Vector3<real_t>(0,0,1)));
p0.setType(0);
data::particle_flags::set(p0.getFlagsRef(), data::particle_flags::INFINITE);
data::particle_flags::set(p0.getFlagsRef(), data::particle_flags::FIXED);
// velocity verlet
kernel::VelocityVerletPreForceUpdate vvPreForce( dt );
kernel::VelocityVerletPostForceUpdate vvPostForce( dt );
// explicit euler
kernel::ExplicitEuler explEuler( dt );
// collision response
collision_detection::AnalyticContactDetection acd;
kernel::DoubleCast double_cast;
kernel::LinearSpringDashpot dem(1);
mpi::ReduceContactHistory rch;
dem.setStiffnessAndDamping(0,0,restitutionCoeff,collisionTime,kappa,Mij);
dem.setFrictionCoefficientStatic(0,0,frictionCoeff_s);
dem.setFrictionCoefficientDynamic(0,0,frictionCoeff_d);
WALBERLA_LOG_DEVEL("begin: vel = " << p.getLinearVelocity() << ", contact vel: " << getVelocityAtWFPoint(0,*accessor,p.getPosition() + Vec3(0,0,-radius)) );
uint_t steps = 0;
real_t maxPenetration = real_t(0);
do
{
if(useVelocityVerlet) vvPreForce(0,*accessor);
real_t penetration;
if (double_cast(0, 1, *accessor, acd, *accessor ))
{
penetration = acd.getPenetrationDepth();
maxPenetration = std::max( maxPenetration, std::abs(penetration));
dem(acd.getIdx1(), acd.getIdx2(), *accessor, acd.getContactPoint(), acd.getContactNormal(), acd.getPenetrationDepth(), dt);
auto force = accessor->getForce(0);
auto torque = accessor->getTorque(0);
WALBERLA_LOG_INFO(steps << ": penetration = " << penetration << " || vel = " << accessor->getLinearVelocity(0) << " || force = " << force << ", torque = " << torque);
}
rch(*ps);
if(useVelocityVerlet) vvPostForce(0,*accessor);
else explEuler(0, *accessor);
++steps;
} while (double_cast(0, 1, *accessor, acd, *accessor ) || p.getLinearVelocity()[2] < 0);
real_t uTout = p.getLinearVelocity()[0] - radius * p.getAngularVelocity()[1];
WALBERLA_LOG_DEVEL("end: linear vel = " << p.getLinearVelocity() << ", angular vel = " << p.getAngularVelocity());
real_t reboundAngle = uTout / uNin;
WALBERLA_LOG_INFO_ON_ROOT("gamma_in = " << impactAngle);
WALBERLA_LOG_INFO_ON_ROOT("gamma_out = " << reboundAngle);
WALBERLA_LOG_INFO_ON_ROOT("Thornton: sliding should occur if " << real_t(2) * impactAngle / ( frictionCoeff_d * ( real_t(1) + restitutionCoeff)) << " >= " << real_t(7) - real_t(1) / kappa );
WALBERLA_LOG_INFO_ON_ROOT("Max penetration = " << maxPenetration << " -> " << maxPenetration / radius * 100. << "% of radius");
std::ofstream file;
file.open( filename.c_str(), std::ios::out | std::ios::app );
file << impactAngle << " " << reboundAngle << "\n";
file.close();
return EXIT_SUCCESS;
}
} //namespace mesa_pd
} //namespace walberla
int main( int argc, char ** argv )
{
return walberla::mesa_pd::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/ObliqueWetCollision.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file ObliqueWetCollision.cpp
//! \ingroup lbm_mesapd_coupling
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include "boundary/all.h"
#include "core/waLBerlaBuildInfo.h"
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/SharedFunctor.h"
#include "core/debug/Debug.h"
#include "core/debug/TestSubsystem.h"
#include "core/math/all.h"
#include "core/timing/RemainingTimeLogger.h"
#include "core/logging/all.h"
#include "core/mpi/Broadcast.h"
#include "core/mpi/Reduce.h"
#include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h"
#include "field/adaptors/AdaptorCreators.h"
#include "field/communication/PackInfo.h"
#include "field/interpolators/FieldInterpolatorCreators.h"
#include "field/interpolators/NearestNeighborFieldInterpolator.h"
#include "lbm/boundary/all.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/Adaptors.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm_mesapd_coupling/mapping/ParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/SimpleBB.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/Reconstructor.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/ExtrapolationDirectionFinder.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/PdfReconstructionManager.h"
#include "lbm_mesapd_coupling/utility/AddForceOnParticlesKernel.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/utility/AverageHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/AddHydrodynamicInteractionKernel.h"
#include "lbm_mesapd_coupling/utility/InitializeHydrodynamicForceTorqueForAveragingKernel.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/LubricationCorrectionKernel.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "mesa_pd/collision_detection/AnalyticContactDetection.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/shape/HalfSpace.h"
#include "mesa_pd/data/shape/Sphere.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/domain/BlockForestDataHandling.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ExplicitEuler.h"
#include "mesa_pd/kernel/LinearSpringDashpot.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/kernel/VelocityVerlet.h"
#include "mesa_pd/mpi/SyncNextNeighbors.h"
#include "mesa_pd/mpi/ReduceProperty.h"
#include "mesa_pd/mpi/ReduceContactHistory.h"
#include "mesa_pd/mpi/ContactFilter.h"
#include "mesa_pd/mpi/notifications/ForceTorqueNotification.h"
#include "mesa_pd/mpi/notifications/HydrodynamicForceTorqueNotification.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/all.h"
#include "field/vtk/all.h"
#include "lbm/vtk/all.h"
#include <functional>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBM.h"
#endif
namespace oblique_wet_collision
{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBM;
#else
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag( "fluid" );
const FlagUID NoSlip_Flag( "no slip" );
const FlagUID MO_Flag( "moving obstacle" );
const FlagUID FormerMO_Flag( "former moving obstacle" );
const FlagUID SimplePressure_Flag("simple pressure");
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using NoSlip_T = lbm::NoSlip< LatticeModel_T, flag_t >;
using MO_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using SimplePressure_T = lbm::SimplePressure< LatticeModel_T, flag_t >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, NoSlip_T, MO_T, SimplePressure_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac,
bool applyOutflowBCAtTop) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ), ac_( ac ), applyOutflowBCAtTop_(applyOutflowBCAtTop) {}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
auto * flagField = block->getData< FlagField_T >( flagFieldID_ );
auto * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
NoSlip_T( "NoSlip", NoSlip_Flag, pdfField ),
MO_T( "MO", MO_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ),
SimplePressure_T( "SimplePressure", SimplePressure_Flag, pdfField, real_t(1) ) );
if(applyOutflowBCAtTop_)
{
const auto simplePressure = flagField->getFlag( SimplePressure_Flag );
CellInterval domainBB = storage->getDomainCellBB();
domainBB.xMin() -= cell_idx_c( FieldGhostLayers );
domainBB.xMax() += cell_idx_c( FieldGhostLayers );
domainBB.yMin() -= cell_idx_c( FieldGhostLayers );
domainBB.yMax() += cell_idx_c( FieldGhostLayers );
domainBB.zMin() -= cell_idx_c( FieldGhostLayers );
domainBB.zMax() += cell_idx_c( FieldGhostLayers );
// TOP
CellInterval top( domainBB.xMin(), domainBB.yMin(), domainBB.zMax(), domainBB.xMax(), domainBB.yMax(), domainBB.zMax() );
storage->transformGlobalToBlockLocalCellInterval( top, *block );
handling->forceBoundary( simplePressure, top );
}
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
bool applyOutflowBCAtTop_;
};
//*******************************************************************************************************************
/*!\brief Evaluating the position and velocity of the sphere
*
*/
//*******************************************************************************************************************
template< typename ParticleAccessor_T>
class SpherePropertyLogger
{
public:
SpherePropertyLogger( const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid,
const std::string & fileNameLogging, const std::string & fileNameForceLogging, bool fileIO,
real_t diameter, real_t uIn, real_t impactRatio, uint_t numberOfSubSteps, real_t fGravX, real_t fGravZ) :
ac_( ac ), sphereUid_( sphereUid ), fileNameLogging_( fileNameLogging ), fileNameForceLogging_( fileNameForceLogging ),
fileIO_(fileIO),
diameter_( diameter ), uIn_( uIn ), impactRatio_(impactRatio), numberOfSubSteps_(numberOfSubSteps), fGravX_(fGravX), fGravZ_(fGravZ),
gap_( real_t(0) ), settlingVelocity_( real_t(0) ), tangentialVelocity_(real_t(0))
{
if ( fileIO_ )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileNameLogging_.c_str() );
file << "#\t D\t uIn\t impactRatio\t positionX\t positionY\t positionZ\t velX\t velY\t velZ\t angX\t angY\t angZ\n";
file.close();
}
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileNameForceLogging_.c_str() );
file << "#\t fGravX\t fGravZ\t fHydX\t fHydZ\t fLubX\t fLubZ\t fColX\t fColZ\t tHydY\t tLubY\t tColY\n";
file.close();
}
}
}
void operator()(const uint_t timeStep, const uint_t subStep, Vector3<real_t> fHyd, Vector3<real_t> fLub, Vector3<real_t> fCol, Vector3<real_t> tHyd, Vector3<real_t> tLub, Vector3<real_t> tCol )
{
real_t curTimestep = real_c(timeStep) + real_c(subStep) / real_c(numberOfSubSteps_);
Vector3<real_t> pos(real_t(0));
Vector3<real_t> transVel(real_t(0));
Vector3<real_t> angVel(real_t(0));
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != std::numeric_limits<size_t>::max())
{
if(!mesa_pd::data::particle_flags::isSet( ac_->getFlags(idx), mesa_pd::data::particle_flags::GHOST))
{
pos = ac_->getPosition(idx);
transVel = ac_->getLinearVelocity(idx);
angVel = ac_->getAngularVelocity(idx);
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( pos, mpi::SUM );
mpi::allReduceInplace( transVel, mpi::SUM );
mpi::allReduceInplace( angVel, mpi::SUM );
}
position_ = pos;
gap_ = pos[2] - real_t(0.5) * diameter_;
settlingVelocity_ = transVel[2];
tangentialVelocity_ = transVel[0] - diameter_ * real_t(0.5) * angVel[1];
if( fileIO_ /* && gap_ < diameter_*/)
{
writeToLoggingFile( curTimestep, pos, transVel, angVel);
writeToForceLoggingFile( curTimestep, fHyd, fLub, fCol, tHyd, tLub, tCol);
}
}
real_t getGapSize() const
{
return gap_;
}
real_t getSettlingVelocity() const
{
return settlingVelocity_;
}
real_t getTangentialVelocity() const
{
return tangentialVelocity_;
}
Vector3<real_t> getPosition() const
{
return position_;
}
private:
void writeToLoggingFile( real_t timestep, Vector3<real_t> position, Vector3<real_t> transVel, Vector3<real_t> angVel )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileNameLogging_.c_str(), std::ofstream::app );
file << timestep << "\t" << diameter_ << "\t" << uIn_ << "\t" << impactRatio_
<< "\t" << position[0] << "\t" << position[1] << "\t" << position[2]
<< "\t" << transVel[0] << "\t" << transVel[1] << "\t" << transVel[2]
<< "\t" << angVel[0] << "\t" << angVel[1] << "\t" << angVel[2]
<< "\n";
file.close();
}
}
void writeToForceLoggingFile( real_t timestep, Vector3<real_t> fHyd, Vector3<real_t> fLub, Vector3<real_t> fCol, Vector3<real_t> tHyd, Vector3<real_t> tLub, Vector3<real_t> tCol )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileNameForceLogging_.c_str(), std::ofstream::app );
file << timestep
<< "\t" << fGravX_ << "\t" << fGravZ_
<< "\t" << fHyd[0] << "\t" << fHyd[2]
<< "\t" << fLub[0] << "\t" << fLub[2]
<< "\t" << fCol[0] << "\t" << fCol[2]
<< "\t" << tHyd[1]
<< "\t" << tLub[1]
<< "\t" << tCol[1]
<< "\n";
file.close();
}
}
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
std::string fileNameLogging_, fileNameForceLogging_;
bool fileIO_;
real_t diameter_, uIn_, impactRatio_;
uint_t numberOfSubSteps_;
real_t fGravX_, fGravZ_;
real_t gap_, settlingVelocity_, tangentialVelocity_;
Vector3<real_t> position_;
};
void createPlaneSetup(const shared_ptr<mesa_pd::data::ParticleStorage> & ps, const shared_ptr<mesa_pd::data::ShapeStorage> & ss, const math::AABB & simulationDomain, bool applyOutflowBCAtTop )
{
// create bounding planes
mesa_pd::data::Particle&& p0 = *ps->create(true);
p0.setPosition(simulationDomain.minCorner());
p0.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p0.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,1) ));
p0.setOwner(mpi::MPIManager::instance()->rank());
p0.setType(0);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
if(!applyOutflowBCAtTop)
{
//only create top plane when no outflow BC should be set there
mesa_pd::data::Particle&& p1 = *ps->create(true);
p1.setPosition(simulationDomain.maxCorner());
p1.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p1.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,-1) ));
p1.setOwner(mpi::MPIManager::instance()->rank());
p1.setType(0);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
}
}
class ForcesOnSphereLogger
{
public:
ForcesOnSphereLogger( const std::string & fileName, bool fileIO, real_t weightForce, uint_t numberOfSubSteps ) :
fileName_( fileName ), fileIO_( fileIO ), weightForce_( weightForce ), numberOfSubSteps_( numberOfSubSteps )
{
if ( fileIO_ )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str() );
file << "#t\t BuoyAndGravF\t HydInteractionF\t LubricationF\t CollisionForce\n";
file.close();
}
}
}
void operator()(const uint_t timeStep, const uint_t subStep, real_t hydForce, real_t lubForce, real_t collisionForce )
{
real_t curTimestep = real_c(timeStep) + real_c(subStep) / real_c(numberOfSubSteps_);
if ( fileIO_ )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str(), std::ofstream::app );
file << std::setprecision(10) << curTimestep << "\t" << weightForce_ << "\t" << hydForce << "\t" << lubForce << "\t" << collisionForce << "\n";
file.close();
}
}
}
private:
std::string fileName_;
bool fileIO_;
real_t weightForce_;
uint_t numberOfSubSteps_;
};
template<typename ParticleAccessor_T>
Vector3<real_t> getForce(walberla::id_t uid, ParticleAccessor_T & ac)
{
auto idx = ac.uidToIdx(uid);
Vector3<real_t> force(0);
if(idx != ac.getInvalidIdx())
{
force = ac.getForce(idx);
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( force, mpi::SUM );
}
return force;
}
template<typename ParticleAccessor_T>
Vector3<real_t> getTorque(walberla::id_t uid, ParticleAccessor_T & ac)
{
auto idx = ac.uidToIdx(uid);
Vector3<real_t> torque(0);
if(idx != ac.getInvalidIdx())
{
torque = ac.getTorque(idx);
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( torque, mpi::SUM );
}
return torque;
}
//////////
// MAIN //
//////////
//*******************************************************************************************************************
/*!\brief PHYSICAL Test case of a sphere settling and colliding with a wall submerged in a viscous fluid.
*
* The collision is oblique, i.e. features normal and tangential contributions.
* There are in principle two ways to carry out this simulation:
* 1) Use some (artificial) gravitational force such that the x(tangential) and z(normal)- components
* have the same ratio as the desired impact ratio of the velocities.
* To have similar normal collision velocities, the option 'useFullGravityInNormalDirection' always applies the same
* gravitational acceleration in normal direction.
* The main problem, however, is that the sphere begins to rotate for increasing impact angles and will thus generate
* a reposing force in normal direction, that will alter the intended impact ratio to larger values.
* Additionally, a reasonable value for the relaxation time, tau, has to be given to avoid too large settling velocities.
*
* 2) Specify a (normal or magnitude-wise) Stokes number and artificially accelerate the sphere to this value.
* Thus, the settling velocity is prescribed and no rotational velocity is allowed.
* This is stopped before the collision to allow for a 'natural' collision.
* To avoid a too large damping by the fluid forces, since no gravitational forces are present to balance them,
* an artificial gravity is assumed and taken as the opposite fluid force acting when the artificial
* acceleration is stopped.
*
* For details see Rettinger, Ruede 2020
*/
//*******************************************************************************************************************
int main( int argc, char **argv )
{
Environment env( argc, argv );
if( !env.config() )
{
WALBERLA_ABORT("Usage: " << argv[0] << " path-to-configuration-file \n");
}
Config::BlockHandle simulationInput = env.config()->getBlock( "Setup" );
const int caseNumber = simulationInput.getParameter<int>("case");
Config::BlockHandle caseDescription = env.config()->getBlock( "Case"+std::to_string(caseNumber) );
const std::string material = caseDescription.getParameter<std::string>("material");
Config::BlockHandle materialDescription = env.config()->getBlock( "Mat_"+material );
const real_t densitySphere_SI = materialDescription.getParameter<real_t>("densitySphere_SI");
const real_t diameter_SI = materialDescription.getParameter<real_t>("diameter_SI");
const real_t restitutionCoeff = materialDescription.getParameter<real_t>("restitutionCoeff");
const real_t frictionCoeff = materialDescription.getParameter<real_t>("frictionCoeff");
const real_t poissonsRatio = materialDescription.getParameter<real_t>("poissonsRatio");
const std::string fluid = caseDescription.getParameter<std::string>("fluid");
Config::BlockHandle fluidDescription = env.config()->getBlock( "Fluid_"+fluid );
const real_t densityFluid_SI = fluidDescription.getParameter<real_t>("densityFluid_SI");
const real_t dynamicViscosityFluid_SI = fluidDescription.getParameter<real_t>("dynamicViscosityFluid_SI");
const std::string simulationVariant = simulationInput.getParameter<std::string>("variant");
Config::BlockHandle variantDescription = env.config()->getBlock( "Variant_"+simulationVariant );
const real_t impactRatio = simulationInput.getParameter<real_t>("impactRatio");
const bool applyOutflowBCAtTop = simulationInput.getParameter<bool>("applyOutflowBCAtTop");
// variant dependent parameters
const Vector3<real_t> domainSizeNonDim = variantDescription.getParameter<Vector3<real_t> >("domainSize");
const Vector3<uint_t> numberOfBlocksPerDirection = variantDescription.getParameter<Vector3<uint_t> >("numberOfBlocksPerDirection");
const real_t initialSphereHeight = variantDescription.getParameter<real_t>("initialSphereHeight");
// LBM
const real_t diameter = simulationInput.getParameter<real_t>("diameter");
const real_t magicNumber = simulationInput.getParameter<real_t>("magicNumber");
const real_t bulkViscRateFactor = simulationInput.getParameter<real_t>("bulkViscRateFactor");
const bool useOmegaBulkAdaption = simulationInput.getParameter<bool>("useOmegaBulkAdaption");
const uint_t numRPDSubCycles = simulationInput.getParameter<uint_t>("numRPDSubCycles");
const bool useLubricationCorrection = simulationInput.getParameter<bool>("useLubricationCorrection");
const real_t lubricationCutOffDistanceNormal = simulationInput.getParameter<real_t>("lubricationCutOffDistanceNormal");
const real_t lubricationCutOffDistanceTangentialTranslational = simulationInput.getParameter<real_t>("lubricationCutOffDistanceTangentialTranslational");
const real_t lubricationCutOffDistanceTangentialRotational = simulationInput.getParameter<real_t>("lubricationCutOffDistanceTangentialRotational");
const real_t lubricationMinimalGapSizeNonDim = simulationInput.getParameter<real_t>("lubricationMinimalGapSizeNonDim");
// Collision Response
const bool useVelocityVerlet = simulationInput.getParameter<bool>("useVelocityVerlet");
const real_t collisionTime = simulationInput.getParameter<real_t>("collisionTime");
const bool averageForceTorqueOverTwoTimeSteps = simulationInput.getParameter<bool>("averageForceTorqueOverTwoTimeSteps");
const bool conserveMomentum = simulationInput.getParameter<bool>("conserveMomentum");
const std::string reconstructorType = simulationInput.getParameter<std::string>("reconstructorType");
const bool fileIO = simulationInput.getParameter<bool>("fileIO");
const uint_t vtkIOFreq = simulationInput.getParameter<uint_t>("vtkIOFreq");
const std::string baseFolder = simulationInput.getParameter<std::string>("baseFolder");
WALBERLA_ROOT_SECTION()
{
if( fileIO )
{
// create base directory if it does not yet exist
filesystem::path tpath( baseFolder );
if( !filesystem::exists( tpath ) )
filesystem::create_directory( tpath );
}
}
//////////////////////////////////////
// SIMULATION PROPERTIES in SI units//
//////////////////////////////////////
const real_t impactAngle = std::atan(impactRatio);
const real_t densityRatio = densitySphere_SI / densityFluid_SI;
const real_t kinematicViscosityFluid_SI = dynamicViscosityFluid_SI / densityFluid_SI;
WALBERLA_LOG_INFO_ON_ROOT("SETUP OF CASE " << caseNumber);
WALBERLA_LOG_INFO_ON_ROOT("Setup (in SI units):");
WALBERLA_LOG_INFO_ON_ROOT(" - impact ratio = " << impactRatio );
WALBERLA_LOG_INFO_ON_ROOT(" - impact angle = " << impactAngle << " (" << impactAngle * real_t(180) / math::pi << " degrees) ");
WALBERLA_LOG_INFO_ON_ROOT(" - sphere: diameter = " << diameter_SI << ", densityRatio = " << densityRatio);
WALBERLA_LOG_INFO_ON_ROOT(" - fluid: density = " << densityFluid_SI << ", dyn. visc = " << dynamicViscosityFluid_SI << ", kin. visc = " << kinematicViscosityFluid_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - domain size = <" << real_c(domainSizeNonDim[0]) * diameter_SI << "," << real_c(domainSizeNonDim[1]) * diameter_SI << ","<< real_c(domainSizeNonDim[2]) * diameter_SI << ">");
//////////////////////////
// NUMERICAL PARAMETERS //
//////////////////////////
WALBERLA_LOG_INFO_ON_ROOT("Setup (in simulation, i.e. lattice, units):");
const real_t dx_SI = diameter_SI / diameter;
const real_t sphereVolume = math::pi / real_t(6) * diameter * diameter * diameter;
real_t dt_SI;
real_t viscosity;
real_t omega;
real_t uIn = real_t(1);
real_t accelerationFactor = real_t(0);
bool applyArtificialGravityAfterAccelerating = false;
bool useFullGravityInNormalDirection = false;
bool artificiallyAccelerateSphere = false;
Vector3<real_t> gravitationalAccelerationVec(real_t(0));
if(simulationVariant=="Acceleration")
{
WALBERLA_LOG_INFO_ON_ROOT("USING MODE OF ARTIFICIALLY ACCELERATED SPHERE");
artificiallyAccelerateSphere = true;
real_t StTarget = variantDescription.getParameter<real_t>("StTarget");
const bool useStTargetInNormalDirection = variantDescription.getParameter<bool>("useStTargetInNormalDirection");
applyArtificialGravityAfterAccelerating = variantDescription.getParameter<bool>("applyArtificialGravityAfterAccelerating");
bool applyUInNormalDirection = variantDescription.getParameter<bool>("applyUInNormalDirection");
uIn = variantDescription.getParameter<real_t>("uIn");
accelerationFactor = variantDescription.getParameter<real_t>("accelerationFactor");
if(applyUInNormalDirection)
{
// the value for uIn is defined as the normal impact velocity, i.e. uNIn
uIn = uIn / std::cos(impactAngle);
}
real_t StTargetN;
if(useStTargetInNormalDirection)
{
StTargetN = StTarget;
StTarget = StTarget / std::cos(impactAngle);
} else
{
StTargetN = StTarget * std::cos(impactAngle);
}
WALBERLA_LOG_INFO_ON_ROOT(" - target St in settling direction = " << StTarget );
WALBERLA_LOG_INFO_ON_ROOT(" - target St in normal direction = " << StTargetN );
const real_t uIn_SI = StTarget * real_t(9) / densityRatio * kinematicViscosityFluid_SI / diameter_SI;
dt_SI = uIn / uIn_SI * dx_SI;
viscosity = kinematicViscosityFluid_SI * dt_SI / ( dx_SI * dx_SI );
//note: no gravity when accelerating artificially
omega = lbm::collision_model::omegaFromViscosity(viscosity);
const real_t uNIn = uIn * std::cos(impactAngle);
const real_t Re_p = diameter * uIn / viscosity;
WALBERLA_LOG_INFO_ON_ROOT(" - target Reynolds number = " << Re_p );
WALBERLA_LOG_INFO_ON_ROOT(" - expected normal impact velocity = " << uNIn );
WALBERLA_LOG_INFO_ON_ROOT(" - expected tangential impact velocity = " << uIn * std::sin(impactAngle) );
} else{
WALBERLA_LOG_INFO_ON_ROOT("USING MODE OF SPHERE SETTLING UNDER GRAVITY");
const real_t gravitationalAcceleration_SI = variantDescription.getParameter<real_t>("gravitationalAcceleration_SI");
useFullGravityInNormalDirection = variantDescription.getParameter<bool>("useFullGravityInNormalDirection");
const real_t tau = variantDescription.getParameter<real_t>("tau");
const real_t ug_SI = std::sqrt((densityRatio-real_t(1)) * gravitationalAcceleration_SI * diameter_SI);
const real_t Ga = ug_SI * diameter_SI / kinematicViscosityFluid_SI;
viscosity = lbm::collision_model::viscosityFromOmega(real_t(1) / tau);
dt_SI = viscosity * dx_SI * dx_SI / kinematicViscosityFluid_SI;
real_t gravitationalAcceleration = gravitationalAcceleration_SI * dt_SI * dt_SI / dx_SI;
omega = real_t(1) / tau;
WALBERLA_LOG_INFO_ON_ROOT(" - ug = " << ug_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - Galileo number = " << Ga );
gravitationalAccelerationVec = useFullGravityInNormalDirection ? Vector3<real_t>( impactRatio, real_t(0), -real_t(1) ) * gravitationalAcceleration
: Vector3<real_t>( std::sin(impactAngle), real_t(0), -std::cos(impactAngle) ) * gravitationalAcceleration;
WALBERLA_LOG_INFO_ON_ROOT(" - g " << gravitationalAcceleration);
}
const real_t omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
const real_t densityFluid = real_t(1);
const real_t densitySphere = densityRatio;
const real_t dx = real_t(1);
const real_t responseTime = densityRatio * diameter * diameter / ( real_t(18) * viscosity );
const real_t particleMass = densitySphere * sphereVolume;
const real_t Mij = particleMass; // * particleMass / ( real_t(2) * particleMass ); // Mij = M for sphere-wall collision
const real_t kappa = real_t(2) * ( real_t(1) - poissonsRatio ) / ( real_t(2) - poissonsRatio ) ;
Vector3<uint_t> domainSize( uint_c(domainSizeNonDim[0] * diameter ), uint_c(domainSizeNonDim[1] * diameter ), uint_c(domainSizeNonDim[2] * diameter ) );
real_t initialSpherePosition = initialSphereHeight * diameter;
WALBERLA_LOG_INFO_ON_ROOT(" - dt_SI = " << dt_SI << " s, dx_SI = " << dx_SI << " m");
WALBERLA_LOG_INFO_ON_ROOT(" - domain size = " << domainSize);
if(applyOutflowBCAtTop) WALBERLA_LOG_INFO_ON_ROOT(" - outflow BC at top");
WALBERLA_LOG_INFO_ON_ROOT(" - sphere: diameter = " << diameter << ", density = " << densitySphere );
WALBERLA_LOG_INFO_ON_ROOT(" - initial sphere position = " << initialSpherePosition );
WALBERLA_LOG_INFO_ON_ROOT(" - fluid: density = " << densityFluid << ", relaxation time (tau) = " << real_t(1) / omega << ", omega = " << omega << ", kin. visc = " << viscosity );
WALBERLA_LOG_INFO_ON_ROOT(" - magic number = " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - omega bulk = " << omegaBulk << ", bulk visc = " << lbm_mesapd_coupling::bulkViscosityFromOmegaBulk(omegaBulk) << " ( bulk visc rate factor (conste) = " << bulkViscRateFactor << ")");
if(useOmegaBulkAdaption) WALBERLA_LOG_INFO_ON_ROOT(" - using omega bulk adaption in vicinity of sphere");
WALBERLA_LOG_INFO_ON_ROOT(" - gravitational acceleration = " << gravitationalAccelerationVec );
WALBERLA_LOG_INFO_ON_ROOT(" - Stokes response time = " << responseTime );
if( artificiallyAccelerateSphere )
{
WALBERLA_LOG_INFO_ON_ROOT(" - artificially accelerating sphere with factor " << accelerationFactor)
if(applyArtificialGravityAfterAccelerating)
{
WALBERLA_LOG_INFO_ON_ROOT(" - applying artificial gravity after accelerating" );
}
}
WALBERLA_LOG_INFO_ON_ROOT(" - integrator = " << (useVelocityVerlet ? "Velocity Verlet" : "Explicit Euler") );
if( vtkIOFreq > 0 )
{
WALBERLA_LOG_INFO_ON_ROOT(" - writing vtk files to folder \"" << baseFolder << "\" with frequency " << vtkIOFreq);
}
WALBERLA_LOG_INFO_ON_ROOT("Collision Response properties:" );
WALBERLA_LOG_INFO_ON_ROOT(" - collision time = " << collisionTime );
WALBERLA_LOG_INFO_ON_ROOT(" - coeff of restitution = " << restitutionCoeff );
WALBERLA_LOG_INFO_ON_ROOT(" - coeff of friction = " << frictionCoeff );
WALBERLA_LOG_INFO_ON_ROOT("Coupling properties:" );
WALBERLA_LOG_INFO_ON_ROOT(" - number of RPD sub cycles = " << numRPDSubCycles );
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction = " << (useLubricationCorrection ? "yes" : "no") );
if( useLubricationCorrection )
{
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction cut off normal = " << lubricationCutOffDistanceNormal );
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction cut off tangential translational = " << lubricationCutOffDistanceTangentialTranslational );
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction cut off tangential rotational = " << lubricationCutOffDistanceTangentialRotational );
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction minimal gap size non dim = " << lubricationMinimalGapSizeNonDim << " ( = " << lubricationMinimalGapSizeNonDim * diameter * real_t(0.5) << " cells )");
}
WALBERLA_LOG_INFO_ON_ROOT(" - average forces over two LBM time steps = " << (averageForceTorqueOverTwoTimeSteps ? "yes" : "no") );
WALBERLA_LOG_INFO_ON_ROOT(" - conserve momentum = " << (conserveMomentum ? "yes" : "no") );
WALBERLA_LOG_INFO_ON_ROOT(" - reconstructor = " << reconstructorType );
///////////////////////////
// BLOCK STRUCTURE SETUP //
///////////////////////////
Vector3<uint_t> cellsPerBlockPerDirection( domainSize[0] / numberOfBlocksPerDirection[0],
domainSize[1] / numberOfBlocksPerDirection[1],
domainSize[2] / numberOfBlocksPerDirection[2] );
for( uint_t i = 0; i < 3; ++i ) {
WALBERLA_CHECK_EQUAL(cellsPerBlockPerDirection[i] * numberOfBlocksPerDirection[i], domainSize[i],
"Unmatching domain decomposition in direction " << i << "!");
}
auto blocks = blockforest::createUniformBlockGrid( numberOfBlocksPerDirection[0], numberOfBlocksPerDirection[1], numberOfBlocksPerDirection[2],
cellsPerBlockPerDirection[0], cellsPerBlockPerDirection[1], cellsPerBlockPerDirection[2], dx,
0, false, false,
true, true, false, //periodicity
false );
WALBERLA_LOG_INFO_ON_ROOT("Domain decomposition:");
WALBERLA_LOG_INFO_ON_ROOT(" - blocks per direction = " << numberOfBlocksPerDirection );
WALBERLA_LOG_INFO_ON_ROOT(" - cells per block = " << cellsPerBlockPerDirection );
//write domain decomposition to file
if( vtkIOFreq > 0 )
{
vtk::writeDomainDecomposition( blocks, "initial_domain_decomposition", baseFolder );
}
//////////////////
// RPD COUPLING //
//////////////////
auto rpdDomain = std::make_shared<mesa_pd::domain::BlockForestDomain>(blocks->getBlockForestPointer());
//init data structures
auto ps = walberla::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = walberla::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
// bounding planes
createPlaneSetup(ps,ss,blocks->getDomain(), applyOutflowBCAtTop);
// create sphere and store Uid
Vector3<real_t> initialPosition( real_t(0.5) * real_c(domainSize[0]), real_t(0.5) * real_c(domainSize[1]), initialSpherePosition );
auto sphereShape = ss->create<mesa_pd::data::Sphere>( diameter * real_t(0.5) );
ss->shapes[sphereShape]->updateMassAndInertia(densitySphere);
walberla::id_t sphereUid = 0;
// create sphere
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), initialPosition ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(initialPosition);
p.setInteractionRadius(diameter * real_t(0.5));
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
p.setType(0);
sphereUid = p.getUid();
}
mpi::allReduceInplace(sphereUid, mpi::SUM);
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumber );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, lambda_d, lambda_e);
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in MRT lattice model and ignoring omega bulk field since not supported!");
LatticeModel_T latticeModel = LatticeModel_T(lbm::collision_model::D3Q19MRT( omegaBulk, omegaBulk, lambda_d, lambda_e, lambda_e, lambda_d ));
#endif
// add PDF field
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field", latticeModel,
Vector3< real_t >( real_t(0) ), real_t(1),
uint_t(1), field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage<FlagField_T>( blocks, "flag field" );
// add particle field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor, applyOutflowBCAtTop), "boundary handling" );
// set up RPD functionality
std::function<void(void)> syncCall = [&ps,&rpdDomain](){
const real_t overlap = real_t( 1.5 );
mesa_pd::mpi::SyncNextNeighbors syncNextNeighborFunc;
syncNextNeighborFunc(*ps, *rpdDomain, overlap);
};
syncCall();
real_t timeStepSizeRPD = real_t(1)/real_t(numRPDSubCycles);
mesa_pd::kernel::ExplicitEuler explEulerIntegrator(timeStepSizeRPD);
mesa_pd::kernel::VelocityVerletPreForceUpdate vvIntegratorPreForce(timeStepSizeRPD);
mesa_pd::kernel::VelocityVerletPostForceUpdate vvIntegratorPostForce(timeStepSizeRPD);
// linear model
mesa_pd::kernel::LinearSpringDashpot linearCollisionResponse(1);
linearCollisionResponse.setStiffnessAndDamping(0,0,restitutionCoeff,collisionTime,kappa,Mij);
//linearCollisionResponse.setFrictionCoefficientStatic(0,0,frictionCoeff); // not used in this test case
linearCollisionResponse.setFrictionCoefficientDynamic(0,0,frictionCoeff);
mesa_pd::mpi::ReduceProperty reduceProperty;
mesa_pd::mpi::ReduceContactHistory reduceAndSwapContactHistory;
// set up coupling functionality
Vector3<real_t> gravitationalForce = (densitySphere - densityFluid) * sphereVolume * gravitationalAccelerationVec;
lbm_mesapd_coupling::AddForceOnParticlesKernel addGravitationalForce(gravitationalForce);
lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel resetHydrodynamicForceTorque;
lbm_mesapd_coupling::AverageHydrodynamicForceTorqueKernel averageHydrodynamicForceTorque;
lbm_mesapd_coupling::LubricationCorrectionKernel lubricationCorrectionKernel(viscosity, [lubricationMinimalGapSizeNonDim](real_t r){ return lubricationMinimalGapSizeNonDim * r;}, lubricationCutOffDistanceNormal,
lubricationCutOffDistanceTangentialTranslational, lubricationCutOffDistanceTangentialRotational );
lbm_mesapd_coupling::ParticleMappingKernel<BoundaryHandling_T> particleMappingKernel(blocks, boundaryHandlingID);
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
///////////////
// TIME LOOP //
///////////////
// map planes into the LBM simulation -> act as no-slip boundaries
ps->forEachParticle(false, lbm_mesapd_coupling::GlobalParticlesSelector(), *accessor, particleMappingKernel, *accessor, NoSlip_Flag);
// map particles into the LBM simulation
ps->forEachParticle(false, lbm_mesapd_coupling::RegularParticlesSelector(), *accessor, movingParticleMappingKernel, *accessor, MO_Flag);
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
blockforest::communication::UniformBufferedScheme< Stencil_T > fullPDFCommunicationScheme( blocks );
fullPDFCommunicationScheme.addPackInfo( make_shared< field::communication::PackInfo< PdfField_T > >( pdfFieldID ) ); // full sync
// create the timeloop
const uint_t timesteps = uint_c( 1000000000 ); // just some large value
SweepTimeloop timeloop( blocks->getBlockStorage(), timesteps );
timeloop.addFuncBeforeTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "Remaining Time Logger" );
// vtk output
if( vtkIOFreq != uint_t(0) )
{
// sphere
auto particleVtkOutput = make_shared<mesa_pd::vtk::ParticleVtkOutput>(ps);
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleLinearVelocity>("velocity");
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleAngularVelocity>("angular velocity");
particleVtkOutput->setParticleSelector( [sphereShape](const mesa_pd::data::ParticleStorage::iterator& pIt) {return pIt->getShapeID() == sphereShape;} ); //limit output to sphere
auto particleVtkWriter = vtk::createVTKOutput_PointData(particleVtkOutput, "Particles", vtkIOFreq, baseFolder, "simulation_step");
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( particleVtkWriter ), "VTK (sphere data)" );
// pdf field, as slice
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", vtkIOFreq, 0, false, baseFolder );
pdfFieldVTK->addBeforeFunction( fullPDFCommunicationScheme );
AABB sliceAABB( real_t(0), real_c(domainSize[1])*real_t(0.5)-real_t(1), real_t(0),
real_c(domainSize[0]), real_c(domainSize[1])*real_t(0.5)+real_t(1), real_c(domainSize[2]) );
vtk::AABBCellFilter aabbSliceFilter( sliceAABB );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
vtk::ChainedFilter combinedSliceFilter;
combinedSliceFilter.addFilter( fluidFilter );
combinedSliceFilter.addFilter( aabbSliceFilter );
pdfFieldVTK->addCellInclusionFilter( combinedSliceFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
// omega bulk field
//timeloop.addFuncBeforeTimeStep( field::createVTKOutput<ScalarField_T, float>( omegaBulkFieldID, *blocks, "omega_bulk_field", vtkIOFreq, uint_t(0), false, baseFolder ), "VTK (omega bulk field)" );
}
// sweep for updating the particle mapping into the LBM simulation
timeloop.add() << Sweep( lbm_mesapd_coupling::makeMovingParticleMapping<PdfField_T, BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, MO_Flag, FormerMO_Flag, lbm_mesapd_coupling::RegularParticlesSelector(), conserveMomentum), "Particle Mapping" );
// sweep for restoring PDFs in cells previously occupied by particles
if( reconstructorType == "EAN" )
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
auto equilibriumAndNonEquilibriumSphereNormalReconstructor = lbm_mesapd_coupling::makeEquilibriumAndNonEquilibriumReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "LBM Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, equilibriumAndNonEquilibriumSphereNormalReconstructor, conserveMomentum)), "PDF Restore" );
}else if( reconstructorType == "Grad" )
{
bool recomputeTargetDensity = false;
auto gradReconstructor = lbm_mesapd_coupling::makeGradsMomentApproximationReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, omega, recomputeTargetDensity, true, true);
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, gradReconstructor, conserveMomentum) ), "PDF Restore" );
}else if( reconstructorType == "Eq" )
{
timeloop.add() << Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, conserveMomentum)), "PDF Restore" );
}else if( reconstructorType == "Ext")
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
#ifdef USE_TRT_LIKE_LATTICE_MODEL
auto extrapolationReconstructor = lbm_mesapd_coupling::makeExtrapolationReconstructor<BoundaryHandling_T, lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder, true>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
#else
auto extrapolationReconstructor = lbm_mesapd_coupling::makeExtrapolationReconstructor<BoundaryHandling_T, lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder, false>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
#endif
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "LBM Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, extrapolationReconstructor, conserveMomentum)), "PDF Restore" );
} else
{
WALBERLA_ABORT("Reconstructor Type " << reconstructorType << " not implemented!");
}
// update bulk omega in all cells to adapt to changed particle position
if(useOmegaBulkAdaption)
{
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, lbm_mesapd_coupling::RegularParticlesSelector>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
real_t adaptionLayerSize = real_t(2);
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, lbm_mesapd_coupling::RegularParticlesSelector());
// initial adaption
for (auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt) {
(*omegaBulkAdapter)(blockIt.get());
}
timeloop.add() << Sweep( makeSharedSweep(omegaBulkAdapter), "Omega Bulk Adapter");
}
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep( BoundaryHandling_T::getBlockSweep( boundaryHandlingID ), "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
timeloop.add() << Sweep( lbmSweep, "LB sweep" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
timeloop.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
// evaluation functionality
std::string loggingFileName( baseFolder + "/LoggingObliqueWetCollision.txt");
std::string forceLoggingFileName( baseFolder + "/ForceLoggingObliqueWetCollision.txt");
if( fileIO )
{
WALBERLA_LOG_INFO_ON_ROOT(" - writing logging output to file \"" << loggingFileName << "\"");
WALBERLA_LOG_INFO_ON_ROOT(" - writing force logging output to file \"" << forceLoggingFileName << "\"");
}
SpherePropertyLogger<ParticleAccessor_T> logger( accessor, sphereUid, loggingFileName, forceLoggingFileName, fileIO, diameter, uIn, impactRatio, numRPDSubCycles, gravitationalForce[0], gravitationalForce[2] );
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
// evaluation quantities
uint_t numBounces = uint_t(0);
uint_t tImpact = uint_t(0);
real_t curVel(real_t(0));
real_t oldVel(real_t(0));
real_t maxSettlingVel(real_t(0));
real_t minGapSize(real_t(0));
real_t actualSt(real_t(0));
real_t actualRe(real_t(0));
WALBERLA_LOG_INFO_ON_ROOT("Running for maximum of " << timesteps << " timesteps!");
const bool useOpenMP = false;
uint_t averagingSampleSize = uint_c(real_t(1) / uIn);
std::vector<Vector3<real_t>> forceValues(averagingSampleSize, Vector3<real_t>(real_t(0)));
// generally: z: normal direction, x: tangential direction
uint_t endTimestep = timesteps; // will be adapted after bounce
// time loop
for (uint_t i = 0; i < endTimestep; ++i )
{
// perform a single simulation step -> this contains LBM and setting of the hydrodynamic interactions
timeloop.singleStep( timeloopTiming );
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
if( averageForceTorqueOverTwoTimeSteps )
{
if( i == 0 )
{
lbm_mesapd_coupling::InitializeHydrodynamicForceTorqueForAveragingKernel initializeHydrodynamicForceTorqueForAveragingKernel;
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, initializeHydrodynamicForceTorqueForAveragingKernel, *accessor );
}
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, averageHydrodynamicForceTorque, *accessor );
}
Vector3<real_t> hydForce(real_t(0));
Vector3<real_t> lubForce(real_t(0));
Vector3<real_t> collisionForce(real_t(0));
Vector3<real_t> hydTorque(real_t(0));
Vector3<real_t> lubTorque(real_t(0));
Vector3<real_t> collisionTorque(real_t(0));
for(auto subCycle = uint_t(0); subCycle < numRPDSubCycles; ++subCycle )
{
timeloopTiming["RPD"].start();
if( useVelocityVerlet )
{
Vector3<real_t> oldForce;
Vector3<real_t> oldTorque;
if(artificiallyAccelerateSphere)
{
// since the pre-force step of VV updates the position based on velocity and old force, we set oldForce to zero here for this step while artificially accelerating
// to not perturb the step after which artificial acceleration is switched off (which requires valid oldForce values then) we store the old force and then re-apply it
auto idx = accessor->uidToIdx(sphereUid);
if(idx != accessor->getInvalidIdx())
{
oldForce = accessor->getOldForce(idx);
accessor->setOldForce(idx, Vector3<real_t>(real_t(0)));
oldTorque = accessor->getOldTorque(idx);
accessor->setOldTorque(idx, Vector3<real_t>(real_t(0)));
}
}
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPreForce, *accessor);
syncCall();
if(artificiallyAccelerateSphere)
{
// re-apply old force
auto idx = accessor->uidToIdx(sphereUid);
if(idx != accessor->getInvalidIdx())
{
accessor->setOldForce(idx, oldForce);
accessor->setOldTorque(idx, oldTorque);
}
}
}
// lubrication correction
ps->forEachParticlePairHalf(useOpenMP, mesa_pd::kernel::ExcludeInfiniteInfinite(), *accessor,
[&lubricationCorrectionKernel,&rpdDomain](const size_t idx1, const size_t idx2, auto& ac)
{
mesa_pd::collision_detection::AnalyticContactDetection acd;
acd.getContactThreshold() = lubricationCorrectionKernel.getNormalCutOffDistance();
mesa_pd::kernel::DoubleCast double_cast;
mesa_pd::mpi::ContactFilter contact_filter;
if (double_cast(idx1, idx2, ac, acd, ac ))
{
if (contact_filter(acd.getIdx1(), acd.getIdx2(), ac, acd.getContactPoint(), *rpdDomain))
{
double_cast(acd.getIdx1(), acd.getIdx2(), ac, lubricationCorrectionKernel, ac, acd.getContactNormal(), acd.getPenetrationDepth());
}
}
},
*accessor );
lubForce = getForce(sphereUid,*accessor);
lubTorque = getTorque(sphereUid,*accessor);
// one could add linked cells here
// collision response
ps->forEachParticlePairHalf(useOpenMP, mesa_pd::kernel::ExcludeInfiniteInfinite(), *accessor,
[&linearCollisionResponse, &rpdDomain, timeStepSizeRPD]
(const size_t idx1, const size_t idx2, auto& ac)
{
mesa_pd::collision_detection::AnalyticContactDetection acd;
mesa_pd::kernel::DoubleCast double_cast;
mesa_pd::mpi::ContactFilter contact_filter;
if (double_cast(idx1, idx2, ac, acd, ac ))
{
if (contact_filter(acd.getIdx1(), acd.getIdx2(), ac, acd.getContactPoint(), *rpdDomain))
{
linearCollisionResponse(acd.getIdx1(), acd.getIdx2(), ac, acd.getContactPoint(), acd.getContactNormal(), acd.getPenetrationDepth(), timeStepSizeRPD);
}
}
},
*accessor );
collisionForce = getForce(sphereUid,*accessor) - lubForce;
collisionTorque = getTorque(sphereUid,*accessor) - lubTorque;
reduceAndSwapContactHistory(*ps);
// add hydrodynamic force
lbm_mesapd_coupling::AddHydrodynamicInteractionKernel addHydrodynamicInteraction;
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addHydrodynamicInteraction, *accessor );
hydForce = getForce(sphereUid,*accessor) - lubForce - collisionForce;
WALBERLA_ASSERT(!std::isnan(hydForce[0]) && !std::isnan(hydForce[1]) && !std::isnan(hydForce[2]), "Found nan value in hydrodynamic force = " << hydForce);
hydTorque = getTorque(sphereUid,*accessor) - lubTorque - collisionTorque;
ps->forEachParticle( useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addGravitationalForce, *accessor );
reduceProperty.operator()<mesa_pd::ForceTorqueNotification>(*ps);
// integration
if( useVelocityVerlet ) ps->forEachParticle( useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPostForce, *accessor);
else ps->forEachParticle( useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, explEulerIntegrator, *accessor);
syncCall();
if( artificiallyAccelerateSphere )
{
lbm_mesapd_coupling::RegularParticlesSelector sphereSelector;
real_t newSphereVel = uIn * (std::exp(-accelerationFactor * real_t(i) / responseTime ) - real_t(1));
ps->forEachParticle(useOpenMP, sphereSelector, *accessor,
[newSphereVel,impactAngle](const size_t idx, ParticleAccessor_T& ac){
ac.setLinearVelocity(idx, Vector3<real_t>( -std::sin(impactAngle), real_t(0), std::cos(impactAngle) ) * newSphereVel);
ac.setAngularVelocity(idx, Vector3<real_t>(real_t(0)));},
*accessor);
}
timeloopTiming["RPD"].end();
// logging
timeloopTiming["Logging"].start();
logger(i, subCycle, hydForce, lubForce, collisionForce, hydTorque, lubTorque, collisionTorque);
timeloopTiming["Logging"].end();
}
// store hyd force to average
forceValues[i % averagingSampleSize] = hydForce;
ps->forEachParticle( useOpenMP, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
// check for termination
oldVel = curVel;
curVel = logger.getSettlingVelocity();
maxSettlingVel = std::min(maxSettlingVel, curVel);
minGapSize = std::min(minGapSize, logger.getGapSize());
// detect the bounce
if( oldVel < real_t(0) && curVel > real_t(0) && logger.getGapSize() < real_t(1))
{
++numBounces;
actualSt = densityRatio * std::abs(maxSettlingVel) * diameter / ( real_t(9) * viscosity );
actualRe = std::abs(maxSettlingVel) * diameter / viscosity;
WALBERLA_LOG_INFO_ON_ROOT("Detected bounce with max settling velocity " << maxSettlingVel << " -> St = " << actualSt );
// end simulation after one non-dim timestep
uint_t remainingTimeSteps = uint_t(real_t(1) * diameter / std::abs(maxSettlingVel));
endTimestep = i + remainingTimeSteps;
WALBERLA_LOG_INFO_ON_ROOT("Will terminate simulation after " << remainingTimeSteps << " time steps, i.e. at time step " << endTimestep);
}
// impact times are measured when the contact between sphere and wall is broken up again
if( tImpact == uint_t(0) && numBounces == uint_t(1) && logger.getGapSize() > real_t(0) )
{
tImpact = i;
WALBERLA_LOG_INFO_ON_ROOT("Detected impact time at time step " << tImpact);
}
// check if sphere is close to bottom plane
if( logger.getGapSize() < real_t(1) * diameter && artificiallyAccelerateSphere) {
WALBERLA_LOG_INFO_ON_ROOT("Switching off acceleration!");
artificiallyAccelerateSphere = false;
if(applyArtificialGravityAfterAccelerating)
{
// to avoid a too large deceleration due to the missing gravitational force, we apply the averaged hydrodynamic force
// (that would have been balanced by the gravitational force) as a kind of artificial gravity
Vector3<real_t> artificialGravitationalForce = -std::accumulate(forceValues.begin(), forceValues.end(), Vector3<real_t>(real_t(0))) / real_t(averagingSampleSize);
WALBERLA_LOG_INFO_ON_ROOT("Applying artificial gravitational and buoyancy force of " << artificialGravitationalForce);
real_t actualGravitationalAcceleration = -artificialGravitationalForce[2] / ( (densitySphere - densityFluid) * sphereVolume );
WALBERLA_LOG_INFO_ON_ROOT("This would correspond to a gravitational acceleration of g = " << actualGravitationalAcceleration << ", g_SI = " << actualGravitationalAcceleration * dx_SI / (dt_SI * dt_SI) );
addGravitationalForce = lbm_mesapd_coupling::AddForceOnParticlesKernel(artificialGravitationalForce);
}
}
}
WALBERLA_LOG_INFO_ON_ROOT("Terminating simulation");
WALBERLA_LOG_INFO_ON_ROOT("Maximum settling velocities: " << maxSettlingVel );
std::string summaryFile(baseFolder + "/Summary.txt");
WALBERLA_LOG_INFO_ON_ROOT("Writing summary file " << summaryFile);
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( summaryFile.c_str());
file << "waLBerla Revision = " << std::string(WALBERLA_GIT_SHA1).substr(0,8) << "\n";
file << "\nInput parameters:\n";
file << "case: " << caseNumber << "\n";
file << "fluid: " << fluid << "\n";
file << "material: " << material << "\n";
file << "variant: " << simulationVariant << "\n";
file << "LBM parameters:\n";
file << " - magic number = " << magicNumber << "\n";
file << " - bulk viscosity rate factor = " << omegaBulk << "\n";
file << " - use omega bulk adaption = " << useOmegaBulkAdaption << " (layer size = 2)\n";
file << "Collision parameters:\n";
file << " - subCycles = " << numRPDSubCycles << "\n";
file << " - collision time (Tc) = " << collisionTime << "\n";
file << "use lubrication correction = " << useLubricationCorrection << "\n";
file << " - minimum gap size non dim = " << lubricationMinimalGapSizeNonDim << "\n";
file << " - lubrication correction cut off normal = " << lubricationCutOffDistanceNormal << "\n";
file << " - lubrication correction cut off tangential translational = " << lubricationCutOffDistanceTangentialTranslational << "\n";
file << " - lubrication correction cut off tangential rotational = " << lubricationCutOffDistanceTangentialRotational << "\n";
file << "reconstructor type " << reconstructorType << "\n";
file << "apply outflow BC at top = " << applyOutflowBCAtTop << "\n";
file << "\nOutput quantities:\n";
file << "actual St = " << actualSt << "\n";
file << "actual Re = " << actualRe << "\n";
file << "impact times = " << tImpact << "\n";
file << "settling velocity = " << maxSettlingVel << "\n";
file << "maximal overlap = " << std::abs(minGapSize) << " (" << std::abs(minGapSize)/diameter << "%)\n";
file.close();
}
timeloopTiming.logResultOnRoot();
return EXIT_SUCCESS;
}
} // namespace oblique_wet_collision
int main( int argc, char **argv ){
oblique_wet_collision::main(argc, argv);
}
apps/benchmarks/FluidParticleCoupling/SettlingSphereInBox.cpp 0 → 100644
View file @ 7a6d3b57
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file SettlingSphereInBox.cpp
//! \ingroup lbm_mesapd_coupling
//! \author Christoph Rettinger <christoph.rettinger@fau.de>
//
//======================================================================================================================
#include "blockforest/Initialization.h"
#include "blockforest/communication/UniformBufferedScheme.h"
#include "boundary/all.h"
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/SharedFunctor.h"
#include "core/debug/Debug.h"
#include "core/debug/TestSubsystem.h"
#include "core/math/all.h"
#include "core/timing/RemainingTimeLogger.h"
#include "core/logging/all.h"
#include "domain_decomposition/SharedSweep.h"
#include "field/AddToStorage.h"
#include "field/StabilityChecker.h"
#include "field/communication/PackInfo.h"
#include "lbm/boundary/all.h"
#include "lbm/communication/PdfFieldPackInfo.h"
#include "lbm/field/AddToStorage.h"
#include "lbm/field/PdfField.h"
#include "lbm/lattice_model/D3Q19.h"
#include "lbm/sweeps/CellwiseSweep.h"
#include "lbm/sweeps/SweepWrappers.h"
#include "lbm_mesapd_coupling/mapping/ParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/MovingParticleMapping.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/boundary/CurvedLinear.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/Reconstructor.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/ExtrapolationDirectionFinder.h"
#include "lbm_mesapd_coupling/momentum_exchange_method/reconstruction/PdfReconstructionManager.h"
#include "lbm_mesapd_coupling/utility/AddForceOnParticlesKernel.h"
#include "lbm_mesapd_coupling/utility/ParticleSelector.h"
#include "lbm_mesapd_coupling/DataTypes.h"
#include "lbm_mesapd_coupling/utility/AverageHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/AddHydrodynamicInteractionKernel.h"
#include "lbm_mesapd_coupling/utility/InitializeHydrodynamicForceTorqueForAveragingKernel.h"
#include "lbm_mesapd_coupling/utility/ResetHydrodynamicForceTorqueKernel.h"
#include "lbm_mesapd_coupling/utility/LubricationCorrectionKernel.h"
#include "lbm_mesapd_coupling/utility/OmegaBulkAdaption.h"
#include "mesa_pd/collision_detection/AnalyticContactDetection.h"
#include "mesa_pd/data/ParticleAccessorWithShape.h"
#include "mesa_pd/data/ParticleStorage.h"
#include "mesa_pd/data/ShapeStorage.h"
#include "mesa_pd/data/DataTypes.h"
#include "mesa_pd/data/shape/HalfSpace.h"
#include "mesa_pd/data/shape/Sphere.h"
#include "mesa_pd/domain/BlockForestDomain.h"
#include "mesa_pd/kernel/DoubleCast.h"
#include "mesa_pd/kernel/ExplicitEuler.h"
#include "mesa_pd/kernel/ParticleSelector.h"
#include "mesa_pd/kernel/VelocityVerlet.h"
#include "mesa_pd/mpi/SyncNextNeighbors.h"
#include "mesa_pd/mpi/ReduceProperty.h"
#include "mesa_pd/mpi/ContactFilter.h"
#include "mesa_pd/mpi/notifications/ForceTorqueNotification.h"
#include "mesa_pd/mpi/notifications/HydrodynamicForceTorqueNotification.h"
#include "mesa_pd/vtk/ParticleVtkOutput.h"
#include "timeloop/SweepTimeloop.h"
#include "vtk/all.h"
#include "field/vtk/all.h"
#include "lbm/vtk/all.h"
#include <functional>
#ifdef WALBERLA_BUILD_WITH_CODEGEN
#include "GeneratedLBM.h"
#endif
namespace settling_sphere_in_box
{
///////////
// USING //
///////////
using namespace walberla;
using walberla::uint_t;
#ifdef WALBERLA_BUILD_WITH_CODEGEN
using LatticeModel_T = lbm::GeneratedLBM;
#else
using LatticeModel_T = lbm::D3Q19< lbm::collision_model::D3Q19MRT>;
#endif
using Stencil_T = LatticeModel_T::Stencil;
using PdfField_T = lbm::PdfField<LatticeModel_T>;
using flag_t = walberla::uint8_t;
using FlagField_T = FlagField<flag_t>;
using ScalarField_T = GhostLayerField< real_t, 1>;
const uint_t FieldGhostLayers = 1;
///////////
// FLAGS //
///////////
const FlagUID Fluid_Flag( "fluid" );
const FlagUID NoSlip_Flag( "no slip" );
const FlagUID MO_Flag( "moving obstacle" );
const FlagUID FormerMO_Flag( "former moving obstacle" );
/////////////////////////////////////
// BOUNDARY HANDLING CUSTOMIZATION //
/////////////////////////////////////
template <typename ParticleAccessor_T>
class MyBoundaryHandling
{
public:
using NoSlip_T = lbm::NoSlip< LatticeModel_T, flag_t >;
using MO_T = lbm_mesapd_coupling::CurvedLinear< LatticeModel_T, FlagField_T, ParticleAccessor_T >;
using Type = BoundaryHandling< FlagField_T, Stencil_T, NoSlip_T, MO_T >;
MyBoundaryHandling( const BlockDataID & flagFieldID, const BlockDataID & pdfFieldID,
const BlockDataID & particleFieldID, const shared_ptr<ParticleAccessor_T>& ac) :
flagFieldID_( flagFieldID ), pdfFieldID_( pdfFieldID ), particleFieldID_( particleFieldID ), ac_( ac ) {}
Type * operator()( IBlock* const block, const StructuredBlockStorage* const storage ) const
{
WALBERLA_ASSERT_NOT_NULLPTR( block );
WALBERLA_ASSERT_NOT_NULLPTR( storage );
auto * flagField = block->getData< FlagField_T >( flagFieldID_ );
auto * pdfField = block->getData< PdfField_T > ( pdfFieldID_ );
auto * particleField = block->getData< lbm_mesapd_coupling::ParticleField_T > ( particleFieldID_ );
const auto fluid = flagField->flagExists( Fluid_Flag ) ? flagField->getFlag( Fluid_Flag ) : flagField->registerFlag( Fluid_Flag );
Type * handling = new Type( "moving obstacle boundary handling", flagField, fluid,
NoSlip_T( "NoSlip", NoSlip_Flag, pdfField ),
MO_T( "MO", MO_Flag, pdfField, flagField, particleField, ac_, fluid, *storage, *block ) );
handling->fillWithDomain( FieldGhostLayers );
return handling;
}
private:
const BlockDataID flagFieldID_;
const BlockDataID pdfFieldID_;
const BlockDataID particleFieldID_;
shared_ptr<ParticleAccessor_T> ac_;
};
//*******************************************************************************************************************
//*******************************************************************************************************************
/*!\brief Evaluating the position and velocity of the sphere
*
*/
//*******************************************************************************************************************
template< typename ParticleAccessor_T>
class SpherePropertyLogger
{
public:
SpherePropertyLogger( const shared_ptr< ParticleAccessor_T > & ac, walberla::id_t sphereUid,
const std::string & fileName, bool fileIO,
real_t dx_SI, real_t dt_SI, real_t diameter, real_t gravitationalForceMag, real_t uref) :
ac_( ac ), sphereUid_( sphereUid ), fileName_( fileName ), fileIO_(fileIO),
dx_SI_( dx_SI ), dt_SI_( dt_SI ), diameter_( diameter ),
gravitationalForceMag_( gravitationalForceMag ), uref_(uref), tref_(diameter / uref),
position_( real_t(0) ), maxVelocity_( real_t(0) )
{
if ( fileIO_ )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str() );
file << "#\t t\t t/tref\t posZ\t gapZ/D\t velZ\t velZ_SI\t velZ/uref\t fZ\t fZ/fGravi\n";
file.close();
}
}
}
void operator()(const uint_t timestep)
{
Vector3<real_t> pos(real_t(0));
Vector3<real_t> transVel(real_t(0));
Vector3<real_t> hydForce(real_t(0));
size_t idx = ac_->uidToIdx(sphereUid_);
if( idx != ac_->getInvalidIdx())
{
if(!mesa_pd::data::particle_flags::isSet( ac_->getFlags(idx), mesa_pd::data::particle_flags::GHOST))
{
pos = ac_->getPosition(idx);
transVel = ac_->getLinearVelocity(idx);
hydForce = ac_->getHydrodynamicForce(idx);
}
}
WALBERLA_MPI_SECTION()
{
mpi::allReduceInplace( pos, mpi::SUM );
mpi::allReduceInplace( transVel, mpi::SUM );
mpi::allReduceInplace( hydForce, mpi::SUM );
}
position_ = pos[2];
maxVelocity_ = std::max(maxVelocity_, -transVel[2]);
if( fileIO_ )
writeToFile( timestep, pos, transVel, hydForce);
}
real_t getPosition() const
{
return position_;
}
real_t getMaxVelocity() const
{
return maxVelocity_;
}
private:
void writeToFile( const uint_t timestep, const Vector3<real_t> & position, const Vector3<real_t> & velocity, const Vector3<real_t> & hydForce )
{
WALBERLA_ROOT_SECTION()
{
std::ofstream file;
file.open( fileName_.c_str(), std::ofstream::app );
auto scaledPosition = position / diameter_;
auto velocity_SI = velocity * dx_SI_ / dt_SI_;
auto normalizedHydForce = hydForce / gravitationalForceMag_;
file << timestep << "\t" << real_c(timestep) * dt_SI_ << "\t" << real_c(timestep) / tref_
<< "\t" << position[2] << "\t" << scaledPosition[2] - real_t(0.5)
<< "\t" << velocity[2] << "\t" << velocity_SI[2] << "\t" << velocity[2] / uref_
<< "\t" << hydForce[2] << "\t" << normalizedHydForce[2]
<< "\n";
file.close();
}
}
shared_ptr< ParticleAccessor_T > ac_;
const walberla::id_t sphereUid_;
std::string fileName_;
bool fileIO_;
real_t dx_SI_, dt_SI_, diameter_, gravitationalForceMag_, uref_, tref_;
real_t position_;
real_t maxVelocity_;
};
void createPlaneSetup(const shared_ptr<mesa_pd::data::ParticleStorage> & ps, const shared_ptr<mesa_pd::data::ShapeStorage> & ss, const math::AABB & simulationDomain)
{
// create bounding planes
mesa_pd::data::Particle p0 = *ps->create(true);
p0.setPosition(simulationDomain.minCorner());
p0.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p0.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,1) ));
p0.setOwner(mpi::MPIManager::instance()->rank());
p0.setType(0);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p0.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p1 = *ps->create(true);
p1.setPosition(simulationDomain.maxCorner());
p1.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p1.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,0,-1) ));
p1.setOwner(mpi::MPIManager::instance()->rank());
p1.setType(0);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p1.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p2 = *ps->create(true);
p2.setPosition(simulationDomain.minCorner());
p2.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p2.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(1,0,0) ));
p2.setOwner(mpi::MPIManager::instance()->rank());
p2.setType(0);
mesa_pd::data::particle_flags::set(p2.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p2.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p3 = *ps->create(true);
p3.setPosition(simulationDomain.maxCorner());
p3.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p3.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(-1,0,0) ));
p3.setOwner(mpi::MPIManager::instance()->rank());
p3.setType(0);
mesa_pd::data::particle_flags::set(p3.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p3.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p4 = *ps->create(true);
p4.setPosition(simulationDomain.minCorner());
p4.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p4.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,1,0) ));
p4.setOwner(mpi::MPIManager::instance()->rank());
p4.setType(0);
mesa_pd::data::particle_flags::set(p4.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p4.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
mesa_pd::data::Particle p5 = *ps->create(true);
p5.setPosition(simulationDomain.maxCorner());
p5.setInteractionRadius(std::numeric_limits<real_t>::infinity());
p5.setShapeID(ss->create<mesa_pd::data::HalfSpace>( Vector3<real_t>(0,-1,0) ));
p5.setOwner(mpi::MPIManager::instance()->rank());
p5.setType(0);
mesa_pd::data::particle_flags::set(p5.getFlagsRef(), mesa_pd::data::particle_flags::INFINITE);
mesa_pd::data::particle_flags::set(p5.getFlagsRef(), mesa_pd::data::particle_flags::FIXED);
}
//////////
// MAIN //
//////////
//*******************************************************************************************************************
/*!\brief Testcase that simulates the settling of a sphere inside a rectangular column filled with viscous fluid
*
* see: ten Cate, Nieuwstad, Derksen, Van den Akker - "Particle imaging velocimetry experiments and lattice-Boltzmann
* simulations on a single sphere settling under gravity" (2002), Physics of Fluids, doi: 10.1063/1.1512918
*/
//*******************************************************************************************************************
int main( int argc, char **argv )
{
debug::enterTestMode();
mpi::Environment env( argc, argv );
///////////////////
// Customization //
///////////////////
// simulation control
bool shortrun = false;
bool funcTest = false;
bool fileIO = true;
uint_t vtkIOFreq = 0;
std::string baseFolder = "vtk_out_SettlingSphere";
std::string fileNameEnding = "";
// physical setup
uint_t fluidType = 1;
//numerical parameters
uint_t numberOfCellsInHorizontalDirection = uint_t(135);
bool averageForceTorqueOverTwoTimeSteps = true;
bool conserveMomentum = false;
uint_t numRPDSubCycles = uint_t(1);
bool useVelocityVerlet = true;
std::string reconstructorType = "Grad"; // Eq, EAN, Ext, Grad
real_t bulkViscRateFactor = real_t(1);
real_t magicNumber = real_t(3)/real_t(16);
real_t characteristicVelocity = real_t(0.02);
bool useOmegaBulkAdaption = false;
real_t adaptionLayerSize = real_t(2);
bool useLubricationCorrection = true;
bool useGalileoParameterization = false;
for( int i = 1; i < argc; ++i )
{
if( std::strcmp( argv[i], "--shortrun" ) == 0 ) { shortrun = true; continue; }
if( std::strcmp( argv[i], "--funcTest" ) == 0 ) { funcTest = true; continue; }
if( std::strcmp( argv[i], "--noLogging" ) == 0 ) { fileIO = false; continue; }
if( std::strcmp( argv[i], "--vtkIOFreq" ) == 0 ) { vtkIOFreq = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--fluidType" ) == 0 ) { fluidType = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--numRPDSubCycles" ) == 0 ) { numRPDSubCycles = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--resolution" ) == 0 ) { numberOfCellsInHorizontalDirection = uint_c( std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--noForceAveraging" ) == 0 ) { averageForceTorqueOverTwoTimeSteps = false; continue; }
if( std::strcmp( argv[i], "--conserveMomentum" ) == 0 ) { conserveMomentum = true; continue; }
if( std::strcmp( argv[i], "--baseFolder" ) == 0 ) { baseFolder = argv[++i]; continue; }
if( std::strcmp( argv[i], "--useEulerIntegrator" ) == 0 ) { useVelocityVerlet = false; continue; }
if( std::strcmp( argv[i], "--reconstructorType" ) == 0 ) { reconstructorType = argv[++i]; continue; }
if( std::strcmp( argv[i], "--bulkViscRateFactor" ) == 0 ) { bulkViscRateFactor = real_c(std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--magicNumber" ) == 0 ) { magicNumber = real_c(std::atof( argv[++i] ) ); continue; }
if( std::strcmp( argv[i], "--velocity" ) == 0 ) { characteristicVelocity = real_c(std::atof( argv[++i] )); continue; }
if( std::strcmp( argv[i], "--fileName" ) == 0 ) { fileNameEnding = argv[++i]; continue; }
if( std::strcmp( argv[i], "--useOmegaBulkAdaption" ) == 0 ) { useOmegaBulkAdaption = true; continue; }
if( std::strcmp( argv[i], "--adaptionLayerSize" ) == 0 ) { adaptionLayerSize = real_c(std::atof( argv[++i] )); continue; }
if( std::strcmp( argv[i], "--noLubricationCorrection" ) == 0 ) { useLubricationCorrection = false; continue; }
if( std::strcmp( argv[i], "--useGalileoParameterization" ) == 0 ) { useGalileoParameterization = true; continue; }
WALBERLA_ABORT("Unrecognized command line argument found: " << argv[i]);
}
if( funcTest )
{
walberla::logging::Logging::instance()->setLogLevel(logging::Logging::LogLevel::WARNING);
}
if( fileIO )
{
// create base directory if it does not yet exist
filesystem::path tpath( baseFolder );
if( !filesystem::exists( tpath ) )
filesystem::create_directory( tpath );
}
//////////////////////////////////////
// SIMULATION PROPERTIES in SI units//
//////////////////////////////////////
// values are mainly taken from the reference paper
const real_t diameter_SI = real_t(15e-3);
const real_t densitySphere_SI = real_t(1120);
real_t densityFluid_SI;
real_t dynamicViscosityFluid_SI;
real_t expectedSettlingVelocity_SI;
// expected velocity given as uMax in experiments of ten Cate (Table 2, E1-E4), with uInfty from Table 1
// > slightly different Re than in ten Cate's Table 1
switch( fluidType )
{
case 1:
// Re_p around 1.5
densityFluid_SI = real_t(970);
dynamicViscosityFluid_SI = real_t(373e-3);
expectedSettlingVelocity_SI = real_t(0.035986);
break;
case 2:
// Re_p around 4.1
densityFluid_SI = real_t(965);
dynamicViscosityFluid_SI = real_t(212e-3);
expectedSettlingVelocity_SI = real_t(0.05718);
break;
case 3:
// Re_p around 11.6
densityFluid_SI = real_t(962);
dynamicViscosityFluid_SI = real_t(113e-3);
expectedSettlingVelocity_SI = real_t(0.087269);
break;
case 4:
// Re_p around 31.9
densityFluid_SI = real_t(960);
dynamicViscosityFluid_SI = real_t(58e-3);
expectedSettlingVelocity_SI = real_t(0.12224);
break;
default:
WALBERLA_ABORT("Only four different fluids are supported! Choose type between 1 and 4.");
}
const real_t kinematicViscosityFluid_SI = dynamicViscosityFluid_SI / densityFluid_SI;
const real_t gravitationalAcceleration_SI = real_t(9.81);
const real_t ug_SI = std::sqrt((densitySphere_SI/densityFluid_SI-real_t(1))*diameter_SI*gravitationalAcceleration_SI);
const real_t GalileoNumber = ug_SI * diameter_SI / (dynamicViscosityFluid_SI/densityFluid_SI);
Vector3<real_t> domainSize_SI(real_t(100e-3), real_t(100e-3), real_t(160e-3));
//shift starting gap a bit upwards to match the reported (plotted) values
const real_t startingGapSize_SI = real_t(120e-3) + real_t(0.25) * diameter_SI;
WALBERLA_LOG_INFO_ON_ROOT("Setup (in SI units):");
WALBERLA_LOG_INFO_ON_ROOT(" - fluid type = " << fluidType );
WALBERLA_LOG_INFO_ON_ROOT(" - domain size = " << domainSize_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - sphere: diameter = " << diameter_SI << ", density = " << densitySphere_SI << ", starting gap size = " << startingGapSize_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - fluid: density = " << densityFluid_SI << ", dyn. visc = " << dynamicViscosityFluid_SI << ", kin. visc = " << kinematicViscosityFluid_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - expected settling velocity = " << expectedSettlingVelocity_SI << " --> Re_p = " << expectedSettlingVelocity_SI * diameter_SI / kinematicViscosityFluid_SI );
WALBERLA_LOG_INFO_ON_ROOT(" - gravitational velocity = " << ug_SI << " --> Ga = " << GalileoNumber );
//////////////////////////
// NUMERICAL PARAMETERS //
//////////////////////////
const real_t dx_SI = domainSize_SI[0] / real_c(numberOfCellsInHorizontalDirection);
const Vector3<uint_t> domainSize( uint_c(floor(domainSize_SI[0] / dx_SI + real_t(0.5)) ),
uint_c(floor(domainSize_SI[1] / dx_SI + real_t(0.5)) ),
uint_c(floor(domainSize_SI[2] / dx_SI + real_t(0.5)) ) );
const real_t diameter = diameter_SI / dx_SI;
const real_t sphereVolume = math::pi / real_t(6) * diameter * diameter * diameter;
const real_t dt_SI = (useGalileoParameterization) ? characteristicVelocity / ug_SI * dx_SI : // this uses Ga for parameterization, where ug is the characteristic velocity
characteristicVelocity / expectedSettlingVelocity_SI * dx_SI; // this uses Re for parameterization (only possible since settling velocity is known)
const real_t viscosity = kinematicViscosityFluid_SI * dt_SI / ( dx_SI * dx_SI );
const real_t omega = lbm::collision_model::omegaFromViscosity(viscosity);
const real_t relaxationTime = real_t(1) / omega;
const real_t omegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, bulkViscRateFactor);
const real_t gravitationalAcceleration = gravitationalAcceleration_SI * dt_SI * dt_SI / dx_SI;
const real_t densityFluid = real_t(1);
const real_t densitySphere = densityFluid * densitySphere_SI / densityFluid_SI;
const real_t expectedSettlingVelocity = expectedSettlingVelocity_SI * dt_SI / dx_SI;
const real_t ug = std::sqrt((densitySphere/densityFluid-real_t(1))*diameter*gravitationalAcceleration);
const real_t GalileoNumber_LBM = ug * diameter / viscosity;
const real_t dx = real_t(1);
const uint_t timesteps = funcTest ? 1 : ( shortrun ? uint_t(200) : uint_t( 250000 ) );
WALBERLA_LOG_INFO_ON_ROOT(" - dx_SI = " << dx_SI << ", dt_SI = " << dt_SI);
WALBERLA_LOG_INFO_ON_ROOT("Setup (in simulation, i.e. lattice, units):");
WALBERLA_LOG_INFO_ON_ROOT(" - domain size = " << domainSize);
WALBERLA_LOG_INFO_ON_ROOT(" - sphere: diameter = " << diameter << ", density = " << densitySphere );
WALBERLA_LOG_INFO_ON_ROOT(" - fluid: density = " << densityFluid << ", relaxation time (tau) = " << relaxationTime << ", omega = " << omega << " kin. visc = " << viscosity );
WALBERLA_LOG_INFO_ON_ROOT(" - magic number = " << magicNumber);
WALBERLA_LOG_INFO_ON_ROOT(" - omegaBulk = " << omegaBulk << ", bulk visc. = " << lbm_mesapd_coupling::bulkViscosityFromOmegaBulk(omegaBulk) << " (bvrf " << bulkViscRateFactor << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - use omega bulk adaption = " << useOmegaBulkAdaption << " (adaption layer size = " << adaptionLayerSize << ")");
WALBERLA_LOG_INFO_ON_ROOT(" - gravitational acceleration = " << gravitationalAcceleration );
WALBERLA_LOG_INFO_ON_ROOT(" - expected settling velocity = " << expectedSettlingVelocity << " --> Re_p = " << expectedSettlingVelocity * diameter / viscosity );
WALBERLA_LOG_INFO_ON_ROOT(" - gravitational velocity = " << ug << " --> Ga = " << GalileoNumber_LBM );
WALBERLA_LOG_INFO_ON_ROOT(" - integrator = " << (useVelocityVerlet ? "Velocity Verlet" : "Explicit Euler") );
WALBERLA_LOG_INFO_ON_ROOT(" - conserve momentum = " << (conserveMomentum ? "yes" : "no") );
WALBERLA_LOG_INFO_ON_ROOT(" - reconstructor type = " << reconstructorType );
WALBERLA_LOG_INFO_ON_ROOT(" - lubrication correction = " << useLubricationCorrection );
if( vtkIOFreq > 0 )
{
WALBERLA_LOG_INFO_ON_ROOT(" - writing vtk files to folder \"" << baseFolder << "\" with frequency " << vtkIOFreq);
}
///////////////////////////
// BLOCK STRUCTURE SETUP //
///////////////////////////
Vector3<uint_t> numberOfBlocksPerDirection( uint_t(1), uint_t(1), uint_t(4) );
Vector3<uint_t> cellsPerBlockPerDirection( domainSize[0] / numberOfBlocksPerDirection[0],
domainSize[1] / numberOfBlocksPerDirection[1],
domainSize[2] / numberOfBlocksPerDirection[2] );
for( uint_t i = 0; i < 3; ++i ) {
WALBERLA_CHECK_EQUAL(cellsPerBlockPerDirection[i] * numberOfBlocksPerDirection[i], domainSize[i],
"Unmatching domain decomposition in direction " << i << "!");
}
auto blocks = blockforest::createUniformBlockGrid( numberOfBlocksPerDirection[0], numberOfBlocksPerDirection[1], numberOfBlocksPerDirection[2],
cellsPerBlockPerDirection[0], cellsPerBlockPerDirection[1], cellsPerBlockPerDirection[2], dx,
0, false, false,
false, false, false, //periodicity
false );
WALBERLA_LOG_INFO_ON_ROOT("Domain decomposition:");
WALBERLA_LOG_INFO_ON_ROOT(" - blocks per direction = " << numberOfBlocksPerDirection );
WALBERLA_LOG_INFO_ON_ROOT(" - cells per block = " << cellsPerBlockPerDirection );
//write domain decomposition to file
if( vtkIOFreq > 0 )
{
vtk::writeDomainDecomposition( blocks, "initial_domain_decomposition", baseFolder );
}
//////////////////
// RPD COUPLING //
//////////////////
auto rpdDomain = std::make_shared<mesa_pd::domain::BlockForestDomain>(blocks->getBlockForestPointer());
//init data structures
auto ps = walberla::make_shared<mesa_pd::data::ParticleStorage>(1);
auto ss = walberla::make_shared<mesa_pd::data::ShapeStorage>();
using ParticleAccessor_T = mesa_pd::data::ParticleAccessorWithShape;
auto accessor = walberla::make_shared<ParticleAccessor_T >(ps, ss);
// bounding planes
createPlaneSetup(ps,ss,blocks->getDomain());
// create sphere and store Uid
Vector3<real_t> initialPosition( real_t(0.5) * real_c(domainSize[0]), real_t(0.5) * real_c(domainSize[1]), startingGapSize_SI / dx_SI + real_t(0.5) * diameter);
auto sphereShape = ss->create<mesa_pd::data::Sphere>( diameter * real_t(0.5) );
ss->shapes[sphereShape]->updateMassAndInertia(densitySphere);
walberla::id_t sphereUid = 0;
if (rpdDomain->isContainedInProcessSubdomain( uint_c(mpi::MPIManager::instance()->rank()), initialPosition ))
{
mesa_pd::data::Particle&& p = *ps->create();
p.setPosition(initialPosition);
p.setInteractionRadius(diameter * real_t(0.5));
p.setOwner(mpi::MPIManager::instance()->rank());
p.setShapeID(sphereShape);
sphereUid = p.getUid();
}
mpi::allReduceInplace(sphereUid, mpi::SUM);
///////////////////////
// ADD DATA TO BLOCKS //
////////////////////////
// add omega bulk field
BlockDataID omegaBulkFieldID = field::addToStorage<ScalarField_T>( blocks, "omega bulk field", omegaBulk, field::fzyx);
// create the lattice model
real_t lambda_e = lbm::collision_model::TRT::lambda_e( omega );
real_t lambda_d = lbm::collision_model::TRT::lambda_d( omega, magicNumber );
#ifdef WALBERLA_BUILD_WITH_CODEGEN
WALBERLA_LOG_INFO_ON_ROOT("Using generated TRT-like lattice model!");
LatticeModel_T latticeModel = LatticeModel_T(omegaBulkFieldID, lambda_d, lambda_e);
#else
WALBERLA_LOG_INFO_ON_ROOT("Using waLBerla built-in MRT lattice model and ignoring omega bulk field since not supported!");
LatticeModel_T latticeModel = LatticeModel_T(lbm::collision_model::D3Q19MRT( omegaBulk, omegaBulk, lambda_d, lambda_e, lambda_e, lambda_d ));
#endif
// add PDF field
BlockDataID pdfFieldID = lbm::addPdfFieldToStorage< LatticeModel_T >( blocks, "pdf field (fzyx)", latticeModel,
Vector3< real_t >( real_t(0) ), real_t(1),
uint_t(1), field::fzyx );
// add flag field
BlockDataID flagFieldID = field::addFlagFieldToStorage<FlagField_T>( blocks, "flag field" );
// add particle field
BlockDataID particleFieldID = field::addToStorage<lbm_mesapd_coupling::ParticleField_T>( blocks, "particle field", accessor->getInvalidUid(), field::fzyx, FieldGhostLayers );
// add boundary handling
using BoundaryHandling_T = MyBoundaryHandling<ParticleAccessor_T>::Type;
BlockDataID boundaryHandlingID = blocks->addStructuredBlockData< BoundaryHandling_T >(MyBoundaryHandling<ParticleAccessor_T>( flagFieldID, pdfFieldID, particleFieldID, accessor), "boundary handling" );
// set up RPD functionality
std::function<void(void)> syncCall = [ps,rpdDomain](){
const real_t overlap = real_t( 1.5 );
mesa_pd::mpi::SyncNextNeighbors syncNextNeighborFunc;
syncNextNeighborFunc(*ps, *rpdDomain, overlap);
};
syncCall();
mesa_pd::kernel::ExplicitEuler explEulerIntegrator(real_t(1)/real_t(numRPDSubCycles));
mesa_pd::kernel::VelocityVerletPreForceUpdate vvIntegratorPreForce(real_t(1)/real_t(numRPDSubCycles));
mesa_pd::kernel::VelocityVerletPostForceUpdate vvIntegratorPostForce(real_t(1)/real_t(numRPDSubCycles));
mesa_pd::mpi::ReduceProperty reduceProperty;
// set up coupling functionality
Vector3<real_t> gravitationalForce( real_t(0), real_t(0), -(densitySphere - densityFluid) * gravitationalAcceleration * sphereVolume );
lbm_mesapd_coupling::AddForceOnParticlesKernel addGravitationalForce(gravitationalForce);
lbm_mesapd_coupling::AddHydrodynamicInteractionKernel addHydrodynamicInteraction;
lbm_mesapd_coupling::ResetHydrodynamicForceTorqueKernel resetHydrodynamicForceTorque;
lbm_mesapd_coupling::AverageHydrodynamicForceTorqueKernel averageHydrodynamicForceTorque;
lbm_mesapd_coupling::LubricationCorrectionKernel lubricationCorrectionKernel(viscosity, [](real_t r){ return real_t(0.0016) * r;});
lbm_mesapd_coupling::RegularParticlesSelector sphereSelector;
lbm_mesapd_coupling::ParticleMappingKernel<BoundaryHandling_T> particleMappingKernel(blocks, boundaryHandlingID);
lbm_mesapd_coupling::MovingParticleMappingKernel<BoundaryHandling_T> movingParticleMappingKernel(blocks, boundaryHandlingID, particleFieldID);
///////////////
// TIME LOOP //
///////////////
// map planes into the LBM simulation -> act as no-slip boundaries
ps->forEachParticle(false, lbm_mesapd_coupling::GlobalParticlesSelector(), *accessor, particleMappingKernel, *accessor, NoSlip_Flag);
// map particles into the LBM simulation
ps->forEachParticle(false, sphereSelector, *accessor, movingParticleMappingKernel, *accessor, MO_Flag);
// setup of the LBM communication for synchronizing the pdf field between neighboring blocks
blockforest::communication::UniformBufferedScheme< Stencil_T > optimizedPDFCommunicationScheme( blocks );
optimizedPDFCommunicationScheme.addPackInfo( make_shared< lbm::PdfFieldPackInfo< LatticeModel_T > >( pdfFieldID ) ); // optimized sync
blockforest::communication::UniformBufferedScheme< Stencil_T > fullPDFCommunicationScheme( blocks );
fullPDFCommunicationScheme.addPackInfo( make_shared< field::communication::PackInfo< PdfField_T > >( pdfFieldID ) ); // full sync
// create the timeloop
SweepTimeloop timeloop( blocks->getBlockStorage(), timesteps );
timeloop.addFuncBeforeTimeStep( RemainingTimeLogger( timeloop.getNrOfTimeSteps() ), "Remaining Time Logger" );
// vtk output
if( vtkIOFreq != uint_t(0) )
{
// spheres
auto particleVtkOutput = make_shared<mesa_pd::vtk::ParticleVtkOutput>(ps);
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleOwner>("owner");
particleVtkOutput->addOutput<mesa_pd::data::SelectParticleLinearVelocity>("velocity");
auto particleVtkWriter = vtk::createVTKOutput_PointData(particleVtkOutput, "Particles", vtkIOFreq, baseFolder, "simulation_step");
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( particleVtkWriter ), "VTK (sphere data)" );
// flag field (written only once in the first time step, ghost layers are also written)
//auto flagFieldVTK = vtk::createVTKOutput_BlockData( blocks, "flag_field", timesteps, FieldGhostLayers, false, baseFolder );
//flagFieldVTK->addCellDataWriter( make_shared< field::VTKWriter< FlagField_T > >( flagFieldID, "FlagField" ) );
//timeloop.addFuncBeforeTimeStep( vtk::writeFiles( flagFieldVTK ), "VTK (flag field data)" );
// pdf field
auto pdfFieldVTK = vtk::createVTKOutput_BlockData( blocks, "fluid_field", vtkIOFreq, 0, false, baseFolder );
pdfFieldVTK->addBeforeFunction( fullPDFCommunicationScheme );
field::FlagFieldCellFilter< FlagField_T > fluidFilter( flagFieldID );
fluidFilter.addFlag( Fluid_Flag );
pdfFieldVTK->addCellInclusionFilter( fluidFilter );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::VelocityVTKWriter< LatticeModel_T, float > >( pdfFieldID, "VelocityFromPDF" ) );
pdfFieldVTK->addCellDataWriter( make_shared< lbm::DensityVTKWriter < LatticeModel_T, float > >( pdfFieldID, "DensityFromPDF" ) );
timeloop.addFuncBeforeTimeStep( vtk::writeFiles( pdfFieldVTK ), "VTK (fluid field data)" );
// omega bulk field
timeloop.addFuncBeforeTimeStep( field::createVTKOutput<ScalarField_T, float>( omegaBulkFieldID, *blocks, "omega_bulk_field", vtkIOFreq, uint_t(0), false, baseFolder ), "VTK (omega bulk field)" );
}
// sweep for updating the particle mapping into the LBM simulation
timeloop.add() << Sweep( lbm_mesapd_coupling::makeMovingParticleMapping<PdfField_T, BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, MO_Flag, FormerMO_Flag, sphereSelector, conserveMomentum), "Particle Mapping" );
// sweep for restoring PDFs in cells previously occupied by particles
if( reconstructorType == "EAN")
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
auto equilibriumAndNonEquilibriumSphereNormalReconstructor = lbm_mesapd_coupling::makeEquilibriumAndNonEquilibriumReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
auto reconstructionManager = lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, equilibriumAndNonEquilibriumSphereNormalReconstructor, conserveMomentum);
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(reconstructionManager), "PDF Restore" );
} else if( reconstructorType == "Ext" )
{
auto sphereNormalExtrapolationDirectionFinder = make_shared<lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder>(blocks);
auto extrapolationSphereNormalReconstructor = lbm_mesapd_coupling::makeExtrapolationReconstructor<BoundaryHandling_T,lbm_mesapd_coupling::SphereNormalExtrapolationDirectionFinder,true>(blocks, boundaryHandlingID, sphereNormalExtrapolationDirectionFinder, uint_t(3), true);
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, extrapolationSphereNormalReconstructor, conserveMomentum) ), "PDF Restore" );
} else if( reconstructorType == "Grad")
{
auto gradReconstructor = lbm_mesapd_coupling::makeGradsMomentApproximationReconstructor<BoundaryHandling_T>(blocks, boundaryHandlingID, omega, false, true, true);
timeloop.add() << BeforeFunction( fullPDFCommunicationScheme, "PDF Communication" )
<< Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, gradReconstructor, conserveMomentum) ), "PDF Restore" );
} else if( reconstructorType == "Eq")
{
timeloop.add() << Sweep( makeSharedSweep(lbm_mesapd_coupling::makePdfReconstructionManager<PdfField_T,BoundaryHandling_T>(blocks, pdfFieldID, boundaryHandlingID, particleFieldID, accessor, FormerMO_Flag, Fluid_Flag, conserveMomentum) ), "PDF Restore" );
} else {
WALBERLA_ABORT("Unknown reconstructor type " << reconstructorType);
}
// update bulk omega in all cells to adapt to changed particle position
if( useOmegaBulkAdaption )
{
using OmegaBulkAdapter_T = lbm_mesapd_coupling::OmegaBulkAdapter<ParticleAccessor_T, decltype(sphereSelector)>;
real_t defaultOmegaBulk = lbm_mesapd_coupling::omegaBulkFromOmega(omega, real_t(1));
shared_ptr<OmegaBulkAdapter_T> omegaBulkAdapter = make_shared<OmegaBulkAdapter_T>(blocks, omegaBulkFieldID, accessor, defaultOmegaBulk, omegaBulk, adaptionLayerSize, sphereSelector);
timeloop.add() << Sweep( makeSharedSweep(omegaBulkAdapter), "Omega Bulk Adapter");
}
// add LBM communication function and boundary handling sweep (does the hydro force calculations and the no-slip treatment)
auto bhSweep = BoundaryHandling_T::getBlockSweep( boundaryHandlingID );
timeloop.add() << BeforeFunction( optimizedPDFCommunicationScheme, "LBM Communication" )
<< Sweep(bhSweep, "Boundary Handling" );
// stream + collide LBM step
#ifdef WALBERLA_BUILD_WITH_CODEGEN
auto lbmSweep = LatticeModel_T::Sweep( pdfFieldID );
timeloop.add() << Sweep( lbmSweep, "LB sweep" );
#else
auto lbmSweep = lbm::makeCellwiseSweep< LatticeModel_T, FlagField_T >( pdfFieldID, flagFieldID, Fluid_Flag );
timeloop.add() << Sweep( makeSharedSweep( lbmSweep ), "cell-wise LB sweep" );
#endif
// evaluation functionality
std::string loggingFileName( baseFolder + "/LoggingSettlingSphere_");
loggingFileName += std::to_string(fluidType);
loggingFileName += "_res" + std::to_string(numberOfCellsInHorizontalDirection);
loggingFileName += "_recon" + reconstructorType;
loggingFileName += "_bvrf" + std::to_string(uint_c(bulkViscRateFactor));
loggingFileName += "_mn" + std::to_string(float(magicNumber));
if( useOmegaBulkAdaption ) loggingFileName += "_uOBA" + std::to_string(uint_c(adaptionLayerSize));
if( useGalileoParameterization ) loggingFileName += "_Ga";
if( !fileNameEnding.empty()) loggingFileName += "_" + fileNameEnding;
loggingFileName += ".txt";
if( fileIO )
{
WALBERLA_LOG_INFO_ON_ROOT(" - writing logging output to file \"" << loggingFileName << "\"");
}
SpherePropertyLogger<ParticleAccessor_T> logger( accessor, sphereUid, loggingFileName, fileIO, dx_SI, dt_SI, diameter, -gravitationalForce[2], characteristicVelocity );
////////////////////////
// EXECUTE SIMULATION //
////////////////////////
WcTimingPool timeloopTiming;
real_t terminationPosition = diameter * real_t(0.501); // right before sphere touches the bottom wall
real_t oldPos = initialPosition[2];
const bool useOpenMP = false;
// time loop
for (uint_t i = 0; i < timesteps; ++i )
{
// perform a single simulation step -> this contains LBM and setting of the hydrodynamic interactions
timeloop.singleStep( timeloopTiming );
timeloopTiming["RPD"].start();
reduceProperty.operator()<mesa_pd::HydrodynamicForceTorqueNotification>(*ps);
if( averageForceTorqueOverTwoTimeSteps )
{
if( i == 0 )
{
lbm_mesapd_coupling::InitializeHydrodynamicForceTorqueForAveragingKernel initializeHydrodynamicForceTorqueForAveragingKernel;
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, initializeHydrodynamicForceTorqueForAveragingKernel, *accessor );
}
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, averageHydrodynamicForceTorque, *accessor );
}
for(auto subCycle = uint_t(0); subCycle < numRPDSubCycles; ++subCycle )
{
if( useVelocityVerlet )
{
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPreForce, *accessor);
syncCall();
}
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addHydrodynamicInteraction, *accessor );
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, addGravitationalForce, *accessor );
// lubrication correction
if(useLubricationCorrection)
{
ps->forEachParticlePairHalf(useOpenMP, mesa_pd::kernel::ExcludeInfiniteInfinite(), *accessor,
[&lubricationCorrectionKernel,rpdDomain](const size_t idx1, const size_t idx2, auto& ac)
{
//TODO change this to storing copy, not reference
mesa_pd::collision_detection::AnalyticContactDetection acd;
acd.getContactThreshold() = lubricationCorrectionKernel.getNormalCutOffDistance();
mesa_pd::kernel::DoubleCast double_cast;
mesa_pd::mpi::ContactFilter contact_filter;
if (double_cast(idx1, idx2, ac, acd, ac ))
{
if (contact_filter(acd.getIdx1(), acd.getIdx2(), ac, acd.getContactPoint(), *rpdDomain))
{
double_cast(idx1, idx2, ac, lubricationCorrectionKernel, ac, acd.getContactNormal(), acd.getPenetrationDepth());
}
}
},
*accessor );
}
reduceProperty.operator()<mesa_pd::ForceTorqueNotification>(*ps);
if( useVelocityVerlet ) ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, vvIntegratorPostForce, *accessor);
else ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectLocal(), *accessor, explEulerIntegrator, *accessor);
syncCall();
}
timeloopTiming["RPD"].end();
// evaluation
timeloopTiming["Logging"].start();
logger(i);
timeloopTiming["Logging"].end();
// reset after logging here
ps->forEachParticle(useOpenMP, mesa_pd::kernel::SelectAll(), *accessor, resetHydrodynamicForceTorque, *accessor );
// check for termination
if ( logger.getPosition() < terminationPosition || oldPos < logger.getPosition() )
{
WALBERLA_LOG_INFO_ON_ROOT("Sphere reached terminal position " << logger.getPosition() << " after " << i << " timesteps!");
break;
}
oldPos = logger.getPosition();
}
timeloopTiming.logResultOnRoot();
// check the result
if ( !funcTest && !shortrun )
{
real_t relErr = std::fabs( expectedSettlingVelocity - logger.getMaxVelocity()) / expectedSettlingVelocity;
WALBERLA_LOG_INFO_ON_ROOT( "Expected maximum settling velocity: " << expectedSettlingVelocity );
WALBERLA_LOG_INFO_ON_ROOT( "Simulated maximum settling velocity: " << logger.getMaxVelocity() );
WALBERLA_LOG_INFO_ON_ROOT( "Relative error: " << relErr );
// the relative error has to be below 10%
WALBERLA_CHECK_LESS( relErr, real_t(0.1) );
}
return EXIT_SUCCESS;
}
} // namespace settling_sphere_in_box
int main( int argc, char **argv ){
settling_sphere_in_box::main(argc, argv);
}