diff --git a/apps/benchmarks/GranularGas/Accessor.h b/apps/benchmarks/GranularGas/Accessor.h new file mode 100644 index 0000000000000000000000000000000000000000..13667332c7553c48142c776ef6fc00be7e4ac065 --- /dev/null +++ b/apps/benchmarks/GranularGas/Accessor.h @@ -0,0 +1,50 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Accessor.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleAccessor.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/data/ShapeStorage.h> + +namespace walberla { +namespace mesa_pd { + +class ParticleAccessorWithShape : public data::ParticleAccessor +{ +public: + ParticleAccessorWithShape(std::shared_ptr<data::ParticleStorage>& ps, std::shared_ptr<data::ShapeStorage>& ss) + : ParticleAccessor(ps) + , ss_(ss) + {} + + const walberla::real_t& getInvMass(const size_t p_idx) const {return ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvMass();} + walberla::real_t& getInvMassRef(const size_t p_idx) {return ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvMass();} + void setInvMass(const size_t p_idx, const walberla::real_t& v) { ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvMass() = v;} + + const auto& getInvInertiaBF(const size_t p_idx) const {return ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvInertiaBF();} + auto& getInvInertiaBFRef(const size_t p_idx) {return ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvInertiaBF();} + void setInvInertiaBF(const size_t p_idx, const Mat3& v) { ss_->shapes[ps_->getShapeIDRef(p_idx)]->getInvInertiaBF() = v;} + + data::BaseShape* getShape(const size_t p_idx) const {return ss_->shapes[ps_->getShapeIDRef(p_idx)].get();} +private: + std::shared_ptr<data::ShapeStorage> ss_; +}; + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/CMakeLists.txt b/apps/benchmarks/GranularGas/CMakeLists.txt index 50ec1c768eb71468e3cc230b65a0854959f76794..0a0f9b4623b330104addd791337977b7ddfb735a 100644 --- a/apps/benchmarks/GranularGas/CMakeLists.txt +++ b/apps/benchmarks/GranularGas/CMakeLists.txt @@ -2,13 +2,13 @@ waLBerla_link_files_to_builddir( *.cfg ) waLBerla_link_files_to_builddir( *.py ) waLBerla_add_executable ( NAME PE_GranularGas - FILES PE_GranularGas.cpp + FILES PE_GranularGas.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp DEPENDS blockforest core pe postprocessing ) waLBerla_add_executable ( NAME MESA_PD_GranularGas - FILES MESA_PD_GranularGas.cpp + FILES MESA_PD_GranularGas.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp sortParticleStorage.cpp CreateParticles.cpp DEPENDS blockforest core pe mesa_pd postprocessing vtk ) waLBerla_add_executable ( NAME MESA_PD_KernelBenchmark - FILES MESA_PD_KernelBenchmark.cpp + FILES MESA_PD_KernelBenchmark.cpp SQLProperties.cpp Parameters.cpp NodeTimings.cpp sortParticleStorage.cpp CreateParticles.cpp DEPENDS blockforest core pe mesa_pd postprocessing vtk ) diff --git a/apps/benchmarks/GranularGas/ConfigGenerator.py b/apps/benchmarks/GranularGas/ConfigGenerator.py new file mode 100644 index 0000000000000000000000000000000000000000..8c3ad3f8feb4085eba3febc2757d43e8d5fd125d --- /dev/null +++ b/apps/benchmarks/GranularGas/ConfigGenerator.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +from jinja2 import Environment, FileSystemLoader +import os + +class Parameter: + def __init__(self, name, type, defValue=""): + """Propery of a data strcuture + + Parameters + ---------- + name : str + name of the property + type : str + type of the property + defValue : str + default value the property should be initialized with + """ + + self.name = name + self.type = type + self.defValue = defValue + + def __str__(self): + return "name: {}, type: {}, defValue: {}".format(self.name, self.type, self.defValue) + +class Config: + def __init__(self): + self.parameters = [] + + def parameterExists(self, name): + for v in self.parameters: + if v.name==name: + return True + return False + + def addParameter(self, name, type, defValue): + if self.parameterExists( name ): + print("parameters already added: " + name) + else: + self.parameters.append( Parameter(name, type, defValue) ) + + def generateFile(self, template): + context = dict() + context["parameters"] = self.parameters + + path = "" + filename = template.replace(".templ", "") + dirname = os.path.dirname(__file__) + env = Environment(loader=FileSystemLoader(dirname)) + print("generating: " + path + filename) + fout = open(path + filename, "wb") + content = env.get_template(template).render(context) + fout.write(content.encode('utf8')) + fout.close() + + def generate(self): + print("="*90) + print("Config File:") + print("") + print("{0: <30}{1: <30}{2: <30}".format("Name", "Type", "Def. Value")) + print("="*90) + for param in self.parameters: + print("{0: <30.29}{1: <30.29}{2: <30.29}".format(param.name, param.type, param.defValue)) + print("="*90) + + self.generateFile("Parameters.templ.h") + self.generateFile("Parameters.templ.cpp") diff --git a/apps/benchmarks/GranularGas/Contact.h b/apps/benchmarks/GranularGas/Contact.h new file mode 100644 index 0000000000000000000000000000000000000000..e26272ce1d69253864f373c8b77a6c01c4af62ae --- /dev/null +++ b/apps/benchmarks/GranularGas/Contact.h @@ -0,0 +1,47 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Contact.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/DataTypes.h> + +namespace walberla { +namespace mesa_pd { + +struct Contact +{ + Contact(const size_t idx1, + const size_t idx2, + const Vec3 contactPoint, + const Vec3 contactNormal, + const real_t penetrationDepth) + : idx1_(idx1) + , idx2_(idx2) + , contactPoint_(contactPoint) + , contactNormal_(contactNormal) + , penetrationDepth_(penetrationDepth) {} + + size_t idx1_; + size_t idx2_; + Vec3 contactPoint_; + Vec3 contactNormal_; + real_t penetrationDepth_; +}; + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/CreateParticles.cpp b/apps/benchmarks/GranularGas/CreateParticles.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4c788c163066a68ce70f0448f458fbbb7877bfc7 --- /dev/null +++ b/apps/benchmarks/GranularGas/CreateParticles.cpp @@ -0,0 +1,57 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file CreateParticles.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include "CreateParticles.h" + +namespace walberla { +namespace mesa_pd { + +data::ParticleStorage::iterator createPlane( data::ParticleStorage& ps, + data::ShapeStorage& ss, + const Vec3& pos, + const Vec3& normal ) +{ + auto p0 = ps.create(true); + p0->getPositionRef() = pos; + p0->getShapeIDRef() = ss.create<data::HalfSpace>( normal ); + p0->getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); + p0->getTypeRef() = 0; + data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::INFINITE); + data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::FIXED); + data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::NON_COMMUNICATING); + return p0; +} + +data::ParticleStorage::iterator createSphere( data::ParticleStorage& ps, + const Vec3& pos, + const real_t& radius, + const uint64_t shapeID) +{ + auto p = ps.create(); + p->getPositionRef() = pos; + p->getInteractionRadiusRef() = radius; + p->getShapeIDRef() = shapeID; + p->getOwnerRef() = walberla::MPIManager::instance()->rank(); + p->getTypeRef() = 0; + return p; +} + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/CreateParticles.h b/apps/benchmarks/GranularGas/CreateParticles.h new file mode 100644 index 0000000000000000000000000000000000000000..230bc6ca415b462e719759e0ae0c1d765b8a2762 --- /dev/null +++ b/apps/benchmarks/GranularGas/CreateParticles.h @@ -0,0 +1,41 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file CreateParticles.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/DataTypes.h> +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/data/ShapeStorage.h> + +#include <core/mpi/MPIManager.h> + +namespace walberla { +namespace mesa_pd { + +data::ParticleStorage::iterator createPlane( data::ParticleStorage& ps, + data::ShapeStorage& ss, + const Vec3& pos, + const Vec3& normal ); + +data::ParticleStorage::iterator createSphere( data::ParticleStorage& ps, + const Vec3& pos, + const real_t& radius, + const uint64_t shapeID); + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/GranularGas.cfg b/apps/benchmarks/GranularGas/GranularGas.cfg index a70c6572cb1e7873d8d2f03f0923fd1a3c06f3e4..2c321312ff72d4174d09d9c0d8a5f058f02d39a7 100644 --- a/apps/benchmarks/GranularGas/GranularGas.cfg +++ b/apps/benchmarks/GranularGas/GranularGas.cfg @@ -1,10 +1,11 @@ GranularGas { simulationCorner < 0, 0, 0 >; - simulationDomain < 6, 6, 6 >; + simulationDomain < 40, 40, 40 >; blocks < 2,2,2 >; isPeriodic < 1, 1, 1 >; initialRefinementLevel 0; + sorting none; radius 0.6; spacing 1.0; diff --git a/apps/benchmarks/GranularGas/MESA_PD_GranularGas.cpp b/apps/benchmarks/GranularGas/MESA_PD_GranularGas.cpp index 51e8a620ea0887a1fecfbd8d92133f41f8b95dd2..94c71049600bd09ce58509a31da367fb7aa3ebf0 100644 --- a/apps/benchmarks/GranularGas/MESA_PD_GranularGas.cpp +++ b/apps/benchmarks/GranularGas/MESA_PD_GranularGas.cpp @@ -18,6 +18,16 @@ // //====================================================================================================================== +#include "Accessor.h" +#include "check.h" +#include "Contact.h" +#include "CreateParticles.h" +#include "NodeTimings.h" +#include "Parameters.h" +#include "SelectProperty.h" +#include "sortParticleStorage.h" +#include "SQLProperties.h" + #include <mesa_pd/vtk/ParticleVtkOutput.h> #include <mesa_pd/collision_detection/AnalyticContactDetection.h> @@ -59,54 +69,6 @@ namespace walberla { namespace mesa_pd { -class SelectRank -{ -public: - using return_type = int; - int operator()(const data::Particle& /*p*/) const { return rank_; } - int operator()(const data::Particle&& /*p*/) const { return rank_; } -private: - int rank_ = walberla::mpi::MPIManager::instance()->rank(); -}; - - -class ParticleAccessorWithShape : public data::ParticleAccessor -{ -public: - ParticleAccessorWithShape(std::shared_ptr<data::ParticleStorage>& ps, std::shared_ptr<data::ShapeStorage>& ss) - : ParticleAccessor(ps) - , ss_(ss) - {} - - const auto& getInvMass(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)]->getInvMass();} - - const auto& getInvInertiaBF(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)]->getInvInertiaBF();} - - data::BaseShape* getShape(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)].get();} -private: - std::shared_ptr<data::ShapeStorage> ss_; -}; - -void createPlane( data::ParticleStorage& ps, - data::ShapeStorage& ss, - const Vec3& pos, - const Vec3& normal ) -{ - auto p0 = ps.create(true); - p0->getPositionRef() = pos; - p0->getShapeIDRef() = ss.create<data::HalfSpace>( normal ); - p0->getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); - p0->getTypeRef() = 0; - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::INFINITE); - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::FIXED); - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::NON_COMMUNICATING); -} - -std::string envToString(const char* env) -{ - return env != nullptr ? std::string(env) : ""; -} - int main( int argc, char ** argv ) { using namespace walberla::timing; @@ -128,40 +90,8 @@ int main( int argc, char ** argv ) if (cfg == nullptr) WALBERLA_ABORT("No config specified!"); const Config::BlockHandle mainConf = cfg->getBlock( "GranularGas" ); - const std::string host = mainConf.getParameter<std::string>("host", "none" ); - WALBERLA_LOG_INFO_ON_ROOT("host: " << host); - - const int jobid = mainConf.getParameter<int>("jobid", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("jobid: " << jobid); - - const real_t spacing = mainConf.getParameter<real_t>("spacing", real_t(1.0) ); - WALBERLA_LOG_INFO_ON_ROOT("spacing: " << spacing); - - const real_t radius = mainConf.getParameter<real_t>("radius", real_t(0.5) ); - WALBERLA_LOG_INFO_ON_ROOT("radius: " << radius); - - bool bBarrier = mainConf.getParameter<bool>("bBarrier", false ); - WALBERLA_LOG_INFO_ON_ROOT("bBarrier: " << bBarrier); - - int64_t numOuterIterations = mainConf.getParameter<int64_t>("numOuterIterations", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("numOuterIterations: " << numOuterIterations); - - int64_t initialRefinementLevel = mainConf.getParameter<int64_t>("initialRefinementLevel", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("initialRefinementLevel: " << initialRefinementLevel); - - int64_t simulationSteps = mainConf.getParameter<int64_t>("simulationSteps", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("simulationSteps: " << simulationSteps); - - real_t dt = mainConf.getParameter<real_t>("dt", real_c(0.01) ); - WALBERLA_LOG_INFO_ON_ROOT("dt: " << dt); - - const int visSpacing = mainConf.getParameter<int>("visSpacing", 1000 ); - WALBERLA_LOG_INFO_ON_ROOT("visSpacing: " << visSpacing); - const std::string path = mainConf.getParameter<std::string>("path", "vtk_out" ); - WALBERLA_LOG_INFO_ON_ROOT("path: " << path); - - const std::string sqlFile = mainConf.getParameter<std::string>("sqlFile", "benchmark.sqlite" ); - WALBERLA_LOG_INFO_ON_ROOT("sqlFile: " << sqlFile); + Parameters params; + loadFromConfig(params, mainConf); WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***"); // create forest @@ -186,29 +116,25 @@ int main( int argc, char ** argv ) auto ps = std::make_shared<data::ParticleStorage>(100); auto ss = std::make_shared<data::ShapeStorage>(); ParticleAccessorWithShape accessor(ps, ss); - data::LinkedCells lc(localDomain.getExtended(spacing), spacing ); + data::LinkedCells lc(localDomain.getExtended(params.spacing), params.spacing ); - auto smallSphere = ss->create<data::Sphere>( radius ); + auto smallSphere = ss->create<data::Sphere>( params.radius ); ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707)); for (auto& iBlk : *forest) { - for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing)) + for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), + Vector3<real_t>(params.spacing) * real_c(0.5), + params.spacing)) { WALBERLA_CHECK(iBlk.getAABB().contains(pt)); - - auto p = ps->create(); - p->getPositionRef() = pt; - p->getInteractionRadiusRef() = radius; - p->getShapeIDRef() = smallSphere; - p->getOwnerRef() = mpiManager->rank(); - p->getTypeRef() = 0; + createSphere(*ps, pt, params.radius, smallSphere); } } int64_t numParticles = int64_c(ps->size()); walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM); WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles); - auto shift = (spacing - radius - radius) * real_t(0.5); + auto shift = (params.spacing - params.radius - params.radius) * real_t(0.5); auto confiningDomain = simulationDomain.getExtended(shift); if (!forest->isPeriodic(0)) @@ -241,7 +167,7 @@ int main( int argc, char ** argv ) WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - START ***"); // Init kernels - kernel::ExplicitEulerWithShape explicitEulerWithShape( dt ); + kernel::ExplicitEulerWithShape explicitEulerWithShape( params.dt ); kernel::InsertParticleIntoLinkedCells ipilc; kernel::SpringDashpot dem(1); dem.setStiffness(0, 0, real_t(0)); @@ -256,8 +182,10 @@ int main( int argc, char ** argv ) // initial sync SNN(*ps, domain); + sortParticleStorage(*ps, params.sorting, lc.domain_, uint_c(lc.numCellsPerDim_[0])); +// vtkWriter->write(); - for (int64_t outerIteration = 0; outerIteration < numOuterIterations; ++outerIteration) + for (int64_t outerIteration = 0; outerIteration < params.numOuterIterations; ++outerIteration) { WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING OUTER ITERATION " << outerIteration << " ***"); @@ -274,9 +202,9 @@ int main( int argc, char ** argv ) int64_t contactsChecked = 0; int64_t contactsDetected = 0; int64_t contactsTreated = 0; - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); timer.start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { // if (i % visSpacing == 0) // { @@ -286,7 +214,7 @@ int main( int argc, char ** argv ) tp["GenerateLinkedCells"].start(); lc.clear(); ps->forEachParticle(true, kernel::SelectAll(), accessor, ipilc, accessor, lc); - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); tp["GenerateLinkedCells"].end(); tp["DEM"].start(); @@ -310,23 +238,23 @@ int main( int argc, char ** argv ) } }, accessor ); - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); tp["DEM"].end(); tp["ReduceForce"].start(); RP.operator()<ForceTorqueNotification>(*ps); - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); tp["ReduceForce"].end(); tp["Euler"].start(); //ps->forEachParticle(false, [&](const size_t idx){WALBERLA_CHECK_EQUAL(ps->getForce(idx), Vec3(0,0,0), *(*ps)[idx] << "\n" << idx);}); ps->forEachParticle(true, kernel::SelectLocal(), accessor, explicitEulerWithShape, accessor); - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); tp["Euler"].end(); tp["SNN"].start(); SNN(*ps, domain); - if (bBarrier) WALBERLA_MPI_BARRIER(); + if (params.bBarrier) WALBERLA_MPI_BARRIER(); tp["SNN"].end(); } timer.end(); @@ -362,7 +290,7 @@ int main( int argc, char ** argv ) { WALBERLA_LOG_INFO_ON_ROOT(*timer_reduced); WALBERLA_LOG_INFO_ON_ROOT("runtime: " << timer_reduced->max()); - PUpS = double_c(numParticles) * double_c(simulationSteps) / double_c(timer_reduced->max()); + PUpS = double_c(numParticles) * double_c(params.simulationSteps) / double_c(timer_reduced->max()); WALBERLA_LOG_INFO_ON_ROOT("PUpS: " << PUpS); } @@ -370,19 +298,10 @@ int main( int argc, char ** argv ) WALBERLA_LOG_INFO_ON_ROOT(*tp_reduced); WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - END ***"); - WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - START ***"); - auto pIt = ps->begin(); - for (auto& iBlk : *forest) + if (params.checkSimulation) { - for (auto it = grid_generator::SCIterator(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing); - it != grid_generator::SCIterator(); - ++it, ++pIt) - { - WALBERLA_CHECK_UNEQUAL(pIt, ps->end()); - WALBERLA_CHECK_FLOAT_EQUAL((*pIt).getPositionRef(), *it); - } + check(*ps, *forest, params.spacing); } - WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - END ***"); WALBERLA_LOG_INFO_ON_ROOT("*** SQL OUTPUT - START ***"); numParticles = 0; @@ -416,6 +335,8 @@ int main( int argc, char ** argv ) walberla::mpi::reduceInplace(local_aabbs, walberla::mpi::SUM); walberla::mpi::reduceInplace(neighbor_subdomains, walberla::mpi::SUM); walberla::mpi::reduceInplace(neighbor_processes, walberla::mpi::SUM); + + uint_t runId = uint_c(-1); WALBERLA_ROOT_SECTION() { std::map< std::string, walberla::int64_t > integerProperties; @@ -424,26 +345,14 @@ int main( int argc, char ** argv ) stringProperties["walberla_git"] = WALBERLA_GIT_SHA1; stringProperties["tag"] = "mesa_pd"; - stringProperties["host"] = host; - integerProperties["jobid"] = jobid; integerProperties["mpi_num_processes"] = mpiManager->numProcesses(); integerProperties["omp_max_threads"] = omp_get_max_threads(); - integerProperties["numOuterIterations"] = numOuterIterations; - integerProperties["simulationSteps"] = simulationSteps; - integerProperties["bBarrier"] = int64_c(bBarrier); - realProperties["PUpS"] = double_c(PUpS); + integerProperties["outerIteration"] = int64_c(outerIteration); integerProperties["num_particles"] = numParticles; integerProperties["num_ghost_particles"] = numGhostParticles; integerProperties["contacts_checked"] = contactsChecked; integerProperties["contacts_detected"] = contactsDetected; integerProperties["contacts_treated"] = contactsTreated; - integerProperties["blocks_x"] = int64_c(forest->getXSize()); - integerProperties["blocks_y"] = int64_c(forest->getYSize()); - integerProperties["blocks_z"] = int64_c(forest->getZSize()); - integerProperties["initialRefinementLevel"] = int64_c(initialRefinementLevel); - realProperties["domain_x"] = double_c(forest->getDomain().xSize()); - realProperties["domain_y"] = double_c(forest->getDomain().ySize()); - realProperties["domain_z"] = double_c(forest->getDomain().zSize()); integerProperties["local_aabbs"] = int64_c(local_aabbs); integerProperties["neighbor_subdomains"] = int64_c(neighbor_subdomains); integerProperties["neighbor_processes"] = int64_c(neighbor_processes); @@ -457,26 +366,18 @@ int main( int argc, char ** argv ) integerProperties["RPReceives"] = RPReceives; realProperties["linkedCellsVolume"] = linkedCellsVolume; integerProperties["numLinkedCells"] = int64_c(numLinkedCells); - realProperties["timer_min"] = timer_reduced->min(); - realProperties["timer_max"] = timer_reduced->max(); - realProperties["timer_average"] = timer_reduced->average(); - realProperties["timer_total"] = timer_reduced->total(); - stringProperties["SLURM_CLUSTER_NAME"] = envToString(std::getenv( "SLURM_CLUSTER_NAME" )); - stringProperties["SLURM_CPUS_ON_NODE"] = envToString(std::getenv( "SLURM_CPUS_ON_NODE" )); - stringProperties["SLURM_CPUS_PER_TASK"] = envToString(std::getenv( "SLURM_CPUS_PER_TASK" )); - stringProperties["SLURM_JOB_ACCOUNT"] = envToString(std::getenv( "SLURM_JOB_ACCOUNT" )); - stringProperties["SLURM_JOB_ID"] = envToString(std::getenv( "SLURM_JOB_ID" )); - stringProperties["SLURM_JOB_CPUS_PER_NODE"] = envToString(std::getenv( "SLURM_JOB_CPUS_PER_NODE" )); - stringProperties["SLURM_JOB_NAME"] = envToString(std::getenv( "SLURM_JOB_NAME" )); - stringProperties["SLURM_JOB_NUM_NODES"] = envToString(std::getenv( "SLURM_JOB_NUM_NODES" )); - stringProperties["SLURM_NTASKS"] = envToString(std::getenv( "SLURM_NTASKS" )); - stringProperties["SLURM_NTASKS_PER_CORE"] = envToString(std::getenv( "SLURM_NTASKS_PER_CORE" )); - stringProperties["SLURM_NTASKS_PER_NODE"] = envToString(std::getenv( "SLURM_NTASKS_PER_NODE" )); - stringProperties["SLURM_NTASKS_PER_SOCKET"] = envToString(std::getenv( "SLURM_NTASKS_PER_SOCKET" )); - - auto runId = postprocessing::storeRunInSqliteDB( sqlFile, integerProperties, stringProperties, realProperties ); - postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *tp_reduced, "Timeloop" ); + saveToSQL(params, integerProperties, realProperties, stringProperties ); + addDomainPropertiesToSQL(*forest, integerProperties, realProperties, stringProperties); + addSlurmPropertiesToSQL(integerProperties, realProperties, stringProperties); + + runId = postprocessing::storeRunInSqliteDB( params.sqlFile, integerProperties, stringProperties, realProperties ); + postprocessing::storeTimingPoolInSqliteDB( params.sqlFile, runId, *tp_reduced, "Timeloop" ); + } + + if (params.storeNodeTimings) + { + storeNodeTimings(runId, params.sqlFile, "NodeTiming", tp); } WALBERLA_LOG_INFO_ON_ROOT("*** SQL OUTPUT - END ***"); } diff --git a/apps/benchmarks/GranularGas/MESA_PD_KernelBenchmark.cpp b/apps/benchmarks/GranularGas/MESA_PD_KernelBenchmark.cpp index bf2b95065ccd7675ac34b9d85f438344ed2f380b..320689a70a15fab555f666e451caee08150034a3 100644 --- a/apps/benchmarks/GranularGas/MESA_PD_KernelBenchmark.cpp +++ b/apps/benchmarks/GranularGas/MESA_PD_KernelBenchmark.cpp @@ -18,6 +18,16 @@ // //====================================================================================================================== +#include "Accessor.h" +#include "check.h" +#include "Contact.h" +#include "CreateParticles.h" +#include "NodeTimings.h" +#include "Parameters.h" +#include "SelectProperty.h" +#include "sortParticleStorage.h" +#include "SQLProperties.h" + #include <mesa_pd/vtk/ParticleVtkOutput.h> #include <mesa_pd/collision_detection/AnalyticContactDetection.h> @@ -34,8 +44,9 @@ #include <mesa_pd/mpi/ContactFilter.h> #include <mesa_pd/mpi/ReduceProperty.h> #include <mesa_pd/mpi/SyncNextNeighbors.h> - #include <mesa_pd/mpi/notifications/ForceTorqueNotification.h> +#include <mesa_pd/sorting/HilbertCompareFunctor.h> +#include <mesa_pd/sorting/LinearizedCompareFunctor.h> #include <blockforest/BlockForest.h> #include <blockforest/Initialization.h> @@ -64,125 +75,6 @@ namespace walberla { namespace mesa_pd { -class SelectRank -{ -public: - using return_type = int; - int operator()(const data::Particle& /*p*/) const { return rank_; } - int operator()(const data::Particle&& /*p*/) const { return rank_; } -private: - int rank_ = walberla::mpi::MPIManager::instance()->rank(); -}; - -struct Contact -{ - Contact(const size_t idx1, - const size_t idx2, - const Vec3 contactPoint, - const Vec3 contactNormal, - const real_t penetrationDepth) - : idx1_(idx1) - , idx2_(idx2) - , contactPoint_(contactPoint) - , contactNormal_(contactNormal) - , penetrationDepth_(penetrationDepth) {} - - size_t idx1_; - size_t idx2_; - Vec3 contactPoint_; - Vec3 contactNormal_; - real_t penetrationDepth_; -}; - - -class ParticleAccessorWithShape : public data::ParticleAccessor -{ -public: - ParticleAccessorWithShape(std::shared_ptr<data::ParticleStorage>& ps, std::shared_ptr<data::ShapeStorage>& ss) - : ParticleAccessor(ps) - , ss_(ss) - {} - - const auto& getInvMass(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)]->getInvMass();} - - const auto& getInvInertiaBF(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)]->getInvInertiaBF();} - - data::BaseShape* getShape(const size_t p_idx) const {return ss_->shapes[ps_->getShapeID(p_idx)].get();} -private: - std::shared_ptr<data::ShapeStorage> ss_; -}; - -void createPlane( data::ParticleStorage& ps, - data::ShapeStorage& ss, - const Vec3& pos, - const Vec3& normal ) -{ - auto p0 = ps.create(true); - p0->getPositionRef() = pos; - p0->getShapeIDRef() = ss.create<data::HalfSpace>( normal ); - p0->getOwnerRef() = walberla::mpi::MPIManager::instance()->rank(); - p0->getTypeRef() = 0; - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::INFINITE); - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::FIXED); - data::particle_flags::set(p0->getFlagsRef(), data::particle_flags::NON_COMMUNICATING); -} - -std::string envToString(const char* env) -{ - return env != nullptr ? std::string(env) : ""; -} - -void storeNodeTimings( const uint_t runId, - const std::string & dbFile, - const std::string & tableName, - const WcTimingPool & tp ) -{ - std::map< std::string, walberla::int64_t > integerProperties; - std::map< std::string, double > realProperties; - std::map< std::string, std::string > stringProperties; - - walberla::mpi::SendBuffer sb; - walberla::mpi::RecvBuffer rb; - - sb << walberla::getHostName(); - sb << int64_t(walberla::mpi::MPIManager::instance()->rank()); - sb << tp; - - walberla::mpi::gathervBuffer(sb, rb); - - WALBERLA_ROOT_SECTION() - { - while (!rb.isEmpty()) - { - integerProperties.clear(); - realProperties.clear(); - stringProperties.clear(); - - std::string hostname; - int64_t rank; - WcTimingPool cTP; - rb >> hostname; - rb >> rank; - rb >> cTP; - - stringProperties["hostname"] = hostname; - integerProperties["rank"] = rank; - for (auto& v : cTP) - { - realProperties[v.first] = v.second.average(); - } - - postprocessing::storeAdditionalRunInfoInSqliteDB( runId, - dbFile, - tableName, - integerProperties, - stringProperties, - realProperties ); - } - } -} - - int main( int argc, char ** argv ) { using namespace walberla::timing; @@ -203,41 +95,8 @@ int main( int argc, char ** argv ) auto cfg = env.config(); if (cfg == nullptr) WALBERLA_ABORT("No config specified!"); const Config::BlockHandle mainConf = cfg->getBlock( "GranularGas" ); - - const std::string host = mainConf.getParameter<std::string>("host", "none" ); - WALBERLA_LOG_INFO_ON_ROOT("host: " << host); - - const int jobid = mainConf.getParameter<int>("jobid", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("jobid: " << jobid); - - const real_t spacing = mainConf.getParameter<real_t>("spacing", real_t(1.0) ); - WALBERLA_LOG_INFO_ON_ROOT("spacing: " << spacing); - - const real_t radius = mainConf.getParameter<real_t>("radius", real_t(0.5) ); - WALBERLA_LOG_INFO_ON_ROOT("radius: " << radius); - - bool bBarrier = mainConf.getParameter<bool>("bBarrier", false ); - WALBERLA_LOG_INFO_ON_ROOT("bBarrier: " << bBarrier); - - int64_t numOuterIterations = mainConf.getParameter<int64_t>("numOuterIterations", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("numOuterIterations: " << numOuterIterations); - - int64_t initialRefinementLevel = mainConf.getParameter<int64_t>("initialRefinementLevel", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("initialRefinementLevel: " << initialRefinementLevel); - - int64_t simulationSteps = mainConf.getParameter<int64_t>("simulationSteps", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("simulationSteps: " << simulationSteps); - - real_t dt = mainConf.getParameter<real_t>("dt", real_c(0.01) ); - WALBERLA_LOG_INFO_ON_ROOT("dt: " << dt); - - const int visSpacing = mainConf.getParameter<int>("visSpacing", 1000 ); - WALBERLA_LOG_INFO_ON_ROOT("visSpacing: " << visSpacing); - const std::string path = mainConf.getParameter<std::string>("path", "vtk_out" ); - WALBERLA_LOG_INFO_ON_ROOT("path: " << path); - - const std::string sqlFile = mainConf.getParameter<std::string>("sqlFile", "benchmark.sqlite" ); - WALBERLA_LOG_INFO_ON_ROOT("sqlFile: " << sqlFile); + Parameters params; + loadFromConfig(params, mainConf); WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***"); // create forest @@ -262,29 +121,25 @@ int main( int argc, char ** argv ) auto ps = std::make_shared<data::ParticleStorage>(100); auto ss = std::make_shared<data::ShapeStorage>(); ParticleAccessorWithShape accessor(ps, ss); - data::LinkedCells lc(localDomain.getExtended(spacing), spacing ); + data::LinkedCells lc(localDomain.getExtended(params.spacing), params.spacing ); - auto smallSphere = ss->create<data::Sphere>( radius ); + auto smallSphere = ss->create<data::Sphere>( params.radius ); ss->shapes[smallSphere]->updateMassAndInertia(real_t(2707)); for (auto& iBlk : *forest) { - for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing)) + for (auto pt : grid_generator::SCGrid(iBlk.getAABB(), + Vector3<real_t>(params.spacing) * real_c(0.5), + params.spacing)) { WALBERLA_CHECK(iBlk.getAABB().contains(pt)); - - auto p = ps->create(); - p->getPositionRef() = pt; - p->getInteractionRadiusRef() = radius; - p->getShapeIDRef() = smallSphere; - p->getOwnerRef() = mpiManager->rank(); - p->getTypeRef() = 0; + createSphere(*ps, pt, params.radius, smallSphere); } } int64_t numParticles = int64_c(ps->size()); walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM); WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles); - auto shift = (spacing - radius - radius) * real_t(0.5); + auto shift = (params.spacing - params.radius - params.radius) * real_t(0.5); auto confiningDomain = simulationDomain.getExtended(shift); if (!forest->isPeriodic(0)) @@ -313,11 +168,12 @@ int main( int argc, char ** argv ) auto vtkWriter = walberla::vtk::createVTKOutput_PointData(vtkOutput, "Bodies", 1, "vtk", "simulation_step", false, false); vtkOutput->addOutput<SelectRank>("rank"); vtkOutput->addOutput<data::SelectParticleOwner>("owner"); + vtkOutput->addOutput<SelectIdx>("idx"); // vtkDomainOutput->write(); WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - START ***"); // Init kernels - kernel::ExplicitEulerWithShape explicitEulerWithShape( dt ); + kernel::ExplicitEulerWithShape explicitEulerWithShape( params.dt ); kernel::InsertParticleIntoLinkedCells ipilc; kernel::SpringDashpot dem(1); dem.setStiffness(0, 0, real_t(0)); @@ -334,8 +190,10 @@ int main( int argc, char ** argv ) // initial sync SNN(*ps, domain); + sortParticleStorage(*ps, params.sorting, lc.domain_, uint_c(lc.numCellsPerDim_[0])); +// vtkWriter->write(); - for (int64_t outerIteration = 0; outerIteration < numOuterIterations; ++outerIteration) + for (int64_t outerIteration = 0; outerIteration < params.numOuterIterations; ++outerIteration) { WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING OUTER ITERATION " << outerIteration << " ***"); @@ -343,7 +201,7 @@ int main( int argc, char ** argv ) WALBERLA_MPI_BARRIER(); tp["GenerateLinkedCells"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { lc.clear(); ps->forEachParticle(true, kernel::SelectAll(), accessor, ipilc, accessor, lc); @@ -355,7 +213,7 @@ int main( int argc, char ** argv ) int64_t contactsTreated = 0; WALBERLA_MPI_BARRIER(); tp["ContactDetection"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { contacts.clear(); contactsChecked = 0; @@ -383,7 +241,7 @@ int main( int argc, char ** argv ) WALBERLA_MPI_BARRIER(); tp["DEM"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { for (auto& c : contacts) { @@ -394,7 +252,7 @@ int main( int argc, char ** argv ) WALBERLA_MPI_BARRIER(); tp["ReduceForce"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { RP.operator()<ForceTorqueNotification>(*ps); } @@ -402,7 +260,7 @@ int main( int argc, char ** argv ) WALBERLA_MPI_BARRIER(); tp["Euler"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { ps->forEachParticle(true, kernel::SelectLocal(), accessor, explicitEulerWithShape, accessor); } @@ -410,7 +268,7 @@ int main( int argc, char ** argv ) WALBERLA_MPI_BARRIER(); tp["SNN"].start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { SNN(*ps, domain); } @@ -418,19 +276,10 @@ int main( int argc, char ** argv ) WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - END ***"); - WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - START ***"); - auto pIt = ps->begin(); - for (auto& iBlk : *forest) + if (params.checkSimulation) { - for (auto it = grid_generator::SCIterator(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing); - it != grid_generator::SCIterator(); - ++it, ++pIt) - { - WALBERLA_CHECK_UNEQUAL(pIt, ps->end()); - WALBERLA_CHECK_FLOAT_EQUAL((*pIt).getPositionRef(), *it); - } + check(*ps, *forest, params.spacing); } - WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - END ***"); WALBERLA_LOG_INFO_ON_ROOT("*** SQL OUTPUT - START ***"); auto SNNBytesSent = SNN.getBytesSent(); @@ -502,26 +351,14 @@ int main( int argc, char ** argv ) stringProperties["walberla_git"] = WALBERLA_GIT_SHA1; stringProperties["tag"] = "mesa_pd"; - stringProperties["host"] = host; - integerProperties["jobid"] = jobid; integerProperties["mpi_num_processes"] = mpiManager->numProcesses(); integerProperties["omp_max_threads"] = omp_get_max_threads(); integerProperties["outerIteration"] = int64_c(outerIteration); - integerProperties["numOuterIterations"] = numOuterIterations; - integerProperties["simulationSteps"] = simulationSteps; - integerProperties["bBarrier"] = int64_c(bBarrier); integerProperties["num_particles"] = numParticles; integerProperties["num_ghost_particles"] = numGhostParticles; integerProperties["contacts_checked"] = contactsChecked; integerProperties["contacts_detected"] = contactsDetected; integerProperties["contacts_treated"] = contactsTreated; - integerProperties["blocks_x"] = int64_c(forest->getXSize()); - integerProperties["blocks_y"] = int64_c(forest->getXSize()); - integerProperties["blocks_z"] = int64_c(forest->getXSize()); - integerProperties["initialRefinementLevel"] = int64_c(initialRefinementLevel); - realProperties["domain_x"] = double_c(forest->getDomain().xSize()); - realProperties["domain_y"] = double_c(forest->getDomain().ySize()); - realProperties["domain_z"] = double_c(forest->getDomain().zSize()); integerProperties["local_aabbs"] = int64_c(local_aabbs); integerProperties["neighbor_subdomains"] = int64_c(neighbor_subdomains); integerProperties["neighbor_processes"] = int64_c(neighbor_processes); @@ -535,25 +372,18 @@ int main( int argc, char ** argv ) integerProperties["RPReceives"] = RPReceives; realProperties["linkedCellsVolume"] = linkedCellsVolume; integerProperties["numLinkedCells"] = int64_c(numLinkedCells); - stringProperties["SLURM_CLUSTER_NAME"] = envToString(std::getenv( "SLURM_CLUSTER_NAME" )); - stringProperties["SLURM_CPUS_ON_NODE"] = envToString(std::getenv( "SLURM_CPUS_ON_NODE" )); - stringProperties["SLURM_CPUS_PER_TASK"] = envToString(std::getenv( "SLURM_CPUS_PER_TASK" )); - stringProperties["SLURM_JOB_ACCOUNT"] = envToString(std::getenv( "SLURM_JOB_ACCOUNT" )); - stringProperties["SLURM_JOB_ID"] = envToString(std::getenv( "SLURM_JOB_ID" )); - stringProperties["SLURM_JOB_CPUS_PER_NODE"] = envToString(std::getenv( "SLURM_JOB_CPUS_PER_NODE" )); - stringProperties["SLURM_JOB_NAME"] = envToString(std::getenv( "SLURM_JOB_NAME" )); - stringProperties["SLURM_JOB_NUM_NODES"] = envToString(std::getenv( "SLURM_JOB_NUM_NODES" )); - stringProperties["SLURM_NTASKS"] = envToString(std::getenv( "SLURM_NTASKS" )); - stringProperties["SLURM_NTASKS_PER_CORE"] = envToString(std::getenv( "SLURM_NTASKS_PER_CORE" )); - stringProperties["SLURM_NTASKS_PER_NODE"] = envToString(std::getenv( "SLURM_NTASKS_PER_NODE" )); - stringProperties["SLURM_NTASKS_PER_SOCKET"] = envToString(std::getenv( "SLURM_NTASKS_PER_SOCKET" )); - stringProperties["SLURM_TASKS_PER_NODE"] = envToString(std::getenv( "SLURM_TASKS_PER_NODE" )); - - - runId = postprocessing::storeRunInSqliteDB( sqlFile, integerProperties, stringProperties, realProperties ); - postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *tp_reduced, "Timeloop" ); + + saveToSQL(params, integerProperties, realProperties, stringProperties ); + addDomainPropertiesToSQL(*forest, integerProperties, realProperties, stringProperties); + addSlurmPropertiesToSQL(integerProperties, realProperties, stringProperties); + + runId = postprocessing::storeRunInSqliteDB( params.sqlFile, integerProperties, stringProperties, realProperties ); + postprocessing::storeTimingPoolInSqliteDB( params.sqlFile, runId, *tp_reduced, "Timeloop" ); + } + if (params.storeNodeTimings) + { + storeNodeTimings(runId, params.sqlFile, "NodeTiming", tp); } - storeNodeTimings(runId, sqlFile, "NodeTiming", tp); WALBERLA_LOG_INFO_ON_ROOT("*** SQL OUTPUT - END ***"); } diff --git a/apps/benchmarks/GranularGas/NodeTimings.cpp b/apps/benchmarks/GranularGas/NodeTimings.cpp new file mode 100644 index 0000000000000000000000000000000000000000..01b1a354b97acd3226f805ff8bd39b938a04cdd7 --- /dev/null +++ b/apps/benchmarks/GranularGas/NodeTimings.cpp @@ -0,0 +1,77 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file NodeTimings.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include "NodeTimings.h" + +namespace walberla { +namespace mesa_pd { + +void storeNodeTimings( const uint_t runId, + const std::string & dbFile, + const std::string & tableName, + const WcTimingPool & tp ) +{ + std::map< std::string, walberla::int64_t > integerProperties; + std::map< std::string, double > realProperties; + std::map< std::string, std::string > stringProperties; + + walberla::mpi::SendBuffer sb; + walberla::mpi::RecvBuffer rb; + + sb << walberla::getHostName(); + sb << int64_t(walberla::mpi::MPIManager::instance()->rank()); + sb << tp; + + walberla::mpi::gathervBuffer(sb, rb); + + WALBERLA_ROOT_SECTION() + { + while (!rb.isEmpty()) + { + integerProperties.clear(); + realProperties.clear(); + stringProperties.clear(); + + std::string hostname; + int64_t rank; + WcTimingPool cTP; + rb >> hostname; + rb >> rank; + rb >> cTP; + + stringProperties["hostname"] = hostname; + integerProperties["rank"] = rank; + for (auto& v : cTP) + { + realProperties[v.first] = v.second.average(); + } + + postprocessing::storeAdditionalRunInfoInSqliteDB( runId, + dbFile, + tableName, + integerProperties, + stringProperties, + realProperties ); + } + } +} + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/NodeTimings.h b/apps/benchmarks/GranularGas/NodeTimings.h new file mode 100644 index 0000000000000000000000000000000000000000..cbe074b1b085b81822f652e805bc67c9c84f5f55 --- /dev/null +++ b/apps/benchmarks/GranularGas/NodeTimings.h @@ -0,0 +1,37 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file NodeTimings.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <core/Abort.h> +#include <core/Hostname.h> +#include <core/mpi/Gatherv.h> +#include <core/logging/Logging.h> +#include <core/timing/TimingPool.h> +#include <postprocessing/sqlite/SQLite.h> + +namespace walberla { +namespace mesa_pd { + +void storeNodeTimings( const uint_t runId, + const std::string & dbFile, + const std::string & tableName, + const WcTimingPool & tp ); + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/PE_GranularGas.cpp b/apps/benchmarks/GranularGas/PE_GranularGas.cpp index ff927ca6fa4238cd33d92dd37e24f4a5dc6ff05f..b2ed75d4a8220bac4a15cfc6614a993d66c96cd1 100644 --- a/apps/benchmarks/GranularGas/PE_GranularGas.cpp +++ b/apps/benchmarks/GranularGas/PE_GranularGas.cpp @@ -18,6 +18,10 @@ // //====================================================================================================================== +#include "NodeTimings.h" +#include "Parameters.h" +#include "SQLProperties.h" + #include <pe/basic.h> #include <pe/vtk/SphereVtkOutput.h> @@ -43,11 +47,6 @@ using namespace walberla::timing; using BodyTuple = std::tuple<Sphere, Plane> ; -std::string envToString(const char* env) -{ - return env != nullptr ? std::string(env) : ""; -} - int main( int argc, char ** argv ) { WcTimingTree tt; @@ -91,41 +90,8 @@ int main( int argc, char ** argv ) auto cfg = env.config(); if (cfg == nullptr) WALBERLA_ABORT("No config specified!"); const Config::BlockHandle mainConf = cfg->getBlock( "GranularGas" ); - - const std::string host = mainConf.getParameter<std::string>("host", "none" ); - WALBERLA_LOG_INFO_ON_ROOT("host: " << host); - - const int jobid = mainConf.getParameter<int>("jobid", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("jobid: " << jobid); - - const real_t spacing = mainConf.getParameter<real_t>("spacing", real_t(1.0) ); - WALBERLA_LOG_INFO_ON_ROOT("spacing: " << spacing); - - const real_t radius = mainConf.getParameter<real_t>("radius", real_t(0.5) ); - WALBERLA_LOG_INFO_ON_ROOT("radius: " << radius); - - bool bBarrier = false; - WALBERLA_LOG_INFO_ON_ROOT("bBarrier: " << bBarrier); - - int64_t numOuterIterations = mainConf.getParameter<int64_t>("numOuterIterations", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("numOuterIterations: " << numOuterIterations); - - int64_t initialRefinementLevel = mainConf.getParameter<int64_t>("initialRefinementLevel", 0 ); - WALBERLA_LOG_INFO_ON_ROOT("initialRefinementLevel: " << initialRefinementLevel); - - int64_t simulationSteps = mainConf.getParameter<int64_t>("simulationSteps", 10 ); - WALBERLA_LOG_INFO_ON_ROOT("simulationSteps: " << simulationSteps); - - real_t dt = mainConf.getParameter<real_t>("dt", real_c(0.01) ); - WALBERLA_LOG_INFO_ON_ROOT("dt: " << dt); - - const int visSpacing = mainConf.getParameter<int>("visSpacing", 1000 ); - WALBERLA_LOG_INFO_ON_ROOT("visSpacing: " << visSpacing); - const std::string path = mainConf.getParameter<std::string>("path", "vtk_out" ); - WALBERLA_LOG_INFO_ON_ROOT("path: " << path); - - const std::string sqlFile = mainConf.getParameter<std::string>("sqlFile", "benchmark.sqlite" ); - WALBERLA_LOG_INFO_ON_ROOT("sqlFile: " << sqlFile); + Parameters params; + loadFromConfig(params, mainConf); WALBERLA_LOG_INFO_ON_ROOT("*** GLOBALBODYSTORAGE ***"); shared_ptr<BodyStorage> globalBodyStorage = make_shared<BodyStorage>(); @@ -223,9 +189,13 @@ int main( int argc, char ** argv ) for (auto& currentBlock : *forest) { - for (auto it = grid_generator::SCIterator(currentBlock.getAABB().getIntersection(generationDomain), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing); it != grid_generator::SCIterator(); ++it) + for (auto it = grid_generator::SCIterator(currentBlock.getAABB().getIntersection(generationDomain), + Vector3<real_t>(params.spacing) * real_c(0.5), + params.spacing); + it != grid_generator::SCIterator(); + ++it) { - SphereID sp = pe::createSphere( *globalBodyStorage, *forest, storageID, 0, *it, radius, material); + SphereID sp = pe::createSphere( *globalBodyStorage, *forest, storageID, 0, *it, params.radius, material); if (sp != nullptr) ++numParticles; } } @@ -255,22 +225,22 @@ int main( int argc, char ** argv ) syncCallWithoutTT(); WALBERLA_LOG_INFO_ON_ROOT("*** SIMULATION - START ***"); - for (int64_t outerIteration = 0; outerIteration < numOuterIterations; ++outerIteration) + for (int64_t outerIteration = 0; outerIteration < params.numOuterIterations; ++outerIteration) { WALBERLA_LOG_INFO_ON_ROOT("*** RUNNING OUTER ITERATION " << outerIteration << " ***"); WcTimer timer; WcTimingPool tp; WALBERLA_MPI_BARRIER(); timer.start(); - for (int64_t i=0; i < simulationSteps; ++i) + for (int64_t i=0; i < params.simulationSteps; ++i) { if( i % 200 == 0 ) { - WALBERLA_LOG_DEVEL_ON_ROOT( "Timestep " << i << " / " << simulationSteps ); + WALBERLA_LOG_DEVEL_ON_ROOT( "Timestep " << i << " / " << params.simulationSteps ); } tp["CR"].start(); - cr->timestep( real_c(dt) ); + cr->timestep( real_c(params.dt) ); tp["CR"].end(); tp["Sync"].start(); syncCallWithoutTT(); @@ -289,7 +259,7 @@ int main( int argc, char ** argv ) { WALBERLA_LOG_INFO_ON_ROOT(*timer_reduced); WALBERLA_LOG_INFO_ON_ROOT("runtime: " << timer_reduced->max()); - PUpS = double_c(numParticles) * double_c(simulationSteps) / double_c(timer_reduced->max()); + PUpS = double_c(numParticles) * double_c(params.simulationSteps) / double_c(timer_reduced->max()); WALBERLA_LOG_INFO_ON_ROOT("PUpS: " << PUpS); } @@ -315,7 +285,9 @@ int main( int argc, char ** argv ) numGhostParticles += shadowStorage.size(); auto bodyIt = localStorage.begin(); - for (auto it = grid_generator::SCIterator(currentBlock.getAABB().getIntersection(generationDomain), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing); + for (auto it = grid_generator::SCIterator(currentBlock.getAABB().getIntersection(generationDomain), + Vector3<real_t>(params.spacing) * real_c(0.5), + params.spacing); it != grid_generator::SCIterator(); ++it, ++bodyIt) { @@ -327,6 +299,7 @@ int main( int argc, char ** argv ) mpi::reduceInplace(numGhostParticles, mpi::SUM); WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - END ***"); + uint_t runId = uint_c(-1); WALBERLA_ROOT_SECTION() { std::map< std::string, walberla::int64_t > integerProperties; @@ -335,15 +308,10 @@ int main( int argc, char ** argv ) stringProperties["walberla_git"] = WALBERLA_GIT_SHA1; stringProperties["tag"] = "pe"; - stringProperties["host"] = host; - integerProperties["jobid"] = jobid; integerProperties["bDEM"] = bDEM; integerProperties["bNN"] = bNN; integerProperties["mpi_num_processes"] = mpi::MPIManager::instance()->numProcesses(); integerProperties["omp_max_threads"] = omp_get_max_threads(); - integerProperties["numOuterIterations"] = numOuterIterations; - integerProperties["simulationSteps"] = simulationSteps; - integerProperties["bBarrier"] = int64_c(bBarrier); realProperties["PUpS"] = double_c(PUpS); realProperties["timer_min"] = timer_reduced->min(); realProperties["timer_max"] = timer_reduced->max(); @@ -351,28 +319,17 @@ int main( int argc, char ** argv ) realProperties["timer_total"] = timer_reduced->total(); integerProperties["num_particles"] = numParticles; integerProperties["num_ghost_particles"] = numGhostParticles; - integerProperties["blocks_x"] = int64_c(forest->getXSize()); - integerProperties["blocks_y"] = int64_c(forest->getYSize()); - integerProperties["blocks_z"] = int64_c(forest->getZSize()); - integerProperties["initialRefinementLevel"] = int64_c(initialRefinementLevel); - realProperties["domain_x"] = double_c(forest->getDomain().xSize()); - realProperties["domain_y"] = double_c(forest->getDomain().ySize()); - realProperties["domain_z"] = double_c(forest->getDomain().zSize()); - stringProperties["SLURM_CLUSTER_NAME"] = envToString(std::getenv( "SLURM_CLUSTER_NAME" )); - stringProperties["SLURM_CPUS_ON_NODE"] = envToString(std::getenv( "SLURM_CPUS_ON_NODE" )); - stringProperties["SLURM_CPUS_PER_TASK"] = envToString(std::getenv( "SLURM_CPUS_PER_TASK" )); - stringProperties["SLURM_JOB_ACCOUNT"] = envToString(std::getenv( "SLURM_JOB_ACCOUNT" )); - stringProperties["SLURM_JOB_ID"] = envToString(std::getenv( "SLURM_JOB_ID" )); - stringProperties["SLURM_JOB_CPUS_PER_NODE"] = envToString(std::getenv( "SLURM_JOB_CPUS_PER_NODE" )); - stringProperties["SLURM_JOB_NAME"] = envToString(std::getenv( "SLURM_JOB_NAME" )); - stringProperties["SLURM_JOB_NUM_NODES"] = envToString(std::getenv( "SLURM_JOB_NUM_NODES" )); - stringProperties["SLURM_NTASKS"] = envToString(std::getenv( "SLURM_NTASKS" )); - stringProperties["SLURM_NTASKS_PER_CORE"] = envToString(std::getenv( "SLURM_NTASKS_PER_CORE" )); - stringProperties["SLURM_NTASKS_PER_NODE"] = envToString(std::getenv( "SLURM_NTASKS_PER_NODE" )); - stringProperties["SLURM_NTASKS_PER_SOCKET"] = envToString(std::getenv( "SLURM_NTASKS_PER_SOCKET" )); - - auto runId = postprocessing::storeRunInSqliteDB( sqlFile, integerProperties, stringProperties, realProperties ); - postprocessing::storeTimingPoolInSqliteDB( sqlFile, runId, *tp_reduced, "Timeloop" ); + + saveToSQL(params, integerProperties, realProperties, stringProperties ); + mesa_pd::addDomainPropertiesToSQL(*forest, integerProperties, realProperties, stringProperties); + mesa_pd::addSlurmPropertiesToSQL(integerProperties, realProperties, stringProperties); + + runId = postprocessing::storeRunInSqliteDB( params.sqlFile, integerProperties, stringProperties, realProperties ); + postprocessing::storeTimingPoolInSqliteDB( params.sqlFile, runId, *tp_reduced, "Timeloop" ); + } + if (params.storeNodeTimings) + { + mesa_pd::storeNodeTimings(runId, params.sqlFile, "NodeTiming", tp); } WALBERLA_LOG_INFO_ON_ROOT("*** SQL OUTPUT - END ***"); } diff --git a/apps/benchmarks/GranularGas/Parameters.cpp b/apps/benchmarks/GranularGas/Parameters.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2080935eab627936e9888afd8e2ac948fcfaf403 --- /dev/null +++ b/apps/benchmarks/GranularGas/Parameters.cpp @@ -0,0 +1,106 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Parameters.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#include "Parameters.h" + +#include <core/logging/Logging.h> + +namespace walberla { + +void loadFromConfig(Parameters& params, const Config::BlockHandle& cfg) +{ + params.sorting = cfg.getParameter<std::string>("sorting", "none" ); + WALBERLA_LOG_INFO_ON_ROOT("sorting: " << params.sorting); + + params.spacing = cfg.getParameter<real_t>("spacing", real_t(1.0) ); + WALBERLA_LOG_INFO_ON_ROOT("spacing: " << params.spacing); + + params.radius = cfg.getParameter<real_t>("radius", real_t(0.5) ); + WALBERLA_LOG_INFO_ON_ROOT("radius: " << params.radius); + + params.bBarrier = cfg.getParameter<bool>("bBarrier", false ); + WALBERLA_LOG_INFO_ON_ROOT("bBarrier: " << params.bBarrier); + + params.storeNodeTimings = cfg.getParameter<bool>("storeNodeTimings", false ); + WALBERLA_LOG_INFO_ON_ROOT("storeNodeTimings: " << params.storeNodeTimings); + + params.checkSimulation = cfg.getParameter<bool>("checkSimulation", false ); + WALBERLA_LOG_INFO_ON_ROOT("checkSimulation: " << params.checkSimulation); + + params.numOuterIterations = cfg.getParameter<int64_t>("numOuterIterations", 10 ); + WALBERLA_LOG_INFO_ON_ROOT("numOuterIterations: " << params.numOuterIterations); + + params.initialRefinementLevel = cfg.getParameter<int64_t>("initialRefinementLevel", 0 ); + WALBERLA_LOG_INFO_ON_ROOT("initialRefinementLevel: " << params.initialRefinementLevel); + + params.simulationSteps = cfg.getParameter<int64_t>("simulationSteps", 10 ); + WALBERLA_LOG_INFO_ON_ROOT("simulationSteps: " << params.simulationSteps); + + params.dt = cfg.getParameter<real_t>("dt", real_t(0.01) ); + WALBERLA_LOG_INFO_ON_ROOT("dt: " << params.dt); + + params.visSpacing = cfg.getParameter<int64_t>("visSpacing", 1000 ); + WALBERLA_LOG_INFO_ON_ROOT("visSpacing: " << params.visSpacing); + + params.path = cfg.getParameter<std::string>("path", "vtk_out" ); + WALBERLA_LOG_INFO_ON_ROOT("path: " << params.path); + + params.sqlFile = cfg.getParameter<std::string>("sqlFile", "benchmark.sqlite" ); + WALBERLA_LOG_INFO_ON_ROOT("sqlFile: " << params.sqlFile); + +} + +void saveToSQL(const Parameters& params, + std::map< std::string, walberla::int64_t >& integerProperties, + std::map< std::string, double >& realProperties, + std::map< std::string, std::string >& stringProperties ) +{ + stringProperties["sorting"] = params.sorting; + + realProperties["spacing"] = double_c(params.spacing); + + realProperties["radius"] = double_c(params.radius); + + + + + integerProperties["numOuterIterations"] = params.numOuterIterations; + + integerProperties["initialRefinementLevel"] = params.initialRefinementLevel; + + integerProperties["simulationSteps"] = params.simulationSteps; + + realProperties["dt"] = double_c(params.dt); + + integerProperties["visSpacing"] = params.visSpacing; + + stringProperties["path"] = params.path; + + stringProperties["sqlFile"] = params.sqlFile; + +} + +} //namespace walberla \ No newline at end of file diff --git a/apps/benchmarks/GranularGas/Parameters.h b/apps/benchmarks/GranularGas/Parameters.h new file mode 100644 index 0000000000000000000000000000000000000000..caae193de89e1a063e13dd16e3f8534d71fa3e12 --- /dev/null +++ b/apps/benchmarks/GranularGas/Parameters.h @@ -0,0 +1,61 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Parameters.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <core/config/Config.h> +#include <core/DataTypes.h> + +#include <string> + +namespace walberla { + +struct Parameters +{ + std::string sorting = "none"; + real_t spacing = real_t(1.0); + real_t radius = real_t(0.5); + bool bBarrier = false; + bool storeNodeTimings = false; + bool checkSimulation = false; + int64_t numOuterIterations = 10; + int64_t initialRefinementLevel = 0; + int64_t simulationSteps = 10; + real_t dt = real_t(0.01); + int64_t visSpacing = 1000; + std::string path = "vtk_out"; + std::string sqlFile = "benchmark.sqlite"; +}; + +void loadFromConfig(Parameters& params, + const Config::BlockHandle& cfg); + +void saveToSQL(const Parameters& params, + std::map< std::string, walberla::int64_t >& integerProperties, + std::map< std::string, double >& realProperties, + std::map< std::string, std::string >& stringProperties ); + +} //namespace walberla \ No newline at end of file diff --git a/apps/benchmarks/GranularGas/Parameters.templ.cpp b/apps/benchmarks/GranularGas/Parameters.templ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8f9be98116d57a52f6d9b9ea200e6ee0882d36f6 --- /dev/null +++ b/apps/benchmarks/GranularGas/Parameters.templ.cpp @@ -0,0 +1,59 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Parameters.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#include "Parameters.h" + +#include <core/logging/Logging.h> + +namespace walberla { + +void loadFromConfig(Parameters& params, const Config::BlockHandle& cfg) +{ + {%- for param in parameters %} + params.{{param.name}} = cfg.getParameter<{{param.type}}>("{{param.name}}", {{param.defValue}} ); + WALBERLA_LOG_INFO_ON_ROOT("{{param.name}}: " << params.{{param.name}}); + {% endfor %} +} + +void saveToSQL(const Parameters& params, + std::map< std::string, walberla::int64_t >& integerProperties, + std::map< std::string, double >& realProperties, + std::map< std::string, std::string >& stringProperties ) +{ + {%- for param in parameters %} + {%- if param.type=="int64_t" %} + integerProperties["{{param.name}}"] = params.{{param.name}}; + {%- endif %} + {%- if param.type=="real_t" %} + realProperties["{{param.name}}"] = double_c(params.{{param.name}}); + {%- endif %} + {%- if param.type=="std::string" %} + stringProperties["{{param.name}}"] = params.{{param.name}}; + {%- endif %} + {% endfor %} +} + +} //namespace walberla diff --git a/apps/benchmarks/GranularGas/Parameters.templ.h b/apps/benchmarks/GranularGas/Parameters.templ.h new file mode 100644 index 0000000000000000000000000000000000000000..29b18935ca1b3ffe708826b141c2cc170ab4c321 --- /dev/null +++ b/apps/benchmarks/GranularGas/Parameters.templ.h @@ -0,0 +1,51 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file Parameters.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +//====================================================================================================================== +// +// THIS FILE IS GENERATED - PLEASE CHANGE THE TEMPLATE !!! +// +//====================================================================================================================== + +#pragma once + +#include <core/config/Config.h> +#include <core/DataTypes.h> + +#include <string> + +namespace walberla { + +struct Parameters +{ + {%- for param in parameters %} + {{param.type}} {{param.name}} = {{param.defValue}}; + {%- endfor %} +}; + +void loadFromConfig(Parameters& params, + const Config::BlockHandle& cfg); + +void saveToSQL(const Parameters& params, + std::map< std::string, walberla::int64_t >& integerProperties, + std::map< std::string, double >& realProperties, + std::map< std::string, std::string >& stringProperties ); + +} //namespace walberla diff --git a/apps/benchmarks/GranularGas/SQLProperties.cpp b/apps/benchmarks/GranularGas/SQLProperties.cpp new file mode 100644 index 0000000000000000000000000000000000000000..08bb587df798ca08ef91ea04815f5fca8526964e --- /dev/null +++ b/apps/benchmarks/GranularGas/SQLProperties.cpp @@ -0,0 +1,104 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SQLProperties.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include "SQLProperties.h" + +#include <blockforest/BlockForest.h> +#include <blockforest/loadbalancing/DynamicDiffusive.h> +#include <blockforest/loadbalancing/DynamicParMetis.h> +#include <core/DataTypes.h> +#include <pe/cr/HCSITS.h> + +namespace walberla { +namespace mesa_pd { + +void addDomainPropertiesToSQL( const ::walberla::blockforest::BlockForest& forest, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & realProperties, + std::map< std::string, std::string > & /*stringProperties*/ ) +{ + integerProperties["numMPIProcesses"] = ::walberla::mpi::MPIManager::instance()->numProcesses(); + + realProperties[ "domainXMin" ] = forest.getDomain().xMin(); + realProperties[ "domainXMax" ] = forest.getDomain().xMax(); + realProperties[ "domainYMin" ] = forest.getDomain().yMin(); + realProperties[ "domainYMax" ] = forest.getDomain().yMax(); + realProperties[ "domainZMin" ] = forest.getDomain().zMin(); + realProperties[ "domainZMax" ] = forest.getDomain().zMax(); + + integerProperties[ "xBlocks" ] = ::walberla::int_c( forest.getXSize() ); + integerProperties[ "yBlocks" ] = ::walberla::int_c( forest.getYSize() ); + integerProperties[ "zBlocks" ] = ::walberla::int_c( forest.getZSize() ); + + integerProperties[ "xPeriodic" ] = ( forest.isXPeriodic() ? 1 : 0 ); + integerProperties[ "yPeriodic" ] = ( forest.isYPeriodic() ? 1 : 0 ); + integerProperties[ "zPeriodic" ] = ( forest.isZPeriodic() ? 1 : 0 ); +} + +void addLoadBalancingPropertiesToSQL( const ::walberla::blockforest::BlockForest& forest, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & /*realProperties*/, + std::map< std::string, std::string > & /*stringProperties*/ ) +{ + integerProperties[ "recalculateBlockLevelsInRefresh" ] = ( forest.recalculateBlockLevelsInRefresh() ? 1 : 0 ); + integerProperties[ "alwaysRebalanceInRefresh" ] = ( forest.alwaysRebalanceInRefresh() ? 1 : 0 ); + integerProperties[ "reevaluateMinTargetLevelsAfterForcedRefinement" ] = ( forest.reevaluateMinTargetLevelsAfterForcedRefinement() ? 1 : 0 ); + integerProperties[ "allowRefreshChangingDepth" ] = ( forest.allowRefreshChangingDepth() ? 1 : 0 ); + integerProperties[ "allowMultipleRefreshCycles" ] = ( forest.allowMultipleRefreshCycles() ? 1 : 0 ); + integerProperties[ "checkForEarlyOutInRefresh" ] = ( forest.checkForEarlyOutInRefresh() ? 1 : 0 ); + integerProperties[ "checkForLateOutInRefresh" ] = ( forest.checkForLateOutInRefresh() ? 1 : 0 ); +} + +void addParMetisPropertiesToSQL( const ::walberla::blockforest::DynamicParMetis& dpm, + std::map< std::string, int64_t > & /*integerProperties*/, + std::map< std::string, double > & realProperties, + std::map< std::string, std::string > & stringProperties ) +{ + stringProperties["metisAlgorithm"] = dpm.algorithmToString(); + stringProperties["metisEdgeSource"] = dpm.edgeSourceToString(); + stringProperties["metisWeightsToUse"] = dpm.weightsToUseToString(); + realProperties["metisipc2redist"] = dpm.getipc2redist(); +} + +std::string envToString(const char* env) +{ + return env != nullptr ? std::string(env) : ""; +} + +void addSlurmPropertiesToSQL( std::map< std::string, int64_t > & /*integerProperties*/, + std::map< std::string, double > & /*realProperties*/, + std::map< std::string, std::string > & stringProperties ) +{ + stringProperties["SLURM_CLUSTER_NAME"] = envToString(std::getenv( "SLURM_CLUSTER_NAME" )); + stringProperties["SLURM_CPUS_ON_NODE"] = envToString(std::getenv( "SLURM_CPUS_ON_NODE" )); + stringProperties["SLURM_CPUS_PER_TASK"] = envToString(std::getenv( "SLURM_CPUS_PER_TASK" )); + stringProperties["SLURM_JOB_ACCOUNT"] = envToString(std::getenv( "SLURM_JOB_ACCOUNT" )); + stringProperties["SLURM_JOB_ID"] = envToString(std::getenv( "SLURM_JOB_ID" )); + stringProperties["SLURM_JOB_CPUS_PER_NODE"] = envToString(std::getenv( "SLURM_JOB_CPUS_PER_NODE" )); + stringProperties["SLURM_JOB_NAME"] = envToString(std::getenv( "SLURM_JOB_NAME" )); + stringProperties["SLURM_JOB_NUM_NODES"] = envToString(std::getenv( "SLURM_JOB_NUM_NODES" )); + stringProperties["SLURM_NTASKS"] = envToString(std::getenv( "SLURM_NTASKS" )); + stringProperties["SLURM_NTASKS_PER_CORE"] = envToString(std::getenv( "SLURM_NTASKS_PER_CORE" )); + stringProperties["SLURM_NTASKS_PER_NODE"] = envToString(std::getenv( "SLURM_NTASKS_PER_NODE" )); + stringProperties["SLURM_NTASKS_PER_SOCKET"] = envToString(std::getenv( "SLURM_NTASKS_PER_SOCKET" )); +} + +} //namespace mesa_pd +} //namespace walberla diff --git a/apps/benchmarks/GranularGas/SQLProperties.h b/apps/benchmarks/GranularGas/SQLProperties.h new file mode 100644 index 0000000000000000000000000000000000000000..f6843ee2e4e6791cd0e691610cfbd2cf4bea6bcb --- /dev/null +++ b/apps/benchmarks/GranularGas/SQLProperties.h @@ -0,0 +1,74 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SQLProperties.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#pragma once + +#include <blockforest/loadbalancing/DynamicDiffusive.h> +#include <core/DataTypes.h> + +#include <map> +#include <string> + +namespace walberla { namespace blockforest{ +class DynamicParMetis; +} } + +namespace walberla { +namespace mesa_pd { + +void addDomainPropertiesToSQL( const walberla::blockforest::BlockForest& forest, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & realProperties, + std::map< std::string, std::string > & /*stringProperties*/ ); + +void addLoadBalancingPropertiesToSQL( const walberla::blockforest::BlockForest& forest, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & /*realProperties*/, + std::map< std::string, std::string > & /*stringProperties*/ ); + +template <typename PhantomBlockWeight_T> +void addDynamicDiffusivePropertiesToSQL( const walberla::blockforest::DynamicDiffusionBalance<PhantomBlockWeight_T>& ddb, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & /*realProperties*/, + std::map< std::string, std::string > & stringProperties ) +{ + integerProperties[ "diffMaxIterations" ] = walberla::int64_c(ddb.getMaxIterations()); + integerProperties[ "diffFlowIterations" ] = walberla::int64_c(ddb.getFlowIterations()); + integerProperties[ "bDiffAbortEarly" ] = ( ddb.checkForEarlyAbort() ? 1 : 0 ); + integerProperties[ "bDiffAdaptInflow" ] = ( ddb.adaptInflowWithGlobalInformation() ? 1 : 0 ); + integerProperties[ "bDiffAdaptOutflow" ] = ( ddb.adaptOutflowWithGlobalInformation() ? 1 : 0 ); + std::string diffModeStr = "unknown"; + if (ddb.getMode() == walberla::blockforest::DynamicDiffusionBalance<PhantomBlockWeight_T>::DIFFUSION_PUSH) diffModeStr = "push"; + if (ddb.getMode() == walberla::blockforest::DynamicDiffusionBalance<PhantomBlockWeight_T>::DIFFUSION_PULL) diffModeStr = "pull"; + if (ddb.getMode() == walberla::blockforest::DynamicDiffusionBalance<PhantomBlockWeight_T>::DIFFUSION_PUSHPULL) diffModeStr = "pushpull"; + stringProperties[ "diffMode" ] = diffModeStr; +} + +void addParMetisPropertiesToSQL( const ::walberla::blockforest::DynamicParMetis& dpm, + std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & realProperties, + std::map< std::string, std::string > & stringProperties ); + +void addSlurmPropertiesToSQL( std::map< std::string, int64_t > & integerProperties, + std::map< std::string, double > & realProperties, + std::map< std::string, std::string > & stringProperties ); + +} //namespace mesa_pd +} //namespace walberla diff --git a/apps/benchmarks/GranularGas/SelectProperty.h b/apps/benchmarks/GranularGas/SelectProperty.h new file mode 100644 index 0000000000000000000000000000000000000000..106e75e84659e59c49423c07042a75fbb3122587 --- /dev/null +++ b/apps/benchmarks/GranularGas/SelectProperty.h @@ -0,0 +1,48 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file SelectProperty.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleStorage.h> + +#include <core/mpi/MPIManager.h> + +namespace walberla { +namespace mesa_pd { + +class SelectRank +{ +public: + using return_type = int; + int operator()(const data::Particle& /*p*/) const { return rank_; } + int operator()(const data::Particle&& /*p*/) const { return rank_; } +private: + int rank_ = walberla::mpi::MPIManager::instance()->rank(); +}; + +class SelectIdx +{ +public: + using return_type = int; + auto operator()(const data::Particle& p) const { return p.getIdx(); } + auto operator()(const data::Particle&& p) const { return p.getIdx(); } +private: +}; + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/check.h b/apps/benchmarks/GranularGas/check.h new file mode 100644 index 0000000000000000000000000000000000000000..31099e229f40645d87b369be18d91df3377dfc9c --- /dev/null +++ b/apps/benchmarks/GranularGas/check.h @@ -0,0 +1,48 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file check.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleStorage.h> + +#include <blockforest/BlockForest.h> +#include <core/grid_generator/SCIterator.h> +#include <core/logging/Logging.h> + +namespace walberla { +namespace mesa_pd { + +void check( data::ParticleStorage& ps, blockforest::BlockForest& forest, real_t spacing ) +{ + WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - START ***"); + auto pIt = ps.begin(); + for (auto& iBlk : forest) + { + for (auto it = grid_generator::SCIterator(iBlk.getAABB(), Vector3<real_t>(spacing, spacing, spacing) * real_c(0.5), spacing); + it != grid_generator::SCIterator(); + ++it, ++pIt) + { + WALBERLA_CHECK_UNEQUAL(pIt, ps.end()); + WALBERLA_CHECK_FLOAT_EQUAL(pIt->getPositionRef(), *it); + } + } + WALBERLA_LOG_INFO_ON_ROOT("*** CHECKING RESULT - END ***"); +} + +} //namespace mesa_pd +} //namespace walberla diff --git a/apps/benchmarks/GranularGas/generateConfig.py b/apps/benchmarks/GranularGas/generateConfig.py new file mode 100755 index 0000000000000000000000000000000000000000..fe13e1768f08f015d151c6e6d41068dc65d8435e --- /dev/null +++ b/apps/benchmarks/GranularGas/generateConfig.py @@ -0,0 +1,21 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +from ConfigGenerator import Config + +cfg = Config() +cfg.addParameter("sorting", "std::string", '"none"') +cfg.addParameter("spacing", "real_t", "real_t(1.0)") +cfg.addParameter("radius", "real_t", "real_t(0.5)") +cfg.addParameter("bBarrier", "bool", "false") +cfg.addParameter("storeNodeTimings", "bool", "false") +cfg.addParameter("checkSimulation", "bool", "false") +cfg.addParameter("numOuterIterations", "int64_t", "10") +cfg.addParameter("initialRefinementLevel", "int64_t", "0") +cfg.addParameter("simulationSteps", "int64_t", "10") +cfg.addParameter("dt", "real_t", "real_t(0.01)") +cfg.addParameter("visSpacing", "int64_t", "1000") +cfg.addParameter("path", "std::string", '"vtk_out"') +cfg.addParameter("sqlFile", "std::string", '"benchmark.sqlite"') + +cfg.generate() diff --git a/apps/benchmarks/GranularGas/sortParticleStorage.cpp b/apps/benchmarks/GranularGas/sortParticleStorage.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3b9f2494c7d97e7042d0118e553e3a7221ee1d90 --- /dev/null +++ b/apps/benchmarks/GranularGas/sortParticleStorage.cpp @@ -0,0 +1,65 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file sortParticleStorage.cpp +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include "sortParticleStorage.h" + +namespace walberla { +namespace mesa_pd { + +class RandomCompareFunctor +{ +public: + double getWeight(const data::Particle p1) const + { + math::seedRandomGenerator(static_cast<std::mt19937::result_type>(p1.getUid())); + return math::realRandom<double>(); + } +}; + + +void sortParticleStorage( data::ParticleStorage& ps, + const std::string& algorithm, + const math::AABB& domain, + const uint64_t cells ) +{ + if( algorithm == "none" ) return; + if( algorithm == "random" ) + { + RandomCompareFunctor rand; + ps.sort(rand); + return; + } + if( algorithm == "linear" ) + { + sorting::LinearizedCompareFunctor linear(domain, Vector3<uint_t>(cells)); + ps.sort(linear); + return; + } + if( algorithm == "hilbert" ) + { + sorting::HilbertCompareFunctor hilbert(domain, 1ul << math::uintMSBPosition(cells)); + ps.sort(hilbert); + return; + } + WALBERLA_ABORT("unknown sorting algorithm: " << algorithm); +} + +} // namespace mesa_pd +} // namespace walberla diff --git a/apps/benchmarks/GranularGas/sortParticleStorage.h b/apps/benchmarks/GranularGas/sortParticleStorage.h new file mode 100644 index 0000000000000000000000000000000000000000..bec6c447df58d0addedb6e34de6d10011474c6fe --- /dev/null +++ b/apps/benchmarks/GranularGas/sortParticleStorage.h @@ -0,0 +1,41 @@ +//====================================================================================================================== +// +// This file is part of waLBerla. waLBerla is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// waLBerla is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file sortParticleStorage.h +//! \author Sebastian Eibl <sebastian.eibl@fau.de> +// +//====================================================================================================================== + +#include <mesa_pd/data/ParticleStorage.h> +#include <mesa_pd/sorting/HilbertCompareFunctor.h> +#include <mesa_pd/sorting/LinearizedCompareFunctor.h> + +#include <core/Abort.h> +#include <core/math/Random.h> +#include <core/logging/Logging.h> + +#include <memory> +#include <string> + +namespace walberla { +namespace mesa_pd { + +void sortParticleStorage( data::ParticleStorage& ps, + const std::string& algorithm, + const math::AABB& domain, + const uint64_t cells ); + +} // namespace mesa_pd +} // namespace walberla