Commit b323b32f authored by Martin Bauer's avatar Martin Bauer

Merge branch 'field_comm_benchmark' into gpu

parents cf92ed15 9bd7bff3
......@@ -65,7 +65,7 @@ IncludeCategories:
Priority: 5
- Regex: '^"domain_decomposition/'
Priority: 6
- Regex: '^"domain_decomposition/'
- Regex: '^"executiontree/'
Priority: 7
- Regex: '^"fft/'
Priority: 8
......
This diff is collapsed.
......@@ -653,7 +653,7 @@ set ( Boost_ADDITIONAL_VERSIONS
"1.45" "1.45.0" "1.46" "1.46.0" "1.46.1" "1.47" "1.47.0" "1.48" "1.48.0" "1.49" "1.49.0"
"1.50" "1.50.0" "1.51" "1.51.0" "1.52" "1.52.0" "1.53" "1.53.0" "1.54" "1.54.0" "1.55" "1.55.0"
"1.56" "1.56.0" "1.57" "1.57.0" "1.58" "1.58.0" "1.59" "1.59.0" "1.60" "1.60.0" "1.61" "1.61.0" "1.62" "1.62.0" "1.63" "1.63.0"
"1.64.0" "1.65.0" "1.65.1" "1.66.0" "1.67.0" "1.68.0" "1.69.0" )
"1.64.0" "1.65.0" "1.65.1" "1.66.0" "1.67.0" "1.68.0" "1.69.0" "1.70.0" )
set ( Boost_USE_STATIC_LIBS OFF CACHE BOOL "Use boost static libraries" )
set ( Boost_USE_MULTITHREADED OFF CACHE BOOL "Use boost multithreaded libraries" )
......@@ -741,11 +741,10 @@ if ( WALBERLA_BUILD_WITH_PYTHON AND NOT WALBERLA_CXX_COMPILER_IS_MSVC)
endif()
if( PYTHON_LIBRARY MATCHES "python3" )
find_library( BOOST_PYTHON_LIBRARY NAMES
boost_python36${_boost_MULTITHREADED}
boost_python35${_boost_MULTITHREADED}
boost_python-py36${_boost_MULTITHREADED} boost_python-py35${_boost_MULTITHREADED}
boost_python37${_boost_MULTITHREADED} boost_python36${_boost_MULTITHREADED} boost_python35${_boost_MULTITHREADED}
boost_python-py37${_boost_MULTITHREADED} boost_python-py36${_boost_MULTITHREADED} boost_python-py35${_boost_MULTITHREADED}
boost_python-py34${_boost_MULTITHREADED} boost_python-py33${_boost_MULTITHREADED}
boost_python-py32${_boost_MULTITHREADED} boost_python3${_boost_MULTITHREADED}
boost_python3${_boost_MULTITHREADED}
boost_python${_boost_MULTITHREADED}
PATHS ${Boost_LIBRARY_DIRS} NO_DEFAULT_PATH )
else()
......@@ -842,14 +841,6 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
add_flag ( CMAKE_MODULE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
add_flag ( CMAKE_EXE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
add_flag ( CMAKE_SHARED_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
# When using Intel MPI, mpi.h has to be included before including the standard library
# therefore we use the -include flag to enforce this.
if ( MPI_C_INCLUDE_PATH MATCHES "intel" )
message (STATUS "Activating IntelMPI include workaround for mpi.h" )
add_flag ( CMAKE_CXX_FLAGS "-include mpi.h" )
add_flag ( CMAKE_C_FLAGS "-include mpi.h" )
endif ( )
endif ( )
endif ( )
......
add_subdirectory( AdaptiveMeshRefinementFluidParticleCoupling )
add_subdirectory( ComplexGeometry )
add_subdirectory( DEM )
add_subdirectory( FieldCommunication )
add_subdirectory( MeshDistance )
add_subdirectory( CouetteFlow )
add_subdirectory( ForcesOnSphereNearPlaneInShearFlow )
add_subdirectory( GranularGas )
add_subdirectory( LennardJones )
add_subdirectory( NonUniformGrid )
add_subdirectory( MotionSingleHeavySphere )
add_subdirectory( PeriodicGranularGas )
add_subdirectory( PoiseuilleChannel )
add_subdirectory( ProbeVsExtraMessage )
add_subdirectory( SchaeferTurek )
......
waLBerla_link_files_to_builddir( "*.dat" )
waLBerla_link_files_to_builddir( "*.py" )
waLBerla_add_executable ( NAME FieldCommunication
DEPENDS blockforest core domain_decomposition field postprocessing )
This diff is collapsed.
Domain
{
cellsPerBlock < 60, 60, 60 >;
domainWeights < 1, 1, 1 >;
blocksPerProcess 1;
}
Communication
{
stencil D3Q19;
ghostLayers 1;
buffered 1;
singleMessage 1;
Fields {
pdf 1;
pdfOpt 1;
vector 0;
scalar 0;
}
localCommunicationMode start;
layout fzyx;
}
Run
{
warmupIterations 2;
iterations 50;
databaseFile FieldCommunication.sqlite;
}
\ No newline at end of file
import waLBerla
import subprocess
import re
from collections import defaultdict
import sys
from datetime import timedelta
base = (32, 16, 2, 64)
BlOCK_SIZES_SQ = [(i, i, i) for i in base]
BLOCK_SIZES_RECT = [(i, i, i // 2) for i in base] + [(i, i // 2, i // 2) for i in base]
time_for_benchmark = 0.25
outer_iterations = 2
db_file = 'FieldCommunication.sqlite'
def supermuc_network_spread():
try:
node_list = subprocess.check_output("scontrol show hostname $SLURM_JOB_NODELIST", shell=True, encoding='utf8')
except subprocess.CalledProcessError:
return defaultdict(lambda: 0)
spread = defaultdict(set)
for s in node_list.split("\n"):
m = re.search("i(\d\d)r(\d\d)c(\d\d)s(\d\d)", s)
if m:
for name, idx in zip(['i', 'r', 'c', 's'], range(1, 5)):
spread[name].add(m.group(idx))
return {k: len(v) for k, v in spread.items()}
sng_network = supermuc_network_spread()
class AlreadySimulated:
def __init__(self, db_file, properties=('processes0*processes1*processes2', 'layout', 'ghostLayers', 'cartesianCommunicator', 'stencil',
'cellsPerBlock0', 'cellsPerBlock1', 'cellsPerBlock2',
'blocksPerProcess', 'localCommunicationMode', 'singleMessage',
'fieldsPdf', 'fieldsPdfOpt', 'fieldsVector', 'fieldsScalar',
'buffered')):
self.properties = properties
import sqlite3
conn = sqlite3.connect(db_file)
self.data = set()
try:
for row in conn.execute("SELECT {} FROM runs;".format(",".join(self.properties))):
self.data.add(row)
except sqlite3.OperationalError:
pass
waLBerla.log_info_on_root("Loaded {} scenarios".format(len(self.data)))
def in_db(self, args):
return args in self.data
@waLBerla.callback("config")
def config(processes=None):
simulated_db = AlreadySimulated(db_file)
isWalberlaRun = processes is None
skipped = 0
simulated = 0
if isWalberlaRun:
processes = waLBerla.mpi.numProcesses()
for layout in ('fzyx', 'zyxf'):
for ghost_layers in (1, 2):
for cartesian_comm in (False, True):
for stencil in ('D3Q19', 'D3Q27', 'D3Q7'):
for cells in BlOCK_SIZES_SQ + BLOCK_SIZES_RECT:
for blocksPerProcess in (1, 2, 4, 8, 16):
for local_comm in ('start', 'noOptimization', 'buffer'):
for single_message in (True, False):
for pdf, pdf_opt, vector, scalar in ([1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1],
[0, 0, 0, 3], [0, 0, 0, 19], [2, 0, 0, 0], [0, 2, 0, 0]):
for buffered in (0, 1):
if blocksPerProcess >= 8 and cells[0] >= 64 and cells[1] >= 64 and cells[2] >= 64:
continue
data = (processes, layout, ghost_layers, int(cartesian_comm), stencil, *cells, blocksPerProcess, local_comm,
int(single_message), pdf, pdf_opt, vector, scalar, buffered)
if simulated_db.in_db(data):
skipped += 1
if skipped % 100 == 0 and isWalberlaRun:
waLBerla.log_info_on_root("Skipped {} scenarios".format(skipped))
continue
else:
simulated += 1
if not isWalberlaRun:
continue
cfg = {
'Domain': {
'cellsPerBlock': cells,
'domainWeights': (1, 1, 1),
'blocksPerProcess': blocksPerProcess,
},
'Communication': {
'buffered': buffered,
'stencil': stencil,
'ghostLayers': ghost_layers,
'cartesianCommunicator': cartesian_comm,
'singleMessage': single_message,
'Fields': {
'pdf': pdf,
'pdfOpt': pdf_opt,
'vector': vector,
'scalar': scalar,
},
'layout': layout,
'localCommunicationMode': local_comm,
},
'Run': {
'warmupIterations': 3,
'iterations': 100,
'outerIterations': outer_iterations,
'databaseFile': db_file,
'timeForBenchmark': time_for_benchmark,
'minIterations': 2,
'maxIterations': 10000,
},
'Database': {
'sngNetworkIslands': sng_network['i'],
'sngNetworkRows': sng_network['r'],
'sngNetworkCabinets': sng_network['c'],
'sngNetworkSlots': sng_network['s'],
}
}
yield cfg
if not isWalberlaRun:
print("Skipped", skipped, "to simulate", simulated)
estimated_seconds = simulated * time_for_benchmark * outer_iterations
print("Estimated time ", timedelta(seconds=estimated_seconds))
if __name__ == '__main__':
for _ in config(int(sys.argv[1])):
pass
waLBerla_link_files_to_builddir( *.cfg )
waLBerla_link_files_to_builddir( *.py )
waLBerla_add_executable ( NAME PE_GranularGas
FILES PE_GranularGas.cpp
DEPENDS blockforest core pe postprocessing )
waLBerla_add_executable ( NAME MESA_PD_GranularGas
FILES MESA_PD_GranularGas.cpp
DEPENDS blockforest core pe mesa_pd postprocessing vtk )
waLBerla_add_executable ( NAME MESA_PD_KernelBenchmark
FILES MESA_PD_KernelBenchmark.cpp
DEPENDS blockforest core pe mesa_pd postprocessing vtk )
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
from mesa_pd.accessor import Accessor
import mesa_pd.data as data
import mesa_pd.kernel as kernel
import mesa_pd.mpi as mpi
import argparse
import numpy as np
import os
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Generate all necessary files for the waLBerla mesa_pd module.')
parser.add_argument('path', help='Where should the files be created?')
parser.add_argument("-f", "--force", help="Generate the files even if not inside a waLBerla directory.",
action="store_true")
args = parser.parse_args()
if ((not os.path.isfile(args.path + "/src/walberla.h")) and (not args.force)):
raise RuntimeError(args.path + " is not the path to a waLBerla root directory! Specify -f to generate the files anyway.")
os.makedirs(args.path + "/src/mesa_pd/common", exist_ok = True)
os.makedirs(args.path + "/src/mesa_pd/data", exist_ok = True)
os.makedirs(args.path + "/src/mesa_pd/domain", exist_ok = True)
os.makedirs(args.path + "/src/mesa_pd/kernel", exist_ok = True)
os.makedirs(args.path + "/src/mesa_pd/mpi/notifications", exist_ok = True)
os.makedirs(args.path + "/src/mesa_pd/vtk", exist_ok = True)
shapes = ["Sphere", "HalfSpace"]
ps = data.ParticleStorage()
ch = data.ContactHistory()
lc = data.LinkedCells()
ss = data.ShapeStorage(ps, shapes)
ps.addProperty("position", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS")
ps.addProperty("linearVelocity", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS")
ps.addProperty("invMass", "walberla::real_t", defValue="real_t(1)", syncMode="COPY")
ps.addProperty("force", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="NEVER")
ps.addProperty("shapeID", "size_t", defValue="", syncMode="COPY")
ps.addProperty("rotation", "walberla::mesa_pd::Rot3", defValue="", syncMode="ALWAYS")
ps.addProperty("angularVelocity", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="ALWAYS")
ps.addProperty("torque", "walberla::mesa_pd::Vec3", defValue="real_t(0)", syncMode="NEVER")
ps.addProperty("type", "uint_t", defValue="0", syncMode="COPY")
ps.addProperty("flags", "walberla::mesa_pd::data::particle_flags::FlagT", defValue="", syncMode="COPY")
ps.addProperty("nextParticle", "int", defValue="-1", syncMode="NEVER")
kernels = []
kernels.append( kernel.DoubleCast(shapes) )
kernels.append( kernel.ExplicitEuler() )
kernels.append( kernel.ExplicitEulerWithShape() )
kernels.append( kernel.ForceLJ() )
kernels.append( kernel.HeatConduction() )
kernels.append( kernel.InsertParticleIntoLinkedCells() )
kernels.append( kernel.LinearSpringDashpot() )
kernels.append( kernel.NonLinearSpringDashpot() )
kernels.append( kernel.SingleCast(shapes) )
kernels.append( kernel.SpringDashpot() )
kernels.append( kernel.TemperatureIntegration() )
kernels.append( kernel.VelocityVerlet() )
kernels.append( kernel.VelocityVerletWithShape() )
ac = Accessor()
for k in kernels:
ac.mergeRequirements(k.getRequirements())
ac.printSummary()
comm = []
comm.append(mpi.BroadcastProperty())
comm.append(mpi.ClearNextNeighborSync())
comm.append(mpi.ReduceContactHistory())
comm.append(mpi.ReduceProperty())
comm.append(mpi.SyncNextNeighbors(ps))
ps.generate(args.path + "/src/mesa_pd/")
ch.generate(args.path + "/src/mesa_pd/")
lc.generate(args.path + "/src/mesa_pd/")
ss.generate(args.path + "/src/mesa_pd/")
for k in kernels:
k.generate(args.path + "/src/mesa_pd/")
for c in comm:
c.generate(args.path + "/src/mesa_pd/")
GranularGas
{
simulationCorner < 0, 0, 0 >;
simulationDomain < 6, 6, 6 >;
blocks < 2,2,2 >;
isPeriodic < 1, 1, 1 >;
radius 0.6;
spacing 1.0;
vMax 0.0;
dt 0.0001;
simulationSteps 500;
visSpacing 100;
HCSITSmaxIterations 10;
HCSITSRelaxationParameter 0.7;
HCSITSErrorReductionParameter 0.8;
HCSITSRelaxationModelStr ApproximateInelasticCoulombContactByDecoupling;
globalLinearAcceleration < 0, 0, 0 >;
}
This diff is collapsed.
This diff is collapsed.
PeriodicGranularGas
GranularGas
{
simulationCorner < 0, 0, 0 >;
simulationDomain < 30, 30, 30 >;
......
import os
import time
import math
import random
import re
from influxdb import InfluxDBClient
from git import Repo
class Upload:
def __init__(self):
try:
self.write_user_pw = os.environ["INFLUXDB_WRITE_USER"]
except KeyError:
import sys
print('Password for the InfluxDB write_user was not set.\n',
'See https://docs.gitlab.com/ee/ci/variables/#secret-variables', file=sys.stderr)
exc_info = sys.exc_info()
raise exc_info[0].with_traceback(exc_info[1], exc_info[2])
self.client = InfluxDBClient('i10grafana.informatik.uni-erlangen.de', 8086,
'pe', self.write_user_pw, 'pe')
def process(self, filename, model, friction, sync, parallelization):
with open(filename) as f:
s = f.read()
m = re.search('PUpS: (\S*)', s)
json_body = [
{
'measurement': 'pe_benchmark',
'tags': {
'host' : os.uname()[1],
'image' : os.environ["DOCKER_IMAGE_NAME"],
'model' : model,
'friction' : friction,
'sync' : sync,
'parallelization' : parallelization
},
'time': int(time.time()),
'fields': {'PUpS': float(m.group(1))}
}
]
print(float(m.group(1)))
self.client.write_points(json_body, time_precision='s')
if __name__ == "__main__":
up = Upload()
up.process("GranularGas_DEM_NN.txt", "DEM", "Coulomb", "next neighbors", "8P1T")
up.process("GranularGas_DEM_SO.txt", "DEM", "Coulomb", "shadow owners", "8P1T")
up.process("GranularGas_HCSITS_NN_IFC.txt", "HCSITS", "InelasticFrictionlessContact", "next neighbors", "8P1T")
up.process("GranularGas_HCSITS_NN_AICCBD.txt", "HCSITS", "ApproximateInelasticCoulombContactByDecoupling", "next neighbors", "8P1T")
up.process("GranularGas_HCSITS_NN_ICCBD.txt", "HCSITS", "InelasticCoulombContactByDecoupling", "next neighbors", "8P1T")
up.process("GranularGas_HCSITS_NN_IGMDC.txt", "HCSITS", "InelasticGeneralizedMaximumDissipationContact", "next neighbors", "8P1T")
up.process("GranularGas_HCSITS_SO_IFC.txt", "HCSITS", "InelasticFrictionlessContact", "shadow owners", "8P1T")
waLBerla_link_files_to_builddir( *.cfg )
waLBerla_link_files_to_builddir( *.py )
waLBerla_add_executable ( NAME PeriodicGranularGas
FILES PeriodicGranularGas.cpp
waLBerla_add_executable ( NAME LennardJones
FILES LennardJones.cpp
DEPENDS blockforest core pe )
//======================================================================================================================
//
// This file is part of waLBerla. waLBerla is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// waLBerla is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
//
//! \file LennardJones.cpp
//! \author Sebastian Eibl <sebastian.eibl@fau.de>
//
//======================================================================================================================
#include <pe/basic.h>
#include <pe/vtk/SphereVtkOutput.h>
#include <blockforest/Initialization.h>
#include <core/Abort.h>
#include <core/Environment.h>
#include <core/grid_generator/SCIterator.h>
#include <core/logging/Logging.h>
#include <core/math/Random.h>
#include <core/waLBerlaBuildInfo.h>
#include <vtk/VTKOutput.h>
#include <functional>
#include <memory>
#include <tuple>
namespace walberla {
using namespace walberla::pe;
using namespace walberla::timing;
using BodyTuple = std::tuple<Sphere> ;
inline
void LJ(RigidBody& bd1, RigidBody& bd2)
{
if (bd1.getSystemID() != bd2.getSystemID())
{
Vec3 dir = bd1.getPosition() - bd2.getPosition();
const real_t inv_d = real_t(1) / length(dir);
dir *= inv_d;
WALBERLA_ASSERT_FLOAT_EQUAL(length(dir),
real_t(1),
"direction not normalized: " << dir);
const real_t sr = real_t(1.21) * inv_d;
const real_t sr6 = (sr*sr*sr)*(sr*sr*sr);
const real_t force = real_t(4) * real_t(2.12) * sr6 * ( sr6 - 1 );
bd1.addForce(force * dir);
}
}
inline
void integrate(RigidBody& bd)
{
bd.setLinearVel( bd.getForce() * real_t(0.01) + bd.getLinearVel() );
bd.setPosition( bd.getLinearVel() * real_t(0.01) + bd.getPosition() );
bd.setForce( Vec3( real_t(0) ) );
}
int main( int argc, char ** argv )
{
Environment env(argc, argv);
logging::Logging::instance()->setStreamLogLevel(logging::Logging::INFO);
logging::Logging::instance()->setFileLogLevel(logging::Logging::INFO);
WALBERLA_LOG_INFO_ON_ROOT( "config file: " << argv[1] )
WALBERLA_LOG_INFO_ON_ROOT( "waLBerla Revision: " << WALBERLA_GIT_SHA1 );
math::seedRandomGenerator( static_cast<unsigned int>(1337 * mpi::MPIManager::instance()->worldRank()) );
WALBERLA_LOG_INFO_ON_ROOT("*** GLOBALBODYSTORAGE ***");
shared_ptr<BodyStorage> globalBodyStorage = make_shared<BodyStorage>();
WALBERLA_LOG_INFO_ON_ROOT("*** BLOCKFOREST ***");
//domain setup
const real_t spacing = real_t(1.0);
math::AABB domain( Vec3(real_t(-0.5), real_t(-0.5), real_t(-0.5)),
Vec3(real_t(+0.5), real_t(+0.5), real_t(+0.5)));
domain.scale(real_t(20));
// create forest
auto forest = blockforest::createBlockForest(domain,
Vector3<uint_t>(1,1,1),
Vector3<bool>(false, false, false));
if (!forest)
{
WALBERLA_LOG_INFO_ON_ROOT( "No BlockForest created ... exiting!");
return EXIT_SUCCESS;
}
WALBERLA_LOG_INFO_ON_ROOT("simulationDomain: " << forest->getDomain());
WALBERLA_LOG_INFO_ON_ROOT("blocks: " << Vector3<uint_t>(forest->getXSize(), forest->getYSize(), forest->getZSize()) );
WALBERLA_LOG_INFO_ON_ROOT("*** BODYTUPLE ***");
// initialize body type ids
SetBodyTypeIDs<BodyTuple>::execute();
WALBERLA_LOG_INFO_ON_ROOT("*** STORAGEDATAHANDLING ***");
// add block data
auto storageID = forest->addBlockData(createStorageDataHandling<BodyTuple>(), "Storage");
auto ccdID = forest->addBlockData(ccd::createHashGridsDataHandling( globalBodyStorage, storageID ), "CCD");
auto fcdID = forest->addBlockData(fcd::createGenericFCDDataHandling<BodyTuple, fcd::AnalyticCollideFunctor>(), "FCD");
WALBERLA_UNUSED(ccdID);
WALBERLA_UNUSED(fcdID);
WALBERLA_LOG_INFO_ON_ROOT("*** SYNCCALL ***");
std::function<void(void)> syncCallWithoutTT;
syncCallWithoutTT = std::bind( pe::syncNextNeighbors<BodyTuple>, std::ref(*forest), storageID, static_cast<WcTimingTree*>(nullptr), real_c(0.1), false );
WALBERLA_LOG_INFO_ON_ROOT("Using NextNeighbor sync!");
WALBERLA_LOG_INFO_ON_ROOT("*** VTK ***");
auto vtkDomainOutput = vtk::createVTKOutput_DomainDecomposition( forest, "domain_decomposition", 1, "vtk_out", "simulation_step" );
auto vtkSphereHelper = make_shared<SphereVtkOutput>(storageID, *forest) ;
auto vtkSphereOutput = vtk::createVTKOutput_PointData(vtkSphereHelper, "Bodies", 1, "vtk_out", "simulation_step", false, false);
WALBERLA_LOG_INFO_ON_ROOT("*** SETUP - START ***");
auto simulationDomain = forest->getDomain();
const auto& generationDomain = simulationDomain; // simulationDomain.getExtended(-real_c(0.5) * spacing);
//initialize particles
uint_t numParticles(0);
for (auto it = grid_generator::SCIterator(generationDomain, Vec3(spacing, spacing, spacing) * real_c(0.5), spacing);
it != grid_generator::SCIterator();
++it)
{
SphereID sp = createSphere(*globalBodyStorage, *forest, storageID, 0, *it, real_t(1));
if (sp!=nullptr)
{
sp->setLinearVel( math::realRandom(real_t(-1), real_t(+1)),
math::realRandom(real_t(-1), real_t(+1)),