From 7fc2ff66bab9372f34d051b860439fa637d0159e Mon Sep 17 00:00:00 2001
From: Grigorii Drozdov <drozd013@umn.edu>
Date: Thu, 22 Apr 2021 21:23:10 -0500
Subject: [PATCH] fix: reduce_to_root restart
---
apps/benchmarks/CNT/01_cnt_film.cpp | 4 +-
apps/benchmarks/CNT/Config.cpp | 5 ++-
apps/benchmarks/CNT/InitializeCNTs.cpp | 51 ++++++++++++--------------
apps/benchmarks/CNT/InitializeCNTs.h | 3 +-
4 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/apps/benchmarks/CNT/01_cnt_film.cpp b/apps/benchmarks/CNT/01_cnt_film.cpp
index 7d1cafd79..9714486f8 100644
--- a/apps/benchmarks/CNT/01_cnt_film.cpp
+++ b/apps/benchmarks/CNT/01_cnt_film.cpp
@@ -126,7 +126,7 @@ int main(int argc, char **argv)
numParticles = generateCNTs(filmSpecimen, ps, *domain);
} else
{
- numParticles = loadCNTs(filmSpecimen.initialConfigurationFile, ps);
+ numParticles = loadCNTs(filmSpecimen.initialConfigurationFile, ps, *domain);
}
walberla::mpi::reduceInplace(numParticles, walberla::mpi::SUM);
WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles);
@@ -304,7 +304,7 @@ int main(int argc, char **argv)
WALBERLA_LOG_INFO_ON_ROOT(CYAN << " SAVING CONFIG " << RESET);
saveConfig(ps,
filmSpecimen.confFolder + "/conf_" + std::to_string((i + 1) * 0.00002) + "ns",
- IO_MODE::USE_MPI_IO);
+ IO_MODE::REDUCE_TO_ROOT);
loopTP["save config"].end();
}
}
diff --git a/apps/benchmarks/CNT/Config.cpp b/apps/benchmarks/CNT/Config.cpp
index ee19d2a97..17e3b0896 100644
--- a/apps/benchmarks/CNT/Config.cpp
+++ b/apps/benchmarks/CNT/Config.cpp
@@ -74,9 +74,10 @@ void saveConfig(const shared_ptr<data::ParticleStorage>& ps,
WALBERLA_ROOT_SECTION()
{
uint_t dataSize = sizeof(mpi::RecvBuffer::ElementType) * rb.size();
+ size_t size = rb.size() / 104;
std::ofstream binfile;
- binfile.open(filename + ".sav",
- std::ios::out | std::ios::binary);
+ binfile.open(filename + ".sav", std::ios::out | std::ios::binary);
+ binfile.write((char*) &size, sizeof(size_t));
binfile.write(reinterpret_cast< const char * >(rb.ptr()), numeric_cast<std::streamsize>(dataSize));
binfile.close();
}
diff --git a/apps/benchmarks/CNT/InitializeCNTs.cpp b/apps/benchmarks/CNT/InitializeCNTs.cpp
index 9ca22484d..ff9d29996 100644
--- a/apps/benchmarks/CNT/InitializeCNTs.cpp
+++ b/apps/benchmarks/CNT/InitializeCNTs.cpp
@@ -116,7 +116,6 @@ int64_t generateCNTs(const FilmSpecimen& spec,
{
auto myRank = mpi::MPIManager::instance()->rank();
- //auto CNT_length = 2_r * spec.spacing * real_c(spec.numSegs);
// Fixed random seed is necessary for coordinated generation on all MPI proc.
auto rand0_1 = math::RealRandom<real_t>(static_cast<std::mt19937::result_type>(spec.seed));
// Create an assembly of CNTs
@@ -137,7 +136,8 @@ int64_t generateCNTs(const FilmSpecimen& spec,
}
int64_t loadCNTs(const std::string& filename,
- const shared_ptr<data::ParticleStorage>& ps)
+ const shared_ptr<data::ParticleStorage>& ps,
+ const domain::IDomain& domain)
{
WALBERLA_LOG_INFO_ON_ROOT(GREEN << "Loading configuration (binary format): " << filename);
const auto numProcesses = mpi::MPIManager::instance()->numProcesses();
@@ -154,15 +154,13 @@ int64_t loadCNTs(const std::string& filename,
if (i != rank) continue; //bad practice but with the current file format we have to do this
std::ifstream binfile;
binfile.open(filename.c_str(), std::ios::in | std::ios::binary);
- int size;
- binfile.read((char *) &size, sizeof(int));
+ size_t size;
+ binfile.read((char *) &size, sizeof(size_t));
std::cout << RED << "size read form binary file is" << size << RESET << std::endl;
- for (int id = 0; id < size; id++)
+ for (size_t id = 0; id < size; id++)
{
- int ID;
- int sID;
- int cID;
- int gID;
+ int64_t sID;
+ int64_t cID;
double x;
double y;
double z;
@@ -174,10 +172,8 @@ int64_t loadCNTs(const std::string& filename,
double wx;
double wy;
double wz;
- binfile.read((char *) &ID, sizeof(int));
- binfile.read((char *) &sID, sizeof(int));
- binfile.read((char *) &cID, sizeof(int));
- binfile.read((char *) &gID, sizeof(int));
+ binfile.read((char *) &sID, sizeof(int64_t));
+ binfile.read((char *) &cID, sizeof(int64_t));
binfile.read((char *) &x, sizeof(double));
binfile.read((char *) &y, sizeof(double));
binfile.read((char *) &z, sizeof(double));
@@ -194,25 +190,26 @@ int64_t loadCNTs(const std::string& filename,
pos[1] = real_c(y);
pos[2] = real_c(z);
- data::Particle &&sp = *ps->create();
- sp.setPosition(pos);
- sp.setOwner(rank);
- sp.setInteractionRadius(kernel::cnt::outer_radius);
- sp.setSegmentID(sID);
- sp.setClusterID(cID);
- sp.getRotationRef().rotate( Vec3(0_r, 1_r, 0_r), -0.5_r * math::pi + real_c(theta));
- sp.getRotationRef().rotate( Vec3(0_r, 0_r, 1_r), real_c(phi));
- sp.setLinearVelocity( Vec3(real_c(vx), real_c(vy), real_c(vz)) );
- sp.setAngularVelocity( Vec3(real_c(wx), real_c(wy), real_c(wz)) );
- numParticles++;
+ if (domain.isContainedInProcessSubdomain(uint_c(rank), pos))
+ {
+ data::Particle&& sp = *ps->create();
+ sp.setPosition(pos);
+ sp.setOwner(rank);
+ sp.setInteractionRadius(kernel::cnt::outer_radius);
+ sp.setSegmentID(sID);
+ sp.setClusterID(cID);
+ sp.getRotationRef().rotate(Vec3(0_r, 1_r, 0_r), -0.5_r * math::pi + real_c(theta));
+ sp.getRotationRef().rotate(Vec3(0_r, 0_r, 1_r), real_c(phi));
+ sp.setLinearVelocity(Vec3(real_c(vx), real_c(vy), real_c(vz)));
+ sp.setAngularVelocity(Vec3(real_c(wx), real_c(wy), real_c(wz)));
+ numParticles++;
+ }
}
}
- mpi::reduceInplace(numParticles, mpi::SUM);
- WALBERLA_LOG_INFO_ON_ROOT("#particles created: " << numParticles);
-
return numParticles;
}
} //namespace mesa_pd
} //namespace walberla
+
diff --git a/apps/benchmarks/CNT/InitializeCNTs.h b/apps/benchmarks/CNT/InitializeCNTs.h
index f8a114eda..b916389e6 100644
--- a/apps/benchmarks/CNT/InitializeCNTs.h
+++ b/apps/benchmarks/CNT/InitializeCNTs.h
@@ -46,7 +46,8 @@ int64_t generateCNTs(const FilmSpecimen& spec,
const domain::IDomain& domain);
int64_t loadCNTs(const std::string& filename,
- const shared_ptr<data::ParticleStorage>& ps);
+ const shared_ptr<data::ParticleStorage>& ps,
+ const domain::IDomain& domain);
} //namespace mesa_pd
} //namespace walberla
--
GitLab