Commit 86814247 authored by Dominik Thoennes's avatar Dominik Thoennes
Browse files

Merge branch 'petsc_comparison' into 'master'

Petsc comparison

See merge request terraneo/tinyhhg!179
parents 78cc875d 2f1b5e92
Pipeline #12133 passed with stage
in 96 minutes and 14 seconds
......@@ -20,8 +20,8 @@ set ( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${tinyhhg_SOURCE_DIR}/cmake/modules
if ( HHG_BUILD_WITH_PETSC )
find_package( PETSc )
include_directories ( ${PETSC_INCLUDE_DIRS} )
link_directories ( ${PETSC_LIBRARY_DIR} )
include_directories ( ${PETSC_INCLUDES} )
link_directories ( ${PETSC_LIBRARIES} )
list ( APPEND SERVICE_LIBS ${PETSC_LIBRARIES} )
set(WALBERLA_BUILD_WITH_MPI ON CACHE BOOL "Build with MPI" FORCE)
message(STATUS "WALBERLA_BUILD_WITH_MPI was force set to ON")
......
#include "core/DataTypes.h"
#include "core/Environment.h"
#include "core/config/Config.h"
#include "core/mpi/MPIManager.h"
#include "tinyhhg_core/FunctionProperties.hpp"
#include "tinyhhg_core/LikwidWrapper.hpp"
#include "tinyhhg_core/VTKWriter.hpp"
#include "tinyhhg_core/communication/Syncing.hpp"
#include "tinyhhg_core/mesh/MeshInfo.hpp"
#include "tinyhhg_core/p1functionspace/P1ConstantOperator.hpp"
#include "tinyhhg_core/p1functionspace/P1Function.hpp"
#include "tinyhhg_core/primitivestorage/SetupPrimitiveStorage.hpp"
#include "tinyhhg_core/primitivestorage/Visualization.hpp"
#include "tinyhhg_core/primitivestorage/loadbalancing/SimpleBalancer.hpp"
using walberla::real_t;
int main( int argc, char* argv[] )
{
LIKWID_MARKER_INIT;
walberla::Environment env( argc, argv );
walberla::MPIManager::instance()->useWorldComm();
LIKWID_MARKER_THREADINIT;
auto cfg = std::make_shared< walberla::config::Config >();
if( env.config() == nullptr )
{
auto defaultFile = "./ApplyBenchmark.prm";
WALBERLA_LOG_PROGRESS_ON_ROOT( "No Parameter file given loading default parameter file: " << defaultFile );
cfg->readParameterFile( defaultFile );
} else
{
cfg = env.config();
}
const walberla::Config::BlockHandle mainConf = cfg->getBlock( "Parameters" );
const uint_t level = mainConf.getParameter< uint_t >( "level" );
std::string meshFileName = mainConf.getParameter< std::string >( "mesh" );
hhg::MeshInfo meshInfo = hhg::MeshInfo::fromGmshFile( meshFileName );
//hhg::MeshInfo::meshUnitSquare( 2 );
hhg::SetupPrimitiveStorage setupStorage( meshInfo, walberla::uint_c( walberla::mpi::MPIManager::instance()->numProcesses() ) );
hhg::loadbalancing::roundRobin( setupStorage );
std::function< real_t( const hhg::Point3D& ) > ones = []( const hhg::Point3D& ) { return 1.0; };
std::function< real_t( const hhg::Point3D& ) > exact = []( const hhg::Point3D& xx ) {
//return 5.0;
return std::sin( walberla::math::PI * xx[0] ) + std::cos( walberla::math::PI * xx[1] );
//return ( real_c(std::rand()) / real_c(RAND_MAX));
};
std::shared_ptr< hhg::PrimitiveStorage > storage = std::make_shared< hhg::PrimitiveStorage >( setupStorage );
if( mainConf.getParameter< bool >( "writeDomain" ) )
{
hhg::writeDomainPartitioningVTK( storage, "./output", "domain" );
}
std::shared_ptr< walberla::WcTimingTree > timingTree( new walberla::WcTimingTree() );
storage->setTimingTree( timingTree );
hhg::P1Function< double > oneFunc( "x", storage, level, level );
oneFunc.interpolate( ones, level );
hhg::P1Function< double > x( "x", storage, level, level );
hhg::P1Function< double > y( "y", storage, level, level );
hhg::P1Function< double > z( "z", storage, level, level );
x.interpolate( exact, level, hhg::Inner );
//hhg::communication::syncFunctionBetweenPrimitives(x,level);
hhg::P1ConstantLaplaceOperator mass( storage, level, level );
const uint_t localDoFs = hhg::numberOfLocalDoFs< hhg::P1FunctionTag >( *storage, level );
const uint_t totalDoFs = numberOfGlobalDoFs< hhg::P1FunctionTag >( *storage, level );
WALBERLA_LOG_INFO_ON_ROOT( "totalDoFs: " << totalDoFs );
WALBERLA_LOG_INFO( "localDoFs: " << localDoFs << " totalDoFs: " << totalDoFs );
walberla::WcTimer timer;
timer.reset();
LIKWID_MARKER_START( "HyTeG-apply" );
mass.apply( x, y, level, hhg::Inner );
LIKWID_MARKER_STOP( "HyTeG-apply" );
timer.end();
double hyteg_apply = timer.last();
WALBERLA_LOG_INFO_ON_ROOT( "HyTeG apply runtime: " << hyteg_apply );
if( mainConf.getParameter< bool >( "VTKOutput" ) )
{
WALBERLA_LOG_INFO_ON_ROOT( "writing VTK output" );
hhg::VTKOutput vtkOutput( "./output", "ApplyBenchmark" );
vtkOutput.add( &x );
vtkOutput.add( &z );
vtkOutput.add( &y );
vtkOutput.write( level );
}
walberla::WcTimingTree tt = timingTree->getReduced();
auto tt2 = tt.getCopyWithRemainder();
if( mainConf.getParameter< bool >( "printTiming" ) )
{
WALBERLA_LOG_INFO_ON_ROOT( tt2 );
}
if( mainConf.getParameter< bool >( "writeJSON" ) )
{
nlohmann::json ttjson = nlohmann::json( tt2 );
std::ofstream o( "ApplyBenchmarkOutput.json" );
o << ttjson;
o.close();
}
WALBERLA_LOG_INFO_ON_ROOT( std::scientific << " | " << meshFileName << " | " << level << " | " << totalDoFs << " | " << hyteg_apply << " | " );
LIKWID_MARKER_CLOSE;
}
Parameters
{
level 6;
VTKOutput false;
writeDomain false;
printTiming false;
writeJSON false;
mesh ../../../data/meshes/quad_4el.msh;
}
\ No newline at end of file
if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/output)
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/output")
endif()
waLBerla_link_files_to_builddir( *.prm )
waLBerla_link_files_to_builddir( *.py )
waLBerla_add_executable( NAME ApplyBenchmark
FILES ApplyBenchmark.cpp
DEPENDS tinyhhg_core core)
\ No newline at end of file
......@@ -6,128 +6,134 @@
#include "tinyhhg_core/LikwidWrapper.hpp"
#include "tinyhhg_core/VTKWriter.hpp"
#include "tinyhhg_core/edgedofspace/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/mesh/MeshInfo.hpp"
#include "tinyhhg_core/mixedoperators/EdgeDoFToVertexDoFOperator/EdgeDoFToVertexDoFApply.hpp"
#include "tinyhhg_core/mixedoperators/EdgeDoFToVertexDoFOperator/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/mixedoperators/VertexDoFToEdgeDoFOperator/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/p1functionspace/VertexDoFMacroFace.hpp"
#include "tinyhhg_core/p1functionspace/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/p2functionspace/P2ConstantOperator.hpp"
#include "tinyhhg_core/p2functionspace/P2Function.hpp"
#include "tinyhhg_core/primitivestorage/SetupPrimitiveStorage.hpp"
#include "tinyhhg_core/primitivestorage/loadbalancing/SimpleBalancer.hpp"
#include "tinyhhg_core/p1functionspace/VertexDoFMacroFace.hpp"
#include "tinyhhg_core/mixedoperators/EdgeDoFToVertexDoFOperator/EdgeDoFToVertexDoFApply.hpp"
#include "tinyhhg_core/p1functionspace/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/edgedofspace/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/mixedoperators/EdgeDoFToVertexDoFOperator/generatedKernels/generatedKernels.hpp"
#include "tinyhhg_core/mixedoperators/VertexDoFToEdgeDoFOperator/generatedKernels/generatedKernels.hpp"
const int USE_GENERATED_KERNELS = 1;
using walberla::real_t;
static void performBenchmark( hhg::P2Function< double > & src, hhg::P2Function< double > & dst, hhg::P2ConstantLaplaceOperator & laplace,
const uint_t & level, Face & face, const uint_t & sampleSize, walberla::WcTimingTree & timingTree )
static void performBenchmark( hhg::P2Function< double >& src,
hhg::P2Function< double >& dst,
hhg::P2ConstantLaplaceOperator& laplace,
const uint_t& level,
Face& face,
const uint_t& sampleSize,
walberla::WcTimingTree& timingTree )
{
const std::string benchInfoString = "level" + (level < 10 ? "0" + std::to_string( level ) : std::to_string( level ) ) + "-" + "sampleSize" + std::to_string( sampleSize );
const std::string benchInfoString = "level" + ( level < 10 ? "0" + std::to_string( level ) : std::to_string( level ) ) + "-" +
"sampleSize" + std::to_string( sampleSize ) +
"numProcs" + std::to_string(walberla::mpi::MPIManager::instance()->numProcesses());
std::string name;
/// Vertex to Vertex
for ( uint_t i = 0; i < sampleSize; i++ )
{
name = "Vertex-to-Vertex-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if ( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getVertexToVertexOpr().getFaceStencilID() )->getPointer( level );
hhg::vertexdof::macroface::generated::applyReplace( dstPtr, srcPtr, stencilPtr, level );
}
else
{
hhg::vertexdof::macroface::apply( level, face,
laplace.getVertexToVertexOpr().getFaceStencilID(),
src.getVertexDoFFunction()->getFaceDataID(),
dst.getVertexDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Edge to Vertex
for ( uint_t i = 0; i < sampleSize; i++ )
{
name = "Edge-to-Vertex-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if ( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getEdgeToVertexOpr().getFaceStencilID() )->getPointer( level );
hhg::EdgeDoFToVertexDoF::generated::applyFaceReplace( dstPtr, srcPtr, stencilPtr, level );
}
else
{
hhg::EdgeDoFToVertexDoF::applyFace( level, face,
laplace.getEdgeToVertexOpr().getFaceStencilID(),
src.getEdgeDoFFunction()->getFaceDataID(),
dst.getVertexDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Edge to Edge
for ( uint_t i = 0; i < sampleSize; i++ )
{
name = "Edge-to-Edge-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if ( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getEdgeToEdgeOpr().getFaceStencilID() )->getPointer( level );
hhg::edgedof::macroface::generated::applyReplace( dstPtr, srcPtr, stencilPtr, level );
}
else
{
hhg::edgedof::macroface::apply( level, face,
laplace.getEdgeToEdgeOpr().getFaceStencilID(),
src.getEdgeDoFFunction()->getFaceDataID(),
dst.getEdgeDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Vertex to Edge
for ( uint_t i = 0; i < sampleSize; i++ )
{
name = "Vertex-to-Edge-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if ( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getVertexToEdgeOpr().getFaceStencilID() )->getPointer( level );
hhg::VertexDoFToEdgeDoF::generated::applyFaceReplace( dstPtr, srcPtr, stencilPtr, level );
}
else
{
hhg::VertexDoFToEdgeDoF::applyFace( level, face,
laplace.getEdgeToVertexOpr().getFaceStencilID(),
src.getVertexDoFFunction()->getFaceDataID(),
dst.getEdgeDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Vertex to Vertex
for( uint_t i = 0; i < sampleSize; i++ )
{
name = "Vertex-to-Vertex-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getVertexToVertexOpr().getFaceStencilID() )->getPointer( level );
hhg::vertexdof::macroface::generated::applyReplace( dstPtr, srcPtr, stencilPtr, level );
} else
{
hhg::vertexdof::macroface::apply( level,
face,
laplace.getVertexToVertexOpr().getFaceStencilID(),
src.getVertexDoFFunction()->getFaceDataID(),
dst.getVertexDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Edge to Vertex
for( uint_t i = 0; i < sampleSize; i++ )
{
name = "Edge-to-Vertex-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getEdgeToVertexOpr().getFaceStencilID() )->getPointer( level );
hhg::EdgeDoFToVertexDoF::generated::applyFaceReplace( dstPtr, srcPtr, stencilPtr, level );
} else
{
hhg::EdgeDoFToVertexDoF::applyFace( level,
face,
laplace.getEdgeToVertexOpr().getFaceStencilID(),
src.getEdgeDoFFunction()->getFaceDataID(),
dst.getVertexDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Edge to Edge
for( uint_t i = 0; i < sampleSize; i++ )
{
name = "Edge-to-Edge-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getEdgeToEdgeOpr().getFaceStencilID() )->getPointer( level );
hhg::edgedof::macroface::generated::applyReplace( dstPtr, srcPtr, stencilPtr, level );
} else
{
hhg::edgedof::macroface::apply( level,
face,
laplace.getEdgeToEdgeOpr().getFaceStencilID(),
src.getEdgeDoFFunction()->getFaceDataID(),
dst.getEdgeDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
/// Vertex to Edge
for( uint_t i = 0; i < sampleSize; i++ )
{
name = "Vertex-to-Edge-Apply-" + benchInfoString;
timingTree.start( name );
LIKWID_MARKER_START( name.c_str() );
if( USE_GENERATED_KERNELS )
{
auto dstPtr = face.getData( dst.getEdgeDoFFunction()->getFaceDataID() )->getPointer( level );
auto srcPtr = face.getData( src.getVertexDoFFunction()->getFaceDataID() )->getPointer( level );
auto stencilPtr = face.getData( laplace.getVertexToEdgeOpr().getFaceStencilID() )->getPointer( level );
hhg::VertexDoFToEdgeDoF::generated::applyFaceReplace( dstPtr, srcPtr, stencilPtr, level );
} else
{
hhg::VertexDoFToEdgeDoF::applyFace( level,
face,
laplace.getEdgeToVertexOpr().getFaceStencilID(),
src.getVertexDoFFunction()->getFaceDataID(),
dst.getEdgeDoFFunction()->getFaceDataID(),
hhg::Replace );
}
LIKWID_MARKER_STOP( name.c_str() );
timingTree.stop( name );
}
}
int main( int argc, char* argv[] )
......@@ -155,20 +161,16 @@ int main( int argc, char* argv[] )
}
const walberla::Config::BlockHandle mainConf = cfg->getBlock( "Parameters" );
const uint_t minLevel = mainConf.getParameter< uint_t >( "minLevel" );
const uint_t maxLevel = mainConf.getParameter< uint_t >( "maxLevel" );
const std::string meshFileName = mainConf.getParameter< std::string >( "mesh" );
const uint_t level = mainConf.getParameter< uint_t >( "level" );
///// Mesh /////
hhg::MeshInfo meshInfo = hhg::MeshInfo::fromGmshFile( meshFileName );
hhg::MeshInfo meshInfo = hhg::MeshInfo::meshFaceChain( uint_c( walberla::MPIManager::instance()->numProcesses() ) );
hhg::SetupPrimitiveStorage setupStorage( meshInfo, walberla::uint_c( walberla::mpi::MPIManager::instance()->numProcesses() ) );
hhg::loadbalancing::roundRobin( setupStorage );
std::shared_ptr< hhg::PrimitiveStorage > storage = std::make_shared< hhg::PrimitiveStorage >( setupStorage );
std::function< real_t( const hhg::Point3D& ) > exact = []( const hhg::Point3D& xx ) {
return std::sin( walberla::math::PI * xx[0] ) + std::cos( walberla::math::PI * xx[1] );
return std::sin( walberla::math::PI * xx[0] ) + std::cos( walberla::math::PI * xx[1] );
};
std::shared_ptr< walberla::WcTimingTree > timingTree( new walberla::WcTimingTree() );
......@@ -176,15 +178,12 @@ int main( int argc, char* argv[] )
///// Functions / operators / allocation /////
hhg::P2Function< double > src( "src", storage, minLevel, maxLevel );
hhg::P2Function< double > dst( "dst", storage, minLevel, maxLevel );
hhg::P2Function< double > src( "src", storage, level, level );
hhg::P2Function< double > dst( "dst", storage, level, level );
hhg::P2ConstantLaplaceOperator laplace( storage, minLevel, maxLevel );
hhg::P2ConstantLaplaceOperator laplace( storage, level, level );
for ( uint_t level = minLevel; level <= maxLevel; level++ )
{
src.interpolate( exact, level, hhg::Inner );
}
src.interpolate( exact, level, hhg::Inner );
///// Apply benchmarks /////
......@@ -192,34 +191,34 @@ int main( int argc, char* argv[] )
WALBERLA_CHECK_EQUAL( macroFaces.size(), 1 );
auto face = storage->getFace( macroFaces.front() );
for ( uint_t level = minLevel; level <= maxLevel; level++ )
WALBERLA_CHECK_LESS_EQUAL( level, 14 );
uint_t sampleSize = ( 1u << ( 14u - level ) );
performBenchmark( src, dst, laplace, level, *face, sampleSize, wcTimingTreeApp );
if( mainConf.getParameter< bool >( "printTiming" ) )
{
WALBERLA_CHECK_LESS_EQUAL( maxLevel, 14 );
uint_t sampleSize = (1 << (14 - level));
performBenchmark( src, dst, laplace, level, *face, sampleSize, wcTimingTreeApp );
auto wcTPReduced = wcTimingTreeApp.getReduced();
WALBERLA_LOG_INFO_ON_ROOT( wcTPReduced );
walberla::WcTimingTree tt = timingTree->getReduced();
auto tt2 = tt.getCopyWithRemainder();
WALBERLA_LOG_INFO_ON_ROOT( tt2 );
nlohmann::json ttJson;
walberla::timing::to_json( ttJson, wcTPReduced );
std::ofstream jsonOutput;
jsonOutput.open( "ApplyPerformanceAnalysis-2D-P2.json" );
jsonOutput << ttJson.dump( 4 );
jsonOutput.close();
}
auto wcTPReduced = wcTimingTreeApp.getReduced();
WALBERLA_LOG_INFO_ON_ROOT( wcTPReduced );
walberla::WcTimingTree tt = timingTree->getReduced();
auto tt2 = tt.getCopyWithRemainder();
WALBERLA_LOG_INFO_ON_ROOT(tt2);
nlohmann::json ttJson;
walberla::timing::to_json( ttJson, wcTPReduced );
std::ofstream jsonOutput;
jsonOutput.open ( "ApplyPerformanceAnalysis-2D-P2.json" );
jsonOutput << ttJson.dump(4);
jsonOutput.close();
if( mainConf.getParameter< bool >( "VTKOutput" ) )
{
WALBERLA_LOG_INFO_ON_ROOT( "Writing VTK output" );
hhg::VTKOutput vtkOutput( "./output", "ApplyPerformanceAnalysis-2D-P2.cpp" );
vtkOutput.add( &src );
vtkOutput.add( &dst );
vtkOutput.write( maxLevel );
vtkOutput.write( level );
}
LIKWID_MARKER_CLOSE;
......
Parameters
{
minLevel 9;
maxLevel 12;
sampleSize 10;
level 9;
VTKOutput false;
mesh ../../../data/meshes/tri_1el.msh;
/// printTiming and write json
printTiming false;
useMeshFile false;
mesh ../../../data/meshes/quad_4el.msh;
/// only relevant if useMeshFile == false
/// if set true the total number of faces is mpi.numProcesses * numberOfFaces
/// if set to false the total number is just numberOfFaces
facesTimesProcs true;
numberOfFaces 1;
}
\ No newline at end of file
import csv
import subprocess
import socket
import collections
#import matplotlib.pyplot as plt
def main():
#procs = range(20)
totalProcs = range(2,20)
if "emmy" in socket.gethostname():
subprocess.call(["source", "~/script/modules/intel.emmy.modules"])
f = open('createScaling.txt','w')
for i in totalProcs:
subprocess.call(["likwid-mpirun", "-np", str(i), "-g", "MEM", "-m", "-O",
"./ApplyPerformanceAnalysis-2D-P2", "ApplyPerformanceAnalysis-2D-P2.prm",
"-Parameters.level=11"], stdout=f)
f.close()
if __name__ == "__main__":
main()
import csv
import subprocess
import socket
import collections
import matplotlib.pyplot as plt
def main():