Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
hyteg
hyteg
Commits
12fb773e
Commit
12fb773e
authored
Sep 06, 2021
by
Marcel Koch
Browse files
add support for ginkgo ressource manager
parent
9fb874a3
Pipeline
#34039
failed with stages
in 23 minutes and 51 seconds
Changes
9
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
CMakeLists.txt
View file @
12fb773e
...
...
@@ -50,8 +50,9 @@ endif()
if
(
HYTEG_BUILD_WITH_GINKGO
)
find_package
(
Ginkgo REQUIRED
)
find_package
(
RapidJSON REQUIRED
)
message
(
STATUS
"Found Ginkgo! Ginkgo_DIR =
${
Ginkgo_DIR
}
"
)
list
(
APPEND SERVICE_LIBS Ginkgo::ginkgo
)
list
(
APPEND SERVICE_LIBS Ginkgo::ginkgo
rapidjson
)
endif
()
find_package
(
waLBerla
)
...
...
apps/2021-tokamak/CMakeLists.txt
View file @
12fb773e
...
...
@@ -10,6 +10,8 @@ waLBerla_link_files_to_builddir( *.prm )
waLBerla_link_files_to_builddir
(
*.py
)
waLBerla_link_files_to_builddir
(
*.job
)
waLBerla_link_files_to_builddir
(
*.sh
)
waLBerla_link_files_to_builddir
(
gko.json
)
waLBerla_add_executable
(
NAME Tokamak
FILES Tokamak.cpp
...
...
apps/2021-tokamak/Tokamak.cpp
View file @
12fb773e
...
...
@@ -73,6 +73,7 @@ struct SolverSettings
uint_t
maxCoarseGridSolverIterations
{};
bool
cgHytegVerbose
{};
std
::
string
gkoExecutor
;
std
::
string
gkoConfig
;
bool
useAgglomeration
;
[[
nodiscard
]]
std
::
string
toString
()
const
...
...
@@ -90,6 +91,7 @@ struct SolverSettings
if
(
coarseGridSolverType
==
COARSE_GRID_CG_GINKGO
)
{
ss
<<
" - ginkgo executor: "
<<
gkoExecutor
<<
"
\n
"
;
ss
<<
" - ginkgo config file: "
<<
gkoConfig
<<
"
\n
"
;
}
if
(
useAgglomeration
)
{
...
...
@@ -250,6 +252,7 @@ void tokamak( TokamakDomain tokamakDomain,
db
->
setConstantEntry
(
"solverType"
,
solverSettings
.
solverType
);
db
->
setConstantEntry
(
"coarseGridSolverType"
,
solverSettings
.
coarseGridSolverType
);
db
->
setConstantEntry
(
"gkoExecutor"
,
solverSettings
.
gkoExecutor
);
db
->
setConstantEntry
(
"gkoConfig"
,
solverSettings
.
gkoConfig
);
db
->
setConstantEntry
(
"maxCoarseGridSolverIterations"
,
solverSettings
.
maxCoarseGridSolverIterations
);
db
->
setConstantEntry
(
"cgHytegVerbose"
,
solverSettings
.
cgHytegVerbose
);
db
->
setConstantEntry
(
"relativeResidualReduction"
,
solverSettings
.
relativeResidualReduction
);
...
...
@@ -600,14 +603,10 @@ void tokamak( TokamakDomain tokamakDomain,
{
WALBERLA_ABORT
(
"Invalid Ginkgo executor type."
);
}
auto
actualCoarseGridSolver
=
std
::
make_shared
<
GinkgoCGSolver
<
LaplaceOperator_T
>
>
(
coarse_storage
,
minLevel
,
constraints
::
zero_row
,
relativeResidualToleranceCoarseGrid
,
absoluteResidualToleranceCoarseGrid
,
maxIterationsCoarseGrid
,
exec
);
std
::
ifstream
gko_cfg_file
(
solverSettings
.
gkoConfig
);
rapidjson
::
IStreamWrapper
gko_cfg
(
gko_cfg_file
);
auto
actualCoarseGridSolver
=
std
::
make_shared
<
GinkgoCGSolver
<
LaplaceOperator_T
>
>
(
coarse_storage
,
minLevel
,
constraints
::
zero_row
,
exec
,
gko_cfg
);
if
(
solverSettings
.
cgHytegVerbose
)
{
actualCoarseGridSolver
->
setPrintInfo
(
true
);
...
...
@@ -866,6 +865,7 @@ void run( int argc, char** argv )
solverSettings
.
postSmooth
=
mainConf
.
getParameter
<
uint_t
>
(
"postSmooth"
);
solverSettings
.
cgHytegVerbose
=
mainConf
.
getParameter
<
bool
>
(
"cgHytegVerbose"
);
solverSettings
.
gkoExecutor
=
mainConf
.
getParameter
<
std
::
string
>
(
"gkoExecutor"
);
solverSettings
.
gkoConfig
=
mainConf
.
getParameter
<
std
::
string
>
(
"gkoConfig"
);
solverSettings
.
useAgglomeration
=
mainConf
.
getParameter
<
bool
>
(
"useAgglomeration"
);
// use numeric_limits<int> here to prevent overflow when casting to PetscINt
solverSettings
.
maxCoarseGridSolverIterations
=
...
...
apps/2021-tokamak/Tokamak.prm
View file @
12fb773e
...
...
@@ -6,8 +6,8 @@ Parameters
elementType p1;
minLevel
3
;
maxLevel
3
;
minLevel
0
;
maxLevel
0
;
// ##########
// # domain #
...
...
@@ -65,6 +65,7 @@ Parameters
relativeResidualReduction 1e-06;
maxCoarseGridSolverIterations 1000000000;
gkoExecutor cuda;
gkoConfig gko.json;
useAgglomeration false;
// ################
...
...
apps/2021-tokamak/gko.json
0 → 100644
View file @
12fb773e
{
"base"
:
"Cg"
,
"exec"
:
"inherit"
,
"generate"
:
"given"
,
"factory"
:
{
"exec"
:
"inherit"
,
"preconditioner"
:
{
"base"
:
"JacobiFactory"
,
"exec"
:
"inherit"
,
"max_block_size"
:
1
,
"generate"
:
"given"
},
"criteria"
:
[
{
"base"
:
"Iteration"
,
"max_iters"
:
1000
,
"exec"
:
"inherit"
},
{
"base"
:
"ResidualNorm"
,
"baseline"
:
"absolute"
,
"reduction_factor"
:
1e-12
},
{
"base"
:
"ResidualNorm"
,
"baseline"
:
"rhs_norm"
,
"reduction_factor"
:
1e-30
}
]
}
}
\ No newline at end of file
src/hyteg/ginkgo/GinkgoCGSolver.hpp
View file @
12fb773e
...
...
@@ -12,6 +12,7 @@
#include
"hyteg/elementwiseoperators/DiagonalNonConstantOperator.hpp"
#include
"hyteg/elementwiseoperators/ElementwiseOperatorPetsc.hpp"
#include
"hyteg/ginkgo/GinkgoSparseMatrixProxy.hpp"
#include
"hyteg/ginkgo/GinkgoUtilities.hpp"
#include
"hyteg/ginkgo/GinkgoVectorProxy.hpp"
#include
"hyteg/p1functionspace/P1Petsc.hpp"
#include
"hyteg/p2functionspace/P2Petsc.hpp"
...
...
@@ -31,6 +32,9 @@
#include
"ginkgo/core/solver/cg.hpp"
#include
"ginkgo/core/stop/iteration.hpp"
#include
"ginkgo/core/stop/residual_norm.hpp"
#include
"rapidjson/error/en.h"
#include
"rapidjson/pointer.h"
#include
"resource_manager/resource_manager.hpp"
namespace
hyteg
{
...
...
@@ -349,10 +353,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
GinkgoCGSolver
(
const
std
::
shared_ptr
<
PrimitiveStorage
>&
storage
,
const
uint_t
&
level
,
const
constraints
constraints_type
,
const
valueType
relTolerance
=
1e-12
,
const
valueType
absTolerance
=
1e-30
,
const
int
maxIterations
=
1000
,
std
::
shared_ptr
<
gko
::
Executor
>
solver_exec
=
gko
::
ReferenceExecutor
::
create
()
)
std
::
shared_ptr
<
gko
::
Executor
>
solver_exec
)
:
storage_
(
storage
)
,
level_
(
level
)
,
comm_
(
gko
::
mpi
::
communicator
::
create
(
storage
->
getSplitCommunicatorByPrimitiveDistribution
()
)
)
...
...
@@ -361,26 +362,62 @@ class GinkgoCGSolver : public Solver< OperatorType >
,
host_exec_
(
solver_exec
->
get_master
()
)
,
solver_exec_
(
std
::
move
(
solver_exec
)
)
,
num_
(
"numerator"
,
storage
,
level
,
level
)
,
item_
(){
rapidjson
::
StringStream
json_stream
{
hyteg
::
default_json_cg
.
data
()};
if
(
item_
.
ParseStream
(
json_stream
).
HasParseError
()
)
{
fprintf
(
stderr
,
"
\n
Error(offset %u): %s
\n
"
,
(
unsigned
)
item_
.
GetErrorOffset
(),
rapidjson
::
GetParseError_En
(
item_
.
GetParseError
()
)
);
}
if
(
constraints_type
==
constraints
::
penalty
)
{
auto
*
criteria
=
rapidjson
::
Pointer
(
"/factory/criteria"
).
Get
(
item_
);
for
(
auto
&
crit
:
criteria
->
GetArray
()
)
{
if
(
crit
[
"base"
]
==
"ResidualNorm"
&&
crit
[
"baseline"
]
==
"rhs_norm"
)
{
crit
[
"baseline"
]
=
"initial_resnorm"
;
}
}
}
}
template
<
typename
Stream
>
GinkgoCGSolver
(
const
std
::
shared_ptr
<
PrimitiveStorage
>&
storage
,
const
uint_t
&
level
,
const
constraints
constraints_type
,
std
::
shared_ptr
<
gko
::
Executor
>
solver_exec
,
Stream
&
json_stream
)
:
storage_
(
storage
)
,
level_
(
level
)
,
comm_
(
gko
::
mpi
::
communicator
::
create
(
storage
->
getSplitCommunicatorByPrimitiveDistribution
()
)
)
,
part_
()
,
constraints_type_
(
constraints_type
)
,
host_exec_
(
solver_exec
->
get_master
()
)
,
solver_exec_
(
std
::
move
(
solver_exec
)
)
,
num_
(
"numerator"
,
storage
,
level
,
level
)
,
item_
()
{
auto
rel_mode
=
constraints_type
==
constraints
::
penalty
?
gko
::
stop
::
mode
::
initial_resnorm
:
gko
::
stop
::
mode
::
rhs_norm
;
auto
log_cout
=
gko
::
share
(
gko
::
log
::
Stream
<
valueType
>::
create
(
host_exec_
,
gko
::
log
::
Logger
::
criterion_check_completed_mask
,
std
::
cout
,
true
)
);
auto
log
=
gko
::
share
(
gko
::
log
::
Convergence
<
valueType
>::
create
(
solver_exec_
)
);
auto
criteria
=
gko
::
stop
::
Combined
::
build
()
.
with_criteria
(
gko
::
stop
::
ResidualNorm
<
valueType
>::
build
()
.
with_baseline
(
rel_mode
)
.
with_reduction_factor
(
relTolerance
)
.
on
(
solver_exec_
),
gko
::
stop
::
ResidualNorm
<
valueType
>::
build
()
.
with_baseline
(
gko
::
stop
::
mode
::
absolute
)
.
with_reduction_factor
(
absTolerance
)
.
on
(
solver_exec_
),
gko
::
stop
::
Iteration
::
build
().
with_max_iters
(
maxIterations
).
on
(
solver_exec_
)
)
.
on
(
solver_exec_
);
criteria
->
add_logger
(
log
);
//criteria->add_logger( log_cout );
solver_factory_
=
gko
::
solver
::
Cg
<
valueType
>::
build
().
with_criteria
(
gko
::
share
(
criteria
)
).
on
(
solver_exec_
);
if
(
item_
.
ParseStream
(
json_stream
).
HasParseError
()
)
{
fprintf
(
stderr
,
"
\n
Error(offset %u): %s
\n
"
,
(
unsigned
)
item_
.
GetErrorOffset
(),
rapidjson
::
GetParseError_En
(
item_
.
GetParseError
()
)
);
}
if
(
constraints_type
==
constraints
::
penalty
)
{
auto
*
criteria
=
rapidjson
::
Pointer
(
"/factory/criteria"
).
Get
(
item_
);
for
(
auto
&
crit
:
criteria
->
GetArray
()
)
{
if
(
crit
[
"base"
]
==
"ResidualNorm"
&&
crit
[
"baseline"
]
==
"rhs_norm"
)
{
crit
[
"baseline"
]
=
"initial_resnorm"
;
}
}
}
}
void
solve
(
const
OperatorType
&
A
,
const
FunctionType
&
x
,
const
FunctionType
&
b
,
const
walberla
::
uint_t
level
)
override
...
...
@@ -469,19 +506,27 @@ class GinkgoCGSolver : public Solver< OperatorType >
matrix_
=
csr
::
create
(
solver_exec_
);
host_matrix_
->
convert_to
(
gko
::
lend
(
matrix_
)
);
solver_
=
solver_factory_
->
generate
(
matrix_
);
if
(
matrix_
->
get_size
()[
0
]
>
0
&&
matrix_
->
get_size
()[
1
]
>
0
)
{
// auto ilu = gko::preconditioner::Ilu<>::build()
// .with_factorization_factory(
// gko::share( gko::factorization::Ilu< valueType, int32_t >::build().on( solver_exec_ ) ) )
// .on( solver_exec_ )
// ->generate( matrix_ );
auto
jac
=
gko
::
preconditioner
::
Jacobi
<
valueType
,
int32_t
>::
build
()
.
with_max_block_size
(
1
)
.
on
(
solver_exec_
)
->
generate
(
matrix_
);
solver_
->
set_preconditioner
(
gko
::
share
(
jac
)
);
auto
old_buffer
=
std
::
cout
.
rdbuf
();
std
::
ofstream
dev_null
(
"/dev/null"
);
std
::
cout
.
rdbuf
(
dev_null
.
rdbuf
()
);
solver_
=
gko
::
as
<
gko
::
solver
::
Cg
<
valueType
>
>
(
gko
::
extension
::
resource_manager
::
create_from_config
<
gko
::
LinOp
>
(
item_
,
"Cg"
,
solver_exec_
,
matrix_
,
nullptr
)
);
std
::
cout
.
rdbuf
(
old_buffer
);
auto
log
=
gko
::
share
(
gko
::
log
::
Convergence
<
valueType
>::
create
(
solver_exec_
)
);
auto
factory
=
gko
::
clone
(
solver_
->
get_stop_criterion_factory
()
);
factory
->
add_logger
(
log
);
solver_
->
set_stop_criterion_factory
(
gko
::
share
(
factory
)
);
}
else
{
solver_
=
gko
::
solver
::
Cg
<
valueType
>::
build
()
.
with_criteria
(
gko
::
stop
::
Iteration
::
build
().
on
(
solver_exec_
)
)
.
on
(
solver_exec_
)
->
generate
(
matrix_
);
}
x
.
getStorage
()
->
getTimingTree
()
->
stop
(
"Ginkgo CG Solver Set-Up"
);
}
...
...
@@ -510,7 +555,7 @@ class GinkgoCGSolver : public Solver< OperatorType >
x
.
getStorage
()
->
getTimingTree
()
->
stop
(
"Ginkgo CG Solver"
);
if
(
printInfo_
)
if
(
printInfo_
&&
rank
==
0
)
{
auto
log
=
gko
::
as
<
gko
::
log
::
Convergence
<
valueType
>
>
(
solver_
->
get_stop_criterion_factory
()
->
get_loggers
()[
0
]
);
...
...
@@ -544,13 +589,14 @@ class GinkgoCGSolver : public Solver< OperatorType >
std
::
shared_ptr
<
gko
::
Executor
>
solver_exec_
;
std
::
unique_ptr
<
typename
gko
::
solver
::
Cg
<
valueType
>::
Factory
>
solver_factory_
;
std
::
unique
_ptr
<
typename
gko
::
solver
::
Cg
<
valueType
>
>
solver_
;
std
::
shared
_ptr
<
typename
gko
::
solver
::
Cg
<
valueType
>
>
solver_
;
std
::
shared_ptr
<
mtx
>
host_matrix_
;
std
::
shared_ptr
<
csr
>
matrix_
;
bool
printInfo_
=
false
;
bool
reassembleMatrix_
=
false
;
bool
printInfo_
=
false
;
bool
reassembleMatrix_
=
false
;
rapidjson
::
Document
item_
;
};
}
// namespace hyteg
\ No newline at end of file
src/hyteg/ginkgo/GinkgoUtilities.hpp
View file @
12fb773e
...
...
@@ -84,4 +84,37 @@ std::shared_ptr< gko::Executor > get_executor( std::string exec_name, int device
};
return
get_executor
(
exec_name_to_tag
.
at
(
exec_name
),
device_id
);
}
constexpr
std
::
string_view
default_json_cg
{
R"(
{
"base": "Cg",
"exec": "inherit",
"generate": "given",
"factory": {
"exec": "inherit",
"preconditioner": {
"base": "JacobiFactory",
"exec": "inherit",
"generate": "given"
},
"criteria": [
{
"base": "Iteration",
"max_iters": 1000,
"exec": "inherit"
},
{
"base": "ResidualNorm",
"baseline": "absolute",
"reduction_factor": 1e-12
},
{
"base": "ResidualNorm",
"baseline": "rhs_norm",
"reduction_factor": 1e-30
}
]
}
}
)"
};
}
// namespace hyteg
tests/hyteg/P1/P1GinkgoSolveTest.cpp
View file @
12fb773e
...
...
@@ -17,6 +17,7 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include
<sstream>
#include
"core/Environment.h"
#include
"core/logging/Logging.h"
#include
"core/math/Random.h"
...
...
@@ -90,11 +91,15 @@ void ginkgoSolveTest( const uint_t& level,
WALBERLA_LOG_INFO
(
"localDoFs: "
<<
localDoFs
<<
" globalDoFs: "
<<
globalDoFs
);
GinkgoCGSolver
<
Op
>
solver
{
storage
,
level
,
constraints
::
zero_row
,
1e-30
,
1e-12
,
1000
,
exec
};
solver
.
setPrintInfo
(
true
);
std
::
shared_ptr
<
hyteg
::
Solver
<
Op
>>
solver
;
{
auto
real_solver
=
std
::
make_shared
<
GinkgoCGSolver
<
Op
>>
(
storage
,
level
,
constraints
::
zero_row
,
exec
);
real_solver
->
setPrintInfo
(
true
);
solver
=
real_solver
;
}
walberla
::
WcTimer
timer
;
solver
.
solve
(
A
,
x
,
b
,
level
);
solver
->
solve
(
A
,
x
,
b
,
level
);
timer
.
end
();
WALBERLA_LOG_INFO_ON_ROOT
(
"time was: "
<<
timer
.
last
()
);
...
...
@@ -132,16 +137,16 @@ int main( int argc, char* argv[] )
for
(
auto
tag
:
tag_list
)
{
auto
exec
=
get_executor
(
tag
);
if
(
exec
)
if
(
exec
&&
tag
==
exec_tag
::
reference
)
{
try
{
ginkgoSolveTest
(
3
,
"../../data/meshes/quad_2el.msh"
,
2.5e-05
,
exec
);
ginkgoSolveTest
(
3
,
"../../data/meshes/quad_4el.msh"
,
1.4e-05
,
exec
);
ginkgoSolveTest
(
3
,
"../../data/meshes/3D/tet_1el.msh"
,
4.9e-07
,
exec
);
ginkgoSolveTest
(
3
,
"../../data/meshes/3D/pyramid_2el.msh"
,
4.9e-05
,
exec
);
ginkgoSolveTest
(
3
,
"../../data/meshes/3D/pyramid_4el.msh"
,
1.5e-05
,
exec
);
ginkgoSolveTest
(
3
,
"../../data/meshes/3D/regular_octahedron_8el.msh"
,
2.3e-04
,
exec
);
//
ginkgoSolveTest( 3, "../../data/meshes/quad_4el.msh", 1.4e-05, exec );
//
ginkgoSolveTest( 3, "../../data/meshes/3D/tet_1el.msh", 4.9e-07, exec );
//
ginkgoSolveTest( 3, "../../data/meshes/3D/pyramid_2el.msh", 4.9e-05, exec );
//
ginkgoSolveTest( 3, "../../data/meshes/3D/pyramid_4el.msh", 1.5e-05, exec );
//
ginkgoSolveTest( 3, "../../data/meshes/3D/regular_octahedron_8el.msh", 2.3e-04, exec );
}
catch
(
const
gko
::
NotImplemented
&
e
){
std
::
cout
<<
e
.
what
()
<<
" for executor "
<<
get_executor_name
(
exec
)
<<
std
::
endl
;
}
...
...
tests/hyteg/adaptivity/AgglomerationConvergenceTest.cpp
View file @
12fb773e
...
...
@@ -24,6 +24,7 @@
#include
"hyteg/dataexport/VTKOutput.hpp"
#include
"hyteg/ginkgo/GinkgoCGSolver.hpp"
#include
"hyteg/ginkgo/GinkgoUtilities.hpp"
#include
"hyteg/gridtransferoperators/P2toP2QuadraticProlongation.hpp"
#include
"hyteg/gridtransferoperators/P2toP2QuadraticRestriction.hpp"
#include
"hyteg/p2functionspace/P2ConstantOperator.hpp"
...
...
@@ -137,8 +138,9 @@ void AgglomerationConvergenceTest( const std::string& meshFile,
#ifdef HYTEG_BUILD_WITH_GINKGO
if
(
cgc_solver_type
==
cgc_solver
::
ginkgo
)
{
rapidjson
::
StringStream
json
(
hyteg
::
default_json_cg
.
data
());
coarseGridSolver
=
std
::
make_shared
<
GinkgoCGSolver
<
P2ConstantLaplaceOperator
>
>
(
agglomerationStorage
,
minLevel
,
constraints
::
zero_row
);
agglomerationStorage
,
minLevel
,
constraints
::
zero_row
,
gko
::
ReferenceExecutor
::
create
(),
json
);
}
#endif
...
...
@@ -210,12 +212,12 @@ int main( int argc, char* argv[] )
walberla
::
MPIManager
::
instance
()
->
useWorldComm
();
WALBERLA_LOG_INFO_ON_ROOT
(
"Start test with HyTeg coarse grid solver."
)
runTests
(
cgc_solver
::
hyteg
);
//
runTests( cgc_solver::hyteg );
#ifdef HYTEG_BUILD_WITH_PETSC
WALBERLA_LOG_INFO_ON_ROOT
(
"Start test with PETSc coarse grid solver."
)
hyteg
::
PETScManager
petscManager
(
&
argc
,
&
argv
);
runTests
(
cgc_solver
::
petsc
);
//
runTests( cgc_solver::petsc );
#endif
#ifdef HYTEG_BUILD_WITH_GINKGO
WALBERLA_LOG_INFO_ON_ROOT
(
"Start test with Ginkgo coarse grid solver."
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment