From 8eaac6a1b43bf8e37e0e9bb4122138ee9902ce15 Mon Sep 17 00:00:00 2001
From: Martin Bauer <martin.bauer@fau.de>
Date: Fri, 1 Dec 2017 14:15:57 +0100
Subject: [PATCH] Updated jobscript generation scripts
---
python/waLBerla/tools/jobscripts/__init__.py | 7 ++-
python/waLBerla/tools/jobscripts/hornet.py | 2 +-
.../tools/jobscripts/pizdaint_hybrid.job | 16 +++++
.../tools/jobscripts/pizdaint_hybrid.py | 61 +++++++++++++++++++
python/waLBerla/tools/jobscripts/supermuc.py | 6 +-
5 files changed, 85 insertions(+), 7 deletions(-)
create mode 100644 python/waLBerla/tools/jobscripts/pizdaint_hybrid.job
create mode 100644 python/waLBerla/tools/jobscripts/pizdaint_hybrid.py
diff --git a/python/waLBerla/tools/jobscripts/__init__.py b/python/waLBerla/tools/jobscripts/__init__.py
index b52cb6f69..0a006cd32 100644
--- a/python/waLBerla/tools/jobscripts/__init__.py
+++ b/python/waLBerla/tools/jobscripts/__init__.py
@@ -3,7 +3,7 @@
"""
from __future__ import print_function, absolute_import, division, unicode_literals
-
+from datetime import timedelta
from waLBerla.tools.jobscripts.hornet import createJobscript as _cr_hornet
from waLBerla.tools.jobscripts.supermuc import createJobscript as _cr_supermuc
from waLBerla.tools.jobscripts.supermuc_phase2 import createJobscript as _cr_supermuc2
@@ -35,7 +35,10 @@ def createJobscript(*args, **kwargs):
"""
if 'machine' not in kwargs:
raise ValueError("Specify which machine to use with 'machine=<supermuc,juqueen,hornet>'")
-
+
+ if 'wall_time' in kwargs and isinstance(kwargs['wall_time'], int):
+ kwargs['wall_time'] = timedelta(seconds=kwargs['wall_time'])
+
if kwargs['machine'].lower() == 'supermuc': return _cr_supermuc ( *args, **kwargs )
if kwargs['machine'].lower() == 'supermuc_phase2': return _cr_supermuc2 ( *args, **kwargs )
if kwargs['machine'].lower() == 'juqueen' : return _cr_juqueen ( *args, **kwargs )
diff --git a/python/waLBerla/tools/jobscripts/hornet.py b/python/waLBerla/tools/jobscripts/hornet.py
index 111335cda..af74febf0 100644
--- a/python/waLBerla/tools/jobscripts/hornet.py
+++ b/python/waLBerla/tools/jobscripts/hornet.py
@@ -9,7 +9,7 @@ def createJobscript( wall_time = None, nodes = None, cores = None, job_class = N
initial_dir = '~', job_name="waLBerla", hyperthreading = 1,
exe_name = None, arguments = [], commands = [], **kwargs ):
- if type(hyperthreading) == bool:
+ if type(hyperthreading) is bool:
hyperthreading = 2 if hyperthreading else 1
diff --git a/python/waLBerla/tools/jobscripts/pizdaint_hybrid.job b/python/waLBerla/tools/jobscripts/pizdaint_hybrid.job
new file mode 100644
index 000000000..a46019185
--- /dev/null
+++ b/python/waLBerla/tools/jobscripts/pizdaint_hybrid.job
@@ -0,0 +1,16 @@
+#!/bin/bash -l
+#SBATCH --job-name={job_name}
+#SBATCH --time={wall_time}
+#SBATCH --nodes={nodes}
+#SBATCH --ntasks-per-core={tasks_per_core}
+#SBATCH --ntasks-per-node={tasks_per_node}
+#SBATCH --cpus-per-task={cpus_per_task}
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+
+module load daint-gpu
+
+export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
+export CRAY_CUDA_MPS=1
+
+
diff --git a/python/waLBerla/tools/jobscripts/pizdaint_hybrid.py b/python/waLBerla/tools/jobscripts/pizdaint_hybrid.py
new file mode 100644
index 000000000..b9d369cc0
--- /dev/null
+++ b/python/waLBerla/tools/jobscripts/pizdaint_hybrid.py
@@ -0,0 +1,61 @@
+from __future__ import print_function, absolute_import, division, unicode_literals
+
+import os
+import math
+
+
+
+def createJobscript( wall_time = None, nodes = None, cores = None, initial_dir=None, job_name="waLBerla",
+ exe_name=None, parameter_files=[], commands=[], hyperthreading=1, **kwargs ):
+ if type(hyperthreading) is bool:
+ hyperthreading = 2 if hyperthreading else 1
+
+ CORES_PER_NODE = 12 * hyperthreading
+
+ if wall_time and wall_time.total_seconds() > 24 * 3600:
+ raise ValueError("No jobs longer that 24h allowed")
+
+ if hyperthreading > 2:
+ raise ValueError("PizDaint supports only two way hyperthreading (requested %d)" %(hyperthreading,) )
+
+ if nodes is not None and cores is not None:
+ raise ValueError("You can either specify nodes or cores - not both.")
+
+ if nodes is None and cores is None:
+ raise ValueError('Specify either cores or nodes.')
+
+ if nodes is None:
+ nodes = math.ceil( cores / CORES_PER_NODE )
+ if cores is None:
+ cores = nodes * CORES_PER_NODE
+
+ if cores > CORES_PER_NODE and cores % CORES_PER_NODE != 0:
+ raise ValueError("When using more than one node, the number of cores has to be a multiple of 12")
+
+ tasks_per_node = min( CORES_PER_NODE, cores )
+
+ template_file = os.path.join( os.path.dirname( os.path.realpath(__file__) ), "pizdaint_hybrid.job" )
+ result = open(template_file).read().format( cores = cores,
+ nodes = nodes,
+ tasks_per_core = hyperthreading,
+ tasks_per_node=tasks_per_node,
+ cpus_per_task = 1, # OpenMP num threads would go here
+ initial_dir = initial_dir,
+ job_name = job_name,
+ wall_time = wall_time )
+
+ exec_line = "srun %s %s \n"
+
+ if exe_name is not None:
+ for param_file in parameter_files:
+ result += exec_line %( cores, exe_name, param_file )
+
+ for exe_paramfile_pair in commands:
+ if type(exe_paramfile_pair) is not tuple:
+ result += exe_paramfile_pair + "\n"
+ else:
+ result += exec_line % exe_paramfile_pair
+
+
+ return result
+
diff --git a/python/waLBerla/tools/jobscripts/supermuc.py b/python/waLBerla/tools/jobscripts/supermuc.py
index 2f1fcdf16..a20ec18ab 100644
--- a/python/waLBerla/tools/jobscripts/supermuc.py
+++ b/python/waLBerla/tools/jobscripts/supermuc.py
@@ -49,11 +49,9 @@ def createJobscript( wall_time = None, nodes = None, cores = None, job_class = N
job_class='micro'
elif nodes <= 512:
job_class= 'general'
- elif nodes <= 2048:
- job_class = 'large'
else:
- job_class = 'special'
-
+ job_class = 'large'
+
tasks_per_node = min( CORES_PER_NODE, cores )
task_affinity = "core" if hyperthreading==1 else "cpu"
--
GitLab