Updated jobscript generation scripts

python/waLBerla/tools/jobscripts/__init__.py

@@ -3,7 +3,7 @@
 """
 
 from __future__ import print_function, absolute_import, division, unicode_literals
-
+from datetime import timedelta
 from waLBerla.tools.jobscripts.hornet          import createJobscript as _cr_hornet
 from waLBerla.tools.jobscripts.supermuc        import createJobscript as _cr_supermuc
 from waLBerla.tools.jobscripts.supermuc_phase2 import createJobscript as _cr_supermuc2
@@ -35,7 +35,10 @@ def createJobscript(*args, **kwargs):
     """
     if 'machine' not in kwargs:
         raise ValueError("Specify which machine to use with 'machine=<supermuc,juqueen,hornet>'")
-    
+
+    if 'wall_time' in kwargs and isinstance(kwargs['wall_time'], int):
+        kwargs['wall_time'] = timedelta(seconds=kwargs['wall_time'])
+
     if kwargs['machine'].lower() == 'supermuc':        return _cr_supermuc  ( *args, **kwargs )
     if kwargs['machine'].lower() == 'supermuc_phase2': return _cr_supermuc2 ( *args, **kwargs )
     if kwargs['machine'].lower() == 'juqueen' :        return _cr_juqueen   ( *args, **kwargs )

python/waLBerla/tools/jobscripts/hornet.py

@@ -9,7 +9,7 @@ def createJobscript( wall_time = None, nodes = None, cores = None, job_class = N
                      initial_dir = '~', job_name="waLBerla", hyperthreading = 1, 
                      exe_name = None, arguments = [], commands = [], **kwargs ):
     
-    if type(hyperthreading) == bool:
+    if type(hyperthreading) is bool:
         hyperthreading = 2 if hyperthreading else 1
 
     

python/waLBerla/tools/jobscripts/pizdaint_hybrid.job

+#!/bin/bash -l
+#SBATCH --job-name={job_name}
+#SBATCH --time={wall_time}
+#SBATCH --nodes={nodes}
+#SBATCH --ntasks-per-core={tasks_per_core}
+#SBATCH --ntasks-per-node={tasks_per_node}
+#SBATCH --cpus-per-task={cpus_per_task}
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+
+module load daint-gpu
+
+export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
+export CRAY_CUDA_MPS=1
+
+

python/waLBerla/tools/jobscripts/pizdaint_hybrid.py

+from __future__ import print_function, absolute_import, division, unicode_literals
+
+import os
+import math
+
+
+
+def createJobscript( wall_time = None, nodes = None, cores = None, initial_dir=None, job_name="waLBerla",
+                     exe_name=None, parameter_files=[], commands=[], hyperthreading=1, **kwargs ):
+    if type(hyperthreading) is bool:
+        hyperthreading = 2 if hyperthreading else 1
+
+    CORES_PER_NODE = 12 * hyperthreading
+
+    if wall_time and wall_time.total_seconds() >  24 * 3600:
+        raise ValueError("No jobs longer that 24h allowed")
+
+    if hyperthreading > 2:
+        raise ValueError("PizDaint supports only two way hyperthreading (requested %d)" %(hyperthreading,) )
+
+    if nodes is not None and cores is not None:
+        raise ValueError("You can either specify nodes or cores - not both.")
+
+    if nodes is None and cores is None:
+        raise ValueError('Specify either cores or nodes.')
+
+    if nodes is None:
+        nodes = math.ceil( cores / CORES_PER_NODE )
+    if cores is None:
+        cores = nodes * CORES_PER_NODE
+
+    if cores > CORES_PER_NODE and cores % CORES_PER_NODE != 0:
+        raise ValueError("When using more than one node, the number of cores has to be a multiple of 12")
+
+    tasks_per_node = min( CORES_PER_NODE, cores )
+
+    template_file = os.path.join(  os.path.dirname( os.path.realpath(__file__)  ), "pizdaint_hybrid.job" )
+    result = open(template_file).read().format( cores = cores,
+                                                nodes = nodes,
+                                                tasks_per_core = hyperthreading,
+                                                tasks_per_node=tasks_per_node,
+                                                cpus_per_task = 1,  # OpenMP num threads would go here
+                                                initial_dir = initial_dir,
+                                                job_name = job_name,
+                                                wall_time = wall_time )
+
+    exec_line = "srun %s %s \n"
+
+    if exe_name is not None:
+        for param_file in parameter_files:
+            result += exec_line %( cores, exe_name, param_file )
+
+    for exe_paramfile_pair in commands:
+        if type(exe_paramfile_pair) is not tuple:
+            result += exe_paramfile_pair + "\n"
+        else:
+            result += exec_line % exe_paramfile_pair
+
+
+    return result
+

python/waLBerla/tools/jobscripts/supermuc.py

@@ -49,11 +49,9 @@ def createJobscript( wall_time = None, nodes = None, cores = None, job_class = N
                 job_class='micro' 
         elif nodes <= 512:
             job_class= 'general'
-        elif nodes <= 2048:
-            job_class = 'large'
         else:
-            job_class = 'special'
-    
+            job_class = 'large'
+
     tasks_per_node = min( CORES_PER_NODE, cores )
     
     task_affinity = "core" if hyperthreading==1 else "cpu"