University of Hamburg - to research, to teach, to educate and form, to homepage
RRZ
Regional Computer
Center (RRZ)

Photo: Thomas Orgis

Example batch jobs for the big partition

Contents

OpenMP job

line
no. 		/sw/batch/examples/big-partition/openmp-job.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 #!/bin/bash
#SBATCH --account=WorkingGroupName_big
#SBATCH --partition=big
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=192
#SBATCH --time=00:05:00
#SBATCH --export=NONE

source /sw/batch/init.sh

export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK  # essential
export OMP_PROC_BIND=spread                  # recommended
export OMP_PLACES=cores                      # recommended
export OMP_SCHEDULE=static                   # recommended
export OMP_DISPLAY_ENV=verbose               # good to know

# If --cpus-per-task != 192 check process binding
# with export OMP_DISPLAY_ENV=verbose ! 

./a.out

exit

MPI job

line
no. 		/sw/batch/examples/big-partition/mpi-job.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
 #!/bin/bash
#SBATCH --account=WorkingGroupName_big
#SBATCH --partition=big
#SBATCH --ntasks=192
#SBATCH --time=00:01:00
#SBATCH --export=NONE

source /sw/batch/init.sh

module switch env env/that-was-used-at-compile-time

mpirun ./a.out

exit

Open-MPI job with more memory per core

line
no. 		/sw/batch/examples/big-partition/mpi-memory-job.openmpi.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 #!/bin/bash
#
# Open-MPI job which needs more memory per MPI-task than 12 GB
#
#     memory per task = --cpus-per-task * 12 GB
#     --ntasks should be a multiple of 24
#     --ntasks * --cpus-per-task should be 192
#
#SBATCH --account=WorkingGroupName_big
#SBATCH --partition=big
#SBATCH --ntasks=96
#SBATCH --cpus-per-task=2
#SBATCH --time=00:05:00
#SBATCH --export=NONE

source /sw/batch/init.sh

module switch env env/that-was-used-at-compile-time

mpirun -map-by node:pe=$SLURM_CPUS_PER_TASK -bind-to core ./a.out

exit

Intel-MPI job with more memory per core

line
no. 		/sw/batch/examples/big-partition/mpi-memory-job.impi.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 #!/bin/bash
#
# Intel-MPI job which needs more memory per MPI-task than 12 GB
#
#     memory per task = --cpus-per-task * 12 GB
#     --ntasks should be a multiple of 24
#     --ntasks * --cpus-per-task should be 192
#
#SBATCH --account=WorkingGroupName_big
#SBATCH --partition=big
#SBATCH --ntasks=96
#SBATCH --cpus-per-task=2
#SBATCH --time=00:05:00
#SBATCH --export=NONE

source /sw/batch/init.sh

module switch env env/that-was-used-at-compile-time

mpirun ./a.out

exit