#SBATCH --nodes=1               # 1 node is used
#SBATCH --ntasks-per-node=1     # 1 MPI tasks
#SBATCH --cpus-per-task=40      # Number of OpenMP threads per MPI task
#SBATCH --gres=gpu:4            # Number of GPUs per node
#SBATCH --hint=nomultithread    # Disable hyperthreading
#SBATCH --job-name=gromacs_tmpi # Jobname
#SBATCH --output=GMX_GenMD.o%j  # Standard output file (%j is the job number)
#SBATCH --error=GMX_GenMD.o%j   # Standard error file
#SBATCH --time=10:00:00         # Expected runtime HH:MM:SS (max 100h for V100, 20h for A100)
##
## Please, refer to comments below for
## more information about these 4 last options.
##SBATCH --account=<account>@v100  # To specify gpu accounting: <account> = echo $IDRPROJ
##SBATCH --partition=<partition>   # To specify partition (see IDRIS web site for more info)
##SBATCH --qos=qos_gpu-dev         # Uncomment for job requiring less than 2 hours
##SBATCH --qos=qos_gpu-t4          # Uncomment for job requiring more than 20h (up to 16 GPU, V100 only)

# Cleans out the modules loaded in interactive and inherited by default
module purge

# Load needed modules
module load gromacs/2022.3-cuda-plumed

# Activate latest GPU developments
export GMX_GPU_PME_PP_COMMS=true
export GMX_FORCE_UPDATE_DEFAULT_GPU=1
export GMX_GPU_DD_COMMS=true

# Run : 4 thread-MPI tasks and 10 threads/task 4 GPUs (--gres=gpu:4) automatically detected.
# Please read the documentation about thread-MPI and latest GPU development
# http://manual.gromacs.org/documentation/current/user-guide/mdrun-performance.html
gmx mdrun -ntmpi 4 -npme 1 -ntomp 10 \
          -update gpu -bonded gpu \
          -nb gpu -pme gpu -pmefft gpu \
          -deffnm 6vxx_nvt -v