#!/usr/bin/env bash

set -euo pipefail

clock=1005

echo '-- refresh base templates'
cd 16-nodes
gridenv-run-make-script run.dwffp32.template.sh run.template.sh dwf_fp32.tok
cd ../8-nodes
gridenv-run-make-script run.dwffp32.template.sh run.template.sh dwf_fp32.tok
cd ..

for clock in $(seq 210 15 1410); do
  echo "====== CLOCK FREQ ${clock}"
  echo '-- create job templates'
  cd 16-nodes
  cp run.dwffp32.template.sh run.dwffp32."${clock}"-16A.sh
  gridenv-token-replace run.dwffp32."${clock}"-16A.sh freq:"${clock}" jobid:16A
  cp run.dwffp32.template.sh run.dwffp32."${clock}"-16B.sh
  gridenv-token-replace run.dwffp32."${clock}"-16B.sh freq:"${clock}" jobid:16B
  cd ../8-nodes
  cp run.dwffp32.template.sh run.dwffp32."${clock}"-8A.sh
  gridenv-token-replace run.dwffp32."${clock}"-8A.sh freq:"${clock}" jobid:8A
  cp run.dwffp32.template.sh run.dwffp32."${clock}"-8B.sh
  gridenv-token-replace run.dwffp32."${clock}"-8B.sh freq:"${clock}" jobid:8B
  cd ..

  echo '-- run jobs'
  cd 16-nodes
  sbatch run.dwffp32."${clock}"-16A.sh
  sbatch run.dwffp32."${clock}"-16B.sh
  cd ../8-nodes
  sbatch run.dwffp32."${clock}"-8A.sh
  sbatch run.dwffp32."${clock}"-8B.sh
  cd ..

  echo '-- waiting for jobs to finish'
  while (( $(squeue -u dc-port1 | grep -c power) > 0 )); do
    sleep 1
  done
done