3GeV run setup

HMC/FTHMC2p1f_3GeV.cc

@@ -58,12 +58,13 @@ int main(int argc, char **argv)
   MD.trajL   = 1.0;
 
   HMCparameters HMCparams;
-  HMCparams.StartTrajectory  = 104;
+  HMCparams.StartTrajectory  = 0;
   HMCparams.Trajectories     = 200;
   HMCparams.NoMetropolisUntil=  20;
   // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
   //  HMCparams.StartingType     =std::string("HotStart");
-  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.StartingType     =std::string("ColdStart");
+  //  HMCparams.StartingType     =std::string("CheckpointStart");
   HMCparams.MD = MD;
   HMCWrapper TheHMC(HMCparams);
 
@@ -91,13 +92,13 @@ int main(int argc, char **argv)
 
   //////////////////////////////////////////////
 
-  const int Ls      = 16;
+  const int Ls      = 12;
-  Real beta         = 2.13;
+  Real beta         = 2.37;
-  Real light_mass   = 0.01;
+  Real light_mass   = 0.0047;
-  Real strange_mass = 0.04;
+  Real strange_mass = 0.0186;
   Real pv_mass      = 1.0;
   RealD M5  = 1.8;
-  RealD b   = 1.0; // Scale factor two
+  RealD b   = 1.0; // Scale factor one, Shamir
   RealD c   = 0.0;
 
   OneFlavourRationalParams OFRp;
@@ -108,7 +109,7 @@ int main(int argc, char **argv)
   OFRp.degree   = 14;
   OFRp.precision= 40;
 
-  std::vector<Real> hasenbusch({ 0.1 });
+  std::vector<Real> hasenbusch({ 0.05, 0.1, 0.25, 0.5 });
 
   auto GridPtr   = TheHMC.Resources.GetCartesian();
   auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
@@ -199,8 +200,7 @@ int main(int argc, char **argv)
   /////////////////////////////////////////////////////////////
   // Gauge action
   /////////////////////////////////////////////////////////////
-  //  GaugeAction.is_smeared = ApplySmearing;
+  GaugeAction.is_smeared = ApplySmearing;
-  GaugeAction.is_smeared = true;
   Level2.push_back(&GaugeAction);
 
   std::cout << GridLogMessage << " ************************************************"<< std::endl;

systems/Lumi/HMC/32cube/fthmc3gev.slurm

@@ -0,0 +1,57 @@
+#!/bin/bash -l
+#SBATCH --job-name=fthmc3ge
+#SBATCH --partition=small-g
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+##SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-node=8
+#SBATCH --time=2:00:00
+#SBATCH --account=project_465000546
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+
+
+#sbatch --dependency=afterany:$SLURM_JOBID fthmc3gev.slurm
+
+CPU_BIND="map_ldom:3,3,1,1,0,0,2,2"
+MEM_BIND="map_mem:3,3,1,1,0,0,2,2"
+echo $CPU_BIND
+
+cat << EOF > ./select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 4 5 6 7)
+export NUMA_MAP=(3 3 1 1 0 0 2 2)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUM=\${NUMA_MAP[\$SLURM_LOCALID]}
+#export HIP_VISIBLE_DEVICES=\$GPU
+export ROCR_VISIBLE_DEVICES=\$GPU
+echo RANK \$SLURM_LOCALID using GPU \$GPU    
+echo NUMA \$SLURM_LOCALID using NUMA \${NUM}
+echo numactl -m \$NUM -N \$NUM \$*
+exec numactl -m \$NUM -N \$NUM \$*
+EOF
+cat ./select_gpu
+
+chmod +x ./select_gpu
+
+root=/scratch/project_465000546/boylepet/Grid/systems/Lumi
+source ${root}/sourceme.sh
+
+export OMP_NUM_THREADS=7
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+#cfg=`ls -rt ckpoint_*lat* | tail -n 1  `
+#traj="${cfg#*.}"
+#cfg=`ls -rt ckpoint_*lat* | tail -n 1  `
+traj=0
+
+vol=32.32.32.64
+mpi=1.2.2.2
+PARAMS="--mpi $mpi --accelerator-threads 16 --comms-sequential --shm 2048 --shm-mpi 0 --grid $vol"
+#HMCPARAMS="--StartingType CheckpointStart --StartingTrajectory $traj --Trajectories 200"
+HMCPARAMS="--StartingType ColdStart --StartingTrajectory $traj --Trajectories 20"
+
+srun ./select_gpu ../FTHMC2p1f_3GeV $HMCPARAMS $PARAMS
+