diff --git a/HMC/FTHMC2p1f_3GeV.cc b/HMC/FTHMC2p1f_3GeV.cc index 7d93d168..a8aa67f8 100644 --- a/HMC/FTHMC2p1f_3GeV.cc +++ b/HMC/FTHMC2p1f_3GeV.cc @@ -58,12 +58,13 @@ int main(int argc, char **argv) MD.trajL = 1.0; HMCparameters HMCparams; - HMCparams.StartTrajectory = 104; + HMCparams.StartTrajectory = 0; HMCparams.Trajectories = 200; HMCparams.NoMetropolisUntil= 20; // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n"; // HMCparams.StartingType =std::string("HotStart"); - HMCparams.StartingType =std::string("CheckpointStart"); + HMCparams.StartingType =std::string("ColdStart"); + // HMCparams.StartingType =std::string("CheckpointStart"); HMCparams.MD = MD; HMCWrapper TheHMC(HMCparams); @@ -91,13 +92,13 @@ int main(int argc, char **argv) ////////////////////////////////////////////// - const int Ls = 16; - Real beta = 2.13; - Real light_mass = 0.01; - Real strange_mass = 0.04; + const int Ls = 12; + Real beta = 2.37; + Real light_mass = 0.0047; + Real strange_mass = 0.0186; Real pv_mass = 1.0; RealD M5 = 1.8; - RealD b = 1.0; // Scale factor two + RealD b = 1.0; // Scale factor one, Shamir RealD c = 0.0; OneFlavourRationalParams OFRp; @@ -108,7 +109,7 @@ int main(int argc, char **argv) OFRp.degree = 14; OFRp.precision= 40; - std::vector hasenbusch({ 0.1 }); + std::vector hasenbusch({ 0.05, 0.1, 0.25, 0.5 }); auto GridPtr = TheHMC.Resources.GetCartesian(); auto GridRBPtr = TheHMC.Resources.GetRBCartesian(); @@ -199,8 +200,7 @@ int main(int argc, char **argv) ///////////////////////////////////////////////////////////// // Gauge action ///////////////////////////////////////////////////////////// - // GaugeAction.is_smeared = ApplySmearing; - GaugeAction.is_smeared = true; + GaugeAction.is_smeared = ApplySmearing; Level2.push_back(&GaugeAction); std::cout << GridLogMessage << " ************************************************"<< std::endl; diff --git a/systems/Lumi/HMC/32cube/fthmc3gev.slurm b/systems/Lumi/HMC/32cube/fthmc3gev.slurm new file mode 100644 index 00000000..4cdc5136 --- /dev/null +++ b/systems/Lumi/HMC/32cube/fthmc3gev.slurm @@ -0,0 +1,57 @@ +#!/bin/bash -l +#SBATCH --job-name=fthmc3ge +#SBATCH --partition=small-g +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +##SBATCH --cpus-per-task=8 +#SBATCH --gpus-per-node=8 +#SBATCH --time=2:00:00 +#SBATCH --account=project_465000546 +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 + + +#sbatch --dependency=afterany:$SLURM_JOBID fthmc3gev.slurm + +CPU_BIND="map_ldom:3,3,1,1,0,0,2,2" +MEM_BIND="map_mem:3,3,1,1,0,0,2,2" +echo $CPU_BIND + +cat << EOF > ./select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 4 5 6 7) +export NUMA_MAP=(3 3 1 1 0 0 2 2) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUM=\${NUMA_MAP[\$SLURM_LOCALID]} +#export HIP_VISIBLE_DEVICES=\$GPU +export ROCR_VISIBLE_DEVICES=\$GPU +echo RANK \$SLURM_LOCALID using GPU \$GPU +echo NUMA \$SLURM_LOCALID using NUMA \${NUM} +echo numactl -m \$NUM -N \$NUM \$* +exec numactl -m \$NUM -N \$NUM \$* +EOF +cat ./select_gpu + +chmod +x ./select_gpu + +root=/scratch/project_465000546/boylepet/Grid/systems/Lumi +source ${root}/sourceme.sh + +export OMP_NUM_THREADS=7 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_GPU_SUPPORT_ENABLED=1 + +#cfg=`ls -rt ckpoint_*lat* | tail -n 1 ` +#traj="${cfg#*.}" +#cfg=`ls -rt ckpoint_*lat* | tail -n 1 ` +traj=0 + +vol=32.32.32.64 +mpi=1.2.2.2 +PARAMS="--mpi $mpi --accelerator-threads 16 --comms-sequential --shm 2048 --shm-mpi 0 --grid $vol" +#HMCPARAMS="--StartingType CheckpointStart --StartingTrajectory $traj --Trajectories 200" +HMCPARAMS="--StartingType ColdStart --StartingTrajectory $traj --Trajectories 20" + +srun ./select_gpu ../FTHMC2p1f_3GeV $HMCPARAMS $PARAMS +