From a7abda89e237459cf850094378f1d0f43a51b590 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 21 May 2020 16:13:59 -0400
Subject: [PATCH] View location & access mode

---
 benchmarks/Benchmark_ITT.cc              | 10 ++++-----
 benchmarks/Benchmark_dwf.cc              | 12 +++++------
 benchmarks/Benchmark_dwf_sweep.cc        |  4 ++--
 benchmarks/Benchmark_memory_asynch.cc    |  2 +-
 benchmarks/Benchmark_memory_bandwidth.cc |  2 +-
 benchmarks/Benchmark_meson_field.cc      | 18 ++++++++--------
 benchmarks/Benchmark_su3_gpu.cc          | 26 ++++++++++++------------
 benchmarks/Benchmark_wilson.cc           | 20 +++++++++---------
 8 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 7ad4a147..adfd4b38 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -252,9 +252,9 @@ public:
       double start=usecond();
       for(int i=0;i<Nloop;i++){
 	z=a*x-y;
-	auto x_v = x.View();
-	auto y_v = y.View();
-	auto z_v = z.View();
+	auto x_v = x.View(CpuWrite);
+	auto y_v = y.View(CpuWrite);
+	auto z_v = z.View(CpuRead);
         x_v[0]=z_v[0]; // force serial dependency to prevent optimise away
         y_v[4]=z_v[4];
       }
@@ -534,8 +534,8 @@ public:
     {
       LatticeGaugeField Umu5d(FGrid); 
       std::vector<LatticeColourMatrix> U(4,FGrid);
-      auto Umu_v = Umu.View();
-      auto Umu5d_v = Umu5d.View();
+      auto Umu_v = Umu.View(CpuRead);
+      auto Umu5d_v = Umu5d.View(CpuWrite);
       for(int ss=0;ss<Umu.Grid()->oSites();ss++){
 	for(int s=0;s<Ls;s++){
 	  Umu5d_v[Ls*ss+s] = Umu_v[ss];
diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index a3146cbc..a69061d6 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -129,8 +129,8 @@ int main (int argc, char ** argv)
   LatticeGaugeField Umu5d(FGrid); 
   std::vector<LatticeColourMatrix> U(4,FGrid);
   {
-    auto Umu5d_v = Umu5d.View();
-    auto Umu_v = Umu.View();
+    auto Umu5d_v = Umu5d.View(CpuWrite);
+    auto Umu_v = Umu.View(CpuRead);
     for(int ss=0;ss<Umu.Grid()->oSites();ss++){
       for(int s=0;s<Ls;s++){
 	Umu5d_v[Ls*ss+s] = Umu_v[ss];
@@ -258,8 +258,8 @@ int main (int argc, char ** argv)
       //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu+1,1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuRead);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] + Gamma(Gmu[mu])*tmp_v[i]; ;
 	}
@@ -268,8 +268,8 @@ int main (int argc, char ** argv)
       tmp =adj(U[mu])*src;
       tmp =Cshift(tmp,mu+1,-1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuRead);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] - Gamma(Gmu[mu])*tmp_v[i]; ;
 	}
diff --git a/benchmarks/Benchmark_dwf_sweep.cc b/benchmarks/Benchmark_dwf_sweep.cc
index 3f108b16..d2c9f711 100644
--- a/benchmarks/Benchmark_dwf_sweep.cc
+++ b/benchmarks/Benchmark_dwf_sweep.cc
@@ -130,8 +130,8 @@ void benchDw(std::vector<int> & latt4, int Ls, int threads,int report )
   LatticeGaugeField Umu5d(FGrid); 
 
   // replicate across fifth dimension
-  auto Umu5d_v = Umu5d.View();
-  auto Umu_v   = Umu.View();
+  auto Umu5d_v = Umu5d.View(CpuWrite);
+  auto Umu_v   = Umu.View(CpuRead);
   for(int ss=0;ss<Umu.Grid()->oSites();ss++){
     for(int s=0;s<Ls;s++){
       Umu5d_v[Ls*ss+s] = Umu_v[ss];
diff --git a/benchmarks/Benchmark_memory_asynch.cc b/benchmarks/Benchmark_memory_asynch.cc
index a29d281b..bb155fbe 100644
--- a/benchmarks/Benchmark_memory_asynch.cc
+++ b/benchmarks/Benchmark_memory_asynch.cc
@@ -79,7 +79,7 @@ int main (int argc, char ** argv)
 
     double start=usecond();
     thread_for(t,threads,{
-      auto x_t = x[t].View();
+      auto x_t = x[t].View(CpuRead);
       sum[t] = x_t[0];
       for(int i=0;i<Nloop;i++){
 	for(auto ss=x_t.begin();ss<x_t.end();ss++){
diff --git a/benchmarks/Benchmark_memory_bandwidth.cc b/benchmarks/Benchmark_memory_bandwidth.cc
index 9b921a77..fcb889e8 100644
--- a/benchmarks/Benchmark_memory_bandwidth.cc
+++ b/benchmarks/Benchmark_memory_bandwidth.cc
@@ -177,7 +177,7 @@ int main (int argc, char ** argv)
       Real nn;      
       double start=usecond();
       for(int i=0;i<Nloop;i++){
-	auto x_v = x.View();
+	auto x_v = x.View(CpuWrite);
 	nn=norm2(x);
 	vsplat(x_v[0]._internal[0],nn);
       }
diff --git a/benchmarks/Benchmark_meson_field.cc b/benchmarks/Benchmark_meson_field.cc
index d37acb7f..6c77b6c0 100644
--- a/benchmarks/Benchmark_meson_field.cc
+++ b/benchmarks/Benchmark_meson_field.cc
@@ -85,11 +85,11 @@ void sliceInnerProductMesonField(std::vector< std::vector<ComplexD> > &mat,
       for(int b=0;b<e2;b++){
 	int ss= so+n*stride+b;
 	for(int i=0;i<Lblock;i++){
-	  auto lhs_v = lhs[i].View();
+	  auto lhs_v = lhs[i].View(CpuRead);
 	  auto left = conjugate(lhs_v[ss]);
 	  for(int j=0;j<Rblock;j++){
 	    int idx = i+Lblock*j+Lblock*Rblock*r;
-	    auto rhs_v = rhs[j].View();
+	    auto rhs_v = rhs[j].View(CpuRead);
 	    auto right = rhs_v[ss];
 	    vector_type vv = left()(0)(0) * right()(0)(0)
 	      +              left()(0)(1) * right()(0)(1)
@@ -221,12 +221,12 @@ void sliceInnerProductMesonFieldGamma(std::vector< std::vector<ComplexD> > &mat,
       for(int b=0;b<e2;b++){
 	int ss= so+n*stride+b;
 	for(int i=0;i<Lblock;i++){
- 	  auto lhs_v=lhs[i].View();
+ 	  auto lhs_v=lhs[i].View(CpuRead);
 	  auto left = conjugate(lhs_v[ss]);
 	  for(int j=0;j<Rblock;j++){
 	  for(int mu=0;mu<Ngamma;mu++){
 	    
-	    auto rhs_v = rhs[j].View();
+	    auto rhs_v = rhs[j].View(CpuRead);
 	    auto right = Gamma(gammas[mu])*rhs_v[ss];
 
 	      vector_type vv = left()(0)(0) * right()(0)(0)
@@ -370,12 +370,12 @@ void sliceInnerProductMesonFieldGamma1(std::vector< std::vector<ComplexD> > &mat
 	int ss= so+n*stride+b;
 	for(int i=0;i<Lblock;i++){
 	  
-	  auto lhs_v=lhs[i].View();
+	  auto lhs_v=lhs[i].View(CpuRead);
 	  auto left = conjugate(lhs_v[ss]);
 	  for(int j=0;j<Rblock;j++){
 
 	    SpinMatrix_v vv;
-	    auto rhs_v = rhs[j].View();
+	    auto rhs_v = rhs[j].View(CpuRead);
 	    auto right = rhs_v[ss];
 	    for(int s1=0;s1<Ns;s1++){
 	    for(int s2=0;s2<Ns;s2++){
@@ -518,12 +518,12 @@ void sliceInnerProductMesonFieldGammaMom(std::vector< std::vector<ComplexD> > &m
 
 	for(int i=0;i<Lblock;i++){
 
-	  auto lhs_v = lhs[i].View();
+	  auto lhs_v = lhs[i].View(CpuRead);
 	  auto left = conjugate(lhs_v[ss]);
 	  for(int j=0;j<Rblock;j++){
 
 	    SpinMatrix_v vv;
-	    auto rhs_v = rhs[j].View();
+	    auto rhs_v = rhs[j].View(CpuRead);
 	    auto right = rhs_v[ss];
 	    for(int s1=0;s1<Ns;s1++){
 	    for(int s2=0;s2<Ns;s2++){
@@ -537,7 +537,7 @@ void sliceInnerProductMesonFieldGammaMom(std::vector< std::vector<ComplexD> > &m
 	    // Trigger unroll
 	    for ( int m=0;m<Nmom;m++){
 	      int idx = m+base;
-	      auto mom_v = mom[m].View();
+	      auto mom_v = mom[m].View(CpuRead);
 	      auto phase = mom_v[ss];
 	      mac(&lvSum[idx],&vv,&phase);
 	    }
diff --git a/benchmarks/Benchmark_su3_gpu.cc b/benchmarks/Benchmark_su3_gpu.cc
index 181d5894..5e2ff61c 100644
--- a/benchmarks/Benchmark_su3_gpu.cc
+++ b/benchmarks/Benchmark_su3_gpu.cc
@@ -66,9 +66,9 @@ int main (int argc, char ** argv)
       LatticeColourMatrix x(&Grid);// random(pRNG,x);
       LatticeColourMatrix y(&Grid);// random(pRNG,y);
 
-      auto x_v = x.View();
-      auto y_v = y.View();
-      auto z_v = z.View();
+      auto x_v = x.View(AcceleratorRead);
+      auto y_v = y.View(AcceleratorRead);
+      auto z_v = z.View(AcceleratorWrite);
       const uint64_t Nsite = x_v.size();
       const uint64_t nsimd = vComplex::Nsimd();
       for(int64_t i=0;i<Nwarm;i++){
@@ -116,9 +116,9 @@ int main (int argc, char ** argv)
       LatticeColourMatrix x(&Grid);// random(pRNG,x);
       LatticeColourMatrix y(&Grid);// random(pRNG,y);
 
-      auto x_v = x.View();
-      auto y_v = y.View();
-      auto z_v = z.View();
+      auto x_v = x.View(AcceleratorWrite);
+      auto y_v = y.View(AcceleratorRead);
+      auto z_v = z.View(AcceleratorRead);
       const uint64_t Nsite = x_v.size();
       const uint64_t nsimd = vComplex::Nsimd();
       for(int64_t i=0;i<Nwarm;i++){
@@ -167,9 +167,9 @@ int main (int argc, char ** argv)
       LatticeColourMatrix x(&Grid);// random(pRNG,x);
       LatticeColourMatrix y(&Grid);// random(pRNG,y);
 
-      auto x_v = x.View();
-      auto y_v = y.View();
-      auto z_v = z.View();
+      auto x_v = x.View(AcceleratorRead);
+      auto y_v = y.View(AcceleratorRead);
+      auto z_v = z.View(AcceleratorWrite);
       const uint64_t Nsite = x_v.size();
       const uint64_t nsimd = vComplex::Nsimd();
       for(int64_t i=0;i<Nwarm;i++){
@@ -220,10 +220,10 @@ int main (int argc, char ** argv)
       LatticeColourMatrix y(&Grid);// random(pRNG,y);
       LatticeColourMatrix w(&Grid);// random(pRNG,y);
 
-      auto x_v = x.View();
-      auto y_v = y.View();
-      auto z_v = z.View();
-      auto w_v = z.View();
+      auto x_v = x.View(AcceleratorRead);
+      auto y_v = y.View(AcceleratorRead);
+      auto z_v = z.View(AcceleratorRead);
+      auto w_v = w.View(AcceleratorWrite);
       const uint64_t Nsite = x_v.size();
       const uint64_t nsimd = vComplex::Nsimd();
       for(int64_t i=0;i<Nwarm;i++){
diff --git a/benchmarks/Benchmark_wilson.cc b/benchmarks/Benchmark_wilson.cc
index 49939620..352f7289 100644
--- a/benchmarks/Benchmark_wilson.cc
+++ b/benchmarks/Benchmark_wilson.cc
@@ -125,8 +125,8 @@ int main (int argc, char ** argv)
       //    ref =  src + Gamma(Gamma::Algebra::GammaX)* src ; // 1-gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuWrite);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] - Gamma(Gmu[mu])*tmp_v[i]; ;
 	}
@@ -135,8 +135,8 @@ int main (int argc, char ** argv)
       tmp =adj(U[mu])*src;
       tmp =Cshift(tmp,mu,-1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuWrite);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] + Gamma(Gmu[mu])*tmp_v[i]; ;
 	}
@@ -187,8 +187,8 @@ int main (int argc, char ** argv)
   for(int ss=0;ss<0;ss++ ){
     for(int i=0;i<Ns;i++){
       for(int j=0;j<Nc;j++){
-	auto ref_v = ref.View();
-	auto result_v = result.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto result_v = result.View(CpuWrite);
 	ComplexF * ref_p = (ComplexF *)&ref_v[ss]()(i)(j);
 	ComplexF * res_p = (ComplexF *)&result_v[ss]()(i)(j);
 	std::cout<<GridLogMessage << ss<< " "<<i<<" "<<j<<" "<< (*ref_p)<<" " <<(*res_p)<<std::endl;
@@ -204,8 +204,8 @@ int main (int argc, char ** argv)
       //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuWrite);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] + Gamma(Gmu[mu])*tmp_v[i]; ;
 	}
@@ -214,8 +214,8 @@ int main (int argc, char ** argv)
       tmp =adj(U[mu])*src;
       tmp =Cshift(tmp,mu,-1);
       {
-	auto ref_v = ref.View();
-	auto tmp_v = tmp.View();
+	auto ref_v = ref.View(CpuWrite);
+	auto tmp_v = tmp.View(CpuWrite);
 	for(int i=0;i<ref_v.size();i++){
 	  ref_v[i]+= tmp_v[i] - Gamma(Gmu[mu])*tmp_v[i]; ;
 	}