From 6616d5d09012cb7097e2703d19dfb039bae8a539 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Wed, 2 Feb 2022 16:38:24 -0500
Subject: [PATCH 1/4] Commit

---
 Grid/stencil/Stencil.h | 107 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 90 insertions(+), 17 deletions(-)

diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h
index c2bc8dab..269ecfe4 100644
--- a/Grid/stencil/Stencil.h
+++ b/Grid/stencil/Stencil.h
@@ -240,6 +240,19 @@ public:
     cobj * mpi_p;
     Integer buffer_size;
   };
+  struct CopyReceiveBuffer {
+    void * from_p;
+    void * to_p;
+    Integer bytes;
+  };
+  struct CachedTransfer {
+    Integer direction;
+    Integer OrthogPlane;
+    Integer DestProc;
+    Integer bytes;
+    Integer lane;
+    void *recv_buf;
+  };
 
 
 protected:
@@ -271,7 +284,8 @@ public:
   std::vector<Merge> MergersSHM;
   std::vector<Decompress> Decompressions;
   std::vector<Decompress> DecompressionsSHM;
-
+  std::vector<CopyReceiveBuffer> CopyReceiveBuffers ;
+  std::vector<CachedTransfer> CachedTransfers;
   ///////////////////////////////////////////////////////////
   // Unified Comms buffers for all directions
   ///////////////////////////////////////////////////////////
@@ -551,8 +565,57 @@ public:
     Mergers.resize(0);
     MergersSHM.resize(0);
     Packets.resize(0);
+    CopyReceiveBuffers.resize(0);
+    CachedTransfers.resize(0);
     calls++;
   }
+  void AddCopy(void *from,void * to, Integer bytes)
+  {
+    CopyReceiveBuffer obj;
+    obj.from_p = from;
+    obj.to_p = to;
+    obj.bytes= bytes;
+    CopyReceiveBuffers.push_back(obj);
+  }
+  void CommsCopy()
+  {
+    //    These are device resident MPI buffers.
+    for(int i=0;i<CopyReceiveBuffers.size();i++){
+      cobj *from=(cobj *)CopyReceiveBuffers[i].from_p;
+      cobj *to  =(cobj *)CopyReceiveBuffers[i].to_p;
+      Integer words = CopyReceiveBuffers[i].bytes/sizeof(cobj);
+      accelerator_forNB(j, words, cobj::Nsimd(), {
+	to[j] = from [j]; // Start with a debug check that these give the same data.
+      });
+    }
+  }
+  
+  Integer CheckForDuplicate(Integer direction, Integer OrthogPlane, Integer DestProc, void *recv_buf,Integer lane,Integer bytes)
+  {
+    CachedTransfer obj;
+    obj.direction   = direction;
+    obj.OrthogPlane = OrthogPlane;
+    obj.DestProc    = DestProc;
+    obj.recv_buf    = recv_buf;
+    obj.bytes       = bytes;
+    obj.lane        = lane;
+    for(int i=0;i<CachedTransfers.size();i++){
+      if (   (CachedTransfers[i].direction  ==direction)
+	   &&(CachedTransfers[i].OrthogPlane==OrthogPlane)
+	   &&(CachedTransfers[i].DestProc   ==DestProc)
+	   &&(CachedTransfers[i].bytes      ==bytes)
+	   &&(CachedTransfers[i].lane       ==lane)
+	     ){
+	std::cout << "Found duplicate copy plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
+	AddCopy(CachedTransfers[i].recv_buf,recv_buf,bytes);
+	return 1;
+      } else {
+	CachedTransfers.push_back(obj);
+	return 0;
+      }
+    }
+    return 0;
+  }
   void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){
     Packet p;
     p.send_buf = xmit;
@@ -578,6 +641,7 @@ public:
     mv.push_back(m);
   }
   template<class decompressor>  void CommsMerge(decompressor decompress)    {
+    CommsCopy();
     CommsMerge(decompress,Mergers,Decompressions);
   }
   template<class decompressor>  void CommsMergeSHM(decompressor decompress) {
@@ -590,8 +654,8 @@ public:
   }
 
   template<class decompressor>
-  void CommsMerge(decompressor decompress,std::vector<Merge> &mm,std::vector<Decompress> &dd) {
-
+  void CommsMerge(decompressor decompress,std::vector<Merge> &mm,std::vector<Decompress> &dd)
+  {
     
     mergetime-=usecond();
     for(int i=0;i<mm.size();i++){
@@ -1011,9 +1075,11 @@ public:
 
       int sx        = (x+sshift)%rd;
       int comm_proc = ((x+sshift)/rd)%pd;
-
+      
       if (comm_proc) {
 
+	
+	
 	int words = buffer_size;
 	if (cbmask != 0x3) words=words>>1;
 
@@ -1045,9 +1111,10 @@ public:
 	  recv_buf=this->u_recv_buf_p;
 	}
 
+
 	cobj *send_buf;
 	send_buf = this->u_send_buf_p; // Gather locally, must send
-
+	
 	////////////////////////////////////////////////////////
 	// Gather locally
 	////////////////////////////////////////////////////////
@@ -1056,23 +1123,27 @@ public:
 	Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so); face_idx++;
 	gathertime+=usecond();
 
-	///////////////////////////////////////////////////////////
-	// Build a list of things to do after we synchronise GPUs
-	// Start comms now???
-	///////////////////////////////////////////////////////////
-	AddPacket((void *)&send_buf[u_comm_offset],
-		  (void *)&recv_buf[u_comm_offset],
-		  xmit_to_rank,
-		  recv_from_rank,
-		  bytes);
+	int duplicate = CheckForDuplicate(dimension,x,comm_proc,(void *)&recv_buf[u_comm_offset],0,bytes);
+	if (!duplicate || 1) { // Force comms for now
 
+	  ///////////////////////////////////////////////////////////
+	  // Build a list of things to do after we synchronise GPUs
+	  // Start comms now???
+	  ///////////////////////////////////////////////////////////
+	  AddPacket((void *)&send_buf[u_comm_offset],
+		    (void *)&recv_buf[u_comm_offset],
+		    xmit_to_rank,
+		    recv_from_rank,
+		    bytes);
+	}
+	
 	if ( compress.DecompressionStep() ) {
 	  AddDecompress(&this->u_recv_buf_p[u_comm_offset],
 			&recv_buf[u_comm_offset],
 			words,Decompressions);
 	}
 	u_comm_offset+=words;
-      }
+	}
     }
     return 0;
   }
@@ -1181,8 +1252,10 @@ public:
 
 	    rpointers[i] = rp;
 
-	    AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
-
+	    int duplicate = CheckForDuplicate(dimension,x,nbr_proc,(void *)rp,i,bytes);
+	    if (!duplicate || 1 ) { // Force comms for now
+	      AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
+	    }
 
 	  } else {
 

From 6283d11d5042ef36d306b28dfafe47ee7eab9d23 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@Hannibal.local>
Date: Tue, 8 Feb 2022 15:22:06 +0000
Subject: [PATCH 2/4] Add the comment line to tell the existance of copied
 data/buffer

---
 Grid/stencil/Stencil.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h
index 269ecfe4..d39345f7 100644
--- a/Grid/stencil/Stencil.h
+++ b/Grid/stencil/Stencil.h
@@ -571,6 +571,7 @@ public:
   }
   void AddCopy(void *from,void * to, Integer bytes)
   {
+    std::cout << "Adding CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<bytes<<std::endl;
     CopyReceiveBuffer obj;
     obj.from_p = from;
     obj.to_p = to;
@@ -584,6 +585,7 @@ public:
       cobj *from=(cobj *)CopyReceiveBuffers[i].from_p;
       cobj *to  =(cobj *)CopyReceiveBuffers[i].to_p;
       Integer words = CopyReceiveBuffers[i].bytes/sizeof(cobj);
+    std::cout << "CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<words*sizeof(cobj)<<std::endl;
       accelerator_forNB(j, words, cobj::Nsimd(), {
 	to[j] = from [j]; // Start with a debug check that these give the same data.
       });
@@ -610,6 +612,7 @@ public:
 	AddCopy(CachedTransfers[i].recv_buf,recv_buf,bytes);
 	return 1;
       } else {
+	std::cout << "No duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
 	CachedTransfers.push_back(obj);
 	return 0;
       }

From fad5a74a4bf0b1c528e9e32684088a93eb127064 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 15 Feb 2022 10:27:39 -0500
Subject: [PATCH 3/4] Bug fix to detection case

---
 Grid/stencil/Stencil.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h
index d39345f7..5a2d5099 100644
--- a/Grid/stencil/Stencil.h
+++ b/Grid/stencil/Stencil.h
@@ -601,6 +601,7 @@ public:
     obj.recv_buf    = recv_buf;
     obj.bytes       = bytes;
     obj.lane        = lane;
+
     for(int i=0;i<CachedTransfers.size();i++){
       if (   (CachedTransfers[i].direction  ==direction)
 	   &&(CachedTransfers[i].OrthogPlane==OrthogPlane)
@@ -611,12 +612,11 @@ public:
 	std::cout << "Found duplicate copy plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
 	AddCopy(CachedTransfers[i].recv_buf,recv_buf,bytes);
 	return 1;
-      } else {
-	std::cout << "No duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
-	CachedTransfers.push_back(obj);
-	return 0;
       }
     }
+
+    std::cout << "No duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
+    CachedTransfers.push_back(obj);
     return 0;
   }
   void AddPacket(void *xmit,void * rcv, Integer to,Integer from,Integer bytes){

From e8b1251b8c29c4219cc7e1cd7083b4f008dfa7e3 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 17 Feb 2022 04:51:13 +0000
Subject: [PATCH 4/4] Staggered fix finished

---
 Grid/stencil/Stencil.h | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h
index 5a2d5099..246bdb36 100644
--- a/Grid/stencil/Stencil.h
+++ b/Grid/stencil/Stencil.h
@@ -251,6 +251,7 @@ public:
     Integer DestProc;
     Integer bytes;
     Integer lane;
+    Integer cb;
     void *recv_buf;
   };
 
@@ -571,7 +572,7 @@ public:
   }
   void AddCopy(void *from,void * to, Integer bytes)
   {
-    std::cout << "Adding CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<bytes<<std::endl;
+    //    std::cout << "Adding CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<bytes<<std::endl;
     CopyReceiveBuffer obj;
     obj.from_p = from;
     obj.to_p = to;
@@ -585,22 +586,23 @@ public:
       cobj *from=(cobj *)CopyReceiveBuffers[i].from_p;
       cobj *to  =(cobj *)CopyReceiveBuffers[i].to_p;
       Integer words = CopyReceiveBuffers[i].bytes/sizeof(cobj);
-    std::cout << "CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<words*sizeof(cobj)<<std::endl;
+      //    std::cout << "CopyReceiveBuffer "<<std::hex<<from<<" "<<to<<std::dec<<" "<<words*sizeof(cobj)<<std::endl;
       accelerator_forNB(j, words, cobj::Nsimd(), {
-	to[j] = from [j]; // Start with a debug check that these give the same data.
+	  coalescedWrite(to[j] ,coalescedRead(from [j]));
       });
     }
   }
   
-  Integer CheckForDuplicate(Integer direction, Integer OrthogPlane, Integer DestProc, void *recv_buf,Integer lane,Integer bytes)
+  Integer CheckForDuplicate(Integer direction, Integer OrthogPlane, Integer DestProc, void *recv_buf,Integer lane,Integer bytes,Integer cb)
   {
     CachedTransfer obj;
     obj.direction   = direction;
     obj.OrthogPlane = OrthogPlane;
     obj.DestProc    = DestProc;
     obj.recv_buf    = recv_buf;
-    obj.bytes       = bytes;
     obj.lane        = lane;
+    obj.bytes       = bytes;
+    obj.cb          = cb;
 
     for(int i=0;i<CachedTransfers.size();i++){
       if (   (CachedTransfers[i].direction  ==direction)
@@ -608,14 +610,15 @@ public:
 	   &&(CachedTransfers[i].DestProc   ==DestProc)
 	   &&(CachedTransfers[i].bytes      ==bytes)
 	   &&(CachedTransfers[i].lane       ==lane)
+	   &&(CachedTransfers[i].cb         ==cb)
 	     ){
-	std::cout << "Found duplicate copy plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
+	//	std::cout << "Found duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<< " bytes "<<bytes <<std::endl;
 	AddCopy(CachedTransfers[i].recv_buf,recv_buf,bytes);
 	return 1;
       }
     }
 
-    std::cout << "No duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<std::endl;
+    //    std::cout << "No duplicate plane dir "<<direction<<" plane "<< OrthogPlane<< " simd "<<lane << " relproc "<<DestProc<<"  bytes "<<bytes<<std::endl;
     CachedTransfers.push_back(obj);
     return 0;
   }
@@ -1126,8 +1129,8 @@ public:
 	Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,u_comm_offset,so); face_idx++;
 	gathertime+=usecond();
 
-	int duplicate = CheckForDuplicate(dimension,x,comm_proc,(void *)&recv_buf[u_comm_offset],0,bytes);
-	if (!duplicate || 1) { // Force comms for now
+	int duplicate = CheckForDuplicate(dimension,sx,comm_proc,(void *)&recv_buf[u_comm_offset],0,bytes,cbmask);
+	if ( (!duplicate) ) { // Force comms for now
 
 	  ///////////////////////////////////////////////////////////
 	  // Build a list of things to do after we synchronise GPUs
@@ -1255,8 +1258,8 @@ public:
 
 	    rpointers[i] = rp;
 
-	    int duplicate = CheckForDuplicate(dimension,x,nbr_proc,(void *)rp,i,bytes);
-	    if (!duplicate || 1 ) { // Force comms for now
+	    int duplicate = CheckForDuplicate(dimension,sx,nbr_proc,(void *)rp,i,bytes,cbmask);
+	    if ( (!duplicate) ) { // Force comms for now
 	      AddPacket((void *)sp,(void *)rp,xmit_to_rank,recv_from_rank,bytes);
 	    }