portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-06-13 12:47:05 +01:00
Code Releases Activity

NUMA improvements worth preserving from AMD EPYC tests

Browse Source
This commit is contained in:
Peter Boyle
2017-07-08 22:27:11 -04:00
parent 7b0237b081
commit 40e119c61c
3 changed files with 45 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/communicator/Communicator_mpi3.cc
View File

@ -37,7 +37,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/mman.h>
//#include <zlib.h>
#include <zlib.h>
#include <numaif.h>
#ifndef SHM_HUGETLB
#define SHM_HUGETLB 04000
#endif
@ -214,6 +215,23 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if ( ptr == MAP_FAILED ) { perror("failed mmap"); assert(0); }
assert(((uint64_t)ptr&0x3F)==0);
int status;
int flags=MPOL_MF_MOVE;
#ifdef KNL
int nodes=1; // numa domain == MCDRAM
// Find out if in SNC2,SNC4 mode ?
#else
int nodes=r; // numa domain == MPI ID
#endif
unsigned long count=1;
for(uint64_t page=0;page<size;page+=4096){
void *pages = (void *) ( page + (uint64_t)ptr );
uint64_t *cow_it = (uint64_t *)pages; *cow_it = 1;
ierr= move_pages(0,count, &pages,&nodes,&status,flags);
if (ierr && (page==0)) perror("numa relocate command failed");
}
ShmCommBufs[r] =ptr;
}