atomicAdd -> lockAdd (ECP-WarpX#4608)

For WarpX's Gordon Bell runs on Fugaku, Stephan Jaure of ATOS optimized atomicAdd using pthread spin locks. The optimized approach has been implemented in amrex::BaseFab::lockAdd, which is now used in WarpX.
ax3l · Jan 16, 2024 · a8f20e3 · a8f20e3
1 parent a3fd142
commit a8f20e3
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 10 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -115,7 +115,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd -
+        cd ../amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
         ccache -s

diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini
@@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more
 
 [AMReX]
 dir = /home/regtester/git/amrex/
-branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14
+branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8
 
 [source]
 dir = /home/regtester/git/WarpX

diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini
@@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det
 
 [AMReX]
 dir = /home/regtester/AMReX_RegTesting/amrex/
-branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14
+branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8
 
 [source]
 dir = /home/regtester/AMReX_RegTesting/warpx

diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp
@@ -701,9 +701,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti,
 #ifndef AMREX_USE_GPU
     // CPU, tiling: atomicAdd local_j<xyz> into j<xyz>
     WARPX_PROFILE_VAR_START(blp_accumulate);
-    (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp());
-    (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp());
-    (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp());
+    (*jx)[pti].lockAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp());
+    (*jy)[pti].lockAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp());
+    (*jz)[pti].lockAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp());
     WARPX_PROFILE_VAR_STOP(blp_accumulate);
 #endif
 }
@@ -996,7 +996,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp,
 #ifndef AMREX_USE_GPU
         // CPU, tiling: atomicAdd local_rho into rho
         WARPX_PROFILE_VAR_START(blp_accumulate);
-        (*rho)[pti].atomicAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc);
+        (*rho)[pti].lockAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc);
         WARPX_PROFILE_VAR_STOP(blp_accumulate);
 #endif
     } else {

diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H
@@ -201,7 +201,7 @@ deposit_charge (typename T_PC::ParIterType& pti,
 #ifndef AMREX_USE_GPU
     // CPU, tiling: atomicAdd local_rho into rho
     ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize);
-    (*rho)[pti].atomicAdd(local_rho, tb, tb, 0, icomp*nc, nc);
+    (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc);
     ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize);
 #endif
 }

diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake
@@ -269,7 +269,7 @@ set(WarpX_amrex_src ""
 set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
     CACHE STRING
     "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)")
-set(WarpX_amrex_branch "f1ec8df75c562d2a4822cea84d284cf8e72c2e14"
+set(WarpX_amrex_branch "255d30f387cf2c1a7eff5a31f703c94de803e8d8"
     CACHE STRING
     "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)")
 

diff --git a/run_test.sh b/run_test.sh
@@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt
 
 # Clone AMReX and warpx-data
 git clone https://github.com/AMReX-Codes/amrex.git
-cd amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd -
+cd amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd -
 # warpx-data contains various required data sets
 git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
 # openPMD-example-datasets contains various required data sets