Update to SUNDIALS v7 (#3835)

This is the rebirth of #3756. But it's compatible with SUNDIALS 6. --------- Co-authored-by: Balos, Cody, J <balos1@llnl.gov> Co-authored-by: Cody Balos <cjbalos@gmail.com> Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
AMReX-Codes · Mar 27, 2024 · 82008f8 · 82008f8
1 parent 2a9e992
commit 82008f8
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 8 deletions.
diff --git a/.github/workflows/sundials.yml b/.github/workflows/sundials.yml
@@ -68,7 +68,7 @@ jobs:
         du -hs ~/.cache/ccache
 
   sundials-cuda:
-    name: CUDA SUNDIALS@6.5.0
+    name: CUDA SUNDIALS@7.0.0
     runs-on: ubuntu-22.04
     env:
        CCACHE_COMPRESS: 1
@@ -97,9 +97,9 @@ jobs:
         export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
         which nvcc || echo "nvcc not in PATH!"
 
-        wget -q https://github.com/LLNL/sundials/archive/refs/tags/v6.5.0.tar.gz
-        tar xfz v6.5.0.tar.gz
-        cd sundials-6.5.0
+        wget -q https://github.com/LLNL/sundials/archive/refs/tags/v7.0.0.tar.gz
+        tar xfz v7.0.0.tar.gz
+        cd sundials-7.0.0
         mkdir instdir builddir
         cd builddir
         cmake ..                                     \
@@ -126,7 +126,7 @@ jobs:
             -DAMReX_GPU_BACKEND=CUDA    \
             -DAMReX_CUDA_ARCH=8.0       \
             -DAMReX_SUNDIALS=ON         \
-            -DSUNDIALS_ROOT=${PWD}/sundials-6.5.0/instdir \
+            -DSUNDIALS_ROOT=${PWD}/sundials-7.0.0/instdir \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache  \
             -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache
         cmake --build build -j 4

diff --git a/Src/Extern/SUNDIALS/AMReX_SUNMemory.cpp b/Src/Extern/SUNDIALS/AMReX_SUNMemory.cpp
@@ -1,13 +1,16 @@
+#include <AMReX_Sundials_Core.H>
+#include <AMReX_SUNMemory.H>
 #include <AMReX.H>
 #include <AMReX_Gpu.H>
-#include <AMReX_SUNMemory.H>
+
 #if defined(AMREX_USE_HIP)
 #include <sunmemory/sunmemory_hip.h>
 #elif defined(AMREX_USE_CUDA)
 #include <sunmemory/sunmemory_cuda.h>
 #elif defined(AMREX_USE_SYCL)
 #include <sunmemory/sunmemory_sycl.h>
 #endif
+#include <sundials/sundials_config.h>
 
 namespace amrex::sundials {
 
@@ -35,7 +38,11 @@ namespace {
 
     int Alloc(SUNMemoryHelper, SUNMemory* memptr, size_t memsize, SUNMemoryType mem_type, void* /*queue*/)
     {
+#if defined(SUNDIALS_VERSION_MAJOR) && (SUNDIALS_VERSION_MAJOR < 7)
         SUNMemory mem = SUNMemoryNewEmpty();
+#else
+        SUNMemory mem = SUNMemoryNewEmpty(*The_Sundials_Context());
+#endif
 
         if (mem == nullptr) { return -1; }
         mem->ptr = nullptr;

diff --git a/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H b/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H
@@ -1,5 +1,6 @@
 #ifndef AMREX_SUNDIALS_INTEGRATOR_H
 #define AMREX_SUNDIALS_INTEGRATOR_H
+#include <AMReX_Config.H>
 #include <AMReX_REAL.H>
 #include <AMReX_Vector.H>
 #include <AMReX_ParmParse.H>
@@ -178,14 +179,30 @@ public:
     {
         initialize_parameters();
         mpi_comm = ParallelContext::CommunicatorSub();
+#if defined(SUNDIALS_VERSION_MAJOR) && (SUNDIALS_VERSION_MAJOR < 7)
         SUNContext_Create(&mpi_comm, &sunctx);
+#else
+#  ifdef AMREX_USE_MPI
+        SUNContext_Create(mpi_comm, &sunctx);
+#  else
+        SUNContext_Create(SUN_COMM_NULL, &sunctx);
+#  endif
+#endif
     }
 
     void initialize ()
     {
         initialize_parameters();
         mpi_comm = ParallelContext::CommunicatorSub();
+#if defined(SUNDIALS_VERSION_MAJOR) && (SUNDIALS_VERSION_MAJOR < 7)
         SUNContext_Create(&mpi_comm, &sunctx);
+#else
+#  ifdef AMREX_USE_MPI
+        SUNContext_Create(mpi_comm, &sunctx);
+#  else
+        SUNContext_Create(SUN_COMM_NULL, &sunctx);
+#  endif
+#endif
     }
 
     virtual ~SundialsIntegrator () {
@@ -607,7 +624,11 @@ public:
         MRIStepSStolerances(mristep_mem, reltol, abstol);
 
         /* Initialize spgmr solver */
+#if defined(SUNDIALS_VERSION_MAJOR) && (SUNDIALS_VERSION_MAJOR < 7)
         LS = SUNLinSol_SPGMR(nv_S, PREC_NONE, 10, sunctx);
+#else
+        LS = SUNLinSol_SPGMR(nv_S, SUN_PREC_NONE, 10, sunctx);
+#endif
         NLS = SUNNonlinSol_FixedPoint(nv_S, 50, sunctx);
 
         if (use_implicit_inner) { ARKStepSetNonlinearSolver(inner_mem, NLS); }

diff --git a/Src/Extern/SUNDIALS/AMReX_Sundials_Core.H b/Src/Extern/SUNDIALS/AMReX_Sundials_Core.H
@@ -3,7 +3,13 @@
 
 #include <AMReX.H>
 #include <AMReX_OpenMP.H>
-#include <sundials/sundials_context.h>
+
+#include <sundials/sundials_config.h>
+#if defined(SUNDIALS_VERSION_MAJOR) && (SUNDIALS_VERSION_MAJOR < 7)
+#  include <sundials/sundials_context.h>
+#else
+#  include <sundials/sundials_context.hpp>
+#endif
 
 namespace amrex::sundials {
 

diff --git a/Tools/CMake/AMReXThirdPartyLibraries.cmake b/Tools/CMake/AMReXThirdPartyLibraries.cmake
@@ -143,9 +143,22 @@ if (AMReX_SUNDIALS)
         message(STATUS "SUNDIALS_FOUND is true, assuming nvecserial or gpu-specific vector found for version 6.0.0 or higher")
     else ()
        set(SUNDIALS_MINIMUM_VERSION 6.0.0 CACHE INTERNAL "Minimum required SUNDIALS version")
-       find_package(SUNDIALS ${SUNDIALS_MINIMUM_VERSION} CONFIG QUIET )
+       set(SUNDIALS_COMPONENTS arkode cvode sunlinsolspgmr sunlinsolspfgmr
+                               nvecserial nvecmanyvector sunnonlinsolfixedpoint)
+       find_package(SUNDIALS CONFIG REQUIRED
+                    COMPONENTS ${SUNDIALS_COMPONENTS}
+                    OPTIONAL_COMPONENTS core) # core only available for >= 7
+       if (SUNDIALS_VERSION VERSION_LESS ${SUNDIALS_MINIMUM_VERSION})
+          message(FATAL_ERROR "SUNDIALS_VERSION ${SUNDIALS_MINIMUM_VERSION} or newer is required. Found version ${SUNDIALS_VERSION}.")
+       endif ()
     endif ()
     foreach(D IN LISTS AMReX_SPACEDIM)
+        if (SUNDIALS_VERSION VERSION_GREATER_EQUAL 7)
+           target_link_libraries(amrex_${D}d PUBLIC SUNDIALS::core)
+        endif ()
+        foreach(comp IN LISTS SUNDIALS_COMPONENTS)
+           target_link_libraries(amrex_${D}d PUBLIC SUNDIALS::${comp})
+        endforeach()
         if (AMReX_GPU_BACKEND STREQUAL "CUDA")
            target_link_libraries(amrex_${D}d PUBLIC SUNDIALS::nveccuda)
         elseif (AMReX_GPU_BACKEND STREQUAL "HIP")