Daniel

CMakeLists.txt

@@ -24,13 +24,6 @@ endif(NOT CMAKE_BUILD_TYPE)
 # Macros and packages
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
-# --- custom targets: ---
-if (NOT TARGET distclean)
-  # Only include distclean if it has not already been defined (by ELEMENTS or
-  # any other package that defines distclean and uses MATAR as a submodule)
-  INCLUDE(cmake/Modules/TargetDistclean.cmake OPTIONAL)
-endif (NOT TARGET distclean)
-
 find_package(Vector)
 if (CMAKE_VECTOR_NOVEC)
   set(VECTOR_C_FLAGS "${VECTOR_NOVEC_C_FLAGS}")
@@ -62,8 +55,18 @@ if(Matar_ENABLE_KOKKOS)
     else()
         find_package(Kokkos REQUIRED)
     endif()
+    if (Matar_ENABLE_MPI)
+        find_package(MPI REQUIRED)
+        add_definitions(-DHAVE_MPI=1)
+        target_link_libraries(matar INTERFACE Kokkos::kokkos MPI::MPI_CXX)
+    else()
         target_link_libraries(matar INTERFACE Kokkos::kokkos)
+    endif()
     add_definitions(-DHAVE_KOKKOS=1)
+elseif(Matar_ENABLE_MPI)
+    find_package(MPI REQUIRED)
+    target_link_libraries(matar INTERFACE MPI::MPI_CXX)
+    add_definitions(-DHAVE_MPI=1)
 endif()
 
 include(CMakePackageConfigHelpers)

cmake/Modules/TargetDistclean.cmake

@@ -31,11 +31,6 @@ IF (UNIX)
    */*/cmake_install.cmake
    install_manifest.txt
    */install_manifest.txt
-   Kokkos
-   kokkos
-   src
-   test
-   mtest
   )
 
   ADD_CUSTOM_COMMAND(

examples/CMakeLists.txt

@@ -1,65 +1,92 @@
 cmake_minimum_required(VERSION 3.8)
 
+# --- custom targets: ---
+
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-find_package(Matar REQUIRED)
+if (NOT TARGET distclean)
+  # Only include distclean if it has not already been defined (by any
+  # other package that defines distclean and uses MATAR as a submodule)
+  INCLUDE(../cmake/Modules/TargetDistclean.cmake OPTIONAL)
+endif (NOT TARGET distclean)
 
-add_executable(mtest main.cpp)
-target_link_libraries(mtest matar)
+set(LINKING_LIBRARIES matar)
+
+find_package(Matar REQUIRED)
+if (MPI)
+    find_package(MPI REQUIRED)
+    add_definitions(-DHAVE_MPI=1)
+    list(APPEND LINKING_LIBRARIES MPI::MPI_CXX)
+    if (GPU_AWARE_MPI)
+        add_definitions(-DHAVE_GPU_AWARE_MPI)
+#    elseif (GPU_SHARED_MEM)
+#        add_definitions(-DHAVE_GPU_SHARED_MEM)
+    endif()
+endif()
 
 if (NOT KOKKOS)
+    add_executable(mtest main.cpp)
+    target_link_libraries(mtest ${LINKING_LIBRARIES})
+
    add_executable(test_for test_for.cpp)
-   target_link_libraries(test_for matar)
+   target_link_libraries(test_for ${LINKING_LIBRARIES})
 
    add_executable(test_shared_ptr test_shared_ptr.cpp)
-   target_link_libraries(test_shared_ptr matar)
+   target_link_libraries(test_shared_ptr ${LINKING_LIBRARIES})
 endif()
 
 if (KOKKOS)
   find_package(Kokkos REQUIRED) #new
 
+  list(APPEND LINKING_LIBRARIES Kokkos::kokkos)
+
+  add_definitions(-DHAVE_KOKKOS=1)
+
+  if (CUDA)
+    add_definitions(-DHAVE_CUDA=1)
+  elseif (HIP)
+    add_definitions(-DHAVE_HIP=1)
+  elseif (OPENMP)
+    add_definitions(-DHAVE_OPENMP=1)
+  elseif (THREADS)
+    add_definitions(-DHAVE_THREADS=1)
+  endif()
+
   add_executable(mtestkokkos main_kokkos.cpp)
-  target_link_libraries(mtestkokkos matar Kokkos::kokkos)
+  target_link_libraries(mtestkokkos ${LINKING_LIBRARIES})
 
   add_executable(test_kokkos_for kokkos_for.cpp)
-  target_link_libraries(test_kokkos_for matar Kokkos::kokkos)
+  target_link_libraries(test_kokkos_for ${LINKING_LIBRARIES})
 
   add_executable(test_dual_types test_dual_types.cpp)
-  target_link_libraries(test_dual_types matar Kokkos::kokkos)
-
-  add_definitions(-DHAVE_KOKKOS=1)
+  target_link_libraries(test_dual_types ${LINKING_LIBRARIES})
 
   add_executable(kokkos_csr CSRKokkos.cpp)
-  target_link_libraries(kokkos_csr matar Kokkos::kokkos)
+  target_link_libraries(kokkos_csr ${LINKING_LIBRARIES})
 
   add_executable(kokkos_csc CSCKokkos.cpp)
-  target_link_libraries(kokkos_csc matar Kokkos::kokkos)
+  target_link_libraries(kokkos_csc ${LINKING_LIBRARIES})
 
   add_executable(mtr_kokkos-simple mtr-kokkos-simple.cpp)
-  target_link_libraries(mtr_kokkos-simple matar Kokkos::kokkos)
-
+  target_link_libraries(mtr_kokkos-simple ${LINKING_LIBRARIES})
 
-  if (CUDA)
-    add_definitions(-DHAVE_CUDA=1)
-  elseif (HIP)
-    add_definitions(-DHAVE_HIP=1)
-  elseif (OPENMP)
-    add_definitions(-DHAVE_OPENMP=1)
+  if (OPENMP)
     add_executable(parallel_hello_world parallel_hello_world.cpp)
-    target_link_libraries(parallel_hello_world matar Kokkos::kokkos)
-  elseif (THREADS)
-    add_definitions(-DHAVE_THREADS=1)
+    target_link_libraries(parallel_hello_world ${LINKING_LIBRARIES})
   endif()
 
 endif()
 
-
 ### HIP Linking error, will add back in after fixed
 if (NOT HIP)
     include_directories(virtualFcnKokkos)
     add_subdirectory(virtualFcnKokkos)
 endif()
 
+# In testing, not working
+#include_directories(gArrayofgArrays)
+#add_subdirectory(gArrayofgArrays)
+
 include_directories(virtualFcnMATAR)
 add_subdirectory(virtualFcnMATAR)
 
@@ -72,17 +99,17 @@ add_subdirectory(halfspace_cooling)
 include_directories(watt-graph)
 add_subdirectory(watt-graph)
 
-include_directories(matar_fortran)
-add_subdirectory(matar_fortran)
+#include_directories(matar_fortran)
+#add_subdirectory(matar_fortran)
 
 include_directories(sparsetests)
 add_subdirectory(sparsetests)
 
 include_directories(test_rocm)
 add_subdirectory(test_rocm)
 
-#include_directories(laplaceMPI)
-#add_subdirectory(laplaceMPI)
+include_directories(laplaceMPI)
+add_subdirectory(laplaceMPI)
 
 #include_directories(phaseField/srcKokkosVerbose)
 #add_subdirectory(phaseField/srcKokkosVerbose)

examples/gArrayofgArrays/CMakeLists.txt

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.1.3)
+
+find_package(Matar REQUIRED)
+
+if (KOKKOS)
+  add_definitions(-DHAVE_KOKKOS=1)
+  find_package(Kokkos REQUIRED)
+
+  add_definitions(-DHAVE_KOKKOS=1)
+  if (CUDA)
+    add_definitions(-DHAVE_CUDA=1)
+  elseif (HIP)
+    add_definitions(-DHAVE_HIP=1)
+  elseif (OPENMP)
+    add_definitions(-DHAVE_OPENMP=1)
+  elseif (THREADS)
+    add_definitions(-DHAVE_THREADS=1)
+  endif()
+
+  add_executable(viewofview main.cpp)
+  target_link_libraries(viewofview matar Kokkos::kokkos)
+
+endif()

src/matar.h → examples/gArrayofgArrays/main.cpp

@@ -1,7 +1,5 @@
-#ifndef MATAR_H
-#define MATAR_H
 /**********************************************************************************************
- © 2020. Triad National Security, LLC. All rights reserved.
+ © 2020. Triad National Security, LLC. All rights reserved.
  This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos
  National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S.
  Department of Energy/National Nuclear Security Administration. All rights in the program are
@@ -13,14 +11,11 @@
  This program is open source under the BSD-3 License.
  Redistribution and use in source and binary forms, with or without modification, are permitted
  provided that the following conditions are met:
-
  1.  Redistributions of source code must retain the above copyright notice, this list of
  conditions and the following disclaimer.
-
  2.  Redistributions in binary form must reproduce the above copyright notice, this list of
  conditions and the following disclaimer in the documentation and/or other materials
  provided with the distribution.
-
  3.  Neither the name of the copyright holder nor the names of its contributors may be used
  to endorse or promote products derived from this software without specific prior
  written permission.
@@ -36,52 +31,80 @@
  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  **********************************************************************************************/
+#include <iostream>
+#include <stdio.h>
+#include "matar.h"
 
+using namespace mtr;
 
-//  ----
-//   Standard data structures (host types)
-//   1. FArray
-//   2. ViewFArray
-//   3. FMatrix
-//   4. ViewFMatrix
-//   5. CArray
-//   6. ViewCArray
-//   7. CMatrix
-//   8. ViewCMatrix
-//   9. RaggedRightArray
-//   10. RaggedDownArray
-//   11. DynamicRaggedRightArray
-//   12. DynamicRaggedDownArray
-//   13. CSRArray
-//   14. CSCArray
+struct thing
+{
+    int a;
+    DCArrayKokkos <int> inner;
+};
 
-//  ----
-//   Kokkos Data structures (device and dual types)
-//   15. FArrayKokkos
-//   16. ViewFArrayKokkos
-//   17. FMatrixKokkos
-//   18. ViewFMatrixKokkos
-//   19. CArrayKokkos
-//   20. ViewCArrayKokkos
-//   21. CMatrixKokkos
-//   22. ViewCMatrixKokkos
-//   23. RaggedRightArrayKokkos
-//   24. RaggedDownArrayKokkos
-//   25. DynamicRaggedRightArrayKokkos
-//   26. DynamicRaggedDownArrayKokkos
-//   27. CSRArrayKokkos
-//   28. CSCArrayKokkos
-//   29. DViewCArrayKokkos
-//   30. DViewCMatrixKokkos
-//   31. DViewFArrayKokkos
-//   32. DViewFMatrixKokkos
+int main(int argc, char* argv[])
+{
+    Kokkos::initialize(argc, argv);
+    { // kokkos scope
+        const size_t num_things = 4;
+        auto array_of_things = DCArrayKokkos <thing> (num_things);
 
+        for (size_t i = 0; i < num_things; i++) {
+            array_of_things.host(i).inner = (DCArrayKokkos<int>*)Kokkos::kokkos_malloc(sizeof(DCArrayKokkos<int>));
+        }
+        // Update device side of array of memory location on GPU
+        array_of_things.update_device();
 
-#include "macros.h"
-#include "host_types.h"
-#include "kokkos_types.h"
-#include "aliases.h"
+/*
+        // Create shapes using `placement new`. Even=Circle, Odd=Square. Radius=i, Length=i.
+        FOR_ALL(i, 0, num_things, {
+            new ((DCArrayKokkos*)array_of_things(i).inner) DCArrayKokkos<int>(8);
+        });
+        Kokkos::fence();
+        // Calculate Area
+        DCArrayKokkos<double> area_array(num_shapes);
+        FOR_ALL(i, 0, num_shapes, {
+            area_array(i) = shape_array(i).shape->area();
+        });
+        Kokkos::fence();
+        area_array.update_host();
 
+        // Check result
+        for (size_t i = 0; i < num_shapes; i++) {
+            double area;
+            if (i % 2 == 0) {
+                area = atan(1) * 4 * i * i;
+                if (area != area_array.host(i)) {
+                    printf("Circle radius=%.3f, calc_area=%.3f, actual_area=%.3f\n", i, area_array.host(i), area);
+                }
+            }
+            else {
+                area = i * i;
+                if (area != area_array.host(i)) {
+                    printf("Square length=%.3f, calc_area=%.3f, actual_area=%.3f\n", i, area_array.host(i), area);
+                }
+            }
 
+            if (area != area_array.host(i)) {
+                throw std::runtime_error("calculated area NOT EQUAL actual area");
+            }
+        }
+        // Destroy shapes
+        FOR_ALL(i, 0, num_things, {
+            array_of_things(i).inner->~DCArrayKokkos();
+        });
+        Kokkos::fence();
 
-#endif // MATAR_H
+        // Free GPU memory
+        for (size_t i = 0; i < num_shapes; i++) {
+            Kokkos::kokkos_free(array_of_things.host(i).inner);
+        }
+*/
+
+        printf("COMPLETED SUCCESSFULLY!!!\n");
+    } // end kokkos scope
+    Kokkos::finalize();
+
+    return 0;
+}

examples/halfspace_cooling/CMakeLists.txt

@@ -9,5 +9,5 @@ if (KOKKOS)
 
   add_definitions(-DHAVE_KOKKOS=1)
 
-  target_link_libraries(halfspace_cooling matar Kokkos::kokkos)
+  target_link_libraries(halfspace_cooling ${LINKING_LIBRARIES})
 endif()

examples/kokkos_for.cpp

@@ -47,22 +47,22 @@ int main()
     // Kokkos::View<int *> arr("ARR", 10);
     CArrayKokkos<int> arr(10);
     FOR_ALL(i, 0, 10, {
-        arr(i) = 314;
+        arr(i) = i;
     });
 
     // Kokkos::View<int **> arr_2D("ARR_2D", 10,10);
     CArrayKokkos<int> arr_2D(10, 10);
     FOR_ALL(i, 0, 10,
          j, 0, 10, {
-        arr_2D(i, j) = 314;
+        arr_2D(i, j) = j * 10 + i;
     });
 
     // Kokkos::View<int ***> arr_3D("ARR_3D", 10,10,10);
     CArrayKokkos<int> arr_3D(10, 10, 10);
     FOR_ALL(i, 0, 10,
          j, 0, 10,
          k, 0, 10, {
-        arr_3D(i, j, k) = 314;
+        arr_3D(i, j, k) = k * 10 * 10 + j * 10 + i;
     });
 
     int loc_sum = 0;