feat(shm): add graph compression interface

CMakeLists.txt

@@ -29,6 +29,8 @@ option(KAMINPAR_BUILD_EXAMPLES "Build examples." OFF)
 
 option(KAMINPAR_BUILD_EXPERIMENTAL_FEATURES "Include experimental features in the build. This might increase compile times drastically." OFF)
 
+option(KAMINPAR_DOWNLOAD_TBB "Download TBB instead of requiring it to be installed." OFF)
+
 # Control how to build
 ######################
 option(KAMINPAR_ENABLE_HEAP_PROFILING "Profile and output heap memory usage." OFF)
@@ -299,7 +301,6 @@ if (CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     include(CTest)
 endif()
 include(FetchContent)
-include(FindGit)
 
 # Fetch KAssert for assertions
 FetchContent_Declare(
@@ -311,6 +312,20 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(kassert)
 
+# Intel TBB
+if (KAMINPAR_DOWNLOAD_TBB)
+    FetchContent_Declare(
+        TBB
+        GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB.git
+        GIT_TAG 0c0ff192a2304e114bc9e6557582dfba101360ff # v2022.0.0
+        EXCLUDE_FROM_ALL
+        UPDATE_DISCONNECTED TRUE
+    )
+    FetchContent_MakeAvailable(TBB)
+else ()
+    find_package(TBB REQUIRED)
+endif ()
+
 # Google Sparsehash
 if (KAMINPAR_BUILD_WITH_SPARSEHASH)
     find_package(Sparsehash REQUIRED)
@@ -408,6 +423,9 @@ if (KAMINPAR_BUILD_DISTRIBUTED)
     add_subdirectory(kaminpar-dist)
 endif ()
 
+# Shared-memory and distributed IO components
+add_subdirectory(kaminpar-io)
+
 # Binaries
 add_subdirectory(kaminpar-cli)
 
@@ -436,13 +454,13 @@ include(CMakePackageConfigHelpers)
 include(GNUInstallDirs)
 
 install(
-    FILES kaminpar-shm/kaminpar.h
+    DIRECTORY include/
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
     COMPONENT KaMinPar_Development
 )
 
 install(
-    TARGETS kassert_base kassert kaminpar_common kaminpar_shm
+    TARGETS kassert_base kassert kaminpar_common kaminpar_shm kaminpar_io
     EXPORT KaMinParTargets
     RUNTIME COMPONENT KaMinPar_Runtime
     LIBRARY COMPONENT KaMinPar_Runtime NAMELINK_COMPONENT KaMinPar_Development

apps/CMakeLists.txt

@@ -1,41 +1,7 @@
-set(COMMON_IO_SOURCE_FILES
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/file_toker.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/binary_util.h)
-
-set(KAMINPAR_IO_SOURCE_FILES
-    ${COMMON_IO_SOURCE_FILES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_metis_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_metis_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_parhip_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_parhip_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_compressed_graph_binary.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_compressed_graph_binary.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_input_validator.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_input_validator.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_io.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_io.cc)
-
-set(DKAMINPAR_IO_SOURCE_FILES  
-    ${COMMON_IO_SOURCE_FILES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_io.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_skagen.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_skagen.cc)
-
 set(COMMON_APP_SOURCE_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/version.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/version.h)
 
-add_library(kaminpar_io ${KAMINPAR_IO_SOURCE_FILES})
-target_include_directories(kaminpar_io PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
-target_link_libraries(kaminpar_io PUBLIC KaMinPar::KaMinPar KaMinPar::KaMinParCLI)
-target_compile_options(kaminpar_io PRIVATE ${KAMINPAR_WARNING_FLAGS})
-
-add_library(KaMinPar::KaMinParIO ALIAS kaminpar_io)
-
 function(add_shm_app target)
     add_executable(${target} ${ARGN} ${COMMON_APP_SOURCE_FILES})
     target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
@@ -48,7 +14,7 @@ endfunction()
 function(add_dist_app target)
     if (TARGET kaminpar_dist)
         add_shm_app(${target} ${ARGN})
-        target_link_libraries(${target} PRIVATE KaMinPar::dKaMinPar KaMinPar::dKaMinParCLI)
+        target_link_libraries(${target} PRIVATE KaMinPar::dKaMinPar KaMinPar::dKaMinParCLI KaMinPar::dKaMinParIO KaGen::KaGen)
     else ()
         message(STATUS "Disabled app: ${target}")
     endif ()
@@ -57,18 +23,7 @@ endfunction()
 add_shm_app(KaMinPar KaMinPar.cc)
 
 if (TARGET kaminpar_dist)
-    add_library(kaminpar_dist_io ${DKAMINPAR_IO_SOURCE_FILES})
-    target_include_directories(kaminpar_dist_io PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
-    target_link_libraries(kaminpar_dist_io PUBLIC 
-        KaMinPar::dKaMinPar 
-        KaMinPar::dKaMinParCLI 
-        KaGen::KaGen)
-    target_compile_options(kaminpar_dist_io PRIVATE ${KAMINPAR_WARNING_FLAGS})
-
-    add_library(KaMinPar::dKaMinParIO ALIAS kaminpar_dist_io)
-
     add_dist_app(dKaMinPar dKaMinPar.cc)
-    target_link_libraries(dKaMinPar PRIVATE KaMinPar::dKaMinParIO)
 
     if (KAMINPAR_BUILD_WITH_BACKWARD)
         target_link_libraries(dKaMinPar PRIVATE Backward::Backward)

apps/KaMinPar.cc

@@ -7,9 +7,9 @@
  ******************************************************************************/
 // clang-format off
 #include "kaminpar-cli/kaminpar_arguments.h"
-#include "kaminpar-shm/kaminpar.h"
 // clang-format on
 
+#include <cstdlib>
 #include <iostream>
 #include <span>
 
@@ -19,17 +19,16 @@
 #include <numa.h>
 #endif // __has_include(<numa.h>)
 
-#include "kaminpar-shm/datastructures/graph.h"
+#include "kaminpar-io/kaminpar_io.h"
+
+#include "kaminpar-shm/graphutils/graph_validator.h"
 #include "kaminpar-shm/graphutils/permutator.h"
+#include "kaminpar-shm/kaminpar.h"
 
 #include "kaminpar-common/heap_profiler.h"
 #include "kaminpar-common/strutils.h"
 #include "kaminpar-common/timer.h"
 
-#include "apps/io/shm_input_validator.h"
-#include "apps/io/shm_io.h"
-#include "apps/io/shm_metis_parser.h"
-#include "apps/io/shm_parhip_parser.h"
 #include "apps/version.h"
 
 #if defined(__linux__)
@@ -164,7 +163,14 @@ The output should be stored in a file and can be used by the -C,--config option.
     app.max_timer_depth = std::numeric_limits<int>::max();
   });
   cli.add_option("-f,--graph-file-format,--input-graph-file-format", app.input_graph_file_format)
-      ->transform(CLI::CheckedTransformer(io::get_graph_file_formats()).description(""))
+      ->transform(CLI::CheckedTransformer(
+          std::unordered_map<std::string, io::GraphFileFormat>{
+              {"metis", io::GraphFileFormat::METIS},
+              {"parhip", io::GraphFileFormat::PARHIP},
+              {"compressed", io::GraphFileFormat::PARHIP},
+          },
+          CLI::ignore_case
+      ))
       ->description(R"(Graph file formats:
   - metis
   - parhip)")
@@ -223,7 +229,14 @@ The output should be stored in a file and can be used by the -C,--config option.
   )
       ->capture_default_str();
   cli.add_option("--output-graph-file-format", app.output_graph_file_format)
-      ->transform(CLI::CheckedTransformer(io::get_graph_file_formats()).description(""))
+      ->transform(CLI::CheckedTransformer(
+          std::unordered_map<std::string, io::GraphFileFormat>{
+              {"metis", io::GraphFileFormat::METIS},
+              {"parhip", io::GraphFileFormat::PARHIP},
+              {"compressed", io::GraphFileFormat::PARHIP},
+          },
+          CLI::ignore_case
+      ))
       ->description(R"(Graph file formats:
   - metis
   - parhip)")
@@ -268,61 +281,51 @@ The output should be stored in a file and can be used by the -C,--config option.
 }
 
 inline void
-output_block_sizes(const ApplicationContext &app, const std::vector<BlockID> &partition) {
-  shm::io::partition::write_block_sizes(app.block_sizes_filename, app.k, partition);
-}
+output_rearranged_graph(const ApplicationContext &app, const std::vector<BlockID> &partition) {
+  if (app.rearranged_graph_filename.empty()) {
+    return;
+  }
 
-inline void output_partition(const ApplicationContext &app, const std::vector<BlockID> &partition) {
-  shm::io::partition::write(app.partition_filename, partition);
-}
+  auto graph = io::read_graph(app.graph_filename, app.input_graph_file_format);
+  if (!graph) {
+    LOG_ERROR << "Could not output rearranged graph as the input graph cannot be read.";
+    return;
+  }
 
-inline void
-output_rearranged_graph(const ApplicationContext &app, const std::vector<BlockID> &partition) {
-  if (!app.rearranged_graph_filename.empty()) {
-    Graph graph =
-        io::read(app.graph_filename, app.input_graph_file_format, NodeOrdering::NATURAL, false);
-    auto &csr_graph = graph.concretize<CSRGraph>();
+  auto &csr_graph = graph.value().csr_graph();
+  auto permutations = shm::graph::compute_node_permutation_by_generic_buckets(
+      csr_graph.n(), app.k, [&](const NodeID u) { return partition[u]; }
+  );
 
-    auto permutations = shm::graph::compute_node_permutation_by_generic_buckets(
-        csr_graph.n(), app.k, [&](const NodeID u) { return partition[u]; }
-    );
+  if (!app.rearranged_mapping_filename.empty()) {
+    io::write_remapping(app.rearranged_mapping_filename, permutations.old_to_new);
+  }
 
-    if (!app.rearranged_mapping_filename.empty()) {
-      shm::io::remapping::write(app.rearranged_mapping_filename, permutations.old_to_new);
-    }
+  StaticArray<EdgeID> tmp_nodes(csr_graph.raw_nodes().size());
+  StaticArray<NodeID> tmp_edges(csr_graph.raw_edges().size());
+  StaticArray<NodeWeight> tmp_node_weights(csr_graph.raw_node_weights().size());
+  StaticArray<EdgeWeight> tmp_edge_weights(csr_graph.raw_edge_weights().size());
+
+  shm::graph::build_permuted_graph(
+      csr_graph.raw_nodes(),
+      csr_graph.raw_edges(),
+      csr_graph.raw_node_weights(),
+      csr_graph.raw_edge_weights(),
+      permutations,
+      tmp_nodes,
+      tmp_edges,
+      tmp_node_weights,
+      tmp_edge_weights
+  );
 
-    StaticArray<EdgeID> tmp_nodes(csr_graph.raw_nodes().size());
-    StaticArray<NodeID> tmp_edges(csr_graph.raw_edges().size());
-    StaticArray<NodeWeight> tmp_node_weights(csr_graph.raw_node_weights().size());
-    StaticArray<EdgeWeight> tmp_edge_weights(csr_graph.raw_edge_weights().size());
-
-    shm::graph::build_permuted_graph(
-        csr_graph.raw_nodes(),
-        csr_graph.raw_edges(),
-        csr_graph.raw_node_weights(),
-        csr_graph.raw_edge_weights(),
-        permutations,
-        tmp_nodes,
-        tmp_edges,
-        tmp_node_weights,
-        tmp_edge_weights
-    );
+  Graph permuted_graph = {std::make_unique<CSRGraph>(
+      std::move(tmp_nodes),
+      std::move(tmp_edges),
+      std::move(tmp_node_weights),
+      std::move(tmp_edge_weights)
+  )};
 
-    Graph permuted_graph = {std::make_unique<CSRGraph>(
-        std::move(tmp_nodes),
-        std::move(tmp_edges),
-        std::move(tmp_node_weights),
-        std::move(tmp_edge_weights)
-    )};
-
-    if (app.output_graph_file_format == io::GraphFileFormat::METIS) {
-      shm::io::metis::write(app.rearranged_graph_filename, permuted_graph);
-    } else if (app.output_graph_file_format == io::GraphFileFormat::PARHIP) {
-      shm::io::parhip::write(app.rearranged_graph_filename, permuted_graph.concretize<CSRGraph>());
-    } else {
-      LOG_WARNING << "Unsupported output graph format";
-    }
-  }
+  io::write_graph(app.rearranged_graph_filename, app.output_graph_file_format, permuted_graph);
 }
 
 inline void print_rss(const ApplicationContext &app) {
@@ -409,19 +412,27 @@ int main(int argc, char *argv[]) {
   // Read the input graph and allocate memory for the partition
   START_HEAP_PROFILER("Input Graph Allocation");
   Graph graph = TIMED_SCOPE("Read input graph") {
-    return io::read(
-        app.graph_filename, app.input_graph_file_format, ctx.node_ordering, ctx.compression.enabled
-    );
+    if (auto graph = io::read_graph(
+            app.graph_filename,
+            app.input_graph_file_format,
+            ctx.compression.enabled,
+            ctx.node_ordering
+        )) {
+      return std::move(*graph);
+    }
+
+    LOG_ERROR << "Failed to read the input graph.";
+    std::exit(EXIT_FAILURE);
   };
 
   if (app.ignore_edge_weights && !ctx.compression.enabled) {
-    auto &csr_graph = graph.concretize<CSRGraph>();
-    graph = {std::make_unique<CSRGraph>(
+    auto &csr_graph = graph.csr_graph();
+    graph = Graph(std::make_unique<CSRGraph>(
         csr_graph.take_raw_nodes(),
         csr_graph.take_raw_edges(),
         csr_graph.take_raw_node_weights(),
-        StaticArray<EdgeWeight>{}
-    )};
+        StaticArray<EdgeWeight>()
+    ));
   } else if (app.ignore_edge_weights) {
     LOG_WARNING << "Ignoring edge weights is currently only supported for uncompressed graphs; "
                    "ignoring option.";
@@ -480,15 +491,15 @@ int main(int argc, char *argv[]) {
 
   // Save graph partition
   if (!app.partition_filename.empty()) {
-    output_partition(app, partition);
+    io::write_partition(app.partition_filename, partition);
   }
 
-  if (!app.rearranged_graph_filename.empty()) {
-    output_rearranged_graph(app, partition);
+  if (!app.block_sizes_filename.empty()) {
+    io::write_block_sizes(app.block_sizes_filename, app.k, partition);
   }
 
-  if (!app.block_sizes_filename.empty()) {
-    output_block_sizes(app, partition);
+  if (!app.rearranged_graph_filename.empty()) {
+    output_rearranged_graph(app, partition);
   }
 
   DISABLE_HEAP_PROFILER();

apps/benchmarks/CMakeLists.txt

@@ -1,9 +1,9 @@
 function(add_shm_benchmark target)
     add_executable(${target} ${ARGN})
-    target_link_libraries(${target} PRIVATE 
-        KaMinPar::KaMinPar 
-        KaMinPar::KaMinParCLI 
-        KaMinPar::KaMinParIO 
+    target_link_libraries(${target} PRIVATE
+        KaMinPar::KaMinPar
+        KaMinPar::KaMinParCLI
+        KaMinPar::KaMinParIO
         KaGen::KaGen)
     target_compile_options(${target} PRIVATE ${KAMINPAR_WARNING_FLAGS})
     message(STATUS "Enabled benchmark: ${target}")
@@ -17,8 +17,8 @@ add_shm_benchmark(shm_compressed_graph_benchmark shm_compressed_graph_benchmark.
 if (KAMINPAR_BUILD_DISTRIBUTED)
     function(add_dist_benchmark target)
         add_shm_benchmark(${target} ${ARGN})
-        target_link_libraries(${target} PRIVATE 
-            KaMinPar::dKaMinPar 
+        target_link_libraries(${target} PRIVATE
+            KaMinPar::dKaMinPar
             KaMinPar::dKaMinParCLI
             KaMinPar::dKaMinParIO)
     endfunction()