feat(shm): add graph compression interface

CMakeLists.txt

@@ -368,6 +368,9 @@ if (KAMINPAR_BUILD_DISTRIBUTED)
     add_subdirectory(kaminpar-dist)
 endif ()
 
+# Shared-memory and distributed IO components
+add_subdirectory(kaminpar-io)
+
 # Binaries
 add_subdirectory(kaminpar-cli)
 
@@ -391,10 +394,12 @@ endif ()
 ################################################################################
 
 add_library(KaMinPar::KaMinPar ALIAS kaminpar_shm)
+add_library(KaMinPar::KaMinParIO ALIAS kaminpar_io)
 add_library(KaMinPar::KaMinParCLI ALIAS kaminpar_cli)
 
 if (KAMINPAR_BUILD_DISTRIBUTED)
     add_library(KaMinPar::dKaMinPar ALIAS kaminpar_dist)
+    add_library(KaMinPar::dKaMinParIO ALIAS kaminpar_dist_io)
     add_library(KaMinPar::dKaMinParCLI ALIAS kaminpar_dist_cli)
 endif ()
 

apps/CMakeLists.txt

@@ -1,41 +1,7 @@
-set(COMMON_IO_SOURCE_FILES
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/file_toker.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/binary_util.h)
-
-set(KAMINPAR_IO_SOURCE_FILES
-    ${COMMON_IO_SOURCE_FILES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_metis_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_metis_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_parhip_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_parhip_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_compressed_graph_binary.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_compressed_graph_binary.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_input_validator.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_input_validator.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_io.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_io.cc)
-
-set(DKAMINPAR_IO_SOURCE_FILES  
-    ${COMMON_IO_SOURCE_FILES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_io.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_skagen.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_skagen.cc)
-
 set(COMMON_APP_SOURCE_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/version.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/version.h)
 
-add_library(kaminpar_io ${KAMINPAR_IO_SOURCE_FILES})
-target_include_directories(kaminpar_io PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
-target_link_libraries(kaminpar_io PUBLIC KaMinPar::KaMinPar KaMinPar::KaMinParCLI)
-target_compile_options(kaminpar_io PRIVATE ${KAMINPAR_WARNING_FLAGS})
-
-add_library(KaMinPar::KaMinParIO ALIAS kaminpar_io)
-
 function(add_shm_app target)
     add_executable(${target} ${ARGN} ${COMMON_APP_SOURCE_FILES})
     target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
@@ -48,7 +14,7 @@ endfunction()
 function(add_dist_app target)
     if (TARGET kaminpar_dist)
         add_shm_app(${target} ${ARGN})
-        target_link_libraries(${target} PRIVATE KaMinPar::dKaMinPar KaMinPar::dKaMinParCLI)
+        target_link_libraries(${target} PRIVATE KaMinPar::dKaMinPar KaMinPar::dKaMinParCLI KaMinPar::dKaMinParIO KaGen::KaGen)
     else ()
         message(STATUS "Disabled app: ${target}")
     endif ()
@@ -57,18 +23,7 @@ endfunction()
 add_shm_app(KaMinPar KaMinPar.cc)
 
 if (TARGET kaminpar_dist)
-    add_library(kaminpar_dist_io ${DKAMINPAR_IO_SOURCE_FILES})
-    target_include_directories(kaminpar_dist_io PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
-    target_link_libraries(kaminpar_dist_io PUBLIC 
-        KaMinPar::dKaMinPar 
-        KaMinPar::dKaMinParCLI 
-        KaGen::KaGen)
-    target_compile_options(kaminpar_dist_io PRIVATE ${KAMINPAR_WARNING_FLAGS})
-
-    add_library(KaMinPar::dKaMinParIO ALIAS kaminpar_dist_io)
-
     add_dist_app(dKaMinPar dKaMinPar.cc)
-    target_link_libraries(dKaMinPar PRIVATE KaMinPar::dKaMinParIO)
 
     if (KAMINPAR_BUILD_WITH_BACKWARD)
         target_link_libraries(dKaMinPar PRIVATE Backward::Backward)

apps/KaMinPar.cc

@@ -10,6 +10,7 @@
 #include "kaminpar-shm/kaminpar.h"
 // clang-format on
 
+#include <cstdlib>
 #include <iostream>
 #include <span>
 
@@ -19,17 +20,18 @@
 #include <numa.h>
 #endif // __has_include(<numa.h>)
 
+#include "kaminpar-io/io.h"
+#include "kaminpar-io/metis_parser.h"
+#include "kaminpar-io/parhip_parser.h"
+
 #include "kaminpar-shm/datastructures/graph.h"
+#include "kaminpar-shm/graphutils/graph_validator.h"
 #include "kaminpar-shm/graphutils/permutator.h"
 
 #include "kaminpar-common/heap_profiler.h"
 #include "kaminpar-common/strutils.h"
 #include "kaminpar-common/timer.h"
 
-#include "apps/io/shm_input_validator.h"
-#include "apps/io/shm_io.h"
-#include "apps/io/shm_metis_parser.h"
-#include "apps/io/shm_parhip_parser.h"
 #include "apps/version.h"
 
 #if defined(__linux__)
@@ -279,10 +281,14 @@ inline void output_partition(const ApplicationContext &app, const std::vector<Bl
 inline void
 output_rearranged_graph(const ApplicationContext &app, const std::vector<BlockID> &partition) {
   if (!app.rearranged_graph_filename.empty()) {
-    Graph graph =
+    auto graph =
         io::read(app.graph_filename, app.input_graph_file_format, NodeOrdering::NATURAL, false);
-    auto &csr_graph = graph.concretize<CSRGraph>();
+    if (!graph) {
+      LOG_ERROR << "Could not output rearranged graph as the input graph cannot be read.";
+      return;
+    }
 
+    auto &csr_graph = graph->concretize<CSRGraph>();
     auto permutations = shm::graph::compute_node_permutation_by_generic_buckets(
         csr_graph.n(), app.k, [&](const NodeID u) { return partition[u]; }
     );
@@ -409,9 +415,17 @@ int main(int argc, char *argv[]) {
   // Read the input graph and allocate memory for the partition
   START_HEAP_PROFILER("Input Graph Allocation");
   Graph graph = TIMED_SCOPE("Read input graph") {
-    return io::read(
-        app.graph_filename, app.input_graph_file_format, ctx.node_ordering, ctx.compression.enabled
-    );
+    if (auto graph = io::read(
+            app.graph_filename,
+            app.input_graph_file_format,
+            ctx.node_ordering,
+            ctx.compression.enabled
+        )) {
+      return std::move(*graph);
+    }
+
+    LOG_ERROR << "Failed to read the input graph.";
+    std::exit(EXIT_FAILURE);
   };
 
   if (app.ignore_edge_weights && !ctx.compression.enabled) {

apps/benchmarks/CMakeLists.txt

@@ -1,9 +1,9 @@
 function(add_shm_benchmark target)
     add_executable(${target} ${ARGN})
-    target_link_libraries(${target} PRIVATE 
-        KaMinPar::KaMinPar 
-        KaMinPar::KaMinParCLI 
-        KaMinPar::KaMinParIO 
+    target_link_libraries(${target} PRIVATE
+        KaMinPar::KaMinPar
+        KaMinPar::KaMinParCLI
+        KaMinPar::KaMinParIO
         KaGen::KaGen)
     target_compile_options(${target} PRIVATE ${KAMINPAR_WARNING_FLAGS})
     message(STATUS "Enabled benchmark: ${target}")
@@ -19,8 +19,8 @@ add_shm_benchmark(shm_compressed_graph_benchmark shm_compressed_graph_benchmark.
 if (KAMINPAR_BUILD_DISTRIBUTED)
     function(add_dist_benchmark target)
         add_shm_benchmark(${target} ${ARGN})
-        target_link_libraries(${target} PRIVATE 
-            KaMinPar::dKaMinPar 
+        target_link_libraries(${target} PRIVATE
+            KaMinPar::dKaMinPar
             KaMinPar::dKaMinParCLI
             KaMinPar::dKaMinParIO)
     endfunction()

apps/benchmarks/shm_compressed_graph_benchmark.cc

@@ -8,16 +8,15 @@
 #include <limits>
 
 #include "kaminpar-cli/CLI11.h"
+#include "kaminpar-io/io.h"
 
-#include "kaminpar-shm/graphutils/parallel_compressed_graph_builder.h"
+#include "kaminpar-shm/graphutils/compressed_graph_builder.h"
 #include "kaminpar-shm/kaminpar.h"
 
 #include "kaminpar-common/console_io.h"
 #include "kaminpar-common/logger.h"
 #include "kaminpar-common/timer.h"
 
-#include "apps/io/shm_io.h"
-
 using namespace kaminpar;
 using namespace kaminpar::shm;
 using namespace kaminpar::shm::io;
@@ -137,25 +136,30 @@ int main(int argc, char *argv[]) {
 
   // Read input graph
   LOG << "Reading the input graph...";
-  CSRGraph graph = io::csr_read(graph_filename, graph_file_format, NodeOrdering::NATURAL);
+  auto graph = io::csr_read(graph_filename, graph_file_format, NodeOrdering::NATURAL);
+  if (!graph) {
+    LOG_ERROR << "Failed to read the input graph";
+    return EXIT_FAILURE;
+  }
 
   LOG << "Compressing the input graph...";
-  CompressedGraph compressed_graph = parallel_compress(graph);
+  CompressedGraph compressed_graph = parallel_compress(*graph);
 
   // Run benchmarks
   LOG << "Running the benchmarks...";
   GLOBAL_TIMER.reset();
-  run_benchmark(graph, compressed_graph);
+  run_benchmark(*graph, compressed_graph);
   STOP_TIMER();
 
   // Print the result summary
   LOG;
   cio::print_delimiter("Result Summary");
 
-  LOG << "Input graph has " << graph.n() << " vertices and " << graph.m()
-      << " edges. Its density is " << ((graph.m()) / (float)(graph.n() * (graph.n() - 1))) << ".";
-  LOG << "Node weights: " << (graph.is_node_weighted() ? "yes" : "no")
-      << ", edge weights: " << (graph.is_edge_weighted() ? "yes" : "no");
+  LOG << "Input graph has " << graph->n() << " vertices and " << graph->m()
+      << " edges. Its density is " << ((graph->m()) / (float)(graph->n() * (graph->n() - 1)))
+      << ".";
+  LOG << "Node weights: " << (graph->is_node_weighted() ? "yes" : "no")
+      << ", edge weights: " << (graph->is_edge_weighted() ? "yes" : "no");
   LOG;
 
   Timer::global().print_human_readable(std::cout);

apps/benchmarks/shm_input_benchmark.cc

@@ -11,17 +11,16 @@
 
 #include <tbb/global_control.h>
 
+#include "kaminpar-io/io.h"
+
 #include "kaminpar-shm/context_io.h"
 #include "kaminpar-shm/graphutils/compressed_graph_builder.h"
-#include "kaminpar-shm/graphutils/parallel_compressed_graph_builder.h"
 
 #include "kaminpar-common/console_io.h"
 #include "kaminpar-common/logger.h"
 #include "kaminpar-common/random.h"
 #include "kaminpar-common/timer.h"
 
-#include "apps/io/shm_io.h"
-
 using namespace kaminpar;
 using namespace kaminpar::shm;
 
@@ -35,6 +34,7 @@ int main(int argc, char *argv[]) {
   io::GraphFileFormat graph_file_format = io::GraphFileFormat::METIS;
   bool compress_in_memory = false;
   int seed = 0;
+  double epsilon = 0.03;
 
   CLI::App app("Shared-memory input benchmark");
   app.add_option("-G,--graph", graph_filename, "Graph file")->required();
@@ -61,7 +61,7 @@ int main(int argc, char *argv[]) {
       ->capture_default_str();
   app.add_option("-s,--seed", seed, "Seed for random number generation")->capture_default_str();
   app.add_option("-k,--k", ctx.partition.k);
-  app.add_option("-e,--epsilon", ctx.partition.epsilon);
+  app.add_option("-e,--epsilon", epsilon)->capture_default_str();
   create_graph_compression_options(&app, ctx);
   CLI11_PARSE(app, argc, argv);
 
@@ -75,27 +75,35 @@ int main(int argc, char *argv[]) {
     SCOPED_HEAP_PROFILER("Read Input Graph");
 
     if (ctx.compression.enabled && compress_in_memory) {
-      CSRGraph csr_graph = TIMED_SCOPE("Read CSR Graph") {
+      auto csr_graph = TIMED_SCOPE("Read CSR Graph") {
         SCOPED_HEAP_PROFILER("Read CSR Graph");
         return io::csr_read(graph_filename, graph_file_format, ctx.node_ordering);
       };
+      if (!csr_graph) {
+        LOG_ERROR << "Failed to read and compress the input graph";
+        std::exit(EXIT_FAILURE);
+      }
 
       SCOPED_TIMER("Compress CSR Graph");
       SCOPED_HEAP_PROFILER("Compress CSR Graph");
 
       const bool sequential_compression = ctx.parallel.num_threads <= 1;
       if (sequential_compression) {
-        Graph graph =
-            Graph(std::make_unique<CompressedGraph>(CompressedGraphBuilder::compress(csr_graph)));
-        ctx.setup(graph);
+        Graph graph = Graph(std::make_unique<CompressedGraph>(compress(*csr_graph)));
+        ctx.partition.setup(graph, ctx.partition.k, epsilon);
       } else {
-        Graph graph = Graph(std::make_unique<CompressedGraph>(parallel_compress(csr_graph)));
-        ctx.setup(graph);
+        Graph graph = Graph(std::make_unique<CompressedGraph>(parallel_compress(*csr_graph)));
+        ctx.partition.setup(graph, ctx.partition.k, epsilon);
       }
     } else {
-      Graph graph =
+      auto graph =
           io::read(graph_filename, graph_file_format, ctx.node_ordering, ctx.compression.enabled);
-      ctx.setup(graph);
+      if (!graph) {
+        LOG_ERROR << "Failed to read the input graph";
+        std::exit(EXIT_FAILURE);
+      }
+
+      ctx.partition.setup(*graph, ctx.partition.k, epsilon);
     }
   };