Align results with those of labs/benchmarks and the Conbench API

.github/workflows/R-CMD-check.yaml

@@ -1,4 +1,8 @@
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 name: R-CMD-check
 

DESCRIPTION

@@ -1,10 +1,11 @@
 Package: arrowbench
 Type: Package
 Title: Tools for Continuous and Interactive Benchmarking
-Version: 0.1.0
+Version: 0.2.0
 Authors@R: c(
     person("Neal", "Richardson", email = "neal.p.richardson@gmail.com", role = "aut"),
-    person("Jonathan", "Keane", email = "jkeane@gmail.com", role = c("aut", "cre"))
+    person("Jonathan", "Keane", email = "jkeane@gmail.com", role = c("aut", "cre")),
+    person("Edward", "Visel", email = "edward.visel@gmail.com", role = "aut", comment = c(ORCID = "0000-0002-2811-6254"))
   )
 Description: Tools for defining benchmarks, running them across a 
     range of parameters, and reporting their results in a standardized form.
@@ -14,6 +15,7 @@ Depends: R (>= 3.5.0)
 Imports:
     arrow,
     bench,
+    callr,
     dbplyr,
     dplyr,
     distro,
@@ -26,9 +28,9 @@ Imports:
     R.utils,
     sessioninfo,
     utils,
+    uuid,
     waldo,
-    withr,
-    callr
+    withr
 Suggests:
     testthat (>= 3.0.0),
     archive,
@@ -43,7 +45,7 @@ Suggests:
     RcppSimdJson,
     readr,
     vroom
-RoxygenNote: 7.1.2
+RoxygenNote: 7.2.2
 Roxygen: list(markdown = TRUE, load = "source")
 Collate: 
     'benchmark.R'

R/benchmark.R

@@ -86,6 +86,12 @@
 #' single case and returning an integer version for the case, or `NULL` to not
 #' append a version; `NA` will raise an error. Changes to version will break
 #' conbench history for a case.
+#' @param batch_id_fun A unary function which takes a dataframe of parameters and
+#' returns a character vector to use as `batch_id` of length 1 or `nrow(params)`
+#' @param tags_fun A unary function which takes a named list of setup parameters
+#' for a single case and returns a named list of tags to send to Conbench for that
+#' case. Can be overwritten to add static tags or postprocess parameters into more
+#' readable forms.
 #' @param packages_used function taking a `data.frame` of setup parameters and
 #' returning a vector of R package names required
 #' @param ... additional attributes or functions, possibly called in `setup()`.
@@ -100,6 +106,8 @@ Benchmark <- function(name,
                       teardown = TRUE,
                       valid_params = function(params) params,
                       case_version = function(params) NULL,
+                      batch_id_fun = function(params) uuid(),
+                      tags_fun = function(params) params,
                       packages_used = function(params) "arrow",
                       ...) {
   stopifnot(is.character(name))
@@ -113,6 +121,8 @@ Benchmark <- function(name,
       teardown = substitute(teardown),
       valid_params = valid_params,
       case_version = case_version,
+      batch_id_fun = batch_id_fun,
+      tags_fun = tags_fun,
       packages_used = packages_used,
       ...),
     class = "Benchmark"
@@ -164,12 +174,12 @@ default_params <- function(bm, ...) {
 #' error status
 #'
 #' @param run An instance of `BenchmarkResults` as returned by `run_benchmark`
-#' or `BenchmarkResult` or `BenchmarkFailure` as returned by `run_one` and `run_bm`
+#' or `BenchmarkResult` as returned by `run_one` and `run_bm`
 #' @return a tibble
 #' @export
 get_params_summary <- function(run) {
-  if (!inherits(run, c("BenchmarkResults", "BenchmarkResult", "BenchmarkFailure"))) {
-    stop("run objects need to be of class BenchmarkResults, BenchmarkResult, or BenchmarkFailure")
+  if (!inherits(run, c("BenchmarkResults", "BenchmarkResult"))) {
+    stop("run objects need to be of class BenchmarkResults or BenchmarkResult")
   }
   run$params_summary
 }

R/bm-dataset-taxi-parquet.R

@@ -4,7 +4,7 @@
 #' * `query` Name of a known query to run; see `dataset_taxi_parquet$cases`
 #'
 #' @export
-dataset_taxi_parquet <- Benchmark("dataset_taxi_parquet",
+dataset_taxi_parquet <- Benchmark("partitioned-dataset-filter",
   setup = function(query = names(dataset_taxi_parquet$cases)) {
     library("dplyr", warn.conflicts = FALSE)
     dataset <- ensure_dataset("taxi_parquet")

R/bm-df-to-table.R

@@ -6,8 +6,13 @@
 #' * `source` A known-file id to use (it will be read in to a data.frame first)
 #'
 #' @export
-df_to_table <- Benchmark("df_to_table",
-  setup = function(source = names(known_sources)) {
+df_to_table <- Benchmark("dataframe-to-table",
+  setup = function(source = c("chi_traffic_2020_Q1",
+                              "type_strings",
+                              "type_dict",
+                              "type_integers",
+                              "type_floats",
+                              "type_nested")) {
     source <- ensure_source(source)
     result_dim <- get_source_attr(source, "dim")
     # Make sure that we're not (accidentally) creating altrep vectors which will

R/bm-read-file.R

@@ -2,35 +2,35 @@
 #'
 #' @section Parameters:
 #' * `source` A known-file id, or a CSV(?) file path to read in
-#' * `format` One of `c("parquet", "feather", "fst")`
+#' * `file_type` One of `c("parquet", "feather", "fst")`
 #' * `compression` One of the values: `r paste(known_compressions, collapse = ", ")`
-#' * `output` One of `c("arrow_table", "data_frame")`
+#' * `output_type` One of `c("arrow_table", "data_frame")`
 #'
 #' @export
-read_file <- Benchmark("read_file",
-  setup = function(source = names(known_sources),
+read_file <- Benchmark("file-read",
+  setup = function(source = c("fanniemae_2016Q4", "nyctaxi_2010-01"),
                    # TODO: break out feather_v1 and feather_v2, feather_v2 only in >= 0.17
-                   format = c("parquet", "feather"),
+                   file_type = c("parquet", "feather"),
                    compression = c("uncompressed", "snappy", "lz4"),
-                   output = c("arrow_table", "data_frame")) {
-    # format defaults to parquet or feather, but can accept fst as well
-    format <- match.arg(format, c("parquet", "feather", "fst"))
-    # the output defaults are retrieved from the function definition (arrow_table and data_frame)
-    output <- match.arg(output)
+                   output_type = c("arrow_table", "data_frame")) {
+    # file_type defaults to parquet or feather, but can accept fst as well
+    file_type <- match.arg(file_type, c("parquet", "feather", "fst"))
+    # the output_type defaults are retrieved from the function definition (arrow_table and data_frame)
+    output_type <- match.arg(output_type)
 
     # ensure that we have the right kind of file available
-    input_file <- ensure_format(source, format, compression)
-    # retrieve the dimnesions for run-checking after the benchmark
+    input_file <- ensure_format(source, file_type, compression)
+    # retrieve the dimensions for run-checking after the benchmark
     result_dim <- get_source_attr(source, "dim")
 
-    # put the necesary variables into a BenchmarkEnvironment to be used when the
+    # put the necessary variables into a BenchmarkEnvironment to be used when the
     # benchmark is running.
     BenchEnvironment(
-      # get the correct read function for the input format
-      read_func = get_read_function(format),
+      # get the correct read function for the input file_type
+      read_func = get_read_function(file_type),
       input_file = input_file,
       result_dim = result_dim,
-      as_data_frame = output == "data_frame"
+      as_data_frame = output_type == "data_frame"
     )
   },
   # delete the results before each iteration
@@ -48,44 +48,44 @@ read_file <- Benchmark("read_file",
   },
   # validate that the parameters given are compatible
   valid_params = function(params) {
-    # make sure that the format and the compression is compatible
-    # and fst doesn't have arrow_table output
-    drop <- !validate_format(params$format, params$compression) |
-            params$output == "arrow_table" & params$format == "fst"
+    # make sure that the file_type and the compression is compatible
+    # and fst doesn't have arrow_table output_type
+    drop <- !validate_format(params$file_type, params$compression) |
+            params$output_type == "arrow_table" & params$file_type == "fst"
     params[!drop,]
   },
-  # packages used when specific formats are used
+  # packages used when specific file_types are used
   packages_used = function(params) {
     pkg_map <- c(
       "feather" = "arrow",
       "parquet" = "arrow",
       "fst" = "fst"
     )
-    pkg_map[params$format]
+    pkg_map[params$file_type]
   }
 )
 
 #' Get a reader
 #'
-#' @param format what format to read
+#' @param file_type what file_type to read
 #'
 #' @return the read function to use
 #' @export
-get_read_function <- function(format) {
+get_read_function <- function(file_type) {
   pkg_map <- c(
     "feather" = "arrow",
     "parquet" = "arrow",
     "fst" = "fst"
   )
-  library(pkg_map[[format]], character.only = TRUE, warn.conflicts = FALSE)
+  library(pkg_map[[file_type]], character.only = TRUE, warn.conflicts = FALSE)
 
-  if (format == "feather") {
+  if (file_type == "feather") {
     return(function(...) arrow::read_feather(...))
-  } else if (format == "parquet") {
+  } else if (file_type == "parquet") {
     return(function(...) arrow::read_parquet(...))
-  } else if (format == "fst") {
+  } else if (file_type == "fst") {
     return(function(..., as_data_frame) fst::read_fst(...))
   } else {
-    stop("Unsupported format: ", format, call. = FALSE)
+    stop("Unsupported file_type: ", file_type, call. = FALSE)
   }
 }

R/bm-tpc-h.R

@@ -14,7 +14,7 @@
 #'
 #' @importFrom waldo compare
 #' @export
-tpc_h <- Benchmark("tpc_h",
+tpc_h <- Benchmark("tpch",
   setup = function(engine = "arrow",
                    query_id = 1:22,
                    format = c("native", "parquet"),
@@ -135,6 +135,18 @@ tpc_h <- Benchmark("tpc_h",
       ( params$engine == "dplyr" & params$format == "native" )
     params[!drop,]
   },
+  batch_id_fun = function(params) {
+    batch_id <- uuid()
+    paste0(batch_id, "-", params$scale_factor, substr(params$format, 1, 1))
+  },
+  tags_fun = function(params) {
+    # for consistency with runs through voltrondata-labs/benchmarks
+    params$query_id <- sprintf("TPCH-%02d", params$query_id)
+    if (params$output == "data_frame") {
+      params$output <- NULL
+    }
+    params
+  },
   # packages used when specific formats are used
   packages_used = function(params) {
     c(params$engine, "dplyr", "lubridate")

R/bm-write-file.R

@@ -2,31 +2,45 @@
 #'
 #' @section Parameters:
 #' * `source` A known-file id, or a CSV(?) file path to read in
-#' * `format` One of `c("parquet", "feather", "fst")`
+#' * `file_type` One of `c("parquet", "feather", "fst")`
 #' * `compression` One of the values: `r paste(known_compressions, collapse = ", ")`
-#' * `input` One of `c("arrow_table", "data_frame")`
+#' * `input_type` One of `c("arrow_table", "data_frame")`
 #'
 #' @export
-write_file <- Benchmark("write_file",
-  setup = function(source = names(known_sources),
-                   format = c("parquet", "feather"),
+write_file <- Benchmark("file-write",
+  setup = function(source = c("fanniemae_2016Q4", "nyctaxi_2010-01"),
+                   file_type = c("parquet", "feather"),
                    compression = c("uncompressed", "snappy", "lz4"),
-                   input = c("arrow_table", "data_frame")) {
+                   input_type = c("arrow_table", "data_frame")) {
     # source defaults are retrieved from the function definition (all available
     # known_sources) and then read the source in as a data.frame
     source <- ensure_source(source)
-    df <- read_source(source, as_data_frame = match.arg(input) == "data_frame")
-    # format defaults to parquet or feather, but can accept fst as well
-    format <- match.arg(format, c("parquet", "feather", "fst"))
+    df <- read_source(source, as_data_frame = match.arg(input_type) == "data_frame")
+    # file_type defaults to parquet or feather, but can accept fst as well
+    file_type <- match.arg(file_type, c("parquet", "feather", "fst"))
 
     # Map string param name to functions
-    write_func <- get_write_function(format, compression)
+    get_write_func <- function(file_type, compression) {
+      force(compression)
+      if (file_type == "feather") {
+        return(function(...) arrow::write_feather(..., compression = compression))
+      } else if (file_type == "parquet") {
+        return(function(...) arrow::write_parquet(..., compression = compression))
+      } else if (file_type == "fst") {
+        # fst is always zstd, just a question of what level of compression
+        level <- ifelse(compression == "uncompressed", 0, 50)
+        return(function(...) fst::write_fst(..., compress = level))
+      } else {
+        stop("Unsupported file_type: ", file_type, call. = FALSE)
+      }
+    }
+    write_func <- get_write_func(file_type, compression)
 
     # put the necessary variables into a BenchmarkEnvironment to be used when
     # the benchmark is running.
     BenchEnvironment(
       write_func = write_func,
-      format = format,
+      file_type = file_type,
       source = source,
       df = df
     )
@@ -46,41 +60,19 @@ write_file <- Benchmark("write_file",
   },
   # validate that the parameters given are compatible
   valid_params = function(params) {
-    # make sure that the format and the compression is compatible
-    # and fst doesn't have arrow_table input
-    drop <- !validate_format(params$format, params$compression) |
-      params$format == "fst" & params$input == "arrow_table"
+    # make sure that the file_type and the compression is compatible
+    # and fst doesn't have arrow_table input_type
+    drop <- !validate_format(params$file_type, params$compression) |
+      params$file_type == "fst" & params$input_type == "arrow_table"
     params[!drop,]
   },
-  # packages used when specific formats are used
+  # packages used when specific file_types are used
   packages_used = function(params) {
     pkg_map <- c(
       "feather" = "arrow",
       "parquet" = "arrow",
       "fst" = "fst"
     )
-    pkg_map[params$format]
+    pkg_map[params$file_type]
   }
 )
-
-#' Get a writer
-#'
-#' @param format format to write
-#' @param compression compression to use
-#'
-#' @return the write function to use
-#' @export
-get_write_function <- function(format, compression) {
-  force(compression)
-  if (format == "feather") {
-    return(function(...) arrow::write_feather(..., compression = compression))
-  } else if (format == "parquet") {
-    return(function(...) arrow::write_parquet(..., compression = compression))
-  } else if (format == "fst") {
-    # fst is always zstd, just a question of what level of compression
-    level <- ifelse(compression == "uncompressed", 0, 50)
-    return(function(...) fst::write_fst(..., compress = level))
-  } else {
-    stop("Unsupported format: ", format, call. = FALSE)
-  }
-}