Merge pull request #156 from pfuehrlich-pik/master

terra-based reading/writing of netcdf files

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '120248823'
+ValidationKey: '120963290'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'magclass: Data Class and Tools for Handling Spatial-Temporal Data'
-version: 6.11.3
-date-released: '2023-11-10'
+version: 6.13.0
+date-released: '2024-01-11'
 abstract: Data class for increased interoperability working with spatial-temporal
   data together with corresponding functions and methods (conversions, basic calculations
   and basic data manipulation). The class distinguishes between spatial, temporal
@@ -30,6 +30,9 @@ authors:
 - family-names: Leip
   given-names: Debbora
   email: leip@pik-potsdam.de
+- family-names: Sauer
+  given-names: Pascal
+  email: pascal.sauer@pik-potsdam.de
 - family-names: Baumstark
   given-names: Lavinia
   email: lavinia@pik-potsdam.de

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: magclass
 Title: Data Class and Tools for Handling Spatial-Temporal Data
-Version: 6.11.3
-Date: 2023-11-10
+Version: 6.13.0
+Date: 2024-01-11
 Authors@R: c(
     person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre")),
     person("Benjamin Leon", "Bodirsky", , "bodirsky@pik-potsdam.de", role = "aut"),
@@ -11,6 +11,7 @@ Authors@R: c(
     person("Stephen", "Bi", role = "aut"),
     person("Kristine", "Karstens", , "karstens@pik-potsdam.de", role = "aut"),
     person("Debbora", "Leip", , "leip@pik-potsdam.de", role = "aut"),
+    person("Pascal", "Sauer", , "pascal.sauer@pik-potsdam.de", role = "aut"),
     person("Lavinia", "Baumstark", , "lavinia@pik-potsdam.de", role = "ctb"),
     person("Christoph", "Bertram", , "bertram@pik-potsdam.de", role = "ctb"),
     person("Anastasis", "Giannousakis", , "giannou@pik-potsdam.de", role = "ctb"),
@@ -52,7 +53,6 @@ Suggests:
     quitte,
     raster,
     rmarkdown,
-    sf,
     terra,
     testthat (>= 3.1.5),
     tibble,

NAMESPACE

@@ -20,6 +20,7 @@ export(add_columns)
 export(add_dimension)
 export(as.RasterBrick)
 export(as.SpatRaster)
+export(as.SpatRasterDataset)
 export(as.SpatVector)
 export(clean_magpie)
 export(collapseDim)
@@ -72,6 +73,7 @@ export(setCells)
 export(setComment)
 export(setItems)
 export(setYears)
+export(spatRasterToDataset)
 export(time_interpolate)
 export(unitsplit)
 export(unwrap)
@@ -117,12 +119,8 @@ importFrom(methods,setGeneric)
 importFrom(methods,setMethod)
 importFrom(methods,signature)
 importFrom(stats,as.formula)
-importFrom(utils,capture.output)
 importFrom(utils,head)
-importFrom(utils,read.csv)
 importFrom(utils,read.table)
 importFrom(utils,tail)
-importFrom(utils,toBibtex)
 importFrom(utils,type.convert)
-importFrom(utils,write.csv)
 importFrom(utils,write.table)

R/as.SpatRaster.R

@@ -21,7 +21,7 @@
 #' @export
 
 
-as.SpatRaster <- function(x, res = NULL) { # nolint: object_name_linter
+as.SpatRaster <- function(x, res = NULL) { # nolint: object_name_linter.
   if (!is.magpie(x)) stop("Input is not a magpie object")
   if (!requireNamespace("terra", quietly = TRUE)) stop("The package \"terra\" is required!")
 

R/as.SpatRasterDataset.R

@@ -0,0 +1,13 @@
+#' as.SpatRasterDataset
+#'
+#' Convert magclass object to a SpatRasterDataset object. Requires the terra package.
+#'
+#' Calls \code{\link{as.SpatRaster}} and then \code{\link{spatRasterToDataset}}.
+#'
+#' @param ... arguments passed to as.SpatRaster
+#' @return A SpatRasterDataset object
+#' @author Pascal Sauer
+#' @export
+as.SpatRasterDataset <- function(...) { # nolint: object_name_linter.
+  return(spatRasterToDataset(as.SpatRaster(...)))
+}

R/as.SpatVector.R

@@ -23,7 +23,7 @@
 #' }
 #' @export
 
-as.SpatVector <- function(x) { # nolint: object_name_linter
+as.SpatVector <- function(x) { # nolint: object_name_linter.
   if (!is.magpie(x)) stop("Input is not a magpie object")
   if (!requireNamespace("terra", quietly = TRUE)) stop("The package \"terra\" is required!")
   if (is.null(attr(x, "geometry"))) {

R/as.magpie.R

@@ -2,8 +2,7 @@
 #' @importFrom data.table as.data.table tstrsplit melt
 
 #' @exportMethod as.magpie
-setGeneric(
-  "as.magpie", function(x, ...) standardGeneric("as.magpie"))
+setGeneric("as.magpie", function(x, ...) standardGeneric("as.magpie"))
 
 setMethod("as.magpie", signature(x = "magpie"), function(x) return(x))
 
@@ -40,8 +39,8 @@ setMethod("as.magpie",
     x <- array(x[magclassdata$half_deg$lpj_index, , , ], dim = c(dim(x)[1:2], dim(x)[3] * dim(x)[4]))
     # convert to previous default standard 59199 cells half degree resolution
     dimnames(x) <- list(paste(magclassdata$half_deg$region, 1:59199, sep = "."),
-      xdimnames[[2]],
-      paste(rep(xdimnames[[3]], xdim[4]), rep(xdimnames[[4]], each = xdim[3]), sep = "."))
+                        xdimnames[[2]],
+                        paste(rep(xdimnames[[3]], xdim[4]), rep(xdimnames[[4]], each = xdim[3]), sep = "."))
     out <- new("magpie", x)
     return(out)
   }
@@ -64,8 +63,7 @@ setMethod("as.magpie",
             }
             out <- new("magpie", y)
             return(out)
-          }
-)
+          })
 
 setMethod("as.magpie", # nolint
   signature(x = "array"),
@@ -116,9 +114,11 @@ setMethod("as.magpie", # nolint
     # Write warning when any type (except type "nothing") is found more than once
     nOccurrence <- lapply(d, length) > 1
     nOccurrence <- nOccurrence[names(nOccurrence) != "nothing"]
-    if (any(nOccurrence) == TRUE) warning("No clear mapping of dimensions to dimension types. First detected ",
-      "possibility is used! Please use arguments temporal and spatial to specify",
-      " which dimensions are what!")
+    if (any(nOccurrence) == TRUE) {
+      warning("No clear mapping of dimensions to dimension types. First detected ",
+              "possibility is used! Please use arguments temporal and spatial to specify",
+              " which dimensions are what!")
+    }
     for (i in which(nOccurrence)) {
       d[[i]] <- d[[i]][1]
     }
@@ -200,18 +200,16 @@ setMethod("as.magpie", # nolint
 )
 
 setMethod("as.magpie",
-  signature(x = "numeric"),
-  function(x, unit = "unknown", ...) {
-    return(copy.attributes(x, as.magpie(as.array(x), ...)))
-  }
-)
+          signature(x = "numeric"),
+          function(x, unit = "unknown", ...) {
+            return(copy.attributes(x, as.magpie(as.array(x), ...)))
+          })
 
 setMethod("as.magpie",
           signature(x = "logical"),
           function(x, unit = "unknown", ...) {
             return(copy.attributes(x, as.magpie(as.array(x), ...)))
-          }
-)
+          })
 
 setMethod("as.magpie",
   signature(x = "NULL"),
@@ -278,8 +276,10 @@ setMethod("as.magpie",
 
       mandatoryColumns <- c("model", "scenario", "region", "variable", "unit", "period", "value")
       factorColumns    <- c("model", "scenario", "region", "variable", "unit")
-      isQuitte <- all(mandatoryColumns %in% names(x)) && all(vapply(x[factorColumns], is.factor, logical(1))) &&
-                  is.numeric(x$value) && (methods::is(x$period, "POSIXct") || is.integer(x$period))
+      isQuitte <- (all(mandatoryColumns %in% names(x)) &&
+                     all(vapply(x[factorColumns], is.factor, logical(1))) &&
+                     is.numeric(x$value) &&
+                     (methods::is(x$period, "POSIXct") || is.integer(x$period)))
       return(isQuitte)
     }
 
@@ -434,12 +434,10 @@ setMethod("as.magpie",
           signature(x = "SpatRaster"),
           function(x, unit = "unknown", temporal = NULL, ...) {
             return(.raster2magpie(x, unit = unit, temporal = temporal))
-          }
-)
+          })
 
 setMethod("as.magpie",
           signature(x = "SpatVector"),
           function(x, unit = "unknown", temporal = NULL, spatial = NULL, ...) {
             return(.raster2magpie(x, unit = unit, temporal = temporal, spatial = spatial))
-          }
-)
+          })

R/read.magpie.R

@@ -3,44 +3,17 @@
 #' Reads a MAgPIE-file and converts it to a 3D array of the structure
 #' (cells,years,datacolumn)
 #'
-#' This function reads from 13 different MAgPIE file_types. "rds" is
-#' a R-default format for storing R objects."cs2" or "cs2b" is the new standard
-#' format for cellular data with or without
-#' header and the first columns (year,regiospatial) or only (regiospatial),
-#' "csv" is the standard format for regional data with or without header
-#' and the first columns (year,region,cellnumber) or only (region,cellnumber).
-#' "cs3" is a format similar to csv and cs2, but with the difference that it supports
-#' multidimensional data in a format which can be read by GAMS, "put" is a
-#' newly supported format which is mosty used for the REMIND-MAgPIE coupling.
-#' This format is only partly supported at the moment.  "asc" is the AsciiGrid
-#' format (for example used for Arc Gis data).  "nc" is the netCDF format (only
-#' "nc" files written by write.magpie can be read).  All these variants are
-#' read without further specification. "magpie" (.m) and "magpie zipped" (.mz)
-#' are new formats developed to allow a less storage intensive management of
-#' MAgPIE-data. The only difference between both formats is that .mz is gzipped
-#' whereas .m is not compressed. So .mz needs less memory, whereas .m might
-#' have a higher compatibility to other languages. \cr\cr Since library version
-#' 1.4 read.magpie can also read regional or global MAgPIE csv-files.
+#' See \code{\link{write.magpie}} for a list of supported file formats.
 #'
 #' @param file_name file name including file ending (wildcards are supported).
 #' Optionally also the full path can be specified here (instead of splitting it
 #' to file_name and file_folder)
 #' @param file_folder folder the file is located in (alternatively you can also
 #' specify the full path in file_name - wildcards are supported)
-#' @param file_type format the data is stored in. Currently 13 formats are
-#' available: "rds" (recommended compressed format),
-#' "cs2" & "cs2b" (cellular standard MAgPIE format), "csv" (regional standard
-#' MAgPIE format), "cs3" (multidimensional format compatible to GAMS), "cs4"
-#' (alternative multidimensional format compatible to GAMS, in contrast to cs3
-#' it can also handle sparse data), "cs5" (more generalized version of cs4),
-#' "csvr", "cs2r", "cs3r" and "cs4r" which are
-#' the same formats as the previous mentioned ones with the only difference
-#' that they have a REMIND compatible format, "m" (binary MAgPIE format
-#' "magpie"), "mz" (compressed binary MAgPIE format "magpie zipped") "put"
-#' (format used primarily for the REMIND-MAgPIE coupling) and "asc",
-#' (ASCII-Grid format as used by ArcGis) . If file_type=NULL the file ending
+#' @param file_type format the data is stored in. If file_type=NULL the file ending
 #' of the file_name is used as format. If format is different to the formats
-#' mentioned standard MAgPIE format is assumed.
+#' mentioned standard MAgPIE format is assumed. See \code{\link{write.magpie}}
+#' for a list of supported file formats.
 #' @param as.array Should the input be transformed to an array? This can be
 #' useful for regional or global inputs, but all advantages of the magpie-class
 #' are lost.
@@ -58,33 +31,13 @@
 #' @return \item{x}{MAgPIE-object}
 #' @note
 #'
-#' The binary MAgPIE formats .m and .mz have the following content/structure
-#' (you only have to care for that if you want to implement
-#' read.magpie/write.magpie functions in other languages): \cr \cr
-#' [ FileFormatVersion | Current file format version number (currently 6) | integer | 2 Byte ] \cr
-#' [ ncharComment | Number of character bytes of the file comment | integer | 4 Byte ] \cr
-#' [ nbyteMetadata | Number of bytes of the serialized metadata | integer | 4 Byte ] \cr
-#' [ ncharSets | Number of characters bytes of all regionnames + 2 delimiter | integer | 2 Byte] \cr
-#' [ nyears | Number of years | integer | 2 Byte ]\cr
-#' [ yearList | All years of the dataset (0, if year is not present) | integer | 2*nyears Byte ] \cr
-#' [ ncells | Number of cells | integer | 4 Byte ]\cr
-#' [ nchar_cell | Number of characters bytes of all regionnames + (nreg-1) for delimiters | integer | 4 Byte ] \cr
-#' [ cells | Cell names saved as cell1\\cell2 (\\n is the delimiter) | character | 1*nchar_cell Byte ] \cr
-#' [ nelem | Total number of data elements | integer | 4 Byte ] \cr
-#' [ ncharData | Number of char. bytes of all datanames + (ndata - 1) for delimiters | integer | 4 Byte ] \cr
-#' [ datanames | Names saved in the format data1\\ndata2 (\\n as del.) | character | 1*ncharData Byte ] \cr
-#' [ data | Data of the MAgPIE array in vectorized form | numeric | 4*nelem Byte ] \cr
-#' [ comment | Comment with additional information about the data | character | 1*ncharComment Byte ] \cr
-#' [ sets | Set names with \\n as delimiter | character | 1*ncharSets Byte] \cr
-#' [ metadata | serialized metadata information | bytes | 1*nbyteMetadata Byte] \cr
+#' See \code{\link{write.magpie}} for the detailed structure of binary MAgPIE formats .m and .mz.
 #'
-#' @author Jan Philipp Dietrich, Stephen Bi, Florian Humpenoeder
+#' @author Jan Philipp Dietrich, Stephen Bi, Florian Humpenoeder, Pascal Sauer
 #' @seealso \code{"\linkS4class{magpie}"}, \code{\link{write.magpie}}
-#' @importFrom methods is new
-#' @importFrom utils read.csv capture.output toBibtex
 #' @export
-read.magpie <- function(file_name, file_folder = "", file_type = NULL, as.array = FALSE, # nolint
-                        comment.char = "*", check.names = FALSE, ...) {                  # nolint
+read.magpie <- function(file_name, file_folder = "", file_type = NULL, # nolint: object_name_linter, cyclocomp_linter.
+                        as.array = FALSE, comment.char = "*", check.names = FALSE, ...) { # nolint: object_name_linter.
 
   .buildFileName <- function(fileName, fileFolder) {
     fileName <- paste0(fileFolder, fileName)
@@ -115,14 +68,14 @@ read.magpie <- function(file_name, file_folder = "", file_type = NULL, as.array
     readMagpie <- readRDS(fileName)
     if (!is.magpie(readMagpie)) stop("File does not contain a magpie object!")
   } else if (fileType %in% c("cs3", "cs3r")) {
-    x <- read.csv(fileName, comment.char = comment.char, check.names = check.names, stringsAsFactors = TRUE)
+    x <- utils::read.csv(fileName, comment.char = comment.char, check.names = check.names, stringsAsFactors = TRUE)
     datacols <- grep("^dummy\\.?[0-9]*$", colnames(x))
     xdimnames <- lapply(x[datacols], function(x) return(as.character(unique(x))))
     xdimnames[[length(xdimnames) + 1]] <- colnames(x)[-datacols]
     names(xdimnames) <- NULL
-    tmparr <- array(NA, dim = sapply(xdimnames, length), dimnames = xdimnames) # nolint
+    tmparr <- array(NA, dim = sapply(xdimnames, length), dimnames = xdimnames) # nolint:undesirable_function_linter.
     for (i in xdimnames[[length(xdimnames)]]) {
-      j <- sapply(cbind(x[datacols], i), as.character) # nolint
+      j <- sapply(cbind(x[datacols], i), as.character) # nolint:undesirable_function_linter.
       .duplicates_check(j)
       tmparr[j] <- x[, i]
     }
@@ -132,8 +85,8 @@ read.magpie <- function(file_name, file_folder = "", file_type = NULL, as.array
     }
     attr(readMagpie, "comment") <- .readComment(fileName, commentChar = comment.char)
   } else if (fileType %in% c("cs4", "cs4r")) {
-    x <- read.csv(fileName, comment.char = comment.char, header = FALSE,
-                  check.names = check.names, stringsAsFactors = TRUE)
+    x <- utils::read.csv(fileName, comment.char = comment.char, header = FALSE,
+                         check.names = check.names, stringsAsFactors = TRUE)
     readMagpie <- as.magpie(x, tidy = TRUE)
     attr(readMagpie, "comment") <- .readComment(fileName, commentChar = comment.char)
   } else if (fileType == "cs5") {
@@ -149,40 +102,40 @@ read.magpie <- function(file_name, file_folder = "", file_type = NULL, as.array
       return(list(comment = comment, metadata = mData))
     }
     m <- .metaExtract(fileName, comment.char)
-    x <- read.csv(fileName, comment.char = comment.char, header = FALSE,
-                  check.names = check.names, stringsAsFactors = FALSE)
+    x <- utils::read.csv(fileName, comment.char = comment.char, header = FALSE,
+                         check.names = check.names, stringsAsFactors = FALSE)
     colnames(x) <- m$metadata$names
     readMagpie <- as.magpie(x, tidy = TRUE,
                             spatial = grep(".spat", m$metadata$dimtype, fixed = TRUE),
                             temporal = grep(".temp", m$metadata$dimtype, fixed = TRUE),
                             data = grep(".data", m$metadata$dimtype, fixed = TRUE))
     attr(readMagpie, "comment") <- m$comment
   } else if (fileType %in% c("asc", "nc", "grd", "tif")) {
-    if (!requireNamespace("raster", quietly = TRUE)) stop("The package \"raster\" is required!")
     if (fileType == "nc") {
-      if (!requireNamespace("ncdf4", quietly = TRUE)) {
-        stop("The package \"ncdf4\" is required!")
+      if (!requireNamespace("terra", quietly = TRUE)) {
+        stop("The package \"terra\" is required!")
       }
-      nc <- ncdf4::nc_open(fileName)
-      var <- names(nc[["var"]])
-      vdim <- vapply(nc[["var"]], function(x) return(x$ndims), integer(1))
-      var <- var[vdim > 0]
-      ncdf4::nc_close(nc)
-      tmp <- list()
-      for (v in var) {
-        suppressSpecificWarnings({
-          warning <- capture.output(tmp[[v]] <- raster::brick(fileName, varname = v, ...))
-        }, "partial match of 'group' to 'groups'", fixed = TRUE)
-        if (length(warning) > 0) {
-          tmp[[v]] <- NULL
-          next
-        }
-        name <- sub("^X([0-9]*)$", "y\\1", names(tmp[[v]]), perl = TRUE)
-        if (length(name) == 1 && name == "layer") name <- "y0"
-        names(tmp[[v]]) <- paste0(name, "..", v)
+
+      x <- terra::rast(fileName)
+      if (all(grepl("Time=[0-9]+", names(x)))) {
+        names(x) <- sub("(.+)_Time=([0-9]+)", "y\\2..\\1", names(x))
+      } else if (all(grepl("_", names(x)))) {
+        names(x) <- vapply(names(x), function(n) {
+          parts <- strsplit(n, "_")[[1]] # e.g. "AFR_3" where 3 means the third entry in terra::time(x)
+          year <- terra::time(x)[as.integer(parts[2])]
+          if (is.na(year)) {
+            year <- as.integer(parts[2])
+          }
+          return(paste0("y", year, "..", parts[1]))
+        }, character(1))
       }
-      readMagpie <- as.magpie(raster::stack(tmp))
+
+      readMagpie <- clean_magpie(as.magpie(x))
+      attr(readMagpie, "crs") <- NULL
     } else {
+      if (!requireNamespace("raster", quietly = TRUE)) {
+        stop("The package \"raster\" is required!")
+      }
       readMagpie <- as.magpie(raster::brick(fileName, ...))
     }
   } else {