diff --git a/.Rbuildignore b/.Rbuildignore index ba4e423a..0738ebe4 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,4 +15,5 @@ vignettes/articles/usecase.Rmd ^pkgdown$ ^.codecov.yml$ +^tests$ \ No newline at end of file diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index c6efaa83..4ad315b0 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -34,7 +34,7 @@ jobs: - {os: windows-latest, r: 'release'} - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - {os: ubuntu-20.04, r: 'release'} - - {os: ubuntu-20.04, r: '3.6'} + - {os: ubuntu-20.04, r: '4.0'} env: R_REMOTES_NO_ERRORS_FROM_WARNINGS: true RSPM: ${{ matrix.config.rspm }} diff --git a/.gitignore b/.gitignore index 449f9e61..bdb97424 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ covr_report.html lib docs pkgdown +.Renviron \ No newline at end of file diff --git a/DESCRIPTION b/DESCRIPTION index ced1fa06..6f73acf4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: climate Title: Interface to Download Meteorological (and Hydrological) Datasets -Version: 1.1.2 +Version: 1.2.0 Authors@R: c(person(given = "Bartosz", family = "Czernecki", role = c("aut", "cre"), @@ -20,28 +20,29 @@ Authors@R: c(person(given = "Bartosz", Description: Automatize downloading of meteorological and hydrological data from publicly available repositories: OGIMET (), University of Wyoming - atmospheric vertical profiling data (), - Polish Institute of Meterology and Water Management - National Research Institute (), + Polish Institute of Meteorology and Water Management - National Research Institute (), and National Oceanic & Atmospheric Administration (NOAA). This package also allows for searching geographical coordinates for each observation and calculate distances to the nearest stations. License: MIT + file LICENSE Encoding: UTF-8 LazyData: true Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.0 Depends: - R (>= 3.5.0) + R (>= 4.0.0) Imports: XML, httr, curl, - data.table + data.table, + stringi Suggests: - testthat, - knitr, - rmarkdown, dplyr, + knitr, + maps, + testthat, tidyr, - maps + rmarkdown URL: https://github.com/bczernecki/climate BugReports: https://github.com/bczernecki/climate/issues VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index 3eb236c4..cc46cf0a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,6 +7,7 @@ export(hydro_imgw_daily) export(hydro_imgw_monthly) export(meteo_imgw) export(meteo_imgw_daily) +export(meteo_imgw_datastore) export(meteo_imgw_hourly) export(meteo_imgw_monthly) export(meteo_noaa_co2) @@ -19,17 +20,22 @@ export(ogimet_daily) export(ogimet_hourly) export(sounding_wyoming) export(spheroid_dist) +export(stations_meteo_imgw_telemetry) export(stations_ogimet) export(test_url) +import(data.table) import(httr) importFrom(XML,readHTMLTable) importFrom(curl,curl_download) importFrom(curl,has_internet) +importFrom(data.table,as.data.table) importFrom(data.table,fread) importFrom(stats,na.omit) importFrom(stats,runif) +importFrom(stringi,stri_trans_general) importFrom(utils,data) importFrom(utils,download.file) +importFrom(utils,globalVariables) importFrom(utils,packageVersion) importFrom(utils,read.csv) importFrom(utils,read.fwf) diff --git a/NEWS.md b/NEWS.md index 274b7d4d..83219cb7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# climate 1.2.0 + +* Corrected encoding problems for some of platforms where IMGW-PIB metadata were not parsed correctly +* Added location of IMGW-PIB telemetry stations within `meteo_imgw_telemetry_stations()` +* Minor bug fixes + + # climate 1.1.1 * Fix problems with downloading `precip` dataset from IMGW-PIB repository after recent changes in metadata diff --git a/R/clean_metadata_meteo.R b/R/clean_metadata_meteo.R index 8821d4c4..d721b2ef 100644 --- a/R/clean_metadata_meteo.R +++ b/R/clean_metadata_meteo.R @@ -6,21 +6,27 @@ #' @param interval temporal interval #' @importFrom utils read.fwf #' @importFrom stats na.omit -#' @keywords internal +#' @importFrom stringi stri_trans_general +#' @keywords internal #' clean_metadata_meteo = function(address, rank = "synop", interval = "hourly") { temp = tempfile() test_url(link = address, output = temp) - a = readLines(temp, warn = FALSE) - a = iconv(a, from = "CP1250", to = "ASCII//TRANSLIT") + + # a = readLines(temp, warn = FALSE, encoding = "CP1250") # doesn't work on mac, + # thus: + # a = iconv(a, from = "CP1250", to = "ASCII//TRANSLIT") + a = read.csv(temp, header = FALSE, stringsAsFactors = FALSE, + fileEncoding = "CP1250")$V1 a = gsub(a, pattern = "\\?", replacement = "") + a = stringi::stri_trans_general(a, 'LATIN-ASCII') # additional workarounds for mac os but not only... a = gsub(x = a, pattern = "'", replacement = "") a = gsub(x = a, pattern = "\\^0", replacement = "") - a = data.frame(V1 = a[nchar(a) > 0], stringsAsFactors = FALSE) + a = data.frame(V1 = a[nchar(a) > 3], stringsAsFactors = FALSE) # this one does not work on windows # a = suppressWarnings(na.omit(read.fwf(address, widths = c(1000), # fileEncoding = "CP1250", stringsAsFactors = FALSE))) @@ -44,8 +50,6 @@ clean_metadata_meteo = function(address, rank = "synop", interval = "hourly") { a$V1 = trimws(substr(a$V1, 1, nchar(a$V1) - 3)) a$V1 = gsub(x = a$V1, pattern = "* ", "") - #strsplit(x = a$V1, split = "/") - #a = a[nchar(a$V1)>2,] # remove empty or almost empty rows a = a[!(is.na(a$field1) & is.na(a$field2)), ] # remove info about status colnames(a)[1] = "parameters" return(a) diff --git a/R/globals.R b/R/globals.R new file mode 100644 index 00000000..d79db94e --- /dev/null +++ b/R/globals.R @@ -0,0 +1,2 @@ +#' @importFrom utils globalVariables +utils::globalVariables(c("i.parameter", "param")) diff --git a/R/imgw_hydro_abbrev.R b/R/imgw_hydro_abbrev.R index 6b1ae49a..e3f82d8d 100644 --- a/R/imgw_hydro_abbrev.R +++ b/R/imgw_hydro_abbrev.R @@ -7,11 +7,10 @@ #' fullname, abbr_eng, and fullname_eng #' #' @format The data contains a data.frame with ca. 20 elements described in three ways: -#' \itemize{ -#' \item{fullname} {original column names as downloaded from the repository} -#' \item{abbr_eng} {shorten column names with abbreviations derived from the most -#' popular scheme used for meteorological parameters} -#' \item{fullname_eng} {detailed description of downloaded meteorological variables} +#' \describe{ +#' \item{fullname}{original column names as downloaded from the repository} +#' \item{abbr_eng}{shorten column names with abbreviations derived from the most popular scheme used for meteorological parameters} +#' \item{fullname_eng}{detailed description of downloaded meteorological variables} #' } #' The object is created mostly to be used altogether with the hydro_shortening_imgw() function #' diff --git a/R/imgw_hydro_stations.R b/R/imgw_hydro_stations.R index fe399c40..2c705118 100644 --- a/R/imgw_hydro_stations.R +++ b/R/imgw_hydro_stations.R @@ -5,10 +5,10 @@ #' coordinates, ID numbers, and elevations #' #' @format The data contains a data.frame with 1304 obs. of 3 variables: -#' \itemize{ -#' \item{id} {Station ID} -#' \item{X} {Longitude} -#' \item{Y} {Latitude} +#' \describe{ +#' \item{id}{Station ID} +#' \item{X}{Longitude} +#' \item{Y}{Latitude} #' } #' The object is in the geographic coordinates using WGS84 (EPSG:4326). #' diff --git a/R/imgw_meteo_abbrev.R b/R/imgw_meteo_abbrev.R index 4ca8fed4..72b11da0 100644 --- a/R/imgw_meteo_abbrev.R +++ b/R/imgw_meteo_abbrev.R @@ -7,11 +7,11 @@ #' fullname, abbr_eng, and fullname_eng #' #' @format The data contains a data.frame with ca. 250 elements described in three ways: -#' \itemize{ -#' \item{fullname} {original column names as downloaded from the repository} -#' \item{abbr_eng} {shorten column names with abbreviations derived from the most +#' \describe{ +#' \item{fullname}{original column names as downloaded from the repository} +#' \item{abbr_eng}{shorten column names with abbreviations derived from the most #' popular scheme used for meteorological parameters} -#' \item{fullname_eng} {detailed description of downloaded meteorological variables} +#' \item{fullname_eng}{detailed description of downloaded meteorological variables} #' } #' The object is created mostly to be used altogether with the meteo_shortening_imgw #' function diff --git a/R/imgw_meteo_stations.R b/R/imgw_meteo_stations.R index 5b1718d0..753fe1f0 100644 --- a/R/imgw_meteo_stations.R +++ b/R/imgw_meteo_stations.R @@ -5,10 +5,12 @@ #' coordinates, ID numbers, and elevations #' #' @format The data contains a data.frame with 1998 obs. of 3 variables: -#' \itemize{ -#' \item{id} {Station ID} -#' \item{X} {Longitude} -#' \item{Y} {Latitude} +#' \describe{ +#' \item{id}{Station ID} +#' \item{X}{Longitude} +#' \item{Y}{Latitude} +#' \item{station}{Station name} +#' \item{id2}{IMGW-PIB ID for station rank} #' } #' The object is in the geographic coordinates using WGS84 (EPSG:4326). #' diff --git a/R/meteo_imgw_daily.R b/R/meteo_imgw_daily.R index baf726d0..1060dac3 100644 --- a/R/meteo_imgw_daily.R +++ b/R/meteo_imgw_daily.R @@ -9,8 +9,7 @@ #' (default status = FALSE - i.e. the status columns are deleted) #' @param coords add coordinates of the station (logical value TRUE or FALSE) #' @param station name of meteorological station(s). -#' It accepts names (characters in CAPITAL LETTERS); stations' IDs (numeric) are -#' no longer valid +#' It accepts names (characters in CAPITAL LETTERS); Stations' IDs (numeric) are no longer valid #' @param col_names three types of column names possible: #' "short" - default, values with shorten names, #' "full" - full English description, @@ -44,11 +43,6 @@ meteo_imgw_daily = function(rank = "synop", coords, station, col_names), - warning = function(w) { - message(paste("Potential problem(s) found. Problems with downloading data.\n", - "\rRun function with argument allow_failure = FALSE", - "to see more details")) - }, error = function(e){ message(paste("Potential error(s) found. Problems with downloading data.\n", "\rRun function with argument allow_failure = FALSE", @@ -116,7 +110,6 @@ meteo_imgw_daily_bp = function(rank, temp = tempfile() temp2 = tempfile() test_url(addresses_to_download[j], temp) - #download.file(addresses_to_download[j], temp) unzip(zipfile = temp, exdir = temp2) file1 = paste(temp2, dir(temp2), sep = "/")[1] if (translit) { @@ -124,28 +117,28 @@ meteo_imgw_daily_bp = function(rank, } else { data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250") } - colnames(data1) = meta[[1]]$parameters file2 = paste(temp2, dir(temp2), sep = "/")[2] if (translit) { - data2 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file2))) + data2 = data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file2)) } else { - data2 = read.csv(file2, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250") + data2 = suppressWarnings(read.csv(file2, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")) } colnames(data2) = meta[[2]]$parameters + unlink(c(temp, temp2)) - # usuwa statusy + # remove statuses if not needed: if (status == FALSE) { data1[grep("^Status", colnames(data1))] = NULL data2[grep("^Status", colnames(data2))] = NULL } - unlink(c(temp, temp2)) - - # moja proba z obejsciem dla wyboru kodu - ttt = merge(data1, data2, by = c("Kod stacji", "Rok", "Miesiac", "Dzien"), - all.x = TRUE) + ttt = base::merge(data1, + data2, + by = c("Kod stacji", "Rok", "Miesiac", "Dzien"), + all.x = TRUE) + ttt = ttt[order(ttt$`Nazwa stacji.x`, ttt$Rok, ttt$Miesiac, ttt$Dzien), ] ### ta część kodu powtarza sie po dużej petli od rank if (!is.null(station)) { @@ -251,7 +244,7 @@ meteo_imgw_daily_bp = function(rank, all_data = do.call(rbind, all_data) if (coords) { - all_data = merge(climate::imgw_meteo_stations, + all_data = merge(climate::imgw_meteo_stations[, 1:3], all_data, by.x = "id", by.y = "Kod stacji", @@ -291,15 +284,16 @@ meteo_imgw_daily_bp = function(rank, } } - # sortowanie w zaleznosci od nazw kolumn - raz jest "kod stacji", raz "id" + # sort output if (sum(grepl(x = colnames(all_data), pattern = "Kod stacji"))) { all_data = all_data[order(all_data$`Kod stacji`, all_data$Rok, all_data$Miesiac, all_data$Dzien), ] } else { all_data = all_data[order(all_data$id, all_data$Rok, all_data$Miesiac, all_data$Dzien), ] } - # # dodanie opcji dla skracania kolumn i usuwania duplikatow: + # remove duplicates and shorten colnames all_data = meteo_shortening_imgw(all_data, col_names = col_names, ...) + rownames(all_data) = NULL return(all_data) } diff --git a/R/meteo_imgw_datastore.R b/R/meteo_imgw_datastore.R new file mode 100644 index 00000000..2bc3256b --- /dev/null +++ b/R/meteo_imgw_datastore.R @@ -0,0 +1,164 @@ +#' IMGW meteorological data from the IMGW datastore repository +#' +#' Downloading hourly (meteorological) data from the telemetric stations +#' available in the danepubliczne.imgw.pl/datastore collection since 2008. +#' Most parameters are collected with 10 minutes interval and thus it is recommended to download only the mandatory years, parameters or stations. +#' For example, 1 year of data with all available parameters requires processing around 4GB of uncompressed data. +#' +#' Data from the IMGW automated (telemetry) systems are non validated by experts and may contain invalid values. +#' +#' +#' @param year numeric vector of years to be downloaded (e.g., 2022:2023) +#' @param parameters - character vector describing which parameters to be downloaded. Default `NULL` means to download all available. +#' \enumerate{ +#' \item "wd" - wind direction (degrees) +#' \item "t2m" - temperature at 2 metres above ground level (degree Celsius) +#' \item "t0m" - ground temperature (degree Celsius) +#' \item "rr_24h" - precipitation totals for last 24 hours (mm) +#' \item "rr_1h" - precipitation totals for last 1 hour (mm) +#' \item "rr_10min" - precipitation totals for last 10 minutes (mm) +#' \item "ws" - wind speed (m/s) +#' \item "ws_max" - maximum wind speed for last 10 minutes (m/s) +#' \item "gust" - wind gust (if present) (m/s) +#' \item "rh" - relative humidity (%) +#' \item "water_in_snow" - water equivalent of melted snow cover (mm) +#' } +#' @param stations - character vector with station names as visible in the `meteo_imgw_telemetry_stations()`. +#' Default `NULL` means to download data for all available stations. +#' @param coords - logical - whether to append the dataset with station full name, longitude, latitude and altitude. Default: TRUE +#' @param allow_failure logical - whether to proceed or stop on failure. By default set to TRUE (i.e. don't stop on error). For debugging purposes change to FALSE +#' @import data.table +#' @export +#' +#' @examples \donttest{ +#' # download only air temperature for selected 2 stations in 2022-2023: +#' imgw_telemetry = meteo_imgw_datastore(year = 2022:2023, +#' parameters = "t2m", +#' stations = c("HALA GĄSIENICOWA", +#' "DOLINA PIĘCIU STAWÓW"), +#' coords = TRUE) +#' } +#' + + +meteo_imgw_datastore = function(year, + parameters = NULL, + stations = NULL, + coords = TRUE, + allow_failure = TRUE) { + + # assertions for year + if (!any(is.character(year) || is.numeric(year))) { + stop("year argument must be character or numeric") + } + + if (!all(as.numeric(year) >= 2008)) { + stop("year argument must be provided and all elements must be >= 2008") + } + + if (allow_failure) { + tryCatch(meteo_imgw_datastore_bp(year, + parameters, + stations, + coords), + error = function(e){ + message(paste("Potential error(s) found. Problems with downloading data.\n", + "\rRun function with argument allow_failure = FALSE", + "to see more details"))}) + } else { + meteo_imgw_datastore_bp(year, + parameters, + stations, + coords) + } +} + +#' @keywords internal +#' @noRd +meteo_imgw_datastore_bp = function(year, + parameters, + stations, + coords) { + # read metadata for stations: + telemetry_stations = stations_meteo_imgw_telemetry() + telemetry_stations$river = NULL + + if (!is.null(stations)) { + telemetry_stations = telemetry_stations[telemetry_stations$name %in% toupper(stations), ] + } + + urls = as.character( + sapply(year, function(x) paste0( + "https://danepubliczne.imgw.pl/datastore/getfiledown/Arch/Telemetria/Meteo/", + x, "/Meteo_", x, "-", sprintf("%02d", 1:12)) + ) + ) + + dict = data.table::data.table( + V2 = c("B00202A", "B00300S", "B00305A", "B00604S", "B00606S", + "B00608S", "B00702A", "B00703A", "B00714A", "B00802A", + "B00910A"), + parameter = c("wd", "t2m", "t0m", "rr_24h", "rr_1h", + "rr_10min", "ws", "ws_max", "gust", "rh", + "water_in_snow") + ) + + if (!is.null(parameters)) { + dict = dict[dict$parameter %in% parameters,] + } + + all_data = NULL + + for (i in seq_along(urls)) { + + temp = tempfile() + temp2 = tempfile() + tmp_dir = file.path(tempdir(), basename(urls[i])) + download.file(paste0(urls[i], ".zip"), temp) + download.file(paste0(urls[i], ".ZIP"), temp2) + + if (file.size(temp) > 1000) { + unzip(zipfile = temp, exdir = tmp_dir) + } else if (file.size(temp2) > 1000) { + unzip(zipfile = temp2, exdir = tmp_dir) + } else { + message("IMGW datastore does not contain data for ", paste(basename(urls[i])), ". skipping\n") + } + + files_to_read = dir(tmp_dir, full.names = TRUE, pattern = paste0(dict$V2, collapse = "|")) + files_to_read = files_to_read[file.size(files_to_read) > 0] + + all_data[[i]] = data.table::rbindlist( + lapply(files_to_read, + function(x) data.table::fread(x, + header = FALSE, + stringsAsFactors = FALSE, + dec = ",", + sep = ";", + select = c("V1", "V2", "V3", "V4") + ) + ), + fill = TRUE) + + if (!is.null(stations)) { + all_data[[i]] = all_data[[i]][all_data[[i]]$V1 %in% telemetry_stations$id, ] + } + + unlink(c(temp, temp2, tmp_dir), recursive = TRUE) + } + + all_data = data.table::rbindlist(all_data, fill = TRUE) + all_data = all_data[dict, on = 'V2', param := i.parameter] + all_data = data.table::dcast(all_data, V1 + V3 ~ param, value.var = "V4") + all_data$V3 = as.POSIXct(all_data$V3, tz = "UTC") + + if (coords) { + class(telemetry_stations$id) = class(all_data$V1) # equalise classes of objects: + all_data = merge(all_data, telemetry_stations, by.x = "V1", by.y = "id", all = FALSE) + data.table::setcolorder(all_data, c("V1", "name", "lon", "lat", "alt")) + } + + colnames(all_data)[which(colnames(all_data) %in% c("V1", "V3"))] = c("id", "date_time") + + return(all_data) +} diff --git a/R/meteo_imgw_hourly.R b/R/meteo_imgw_hourly.R index d2ca068a..026c5de0 100644 --- a/R/meteo_imgw_hourly.R +++ b/R/meteo_imgw_hourly.R @@ -44,11 +44,6 @@ meteo_imgw_hourly = function(rank = "synop", coords, station, col_names, ...), - warning = function(w) { - message(paste("Potential problem(s) found. Problems with downloading data.\n", - "\rRun function with argument allow_failure = FALSE", - "to see more details")) - }, error = function(e){ message(paste("Potential error(s) found. Problems with downloading data.\n", "\rRun function with argument allow_failure = FALSE", @@ -75,8 +70,8 @@ meteo_imgw_hourly_bp = function(rank, translit = check_locale() stopifnot(rank == "synop" | rank == "climate") # dla terminowek tylko synopy i klimaty maja dane base_url = "https://danepubliczne.imgw.pl/data/dane_pomiarowo_obserwacyjne/" - interval = "hourly" # to mozemy ustawic na sztywno - interval_pl = "terminowe" # to mozemy ustawic na sztywno + interval = "hourly" + interval_pl = "terminowe" meta = meteo_metadata_imgw(interval = "hourly", rank = rank) rank_pl = switch(rank, synop = "synop", climate = "klimat", precip = "opad") temp = tempfile() @@ -84,7 +79,7 @@ meteo_imgw_hourly_bp = function(rank, output = temp) a = readLines(temp, warn = FALSE) unlink(temp) - + ind = grep(readHTMLTable(a)[[1]]$Name, pattern = "/") catalogs = as.character(readHTMLTable(a)[[1]]$Name[ind]) @@ -108,20 +103,20 @@ meteo_imgw_hourly_bp = function(rank, ind = grep(readHTMLTable(folder_contents)[[1]]$Name, pattern = "zip") files = as.character(readHTMLTable(folder_contents)[[1]]$Name[ind]) + addresses_to_download = paste0(address, files) for (j in seq_along(addresses_to_download)) { temp = tempfile() temp2 = tempfile() test_url(addresses_to_download[j], temp) - #download.file(addresses_to_download[j], temp) unzip(zipfile = temp, exdir = temp2) file1 = paste(temp2, dir(temp2), sep = "/") if (translit) { data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1))) } else { - data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250") + data1 = suppressWarnings(read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")) } colnames(data1) = meta[[1]]$parameters @@ -179,13 +174,16 @@ meteo_imgw_hourly_bp = function(rank, all_data = do.call(rbind, all_data) if (coords) { - all_data = merge(climate::imgw_meteo_stations, all_data, by.x = "id", by.y = "Kod stacji", all.y = TRUE) + all_data = merge(climate::imgw_meteo_stations[,1:3], + all_data, + by.x = "id", + by.y = "Kod stacji", + all.y = TRUE) } # dodaje rank rank_code = switch(rank, synop = "SYNOPTYCZNA", climate = "KLIMATYCZNA") all_data = cbind(data.frame(rank_code = rank_code), all_data) - all_data = all_data[all_data$Rok %in% year, ] # przyciecie tylko do wybranych lat gdyby sie pobralo za duzo #station selection @@ -217,7 +215,8 @@ meteo_imgw_hourly_bp = function(rank, all_data = all_data[order(all_data$id, all_data$Rok, all_data$Miesiac, all_data$Dzien, all_data$Godzina), ] } - # dodanie opcji dla skracania kolumn i usuwania duplikatow: + # extra option for shortening colnames and removing duplicates all_data = meteo_shortening_imgw(all_data, col_names = col_names, ...) + rownames(all_data) = NULL return(all_data) -} # end of function +} diff --git a/R/meteo_imgw_monthly.R b/R/meteo_imgw_monthly.R index c98353b5..4b37858b 100644 --- a/R/meteo_imgw_monthly.R +++ b/R/meteo_imgw_monthly.R @@ -53,11 +53,6 @@ meteo_imgw_monthly = function(rank = "synop", station, col_names, ...), - warning = function(w) { - message(paste("Potential problem(s) found. Problems with downloading data.\n", - "\rRun function with argument allow_failure = FALSE", - "to see more details")) - }, error = function(e){ message(paste("Potential error(s) found. Problems with downloading data.\n", "\rRun function with argument allow_failure = FALSE", @@ -133,7 +128,7 @@ meteo_imgw_monthly_bp = function(rank, if (translit) { data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1))) } else { - data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250") + data1 = suppressWarnings(read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")) } colnames(data1) = meta[[1]]$parameters @@ -174,7 +169,11 @@ meteo_imgw_monthly_bp = function(rank, all_data = all_data[all_data$Rok %in% year, ] if (coords) { - all_data = merge(climate::imgw_meteo_stations, all_data, by.x = "id", by.y = "Kod stacji", all.y = TRUE) + all_data = merge(climate::imgw_meteo_stations[, 1:3], + all_data, + by.x = "id", + by.y = "Kod stacji", + all.y = TRUE) } # add rank @@ -208,6 +207,7 @@ meteo_imgw_monthly_bp = function(rank, # adding option to shorten columns and removing duplicates: all_data = meteo_shortening_imgw(all_data, col_names = col_names, ...) + rownames(all_data) = NULL return(all_data) # clipping to selected years only } diff --git a/R/meteo_metadata_imgw.R b/R/meteo_metadata_imgw.R index 6a932627..ad9c6d3a 100644 --- a/R/meteo_metadata_imgw.R +++ b/R/meteo_metadata_imgw.R @@ -41,7 +41,7 @@ meteo_metadata_imgw = function(interval, rank) { # interval can be: monthly, hou rank = "precip", interval = "daily") } - } + } # end of daily interval if (interval == "monthly") { @@ -66,7 +66,7 @@ meteo_metadata_imgw = function(interval, rank) { # interval can be: monthly, hou } # koniec MIESIECZNYCH - ## rozpoczecie dla danych TERMINOWYCH: + ## hourly data section: if (interval == "hourly") { if (rank == "synop") b[[1]] = clean_metadata_meteo(paste0(base_url, "dane_meteorologiczne/terminowe/synop/s_t_format.txt"), rank = "synop", interval = "hourly") @@ -76,6 +76,5 @@ meteo_metadata_imgw = function(interval, rank) { # interval can be: monthly, hou stop("The precipitation stations ('precip') does not provide hourly data.", call. = FALSE) } } - return(b) } diff --git a/R/meteo_shortening_imgw.R b/R/meteo_shortening_imgw.R index 3b8c9183..283c5cbb 100644 --- a/R/meteo_shortening_imgw.R +++ b/R/meteo_shortening_imgw.R @@ -34,7 +34,9 @@ meteo_shortening_imgw = function(data, col_names = "short", remove_duplicates = if (col_names != "polish") { abbrev = climate::imgw_meteo_abbrev orig_columns = trimws(gsub("\\s+", " ", colnames(data))) # remove double spaces + orig_columns = trimws(gsub("\\[.*?]", "", orig_columns)) # remove brackets and content inside + abbrev$fullname = trimws(gsub("\\[.*?]", "", abbrev$fullname)) matches = match(orig_columns, abbrev$fullname) matches = matches[!is.na(matches)] @@ -48,7 +50,6 @@ meteo_shortening_imgw = function(data, col_names = "short", remove_duplicates = colnames(data)[orig_columns %in% abbrev$fullname] = abbrev$fullname_eng[matches] } } - + return(data) - -} # end of function +} diff --git a/R/nearest_stations_ogimet.R b/R/nearest_stations_ogimet.R index d9d5f22a..0b0b63fd 100644 --- a/R/nearest_stations_ogimet.R +++ b/R/nearest_stations_ogimet.R @@ -109,7 +109,7 @@ nearest_stations_ogimet_bp = function(country = country, b = strsplit(a, "Decoded synops since") - b1 = lapply(b, function(x) substr(x, 1, 400)) + b1 = lapply(b, function(x) substr(iconv(x, from = 'UTF-8', to = 'ASCII//TRANSLIT'), 1, 400)) b1[[1]] = b1[[1]][-1] # header b21 = unlist(lapply(gregexpr('Lat=', b1[[1]], fixed = TRUE), function(x) x[1])) diff --git a/R/stations_meteo_imgw_telemetry.R b/R/stations_meteo_imgw_telemetry.R new file mode 100644 index 00000000..efd0b9e0 --- /dev/null +++ b/R/stations_meteo_imgw_telemetry.R @@ -0,0 +1,55 @@ +#' IMGW telemetry stations +#' +#' Retrieving current metadata for stations used in the telemetric systems of the IMGW-PIB datastore (danepubliczne.imgw.pl/datastore) +#' +#' @return data table with metadata for over 500 stations. Metadata contains: station ID, station name, river, latitude, longitude, altitude +#' @importFrom data.table as.data.table +#' @export +#' +#' @examples \donttest{ +#' telemetry_stations = stations_meteo_imgw_telemetry() +#' } +#' + +stations_meteo_imgw_telemetry = function() { + + url = "https://danepubliczne.imgw.pl/datastore/getfiledown/Arch/Telemetria/Meteo/kody_stacji.csv" + + telemetry_stations = tryCatch(expr = suppressWarnings( + read.csv(url, + fileEncoding = "CP1250", + sep = ";", + stringsAsFactors = FALSE) + ), + error = function(e) { + message(paste0("Problems with downloading data from:\n", url)) + }) + + if (!is.null(telemetry_stations)) { + colnames(telemetry_stations) = c("no", "id", "name", "river", "lat", "lon", "alt") + # extra fix for columns wrongly signed in IMGW datatabase (i.e. missing semicolon): + fix_needed = grep(x = substr(telemetry_stations$river, 1, 1), '^[0-9]$') + + telemetry_stations[fix_needed, "alt"] = telemetry_stations[fix_needed, "lon"] + telemetry_stations[fix_needed, "lon"] = telemetry_stations[fix_needed, "lat"] + telemetry_stations[fix_needed, "lat"] = telemetry_stations[fix_needed, "river"] + telemetry_stations[fix_needed, "river"] = NA + + telemetry_stations$lon = suppressWarnings(coordinates_to_decimal(telemetry_stations$lon)) + telemetry_stations$lat = suppressWarnings(coordinates_to_decimal(telemetry_stations$lat)) + telemetry_stations$alt = as.numeric(gsub(x = telemetry_stations$alt, " ", "")) + telemetry_stations = as.data.table(telemetry_stations[,-1]) + telemetry_stations$id = as.character(telemetry_stations$id) + } + return(telemetry_stations) +} + + +#' @keywords internal +#' @noRd +coordinates_to_decimal = function(lonlat) { + converted = unlist(lapply(strsplit(lonlat, " "), function(d) as.numeric(d[[1]]))) + + unlist(lapply(strsplit(lonlat, " "), function(m) as.numeric(m[[2]]) * 0.01667)) + + unlist(lapply(strsplit(lonlat, " "), function(s) as.numeric(s[[3]]) * 0.0001667)) + return(converted) +} diff --git a/R/stations_ogimet.R b/R/stations_ogimet.R index fa60c62e..bf617ec7 100644 --- a/R/stations_ogimet.R +++ b/R/stations_ogimet.R @@ -38,7 +38,7 @@ stations_ogimet = function(country = "United+Kingdom", stations_ogimet_bp = function(country = country, date = date, add_map = add_map) { if (length(country) != 1) { - stop("To many country selected. Please choose one country") + stop("Too many countries selected. Please choose only one country") } if (length(date) != 1) { @@ -52,64 +52,64 @@ stations_ogimet_bp = function(country = country, date = date, add_map = add_map) ndays = 1 linkpl2 = paste0("http://ogimet.com/cgi-bin/gsynres?lang=en&state=",country,"&osum=no&fmt=html&ord=REV&ano=",year,"&mes=",month,"&day=",day,"&hora=06&ndays=1&Send=send") - temp = tempfile() - test_url(link = linkpl2, output = temp) + temp = tempfile() + test_url(link = linkpl2, output = temp) - # run only if downloaded file is valid - if (!is.na(file.size(temp)) & (file.size(temp) > 0)) { + # run only if downloaded file is valid + if (!is.na(file.size(temp)) & (file.size(temp) > 0)) { - a = readLines(temp) - a = paste(a, sep = "", collapse = "") - - b = strsplit(a, "Decoded synops since") - - b1 = lapply(b, function(x) substr(x, 1, 400)) - b1[[1]] = b1[[1]][-1] # header - - b21 = unlist(lapply(gregexpr('Lat=', b1[[1]], fixed = TRUE), function(x) x[1])) - - pattern = paste0(" (", gsub(x = country, pattern = "+", replacement = " ", fixed = TRUE)) - b22 = unlist(lapply(gregexpr(pattern = pattern, b1[[1]], fixed = TRUE), function(x) x[1])) - - b1 = data.frame(str = b1[[1]], start = b21, stop = b22 - 1, stringsAsFactors = FALSE) - - res = substr(b1$str, b1$start, b1$stop) - - station_names = unlist(lapply(strsplit(res, " - "), function(x) x[length(x)])) - - - res = gsub(x = res, pattern = ", CAPTION, '", replacement = '', fixed = TRUE) - res = gsub(x = res, pattern = " m'", replacement = ' ', fixed = TRUE) - res = gsub(x = res, pattern = " - ", replacement = ' ', fixed = TRUE) - res = gsub(x = res, pattern = "Lat=", replacement = '', fixed = TRUE) - res = gsub(x = res, pattern = "Lon=", replacement = ' ', fixed = TRUE) - res = gsub(x = res, pattern = "Alt=", replacement = ' ', fixed = TRUE) - - res = suppressWarnings(do.call("rbind", strsplit(res, " "))) - - res1 = res[, c(1, 3, 5:7)] - - lat = as.numeric(substr(res1[, 1], 1, 2)) + - (as.numeric(substr(res1[,1], 4, 5))/100) * 1.6667 - - lon_hemisphere = gsub("[0-9]", "\\1", res1[, 2]) - lon_hemisphere = gsub("-", "", lon_hemisphere) - lon_hemisphere = ifelse(lon_hemisphere == "W", -1, 1) - - lat_hemisphere = gsub("[0-9]", "\\1", res1[, 1]) - lat_hemisphere = gsub("-", "", lat_hemisphere) - lat_hemisphere = ifelse(lat_hemisphere == "S", -1, 1) - - lon = as.numeric(substr(res1[, 2], 1, 3)) + (as.numeric(substr(res1[, 2], 5, 6)) / - 100) * 1.6667 - lon = lon * lon_hemisphere - - lat = as.numeric(substr(res1[, 1], 1, 2)) + (as.numeric(substr(res1[, 1], 4, 5)) / - 100) * 1.6667 - lat = lat * lat_hemisphere + a = readLines(temp) + a = paste(a, sep = "", collapse = "") + b = strsplit(a, "Decoded synops since") + + b1 = lapply(b, function(x) substr(x, 1, 400)) + b1[[1]] = b1[[1]][-1] # header + + b21 = unlist(lapply(gregexpr('Lat=', b1[[1]], fixed = TRUE), function(x) x[1])) + + pattern = paste0(" (", gsub(x = country, pattern = "+", replacement = " ", fixed = TRUE)) + b22 = unlist(lapply(gregexpr(pattern = pattern, b1[[1]], fixed = TRUE), function(x) x[1])) + + b1 = data.frame(str = b1[[1]], start = b21, stop = b22 - 1, stringsAsFactors = FALSE) - res = data.frame(wmo_id = res1[, 4], station_names = station_names, - lon = lon, lat = lat, alt = as.numeric(res1[, 3])) + res = substr(b1$str, b1$start, b1$stop) + + station_names = unlist(lapply(strsplit(res, " - ", fixed = TRUE), function(x) x[length(x)])) + + res = gsub(x = res, pattern = ", CAPTION, '", replacement = '', fixed = TRUE) + res = gsub(x = res, pattern = " m'", replacement = ' ', fixed = TRUE) + res = gsub(x = res, pattern = " - ", replacement = ' ', fixed = TRUE) + res = gsub(x = res, pattern = "Lat=", replacement = '', fixed = TRUE) + res = gsub(x = res, pattern = "Lon=", replacement = ' ', fixed = TRUE) + res = gsub(x = res, pattern = "Alt=", replacement = ' ', fixed = TRUE) + res = gsub(x = res, pattern = " / ", replacement = '/', fixed = TRUE) + + res = suppressWarnings(do.call("rbind", strsplit(res, " "))) + res1 = matrix(res[, c(1, 3, 5:7)], ncol = 5) + + lat = suppressWarnings( + as.numeric(substr(res1[, 1], 1, 2)) + + (as.numeric(substr(res1[, 1], 4, 5)) / 100) * 1.6667 + ) + + lon_hemisphere = gsub("[0-9]", "\\1", res1[, 2]) + lon_hemisphere = gsub("-", "", lon_hemisphere) + lon_hemisphere = ifelse(lon_hemisphere == "W", -1, 1) + + lat_hemisphere = gsub("[0-9]", "\\1", res1[, 1]) + lat_hemisphere = gsub("-", "", lat_hemisphere) + lat_hemisphere = ifelse(lat_hemisphere == "S", -1, 1) + + lon = suppressWarnings( + as.numeric(substr(res1[, 2], 1, 3)) + + (as.numeric(substr(res1[, 2], 5, 6)) / 100) * 1.6667 + ) + lon = lon * lon_hemisphere + + lat = lat * lat_hemisphere + + res = data.frame(wmo_id = res1[, 4], station_names = station_names, + lon = lon, lat = lat, alt = as.numeric(res1[, 3])) } else { res = NULL diff --git a/README.md b/README.md index 520c37eb..764992eb 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ and hydrological data from publicly available repositories: - OGIMET [(ogimet.com)](http://ogimet.com/index.phtml.en) - up-to-date collection of SYNOP dataset - University of Wyoming - atmospheric vertical profiling data (http://weather.uwyo.edu/upperair/) - National Oceanic & Atmospheric Administration - Earth System Research Laboratories - Global Monitoring Laboratory [(NOAA)](https://gml.noaa.gov/ccgg/trends/) -- Polish Institute of Meterology and Water Management - National Research Institute [(IMGW-PIB)](https://dane.imgw.pl/) +- Polish Institute of Meteorology and Water Management - National Research Institute [(IMGW-PIB)](https://dane.imgw.pl/) - National Oceanic & Atmospheric Administration - National Climatic Data Center - Integrated Surface Hourly (ISH) [(NOAA)](https://www1.ncdc.noaa.gov/pub/data/noaa/) ## Installation diff --git a/data/imgw_meteo_stations.rda b/data/imgw_meteo_stations.rda index b0297da9..78f97570 100644 Binary files a/data/imgw_meteo_stations.rda and b/data/imgw_meteo_stations.rda differ diff --git a/inst/CITATION b/inst/CITATION index 81a84f68..ae4c1415 100644 --- a/inst/CITATION +++ b/inst/CITATION @@ -1,6 +1,6 @@ citHeader("To cite the climate package in publications, please use:") -citEntry(entry = "manual", +bibentry(bibtype = "Article", author = c(person("Bartosz", "Czernecki"), person("Arkadiusz", "Głogowski"), person("Jakub", "Nowosad")), @@ -10,8 +10,8 @@ citEntry(entry = "manual", pages = {"394"}, volume = {"12"}, doi = {"10.3390/su12010394"}, - note = "R package version 0.9.1", - url = "https://github.com/bczernecki/climate/", + #note = "R package version 1.2.0", + #url = "https://github.com/bczernecki/climate/", textVersion = "Czernecki, B.; Głogowski, A.; Nowosad, J. Climate: An R Package to Access Free In-Situ Meteorological and Hydrological Datasets for Environmental Assessment. Sustainability 2020, 12, 394. https://doi.org/10.3390/su12010394" ) diff --git a/man/climate-package.Rd b/man/climate-package.Rd index 19dd9d7e..a244cc32 100644 --- a/man/climate-package.Rd +++ b/man/climate-package.Rd @@ -8,7 +8,7 @@ \description{ \if{html}{\figure{logo.png}{options: style='float: right' alt='logo' width='120'}} -Automatize downloading of meteorological and hydrological data from publicly available repositories: OGIMET (\url{http://ogimet.com/index.phtml.en}), University of Wyoming - atmospheric vertical profiling data (\url{http://weather.uwyo.edu/upperair/}), Polish Institute of Meterology and Water Management - National Research Institute (\url{https://danepubliczne.imgw.pl}), and National Oceanic & Atmospheric Administration (NOAA). This package also allows for searching geographical coordinates for each observation and calculate distances to the nearest stations. +Automatize downloading of meteorological and hydrological data from publicly available repositories: OGIMET (\url{http://ogimet.com/index.phtml.en}), University of Wyoming - atmospheric vertical profiling data (\url{http://weather.uwyo.edu/upperair/}), Polish Institute of Meteorology and Water Management - National Research Institute (\url{https://danepubliczne.imgw.pl}), and National Oceanic & Atmospheric Administration (NOAA). This package also allows for searching geographical coordinates for each observation and calculate distances to the nearest stations. } \seealso{ Useful links: diff --git a/man/imgw_hydro_abbrev.Rd b/man/imgw_hydro_abbrev.Rd index 93ddebcc..644f7b1f 100644 --- a/man/imgw_hydro_abbrev.Rd +++ b/man/imgw_hydro_abbrev.Rd @@ -7,11 +7,10 @@ from the danepubliczne.imgw.pl collection} \format{ The data contains a data.frame with ca. 20 elements described in three ways: -\itemize{ -\item{fullname} {original column names as downloaded from the repository} -\item{abbr_eng} {shorten column names with abbreviations derived from the most -popular scheme used for meteorological parameters} -\item{fullname_eng} {detailed description of downloaded meteorological variables} +\describe{ +\item{fullname}{original column names as downloaded from the repository} +\item{abbr_eng}{shorten column names with abbreviations derived from the most popular scheme used for meteorological parameters} +\item{fullname_eng}{detailed description of downloaded meteorological variables} } The object is created mostly to be used altogether with the hydro_shortening_imgw() function } diff --git a/man/imgw_hydro_stations.Rd b/man/imgw_hydro_stations.Rd index 403641b5..45b6b5c9 100644 --- a/man/imgw_hydro_stations.Rd +++ b/man/imgw_hydro_stations.Rd @@ -6,10 +6,10 @@ \title{Location of the hydrological stations from the danepubliczne.imgw.pl collection} \format{ The data contains a data.frame with 1304 obs. of 3 variables: -\itemize{ -\item{id} {Station ID} -\item{X} {Longitude} -\item{Y} {Latitude} +\describe{ +\item{id}{Station ID} +\item{X}{Longitude} +\item{Y}{Latitude} } The object is in the geographic coordinates using WGS84 (EPSG:4326). } diff --git a/man/imgw_meteo_abbrev.Rd b/man/imgw_meteo_abbrev.Rd index b05e011a..7ca53bd6 100644 --- a/man/imgw_meteo_abbrev.Rd +++ b/man/imgw_meteo_abbrev.Rd @@ -7,11 +7,11 @@ for the meteorological data from the danepubliczne.imgw.pl collection} \format{ The data contains a data.frame with ca. 250 elements described in three ways: -\itemize{ -\item{fullname} {original column names as downloaded from the repository} -\item{abbr_eng} {shorten column names with abbreviations derived from the most +\describe{ +\item{fullname}{original column names as downloaded from the repository} +\item{abbr_eng}{shorten column names with abbreviations derived from the most popular scheme used for meteorological parameters} -\item{fullname_eng} {detailed description of downloaded meteorological variables} +\item{fullname_eng}{detailed description of downloaded meteorological variables} } The object is created mostly to be used altogether with the meteo_shortening_imgw function diff --git a/man/imgw_meteo_stations.Rd b/man/imgw_meteo_stations.Rd index 30261938..f4654561 100644 --- a/man/imgw_meteo_stations.Rd +++ b/man/imgw_meteo_stations.Rd @@ -6,10 +6,12 @@ \title{Location of the meteorological stations from the danepubliczne.imgw.pl collection} \format{ The data contains a data.frame with 1998 obs. of 3 variables: -\itemize{ -\item{id} {Station ID} -\item{X} {Longitude} -\item{Y} {Latitude} +\describe{ +\item{id}{Station ID} +\item{X}{Longitude} +\item{Y}{Latitude} +\item{station}{Station name} +\item{id2}{IMGW-PIB ID for station rank} } The object is in the geographic coordinates using WGS84 (EPSG:4326). } diff --git a/man/meteo_imgw_daily.Rd b/man/meteo_imgw_daily.Rd index b1ad6b2a..718bb1db 100644 --- a/man/meteo_imgw_daily.Rd +++ b/man/meteo_imgw_daily.Rd @@ -26,8 +26,7 @@ meteo_imgw_daily( \item{coords}{add coordinates of the station (logical value TRUE or FALSE)} \item{station}{name of meteorological station(s). -It accepts names (characters in CAPITAL LETTERS); stations' IDs (numeric) are -no longer valid} +It accepts names (characters in CAPITAL LETTERS); Stations' IDs (numeric) are no longer valid} \item{col_names}{three types of column names possible: "short" - default, values with shorten names, diff --git a/man/meteo_imgw_datastore.Rd b/man/meteo_imgw_datastore.Rd new file mode 100644 index 00000000..4b6f2790 --- /dev/null +++ b/man/meteo_imgw_datastore.Rd @@ -0,0 +1,65 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/meteo_imgw_datastore.R +\name{meteo_imgw_datastore} +\alias{meteo_imgw_datastore} +\title{IMGW meteorological data from the IMGW datastore repository} +\usage{ +meteo_imgw_datastore( + year, + parameters = NULL, + stations = NULL, + coords = TRUE, + allow_failure = TRUE +) +} +\arguments{ +\item{year}{numeric vector of years to be downloaded (e.g., 2022:2023)} + +\item{parameters}{\itemize{ +\item character vector describing which parameters to be downloaded. Default \code{NULL} means to download all available. +\enumerate{ +\item "wd" - wind direction (degrees) +\item "t2m" - temperature at 2 metres above ground level (degree Celsius) +\item "t0m" - ground temperature (degree Celsius) +\item "rr_24h" - precipitation totals for last 24 hours (mm) +\item "rr_1h" - precipitation totals for last 1 hour (mm) +\item "rr_10min" - precipitation totals for last 10 minutes (mm) +\item "ws" - wind speed (m/s) +\item "ws_max" - maximum wind speed for last 10 minutes (m/s) +\item "gust" - wind gust (if present) (m/s) +\item "rh" - relative humidity (\%) +\item "water_in_snow" - water equivalent of melted snow cover (mm) +} +}} + +\item{stations}{\itemize{ +\item character vector with station names as visible in the \code{meteo_imgw_telemetry_stations()}. +Default \code{NULL} means to download data for all available stations. +}} + +\item{coords}{\itemize{ +\item logical - whether to append the dataset with station full name, longitude, latitude and altitude. Default: TRUE +}} + +\item{allow_failure}{logical - whether to proceed or stop on failure. By default set to TRUE (i.e. don't stop on error). For debugging purposes change to FALSE} +} +\description{ +Downloading hourly (meteorological) data from the telemetric stations +available in the danepubliczne.imgw.pl/datastore collection since 2008. +Most parameters are collected with 10 minutes interval and thus it is recommended to download only the mandatory years, parameters or stations. +For example, 1 year of data with all available parameters requires processing around 4GB of uncompressed data. +} +\details{ +Data from the IMGW automated (telemetry) systems are non validated by experts and may contain invalid values. +} +\examples{ +\donttest{ +# download only air temperature for selected 2 stations in 2022-2023: +imgw_telemetry = meteo_imgw_datastore(year = 2022:2023, + parameters = "t2m", + stations = c("HALA GĄSIENICOWA", + "DOLINA PIĘCIU STAWÓW"), + coords = TRUE) +} + +} diff --git a/man/stations_meteo_imgw_telemetry.Rd b/man/stations_meteo_imgw_telemetry.Rd new file mode 100644 index 00000000..17ca8b88 --- /dev/null +++ b/man/stations_meteo_imgw_telemetry.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/stations_meteo_imgw_telemetry.R +\name{stations_meteo_imgw_telemetry} +\alias{stations_meteo_imgw_telemetry} +\title{IMGW telemetry stations} +\usage{ +stations_meteo_imgw_telemetry() +} +\value{ +data table with metadata for over 500 stations. Metadata contains: station ID, station name, river, latitude, longitude, altitude +} +\description{ +Retrieving current metadata for stations used in the telemetric systems of the IMGW-PIB datastore (danepubliczne.imgw.pl/datastore) +} +\examples{ +\donttest{ + telemetry_stations = stations_meteo_imgw_telemetry() +} + +} diff --git a/tests/testthat/test-meteo_imgw.R b/tests/testthat/test-meteo_imgw.R index 62276389..93e9d832 100644 --- a/tests/testthat/test-meteo_imgw.R +++ b/tests/testthat/test-meteo_imgw.R @@ -2,23 +2,29 @@ context("meteo_imgw") y <- 2018 test_that("meteo_imgw works!", { - x <- meteo_imgw("hourly", "synop", year = y) - x <- meteo_imgw("hourly", "climate", year = y) - expect_message(x <- meteo_imgw("hourly", "precip", year = y)) - x <- meteo_imgw("daily", "synop", year = y) - x <- meteo_imgw("daily", "climate", year = y) - x <- meteo_imgw("daily", "precip", year = y) - x <- meteo_imgw("monthly", "synop", year = y) - x <- meteo_imgw("monthly", "climate", year = y) - x <- meteo_imgw("monthly", "precip", year = y) - x <- meteo_imgw("monthly", "synop", year = y, status = TRUE) - x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE) - x <- meteo_imgw("monthly", "synop", year = y, col_names = "full") - x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE, col_names = "full") - x <- meteo_imgw("monthly", "synop", year = y, col_names = "polish") - x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE, col_names = "polish") - x <- meteo_imgw("monthly", "synop", year = y, station = "BIAŁYSTOK") - x2 <- meteo_imgw("monthly", "synop", year = y, station = 353230295) - testthat::expect_message(x <- suppressWarnings(meteo_imgw_daily(rank = "synop", year = 2001, station = "blabla"))) + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + x <- meteo_imgw("hourly", "synop", year = y) + x <- meteo_imgw("hourly", "climate", year = y) + expect_message(x <- meteo_imgw("hourly", "precip", year = y)) + x <- meteo_imgw("daily", "synop", year = y) + x <- meteo_imgw("daily", "climate", year = y) + x <- meteo_imgw("daily", "precip", year = y) + x <- meteo_imgw("monthly", "synop", year = y) + x <- meteo_imgw("monthly", "climate", year = y) + x <- meteo_imgw("monthly", "precip", year = y) + x <- meteo_imgw("monthly", "synop", year = y, status = TRUE) + x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE) + x <- meteo_imgw("monthly", "synop", year = y, col_names = "full") + x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE, col_names = "full") + x <- meteo_imgw("monthly", "synop", year = y, col_names = "polish") + x <- meteo_imgw("monthly", "synop", year = y, coords = TRUE, col_names = "polish") + x <- meteo_imgw("monthly", "synop", year = y, station = "BIAŁYSTOK") + x2 <- meteo_imgw("monthly", "synop", year = y, station = 353230295) + + testthat::expect_message(x <- suppressWarnings(meteo_imgw_daily(rank = "synop", year = 2001, station = "blabla"))) + } }) diff --git a/tests/testthat/test-meteo_imgw_daily.R b/tests/testthat/test-meteo_imgw_daily.R index fe06bcf8..1c1ead0c 100644 --- a/tests/testthat/test-meteo_imgw_daily.R +++ b/tests/testthat/test-meteo_imgw_daily.R @@ -2,25 +2,42 @@ context("meteo_imgw_daily") test_that("meteo_imgw_daily", { - y <- 1900 # year not supported - expect_message(meteo_imgw_daily(rank = "synop", year = y, status = TRUE, - coords = TRUE, allow_failure = TRUE)) + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + y <- 1900 # year not supported + expect_message(meteo_imgw_daily(rank = "synop", year = y, status = TRUE, + coords = TRUE, allow_failure = TRUE)) + } }) test_that("check_column_with_coordinates", { - station_with_coordinates = meteo_imgw_daily(rank = "precip", - year = 2002, - coords = TRUE, - station = "IMBRAMOWICE") - if (is.data.frame(station_with_coordinates)) { - expect_true(any(colnames(station_with_coordinates) %in% c("X", "Y"))) + + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + station_with_coordinates = meteo_imgw_daily(rank = "precip", + year = 2002, + coords = TRUE, + station = "IMBRAMOWICE") + if (is.data.frame(station_with_coordinates)) { + expect_true(any(colnames(station_with_coordinates) %in% c("X", "Y"))) + } } }) test_that("check_message_for_non_existing_station", { - expect_message(meteo_imgw_daily(rank = "precip", - year = 2002, - coords = TRUE, - station = 9999)) + + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + expect_message(meteo_imgw_daily(rank = "precip", + year = 2002, + coords = TRUE, + station = 9999)) + } }) diff --git a/tests/testthat/test-meteo_imgw_datastore.R b/tests/testthat/test-meteo_imgw_datastore.R new file mode 100644 index 00000000..8b567da0 --- /dev/null +++ b/tests/testthat/test-meteo_imgw_datastore.R @@ -0,0 +1,19 @@ +context("meteo-imgw-datastore") + + +test_that("test-meteo_imgw_datastore", { + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + + imgw_telemetry = meteo_imgw_datastore(year = 2023, + parameters = "t2m", + stations = "PSZENNO") + + if (is.data.frame(imgw_telemetry) & nrow(imgw_telemetry) > 0) { + testthat::expect_true(is.data.frame(imgw_telemetry)) + testthat::expect_true(nrow(imgw_telemetry) > 50000) + } + } +}) diff --git a/tests/testthat/test-meteo_metadata_imgw.R b/tests/testthat/test-meteo_metadata_imgw.R index d82d2dd9..9e99236a 100644 --- a/tests/testthat/test-meteo_metadata_imgw.R +++ b/tests/testthat/test-meteo_metadata_imgw.R @@ -2,26 +2,31 @@ context("meteo-metadata") test_that("tests to be re-written meteo_metadata_imgw", { #skip("meteo-metadata skipping") - m_hs <- meteo_metadata_imgw("hourly", "synop") - m_hc <- meteo_metadata_imgw("hourly", "climate") - m_ds <- meteo_metadata_imgw("daily", "synop") - m_dc <- meteo_metadata_imgw("daily", "climate") - m_dp <- meteo_metadata_imgw("daily", "precip") - m_ms <- meteo_metadata_imgw("monthly", "synop") - m_mc <- meteo_metadata_imgw("monthly", "climate") - m_mp <- meteo_metadata_imgw("monthly", "precip") - - expect_error(meteo_metadata_imgw("hourly", "precip")) - expect_equal(dim(m_hs[[1]]), c(107, 3)) - expect_equal(dim(m_hc[[1]]), c(22, 3)) - expect_equal(dim(m_ds[[1]]), c(65, 3)) - expect_equal(dim(m_ds[[2]]), c(23, 3)) - expect_equal(dim(m_dc[[1]]), c(18, 3)) - expect_equal(dim(m_dc[[2]]), c(13, 3)) - expect_equal(dim(m_dp[[1]]), c(16, 3)) - expect_equal(dim(m_ms[[1]]), c(60, 3)) - expect_equal(dim(m_ms[[2]]), c(22, 3)) - expect_equal(dim(m_mc[[1]]), c(27, 3)) - expect_equal(dim(m_mc[[2]]), c(12, 3)) - expect_equal(dim(m_mp[[1]]), c(14, 3)) + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + m_hs <- meteo_metadata_imgw("hourly", "synop") + m_hc <- meteo_metadata_imgw("hourly", "climate") + m_ds <- meteo_metadata_imgw("daily", "synop") + m_dc <- meteo_metadata_imgw("daily", "climate") + m_dp <- meteo_metadata_imgw("daily", "precip") + m_ms <- meteo_metadata_imgw("monthly", "synop") + m_mc <- meteo_metadata_imgw("monthly", "climate") + m_mp <- meteo_metadata_imgw("monthly", "precip") + + expect_error(meteo_metadata_imgw("hourly", "precip")) + expect_equal(dim(m_hs[[1]]), c(107, 3)) + expect_equal(dim(m_hc[[1]]), c(22, 3)) + expect_equal(dim(m_ds[[1]]), c(65, 3)) + expect_equal(dim(m_ds[[2]]), c(23, 3)) + expect_equal(dim(m_dc[[1]]), c(18, 3)) + expect_equal(dim(m_dc[[2]]), c(13, 3)) + expect_equal(dim(m_dp[[1]]), c(16, 3)) + expect_equal(dim(m_ms[[1]]), c(60, 3)) + expect_equal(dim(m_ms[[2]]), c(22, 3)) + expect_equal(dim(m_mc[[1]]), c(27, 3)) + expect_equal(dim(m_mc[[2]]), c(12, 3)) + expect_equal(dim(m_mp[[1]]), c(14, 3)) + } }) diff --git a/tests/testthat/test-meteo_ogimet.R b/tests/testthat/test-meteo_ogimet.R index 9eab933a..785911b5 100644 --- a/tests/testthat/test-meteo_ogimet.R +++ b/tests/testthat/test-meteo_ogimet.R @@ -41,21 +41,25 @@ test_that("meteo_ogimet works!", { # only wind measurement are present: - testthat::expect_error( - meteo_ogimet( - date = c(as.Date("2020-02-01"), Sys.Date() - 1), - # date = c(Sys.Date() - 7, Sys.Date() - 1), - interval = "daily", - coords = FALSE, - station = "06683", allow_failure = FALSE) - ) - - # check change between years: - multiyr = ogimet_daily(date = c( as.Date("2022-11-15"), - as.Date("2023-02-01")), - station = 12330) - if (is.data.frame(multiyr)) { - testthat::expect_true(nrow(multiyr) > 70) + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + testthat::expect_error( + meteo_ogimet( + date = c(as.Date("2020-02-01"), Sys.Date() - 1), + # date = c(Sys.Date() - 7, Sys.Date() - 1), + interval = "daily", + coords = FALSE, + station = "06683", allow_failure = FALSE) + ) + + # check change between years: + multiyr = ogimet_daily(date = c( as.Date("2022-11-15"), + as.Date("2023-02-01")), + station = 12330) + if (is.data.frame(multiyr)) { + testthat::expect_true(nrow(multiyr) > 70) + } } - }) diff --git a/tests/testthat/test-nearest_stations_noaa.R b/tests/testthat/test-nearest_stations_noaa.R index 563bf51d..42746958 100644 --- a/tests/testthat/test-nearest_stations_noaa.R +++ b/tests/testthat/test-nearest_stations_noaa.R @@ -1,41 +1,50 @@ test_that("nearest_stations_noaa", { - x = nearest_stations_noaa(country = "SRI LANKA", - point = c(80, 6), - add_map = TRUE, - no_of_stations = 10) - if (is.data.frame(x)) { - testthat::expect_true(nrow(x) <= 10) + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + x = nearest_stations_noaa(country = "SRI LANKA", + point = c(80, 6), + add_map = TRUE, + no_of_stations = 10) + if (is.data.frame(x)) { + testthat::expect_true(nrow(x) <= 10) + } } }) test_that("nearest_stations_noaa_errors", { - testthat::expect_message(nearest_stations_noaa(allow_failure = TRUE)) - testthat::expect_error(nearest_stations_noaa(allow_failure = FALSE)) - testthat::expect_error(nearest_stations_noaa(country = "POLAND", - point = c(10, 20, 30), - allow_failure = FALSE)) - + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + testthat::expect_message(nearest_stations_noaa(allow_failure = TRUE)) + testthat::expect_error(nearest_stations_noaa(allow_failure = FALSE)) + testthat::expect_error(nearest_stations_noaa(country = "POLAND", + point = c(10, 20, 30), + allow_failure = FALSE)) + # testthat::expect_message(nearest_stations_noaa(country = "POLAND", # point = 1, # allow_failure = FALSE)) # # check if this is query is retrieving data for Poland: - x = nearest_stations_noaa(country = "POLAND", point = c(19, 52)) + x = nearest_stations_noaa(country = "POLAND", point = c(19, 52)) + + if (is.data.frame(x)) { + testthat::expect_equal(unique(x$countries), "POLAND") + } - if (is.data.frame(x)) { - testthat::expect_equal(unique(x$countries), "POLAND") - } - - testthat::expect_error(nearest_stations_noaa(country = "POLAND", - point = c(30, 50), - date = c(Sys.Date() - 7, Sys.Date() - 1), - allow_failure = FALSE)) - - testthat::expect_message(nearest_stations_noaa(country = "SOVIET UNION", - point = c(30, 50), - date = c(Sys.Date()), - allow_failure = TRUE)) - + testthat::expect_error(nearest_stations_noaa(country = "POLAND", + point = c(30, 50), + date = c(Sys.Date() - 7, Sys.Date() - 1), + allow_failure = FALSE)) + + testthat::expect_message(nearest_stations_noaa(country = "SOVIET UNION", + point = c(30, 50), + date = c(Sys.Date()), + allow_failure = TRUE)) + } }) diff --git a/tests/testthat/test-stations_meteo_imgw_telemetry.R b/tests/testthat/test-stations_meteo_imgw_telemetry.R new file mode 100644 index 00000000..f733ebb6 --- /dev/null +++ b/tests/testthat/test-stations_meteo_imgw_telemetry.R @@ -0,0 +1,16 @@ +context("stations_meteo_imgw_telemetry") + + +test_that("test-stations_meteo_imgw_telemetry", { + if (!curl::has_internet()) { + message("No internet connection! \n") + return(invisible(NULL)) + } else { + + df = stations_meteo_imgw_telemetry() + if (is.data.frame(df) & nrow(df) > 0) { + testthat::expect_true(is.data.frame(df)) + } + + } +}) diff --git a/vignettes/getstarted.Rmd b/vignettes/getstarted.Rmd index feb70543..75fe552d 100644 --- a/vignettes/getstarted.Rmd +++ b/vignettes/getstarted.Rmd @@ -22,7 +22,7 @@ and hydrological data from publicly available repositories: - OGIMET [(ogimet.com)](http://ogimet.com/index.phtml.en) - University of Wyoming - atmospheric vertical profiling data (http://weather.uwyo.edu/upperair/). -- Polish Institute of Meterology and Water Management - National Research Institute [(IMGW-PIB)](https://dane.imgw.pl/) +- Polish Institute of Meteorology and Water Management - National Research Institute [(IMGW-PIB)](https://dane.imgw.pl/) - National Oceanic & Atmospheric Administration - Earth System Research Laboratory - Global Monitoring Division [(NOAA)](https://gml.noaa.gov/ccgg/trends/) - National Oceanic & Atmospheric Administration - National Climatic Data Center - Integrated Surface Hourly (ISH) [(NOAA)](https://www1.ncdc.noaa.gov/pub/data/noaa/)