Skip to content

Commit

Permalink
fix: simplify read and fix IMGW tests
Browse files Browse the repository at this point in the history
  • Loading branch information
bczernecki committed Oct 28, 2024
1 parent 549de33 commit 031aa30
Show file tree
Hide file tree
Showing 12 changed files with 98 additions and 144 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ covr_report.html
lib
docs
pkgdown
.Renviron
.Renviron
test-out.txt
2 changes: 1 addition & 1 deletion NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* adjusting code to recognize different encoding and directory structure
* adjusting changes in metadata
* removed option to download data for "semiannual and annual" time resolutions due to inconsistencies in the data
* Fix unit tests for ogimet-related datasets
* Fix unit tests for ogimet- and IMGW-related datasets

# climate 1.2.1

Expand Down
36 changes: 3 additions & 33 deletions R/hydro_imgw_daily.R
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
hydro_imgw_daily = function(year,
coords = FALSE,
station = NULL,
col_names= "short",
col_names = "short",
allow_failure = TRUE,
...) {

Expand Down Expand Up @@ -103,22 +103,7 @@ hydro_imgw_daily_bp = function(year,
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]

if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f ISO-8859-2 -t ASCII//TRANSLIT", file1)))
} else {
data1 = tryCatch(expr = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ",",
fileEncoding = "CP1250"),
warning = function(w) {
read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
if (ncol(data1) == 1) {
data1 = tryCatch(expr = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";",
fileEncoding = "UTF-8"),
warning = function(w) {
read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
}
}
data1 = imgw_read(translit, file1)
# extra exception for a current year according to information provided by IMGW-PIB:, i.e.:
# "Do czasu zakonczenia kontroli przeplywow rekordy z danymi z roku 2020 maja format:
# Kod stacji #Nazwa stacji #Nazwa rzeki/jeziora #Rok hydrologiczny #Wskaznik miesiaca w roku hydrologicznym
Expand All @@ -141,22 +126,7 @@ hydro_imgw_daily_bp = function(year,
test_url(address, temp)
unzip(zipfile = temp, exdir = temp2)
file2 = paste(temp2, dir(temp2), sep = "/")[1]
if (translit) {
data2 = as.data.frame(data.table::fread(cmd = paste("iconv -f ISO-8859-2 -t ASCII//TRANSLIT", file2)))
} else {
data2 = tryCatch(expr = read.csv(file2, header = FALSE, stringsAsFactors = FALSE, sep = ",",
fileEncoding = "CP1250"),
warning = function(w) {
read.csv(file2, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
if (ncol(data2) == 1) {
data2 = tryCatch(expr = read.csv(file2, header = FALSE, stringsAsFactors = FALSE, sep = ";",
fileEncoding = "UTF-8"),
warning = function(w) {
read.csv(file2, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
}
}
data2 = imgw_read(translit, file2)
colnames(data2) = meta[[2]][, 1]
zjaw_data = rbind(zjaw_data, data2)
}
Expand Down
19 changes: 1 addition & 18 deletions R/hydro_imgw_monthly.R
Original file line number Diff line number Diff line change
Expand Up @@ -85,24 +85,7 @@ hydro_imgw_monthly_bp = function(year,
test_url(adres, temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]

if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f ISO-8859-2 -t ASCII//TRANSLIT", file1)))
} else {
data1 = tryCatch(expr = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ",",
fileEncoding = "CP1250"),
warning = function(w) {
read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
if (ncol(data1) == 1) {
data1 = tryCatch(expr = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";",
fileEncoding = "UTF-8"),
warning = function(w) {
read.csv(file1, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
}
}

data1 = imgw_read(translit, file1)
colnames(data1) = meta[[1]][, 1]
all_data[[i]] = data1
}
Expand Down
31 changes: 31 additions & 0 deletions R/imgw_read.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#' Read IMGW hydrological and meteorological raw files that can be saved in different formats
#'
#' Internal function for reading IMGW files
#' @param translit logical whether translit detected and iconv needed for reading
#' @param fpath path to unzipped CSV-alike file
#'
#' @keywords internal
#' @noRd

imgw_read = function(translit, fpath) {

if (translit) {
data = as.data.frame(data.table::fread(cmd = paste("iconv -f ISO-8859-2 -t ASCII//TRANSLIT", fpath)))
} else {
data = tryCatch(expr = read.csv(fpath, header = FALSE, stringsAsFactors = FALSE, sep = ",",
fileEncoding = "CP1250"),
warning = function(w) {
read.csv(fpath, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})

if (ncol(data) == 1) {
data = tryCatch(expr = read.csv(fpath, header = FALSE, stringsAsFactors = FALSE, sep = ";",
fileEncoding = "UTF-8"),
warning = function(w) {
read.csv(fpath, header = FALSE, stringsAsFactors = FALSE, sep = ";")
})
}

}
return(data)
}
34 changes: 6 additions & 28 deletions R/meteo_imgw_daily.R
Original file line number Diff line number Diff line change
Expand Up @@ -113,20 +113,12 @@ meteo_imgw_daily_bp = function(rank,
test_url(addresses_to_download[j], temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]
if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}
data1 = imgw_read(translit, file1)
colnames(data1) = meta[[1]]$parameters

file2 = paste(temp2, dir(temp2), sep = "/")[2]
if (file.exists(file2)) {
if (translit) {
data2 = data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file2))
} else {
data2 = suppressWarnings(read.csv(file2, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250"))
}
data2 = imgw_read(translit, file2)
colnames(data2) = meta[[2]]$parameters
}

Expand Down Expand Up @@ -174,19 +166,11 @@ meteo_imgw_daily_bp = function(rank,
test_url(addresses_to_download[j], temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]
if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}
data1 = imgw_read(translit, file1)
colnames(data1) = meta[[1]]$parameters

file2 = paste(temp2, dir(temp2), sep = "/")[2]
if (translit) {
data2 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file2)))
} else {
data2 = read.csv(file2, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}
data2 = imgw_read(translit, file2)
colnames(data2) = meta[[2]]$parameters

# usuwa statusy
Expand Down Expand Up @@ -222,15 +206,9 @@ meteo_imgw_daily_bp = function(rank,
temp2 = tempfile()
test_url(addresses_to_download[j], temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]
if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}

data1 = imgw_read(translit, file1)
colnames(data1) = meta[[1]]$parameters
# usuwa statusy
# remove status
if (status == FALSE) {
data1[grep("^Status", colnames(data1))] = NULL
}
Expand Down
24 changes: 7 additions & 17 deletions R/meteo_imgw_hourly.R
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,7 @@ meteo_imgw_hourly_bp = function(rank,
test_url(addresses_to_download[j], temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")

if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = suppressWarnings(read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250"))
}
data1 = imgw_read(translit, file1)

colnames(data1) = meta[[1]]$parameters

Expand Down Expand Up @@ -153,24 +148,19 @@ meteo_imgw_hourly_bp = function(rank,
test_url(addresses_to_download[j], temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")

if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}

data1 = imgw_read(translit, file1)

colnames(data1) = meta[[1]]$parameters
# usuwa statusy
# remove status
if (status == FALSE) {
data1[grep("^Status", colnames(data1))] = NULL
}

unlink(c(temp, temp2))
all_data[[length(all_data) + 1]] = data1
} # koniec petli po zipach do pobrania
} # koniec if'a dla klimatu
} # koniec petli po glownych catalogach danych dobowych
} # end of looping for zip files
} # end of if statement for climate
} # end of loop over directories

all_data = do.call(rbind, all_data)

Expand Down
16 changes: 3 additions & 13 deletions R/meteo_imgw_monthly.R
Original file line number Diff line number Diff line change
Expand Up @@ -123,25 +123,15 @@ meteo_imgw_monthly_bp = function(rank,
#download.file(address, temp)
unzip(zipfile = temp, exdir = temp2)
file1 = paste(temp2, dir(temp2), sep = "/")[1]

if (translit) {
data1 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file1)))
} else {
data1 = suppressWarnings(read.csv(file1, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250"))
}
data1 = imgw_read(translit, file1)

colnames(data1) = meta[[1]]$parameters

if (rank != "precip") { # w opadowkach jest tylko jeden plik
file2 = paste(temp2, dir(temp2), sep = "/")[2]
if (file.exists(file2)) {
if (translit) {
data2 = as.data.frame(data.table::fread(cmd = paste("iconv -f CP1250 -t ASCII//TRANSLIT", file2)))
} else {
data2 = read.csv(file2, header = FALSE, stringsAsFactors = FALSE, fileEncoding = "CP1250")
}

colnames(data2) = meta[[2]]$parameters
data2 = imgw_read(translit, file2)
colnames(data2) = meta[[2]]$parameters
}
}

Expand Down
16 changes: 10 additions & 6 deletions tests/testthat/test-hydro_imgw.R
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,22 @@ test_that("hydro_imgw_not_available", {
coord = TRUE,
allow_failure = FALSE)

if (is.data.frame(h2022_2023) & nrow(h2022_2023 > 50000)) {
testthat::expect_true(is.data.frame(h2022_2023))
testthat::expect_true(nrow(h2022_2023) > 50000)
if (!is.null(h2022_2023)) {
if (is.data.frame(h2022_2023) & nrow(h2022_2023 > 50000)) {
testthat::expect_true(is.data.frame(h2022_2023))
testthat::expect_true(nrow(h2022_2023) > 50000)
}
}

h2022_2023d = hydro_imgw(interval = "daily",
year = 2022:2023,
coord = TRUE,
allow_failure = FALSE)
if (is.data.frame(h2022_2023d) & nrow(h2022_2023d > 50000)) {
testthat::expect_true(is.data.frame(h2022_2023d))
testthat::expect_true(nrow(h2022_2023d) > 50000)
if (!is.null(h2022_2023d)) {
if (is.data.frame(h2022_2023d) & nrow(h2022_2023d > 50000)) {
testthat::expect_true(is.data.frame(h2022_2023d))
testthat::expect_true(nrow(h2022_2023d) > 50000)
}
}

})
9 changes: 5 additions & 4 deletions tests/testthat/test-meteo_imgw_datastore.R
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ test_that("test-meteo_imgw_datastore", {
imgw_telemetry = meteo_imgw_datastore(year = 2023,
parameters = "t2m",
stations = "PSZENNO")

if (is.data.frame(imgw_telemetry) & nrow(imgw_telemetry) > 0) {
testthat::expect_true(is.data.frame(imgw_telemetry))
testthat::expect_true(nrow(imgw_telemetry) > 50000)
if (!is.null(imgw_telemetry)) {
if (is.data.frame(imgw_telemetry) & nrow(imgw_telemetry) > 0) {
testthat::expect_true(is.data.frame(imgw_telemetry))
testthat::expect_true(nrow(imgw_telemetry) > 50000)
}
}
}
})
46 changes: 25 additions & 21 deletions tests/testthat/test-meteo_metadata_imgw.R
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
context("meteo-metadata")

test_that("tests to be re-written meteo_metadata_imgw", {
test_that("meteo_metadata_imgw tests", {
#skip("meteo-metadata skipping")
if (!curl::has_internet()) {
message("No internet connection! \n")
return(invisible(NULL))
} else {
m_hs <- meteo_metadata_imgw("hourly", "synop")
m_hc <- meteo_metadata_imgw("hourly", "climate")
m_ds <- meteo_metadata_imgw("daily", "synop")
m_dc <- meteo_metadata_imgw("daily", "climate")
m_dp <- meteo_metadata_imgw("daily", "precip")
m_ms <- meteo_metadata_imgw("monthly", "synop")
m_mc <- meteo_metadata_imgw("monthly", "climate")
m_mp <- meteo_metadata_imgw("monthly", "precip")
m_hs = meteo_metadata_imgw("hourly", "synop")
m_hc = meteo_metadata_imgw("hourly", "climate")
m_ds = meteo_metadata_imgw("daily", "synop")
m_dc = meteo_metadata_imgw("daily", "climate")
m_dp = meteo_metadata_imgw("daily", "precip")
m_ms = meteo_metadata_imgw("monthly", "synop")
m_mc = meteo_metadata_imgw("monthly", "climate")
m_mp = meteo_metadata_imgw("monthly", "precip")

expect_error(meteo_metadata_imgw("hourly", "precip"))
expect_equal(dim(m_hs[[1]]), c(107, 3))
expect_equal(dim(m_hc[[1]]), c(22, 3))
expect_equal(dim(m_ds[[1]]), c(65, 3))
expect_equal(dim(m_ds[[2]]), c(23, 3))
expect_equal(dim(m_dc[[1]]), c(18, 3))
expect_equal(dim(m_dc[[2]]), c(13, 3))
expect_equal(dim(m_dp[[1]]), c(16, 3))
expect_equal(dim(m_ms[[1]]), c(60, 3))
expect_equal(dim(m_ms[[2]]), c(22, 3))
expect_equal(dim(m_mc[[1]]), c(27, 3))
expect_equal(dim(m_mc[[2]]), c(12, 3))
expect_equal(dim(m_mp[[1]]), c(14, 3))

if (is.list(m_hs) && is.list(m_ds) && is.list(m_ds) && is.list(m_dc) &&
is.list(m_dp) && is.list(m_ms) && is.list(m_mc) && is.list(m_mp)) {
expect_equal(dim(m_hs[[1]]), c(107, 3))
expect_equal(dim(m_hc[[1]]), c(22, 3))
expect_equal(dim(m_ds[[1]]), c(65, 3))
expect_equal(dim(m_ds[[2]]), c(23, 3))
expect_equal(dim(m_dc[[1]]), c(18, 3))
expect_equal(dim(m_dc[[2]]), c(13, 3))
expect_equal(dim(m_dp[[1]]), c(16, 3))
expect_equal(dim(m_ms[[1]]), c(60, 3))
expect_equal(dim(m_ms[[2]]), c(22, 3))
expect_equal(dim(m_mc[[1]]), c(27, 3))
expect_equal(dim(m_mc[[2]]), c(12, 3))
expect_equal(dim(m_mp[[1]]), c(14, 3))
}
}
})
6 changes: 4 additions & 2 deletions tests/testthat/test-stations_meteo_imgw_telemetry.R
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ test_that("test-stations_meteo_imgw_telemetry", {
} else {

df = stations_meteo_imgw_telemetry()
if (is.data.frame(df) & nrow(df) > 0) {
testthat::expect_true(is.data.frame(df))
if (!is.null(df)) {
if (is.data.frame(df) & nrow(df) > 0) {
testthat::expect_true(is.data.frame(df))
}
}

}
Expand Down

0 comments on commit 031aa30

Please sign in to comment.