quick test deploy with service level data only

dfe-analytical-services · Feb 10, 2025 · b9b3e03 · b9b3e03
1 parent cc73e42
commit b9b3e03
Show file tree

Hide file tree

Showing 12 changed files with 425 additions and 300 deletions.
diff --git a/README.md b/README.md
@@ -26,9 +26,23 @@ Code used to extract source data, process it, and save a permanent store for usa
 
 ### iii. Access requirements
 
-To run the dashboard locally:
+Old data:
 - Access to the MA_SDT_NS_DATA database
 
+Latest data:
+- Access the statistics services area of the unity catalog
+
+To set up access to the app, and understand how the app itself is connected to the SQL warehouse on the server, look at:
+- [Connecting to a SQL warehouse from R Studio](https://dfe-analytical-services.github.io/analysts-guide/ADA/databricks_rstudio_sql_warehouse.html)
+- [R Shiny app databricks connection guide](https://rsconnect/rsc/posit-connect-guidance/_book/databricks-connections.html).
+
+If you don't have access to the source data, you can still run the dashboard using:
+```r
+# This makes the app think it's in test mode and will read in the test data in the repo
+# instead of connecting to the databases
+withr::with_envvar(c(TESTTHAT = "true"), shiny::runApp())
+```
+
 ## Contributing to the dashboard
 
 ### Running locally

diff --git a/config.yml b/config.yml
@@ -6,6 +6,7 @@ default:
     uid: ""
     pwd: ""
     trusted: "Yes"
+    sql_warehouse_id: !expr Sys.getenv("DATABRICKS_SQL_WAREHOUSE_ID")
 
 production:
   db_connection:
@@ -15,3 +16,4 @@ production:
     uid: !expr Sys.getenv("SDT_UID")
     pwd: !expr Sys.getenv("SDT_PWD")
     trusted: "No"
+    sql_warehouse_id: !expr Sys.getenv("DATABRICKS_SQL_WAREHOUSE_ID")
diff --git a/...tion-statistics-analytics-dashboard.Rproj → ees-analytics-dashboard.Rproj b/...tion-statistics-analytics-dashboard.Rproj → ees-analytics-dashboard.Rproj
diff --git a/global.R b/global.R
@@ -25,9 +25,8 @@ shhh(library(dfeR))
 
 # Database connection
 shhh(library(dbplyr))
-shhh(library(DBI))
-shhh(library(config))
 shhh(library(odbc))
+shhh(library(pool))
 
 # Data vis
 shhh(library(plotly))
@@ -39,9 +38,29 @@ if ("meaning of life" == "42") {
   library(git2r)
   library(rsconnect)
   library(shinytest2)
+  # we should never run library(config)
+  # their own docs recommend only calling using config::
+  # this is due to function / namespace clashes and masking
+  library(config)
 }
 
-message("...library calls done, setting up global variables...")
+message("...library calls done, setting up database connection...")
+
+# Load data ====================================================================
+if (Sys.getenv("TESTTHAT") == "") {
+  config <- config::get("db_connection")
+
+  pool <- pool::dbPool(
+    drv = odbc::databricks(),
+    httpPath = config$sql_warehouse_id
+  )
+
+  onStop(function() {
+    pool::poolClose(pool)
+  })
+
+  message("...connected to database...")
+}
 
 # Global variables ============================================================
 link_guidance <- tags$a(
@@ -79,63 +98,4 @@ all_time_date <- "2020-04-03"
 # Custom functions ============================================================
 source("R/utils.R")
 
-message("...global variables set, loading data...")
-
-# Load in data ================================================================
-
-# File paths are relative to analytics-dashboard/ directory
-
-if (Sys.getenv("TEST_MODE") == "") {
-  config <- config::get("db_connection")
-  connection <- dbConnect(odbc::odbc(),
-    Driver = config$driver,
-    Server = config$server,
-    Database = config$database,
-    UID = config$uid,
-    PWD = config$pwd,
-    Trusted_Connection = config$trusted,
-    encoding = "UTF-8"
-  )
-
-  message("...connected to database...")
-
-  joined_data1 <- tbl(connection, "ees_analytics_page_data") %>%
-    as.data.frame()
-
-  message("...page data loaded, loading publication aggregations...")
-
-  pub_agg1 <- tbl(connection, "ees_analytics_publication_agg") %>%
-    as.data.frame()
-
-  message("...publication aggregations loaded, loading service data...")
-
-  combined_data1 <- tbl(connection, "ees_analytics_service_data") %>%
-    as.data.frame()
-
-  message("...service data loaded, loading publication spine...")
-
-  pubs1 <- read_csv("data/pubs.csv", show_col_types = FALSE)
-
-  message("Complete!")
-} else if (Sys.getenv("TEST_MODE") == "TRUE") {
-  message("...in test mode...")
-
-  joined_data1 <- arrow::read_parquet(
-    "tests/testdata/joined_data_0.parquet"
-  )
-
-  pub_agg1 <- arrow::read_parquet(
-    "tests/testdata/publication_aggregation_0.parquet"
-  )
-  combined_data1 <- arrow::read_parquet(
-    "tests/testdata/combined_data_0.parquet"
-  )
-
-  pubs1 <- arrow::read_parquet(
-    "tests/testdata/pub_spine_0.parquet"
-  )
-
-  message("Complete!")
-} else {
-  message("...no data loaded. TEST_MODE = ", Sys.getenv("TEST_MODE"))
-}
+message("...global variables set!")
diff --git a/manifest.json b/manifest.json
diff --git a/renv.lock b/renv.lock
@@ -528,14 +528,14 @@
     },
     "data.table": {
       "Package": "data.table",
-      "Version": "1.16.2",
+      "Version": "1.16.4",
       "Source": "Repository",
-      "Repository": "RSPM",
+      "Repository": "CRAN",
       "Requirements": [
         "R",
         "methods"
       ],
-      "Hash": "2e00b378fc3be69c865120d9f313039a"
+      "Hash": "38bbf05fc2503143db4c734a7e5cab66"
     },
     "dbplyr": {
       "Package": "dbplyr",
@@ -1104,16 +1104,16 @@
     },
     "lubridate": {
       "Package": "lubridate",
-      "Version": "1.9.3",
+      "Version": "1.9.4",
       "Source": "Repository",
-      "Repository": "CRAN",
+      "Repository": "RSPM",
       "Requirements": [
         "R",
         "generics",
         "methods",
         "timechange"
       ],
-      "Hash": "680ad542fbcf801442c83a6ac5a2126c"
+      "Hash": "be38bc740fc51783a78edb5a157e4104"
     },
     "magrittr": {
       "Package": "magrittr",
@@ -1344,6 +1344,21 @@
       ],
       "Hash": "a1ac5c03ad5ad12b9d1597e00e23c3dd"
     },
+    "pool": {
+      "Package": "pool",
+      "Version": "1.0.4",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "DBI",
+        "R",
+        "R6",
+        "later",
+        "methods",
+        "rlang"
+      ],
+      "Hash": "385e70e224a9b47a1df76bd893a46a46"
+    },
     "praise": {
       "Package": "praise",
       "Version": "1.0.0",
@@ -1760,7 +1775,7 @@
     },
     "testthat": {
       "Package": "testthat",
-      "Version": "3.2.1.1",
+      "Version": "3.2.3",
       "Source": "Repository",
       "Repository": "RSPM",
       "Requirements": [
@@ -1785,7 +1800,7 @@
         "waldo",
         "withr"
       ],
-      "Hash": "3f6e7e5e2220856ff865e4834766bf2b"
+      "Hash": "42f889439ccb14c55fc3d75c9c755056"
     },
     "tibble": {
       "Package": "tibble",