From cf93ce652ea06453814063d1d752f555f12f0b7d Mon Sep 17 00:00:00 2001
From: adeveloper-wq <dajesch@gmail.com>
Date: Fri, 28 Jun 2024 13:57:12 +0200
Subject: [PATCH] Add separate FROST http endpoint URL for the pre-fetch of
 observations

---
 .env                      |  3 ++-
 env/env.go                | 10 +++++++---
 env/env_test.go           |  6 ++++--
 observations/sync.go      |  2 +-
 observations/sync_test.go |  2 +-
 things/sync.go            |  2 +-
 6 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/.env b/.env
index cba03b1..ccd25c9 100644
--- a/.env
+++ b/.env
@@ -4,7 +4,8 @@ PREDICTION_MQTT_USERNAME=
 PREDICTION_MQTT_PASSWORD=
 
 # The FROST server config.
-SENSORTHINGS_URL=https://tld.iot.hamburg.de/v1.1/
+SENSORTHINGS_URL_THINGS=https://tld.iot.hamburg.de/v1.1/ # The URL of the SensorThings API. Used to fetch the things.
+SENSORTHINGS_URL_OBSERVATIONS=https://tld.iot.hamburg.de/v1.1/ # The URL of the SensorThings API. Used to pre-fetch the observations. Can be the same as the things URL.
 SENSORTHINGS_MQTT_URL=tcp://tld.iot.hamburg.de:1883
 
 # The path under which all resources will be stored for the web API.
diff --git a/env/env.go b/env/env.go
index 9c1dfb8..2138660 100644
--- a/env/env.go
+++ b/env/env.go
@@ -32,8 +32,11 @@ func loadOptional(name string, validate func(string) *error) string {
 // The path under which the history files are stored, from the environment variable.
 var StaticPath string
 
-// The SensorThings API base URL.
-var SensorThingsBaseUrl string
+// The SensorThings API base URL used for fetching things.
+var SensorThingsBaseUrlThings string
+
+// The SensorThings API base URL used for pre-fetching observations.
+var SensorThingsBaseUrlObservations string
 
 // The URL to the observation MQTT broker from the environment variable.
 var SensorThingsObservationMqttUrl string
@@ -85,7 +88,8 @@ var emptyValidator = func(value string) *error {
 
 func Init() {
 	StaticPath = loadRequired("STATIC_PATH", staticPathValidator)
-	SensorThingsBaseUrl = loadRequired("SENSORTHINGS_URL", sensorThingsBaseUrlValidator)
+	SensorThingsBaseUrlThings = loadRequired("SENSORTHINGS_URL_THINGS", sensorThingsBaseUrlValidator)
+	SensorThingsBaseUrlObservations = loadRequired("SENSORTHINGS_URL_OBSERVATIONS", sensorThingsBaseUrlValidator)
 	SensorThingsObservationMqttUrl = loadRequired("SENSORTHINGS_MQTT_URL", sensorThingsObservationMqttUrlValidator)
 	PredictionMqttUrl = loadRequired("PREDICTION_MQTT_URL", predictionMqttUrlValidator)
 	PredictionMqttUsername = loadOptional("PREDICTION_MQTT_USERNAME", emptyValidator)
diff --git a/env/env_test.go b/env/env_test.go
index ea9a8c5..e332a23 100644
--- a/env/env_test.go
+++ b/env/env_test.go
@@ -40,7 +40,8 @@ func TestLoadRequiredPanics(t *testing.T) {
 
 func TestRequiredEnvComplete(t *testing.T) {
 	t.Setenv("STATIC_PATH", "/usr/share/nginx/html")
-	t.Setenv("SENSORTHINGS_URL", "https://tld.iot.hamburg.de/v1.1/")
+	t.Setenv("SENSORTHINGS_URL_THINGS", "https://tld.iot.hamburg.de/v1.1/")
+	t.Setenv("SENSORTHINGS_URL_OBSERVATIONS", "https://tld.iot.hamburg.de/v1.1/")
 	t.Setenv("SENSORTHINGS_MQTT_URL", "tcp://tld.iot.hamburg.de:1883")
 	t.Setenv("PREDICTION_MQTT_URL", "tcp://predictor-mosquitto:1883")
 	Init()
@@ -62,7 +63,8 @@ func TestValidatorTriggers(t *testing.T) {
 		}
 	}()
 	t.Setenv("STATIC_PATH", "/usr/share/nginx/html/") // TYPO
-	t.Setenv("SENSORTHINGS_URL", "https://tld.iot.hamburg.de/v1.1/")
+	t.Setenv("SENSORTHINGS_URL_THINGS", "https://tld.iot.hamburg.de/v1.1/")
+	t.Setenv("SENSORTHINGS_URL_OBSERVATIONS", "https://tld.iot.hamburg.de/v1.1/")
 	t.Setenv("SENSORTHINGS_MQTT_URL", "tcp://tld.iot.hamburg.de:1883")
 	t.Setenv("PREDICTION_MQTT_URL", "tcp://predictor-mosquitto:1883")
 	Init()
diff --git a/observations/sync.go b/observations/sync.go
index f020e66..823eab1 100644
--- a/observations/sync.go
+++ b/observations/sync.go
@@ -13,7 +13,7 @@ import (
 
 func prefetchMostRecentObservationsPage(page int) (more bool) {
 	elementsPerPage := 100
-	pageUrl := env.SensorThingsBaseUrl + "Datastreams?" + url.QueryEscape(
+	pageUrl := env.SensorThingsBaseUrlObservations + "Datastreams?" + url.QueryEscape(
 		"$filter="+
 			"properties/serviceName eq 'HH_STA_traffic_lights' "+
 			"and (properties/layerName eq 'signal_program') "+
diff --git a/observations/sync_test.go b/observations/sync_test.go
index f672fe1..88c09ba 100644
--- a/observations/sync_test.go
+++ b/observations/sync_test.go
@@ -38,7 +38,7 @@ func TestPrefetchMostRecentObservations(t *testing.T) {
 	}))
 	defer testServer.Close()
 
-	env.SensorThingsBaseUrl = fmt.Sprintf("%s/", testServer.URL)
+	env.SensorThingsBaseUrlObservations = fmt.Sprintf("%s/", testServer.URL)
 	PrefetchMostRecentObservations()
 
 	signalProgramCycles.Range(func(k, v interface{}) bool {
diff --git a/things/sync.go b/things/sync.go
index 6e95365..4990e57 100644
--- a/things/sync.go
+++ b/things/sync.go
@@ -47,7 +47,7 @@ var BikeDetectorDatastreams = &sync.Map{}
 
 func syncThingsPage(page int) (more bool) {
 	elementsPerPage := 100
-	pageUrl := env.SensorThingsBaseUrl + "Things?" + url.QueryEscape(
+	pageUrl := env.SensorThingsBaseUrlThings + "Things?" + url.QueryEscape(
 		"$filter="+
 			"Datastreams/properties/serviceName eq 'HH_STA_traffic_lights' "+
 			"and (Datastreams/properties/layerName eq 'primary_signal' "+