Update Reporter to output both UA and GA4 data in reports

.circleci/config.yml

@@ -26,7 +26,6 @@ jobs:
           name: run tests
           command: npm test
 
-
   develop_deploy:
     docker:
       - image: cimg/node:16.19.1
@@ -50,6 +49,11 @@ jobs:
               sudo curl -v -L -o cf8-cli-installer_8.7.4_x86-64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=8.7.4'
               sudo dpkg -i cf8-cli-installer_8.7.4_x86-64.deb
 
+      - run:
+          name: Write Google GA4 Credentails file from value in CircleCI env var.
+          command: |
+              echo $GA4_CREDS > ./my-analytics-ga4-65057af58daa.json
+
       - run:
           name: deploy
           command: |
@@ -83,6 +87,11 @@ jobs:
               sudo curl -v -L -o cf8-cli-installer_8.7.4_x86-64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=8.7.4'
               sudo dpkg -i cf8-cli-installer_8.7.4_x86-64.deb
 
+      - run:
+          name: Write Google GA4 Credentails file from value in CircleCI env var.
+          command: |
+              echo $GA4_CREDS > ./my-analytics-ga4-65057af58daa.json
+
       - run:
           name: deploy
           command: |

.mocharc.yml

@@ -0,0 +1,14 @@
+diff: true
+extension: ['js']
+package: './package.json'
+slow: '75'
+spec:
+  - 'test/**/*.js'
+  - 'ua/test/**/*.js'
+timeout: '2000'
+ui: 'bdd'
+watch-files:
+  - 'src/**/*.js'
+  - 'test/**/*.js'
+  - 'ua/src/**/*.js'
+  - 'ua/test/**/*.js'

.nvmrc

@@ -0,0 +1 @@
+20.11

README.md

@@ -33,7 +33,7 @@ alias analytics="docker run -t -v ${HOME}:${HOME} -e ANALYTICS_REPORT_EMAIL -e A
 To make this command working as expected you should export the env vars as follows:
 
 ```bash
-export ANALYTICS_REPORT_EMAIL=  "your-report-email"
+export ANALYTICS_REPORT_EMAIL="your-report-email"
 export ANALYTICS_REPORT_IDS="your-report-ids"
 export ANALYTICS_KEY="your-key"
 ```
@@ -180,47 +180,68 @@ A report might look something like this:
   "name": "devices",
   "query": {
     "dimensions": [
-      "ga:date",
-      "ga:deviceCategory"
+      {
+        "name": "date"
+      },
+      {
+        "name": "deviceCategory"
+      }
     ],
     "metrics": [
-      "ga:sessions"
+      {
+        "name": "sessions"
+      }
     ],
-    "start-date": "90daysAgo",
-    "end-date": "yesterday",
-    "sort": "ga:date"
+    "dateRanges": [
+      {
+        "startDate": "90daysAgo",
+        "endDate": "yesterday"
+      }
+    ],
+    "orderBys": [
+      {
+        "dimension": {
+          "dimensionName": "date"
+        },
+        "desc": true
+      }
+    ],
+    "samplingLevel": "HIGHER_PRECISION",
+    "limit": "10000",
+    "property": "properties/393249053"
   },
   "meta": {
     "name": "Devices",
-    "description": "Weekly desktop/mobile/tablet visits by day for all sites."
+    "description": "90 days of desktop/mobile/tablet visits for all sites."
   },
   "data": [
     {
-      "date": "2014-10-14",
-      "device": "desktop",
-      "visits": "11495462"
+      "date": "2023-12-25",
+      "device": "mobile",
+      "visits": "13681896"
     },
     {
-      "date": "2014-10-14",
-      "device": "mobile",
-      "visits": "2499586"
+      "date": "2023-12-25",
+      "device": "desktop",
+      "visits": "5775002"
     },
     {
-      "date": "2014-10-14",
+      "date": "2023-12-25",
       "device": "tablet",
-      "visits": "976396"
+      "visits": "367039"
     },
-    // ...
+   ...
   ],
   "totals": {
+    "visits": 3584551745,
     "devices": {
-      "mobile": 213920363,
-      "desktop": 755511646,
-      "tablet": 81874189
-    },
-    "start_date": "2014-10-14",
-    "end_date": "2015-01-11"
-  }
+      "mobile": 2012722956,
+      "desktop": 1513968883,
+      "tablet": 52313579,
+      "smart tv": 5546327
+    }
+  },
+  "taken_at": "2023-12-26T20:52:50.062Z"
 }
 ```
 
@@ -344,6 +365,27 @@ Compose:
 docker-compose up
 ```
 
+#### Running unit tests locally
+
+The unit tests require a postgres database to be running and accepting
+connections at 127.0.0.1:5432
+
+To run the test database locally with docker:
+
+```shell
+docker-compose -f docker-compose.test.yml up
+```
+
+The test scripts run database migrations and then the tests themselves.  These
+require database connection details to be provided in the shell environment:
+
+```shell
+POSTGRES_PASSWORD=123abc \
+POSTGRES_USER=analytics \
+POSTGRES_DATABASE=analytics_reporter_test \
+npm test
+```
+
 ### Public domain
 
 This project is in the worldwide [public domain](LICENSE.md). As stated in [CONTRIBUTING](CONTRIBUTING.md):

deploy/cron.js

@@ -1,116 +1,102 @@
-if (process.env.NODE_ENV !== 'production') {
-  require('dotenv').config()
+if (process.env.NODE_ENV !== "production") {
+  require("dotenv").config();
 }
 
 if (process.env.NEW_RELIC_APP_NAME) {
-	require("newrelic")
+  require("newrelic");
 }
 
 const spawn = require("child_process").spawn;
-const winston = require("winston")
-
-const logger = winston.createLogger({
-	level: 'debug',
-	format: winston.format.json(),
-	transports: [new winston.transports.Console({level: 'debug'})],
-});
+const logger = require('../src/logger').initialize();
 
 logger.info("===================================");
 logger.info("=== STARTING ANALYTICS-REPORTER ===");
 logger.info("    Running /deploy/cron.js");
 logger.info("===================================");
 
-const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`
-
-var api_run = function() {
-	logger.info("about to run api.sh");
-
-	var api = spawn(`${scriptRootPath}/api.sh`)
-	api.stdout.on("data", (data) => {
-		logger.info("[api.sh]", data.toString().trim())
-	})
-	api.stderr.on("data", (data) => {
-		logger.info("[api.sh]", data.toString().trim())
-	})
-	api.on("exit", (code) => {
-		logger.info("api.sh exitted with code:", code)
-	})
-}
+const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`;
+const scriptUARootPath = `${process.env.ANALYTICS_UA_ROOT_PATH}/deploy`;
 
-var daily_run = function() {
-	logger.info("about to run daily.sh");
-
-	var daily = spawn(`${scriptRootPath}/daily.sh`)
-	daily.stdout.on("data", (data) => {
-		logger.info("[daily.sh]", data.toString().trim())
-	})
-	daily.stderr.on("data", (data) => {
-		logger.info("[daily.sh]", data.toString().trim())
-	})
-	daily.on("exit", (code) => {
-		logger.info("daily.sh exitted with code:", code)
-	})
-}
+const runScriptWithLogName = (scriptPath, scriptLoggingName) => {
+  logger.info(`Beginning: ${scriptLoggingName}`);
+  logger.info(`File path: ${scriptPath}`);
+  const childProcess = spawn(scriptPath);
 
-var hourly_run = function(){
-	logger.info("about to run hourly.sh");
-
-	var hourly = spawn(`${scriptRootPath}/hourly.sh`)
-	hourly.stdout.on("data", (data) => {
-		logger.info("[hourly.sh]", data.toString().trim())
-	})
-	hourly.stderr.on("data", (data) => {
-		logger.info("[hourly.sh]", data.toString().trim())
-	})
-	hourly.on("exit", (code) => {
-		logger.info("hourly.sh exitted with code:", code)
-	})
-}
+  childProcess.stdout.on("data", (data) => {
+    logger.info(`[${scriptLoggingName}]`);
+    // Writes logging output from child processes to console.
+    console.log(data.toString().trim())
+  });
 
-var realtime_run = function(){
-	logger.info("about to run realtime.sh");
-
-	var realtime = spawn(`${scriptRootPath}/realtime.sh`)
-	realtime.stdout.on("data", (data) => {
-		logger.info("[realtime.sh]", data.toString().trim())
-	})
-	realtime.stderr.on("data", (data) => {
-		logger.info("[realtime.sh]", data.toString().trim())
-	})
-	realtime.on("exit", (code) => {
-		logger.info("realtime.sh exitted with code:", code)
-	})
+  childProcess.stderr.on("data", (data) => {
+    logger.error(`[${scriptLoggingName}]`);
+    // Writes error logging output from child processes to console.
+    console.log(data.toString().trim())
+  });
+
+  childProcess.on("exit", (code) => {
+    logger.info(`${scriptLoggingName} exitted with code: ${code}`);
+  });
 }
 
+const api_ua_run = () => {
+  runScriptWithLogName(`${scriptUARootPath}/api.sh`, 'ua - api.sh')
+};
+
+const api_run = () => {
+  runScriptWithLogName(`${scriptRootPath}/api.sh`, 'api.sh')
+};
+
+const daily_run = () => {
+  runScriptWithLogName(`${scriptRootPath}/daily.sh`, 'daily.sh')
+};
+
+const hourly_run = () => {
+  runScriptWithLogName(`${scriptRootPath}/hourly.sh`, 'hourly.sh')
+};
+
+const realtime_run = () => {
+  runScriptWithLogName(`${scriptRootPath}/realtime.sh`, 'realtime.sh')
+};
+
 /**
-	Daily reports run every morning at 10 AM UTC.
-	This calculates the offset between now and then for the next scheduled run.
+  Daily reports run every morning at 10 AM UTC.
+  This calculates the offset between now and then for the next scheduled run.
 */
-var calculateNextDailyRunTimeOffset = function(){
-	const currentTime = new Date();
-	const nextRunTime = new Date(
-		currentTime.getFullYear(),
-		currentTime.getMonth(),
-		currentTime.getDate() + 1,
-		10 - currentTime.getTimezoneOffset() / 60
-	);
-	return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24)
-}
+const calculateNextDailyRunTimeOffset = () => {
+  const currentTime = new Date();
+  const nextRunTime = new Date(
+    currentTime.getFullYear(),
+    currentTime.getMonth(),
+    currentTime.getDate() + 1,
+    10 - currentTime.getTimezoneOffset() / 60
+  );
+  return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24);
+};
 
-logger.info("starting cron.js!");
+/**
+ * All scripts run immediately upon application start, then run again at
+ * intervals going forward.
+ */
 api_run();
+api_ua_run();
 daily_run();
 hourly_run();
 realtime_run();
-//api
-setInterval(api_run,1000 * 60 * 60 * 24)
-//daily
+
+// daily
 setTimeout(() => {
-	// Run at 10 AM UTC, then every 24 hours afterwards
-	daily_run();
-	setInterval(daily_run, 1000 * 60 * 60 * 24);
+  // Run at 10 AM UTC, then every 24 hours afterwards
+  daily_run();
+  setInterval(daily_run, 1000 * 60 * 60 * 24);
+  //api
+  api_run();
+  setInterval(api_run, 1000 * 60 * 60 * 24);
+  //ua api
+  api_ua_run();
+  setInterval(api_ua_run, 1000 * 60 * 60 * 24);
 }, calculateNextDailyRunTimeOffset());
 //hourly
-setInterval(hourly_run,1000 * 60 * 60);
+setInterval(hourly_run, 1000 * 60 * 60);
 //realtime
-setInterval(realtime_run,1000 * 60 * 5);
+setInterval(realtime_run, 1000 * 60 * 5);

deploy/envs/agency-international-development.env

@@ -1,4 +1,5 @@
 # Agency for International Development
-export ANALYTICS_REPORT_IDS="ga:68380943"
+# USAID Agency
+export ANALYTICS_REPORT_IDS="395450427"
 export AGENCY_NAME=agency-international-development
 export AWS_BUCKET_PATH=data/agency-international-development

deploy/envs/agriculture.env

@@ -1,5 +1,6 @@
 # Department of Agriculture
-export ANALYTICS_REPORT_IDS="ga:65240995"
+# USDA Agency
+export ANALYTICS_REPORT_IDS="395461442"
 export AGENCY_NAME=agriculture
 export AWS_BUCKET_PATH=data/agriculture
 

deploy/envs/commerce.env

@@ -1,4 +1,5 @@
 # Department of Commerce
-export ANALYTICS_REPORT_IDS="ga:66877186"
+# DOC Agency
+export ANALYTICS_REPORT_IDS="395253935"
 export AGENCY_NAME=commerce
 export AWS_BUCKET_PATH=data/commerce

deploy/envs/defense.env

@@ -1,4 +1,5 @@
 # Department of Defense
-export ANALYTICS_REPORT_IDS="ga:67120289"
+# DOD Agency
+export ANALYTICS_REPORT_IDS="395251747"
 export AGENCY_NAME=defense
 export AWS_BUCKET_PATH=data/defense