Merge pull request #310 from TWilkin/294.persistence-indexes

#111/#294/#297 Add health check and indexes to persistence

.github/scripts/version.sh

@@ -63,7 +63,7 @@ update_version() {
     increase_version $helmVersion "macro"
     helmVersion=$newVersion
     echo "Increasing helm chart to v$helmVersion"
-    set_chart_version $helmPath $powerpiVersion $helmVersion $subchartVersion
+    set_chart_version $helmPath $powerpiVersion $helmVersion $service $subchartVersion
 
     # commit the change
     echo "Committing version changes"
@@ -81,14 +81,15 @@ set_chart_version() {
     local path=$1
     local appVersion=$2
     local chartVersion=$3
-    local subchartVersion=$4
+    local service=$4
+    local subchartVersion=$5
 
     yq e -i ".appVersion = \"$appVersion\"" $path
     yq e -i ".version = \"$chartVersion\"" $path
 
     if [ ! -z $subchartVersion ]
     then
-        yq e -i "(.dependencies[] | select(.name == \"energy-monitor\").version) = \"$subchartVersion\"" $path
+        yq e -i "(.dependencies[] | select(.name == \"$service\").version) = \"$subchartVersion\"" $path
     fi
 
     git add $path

common/node/common-test/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@powerpi/common-test",
-    "version": "0.0.3",
+    "version": "0.0.4",
     "description": "PowerPi Common Typescript Testing library",
     "private": true,
     "license": "GPL-3.0-only",
@@ -28,6 +28,6 @@
         "typescript": "^4.7.4"
     },
     "dependencies": {
-        "@powerpi/common": "^0.0.11"
+        "@powerpi/common": "^0.0.13"
     }
 }

common/node/common/README.md

@@ -19,7 +19,16 @@ yarn build:common
 
 ## Testing
 
-There are currently no automated tests for this library.
+This library can be tested by executing the following commands.
+
+```bash
+# From the root of your PowerPi checkout
+# Download the dependencies
+yarn
+
+# Run the tests
+yarn test:common
+```
 
 ## Local Execution
 

common/node/common/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+    preset: "ts-jest",
+    testEnvironment: "node",
+};

common/node/common/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@powerpi/common",
-    "version": "0.0.12",
+    "version": "0.0.13",
     "description": "PowerPi Common Typescript library",
     "private": true,
     "license": "GPL-3.0-only",
@@ -13,10 +13,12 @@
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "scripts": {
-        "build": "tsc"
+        "build": "tsc && cp src/health.sh dist/",
+        "test": "jest"
     },
     "files": [
-        "dist/src"
+        "dist/src",
+        "dist/health.sh"
     ],
     "devDependencies": {
         "@types/dateformat": "^5.0.0",

common/node/common/src/health.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# the number of seconds the health check file can be since last modification without being unhealthy
+AGE=$1
+
+# the path to the health check file
+HEALTH_FILE=$2
+
+if [ -z $AGE ]
+then
+    AGE=12
+fi
+
+if [ -z $HEALTH_FILE ]
+then
+    HEALTH_FILE=/home/node/app/powerpi_health
+fi
+
+if [ ! -f $HEALTH_FILE ]
+then
+    # the file doesn't exist, that's unhealthy
+    exit 1
+fi
+
+NOW=$(date +%s)
+HEALTH=$(stat $HEALTH_FILE -c %Y)
+SINCE=$(expr $NOW - $HEALTH)
+
+if [ $SINCE -gt $AGE ]
+then
+    # the health file is older than AGE, that's unhealthy
+    exit 2
+fi
+
+# it's newer than AGE, that's healthy
+exit 0

common/node/common/src/services/FileService.ts

@@ -0,0 +1,20 @@
+import { open, readFile, utimes } from "fs/promises";
+import { Service } from "typedi";
+
+@Service()
+export default class FileService {
+    public async read(path: string) {
+        return await readFile(path);
+    }
+
+    public async touch(path: string) {
+        const now = new Date();
+
+        try {
+            await utimes(path, now, now);
+        } catch (e) {
+            const handle = await open(path, "a");
+            await handle.close();
+        }
+    }
+}

common/node/common/src/services/config.ts

@@ -1,6 +1,5 @@
 import fs from "fs";
 import Container, { Service } from "typedi";
-import util from "util";
 import {
     IDeviceConfigFile,
     IFloorplanConfigFile,
@@ -9,6 +8,7 @@ import {
 } from "../models/config";
 import { Device, IDevice } from "../models/device";
 import { ISensor, Sensor } from "../models/sensor";
+import FileService from "./FileService";
 import { IntervalParserService } from "./interval";
 
 export enum ConfigFileType {
@@ -19,16 +19,13 @@ export enum ConfigFileType {
     Users = "users",
 }
 
-// allow reading of files using await
-const readAsync = util.promisify(fs.readFile);
-
 @Service()
 export class ConfigService {
     protected interval: IntervalParserService;
 
     private configs: { [key in ConfigFileType]?: { data: object; checksum: string } };
 
-    constructor() {
+    constructor(protected readonly fs: FileService) {
         this.interval = Container.get(IntervalParserService);
 
         this.configs = {};
@@ -76,6 +73,10 @@ export class ConfigService {
         });
     }
 
+    get healthCheckFile() {
+        return process.env["HEALTH_CHECK_FILE"] ?? "/home/node/app/powerpi_health";
+    }
+
     get configWaitTime() {
         const str = process.env["CONFIG_WAIT_TIME"];
         if (str) {
@@ -151,7 +152,7 @@ export class ConfigService {
     }
 
     protected async readFile(filePath: string) {
-        return (await readAsync(filePath)).toString().trim();
+        return (await this.fs.read(filePath)).toString().trim();
     }
 
     private get databaseHost() {

common/node/common/src/services/index.ts

@@ -1,4 +1,5 @@
 export * from "./config";
+export { default as FileService } from "./FileService";
 export * from "./interval";
 export * from "./logger";
 export * from "./mqtt";

common/node/common/test/services/FileService.test.ts

@@ -0,0 +1,29 @@
+import fs from "fs";
+import path from "path";
+import { FileService } from "../../src";
+
+describe("FileService", () => {
+    let subject: FileService;
+
+    beforeEach(() => (subject = new FileService()));
+
+    test("read", async () => {
+        const result = await subject.read(__filename);
+
+        expect(result).toBeDefined();
+
+        const str = new TextDecoder().decode(result);
+        expect(str).toContain('test("read",');
+    });
+
+    test("touch", async () => {
+        const tmpDir = fs.mkdtempSync(__filename);
+        const file = path.join(tmpDir, "touchy");
+
+        expect(fs.existsSync(file)).toBeFalsy();
+
+        await subject.touch(file);
+
+        expect(fs.existsSync(file)).toBeTruthy();
+    });
+});

kubernetes/Chart.yaml

@@ -2,70 +2,54 @@ apiVersion: v2
 name: powerpi
 description: A Helm chart for deploying the PowerPi home automation stack
 type: application
-version: 0.1.6
-appVersion: 0.1.4
+version: 0.1.7
+appVersion: 0.1.5
 dependencies:
   # services
   - name: babel-fish
     version: 0.1.0
     condition: global.voiceAssistant
-
   - name: clacks-config
     version: 0.1.0
     condition: global.config
-
   - name: database
-    version: 0.1.2
+    version: 0.1.3
     condition: global.persistence
-
   - name: deep-thought
     version: 0.1.1
-
   - name: energy-monitor
     version: 0.1.1
     condition: global.energy-monitor
-
   - name: freedns
     version: 0.1.0
     condition: global.freeDNS
-
   - name: light-fantastic
     version: 0.1.0
     condition: global.scheduler
-
   - name: mosquitto
     version: 0.1.2
-
   - name: persistence
-    version: 0.1.0
+    version: 0.1.1
     condition: global.persistence
-
   - name: smarter-device-manager
     version: 0.1.1
-
   - name: ui
     version: 0.1.2
-
   # controllers
   - name: energenie-controller
     version: 0.1.1
     condition: global.energenie
-
   - name: harmony-controller
     version: 0.1.1
     condition: global.harmony
-
   - name: lifx-controller
     version: 0.1.1
     condition: global.lifx
-
   - name: macro-controller
     version: 0.1.1
-
   - name: node-controller
     version: 0.1.1
     condition: global.node
-
   - name: zigbee-controller
     version: 0.1.1
     condition: global.zigbee

kubernetes/charts/database/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: database
 description: A Helm chart for deploying the Postgres database for PowerPi
 type: application
-version: 0.1.2
+version: 0.1.3
 appVersion: 15.2-alpine3.17

kubernetes/charts/database/templates/backup-job.yaml

@@ -7,7 +7,7 @@
 }}
 
 {{- $data := dict
-  "Name" "database-backup-job"
+  "Name" "database-backup"
   "Image" "postgres"
   "RequestMemory" "50Mi"
   "NodeSelector" $nodeSelector

kubernetes/charts/persistence/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: persistence
 description: A Helm chart for the PowerPi database message persistence service
 type: application
-version: 0.1.0
-appVersion: 0.1.5
+version: 0.1.1
+appVersion: 0.2.0

kubernetes/charts/persistence/templates/deployment.yaml

@@ -22,5 +22,13 @@
       )
     )
   )
+  "Probe" (dict
+    "Command" (list
+      "/bin/sh"
+      "/home/node/app/node_modules/@powerpi/common/dist/health.sh"
+    )
+    "ReadinessInitialDelay" 10
+    "LivenessInitialDelay" 20
+  )
 }}
 {{- include "powerpi.deployment" (merge (dict "Params" $data) . )}}

package.json

@@ -52,7 +52,9 @@
         "start:persistence": "yarn workspace @powerpi/persistence start:dev",
         "start:ui": "yarn workspace @powerpi/ui start",
         "test:babel-fish": "yarn workspace @powerpi/babel-fish test",
+        "test:common": "yarn workspace @powerpi/common test",
         "test:clacks-config": "yarn workspace @powerpi/clacks-config test",
+        "test:persistence": "yarn workspace @powerpi/persistence test",
         "test:ui": "yarn workspace @powerpi/ui test"
     },
     "devDependencies": {

services/babel-fish/package.json

@@ -27,14 +27,14 @@
         "@jovotech/platform-core": "^4.2.22",
         "@jovotech/server-express": "^4.0.0",
         "@powerpi/api": "^0.0.17",
-        "@powerpi/common": "^0.0.12",
+        "@powerpi/common": "^0.0.13",
         "source-map-support": "^0.5.19"
     },
     "devDependencies": {
         "@jovotech/cli-core": "^4.0.0",
         "@jovotech/db-filedb": "^4.0.0",
         "@jovotech/filebuilder": "^0.0.1",
-        "@powerpi/common-test": "^0.0.3",
+        "@powerpi/common-test": "^0.0.4",
         "@types/express": "^4.17.11",
         "ts-node": "^10.9.1",
         "tsc-watch": "^4.2.9"

services/clacks-config/README.md

@@ -201,7 +201,7 @@ yarn
 yarn build:common
 yarn build:common-test
 
-# Run the service locally
+# Run the tests
 yarn test:clacks-config
 ```