Merge pull request #57 from tago-io/TCORE-237_Postgres_Plugin

TCORE-237 Refactor and Test Postgres Plugin for Monorepo

package.json

@@ -38,7 +38,8 @@
     "uuid": "10.0.0",
     "zod": "3.23.8",
     "systeminformation": "5.23.5",
-    "express": "4.21.1"
+    "express": "4.21.1",
+    "qs": "6.10.1"
   },
   "devDependencies": {
     "@biomejs/biome": "1.9.3",
@@ -50,6 +51,7 @@
     "@types/semver": "7.5.8",
     "@types/uuid": "10.0.0",
     "@types/express": "5.0.0",
+    "@types/axios": "0.14.0",
     "concurrently": "9.0.1",
     "esbuild": "0.24.0",
     "jest": "29.7.0",

packages/console/package.json

@@ -17,7 +17,6 @@
     "mobx-react": "7.3.0",
     "multer": "1.4.5-lts.1",
     "polished": "4.1.3",
-    "qs": "6.10.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-helmet": "6.1.0",
@@ -35,7 +34,6 @@
     "@testing-library/react": "12.0.0",
     "@types/lodash.clonedeep": "4.5.6",
     "@types/multer": "1.4.7",
-    "@types/qs": "6.9.7",
     "@types/react": "17.0.14",
     "@types/react-dom": "17.0.9",
     "@types/react-helmet": "6.1.4",

packages/sdk/src/Types/Plugin/Plugin.types.ts

@@ -34,6 +34,7 @@ export const zPluginType = z.enum([
   "service",
   "sidebar-button",
   "sidebar-footer-button",
+  "sqlite"
 ]);
 
 /**

packages/server/Services/Plugins.ts

@@ -156,7 +156,7 @@ async function getInstalledInsidePlugins(
     );
     const isInstalledDatabasePlugin =
       settings.database_plugin?.split(":")[0] === md5(getPackage.name);
-    const isDatabase = getPackage?.tcore?.types?.includes("database");
+    const isDatabase = getPackage?.tcore?.types?.includes("sqlite");
     const isStore = getPackage?.tcore?.store;
 
     if (

plugins/freezer-simulator/package.json

@@ -4,9 +4,6 @@
   "private": false,
   "main": "./src/index.ts",
   "type": "module",
-  "engines": {
-    "tcore": ">= 0.7.0 <= 1.0.0"
-  },
   "tcore": {
     "name": "Freezer Simulator",
     "short_description": "Ice cream freezer that simulates internal temperature, door and compressor status",

plugins/network-chirpstack/package.json

@@ -4,9 +4,6 @@
   "private": false,
   "main": "./src/index.ts",
   "type": "module",
-  "engines": {
-    "tcore": ">= 0.7.0 < 1.0.0"
-  },
   "tcore": {
     "name": "Chirpstack LoRaWAN",
     "short_description": "Adds support for Chirpstack LoRaWAN",

standalone-plugins/postgres/docs/database_model.md → plugins/postgres/docs/database_model.md

@@ -101,6 +101,7 @@ erDiagram
     JSON metadata
     TIMESTAMP time
     TIMESTAMP created_at
+    VARCHAR_100 serie
   }
 
 ```

plugins/postgres/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@tago-io/tcore-plugin-postgresql",
+  "version": "0.7.0",
+  "main": "./src/index.ts",
+  "type": "module",
+  "tcore": {
+    "name": "PostgreSQL",
+    "short_description": "Integrates the PostgreSQL Database with TagoCore",
+    "full_description": "./README.md",
+    "icon": "./assets/icon.png",
+    "cluster": true,
+    "types": ["database"],
+    "permissions": ["action", "analysis", "device-data", "device"]
+  },
+  "scripts": {
+    "migrate": "knex migrate:make --client=pg --migrations-directory=src/Migrations"
+  },
+  "dependencies": {
+    "knex": "3.1.0",
+    "pg": "8.13.0",
+    "pg-native": "3.2.0",
+    "pg-query-stream": "4.7.0"
+  },
+  "devDependencies": {
+    "@types/pg": "8.11.10"
+  }
+}

standalone-plugins/postgres/src/Database/index.ts → plugins/postgres/src/Database/index.ts

@@ -1,10 +1,27 @@
-import path from "path";
-import fs from "fs";
-import knex, { Knex } from "knex";
-import { DatabaseModule } from "@tago-io/tcore-sdk";
-import PostgreSQLDialect from "knex/lib/dialects/postgres";
-import { Config } from "../types";
-import("./typeParser");
+import fs from "node:fs";
+import path from "node:path";
+import type { DatabaseModule } from "@tago-io/tcore-sdk";
+import knex, { type Knex } from "knex";
+import type { Config } from "../types.ts";
+import pg from "pg";
+import { fileURLToPath } from 'node:url';
+import { dirname } from 'node:path';
+
+const __filename = fileURLToPath(import.meta.url);
+const dirname__ = dirname(__filename);
+
+const types = pg.types;
+// @ts-ignore
+const { arrayParser } = types;
+
+const int8Parser = (val) => Number.parseInt(val, 10);
+
+const _INT8 = 1016;
+
+// ? Int8 in node is string, this workaround make it as Number.
+types.setTypeParser(types.builtins.INT8, int8Parser);
+// @ts-ignore
+types.setTypeParser(_INT8, (array) => arrayParser.create(array, int8Parser).parse());
 
 export interface IDatabaseConnection {
   read: Knex;
@@ -17,7 +34,7 @@ const deviceDB = {} as IDatabaseConnection;
 
 const migrationConfig: Knex.MigratorConfig = {
   tableName: "migrations",
-  directory: path.join(__dirname, "Migrations"),
+  directory: path.join(dirname__, "..", "Migrations"),
   // ! FIX Error: The migration directory is corrupt, the following files are missing:
   disableMigrationsListValidation: true,
 };
@@ -29,14 +46,15 @@ interface IConnectionOptions {
 
 function createConnection(
   connection: Knex.PgConnectionConfig,
-  options?: IConnectionOptions
+  options?: IConnectionOptions,
 ): Knex {
   return knex({
-    client: PostgreSQLDialect,
+    client: "pg",
     connection: {
       charset: "utf8",
       timezone: "utc",
       ...connection,
+      types,
     },
     pool: {
       max: options?.pool?.max_pool,
@@ -49,7 +67,7 @@ function createConnection(
 /**
  * Sets up the knex connection.
  */
-export async function setupKnex(this: DatabaseModule, config: Config) {
+async function setupKnex(this: DatabaseModule, config: Config) {
   config.device_schema = "tcore_device";
   config.main_schema = "tcore";
 
@@ -184,7 +202,7 @@ function setupConnections(config: Config) {
 /**
  * Destroys the knex connection.
  */
-export async function destroyKnex(this: DatabaseModule) {
+async function destroyKnex(this: DatabaseModule) {
   try {
     this.showMessage("info", "Closing connections");
 
@@ -211,7 +229,6 @@ async function testDatabaseConnection() {
 
     await mainDB?.read?.raw("SELECT 1");
     await mainDB?.write?.raw("SELECT 1");
-
     await deviceDB?.read?.raw("SELECT 1");
     await deviceDB?.write?.raw("SELECT 1");
 
@@ -220,19 +237,19 @@ async function testDatabaseConnection() {
     if (typeof e?.code === "string") {
       switch (e.code as string) {
         case "ENOTFOUND":
-          throw new Error(`Could not connect to database`);
+          throw new Error("Could not connect to database");
         case "ECONNREFUSED":
           throw new Error(
-            `Connection to ${e.address}:${e.port} refused. Check that the hostname and port are correct and that the postmaster is accepting TCP/IP connections`
+            `Connection to ${e.address}:${e.port} refused. Check that the hostname and port are correct and that the postmaster is accepting TCP/IP connections`,
           );
         case "28P01":
-          throw new Error(`Authorization denied`);
+          throw new Error("Authorization denied");
         case "3D000":
-          throw new Error(`Database does not exist`);
+          throw new Error("Database does not exist");
       }
     }
     throw e;
   }
 }
 
-export { mainDB, deviceDB };
+export { mainDB, deviceDB, setupKnex, destroyKnex };

standalone-plugins/postgres/src/Helpers/DeviceDatabase.ts → plugins/postgres/src/Helpers/DeviceDatabase.ts

@@ -1,6 +1,6 @@
-import { TDeviceType, TGenericID } from "@tago-io/tcore-sdk/types";
-import { Knex } from "knex";
-import { deviceDB } from "../Database";
+import type { TDeviceType, TGenericID } from "@tago-io/tcore-sdk/types";
+import type { Knex } from "knex";
+import { deviceDB } from "../Database/index.ts";
 
 /**
  * Creates a new connection. This will check if the file exists and
@@ -57,7 +57,7 @@ type Tables = "data";
 
 function getConnection(client: Knex, id: string) {
   return (table: Tables) => {
-    if (table == "data") {
+    if (table === "data") {
       return client(id);
     }
   };

standalone-plugins/postgres/src/Helpers/PluginConnection.ts → plugins/postgres/src/Helpers/PluginConnection.ts

@@ -1,5 +1,5 @@
-import { Knex } from "knex";
-import { mainDB } from "../Database";
+import type { Knex } from "knex";
+import { mainDB } from "../Database/index.ts";
 
 const schemaName = "tcore_plugin_storage" as const;
 

standalone-plugins/postgres/src/Helpers/applyTagFilter.ts → plugins/postgres/src/Helpers/applyTagFilter.ts

@@ -1,4 +1,4 @@
-import { mainDB } from "../Database";
+import { mainDB } from "../Database/index.ts";
 
 /**
  * Applies the `tags` filter.
@@ -22,31 +22,31 @@ function applyTagFilter(query: any, filter: any, resourceName: string) {
             `"${mainDB.read
               .raw(":value", { value: item.key.replace(/\*/g, "%") })
               .toString()
-              .slice(1, -1)}"`
+              .slice(1, -1)}"`,
           ),
           value: mainDB.read.raw(
             `"${mainDB.read
               .raw(":value", { value: item.value.replace(/\*/g, "%") })
               .toString()
-              .slice(1, -1)}"`
+              .slice(1, -1)}"`,
           ),
-        }
+        },
       );
     } else if (item.key) {
       query.whereRaw(
         `replace(:resourceName:.tags::text, '\\', '') ilike '%"key": :key%'`,
         {
           resourceName,
           key: item.key,
-        }
+        },
       );
     } else if (item.value) {
       query.whereRaw(
         `replace(:resourceName:.tags::text, '\\', '') ilike '%"value": :value%'`,
         {
           resourceName,
           value: item.value,
-        }
+        },
       );
     }
   }

standalone-plugins/postgres/src/Helpers/replaceFilterWildcards.ts → plugins/postgres/src/Helpers/replaceFilterWildcards.ts

@@ -1,7 +1,7 @@
-const asterisks = new RegExp("\\*", "g");
-const underlines = new RegExp("\\_", "g");
-const percents = new RegExp("\\%", "g");
-const duplePercents = new RegExp("\\%\\%", "g");
+const asterisks = /\*/g;
+const underlines = /\_/g;
+const percents = /\%/g;
+const duplePercents = /\%\%/g;
 
 /**
  * Replaces the filter wildcards into a format understood by SQLite.

standalone-plugins/postgres/src/Migrations/20220224145409_0.3.0.ts → plugins/postgres/src/Migrations/20220224145409_0.3.0.ts

@@ -1,4 +1,4 @@
-import { Knex } from "knex";
+import type { Knex } from "knex";
 
 /**
  * Bumps up to version 0.0.1 of tagocore.

standalone-plugins/postgres/src/Migrations/20220407173306_0.3.3.ts → plugins/postgres/src/Migrations/20220407173306_0.3.3.ts

@@ -1,4 +1,4 @@
-import { Knex } from "knex";
+import type { Knex } from "knex";
 
 /**
  * Bumps up to version 0.3.3.

standalone-plugins/postgres/src/Migrations/20220526170216_0.4.1.ts → plugins/postgres/src/Migrations/20220526170216_0.4.1.ts

@@ -1,4 +1,4 @@
-import { Knex } from "knex";
+import type { Knex } from "knex";
 
 /**
  * Bumps up to version 0.4.1.

standalone-plugins/postgres/src/Migrations/20220617162847_0.5.0.ts → plugins/postgres/src/Migrations/20220617162847_0.5.0.ts

@@ -1,4 +1,4 @@
-import { Knex } from "knex";
+import type { Knex } from "knex";
 
 /**
  * Bumps up to version 0.5.0.

standalone-plugins/postgres/src/Migrations/20220725164457_0.6.0.ts → plugins/postgres/src/Migrations/20220725164457_0.6.0.ts

@@ -1,4 +1,4 @@
-import { Knex } from "knex";
+import type { Knex } from "knex";
 
 /**
  * Bumps up to version 0.6.0.

standalone-plugins/postgres/src/Providers/Account/createAccount.ts → plugins/postgres/src/Providers/Account/createAccount.ts

@@ -1,5 +1,5 @@
-import { IDatabaseCreateAccountData } from "@tago-io/tcore-sdk/build/Types";
-import { mainDB } from "../../Database";
+import type { IDatabaseCreateAccountData } from "@tago-io/tcore-sdk/types";
+import { mainDB } from "../../Database/index.ts";
 
 /**
  * Creates a new account.

standalone-plugins/postgres/src/Providers/Account/createAccountToken.ts → plugins/postgres/src/Providers/Account/createAccountToken.ts

@@ -1,15 +1,15 @@
-import {
+import type {
   IDatabaseAccountCreateTokenData,
   TGenericID,
 } from "@tago-io/tcore-sdk/types";
-import { mainDB } from "../../Database";
+import { mainDB } from "../../Database/index.ts";
 
 /**
  * Generates and retrieves a new account token.
  */
 async function createAccountToken(
   accountID: TGenericID,
-  data: IDatabaseAccountCreateTokenData
+  data: IDatabaseAccountCreateTokenData,
 ): Promise<void> {
   const object = { ...data, account_id: accountID };
   await mainDB.write.insert(object).into("account_token");

standalone-plugins/postgres/src/Providers/Account/getAccountAmount.ts → plugins/postgres/src/Providers/Account/getAccountAmount.ts

@@ -1,4 +1,4 @@
-import { mainDB } from "../../Database";
+import { mainDB } from "../../Database/index.ts";
 
 /**
  * Retrieves the amount of accounts registered.

standalone-plugins/postgres/src/Providers/Account/getAccountByToken.ts → plugins/postgres/src/Providers/Account/getAccountByToken.ts

@@ -1,6 +1,6 @@
-import { IAccount } from "@tago-io/tcore-sdk/types";
-import { mainDB } from "../../Database";
-import getAccountInfo from "./getAccountInfo";
+import type { IAccount } from "@tago-io/tcore-sdk/types";
+import { mainDB } from "../../Database/index.ts";
+import getAccountInfo from "./getAccountInfo.ts";
 
 /**
  * Retrieves all the information of a single account via its token.

standalone-plugins/postgres/src/Providers/Account/getAccountByUsername.ts → plugins/postgres/src/Providers/Account/getAccountByUsername.ts

@@ -1,11 +1,11 @@
-import { IAccount } from "@tago-io/tcore-sdk/types";
-import { mainDB } from "../../Database";
+import type { IAccount } from "@tago-io/tcore-sdk/types";
+import { mainDB } from "../../Database/index.ts";
 
 /**
  * Retrieves all the information of a single account via its username.
  */
 async function getAccountByUsername(
-  username: string
+  username: string,
 ): Promise<IAccount | null> {
   const response = await mainDB.read
     .select("*")

standalone-plugins/postgres/src/Providers/Account/getAccountInfo.ts → plugins/postgres/src/Providers/Account/getAccountInfo.ts

@@ -1,5 +1,5 @@
-import { IAccount, TGenericID } from "@tago-io/tcore-sdk/types";
-import { mainDB } from "../../Database";
+import type { IAccount, TGenericID } from "@tago-io/tcore-sdk/types";
+import { mainDB } from "../../Database/index.ts";
 
 /**
  * Retrieves all the information of a single account.