AB#26925 Sync stores and layers to geoserver

docker-compose.override.yml

@@ -6,6 +6,7 @@ services:
     environment:
       - NODE_ENV=development
       - LOCAL_PORT_IBF_SERVICE=${LOCAL_PORT_IBF_SERVICE}
+      - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_ADMIN_PASSWORD}
     ports:
       - ${LOCAL_PORT_IBF_SERVICE}:3000
     depends_on:

services/API-service/migration/1710512991479-rename-mock-rasters.ts

@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export class RenameMockRasters1710512991479 implements MigrationInterface {
+  public async up(_queryRunner: QueryRunner): Promise<void> {
+    const directoryPath = './geoserver-volume/raster-files/mock-output/';
+    const ugandaFloodsBasicPath = `${directoryPath}/flood_extent_day_UGA.tif`;
+    if (fs.existsSync(ugandaFloodsBasicPath)) {
+      // Do not run the migration another time if it has already been run on test servers
+      return;
+    }
+
+    if (fs.existsSync(directoryPath)) {
+      const files = fs.readdirSync(directoryPath);
+      console.log('🚀 ~ RenameMockRasters1710512991479 ~ up ~ files:', files);
+
+      files.forEach((file) => {
+        if (!file.includes('hour_MWI')) {
+          const newFilename = file.replace(
+            /_[0-9]+-(hour|day|month)_/g,
+            '_$1_',
+          );
+          fs.renameSync(
+            path.join(directoryPath, file),
+            path.join(directoryPath, newFilename),
+          );
+        }
+      });
+    } else {
+      console.log(`Directory ${directoryPath} does not exist`);
+    }
+  }
+
+  public async down(_queryRunner: QueryRunner): Promise<void> {
+    // If you want to revert the renaming, you would need to implement the logic here
+  }
+}

services/API-service/package.json

@@ -19,7 +19,8 @@
     "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js",
     "migration:generate": "npm run typeorm migration:generate -- -d ./appdatasource.ts",
     "migration:run": "npm run typeorm migration:run -- -d ./appdatasource.ts",
-    "migration:revert": "npm run typeorm migration:revert -- -d ./appdatasource.ts"
+    "migration:revert": "npm run typeorm migration:revert -- -d ./appdatasource.ts",
+    "migration:create": "npm run typeorm migration:create -- "
   },
   "private": true,
   "dependencies": {

services/API-service/src/api/admin-area-dynamic-data/admin-area-dynamic-data.service.ts

@@ -15,6 +15,7 @@ import fs from 'fs';
 import { CountryEntity } from '../country/country.entity';
 import { HelperService } from '../../shared/helper.service';
 import { EventAreaService } from '../admin-area/services/event-area.service';
+import { DisasterTypeGeoServerMapper } from '../../scripts/disaster-type-geoserver-file.mapper';
 
 @Injectable()
 export class AdminAreaDynamicDataService {
@@ -231,16 +232,9 @@ export class AdminAreaDynamicDataService {
     data: any,
     disasterType: DisasterType,
   ): Promise<void> {
-    let subfolder: string;
-    if (
-      [DisasterType.Floods, DisasterType.FlashFloods].includes(disasterType)
-    ) {
-      subfolder = 'flood_extents';
-    } else if (
-      [DisasterType.HeavyRain, DisasterType.Drought].includes(disasterType)
-    ) {
-      subfolder = 'rainfall_extents';
-    } else {
+    const subfolder =
+      DisasterTypeGeoServerMapper.getSubfolderForDisasterType(disasterType);
+    if (subfolder === '') {
       throw new HttpException(
         'Disaster Type not allowed',
         HttpStatus.BAD_REQUEST,

services/API-service/src/config.ts

@@ -14,6 +14,8 @@ const rootUrl =
   process.env.NODE_ENV === 'development'
     ? `http://localhost:${process.env.LOCAL_PORT_IBF_SERVICE}/`
     : process.env.EXTERNAL_API_SERVICE_URL;
+export const INTERNAL_GEOSERVER_API_URL =
+  'http://ibf-geoserver:8080/geoserver/rest';
 export const EXTERNAL_API = {
   root: rootUrl,
   whatsAppStatus: baseApiUrl + API_PATHS.whatsAppStatus,

services/API-service/src/scripts/disaster-type-geoserver-file.mapper.ts

@@ -0,0 +1,73 @@
+import { DisasterType } from '../api/disaster/disaster-type.enum';
+
+export class DisasterTypeGeoServerMapper {
+  static getLayerStorePrefixForDisasterType(
+    disasterType: DisasterType
+  ): string {
+    if (disasterType === DisasterType.Floods) {
+      return 'flood_extent';
+    } else if (disasterType === DisasterType.HeavyRain) {
+      return 'rainfall_extent';
+    } else if (disasterType === DisasterType.Drought) {
+      return 'rainfall_forecast';
+    } else if (disasterType === DisasterType.FlashFloods) {
+      return 'flood_extent';
+    }
+    return '';
+  }
+
+  static getDestFilePrefixForDisasterType(
+    disasterType: DisasterType,
+    countryCode: string,
+  ): string {
+    if (disasterType === DisasterType.Floods) {
+      return 'flood_extent';
+    } else if (disasterType === DisasterType.HeavyRain) {
+      if (countryCode === 'EGY') {
+        return 'rain_rp';
+      } else if (countryCode === 'UGA') {
+        return 'rainfall_extent';
+      }
+    } else if (disasterType === DisasterType.Drought) {
+      return 'rain_rp';
+    } else if (disasterType === DisasterType.FlashFloods) {
+      return 'flood_extent';
+    }
+    return '';
+  }
+
+  static getSubfolderForDisasterType(disasterType: DisasterType): string {
+    let subfolder = '';
+    if (
+      [DisasterType.Floods, DisasterType.FlashFloods].includes(disasterType)
+    ) {
+      subfolder = 'flood_extents';
+    } else if (
+      [DisasterType.HeavyRain, DisasterType.Drought].includes(disasterType)
+    ) {
+      subfolder = 'rainfall_extents';
+    }
+    return subfolder;
+  }
+
+  // DOES not work for heavy rain as it will be phased out
+  static getStyleForCountryAndDisasterType(
+    countryCode: string,
+    disasterType: DisasterType,
+  ): string {
+    if (disasterType === DisasterType.Drought) {
+      return 'rainfall_extent_drought';
+    } else if (disasterType === DisasterType.FlashFloods) {
+      return 'flood_extent_MWI_flash-floods';
+    } else if (disasterType === DisasterType.Floods) {
+      if (countryCode === 'PHL') {
+        return 'flood_extent_PHL';
+      } else {
+        return 'flood_extent';
+      }
+    }
+    throw new Error(
+      `No style found for disaster type' ${disasterType} and country code ${countryCode}`,
+    );
+  }
+}

services/API-service/src/scripts/geoserver-sync.service.ts

@@ -0,0 +1,206 @@
+import { HttpService } from '@nestjs/axios';
+import { Injectable } from '@nestjs/common';
+import { firstValueFrom, map } from 'rxjs';
+import { INTERNAL_GEOSERVER_API_URL } from '../config';
+import countries from './json/countries.json';
+import { DisasterType } from '../api/disaster/disaster-type.enum';
+import { DisasterTypeGeoServerMapper } from './disaster-type-geoserver-file.mapper';
+import * as fs from 'fs';
+
+const workspaceName = 'ibf-system';
+
+class RecourceNameObject {
+  resourceName: string;
+  disasterType: DisasterType;
+  countryCodeISO3: string;
+}
+
+@Injectable()
+export class GeoseverSyncService {
+  constructor(private httpService: HttpService) {}
+
+  public async sync(
+    countryCodeISO3?: string,
+    disasterType?: DisasterType,
+  ): Promise<void> {
+    const filteredCountries = countries.filter((country: any) => {
+      return countryCodeISO3
+        ? country.countryCodeISO3 === countryCodeISO3
+        : true;
+    });
+    // also filter by disaster type
+    for (const country of filteredCountries) {
+      const disasterSettings = country.countryDisasterSettings.filter(
+        (disasterSetting: any) => {
+          return disasterType
+            ? disasterSetting.disasterType === disasterType
+            : true;
+        },
+      );
+      country.countryDisasterSettings = disasterSettings;
+    }
+    const geoserverResourceNameObjects =
+      this.generateGeoserverResourceNames(filteredCountries);
+    await this.syncStores(geoserverResourceNameObjects);
+    await this.syncLayers(geoserverResourceNameObjects);
+  }
+
+  private async syncStores(expectedStoreNameObjects: RecourceNameObject[]) {
+    const foundStoreNames = await this.getStoreNamesFromGeoserver(
+      workspaceName,
+    );
+    const missingStoreNames = expectedStoreNameObjects.filter(
+      (o) => !foundStoreNames.includes(o.resourceName),
+    );
+    await this.postStoreNamesToGeoserver(missingStoreNames);
+  }
+
+  private generateGeoserverResourceNames(
+    filteredCountries: any[],
+  ): RecourceNameObject[] {
+    const resourceNameObjects = [];
+    for (const country of filteredCountries) {
+      resourceNameObjects.push(...this.generateStoreNameForCountry(country));
+    }
+    return resourceNameObjects;
+  }
+
+  private generateStoreNameForCountry(country: any): RecourceNameObject[] {
+    const resourceNameObjects = [];
+    const countryCode = country.countryCodeISO3;
+    for (const disasterSetting of country.countryDisasterSettings) {
+      if (disasterSetting.disasterType == DisasterType.Floods) {
+        for (const leadTime of disasterSetting.activeLeadTimes) {
+          const disasterTypeStorePrefix =
+            DisasterTypeGeoServerMapper.getLayerStorePrefixForDisasterType(
+              disasterSetting.disasterType
+            );
+          const resourceName = `${disasterTypeStorePrefix}_${leadTime}_${countryCode}`;
+          resourceNameObjects.push({
+            resourceName: resourceName,
+            disasterType: disasterSetting.disasterType,
+            countryCodeISO3: countryCode,
+          });
+        }
+      }
+    }
+    return resourceNameObjects;
+  }
+
+  private async getStoreNamesFromGeoserver(workspaceName: string) {
+    const data = await this.get(`workspaces/${workspaceName}/coveragestores`);
+    const storeNames = data.coverageStores.coverageStore.map(
+      (store: any) => store.name,
+    );
+    return storeNames;
+  }
+
+  private async postStoreNamesToGeoserver(
+    resourceNameObjects: RecourceNameObject[],
+  ) {
+    for (const resourceNameObject of resourceNameObjects) {
+      const subfolder = DisasterTypeGeoServerMapper.getSubfolderForDisasterType(
+        resourceNameObject.disasterType,
+      );
+      const url = `workspaces/${workspaceName}/coveragestores`; // replace with the correct API endpoint
+      const body = {
+        coverageStore: {
+          name: resourceNameObject.resourceName,
+          workspace: workspaceName,
+          enabled: true,
+          type: 'GeoTIFF',
+          url: `file:workspaces/ibf-system/ibf-pipeline/output/${subfolder}/${resourceNameObject.resourceName}.tif`,
+        },
+      };
+      const result = await this.post(url, body);
+      console.log(
+        'Updated geoserver with ',
+        result,
+        'please commit the resulting config changes of geoserver to git.',
+      );
+    }
+  }
+
+  public async syncLayers(expectedLayerNames: RecourceNameObject[]) {
+    const foundLayerNames = await this.getLayerNamesFromGeoserver(
+      workspaceName,
+    );
+    const missingLayerNames = expectedLayerNames.filter(
+      (o) => !foundLayerNames.includes(o.resourceName),
+    );
+    await this.postLayerNamesToGeoserver(missingLayerNames);
+  }
+
+  private async getLayerNamesFromGeoserver(workspaceName: string) {
+    const data = await this.get(`workspaces/${workspaceName}/layers`);
+    const layerNames = data.layers.layer.map((layer: any) => layer.name);
+    return layerNames;
+  }
+
+  private async postLayerNamesToGeoserver(
+    resourceNameObjects: RecourceNameObject[],
+  ) {
+    for (const resourceNameObject of resourceNameObjects) {
+      const publishLayerUrl = `workspaces/${workspaceName}/coveragestores/${resourceNameObject.resourceName}/coverages`;
+      const publishLayerBody = {
+        coverage: {
+          name: resourceNameObject.resourceName,
+          title: resourceNameObject.resourceName,
+          nativeName: resourceNameObject.resourceName,
+        },
+      };
+      await this.post(publishLayerUrl, publishLayerBody);
+      // Set the default style for the layer
+      const styleName =
+        DisasterTypeGeoServerMapper.getStyleForCountryAndDisasterType(
+          resourceNameObject.countryCodeISO3,
+          resourceNameObject.disasterType,
+        );
+      const styleUrl = `layers/${resourceNameObject.resourceName}`;
+      const body = {
+        layer: {
+          defaultStyle: {
+            name: `${workspaceName}:${styleName}`,
+          },
+        },
+      };
+      await this.put(styleUrl, body);
+    }
+  }
+
+  private async post(path: string, body: any) {
+    const url = `${INTERNAL_GEOSERVER_API_URL}/${path}`;
+    const headers = this.getHeaders();
+    const result = await firstValueFrom(
+      this.httpService.post(url, body, { headers }),
+    );
+    return result.data;
+  }
+
+  private async put(path: string, body: any) {
+    const url = `${INTERNAL_GEOSERVER_API_URL}/${path}`;
+    const headers = this.getHeaders();
+    const result = await firstValueFrom(
+      this.httpService.put(url, body, { headers }),
+    );
+    return result.data;
+  }
+
+  private async get(path: string) {
+    const url = `${INTERNAL_GEOSERVER_API_URL}/${path}`;
+    const headers = this.getHeaders();
+    const result = await firstValueFrom(this.httpService.get(url, { headers }));
+    return result.data;
+  }
+
+  private getHeaders() {
+    const username = 'admin';
+    return {
+      Authorization:
+        'Basic ' +
+        Buffer.from(
+          username + ':' + process.env.GEOSERVER_ADMIN_PASSWORD,
+        ).toString('base64'),
+    };
+  }
+}

services/API-service/src/scripts/json/countries.json

@@ -9,7 +9,7 @@
         "disasterType": "floods",
         "adminLevels": [2, 3, 4],
         "defaultAdminLevel": 2,
-        "activeLeadTimes": ["5-day"],
+        "activeLeadTimes": ["1-day", "2-day", "3-day", "4-day", "5-day", "6-day", "7-day"],
         "eapLink": "https://docs.google.com/document/d/1z4KfTIF1aJKgx-te8gPY6Scr2FcYR51x/edit#bookmark=id.j5clfgl1ywrd",
         "eapAlertClasses": {
           "no": {