From beb480dd4023df8e3421148539c410b569cc37e6 Mon Sep 17 00:00:00 2001 From: jannisvisser Date: Fri, 28 May 2021 12:45:29 +0200 Subject: [PATCH] refactor: move get-admin-regions from sql to typeorm AB#7940 --- database/init.sql | 1 - .../src/app/services/aggregates.service.ts | 42 ++++-- .../src/app/services/api.service.ts | 15 +- .../api/admin-area/admin-area.controller.ts | 36 +++-- .../src/api/admin-area/admin-area.service.ts | 135 ++++++++++++++---- .../api/admin-area/sql/get-admin-regions.sql | 44 ------ services/API-service/src/shared/data.model.ts | 7 +- .../pipeline/lib/pipeline/forecast.py | 4 +- 8 files changed, 181 insertions(+), 103 deletions(-) delete mode 100644 services/API-service/src/api/admin-area/sql/get-admin-regions.sql diff --git a/database/init.sql b/database/init.sql index 6ce50620df..2b8af2df9b 100644 --- a/database/init.sql +++ b/database/init.sql @@ -1,3 +1,2 @@ -CREATE SCHEMA IF NOT EXISTS "IBF-API"; CREATE SCHEMA IF NOT EXISTS "IBF-app"; CREATE EXTENSION "uuid-ossp"; diff --git a/interfaces/IBF-dashboard/src/app/services/aggregates.service.ts b/interfaces/IBF-dashboard/src/app/services/aggregates.service.ts index c987ada2aa..85e6affb4e 100644 --- a/interfaces/IBF-dashboard/src/app/services/aggregates.service.ts +++ b/interfaces/IBF-dashboard/src/app/services/aggregates.service.ts @@ -82,20 +82,20 @@ export class AggregatesService { indicator: Indicator, ) => { if (indicator.aggregateIndicator) { - if (indicator.name in feature.properties) { - aggregate[indicator.name] = feature.properties[indicator.name]; - } else if (indicator.name in feature.properties.indicators) { - aggregate[indicator.name] = - feature.properties.indicators[indicator.name]; + const foundIndicator = feature.records.find( + (a) => a.indicator === indicator.name, + ); + if (foundIndicator) { + aggregate[indicator.name] = foundIndicator.value; } else { aggregate[indicator.name] = 0; } } }; - private onEachAdminFeature = (feature) => { + private onEachPlaceCode = (feature) => { const aggregate = { - placeCode: feature.properties.placeCode, + placeCode: feature.placeCode, }; this.indicators.forEach( @@ -105,19 +105,39 @@ export class AggregatesService { return aggregate; }; - private onAdminRegions = (adminRegions) => { - this.aggregates = adminRegions.features.map(this.onEachAdminFeature); + private onAggregatesData = (records) => { + const groupsByPlaceCode = this.aggregateOnPlaceCode(records); + this.aggregates = groupsByPlaceCode.map(this.onEachPlaceCode); }; + private aggregateOnPlaceCode(array) { + const groupsByPlaceCode = []; + array.forEach((record) => { + if ( + groupsByPlaceCode.map((i) => i.placeCode).includes(record.placeCode) + ) { + groupsByPlaceCode + .find((i) => i.placeCode === record.placeCode) + .records.push(record); + } else { + groupsByPlaceCode.push({ + placeCode: record.placeCode, + records: [record], + }); + } + }); + return groupsByPlaceCode; + } + loadAggregateInformation(): void { if (this.country) { this.apiService - .getAdminRegions( + .getAggregatesData( this.country.countryCodeISO3, this.timelineService.activeLeadTime, this.adminLevelService.adminLevel, ) - .subscribe(this.onAdminRegions); + .subscribe(this.onAggregatesData); } } diff --git a/interfaces/IBF-dashboard/src/app/services/api.service.ts b/interfaces/IBF-dashboard/src/app/services/api.service.ts index 84410541d2..b0ae8a09c2 100644 --- a/interfaces/IBF-dashboard/src/app/services/api.service.ts +++ b/interfaces/IBF-dashboard/src/app/services/api.service.ts @@ -160,13 +160,26 @@ export class ApiService { adminLevel: AdminLevel = AdminLevel.adm1, ): Observable { return this.get( - `admin-areas/per-leadtime/${countryCodeISO3}/${adminLevel}/${ + `admin-areas/${countryCodeISO3}/${adminLevel}/${ leadTime ? leadTime : '' }`, false, ); } + getAggregatesData( + countryCodeISO3: string, + leadTime: string, + adminLevel: AdminLevel = AdminLevel.adm1, + ): Observable { + return this.get( + `admin-areas/aggregates/${countryCodeISO3}/${adminLevel}/${ + leadTime ? leadTime : '{leadTime}' + }`, + false, + ); + } + getTriggeredAreas(countryCodeISO3: string) { return this.get(`event/triggered-areas/${countryCodeISO3}`, false); } diff --git a/services/API-service/src/api/admin-area/admin-area.controller.ts b/services/API-service/src/api/admin-area/admin-area.controller.ts index c595581b88..353b8eaf66 100644 --- a/services/API-service/src/api/admin-area/admin-area.controller.ts +++ b/services/API-service/src/api/admin-area/admin-area.controller.ts @@ -8,6 +8,7 @@ import { import { GeoJson } from '../../shared/geo.model'; import { RolesGuard } from '../../roles.guard'; import { AdminAreaService } from './admin-area.service'; +import { AggregateDataRecord } from 'src/shared/data.model'; @ApiBearerAuth() @UseGuards(RolesGuard) @@ -20,37 +21,42 @@ export class AdminAreaController { this.adminAreaService = adminAreaService; } - // NOTE: this endpoint is to be used by the IBF-pipeline to read this data from DB (instead of current way > TO DO) @ApiOperation({ - summary: 'Get admin-areas by country', + summary: 'Get admin-areas by country raw', }) @ApiParam({ name: 'countryCodeISO3', required: true, type: 'string' }) - @Get(':countryCodeISO3') - public async getAdminAreas(@Param() params): Promise { - return await this.adminAreaService.getAdminAreas(params.countryCodeISO3); + @Get('raw/:countryCodeISO3') + public async getAdminAreasRaw(@Param() params): Promise { + return await this.adminAreaService.getAdminAreasRaw(params.countryCodeISO3); } - @ApiOperation({ summary: 'Get admin-area by leadTime' }) + @ApiOperation({ + summary: 'Get admin-areas by country as geojson for dashboard', + }) @ApiParam({ name: 'countryCodeISO3', required: true, type: 'string' }) @ApiParam({ name: 'leadTime', required: false, type: 'string' }) @ApiParam({ name: 'adminLevel', required: true, type: 'number' }) - @Get('per-leadtime/:countryCodeISO3/:adminLevel/:leadTime?') - public async getAdminAreaData(@Param() params): Promise { - return await this.adminAreaService.getAdminAreasPerLeadTime( + @Get(':countryCodeISO3/:adminLevel/:leadTime?') + public async getAdminAreas(@Param() params): Promise { + return await this.adminAreaService.getAdminAreas( params.countryCodeISO3, params.leadTime, params.adminLevel, ); } - @ApiOperation({ - summary: 'Get Glofas station to admin-area mapping by country', - }) + @ApiOperation({ summary: 'Get admin-area by leadTime' }) @ApiParam({ name: 'countryCodeISO3', required: true, type: 'string' }) - @Get('station-mapping/:countryCodeISO3') - public async getStationMapping(@Param() params): Promise { - return await this.adminAreaService.getStationAdminAreaMappingByCountry( + @ApiParam({ name: 'leadTime', required: false, type: 'string' }) + @ApiParam({ name: 'adminLevel', required: true, type: 'number' }) + @Get('aggregates/:countryCodeISO3/:adminLevel/:leadTime?') + public async getAggregatesData( + @Param() params, + ): Promise { + return await this.adminAreaService.getAggregatesData( params.countryCodeISO3, + params.leadTime, + params.adminLevel, ); } } diff --git a/services/API-service/src/api/admin-area/admin-area.service.ts b/services/API-service/src/api/admin-area/admin-area.service.ts index 12a1817fcd..1cde71a3f2 100644 --- a/services/API-service/src/api/admin-area/admin-area.service.ts +++ b/services/API-service/src/api/admin-area/admin-area.service.ts @@ -3,12 +3,13 @@ import { InjectRepository } from '@nestjs/typeorm'; import { GeoJson } from '../../shared/geo.model'; import { HelperService } from '../../shared/helper.service'; import { EntityManager, Repository } from 'typeorm'; -import fs from 'fs'; import { LeadTime } from '../admin-area-dynamic-data/enum/lead-time.enum'; import { AdminAreaEntity } from './admin-area.entity'; import { CountryEntity } from '../country/country.entity'; import { EventService } from '../event/event.service'; -import { AdminAreaRecord } from 'src/shared/data.model'; +import { AggregateDataRecord } from 'src/shared/data.model'; +import { AdminAreaDynamicDataEntity } from '../admin-area-dynamic-data/admin-area-dynamic-data.entity'; +import { AdminAreaDataEntity } from '../admin-area-data/admin-area-data.entity'; @Injectable() export class AdminAreaService { @@ -32,18 +33,18 @@ export class AdminAreaService { this.eventService = eventService; } - public async getAdminAreas(countryCodeISO3): Promise { + public async getAdminAreasRaw(countryCodeISO3): Promise { return await this.adminAreaRepository.find({ - select: ['countryCodeISO3', 'name', 'placeCode', 'geom'], + select: ['countryCodeISO3', 'name', 'placeCode', 'geom', 'glofasStation'], where: { countryCodeISO3: countryCodeISO3 }, }); } - public async getAdminAreasPerLeadTime( + private async getTriggeredPlaceCodes( countryCodeISO3: string, leadTime: string, - adminLevel: number, - ): Promise { + ) { + console.log('getTriggeredPlaceCodes: ', countryCodeISO3); if (!leadTime) { leadTime = await this.getDefaultLeadTime(countryCodeISO3); } @@ -51,41 +52,119 @@ export class AdminAreaService { await this.eventService.getTriggerPerLeadtime(countryCodeISO3) )[leadTime]; - let placeCodes; + let placeCodes = []; if (parseInt(trigger) === 1) { placeCodes = ( await this.eventService.getTriggeredAreas(countryCodeISO3) - ).map((triggeredArea): string => "'" + triggeredArea.placeCode + "'"); + ).map((triggeredArea): string => triggeredArea.placeCode); } + return placeCodes; + } - const baseQuery = fs - .readFileSync('./src/api/admin-area/sql/get-admin-regions.sql') - .toString(); - const query = baseQuery.concat( - placeCodes && placeCodes.length > 0 - ? ' and geo."placeCode" in (' + placeCodes.toString() + ')' - : '', + public async getAggregatesData( + countryCodeISO3: string, + leadTime: string, + adminLevel: number, + ): Promise { + console.log('getAggregatesData: ', countryCodeISO3); + const placeCodes = await this.getTriggeredPlaceCodes( + countryCodeISO3, + leadTime, ); - const adminAreas: AdminAreaRecord[] = await this.manager.query(query, [ + let staticIndicatorsScript = this.adminAreaRepository + .createQueryBuilder('area') + .select(['area."placeCode"']) + .leftJoin(AdminAreaDataEntity, 'data', 'area.placeCode = data.placeCode') + .addSelect(['data."indicator"', 'data."value"']) + .where('area."countryCodeISO3" = :countryCodeISO3', { + countryCodeISO3: countryCodeISO3, + }) + .andWhere('area."adminLevel" = :adminLevel', { adminLevel: adminLevel }); + if (placeCodes.length) { + staticIndicatorsScript = staticIndicatorsScript.andWhere( + 'area."placeCode" IN (:...placeCodes)', + { placeCodes: placeCodes }, + ); + } + const staticIndicators = await staticIndicatorsScript.getRawMany(); + + let dynamicIndicatorsScript = this.adminAreaRepository + .createQueryBuilder('area') + .select(['area."placeCode"']) + .leftJoin( + AdminAreaDynamicDataEntity, + 'dynamic', + 'area.placeCode = dynamic.placeCode', + ) + .addSelect(['dynamic."indicator"', 'dynamic."value"']) + .where('area."countryCodeISO3" = :countryCodeISO3', { + countryCodeISO3: countryCodeISO3, + }) + .andWhere('date = current_date') + .andWhere('dynamic."leadTime" = :leadTime', { leadTime: leadTime }) + .andWhere('area."adminLevel" = :adminLevel', { adminLevel: adminLevel }); + if (placeCodes.length) { + dynamicIndicatorsScript = dynamicIndicatorsScript.andWhere( + 'area."placeCode" IN (:...placeCodes)', + { placeCodes: placeCodes }, + ); + } + const dynamicIndicators = await dynamicIndicatorsScript.getRawMany(); + + return staticIndicators.concat(dynamicIndicators); + } + + public async getAdminAreas( + countryCodeISO3: string, + leadTime: string, + adminLevel: number, + ): Promise { + console.log('getAdminAreas: ', countryCodeISO3); + const placeCodes = await this.getTriggeredPlaceCodes( countryCodeISO3, leadTime, - adminLevel, - ]); + ); - return this.helperService.toGeojson(adminAreas); - } + let adminAreasScript = this.adminAreaRepository + .createQueryBuilder('area') + .select([ + 'area."placeCode"', + 'area."name"', + 'ST_AsGeoJSON(area.geom)::json As geom', + 'area."countryCodeISO3"', + ]) + .leftJoin( + AdminAreaDynamicDataEntity, + 'dynamic', + 'area.placeCode = dynamic.placeCode', + ) + .addSelect([ + 'dynamic.value AS "population_affected"', + 'dynamic."leadTime"', + 'dynamic."date"', + ]) + .where('area."countryCodeISO3" = :countryCodeISO3', { + countryCodeISO3: countryCodeISO3, + }) + .andWhere('dynamic."leadTime" = :leadTime', { leadTime: leadTime }) + .andWhere('area."adminLevel" = :adminLevel', { adminLevel: adminLevel }) + .andWhere('date = current_date'); - public async getStationAdminAreaMappingByCountry( - countryCodeISO3, - ): Promise { - return await this.adminAreaRepository.find({ - select: ['countryCodeISO3', 'name', 'placeCode', 'glofasStation'], - where: { countryCodeISO3: countryCodeISO3 }, - }); + if (placeCodes.length) { + adminAreasScript = adminAreasScript.andWhere( + 'area."placeCode" IN (:...placeCodes)', + { placeCodes: placeCodes }, + ); + } + + const adminAreas = await adminAreasScript.getRawMany(); + + return this.helperService.toGeojson(adminAreas); } private async getDefaultLeadTime(countryCodeISO3: string): Promise { + console.log('getDefaultLeadTime: ', countryCodeISO3); const findOneOptions = { countryCodeISO3: countryCodeISO3, }; diff --git a/services/API-service/src/api/admin-area/sql/get-admin-regions.sql b/services/API-service/src/api/admin-area/sql/get-admin-regions.sql deleted file mode 100644 index ceea4aa24b..0000000000 --- a/services/API-service/src/api/admin-area/sql/get-admin-regions.sql +++ /dev/null @@ -1,44 +0,0 @@ -with pivot as ( - select aa."placeCode" - ,max(case when indicator = 'population_over65' then value end) as population_over65 - ,max(case when indicator = 'female_head_hh' then value end) as female_head_hh - ,max(case when indicator = 'population_u8' then value end) as population_u8 - ,max(case when indicator = 'poverty_incidence' then value end) as poverty_incidence - ,max(case when indicator = 'roof_type' then value end) as roof_type - ,max(case when indicator = 'wall_type' then value end) as wall_type - ,max(case when indicator = 'Weighted Vulnerability Index' then value end) as vulnerability_index - ,max(case when indicator = 'covid_risk' then value end) as covid_risk - ,max(case when indicator = 'population_u9' then value end) as population_u9 - ,max(case when indicator = 'dengue_incidence_average' then value end) as dengue_incidence_average - ,max(case when indicator = 'dengue_cases_average' then value end) as dengue_cases_average - from "IBF-app"."admin-area" aa - left join "IBF-app"."admin-area-data" aad - on aa."placeCode" = aad."placeCode" - group by 1 -) -select geo."placeCode" - ,geo."name" - ,ST_AsGeoJSON(geo.geom)::json As geom - ,geo."countryCodeISO3" - , date - , "leadTime" - , population_affected - , row_to_json(pivot.*) as indicators -from "IBF-app"."admin-area" geo -left join ( - select "countryCodeISO3" - ,"leadTime" - ,date - ,"placeCode" - ,value as population_affected - from "IBF-app"."admin-area-dynamic-data" - where date = current_date - and indicator = 'population_affected' -) ca - on geo."placeCode" = ca."placeCode" - and geo."countryCodeISO3" = ca."countryCodeISO3" -left join pivot - on geo."placeCode" = pivot."placeCode" -where geo."countryCodeISO3" = $1 -and "leadTime" = $2 -and "adminLevel" = $3 diff --git a/services/API-service/src/shared/data.model.ts b/services/API-service/src/shared/data.model.ts index 05c9120a03..102afdd760 100644 --- a/services/API-service/src/shared/data.model.ts +++ b/services/API-service/src/shared/data.model.ts @@ -8,7 +8,12 @@ export class AdminAreaRecord { public date: Date; public leadTime: string; public population_affected: number; - public indicators: object; +} + +export class AggregateDataRecord { + public placeCode: string; + public indicator: string; + public value: number; } export class TriggeredArea { diff --git a/services/IBF-pipeline/pipeline/lib/pipeline/forecast.py b/services/IBF-pipeline/pipeline/lib/pipeline/forecast.py index c3a79f6e79..718c7a7f52 100644 --- a/services/IBF-pipeline/pipeline/lib/pipeline/forecast.py +++ b/services/IBF-pipeline/pipeline/lib/pipeline/forecast.py @@ -16,7 +16,7 @@ def __init__(self, leadTimeLabel, leadTimeValue, countryCodeISO3, model): self.leadTimeValue = leadTimeValue self.db = DatabaseManager(leadTimeLabel, countryCodeISO3) - admin_area_json = self.db.apiGetRequest('admin-areas',countryCodeISO3=countryCodeISO3) + admin_area_json = self.db.apiGetRequest('admin-areas/raw',countryCodeISO3=countryCodeISO3) for index in range(len(admin_area_json)): admin_area_json[index]['geometry'] = admin_area_json[index]['geom'] admin_area_json[index]['properties'] = { @@ -27,7 +27,7 @@ def __init__(self, leadTimeLabel, leadTimeValue, countryCodeISO3, model): if model == 'glofas': self.glofas_stations = self.db.apiGetRequest('glofas-stations',countryCodeISO3=countryCodeISO3) - self.district_mapping = self.db.apiGetRequest('admin-areas/station-mapping',countryCodeISO3=countryCodeISO3) + self.district_mapping = self.db.apiGetRequest('admin-areas/raw',countryCodeISO3=countryCodeISO3) self.glofasData = GlofasData(leadTimeLabel, leadTimeValue, countryCodeISO3, self.glofas_stations,self.district_mapping) self.floodExtent = FloodExtent(leadTimeLabel, leadTimeValue, countryCodeISO3,self.district_mapping,self.admin_area_gdf) self.exposure = Exposure(leadTimeLabel, countryCodeISO3,self.admin_area_gdf,self.district_mapping)