Merge pull request #90 from ciatph/dev

v1.3.3

README.md

@@ -2,7 +2,7 @@
 
 The **ph-municipalities** NPM package has **NPM scripts** that allow interactive querying of Philippines municipalities included in one or more provinces or from a whole region, with an option of writing them to JSON files from the command line.
 
-It uses `/app/data/day1.xlsx` (downloaded and stored as of this 20220808) from PAGASA's [10-day weather forecast Excel files](https://www.pagasa.dost.gov.ph/climate/climate-prediction/10-day-climate-forecast) as the default data source.
+It uses a **PAGASA 10-day weather forecast Excel** file in `/app/data/day1.xlsx` (downloaded and stored as of this 20220808) from PAGASA's [10-Day Climate Forecast web page](https://www.pagasa.dost.gov.ph/climate/climate-prediction/10-day-climate-forecast) as the default data source, syncing with the **PAGASA Seasonal weather forecast** regions and province names defined in the manually-encoded `/app/config/regions.json` file to determine the region names.
 
 It also asks users to key in the file download URL of a remote PAGASA 10-Day weather forecast Excel file should they want to use another Excel file for a new and updated data source.
 

app/__tests__/classInitialization/fileLoading.js

@@ -33,7 +33,7 @@ describe('Class intialization using DEFAULT config', () => {
     jest.setTimeout(15000)
 
     const excelFile = new ExcelFile({
-      pathToFile: path.join(__dirname, 'excelfiledownload.xlsx'),
+      pathToFile: path.join(__dirname, 'excelfiledownload4.xlsx'),
       url: process.env.EXCEL_FILE_URL
     })
 
@@ -89,7 +89,7 @@ describe('Class intialization using CUSTOM config', () => {
     jest.setTimeout(15000)
 
     const excelFile = new ExcelFile({
-      pathToFile: path.join(__dirname, 'excelfiledownload.xlsx'),
+      pathToFile: path.join(__dirname, 'excelfiledownload5.xlsx'),
       url: process.env.EXCEL_FILE_URL,
       settings: config
     })

app/__tests__/municipalities/createMunicipalityInstance.js

@@ -23,9 +23,7 @@ const createMunicipalityInstance = (excelFile) => {
     // Unique provinces/municipalities from the 10-day Excel file
     const excel = {}
 
-    excel.provinces = excelFile.datalist
-      .map(item => item.province)
-      .filter((x, i, a) => a.indexOf(x) === i)
+    excel.provinces = excelFile.listAllProvinces(true)
 
     excel.municipalities = excelFile
       .listMunicipalities({ provinces: excel.provinces })
@@ -38,14 +36,14 @@ const createMunicipalityInstance = (excelFile) => {
 
     // Unique provinces/municipalities from the config file (PAGASA Seasonal)
     const config = {}
-    config.provinces = new Set(excelFile.listRegions('provinces').flat())
-    config.municipalities = excelFile.listMunicipalities({ provinces: [...config.provinces] })
+    config.provinces = excelFile.listAllProvinces()
+    config.municipalities = excelFile.listMunicipalities({ provinces: config.provinces })
     config.countMunicipalities = Object.values(config.municipalities).reduce((sum, item) => sum + item.length, 0)
 
     // Municipalities exist in 10-day Excel but not in the municipalities list built from the PAGASA seasonal config
     const hasMissingInConfig = (excel.countMunicipalities - config.countMunicipalities) > 0
     // Province(s) exist in the PAGASA seasonal config but not in the 10-day Excel
-    const hasMissingInExcel = (config.provinces.size - excel.provinces.length) > 0
+    const hasMissingInExcel = (config.provinces.length - excel.provinces.length) > 0
 
     logger.log(
       `[INFO]: Parsed municipalities from config: ${config.countMunicipalities}\n` +

app/__tests__/municipalities/municipalitiesCount.js

@@ -40,7 +40,7 @@ describe('Municipalities total count match', () => {
 
     // Process missing PAGASA seasonal config provinces/municipalities
     if (hasMissingInConfig) {
-      const fromConfig = excel.provinces.filter(item => !config.provinces.has(item))
+      const fromConfig = excel.provinces.filter(item => !config.provinces.includes(item))
 
       const countMissingConfig = Object.values(
         excelFile.listMunicipalities({ provinces: fromConfig })

app/__tests__/municipalities/municipalitiesPerProvinceCount.js

@@ -0,0 +1,111 @@
+require('dotenv')
+const path = require('path')
+
+const ExcelFile = require('../../src/classes/excel')
+const ColorLog = require('../../src/classes/colorlog')
+const logger = new ColorLog()
+
+const checkClass = require('../classInitialization/checkClass')
+const { arrayToString } = require('../../src/lib/utils')
+
+/* eslint-disable no-undef */
+describe('Municipalities per province count match', () => {
+  // Test using the latest 10-day PAGASA Excel file
+  const excelFile = new ExcelFile({
+    pathToFile: path.join(__dirname, 'excelfiledownload2.xlsx'),
+    url: process.env.EXCEL_FILE_URL
+  })
+
+  it('number of parsed/processed municipalities per province should match per province count from original data', async () => {
+    jest.setTimeout(15000)
+
+    // Start file download
+    await excelFile.init()
+
+    checkClass({
+      excelInstance: excelFile,
+      isRemote: true,
+      classType: ExcelFile
+    })
+
+    // Parsed/processed provinces from the Excel file
+    const allProvinces = excelFile.listAllProvinces(true)
+
+    // Provinces from the PAGASA seasonal config file
+    const allProvincesConfig = excelFile.listAllProvinces()
+
+    // Parsed/processed municipalities by province
+    const allMunicipalities = excelFile.listMunicipalities({
+      provinces: allProvinces
+    })
+
+    // Total parsed municipalities count
+    const countMunicipalities = Object.values(allMunicipalities)
+      .reduce(
+        (sum, item) => (sum += item.length), 0
+      )
+
+    const missing = []
+    const matches = []
+
+    // Compare parsed municipalities per province count vs. original (unprocessed) data
+    allProvinces.forEach((province) => {
+      // Total parsed count
+      const countParsed = allMunicipalities[province].length
+
+      // Total count from unprocessed data
+      const countRaw = excelFile.data.filter(row =>
+        row[excelFile.options.SHEETJS_COL]?.includes(`(${province.trim()})`)
+      )
+
+      if (countRaw.length > 0) {
+        const countLog = {
+          province,
+          loaded: countRaw.length,
+          parsed: countParsed
+        }
+
+        if (countParsed !== countRaw.length) {
+          missing.push(countLog)
+        } else {
+          matches.push(countLog)
+        }
+      }
+    })
+
+    expect(missing).toHaveLength(0)
+
+    let totalLoaded = 0
+    let totalParsed = 0
+
+    const message = matches.reduce((list, item) => {
+      totalLoaded += item.loaded
+      totalParsed += item.parsed
+      list += `${item.province}: loaded=${item.loaded}, parsed=${item.parsed}\n`
+      return list
+    }, [])
+
+    const total = `Total loaded: ${totalLoaded}\nTotal parsed: ${totalParsed}/${countMunicipalities}`
+
+    logger.log(message)
+    logger.log(total, { color: ColorLog.COLORS.TEXT.CYAN })
+
+    // Log missing/mismatching provinces from 10-day Excel and PAGASA seasonal config file
+    if (allProvinces.length !== allProvincesConfig.length) {
+      const missingInParsed = allProvincesConfig.filter(item => !allProvinces.includes(item))
+      const missingInConfig = allProvinces.filter(item => !allProvincesConfig.includes(item))
+
+      if (missingInParsed.length > 0) {
+        let msg = `[WARNING]: ${missingInParsed.length} province(s) in the 10-Day Excel file are missing\n`
+        msg += `in the (PAGASA seasonal) config: ${arrayToString(missingInParsed)}`
+        logger.log(msg, { color: ColorLog.COLORS.TEXT.YELLOW })
+      }
+
+      if (missingInConfig.length > 0) {
+        let msg = `[WARNING]: ${missingInConfig.length} province(s) in the (PAGASA seasonal) config are missing\n`
+        msg += `in the 10-Day Excel file: ${arrayToString(missingInConfig)}`
+        logger.log(msg, { color: ColorLog.COLORS.TEXT.YELLOW })
+      }
+    }
+  })
+})

app/__tests__/provinces/createInstances.js

@@ -20,13 +20,11 @@ const createInstances = (excelFile) => {
 
   try {
     // Unique provinces from the Excel file
-    const allExcelProvinces = excelFile.datalist
-      .map(item => item.province)
-      .filter((x, i, a) => a.indexOf(x) === i)
+    const allExcelProvinces = excelFile.listAllProvinces(true)
     const uniqueExcelProvinces = new Set(allExcelProvinces)
 
     // Unique provinces from the config file (PAGASA Seasonal)
-    const allProvinces = excelFile.listRegions('provinces').flat()
+    const allProvinces = excelFile.listAllProvinces()
     const uniqueProvinces = new Set(allProvinces)
 
     // Provinces present in the config (PAGASA seasonal) but missing in the 10-Day Excel file

app/__tests__/provinces/testProvinceCount.js

@@ -11,7 +11,7 @@ const updateInstances = require('./updateInstances')
 describe('Provinces names and count match', () => {
   // Test using the latest 10-day PAGASA Excel file
   const excelFile = new ExcelFile({
-    pathToFile: path.join(__dirname, 'excelfiledownload.xlsx'),
+    pathToFile: path.join(__dirname, 'excelfiledownload3.xlsx'),
     url: process.env.EXCEL_FILE_URL
   })
 

app/__tests__/provinces/updateInstances.js

@@ -51,7 +51,7 @@ const updateInstances = ({
       let msg = `[INFO]: Original provinces count are: ${allProvinces.length} (PAGASA seasonal config) vs. ${allExcelProvinces.length} (10-Day Excel file)\n`
       msg += '[INFO]: Removed incosistent provinces in the config and Excel file only during checking/testing (see yellow WARNINGs)\n'
       msg += `[INFO]: Modified provinces count are: ${uniqueProvinces.size} (PAGASA seasonal config) vs. ${uniqueExcelProvinces.size} (10-Day Excel file)\n\n`
-      msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or you may extend and override\n'
+      msg += '[NOTE]: If you believe these INFOs are incorrect, feel free to reach out or you may extend and override\n'
       msg += 'the ExcelFile or ExcelFactory classes in your scripts to customize this behaviour and other settings.'
 
       logger.log(msg, {

app/src/classes/excel/index.js

@@ -467,13 +467,39 @@ class ExcelFile {
   /**
    * Lists the province names of a region defined in the settings file
    * @param {String} regionName - Region name that matches with the `/app/config/regions.json` file's `data[N].name`
-   * @returns {String[]}  A list provinces under the `regionName`.
+   * @returns {String[]}  List provinces under the `regionName`.
    */
   listProvinces (regionName) {
     return this.#settings.data
       .find(region => region.name === regionName)?.provinces ?? []
   }
 
+  /**
+   * Lists the province names of a region defined in the settings (PAGASA seasonal config) file or from the parsed Excel file
+   * @param {String} region - Region name that matches with the `/app/config/regions.json` file's `data[N].name`
+   * @param {Bool} fromExcel - Flag to return the province names from the parsed 10-day Excel file. Defaults to `false`.
+   *    - Note: Province names from a "remote" Excel file may change without notice.
+   *    - It may differ from the contents of the "default" settings (PAGASA seasonal config) file.
+   *    - If the province names from the "remote" Excel file and "default" settings (PAGASA seasonal config) file vary,
+   *      consider initializing an `ExcelFile` or `ExcelFactory` class with a custom settings config file following
+   *      the format of the default settings file in `/app/config/regions.json`
+   * @returns {String[]}  List of all provinces from a 10-day Excel file.
+   */
+  listAllProvinces (fromExcel = false) {
+    if (fromExcel) {
+      // Return unique province names from the parsed Excel file
+      return this.#datalist
+        .map(item => item.province)
+        .filter((x, i, a) => a.indexOf(x) === i)
+    } else {
+      // Return province names from the PAGASA seasonal config file
+      return this.listRegions().reduce((list, region) => {
+        const provinces = this.listProvinces(region)
+        return [...list, ...provinces]
+      }, [])
+    }
+  }
+
   /**
    * Lists the region names defined in the settings file
    * @param {Object} key - Key name of the region data definition key.