Updates to improve error catching

openaq · Sep 19, 2024 · 8585b43 · 8585b43
1 parent c0b2d28
commit 8585b43
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/fetcher/lib/utils.js b/fetcher/lib/utils.js
@@ -210,10 +210,16 @@ function checkResponseData(data, start_timestamp, end_timestamp) {
 const fetchFileLocal = async (file) => {
     const filepath = file.path;
     const data = [];
-    return new Promise((resolve) => {
+    console.debug('fetching file', filepath);
+    return new Promise((resolve, reject) => {
         fs.createReadStream(filepath)
+            .on('error', (e) => {
+                reject(e);
+            })
             .pipe(csv())
-            .on('data', (row) => data.push(row))
+            .on('data', (row) => {
+                data.push(row);
+            })
             .on('end', () => {
                 resolve(data);
             });
@@ -223,8 +229,10 @@ const fetchFileLocal = async (file) => {
 const fetchFile = async (file) => {
     const source = file.source;
     let data;
-    if (source === 'local') {
+    if (['local','undefined',undefined,null].includes(source)) {
         data = await fetchFileLocal(file);
+    } else {
+        throw new Error(`No support for ${source}`);
     }
     return data;
 };

diff --git a/fetcher/providers/generic.js b/fetcher/providers/generic.js
@@ -376,6 +376,9 @@ class Client {
      */
     async processData(file) {
         const data = await this.fetchData(file);
+        if (!data) {
+            throw new Error('No data was returned from file');
+        }
         if (file.type === 'locations') {
             this.processLocationsData(data);
         } else if (file.type === 'sensors') {