Feature/update clarity (#35)

* Incorporate clarity's updates

* Added date to the key/file path

* Some bug fixes and adding date to key path

* Removing static path

* Linting fixes

fetcher/lib/measure.js

@@ -24,6 +24,7 @@ class Measures {
         if (!this.from || measure.timestamp < this.from) {
             this.from = measure.timestamp;
         }
+
         this.measures.push(measure);
     }
 

fetcher/lib/providers.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const dayjs = require('dayjs');
 const { SNSClient, PublishCommand } = require('@aws-sdk/client-sns');
 
 const {
@@ -9,6 +10,7 @@ const {
     fetchSecret,
     getObject,
     putObject,
+    putFile,
     prettyPrintStation
 } = require('./utils');
 
@@ -148,12 +150,41 @@ class Providers {
 
         if (DRYRUN) {
             console.log(`Would have saved ${measures.length} measurements to '${Bucket}/${Key}'`);
+            putFile(compressedString, Key);
             return new Promise((y) => y(true));
         }
         if (VERBOSE) console.debug(`Saving measurements to ${Bucket}/${Key}`);
 
         return await putObject(compressedString, Bucket, Key, false, 'text/csv', 'gzip');
     }
+
+    /**
+     * Given a measures object, save it to s3
+     *
+     * @param {string} provider The name of the provider (ie purpleair)
+     * @param {Measures} data A object with measures
+     * @param {string} id An optional identifier to use when creating filename
+     */
+    static async put_measures_json(provider, data, id) {
+        if (!data.measures.length && !data.locations.length) {
+            return console.warn('Nothing found, not uploading to S3.');
+        }
+        const Bucket = process.env.BUCKET;
+        const today = dayjs().format('YYYY-MM-DD');
+        const filename = id || `${Math.floor(Date.now() / 1000)}-${Math.random().toString(36).substring(8)}`;
+        const Key = `${process.env.STACK}/measures/${provider}/${today}/${filename}.json.gz`;
+        const compressedString = await gzip(JSON.stringify(data));
+
+
+        if (DRYRUN) {
+            console.log(`Would have saved ${data.measures.length} measurements and ${data.locations.length} stations to '${Bucket}/${Key}'`);
+            putFile(compressedString, Key);
+            return new Promise((y) => y(true));
+        }
+        if (VERBOSE) console.debug(`Saving measurements to ${Bucket}/${Key}`);
+
+        return await putObject(compressedString, Bucket, Key, false, 'application/json', 'gzip');
+    }
 }
 
 module.exports = Providers;

fetcher/lib/utils.js

@@ -1,6 +1,9 @@
 const zlib = require('zlib');
 const { promisify } = require('util');
 const request = promisify(require('request'));
+const fs = require('node:fs');
+const path = require('path');
+const homedir = require('os').homedir();
 
 const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager');
 const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3');
@@ -63,6 +66,20 @@ async function putObject(text, Bucket, Key, gzip = true, ContentType = 'applicat
     return await s3.send(cmd);
 }
 
+
+/**
+ *
+ * @param {string} text the string to be saved to a file
+ * @param {string} key the the file path. Usually the same key that would be used in the cloud storage
+ */
+async function putFile(text, key) {
+    const fpath = path.join(homedir, `Downloads/${key}`);
+    await fs.mkdirSync(path.dirname(fpath), { recursive: true });
+    await fs.writeFileSync(fpath, text);
+}
+
+
+
 /**
  * Retrieve secret from AWS Secrets Manager
  * @param {string} source The source object for which we are fetching a secret.
@@ -199,6 +216,7 @@ module.exports = {
     DRYRUN,
     getObject,
     putObject,
+    putFile,
     prettyPrintStation,
     checkResponseData
 };