Skip to content

Commit

Permalink
linting fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@caparker
caparker committed Sep 19, 2024
1 parent edec244 commit c0b2d28
Show file tree
Hide file tree
Showing 6 changed files with 1,886 additions and 3,087 deletions.
2 changes: 1 addition & 1 deletion .eslintrc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"no-var": "error",
"prefer-const": "error",
"array-bracket-spacing": [ "error", "never" ],
"comma-dangle": [ "error", "never" ],
"comma-dangle": [ "error", "only-multiline" ],
"computed-property-spacing": [ "error", "never" ],
"eol-last": "error",
"eqeqeq": [ "error", "smart" ],
Expand Down
47 changes: 21 additions & 26 deletions fetcher/lib/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/clien
const VERBOSE = !!process.env.VERBOSE;
const DRYRUN = !!process.env.DRYRUN;


const s3 = new S3Client({
maxRetries: 10
});
Expand Down Expand Up @@ -186,11 +187,11 @@ function checkResponseData(data, start_timestamp, end_timestamp) {
const fdata = data.filter((d) => {
return (
(d.time / 1000 >= start_timestamp || !start_timestamp) &&
d.time / 1000 <= end_timestamp
d.time / 1000 <= end_timestamp
);
});
if (fdata.length < n) {
// submit warning so we can track this
// submit warning so we can track this
const requested_start = new Date(
start_timestamp * 1000
).toISOString();
Expand All @@ -206,34 +207,28 @@ function checkResponseData(data, start_timestamp, end_timestamp) {
return fdata;
}

async function fetchFile(file) {
const source = file.source;
var data;
if(source == 'google-bucket') {
data = await fetchFileGoogleBucket(file);
} else {
data = await fetchFileLocal(file);
}
if(DRYRUN) {
//writeJson(data, `raw/${file.path}`)
}
return data;
const fetchFileLocal = async (file) => {
const filepath = file.path;
const data = [];
return new Promise((resolve) => {
fs.createReadStream(filepath)
.pipe(csv())
.on('data', (row) => data.push(row))
.on('end', () => {
resolve(data);
});
});
};

const fetchFileLocal = async file => {
const filepath = file.path;
const data = [];
return new Promise((resolve, reject) => {
fs.createReadStream(filepath)
.pipe(csv())
.on('data', row => data.push(row))
.on('end', () => {
resolve(data);
});
});
const fetchFile = async (file) => {
const source = file.source;
let data;
if (source === 'local') {
data = await fetchFileLocal(file);
}
return data;
};


module.exports = {
fetchSecret,
fetchFile,
Expand Down
70 changes: 33 additions & 37 deletions fetcher/providers/generic.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const Providers = require('../lib/providers');
const { request, fetchFile } = require('../lib/utils');
const { fetchFile } = require('../lib/utils');
const { Measures, FixedMeasure } = require('../lib/measure');
const { Measurand } = require('../lib/measurand');

Expand All @@ -15,7 +15,7 @@ const truthy = (value) => {
* @returns {*} - stripped value
*/
const stripWhitespace = (value) => {
if(typeof(value) == 'string') {
if (typeof(value) === 'string') {
return value.replace(/^[ ]+|[ ]+$/,'');
} else {
return value;
Expand Down Expand Up @@ -60,18 +60,17 @@ const stripNulls = (obj) => {
* @param {*} target - Most likely the class object (this)
* @param {*} data - The values that we are assiging to the target
* @param {*[]} [accepted=[]] - I list of keys that will be accepted to the metadata
* @returns {*} -
*/
const classAssign = (target, data, accepted = []) => {
const keys = Object.keys(target);
for( const [k,v] of Object.entries(data)) {
if(keys.includes(k)) {
for ( const [k,v] of Object.entries(data)) {
if (keys.includes(k)) {
target[k] = stripWhitespace(v);
} else if(accepted.includes(k)) {
if(!target.metadata) target.metadata = {};
} else if (accepted.includes(k)) {
if (!target.metadata) target.metadata = {};
target.metadata[k] = stripWhitespace(v);
}
};
}
};


Expand Down Expand Up @@ -101,13 +100,13 @@ class Location {
*/
getSystem(data) {
let key;
if(typeof(data) == 'string') {
if (typeof(data) === 'string') {
key = data;
data = { system_id: key };
} else {
key = data.system_id;
}
if(!this.systems[key]) {
if (!this.systems[key]) {
this.systems[key] = new System({ ...data });
}
return this.systems[key];
Expand All @@ -122,11 +121,8 @@ class Location {
add(sensor) {
// first we get the system name
// e.g. :provider/:site/:manufacturer-:model
const sensor_id = sensor.sensor_id;
const system_id = sensor.system_id;
const sys = this.getSystem(sensor);
const s = sys.add(sensor);
return s;
return sys.add(sensor);
}

/**
Expand All @@ -141,7 +137,7 @@ class Location {
lon: this.lon,
ismobile: this.ismobile,
metadata: this.metadata,
systems: Object.values(this.systems).map(s => s.json()),
systems: Object.values(this.systems).map((s) => s.json()),
});
}
}
Expand All @@ -168,7 +164,7 @@ class System {
*/
add(sensor) {
const sensor_id = sensor.sensor_id;
if(!this.sensors[sensor_id]) {
if (!this.sensors[sensor_id]) {
this.sensors[sensor_id] = new Sensor(sensor);
}
return this.sensors[sensor_id];
Expand All @@ -183,7 +179,7 @@ class System {
system_id: this.system_id,
manufacturer_name: this.manufacturer_name,
model_name: this.model_name,
sensors: Object.values(this.sensors).map(s => s.json()),
sensors: Object.values(this.sensors).map((s) => s.json()),
});
}

Expand Down Expand Up @@ -277,10 +273,10 @@ class Client {
const model = cleanKey(row[this.model_key]);
const location_id = this.getLocationId(row);
let key = null;
if(manufacturer && model) {
key = `${manufacturer}::${model}`;;
} else if(!manufacturer & !model) {
key = `default`;
if (manufacturer && model) {
key = `${manufacturer}::${model}`;
} else if (!manufacturer & !model) {
key = 'default';
} else {
key = `${manufacturer || model}`;
}
Expand Down Expand Up @@ -323,12 +319,12 @@ class Client {
getLocation(key) {
let loc = null;
let data = {};
if(typeof(key) == 'object') {
if (typeof(key) === 'object') {
data = { ...key };
key = this.getLocationId(data);
}
loc = this.locations[key];
if(!loc) {
if (!loc) {
loc = this.addLocation({ location_id: key, ...data });
}

Expand Down Expand Up @@ -370,7 +366,7 @@ class Client {
// if its binary than it should be an uploaded file
// if its an object then ...
return fetchFile(f);
};
}


/**
Expand All @@ -380,13 +376,13 @@ class Client {
*/
async processData(file) {
const data = await this.fetchData(file);
if(file.type == 'locations') {
if (file.type === 'locations') {
this.processLocationsData(data);
} else if(file.type == "sensors") {
} else if (file.type === 'sensors') {
this.processSensorsData(data);
} else if(file.type == "measurements") {
} else if (file.type === 'measurements') {
this.processMeasurementsData(data);
} else if(file.type == "flags") {
} else if (file.type === 'flags') {
this.processFlagsData(data);
}
}
Expand All @@ -399,7 +395,7 @@ class Client {
*/
addLocation(data) {
const key = this.getLocationId(data);
if(!this.locations[key]) {
if (!this.locations[key]) {
this.locations[key] = new Location({
location_id: key,
label: this.getLabel(data),
Expand Down Expand Up @@ -435,7 +431,6 @@ class Client {
*/
async processSensorsData(sensors) {
console.debug(`Processing ${sensors.length} sensors`);
const sensor_attributes = ['manufacturer_name', 'model_name', 'interval_seconds'];
sensors.map((d) => {
try {

Expand Down Expand Up @@ -466,10 +461,10 @@ class Client {
let params = [];
let long_format = false;

if(measurements.length) {
if (measurements.length) {
const keys = Object.keys(measurements[0]);
long_format = keys.includes(this.parameter_key) && keys.includes(this.value_key);
if(long_format) {
if (long_format) {
params = [this.parameter_key];
} else {
params = Object.keys(this.parameters);
Expand All @@ -480,15 +475,15 @@ class Client {
try {
const datetime = this.getDatetime(meas);
const location = meas[this.location_key];
params.map( p => {
params.map((p) => {
const value = long_format ? meas[this.value_key] : meas[p];
const metric = long_format ? meas[p] : p;
const m = {
location,
value,
metric,
};
if(m.value) {
if (m.value) {
this.measures.push({
sensor_id: this.getSensorId(m),
timestamp: datetime,
Expand All @@ -513,6 +508,7 @@ class Client {
flags.map((d) => {
try {
// coming soon
console.log(d);
} catch (e) {
console.warn(`Error adding flag: ${e.message}`);
}
Expand All @@ -533,7 +529,7 @@ class Client {
matching_method: 'ingest-id'
},
measures: this.measures.measures,
locations: Object.values(this.locations).map(l=>l.json())
locations: Object.values(this.locations).map((l)=>l.json())
};
}

Expand Down Expand Up @@ -573,9 +569,9 @@ module.exports = {
await client.processData(file);
}));
// fetch and process the data
//await client.fetchData();
// await client.fetchData();
// and then push it to the
//console.dir(client.data(), { depth: null });
// console.dir(client.data(), { depth: null });
Providers.put_measures_json(client.provider, client.data());
return client.summary();
},
Expand Down
2 changes: 1 addition & 1 deletion fetcher/providers/smartsense.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class SmartSenseApi {
/**
* Provide a sensor based ingest id
* @param {object} meas
* @param {object} measurand
* @param {object} uid
* @returns {string}
*/
getSensorId(meas, uid) {
Expand Down
Loading

0 comments on commit c0b2d28

Please sign in to comment.