This repository has been archived by the owner on Jan 15, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathindex.js
56 lines (51 loc) · 2.02 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
var csv = require('csv-parser');
var fse = require('fs-extra');
var path = require('path');
var followRedirects = require('follow-redirects');
followRedirects.maxRedirects = 10;
var https = followRedirects.https;
var sheets = JSON.parse(fse.readFileSync('sheets.json'));
function editLinkToCsvLink(url) {
var newUrl = url.replace('edit#gid=0', 'export?format=csv');
newUrl = newUrl.replace('edit?usp=sharing','export?format=csv');
return newUrl;
}
function rowFileName(rowData, headers) {
const exclusions = ['Link', 'Comment', 'Status', 'Owner'];
var fileNameComponents = [];
for (var i = 0; i < headers.length; i++) {
if (!exclusions.includes(headers[i])) {
if (rowData[headers[i]] && rowData[headers[i]].length > 0) {
fileNameComponents.push(rowData[headers[i]]);
}
}
}
return fileNameComponents.join('-').split(/[/:]/).join('_').split(' ').join('');
}
sheets["sheets"].forEach(s => {
var folder = path.join('out', s.name.replace(' ', ''));
fse.ensureDirSync(folder);
https.get(s.url, function (response) {
var headers;
response
.pipe(csv())
.on('headers', (h) => {
headers = h;
})
.on('data', (data) => {
try {
https.get(editLinkToCsvLink(data.Link), function(sheetResponse) {
if (sheetResponse.statusCode == 200) {
var file = fse.createWriteStream(path.join(folder, rowFileName(data, headers) + ".csv"));
sheetResponse.pipe(file);
} else {
console.log('Error accessing file for ' + rowFileName(data, headers) + ', ' + editLinkToCsvLink(data.Link));
}
});
}
catch(err) {
console.log(`Error : [${err}] for ${rowFileName(data, headers)}, ${editLinkToCsvLink(data.Link)}`);
}
});
});
});