Skip to content

Commit a36f15b

Browse files
antoinedcantoinedc
authored
Timescaledb (#325)
* introduce tsdb * better charting * new charts * more analytics * improve cahrts * user hypertables for overview stats * better sync of analytoc events * tests * fix tests * fix integrity checks tests * more test fix * backend tests * fix frontend snaps * frontend tests * fix some tests * small fix * handle deletions
1 parent 7792ea8 commit a36f15b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+3206
-1879
lines changed

‎.github/workflows/test_and_deploy.yml

-9
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@ jobs:
3333
run: |
3434
cd run
3535
npm run test tests/
36-
env:
37-
ENCRYPTION_KEY: 382A5C31A96D38E3DF430E5101E8D07D
38-
ENCRYPTION_JWT_SECRET: 26F95488BA7D7E545B1B8669990739BB21A0A6D3EFB4910C0460B068BDDD3E1C
39-
AUTH_SECRET: 123
40-
STRIPE_WEBHOOK_SECRET: 123
41-
STRIPE_SECRET_KEY: 123
42-
ETHERSCAN_API_TOKEN: 123
43-
APP_URL: http://ethernal.com
44-
APP_DOMAIN: ethernal.com
4536
4637
- name: Install pm2 server deps
4738
run: |

‎docker-compose.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ volumes:
66

77
services:
88
postgres:
9-
image: timescale/timescaledb:latest-pg14
9+
image: timescale/timescaledb-ha:pg14
1010
ports:
1111
- "5433:5432"
1212
restart: always
1313
volumes:
1414
- db:/var/lib/postgresql/data
1515
environment:
16-
- POSTGRES_PASSWORD=password
16+
- POSTGRES_PASSWORD=postgres
1717

1818
web: &backend
1919
volumes:

‎package.json

+3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
"ace-mode-solidity": "^0.1.1",
3232
"axios": "^0.21.1",
3333
"axios-cache-interceptor": "1",
34+
"chart.js": "3.9.1",
35+
"chartjs-plugin-zoom": "^2.0.1",
3436
"chokidar": "^3.5.0",
3537
"codemirror": "^6.0.1",
3638
"codemirror-solidity": "^0.2.5",
@@ -50,6 +52,7 @@
5052
"stripe": "^8.176.0",
5153
"v-stripe-elements": "^1.2.0",
5254
"vue": "^2.6.11",
55+
"vue-chartjs": "4.1.2",
5356
"vue-json-pretty": "^2.2.4",
5457
"vue-moment": "^4.1.0",
5558
"vue-router": "^3.4.9",

‎public/index.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900">
1010
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Exo:600">
1111
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css">
12-
<% if (htmlWebpackPlugin.options.feedbackDomain) { %>
12+
<% if (htmlWebpackPlugin.options.feedbackDomain && htmlWebpackPlugin.options.isProduction) { %>
1313
<!-- Smartsupp Live Chat script -->
1414
<script type="text/javascript">
1515
if (window.location.host == 'app.<%= htmlWebpackPlugin.options.feedbackDomain %>') {

‎run/api/contracts.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,10 @@ const { sanitize } = require('../lib/utils');
66
const workspaceAuthMiddleware = require('../middlewares/workspaceAuth');
77
const authMiddleware = require('../middlewares/auth');
88
const processContractVerification = require('../lib/processContractVerification');
9-
const { holderHistory, cumulativeSupply, transferVolume, holders, transfers } = require('./modules/tokens');
9+
const { holderHistory, circulatingSupply, holders, transfers } = require('./modules/tokens');
1010

1111
router.get('/:address/holderHistory', workspaceAuthMiddleware, holderHistory);
12-
router.get('/:address/cumulativeSupply', workspaceAuthMiddleware, cumulativeSupply);
13-
router.get('/:address/transferVolume', workspaceAuthMiddleware, transferVolume);
12+
router.get('/:address/circulatingSupply', workspaceAuthMiddleware, circulatingSupply);
1413
router.get('/:address/holders', workspaceAuthMiddleware, holders);
1514
router.get('/:address/transfers', workspaceAuthMiddleware, transfers);
1615

‎run/api/explorers.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY);
1+
const { getAppDomain, getDefaultPlanSlug, getDefaultExplorerTrialDays, getStripeSecretKey } = require('../lib/env');
2+
const stripe = require('stripe')(getStripeSecretKey());
23
const express = require('express');
34
const router = express.Router();
45
const { isStripeEnabled } = require('../lib/flags');
5-
const { getAppDomain, getDefaultPlanSlug, getDefaultExplorerTrialDays } = require('../lib/env');
66
const { ProviderConnector } = require('../lib/rpc');
77
const { Explorer } = require('../models');
88
const { withTimeout } = require('../lib/utils');

‎run/api/modules/tokens.js

+4-25
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,17 @@ const holderHistory = async (req, res) => {
1616
}
1717
};
1818

19-
const cumulativeSupply = async (req, res) => {
19+
const circulatingSupply = async (req, res) => {
2020
const data = req.query;
2121
try {
2222
if (!data.workspace || !data.from || !data.to)
2323
throw new Error('Missing parameters.');
2424

25-
const volume = await db.getTokenCumulativeSupply(data.workspace.id, req.params.address, data.from, data.to);
25+
const volume = await db.getTokenCirculatingSupply(data.workspace.id, req.params.address, data.from, data.to);
2626

2727
res.status(200).json(volume);
2828
} catch(error) {
29-
logger.error(error.message, { location: 'get.api.modules.tokens.cumulativeSupply', error: error, data: data });
30-
res.status(400).send(error.message);
31-
}
32-
};
33-
34-
const transferVolume = async (req, res) => {
35-
const data = req.query;
36-
try {
37-
if (!data.workspace || !data.from || !data.to)
38-
throw new Error('Missing parameters.');
39-
40-
const volume = await db.getTokenTransferVolume(data.workspace.id, req.params.address, data.from, data.to);
41-
42-
res.status(200).json(volume);
43-
} catch(error) {
44-
logger.error(error.message, { location: 'get.api.modules.tokens.transferVolume', error: error, data: data });
29+
logger.error(error.message, { location: 'get.api.modules.tokens.circulatingSupply', error: error, data: data });
4530
res.status(400).send(error.message);
4631
}
4732
};
@@ -76,10 +61,4 @@ const transfers = async (req, res) => {
7661
}
7762
};
7863

79-
module.exports = {
80-
holderHistory: holderHistory,
81-
cumulativeSupply: cumulativeSupply,
82-
transferVolume: transferVolume,
83-
holders: holders,
84-
transfers: transfers
85-
};
64+
module.exports = { holderHistory, circulatingSupply, holders, transfers };

‎run/api/stats.js

+101-11
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,122 @@ const db = require('../lib/firebase');
44
const logger = require('../lib/logger');
55
const workspaceAuthMiddleware = require('../middlewares/workspaceAuth');
66

7-
router.get('/wallets', workspaceAuthMiddleware, async (req, res) => {
7+
router.get('/transactions', workspaceAuthMiddleware, async (req, res) => {
88
const data = req.query;
99
try {
10-
if (!data.workspace)
10+
if (!data.workspace || !data.from || !data.to)
1111
throw new Error('Missing parameters.');
1212

13-
const wallets = await db.getWalletVolume(data.workspace.id);
13+
const transactions = await db.getTransactionVolume(data.workspace.id, data.from, data.to);
1414

15-
res.status(200).json(wallets);
15+
res.status(200).json(transactions);
1616
} catch(error) {
17-
logger.error(error.message, { location: 'get.api.stats.wallets', error: error, data: data });
17+
logger.error(error.message, { location: 'get.api.stats.transactions', error: error, data: data });
1818
res.status(400).send(error.message);
1919
}
2020
});
2121

22-
router.get('/transactions', workspaceAuthMiddleware, async (req, res) => {
22+
router.get('/tokenTransferVolume', workspaceAuthMiddleware, async (req, res) => {
2323
const data = req.query;
2424
try {
25-
if (!data.workspace)
25+
if (!data.workspace || !data.from || !data.to)
2626
throw new Error('Missing parameters.');
2727

28-
const transactions = await db.getTransactionVolume(data.workspace.id);
28+
const transfers = await db.getTokenTransferVolume(data.workspace.id, data.from, data.to, data.address, data.type);
2929

30-
res.status(200).json(transactions);
30+
res.status(200).json(transfers);
3131
} catch(error) {
32-
logger.error(error.message, { location: 'get.api.stats.transactions', error: error, data: data });
32+
logger.error(error.message, { location: 'get.api.stats.tokenTransferVolume', error: error, data: data });
33+
res.status(400).send(error.message);
34+
}
35+
});
36+
37+
router.get('/uniqueWalletCount', workspaceAuthMiddleware, async (req, res) => {
38+
const data = req.query;
39+
try {
40+
if (!data.workspace || !data.from || !data.to)
41+
throw new Error('Missing parameters.');
42+
43+
const uniqueWalletCount = await db.getUniqueWalletCount(data.workspace.id, data.from, data.to);
44+
45+
res.status(200).json(uniqueWalletCount);
46+
} catch(error) {
47+
logger.error(error.message, { location: 'get.api.stats.uniqueWalletCount', error: error, data: data });
48+
res.status(400).send(error.message);
49+
}
50+
});
51+
52+
router.get('/cumulativeWalletCount', workspaceAuthMiddleware, async (req, res) => {
53+
const data = req.query;
54+
try {
55+
if (!data.workspace || !data.from || !data.to)
56+
throw new Error('Missing parameters.');
57+
58+
const cumulativeWalletCount = await db.getCumulativeWalletCount(data.workspace.id, data.from, data.to);
59+
60+
res.status(200).json(cumulativeWalletCount);
61+
} catch(error) {
62+
logger.error(error.message, { location: 'get.api.stats.cumulativeWalletCount', error: error, data: data });
63+
res.status(400).send(error.message);
64+
}
65+
});
66+
67+
router.get('/deployedContractCount', workspaceAuthMiddleware, async (req, res) => {
68+
const data = req.query;
69+
try {
70+
if (!data.workspace || !data.from || !data.to)
71+
throw new Error('Missing parameters.');
72+
73+
const deployedContractCount = await db.getDeployedContractCount(data.workspace.id, data.from, data.to);
74+
75+
res.status(200).json(deployedContractCount);
76+
} catch(error) {
77+
logger.error(error.message, { location: 'get.api.stats.deployedContractCount', error: error, data: data });
78+
res.status(400).send(error.message);
79+
}
80+
});
81+
82+
router.get('/cumulativeDeployedContractCount', workspaceAuthMiddleware, async (req, res) => {
83+
const data = req.query;
84+
try {
85+
if (!data.workspace || !data.from || !data.to)
86+
throw new Error('Missing parameters.');
87+
88+
const cumulativeDeployedContractCount = await db.getCumulativeDeployedContractCount(data.workspace.id, data.from, data.to);
89+
90+
res.status(200).json(cumulativeDeployedContractCount);
91+
} catch(error) {
92+
logger.error(error.message, { location: 'get.api.stats.getCumulativeDeployedContractCount', error: error, data: data });
93+
res.status(400).send(error.message);
94+
}
95+
});
96+
97+
router.get('/averageGasPrice', workspaceAuthMiddleware, async (req, res) => {
98+
const data = req.query;
99+
try {
100+
if (!data.workspace || !data.from || !data.to)
101+
throw new Error('Missing parameters.');
102+
103+
const avgGasPrice = await db.getAverageGasPrice(data.workspace.id, data.from, data.to);
104+
105+
res.status(200).json(avgGasPrice);
106+
} catch(error) {
107+
logger.error(error.message, { location: 'get.api.stats.averageGasPrice', error: error, data: data });
108+
res.status(400).send(error.message);
109+
}
110+
});
111+
112+
router.get('/averageTransactionFee', workspaceAuthMiddleware, async (req, res) => {
113+
const data = req.query;
114+
try {
115+
if (!data.workspace || !data.from || !data.to)
116+
throw new Error('Missing parameters.');
117+
118+
const avgTransactionFee = await db.getAverageTransactionFee(data.workspace.id, data.from, data.to);
119+
120+
res.status(200).json(avgTransactionFee);
121+
} catch(error) {
122+
logger.error(error.message, { location: 'get.api.stats.averageTransactionFee', error: error, data: data });
33123
res.status(400).send(error.message);
34124
}
35125
});
@@ -41,7 +131,7 @@ router.get('/global', workspaceAuthMiddleware, async (req, res) => {
41131
throw new Error('Missing parameters.');
42132

43133
const ts24hago = new Date(new Date().getTime() - (24 * 3600 *1000));
44-
const txCount24h = await db.getTxCount(data.workspace.id, ts24hago);
134+
const txCount24h = await db.getTotalTxCount(data.workspace.id, ts24hago);
45135
const txCountTotal = await db.getTotalTxCount(data.workspace.id);
46136
const activeWalletCount = await db.getActiveWalletCount(data.workspace.id);
47137

‎run/api/status.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ router.get('/', workspaceAuthMiddleware, async (req, res, next) => {
1616
if ((workspace.integrityCheckStartBlockNumber === null || workspace.integrityCheckStartBlockNumber === undefined) && !workspace.rpcHealthCheckEnabled)
1717
throw new Error('Status is not available on this workspace');
1818

19-
const integrityCheck = workspace.integrityCheck || {};
20-
const rpcHealthCheck = workspace.rpcHealthCheck || {};
19+
const integrityCheck = workspace.integrityCheck || {};
20+
const rpcHealthCheck = workspace.rpcHealthCheck || {};
2121
const result = sanitize({
2222
syncStatus: integrityCheck.status,
2323
latestCheckedBlock: integrityCheck.block && integrityCheck.block.number,

‎run/api/users.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY);
1+
const { getStripeSecretKey } = require('../lib/env');
2+
const stripe = require('stripe')(getStripeSecretKey());
23
const logger = require('../lib/logger');
34
const Analytics = require('../lib/analytics');
45
const { isStripeEnabled, isSendgridEnabled, isFirebaseAuthEnabled } = require('../lib/flags');

‎run/config/database.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ module.exports = {
66
"database": "ethernal",
77
"port": process.env.DB_PORT,
88
"password": process.env.DB_PASSWORD,
9+
"port": process.env.DB_PORT,
910
"dialect": "postgres",
1011
"logging": function(sql, sequelizeObject) {
1112
logger.debug(sql, { instance: sequelizeObject.instance });
1213
}
1314
},
1415
production: {
15-
"username": process.env.DB_USERNAME,
16+
"username": process.env.DB_USER,
1617
"password": process.env.DB_PASSWORD,
1718
"database": process.env.DB_NAME,
1819
"host": process.env.DB_HOST,

‎run/jobs/integrityCheck.js

+1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ module.exports = async job => {
129129
will be able to start integrity checks from it.
130130
*/
131131
if (!lowerBlock) {
132+
await workspace.safeDeleteIntegrityCheck();
132133
return await enqueue(`blockSync`, `blockSync-${workspace.id}-${workspace.integrityCheckStartBlockNumber}`, {
133134
userId: workspace.user.firebaseUserId,
134135
workspace: workspace.name,

‎run/lib/crypto.js

+6-8
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
const crypto = require('crypto');
22
const jwt = require('jsonwebtoken');
3+
const { getEncryptionKey, getEncryptionJwtSecret } = require('./env');
34
const { FirebaseScrypt } = require('firebase-scrypt');
45
const ALGORITHM = 'aes-256-cbc';
5-
const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY;
66
const IV_LENGTH = 16;
77
const firebaseParameters = {
88
algorithm: 'SCRYPT',
@@ -15,7 +15,7 @@ const firebaseParameters = {
1515
module.exports = {
1616
encrypt: (data) => {
1717
const iv = crypto.randomBytes(IV_LENGTH);
18-
const cipher = crypto.createCipheriv(ALGORITHM, Buffer.from(ENCRYPTION_KEY), iv);
18+
const cipher = crypto.createCipheriv(ALGORITHM, Buffer.from(getEncryptionKey()), iv);
1919

2020
let encryptedData = cipher.update(data);
2121
encryptedData = Buffer.concat([encryptedData, cipher.final()]);
@@ -27,20 +27,18 @@ module.exports = {
2727
const iv = Buffer.from(textParts.shift(), 'hex');
2828

2929
const encryptedText = Buffer.from(textParts.join(':'), 'hex');
30-
const decipher = crypto.createDecipheriv('aes-256-cbc', Buffer.from(ENCRYPTION_KEY), iv);
30+
const decipher = crypto.createDecipheriv('aes-256-cbc', Buffer.from(getEncryptionKey()), iv);
3131
decipher.setAutoPadding(false);
3232
let decrypted = decipher.update(encryptedText);
3333
decrypted = Buffer.concat([decrypted, decipher.final()]);
3434

3535
return decrypted.toString();
3636
},
37-
encode: (data) => {
38-
const jwtSecret = process.env.ENCRYPTION_JWT_SECRET;
39-
return jwt.sign(data, jwtSecret);
37+
encode: (data) => {;
38+
return jwt.sign(data, getEncryptionJwtSecret());
4039
},
4140
decode: (token) => {
42-
const jwtSecret = process.env.ENCRYPTION_JWT_SECRET;
43-
return jwt.verify(token, jwtSecret);
41+
return jwt.verify(token, getEncryptionJwtSecret());
4442
},
4543
firebaseHash: async (password) => {
4644
const salt = crypto.randomBytes(12).toString('base64');

‎run/lib/env.js

+4-1
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@ module.exports = {
1212
getDemoUserId: () => process.env.DEMO_USER_ID,
1313
getDemoTrialSlug: () => process.env.DEMO_TRIAL_SLUG,
1414
getStripeSecretKey: () => process.env.STRIPE_SECRET_KEY,
15+
getStripeWebhookSecret: () => process.env.STRIPE_WEBHOOK_SECRET,
1516
getDefaultExplorerTrialDays: () => process.env.DEFAULT_EXPLORER_TRIAL_DAYS || 7,
1617
getPostHogApiKey: () => process.env.POST_HOG_API_KEY,
1718
getPostHogApiHost: () => process.env.POST_HOG_API_HOST,
1819
getMaxBlockForSyncReset: () => parseInt(process.env.MAX_BLOCK_FOR_SYNC_RESET) || 10,
19-
getMaxContractForReset: () => parseInt(process.env.MAX_CONTRACT_FOR_RESET) || 5
20+
getMaxContractForReset: () => parseInt(process.env.MAX_CONTRACT_FOR_RESET) || 5,
21+
getEncryptionKey: () => process.env.ENCRYPTION_KEY,
22+
getEncryptionJwtSecret: () => process.env.ENCRYPTION_JWT_SECRET
2023
};

0 commit comments

Comments
 (0)