diff --git a/.env_example b/.env_example index bd9638e1..40c3eb73 100644 --- a/.env_example +++ b/.env_example @@ -2,4 +2,5 @@ TETU_MATIC_RPC_URL=https://polygon-mainnet.g.alchemy.com/v2/key TETU_PRIVATE_KEY=deployer_key TETU_NETWORK_SCAN_KEY=key TETU_MATIC_FORK_BLOCK=37637584 +TETU_EXCLUDE_ERROR_LOGS=Return amount is not:separate:AS-1 Need rebalance diff --git a/package-lock.json b/package-lock.json index e16631df..3437992d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "tetu-v2-strategies-polygon", "version": "2.0.0", - "license": "MIT", + "license": "BUSL-1.1", "dependencies": { "@gelatonetwork/web3-functions-sdk": "^2.0.5", "@nomicfoundation/hardhat-chai-matchers": "^1.0.6", @@ -54,6 +54,7 @@ "ky": "^0.33.3", "lodash": "^4.17.21", "mocha": "^10.2.0", + "node-cron": "^3.0.3", "pg": "^8.10.0", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -71,6 +72,9 @@ "web3": "^1.8.2", "yargs": "^17.7.1" }, + "devDependencies": { + "@types/node-cron": "^3.0.11" + }, "engines": { "node": "^18.x.x" } @@ -6234,6 +6238,12 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.18.tgz", "integrity": "sha512-/4QOuy3ZpV7Ya1GTRz5CYSz3DgkKpyUptXuQ5PPce7uuyJAOR7r9FhkmxJfvcNUXyklbC63a+YvB3jxy7s9ngw==" }, + "node_modules/@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, "node_modules/@types/node-fetch": { "version": "2.6.5", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz", @@ -22711,6 +22721,17 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -33416,6 +33437,12 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.18.tgz", "integrity": "sha512-/4QOuy3ZpV7Ya1GTRz5CYSz3DgkKpyUptXuQ5PPce7uuyJAOR7r9FhkmxJfvcNUXyklbC63a+YvB3jxy7s9ngw==" }, + "@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, "@types/node-fetch": { "version": "2.6.5", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz", @@ -45795,6 +45822,14 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, + "node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "requires": { + "uuid": "8.3.2" + } + }, "node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", diff --git a/package.json b/package.json index c9a4c60f..1099edaa 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "ky": "^0.33.3", "lodash": "^4.17.21", "mocha": "^10.2.0", + "node-cron": "^3.0.3", "pg": "^8.10.0", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -101,5 +102,8 @@ }, "engines": { "node": "^18.x.x" + }, + "devDependencies": { + "@types/node-cron": "^3.0.11" } } diff --git a/scripts/rebalance.ts b/scripts/rebalance.ts index e401bf41..2eb0764c 100644 --- a/scripts/rebalance.ts +++ b/scripts/rebalance.ts @@ -19,6 +19,9 @@ import { subscribeTgBot } from './telegram/tg-subscribe'; import { Misc } from './utils/Misc'; import { NSRUtils } from './utils/NSRUtils'; import { formatUnits } from 'ethers/lib/utils'; +import cron from 'node-cron'; +import { Simulate } from 'react-dom/test-utils'; +import error = Simulate.error; // test rebalance debt // NODE_OPTIONS=--max_old_space_size=4096 hardhat run scripts/special/prepareTestEnvForUniswapV3ReduceDebtW3F.ts @@ -32,6 +35,7 @@ const MAX_ERROR_LENGTH = 1000; const DELAY_BETWEEN_NSRS = 60; const DELAY_AFTER_NSR = 10; const DELAY_NEED_NSR_CONFIRM = 300; +const DEFAULT_ENV_SEPARATOR = ':separate:'; dotEnvConfig(); // tslint:disable-next-line:no-var-requires @@ -58,10 +62,92 @@ const argv = require('yargs/yargs')() type: 'number', default: 60_000, }, + cronDailyReport: { + type: 'string', + default: '22 19 * * *' + }, + maxErrorCount: { + type: 'number', + default: 100 + }, + excludeErrorLogs: { + type: 'string', + default: '' + } }).argv; +enum EventType { + Rebalance = 'Rebalance success', + NSR = 'NSR success', + ErrorNSR = 'Error NSR', + ErrorExecute = 'Error EXECUTE', + Empty = 'Empty result', + ErrorProcessing = 'Error inside strategy processing', + ErrorRebalance = 'Error in debt rebalance', + ErrorFetching = 'Error fetch from url' +} + +const eventLogs = new Map(); +let excludeErrorLogs: string[] = []; + +async function logEvent(eventType: EventType, params: string[], isError = false, message: string = '') { + if (isError && !excludeErrorLogs.some(excludeErrorLog => excludeErrorLog.toLocaleLowerCase() === message)) { + await sendMessageToTelegram(message); + return; + } + if (!eventLogs.has(eventType)) { + eventLogs.set(eventType, []); + } + const events = eventLogs.get(eventType)!; + + eventLogs.get(eventType)!.push(params); + + if (eventType.toString().startsWith('Error') && events.length >= argv.maxErrorCount) { + const params = new Set(events.map(item => item.join(' - '))); + + await sendMessageToTelegram(`${eventType}: ${events.length} time(s)\n\n${Array.from(params).join('\n')}`); + eventLogs.set(eventType, []); + } +} + +async function sendDailyReport() { + let report = 'Daily Report:\n\n'; + eventLogs.forEach((params, eventType) => { + report += '---------------------------------------------------------------------------------\n' + + const countMap = new Map(); + + params.forEach(items => { + const key = `${items.join(' - ')}`; + countMap.set(key, (countMap.get(key) || 0) + 1); + }); + + report += `${eventType} Events:\n`; + countMap.forEach((count, strategyInfo) => { + report += ` - ${strategyInfo}: ${count} time(s)\n`; + }); + report += '---------------------------------------------------------------------------------\n\n' + }); + + + if (report) { + await sendMessageToTelegram(report); + } + + // clear logs + eventLogs.clear(); +} + + +// ------------ SEND REPORT +cron.schedule(argv.cronDailyReport, async () => { + await sendDailyReport(); +}); + + async function main() { console.log('Strategies NSR and debt rebalancer'); + excludeErrorLogs = argv.excludeErrorLogs.split(DEFAULT_ENV_SEPARATOR); if (!['localhost', 'matic'].includes(hre.network.name)) { console.log('Unsupported network', hre.network.name); @@ -149,7 +235,7 @@ async function main() { ); console.log('NSR success!', strategyName, strategyAddress); if (argv.nsrMsgSuccess) { - await sendMessageToTelegram(`NSR success! ${strategyName} ${strategyAddress}`); + await logEvent(EventType.NSR, [strategyName, strategyAddress]); } now = await Misc.getBlockTsFromChain(); @@ -157,8 +243,12 @@ async function main() { await sleep(DELAY_AFTER_NSR * 1000); } catch (e) { console.log('Error NSR', strategyName, strategyAddress, e); - await sendMessageToTelegram(`Error NSR ${strategyName} ${strategyAddress} ${(e as string).toString() - .substring(0, MAX_ERROR_LENGTH)}`); + await logEvent( + EventType.ErrorNSR, + [strategyName, strategyAddress], + true, + `Error NSR ${strategyName} ${strategyAddress} ${(e as string).toString().substring(0, MAX_ERROR_LENGTH)}` + ); } } else { if (needNSRTimestamp[strategyAddress] !== 0) { @@ -211,31 +301,43 @@ async function main() { ); console.log('Rebalance success!', strategyName, strategyAddress); if (argv.rebalanceDebtMsgSuccess) { - await sendMessageToTelegram(`Rebalance success! ${strategyName} ${strategyAddress}`); + await logEvent(EventType.Rebalance, [strategyName, strategyAddress]) } } catch (e) { console.log('Error EXECUTE', strategyName, strategyAddress, e); - await sendMessageToTelegram(`Error EXECUTE ${strategyName} ${strategyAddress} ${(e as string).toString() - .substring(0, MAX_ERROR_LENGTH)}`); + await logEvent( + EventType.ErrorExecute, + [strategyName, strategyAddress], + true, + `Error EXECUTE ${strategyName} ${strategyAddress} ${(e as string).toString().substring(0, MAX_ERROR_LENGTH)}` + ); } } else { console.log('Result can not be executed:', strategyName, result.message); } } else { console.log('Empty result!', strategyName); - await sendMessageToTelegram('Empty result! ' + strategyName); + await logEvent(EventType.Empty, [strategyName, strategyAddress]); } } catch (e) { console.log('Error inside strategy processing', strategyAddress, e); - await sendMessageToTelegram(`Error inside strategy processing ${strategyAddress} ${(e as string).toString() - .substring(0, MAX_ERROR_LENGTH)}`); + await logEvent( + EventType.ErrorProcessing, + [strategyAddress], + true, + `Error inside strategy processing ${strategyAddress} ${(e as string).toString().substring(0, MAX_ERROR_LENGTH)}` + ); } } } } catch (e) { console.log('error in debt rebalance loop', e); - await sendMessageToTelegram(`error in debt rebalance loop ${(e as string).toString() - .substring(0, MAX_ERROR_LENGTH)}`); + await logEvent( + EventType.ErrorRebalance, + [], + true, + `error in debt rebalance loop ${(e as string).toString().substring(0, MAX_ERROR_LENGTH)}` + ); } await sleep(argv.rebalanceDebtLoopDelay); @@ -252,7 +354,13 @@ const fetchFuncAxios = async(url: string) => { await sendMessageToTelegram(`wrong response for fetch ${url} ${r.data}`); } } catch (e) { - await sendMessageToTelegram(`error fetch ${url}`); + console.log(`error fetch ${url}`, e); + await logEvent( + EventType.ErrorFetching, + [url], + true, + `error fetch ${url}` + ); throw e; } };