diff --git a/README.md b/README.md index 9bd5db8db..e706545e5 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ You can override default port numbering system by setting `PORT` and `P2P_PORT` ``` gcloud init # For genesis deploy -bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-only|--skip-kill] +bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only] # For incremental deploy bash deploy_blockchain_incremental_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] ``` @@ -130,7 +130,7 @@ BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/afan-shard MIN_NUM_VALIDATORS=1 DEBUG= ``` gcloud init # For genesis deploy -bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-only|--skip-kill] +bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only] # For incremental deploy bash deploy_blockchain_incremental_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] ``` diff --git a/blockchain-configs/afan-shard/node_params.json b/blockchain-configs/afan-shard/node_params.json index e6e2cfb0c..dfff19fc3 100644 --- a/blockchain-configs/afan-shard/node_params.json +++ b/blockchain-configs/afan-shard/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 3, diff --git a/blockchain-configs/base/node_params.json b/blockchain-configs/base/node_params.json index 095343862..23fd32365 100644 --- a/blockchain-configs/base/node_params.json +++ b/blockchain-configs/base/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/he-shard/node_params.json b/blockchain-configs/he-shard/node_params.json index 050f07072..c917e507d 100644 --- a/blockchain-configs/he-shard/node_params.json +++ b/blockchain-configs/he-shard/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/mainnet-prod/node_params.json b/blockchain-configs/mainnet-prod/node_params.json index ee91f27f3..5bbbd3e59 100644 --- a/blockchain-configs/mainnet-prod/node_params.json +++ b/blockchain-configs/mainnet-prod/node_params.json @@ -46,7 +46,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, @@ -56,7 +56,7 @@ "ON_MEMORY_CHAIN_LENGTH": 10, "P2P_HEARTBEAT_INTERVAL_MS": 15000, "P2P_MESSAGE_TIMEOUT_MS": 600000, - "P2P_PORT": 5000, + "P2P_PORT": 4997, "P2P_WAIT_FOR_ADDRESS_TIMEOUT_MS": 10000, "PEER_CANDIDATE_JSON_RPC_URL": "https://mainnet-api.ainetwork.ai/json-rpc", "PEER_CANDIDATES_CONNECTION_INTERVAL_MS": 20000, diff --git a/blockchain-configs/sim-shard/node_params.json b/blockchain-configs/sim-shard/node_params.json index 35c97a9d3..a23e866f2 100644 --- a/blockchain-configs/sim-shard/node_params.json +++ b/blockchain-configs/sim-shard/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/testnet-dev/node_params.json b/blockchain-configs/testnet-dev/node_params.json index 2cd2de037..30dabfde6 100644 --- a/blockchain-configs/testnet-dev/node_params.json +++ b/blockchain-configs/testnet-dev/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/testnet-exp/node_params.json b/blockchain-configs/testnet-exp/node_params.json index 12f0a6110..0f781d236 100644 --- a/blockchain-configs/testnet-exp/node_params.json +++ b/blockchain-configs/testnet-exp/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/testnet-prod/node_params.json b/blockchain-configs/testnet-prod/node_params.json index e7983699b..8683b50de 100644 --- a/blockchain-configs/testnet-prod/node_params.json +++ b/blockchain-configs/testnet-prod/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, @@ -57,7 +57,7 @@ "ON_MEMORY_CHAIN_LENGTH": 10, "P2P_HEARTBEAT_INTERVAL_MS": 15000, "P2P_MESSAGE_TIMEOUT_MS": 600000, - "P2P_PORT": 5000, + "P2P_PORT": 4998, "P2P_WAIT_FOR_ADDRESS_TIMEOUT_MS": 10000, "PEER_CANDIDATE_JSON_RPC_URL": "https://testnet-api.ainetwork.ai/json-rpc", "PEER_CANDIDATES_CONNECTION_INTERVAL_MS": 20000, diff --git a/blockchain-configs/testnet-sandbox/node_params.json b/blockchain-configs/testnet-sandbox/node_params.json index 04af90f42..d785df4a7 100644 --- a/blockchain-configs/testnet-sandbox/node_params.json +++ b/blockchain-configs/testnet-sandbox/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, diff --git a/blockchain-configs/testnet-staging/node_params.json b/blockchain-configs/testnet-staging/node_params.json index 1d5bc123a..d00659f37 100644 --- a/blockchain-configs/testnet-staging/node_params.json +++ b/blockchain-configs/testnet-staging/node_params.json @@ -47,7 +47,7 @@ "MAX_FINALIZED_BLOCK_INFO_ON_MEM": 1000, "MAX_JSON_RPC_API_READ_RATE_LIMIT": 10, "MAX_JSON_RPC_API_WRITE_RATE_LIMIT": 1, - "MAX_NUM_EVENT_CHANNELS": 20, + "MAX_NUM_EVENT_CHANNELS": 30, "MAX_NUM_EVENT_FILTERS": 40, "MAX_NUM_EVENT_FILTERS_PER_CHANNEL": 5, "MAX_NUM_INBOUND_CONNECTION": 6, @@ -57,7 +57,7 @@ "ON_MEMORY_CHAIN_LENGTH": 10, "P2P_HEARTBEAT_INTERVAL_MS": 15000, "P2P_MESSAGE_TIMEOUT_MS": 600000, - "P2P_PORT": 5000, + "P2P_PORT": 4999, "P2P_WAIT_FOR_ADDRESS_TIMEOUT_MS": 10000, "PEER_CANDIDATE_JSON_RPC_URL": "https://staging-api.ainetwork.ai/json-rpc", "PEER_CANDIDATES_CONNECTION_INTERVAL_MS": 20000, diff --git a/client/index.js b/client/index.js index 26d6a5e55..08f6b61ff 100755 --- a/client/index.js +++ b/client/index.js @@ -97,7 +97,7 @@ app.get('/metrics', async (req, res, next) => { .end(); }); -// Used in wait_until_node_sync_gcp.sh +// Used in wait_until_node_sync.sh app.get('/last_block_number', (req, res, next) => { const beginTime = Date.now(); const result = node.bc.lastBlockNumber(); diff --git a/client/protocol_versions.json b/client/protocol_versions.json index 77eb4c4f2..6f2326201 100644 --- a/client/protocol_versions.json +++ b/client/protocol_versions.json @@ -155,5 +155,8 @@ }, "1.4.2": { "min": "1.0.0" + }, + "1.5.0": { + "min": "1.0.0" } } \ No newline at end of file diff --git a/common/file-util.js b/common/file-util.js index 2ab4f2087..85557f792 100644 --- a/common/file-util.js +++ b/common/file-util.js @@ -257,7 +257,7 @@ class FileUtil { }); }); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err.stack}`); return false; } } @@ -288,7 +288,7 @@ class FileUtil { }); }); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err.stack}`); return null; } } @@ -299,7 +299,7 @@ class FileUtil { const zippedFs = fs.readFileSync(filePath); return FileUtil.buildObjectFromChunks(JSON.parse(zlib.gunzipSync(zippedFs).toString()).docs); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err.stack}`); return null; } } @@ -314,7 +314,7 @@ class FileUtil { const zippedFs = fs.readFileSync(filePath); return JSON.parse(zlib.gunzipSync(zippedFs).toString()); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err.stack}`); return null; } } @@ -325,7 +325,7 @@ class FileUtil { const fileStr = fs.readFileSync(filePath); return JSON.parse(fileStr); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${filePath}: ${err.stack}`); return null; } } @@ -399,7 +399,7 @@ class FileUtil { try { return Number(fs.readFileSync(h2nPath).toString()); } catch (err) { - logger.error(`[${LOG_HEADER}] Error while reading ${h2nPath}: ${err}`); + logger.error(`[${LOG_HEADER}] Error while reading ${h2nPath}: ${err.stack}`); return -1; } } diff --git a/copy_blockchain_data_gcp.sh b/copy_blockchain_data_gcp.sh index 965970d58..b53c4ae07 100644 --- a/copy_blockchain_data_gcp.sh +++ b/copy_blockchain_data_gcp.sh @@ -2,7 +2,8 @@ function usage() { printf "Usage: bash copy_blockchain_data_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] [download|upload]\n" - printf "Example: bash copy_blockchain_data_gcp.sh spring 5 download\n" + printf "Example: bash copy_blockchain_data_gcp.sh spring 0 download\n" + printf "Example: bash copy_blockchain_data_gcp.sh spring 1 upload\n" printf "\n" exit } @@ -146,7 +147,7 @@ function upload_data() { # 2. Extract tgz file for node printf "\n\n<<< Extracting tgz file for node $node_index >>>\n\n" - TGZ_CMD="gcloud compute ssh $node_target_addr --command 'cd /home; sudo mkdir -p ain_blockchain_data; sudo chown runner:runner ain_blockchain_data; sudo chmod 777 ain_blockchain_data; cd ain_blockchain_data; gzip -dc ~/ain_blockchain_data.tar.gz | tar xvf -' --project $PROJECT_ID --zone $node_zone" + TGZ_CMD="gcloud compute ssh $node_target_addr --command 'cd /home; sudo mkdir -p ain_blockchain_data; sudo chown $GCP_USER:$GCP_USER ain_blockchain_data; sudo chmod 777 ain_blockchain_data; cd ain_blockchain_data; sudo rm -rf chains snapshots; gzip -dc ~/ain_blockchain_data.tar.gz | tar xvf -' --project $PROJECT_ID --zone $node_zone" printf "TGZ_CMD=$TGZ_CMD\n\n" eval $TGZ_CMD diff --git a/copy_blockchain_data_onprem.sh b/copy_blockchain_data_onprem.sh new file mode 100644 index 000000000..72b62b67d --- /dev/null +++ b/copy_blockchain_data_onprem.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +function usage() { + printf "Usage: bash copy_blockchain_data_onprem.sh [staging|spring|mainnet] [download|upload]\n" + printf "Example: bash copy_blockchain_data_onprem.sh staging 0 download\n" + printf "Example: bash copy_blockchain_data_onprem.sh staging 1 upload\n" + printf "\n" + exit +} + +if [[ $# -lt 3 ]] || [[ $# -gt 3 ]]; then + usage +fi + +if [[ "$1" = 'staging' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid argument: $1\n" + exit +fi +printf "\n" +printf "SEASON=$SEASON\n" + +ONPREM_USER="nvidia" +printf "ONPREM_USER=$ONPREM_USER\n" + +number_re='^[0-9]+$' +if ! [[ $2 =~ $number_re ]] ; then + printf "\n" + printf "Invalid argument: $2\n" + exit +fi +NODE_INDEX=$2 +if [[ $NODE_INDEX -lt 0 ]] || [[ $NODE_INDEX -gt 9 ]]; then + printf "\n" + printf "Out-of-range argument: $NODE_INDEX\n" + exit +fi +printf "NODE_INDEX=$NODE_INDEX\n" + +if [[ "$3" = 'download' ]] || [[ "$3" = 'upload' ]]; then + COMMAND="$3" +else + printf "\n" + printf "Invalid argument: $3\n" + printf "\n" + usage +fi +printf "COMMAND=$COMMAND\n" + +# Get confirmation. +if [[ "$SEASON" = "mainnet" ]]; then + printf "\n" + printf "Do you want to proceed for $SEASON? Enter [mainnet]: " + read CONFIRM + printf "\n\n" + if [[ ! $CONFIRM = "mainnet" ]] + then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +else + printf "\n" + read -p "Do you want to proceed for $SEASON? [y/N]: " -n 1 -r + printf "\n\n" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +fi + +# Read node ip addresses and passwords +IFS=$'\n' read -d '' -r -a NODE_IP_LIST < ./ip_addresses/${SEASON}_onprem_ip.txt +IFS=$'\n' read -d '' -r -a NODE_PW_LIST < ./ip_addresses/${SEASON}_onprem_pw.txt + +function download_data() { + local node_index="$1" + local node_target_addr="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" + local node_login_pw="${NODE_PW_LIST[${node_index}]}" + + printf "\n* >> Downloading data from node $node_index ($node_target_addr) *********************************************************\n\n" + + printf "node_target_addr='$node_target_addr'\n" + + # 1. Create tgz file for node + printf "\n\n<<< Creating tgz file for node $node_index >>>\n\n" + TGZ_CMD="ssh $node_target_addr 'sudo -S ls -la; cd /home/${SEASON}/ain_blockchain_data; tar cvf - chains snapshots | gzip -c > ~/ain_blockchain_data.tar.gz'" + printf "TGZ_CMD=$TGZ_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${TGZ_CMD}" + + # 2. Copy tgz file from node + printf "\n\n<<< Copying tgz file from node $node_index >>>\n\n" + SCP_CMD="scp -r $node_target_addr:~/ain_blockchain_data.tar.gz ." + printf "SCP_CMD=$SCP_CMD\n\n" + eval "sshpass -f <(printf '%s\n' ${node_login_pw}) ${SCP_CMD}" + + # 3. Clean up tgz file for node + printf "\n\n<<< Cleaning up tgz file for node $node_index >>>\n\n" + CLEANUP_CMD="ssh $node_target_addr 'rm ~/ain_blockchain_data.tar.gz'" + printf "CLEANUP_CMD=$CLEANUP_CMD\n\n" + eval "sshpass -f <(printf '%s\n' ${node_login_pw}) ${CLEANUP_CMD}" +} + +function upload_data() { + local node_index="$1" + local node_target_addr="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" + local node_login_pw="${NODE_PW_LIST[${node_index}]}" + + printf "\n* >> Uploading data from node $node_index ($node_target_addr) *********************************************************\n\n" + + printf "node_target_addr='$node_target_addr'\n" + + # 1. Copy tgz file to node + printf "\n\n<<< Copying tgz file to node $node_index >>>\n\n" + SCP_CMD="scp -r ./ain_blockchain_data.tar.gz $node_target_addr:~" + printf "SCP_CMD=$SCP_CMD\n\n" + eval "sshpass -f <(printf '%s\n' ${node_login_pw}) ${SCP_CMD}" + + # 2. Extract tgz file for node + printf "\n\n<<< Extracting tgz file for node $node_index >>>\n\n" + TGZ_CMD="ssh $node_target_addr 'sudo -S ls -la; cd /home; sudo mkdir -p ${SEASON}/ain_blockchain_data; sudo chown $ONPREM_USER:$ONPREM_USER ${SEASON} ${SEASON}/ain_blockchain_data; sudo chmod 777 ${SEASON} ${SEASON}/ain_blockchain_data; cd ${SEASON}/ain_blockchain_data; sudo rm -rf chains snapshots; gzip -dc ~/ain_blockchain_data.tar.gz | tar xvf -'" + printf "TGZ_CMD=$TGZ_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${TGZ_CMD}" + + # 3. Clean up tgz file for node + printf "\n\n<<< Cleaning up tgz file for node $node_index >>>\n\n" + CLEANUP_CMD="ssh $node_target_addr 'rm ~/ain_blockchain_data.tar.gz'" + printf "CLEANUP_CMD=$CLEANUP_CMD\n\n" + eval "sshpass -f <(printf '%s\n' ${node_login_pw}) ${CLEANUP_CMD}" +} + +if [[ "$COMMAND" = 'upload' ]]; then + upload_data "$NODE_INDEX" +else + download_data "$NODE_INDEX" +fi diff --git a/deploy_blockchain_genesis_gcp.sh b/deploy_blockchain_genesis_gcp.sh index 78b670338..54a5f9dad 100644 --- a/deploy_blockchain_genesis_gcp.sh +++ b/deploy_blockchain_genesis_gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash if [[ $# -lt 4 ]] || [[ $# -gt 10 ]]; then - printf "Usage: bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-only|--skip-kill]\n" + printf "Usage: bash deploy_blockchain_genesis_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only]\n" printf "Example: bash deploy_blockchain_genesis_gcp.sh dev 0 -1 4 --keystore --no-keep-code\n" printf "Example: bash deploy_blockchain_genesis_gcp.sh dev 0 0 0 --keystore --keep-code\n" printf "Example: bash deploy_blockchain_genesis_gcp.sh dev 0 -1 -1 --setup --keystore --no-keep-code\n" @@ -72,17 +72,9 @@ function parse_options() { CHOWN_DATA_OPTION="$option" elif [[ $option = '--no-chown-data' ]]; then CHOWN_DATA_OPTION="$option" - elif [[ $option = '--kill-only' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-job' ]]; then KILL_OPTION="$option" - elif [[ $option = '--skip-kill' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-only' ]]; then KILL_OPTION="$option" else printf "Invalid options: $option\n" @@ -97,7 +89,7 @@ KEEP_CODE_OPTION="--keep-code" KEEP_DATA_OPTION="--keep-data" SYNC_MODE_OPTION="--fast-sync" CHOWN_DATA_OPTION="--no-chown-data" -KILL_OPTION="" +KILL_OPTION="--kill-job" ARG_INDEX=5 while [ $ARG_INDEX -le $# ]; do @@ -225,7 +217,7 @@ function inject_account() { # deploy files FILES_FOR_TRACKER="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ logger/ tracker-server/ traffic/ package.json setup_blockchain_ubuntu_gcp.sh start_tracker_genesis_gcp.sh start_tracker_incremental_gcp.sh" -FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync_gcp.sh stop_local_blockchain.sh" +FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync.sh stop_local_blockchain.sh" TRACKER_TARGET_ADDR="${GCP_USER}@${SEASON}-tracker-taiwan" NODE_0_TARGET_ADDR="${GCP_USER}@${SEASON}-node-0-taiwan" @@ -326,47 +318,43 @@ if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then fi fi -if [[ $KILL_OPTION = "--skip-kill" ]]; then - printf "\nSkipping process kill...\n" -else - # kill any processes still alive - printf "\nKilling tracker / blockchain node jobs...\n" - - # Tracker server is killed with PARENT_NODE_INDEX_BEGIN = -1 - if [[ $PARENT_NODE_INDEX_BEGIN = -1 ]]; then - printf "\n* >> Killing tracker job (${TRACKER_TARGET_ADDR}) *********************************************************\n\n" - gcloud compute ssh $TRACKER_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $TRACKER_ZONE - fi - - begin_index=$PARENT_NODE_INDEX_BEGIN - if [[ $begin_index -lt 0 ]]; then - begin_index=0 - fi - if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then - for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do - NODE_TARGET_ADDR=NODE_${node_index}_TARGET_ADDR - NODE_ZONE=NODE_${node_index}_ZONE +# kill any processes still alive +printf "\nKilling tracker / blockchain node jobs...\n" - printf "\n* >> Killing node $node_index job (${!NODE_TARGET_ADDR}) *********************************************************\n\n" - gcloud compute ssh ${!NODE_TARGET_ADDR} --command "sudo killall node" --project $PROJECT_ID --zone ${!NODE_ZONE} - done - fi +# Tracker server is killed with PARENT_NODE_INDEX_BEGIN = -1 +if [[ $PARENT_NODE_INDEX_BEGIN = -1 ]]; then + printf "\n* >> Killing tracker job (${TRACKER_TARGET_ADDR}) *********************************************************\n\n" + gcloud compute ssh $TRACKER_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $TRACKER_ZONE +fi - if [[ $NUM_SHARDS -gt 0 ]]; then - for i in $(seq $NUM_SHARDS); do - printf "shard #$i\n" +begin_index=$PARENT_NODE_INDEX_BEGIN +if [[ $begin_index -lt 0 ]]; then + begin_index=0 +fi +if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then + for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do + NODE_TARGET_ADDR=NODE_${node_index}_TARGET_ADDR + NODE_ZONE=NODE_${node_index}_ZONE - SHARD_TRACKER_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-tracker-taiwan" - SHARD_NODE_0_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-0-taiwan" - SHARD_NODE_1_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-1-oregon" - SHARD_NODE_2_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-2-singapore" + printf "\n* >> Killing node $node_index job (${!NODE_TARGET_ADDR}) *********************************************************\n\n" + gcloud compute ssh ${!NODE_TARGET_ADDR} --command "sudo killall node" --project $PROJECT_ID --zone ${!NODE_ZONE} + done +fi - gcloud compute ssh $SHARD_TRACKER_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $TRACKER_ZONE - gcloud compute ssh $SHARD_NODE_0_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_0_ZONE - gcloud compute ssh $SHARD_NODE_1_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_1_ZONE - gcloud compute ssh $SHARD_NODE_2_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_2_ZONE - done - fi +if [[ $NUM_SHARDS -gt 0 ]]; then + for i in $(seq $NUM_SHARDS); do + printf "shard #$i\n" + + SHARD_TRACKER_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-tracker-taiwan" + SHARD_NODE_0_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-0-taiwan" + SHARD_NODE_1_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-1-oregon" + SHARD_NODE_2_TARGET_ADDR="${GCP_USER}@${SEASON}-shard-${i}-node-2-singapore" + + gcloud compute ssh $SHARD_TRACKER_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $TRACKER_ZONE + gcloud compute ssh $SHARD_NODE_0_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_0_ZONE + gcloud compute ssh $SHARD_NODE_1_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_1_ZONE + gcloud compute ssh $SHARD_NODE_2_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $NODE_2_ZONE + done fi # If --kill-only, do not proceed any further @@ -446,9 +434,8 @@ if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -g printf "REST_FUNC_OPTION=$REST_FUNC_OPTION\n" printf "EVENT_HANDLER_OPTION=$EVENT_HANDLER_OPTION\n" - printf "\n" START_NODE_CMD="gcloud compute ssh ${!NODE_TARGET_ADDR} --command '$START_NODE_CMD_BASE $SEASON $GCP_USER 0 $node_index $KEEP_CODE_OPTION $KEEP_DATA_OPTION $SYNC_MODE_OPTION $CHOWN_DATA_OPTION $ACCOUNT_INJECTION_OPTION $JSON_RPC_OPTION $UPDATE_FRONT_DB_OPTION $REST_FUNC_OPTION $EVENT_HANDLER_OPTION' --project $PROJECT_ID --zone ${!NODE_ZONE}" - printf "START_NODE_CMD=$START_NODE_CMD\n" + printf "\nSTART_NODE_CMD=$START_NODE_CMD\n" eval $START_NODE_CMD sleep 5 inject_account "$node_index" diff --git a/deploy_blockchain_genesis_onprem.sh b/deploy_blockchain_genesis_onprem.sh index 5272966f0..10ba48735 100644 --- a/deploy_blockchain_genesis_onprem.sh +++ b/deploy_blockchain_genesis_onprem.sh @@ -1,33 +1,25 @@ #!/bin/bash if [[ $# -lt 4 ]] || [[ $# -gt 10 ]]; then - printf "Usage: bash deploy_blockchain_genesis_onprem.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-only|--skip-kill]\n" - printf "Example: bash deploy_blockchain_genesis_onprem.sh dev 0 -1 4 --keystore --no-keep-code\n" - printf "Example: bash deploy_blockchain_genesis_onprem.sh dev 0 0 0 --keystore --keep-code\n" - printf "Example: bash deploy_blockchain_genesis_onprem.sh dev 0 -1 -1 --setup --keystore --no-keep-code\n" - printf "Example: bash deploy_blockchain_genesis_onprem.sh dev 0 0 0 --setup --keystore --no-keep-code\n" - printf "Note: = -1 is for tracker\n" + printf "Usage: bash deploy_blockchain_genesis_onprem.sh [staging|spring|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only]\n" + printf "Example: bash deploy_blockchain_genesis_onprem.sh staging 0 0 4 --keystore --no-keep-code\n" + printf "Example: bash deploy_blockchain_genesis_onprem.sh staging 0 0 0 --keystore --keep-code\n" + #printf "Example: bash deploy_blockchain_genesis_onprem.sh staging 0 -1 -1 --setup --keystore --no-keep-code\n" + printf "Example: bash deploy_blockchain_genesis_onprem.sh staging 0 0 0 --setup --keystore --no-keep-code\n" + #printf "Note: = -1 is for tracker\n" printf "Note: is inclusive\n" printf "\n" exit fi printf "\n[[[[[ deploy_blockchain_genesis_onprem.sh ]]]]]\n\n" -if [[ "$1" = 'dev' ]] || [[ "$1" = 'staging' ]] || [[ "$1" = 'sandbox' ]] || [[ "$1" = 'exp' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'summer' ]] || [[ "$1" = 'mainnet' ]]; then +if [[ "$1" = 'staging' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'mainnet' ]]; then SEASON="$1" - if [[ "$1" = 'mainnet' ]]; then - PROJECT_ID="mainnet-prod-ground" - elif [[ "$1" = 'spring' ]] || [[ "$1" = 'summer' ]]; then - PROJECT_ID="testnet-prod-ground" - else - PROJECT_ID="testnet-$1-ground" - fi else printf "Invalid project/season argument: $1\n" exit fi printf "SEASON=$SEASON\n" -printf "PROJECT_ID=$PROJECT_ID\n" ONPREM_USER="nvidia" printf "ONPREM_USER=$ONPREM_USER\n" @@ -37,10 +29,16 @@ if [[ ! $2 =~ $number_re ]] ; then printf "Invalid <# of Shards> argument: $2\n" exit fi -NUM_SHARDS=$2 -printf "NUM_SHARDS=$NUM_SHARDS\n" +if [[ $3 -lt 0 ]] || [[ $3 -gt 4 ]]; then + printf "Invalid argument: $3\n" + exit +fi PARENT_NODE_INDEX_BEGIN=$3 printf "PARENT_NODE_INDEX_BEGIN=$PARENT_NODE_INDEX_BEGIN\n" +if [[ $4 -lt 0 ]] || [[ $4 -gt 4 ]]; then + printf "Invalid argument: $4\n" + exit +fi PARENT_NODE_INDEX_END=$4 printf "PARENT_NODE_INDEX_END=$PARENT_NODE_INDEX_END\n" printf "\n" @@ -72,17 +70,9 @@ function parse_options() { CHOWN_DATA_OPTION="$option" elif [[ $option = '--no-chown-data' ]]; then CHOWN_DATA_OPTION="$option" - elif [[ $option = '--kill-only' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-job' ]]; then KILL_OPTION="$option" - elif [[ $option = '--skip-kill' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-only' ]]; then KILL_OPTION="$option" else printf "Invalid options: $option\n" @@ -97,7 +87,7 @@ KEEP_CODE_OPTION="--keep-code" KEEP_DATA_OPTION="--keep-data" SYNC_MODE_OPTION="--fast-sync" CHOWN_DATA_OPTION="--no-chown-data" -KILL_OPTION="" +KILL_OPTION="--kill-job" ARG_INDEX=5 while [ $ARG_INDEX -le $# ]; do @@ -110,6 +100,11 @@ if [[ $SETUP_OPTION = "--setup" ]] && [[ ! $KEEP_CODE_OPTION = "--no-keep-code" exit fi +if [[ $PARENT_NODE_INDEX_BEGIN -lt 0 ]]; then + printf "Please use deploy_blockchain_incremental_gcp.sh instead for the tracker job.\n" + exit +fi + printf "SETUP_OPTION=$SETUP_OPTION\n" printf "ACCOUNT_INJECTION_OPTION=$ACCOUNT_INJECTION_OPTION\n" printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" @@ -228,8 +223,8 @@ function inject_account() { } # deploy files -#FILES_FOR_TRACKER="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ logger/ tracker-server/ traffic/ package.json setup_blockchain_ubuntu_onprem.sh start_tracker_genesis_gcp.sh start_tracker_incremental_gcp.sh" -FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_onprem.sh start_node_genesis_onprem.sh start_node_incremental_gcp.sh wait_until_node_sync_gcp.sh stop_local_blockchain.sh" +#FILES_FOR_TRACKER="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ logger/ tracker-server/ traffic/ package.json setup_blockchain_ubuntu_onprem.sh start_tracker_genesis_onprem.sh start_tracker_incremental_onprem.sh" +FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_onprem.sh start_node_genesis_onprem.sh start_node_incremental_onprem.sh wait_until_node_sync.sh stop_local_blockchain.sh" printf "###############################################################################\n" printf "# Deploying parent blockchain #\n" @@ -252,11 +247,12 @@ if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n" - printf "NODE_TARGET_ADDR=${NODE_TARGET_ADDR}\n" - printf "\n* >> Deploying files for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" - sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) scp -rv $FILES_FOR_NODE ${NODE_TARGET_ADDR}:~/ain-blockchain/ + printf "\n\n* >> Deploying files for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" + printf "FILES_FOR_NODE=${FILES_FOR_NODE}\n\n" + + echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh $NODE_TARGET_ADDR "sudo -S rm -rf ~/ain-blockchain; mkdir ~/ain-blockchain; chmod -R 777 ~/ain-blockchain" + sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) scp -r $FILES_FOR_NODE ${NODE_TARGET_ADDR}:~/ain-blockchain/ done fi fi @@ -277,11 +273,10 @@ if [[ $SETUP_OPTION = "--setup" ]]; then for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n" - printf "NODE_TARGET_ADDR=${NODE_TARGET_ADDR}\n" - printf "\n* >> Setting up parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" - echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh -v ${NODE_TARGET_ADDR} "cd ./ain-blockchain; . setup_blockchain_ubuntu_onprem.sh" + printf "\n\n* >> Setting up parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" + + echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh ${NODE_TARGET_ADDR} "cd ./ain-blockchain; . setup_blockchain_ubuntu_onprem.sh" done fi fi @@ -302,20 +297,16 @@ if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n" - printf "NODE_TARGET_ADDR=${NODE_TARGET_ADDR}\n" - printf "\n* >> Installing node modules for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" - sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh -v ${NODE_TARGET_ADDR} "cd ./ain-blockchain; yarn install --ignore-engines" + printf "\n\n* >> Installing node modules for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" + + sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh ${NODE_TARGET_ADDR} "cd ./ain-blockchain; yarn install --ignore-engines" done fi fi -if [[ $KILL_OPTION = "--skip-kill" ]]; then - printf "\nSkipping process kill...\n" -else - # kill any processes still alive - printf "\nKilling tracker / blockchain node jobs...\n" +# kill any processes still alive +printf "\n\nKilling tracker / blockchain node jobs...\n" # # Tracker server is killed with PARENT_NODE_INDEX_BEGIN = -1 # if [[ $PARENT_NODE_INDEX_BEGIN = -1 ]]; then @@ -323,21 +314,19 @@ else # gcloud compute ssh $TRACKER_TARGET_ADDR --command "sudo killall node" --project $PROJECT_ID --zone $TRACKER_ZONE # fi - begin_index=$PARENT_NODE_INDEX_BEGIN - if [[ $begin_index -lt 0 ]]; then - begin_index=0 - fi - if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then - for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do - NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" - NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n" - printf "NODE_TARGET_ADDR=${NODE_TARGET_ADDR}\n" +begin_index=$PARENT_NODE_INDEX_BEGIN +if [[ $begin_index -lt 0 ]]; then + begin_index=0 +fi +if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then + for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do + NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" + NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n* >> Killing node $node_index job (${NODE_TARGET_ADDR}) *********************************************************\n\n" - echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh -v ${NODE_TARGET_ADDR} "sudo -S killall node" - done - fi + printf "\n* >> Killing node $node_index job (${NODE_TARGET_ADDR}) *********************************************************\n\n" + + echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh ${NODE_TARGET_ADDR} "sudo -S pkill -f client/${SEASON}-ain-blockchain-index.js" + done fi # If --kill-only, do not proceed any further @@ -345,14 +334,14 @@ if [[ $KILL_OPTION = "--kill-only" ]]; then exit fi -printf "\nStarting blockchain servers...\n\n" +printf "\n\nStarting blockchain servers...\n\n" if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then GO_TO_PROJECT_ROOT_CMD="cd ./ain-blockchain" else - GO_TO_PROJECT_ROOT_CMD="cd \$(find /home/ain-blockchain* -maxdepth 0 -type d)" + GO_TO_PROJECT_ROOT_CMD="cd \$(find /home/${SEASON}/ain-blockchain* -maxdepth 0 -type d)" fi -#START_TRACKER_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_tracker_genesis_gcp.sh" +#START_TRACKER_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_tracker_genesis_onprem.sh" START_NODE_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_node_genesis_onprem.sh" printf "\n" #printf "START_TRACKER_CMD_BASE=$START_TRACKER_CMD_BASE\n" @@ -378,19 +367,17 @@ if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -g for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do NODE_TARGET_ADDR="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" NODE_LOGIN_PW="${NODE_PW_LIST[${node_index}]}" - printf "\n" - printf "NODE_TARGET_ADDR=${NODE_TARGET_ADDR}\n" if [[ $KEEP_DATA_OPTION = "--no-keep-data" ]]; then - printf "\n* >> Removing old data for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" + printf "\n\n* >> Removing old data for parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" - CHAINS_DIR=/home/ain_blockchain_data/chains - SNAPSHOTS_DIR=/home/ain_blockchain_data/snapshots - LOGS_DIR=/home/ain_blockchain_data/logs - echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh -v ${NODE_TARGET_ADDR} "sudo -S rm -rf $CHAINS_DIR $SNAPSHOTS_DIR $LOGS_DIR" + CHAINS_DIR=/home/${SEASON}/ain_blockchain_data/chains + SNAPSHOTS_DIR=/home/${SEASON}/ain_blockchain_data/snapshots + LOGS_DIR=/home/${SEASON}/ain_blockchain_data/logs + echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ssh ${NODE_TARGET_ADDR} "sudo -S rm -rf $CHAINS_DIR $SNAPSHOTS_DIR $LOGS_DIR" fi - printf "\n* >> Starting parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" + printf "\n\n* >> Starting parent node $node_index (${NODE_TARGET_ADDR}) *********************************************************\n\n" if [[ $node_index -ge $JSON_RPC_NODE_INDEX_GE ]] && [[ $node_index -le $JSON_RPC_NODE_INDEX_LE ]]; then JSON_RPC_OPTION="--json-rpc" @@ -419,9 +406,8 @@ if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -g printf "REST_FUNC_OPTION=$REST_FUNC_OPTION\n" printf "EVENT_HANDLER_OPTION=$EVENT_HANDLER_OPTION\n" - printf "\n" - START_NODE_CMD="ssh -v ${NODE_TARGET_ADDR} '$START_NODE_CMD_BASE $SEASON $ONPREM_USER 0 $node_index $KEEP_CODE_OPTION $KEEP_DATA_OPTION $SYNC_MODE_OPTION $CHOWN_DATA_OPTION $ACCOUNT_INJECTION_OPTION $JSON_RPC_OPTION $UPDATE_FRONT_DB_OPTION $REST_FUNC_OPTION $EVENT_HANDLER_OPTION'" - printf "START_NODE_CMD=$START_NODE_CMD\n" + START_NODE_CMD="ssh ${NODE_TARGET_ADDR} '$START_NODE_CMD_BASE $SEASON $ONPREM_USER 0 $node_index $KEEP_CODE_OPTION $KEEP_DATA_OPTION $SYNC_MODE_OPTION $CHOWN_DATA_OPTION $ACCOUNT_INJECTION_OPTION $JSON_RPC_OPTION $UPDATE_FRONT_DB_OPTION $REST_FUNC_OPTION $EVENT_HANDLER_OPTION'" + printf "\nSTART_NODE_CMD=$START_NODE_CMD\n" eval "echo ${NODE_LOGIN_PW} | sshpass -f <(printf '%s\n' ${NODE_LOGIN_PW}) ${START_NODE_CMD}" sleep 5 inject_account "$node_index" diff --git a/deploy_blockchain_incremental_gcp.sh b/deploy_blockchain_incremental_gcp.sh index 0c0e71165..4fa960546 100644 --- a/deploy_blockchain_incremental_gcp.sh +++ b/deploy_blockchain_incremental_gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash -if [[ $# -lt 4 ]] || [[ $# -gt 11 ]]; then - printf "Usage: bash deploy_blockchain_incremental_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data]\n" +if [[ $# -lt 4 ]] || [[ $# -gt 12 ]]; then + printf "Usage: bash deploy_blockchain_incremental_gcp.sh [dev|staging|sandbox|exp|spring|summer|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only]\n" printf "Example: bash deploy_blockchain_incremental_gcp.sh dev 0 -1 4 --keystore --no-keep-code\n" printf "Example: bash deploy_blockchain_incremental_gcp.sh dev 0 0 0 --keystore --keep-code\n" printf "Example: bash deploy_blockchain_incremental_gcp.sh dev 0 -1 -1 --setup --keystore --no-keep-code\n" @@ -71,6 +71,10 @@ function parse_options() { CHOWN_DATA_OPTION="$option" elif [[ $option = '--no-chown-data' ]]; then CHOWN_DATA_OPTION="$option" + elif [[ $option = '--kill-job' ]]; then + KILL_OPTION="$option" + elif [[ $option = '--kill-only' ]]; then + KILL_OPTION="$option" else printf "Invalid option: $option\n" exit @@ -84,6 +88,7 @@ KEEP_CODE_OPTION="--keep-code" KEEP_DATA_OPTION="--keep-data" SYNC_MODE_OPTION="--fast-sync" CHOWN_DATA_OPTION="--no-chown-data" +KILL_OPTION="--kill-job" ARG_INDEX=5 while [ $ARG_INDEX -le $# ]; do @@ -102,6 +107,7 @@ printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" printf "KEEP_DATA_OPTION=$KEEP_DATA_OPTION\n" printf "SYNC_MODE_OPTION=$SYNC_MODE_OPTION\n" printf "CHOWN_DATA_OPTION=$CHOWN_DATA_OPTION\n" +printf "KILL_OPTION=$KILL_OPTION\n" # Json-RPC-enabled blockchain nodes JSON_RPC_NODE_INDEX_GE=0 @@ -153,26 +159,28 @@ else fi fi -# Read node urls -IFS=$'\n' read -d '' -r -a NODE_URL_LIST < ./ip_addresses/$SEASON.txt -if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then - # Get keystore password - printf "Enter keystore password: " - read -s KEYSTORE_PW - printf "\n\n" - if [[ $SEASON = "mainnet" ]]; then - KEYSTORE_DIR="mainnet_prod_keys" - elif [[ $SEASON = "spring" ]] || [[ $SEASON = "summer" ]]; then - KEYSTORE_DIR="testnet_prod_keys" - else - KEYSTORE_DIR="testnet_dev_staging_keys" +if [[ ! $KILL_OPTION = '--kill-only' ]]; then + # Read node urls + IFS=$'\n' read -d '' -r -a NODE_URL_LIST < ./ip_addresses/$SEASON.txt + if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then + # Get keystore password + printf "Enter keystore password: " + read -s KEYSTORE_PW + printf "\n\n" + if [[ $SEASON = "mainnet" ]]; then + KEYSTORE_DIR="mainnet_prod_keys" + elif [[ $SEASON = "spring" ]] || [[ $SEASON = "summer" ]]; then + KEYSTORE_DIR="testnet_prod_keys" + else + KEYSTORE_DIR="testnet_dev_staging_keys" + fi + elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then + IFS=$'\n' read -d '' -r -a MNEMONIC_LIST < ./testnet_mnemonics/$SEASON.txt fi -elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then - IFS=$'\n' read -d '' -r -a MNEMONIC_LIST < ./testnet_mnemonics/$SEASON.txt fi FILES_FOR_TRACKER="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ logger/ tracker-server/ traffic/ package.json setup_blockchain_ubuntu_gcp.sh start_tracker_genesis_gcp.sh start_tracker_incremental_gcp.sh" -FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync_gcp.sh stop_local_blockchain.sh" +FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync.sh stop_local_blockchain.sh" NUM_SHARD_NODES=3 @@ -196,8 +204,19 @@ function deploy_tracker() { printf "TRACKER_TARGET_ADDR='$TRACKER_TARGET_ADDR'\n" printf "TRACKER_ZONE='$TRACKER_ZONE'\n" + # 0. Kill jobs for tracker (if necessary) + if [[ $KILL_OPTION = "--kill-only" ]]; then + printf "\n\n<<< Killing tracker job (${TRACKER_TARGET_ADDR}) *********************************************************\n\n" + + KILL_CMD="gcloud compute ssh $TRACKER_TARGET_ADDR --command 'sudo killall node' --project $PROJECT_ID --zone $TRACKER_ZONE" + printf "KILL_CMD=$KILL_CMD\n\n" + eval $KILL_CMD + + return 0 + fi + + # 1. Copy files for tracker (if necessary) if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then - # 1. Copy files for tracker printf "\n\n[[[ Copying files for tracker ]]]\n\n" gcloud compute ssh $TRACKER_TARGET_ADDR --command "sudo rm -rf ~/ain-blockchain; sudo mkdir ~/ain-blockchain; sudo chmod -R 777 ~/ain-blockchain" --project $PROJECT_ID --zone $TRACKER_ZONE SCP_CMD="gcloud compute scp --recurse $FILES_FOR_TRACKER ${TRACKER_TARGET_ADDR}:~/ain-blockchain --project $PROJECT_ID --zone $TRACKER_ZONE" @@ -205,9 +224,9 @@ function deploy_tracker() { eval $SCP_CMD fi + # 2. Set up tracker (if necessary) # ssh into each instance, set up the ubuntu VM instance (ONLY NEEDED FOR THE FIRST TIME) if [[ $SETUP_OPTION = "--setup" ]]; then - # 2. Set up tracker printf "\n\n[[[ Setting up tracker ]]]\n\n" SETUP_CMD="gcloud compute ssh $TRACKER_TARGET_ADDR --command 'cd ./ain-blockchain; . setup_blockchain_ubuntu_gcp.sh' --project $PROJECT_ID --zone $TRACKER_ZONE" printf "SETUP_CMD=$SETUP_CMD\n\n" @@ -230,13 +249,21 @@ function deploy_node() { local node_target_addr=${NODE_TARGET_ADDR_LIST[${node_index}]} local node_zone=${NODE_ZONE_LIST[${node_index}]} - printf "\n* >> Deploying files for node $node_index ($node_target_addr) *********************************************************\n\n" + printf "\n\n* >> Deploying files for node $node_index ($node_target_addr) *********************************************************\n\n" + + # 0. Kill jobs for node (if necessary) + if [[ $KILL_OPTION = "--kill-only" ]]; then + printf "\n\n<<< Killing node $node_index job (${node_target_addr}) *********************************************************\n\n" - printf "node_target_addr='$node_target_addr'\n" - printf "node_zone='$node_zone'\n" + KILL_CMD="gcloud compute ssh $node_target_addr --command 'sudo killall node' --project $PROJECT_ID --zone $node_zone" + printf "\n\nKILL_CMD=$KILL_CMD\n\n" + eval $KILL_CMD + + return 0 + fi + # 1. Copy files for node (if necessary) if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then - # 1. Copy files for node printf "\n\n<<< Copying files for node $node_index >>>\n\n" gcloud compute ssh $node_target_addr --command "sudo rm -rf ~/ain-blockchain; sudo mkdir ~/ain-blockchain; sudo chmod -R 777 ~/ain-blockchain" --project $PROJECT_ID --zone $node_zone SCP_CMD="gcloud compute scp --recurse $FILES_FOR_NODE ${node_target_addr}:~/ain-blockchain --project $PROJECT_ID --zone $node_zone" @@ -244,9 +271,9 @@ function deploy_node() { eval $SCP_CMD fi + # 2. Set up node (if necessary) # ssh into each instance, set up the ubuntu VM instance (ONLY NEEDED FOR THE FIRST TIME) if [[ $SETUP_OPTION = "--setup" ]]; then - # 2. Set up node printf "\n\n<<< Setting up node $node_index >>>\n\n" SETUP_CMD="gcloud compute ssh $node_target_addr --command 'cd ./ain-blockchain; . setup_blockchain_ubuntu_gcp.sh' --project $PROJECT_ID --zone $node_zone" printf "SETUP_CMD=$SETUP_CMD\n\n" @@ -283,16 +310,15 @@ function deploy_node() { printf "REST_FUNC_OPTION=$REST_FUNC_OPTION\n" printf "EVENT_HANDLER_OPTION=$EVENT_HANDLER_OPTION\n" - printf "\n" START_NODE_CMD="gcloud compute ssh $node_target_addr --command '$START_NODE_CMD_BASE $SEASON $GCP_USER 0 $node_index $KEEP_CODE_OPTION $KEEP_DATA_OPTION $SYNC_MODE_OPTION $CHOWN_DATA_OPTION $ACCOUNT_INJECTION_OPTION $JSON_RPC_OPTION $UPDATE_FRONT_DB_OPTION $REST_FUNC_OPTION $EVENT_HANDLER_OPTION' --project $PROJECT_ID --zone $node_zone" - printf "START_NODE_CMD=$START_NODE_CMD\n\n" + printf "\n\nSTART_NODE_CMD=$START_NODE_CMD\n\n" eval $START_NODE_CMD # 4. Inject node account sleep 5 if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then local node_url=${NODE_URL_LIST[${node_index}]} - printf "\n* >> Initializing account for node $node_index ($node_target_addr) ********************\n\n" + printf "\n\n* >> Initializing account for node $node_index ($node_target_addr) ********************\n\n" printf "node_url='$node_url'\n" KEYSTORE_FILE_PATH="$KEYSTORE_DIR/keystore_node_$node_index.json" @@ -304,7 +330,7 @@ function deploy_node() { elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then local node_url=${NODE_URL_LIST[${node_index}]} local MNEMONIC=${MNEMONIC_LIST[${node_index}]} - printf "\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" + printf "\n\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" printf "node_url='$node_url'\n" { @@ -314,7 +340,7 @@ function deploy_node() { } | node inject_node_account.js $node_url $ACCOUNT_INJECTION_OPTION else local node_url=${NODE_URL_LIST[${node_index}]} - printf "\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" + printf "\n\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" printf "node_url='$node_url'\n" local GENESIS_ACCOUNTS_PATH="blockchain-configs/base/genesis_accounts.json" if [[ "$SEASON" = "spring" ]] || [[ "$SEASON" = "summer" ]]; then @@ -326,7 +352,7 @@ function deploy_node() { # 5. Wait until node is synced printf "\n\n<<< Waiting until node $node_index is synced >>>\n\n" - WAIT_CMD="gcloud compute ssh $node_target_addr --command 'cd \$(find /home/ain-blockchain* -maxdepth 0 -type d); . wait_until_node_sync_gcp.sh' --project $PROJECT_ID --zone $node_zone" + WAIT_CMD="gcloud compute ssh $node_target_addr --command 'cd \$(find /home/ain-blockchain* -maxdepth 0 -type d); . wait_until_node_sync.sh $node_url' --project $PROJECT_ID --zone $node_zone" printf "WAIT_CMD=$WAIT_CMD\n\n" eval $WAIT_CMD } @@ -355,18 +381,8 @@ if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then else GO_TO_PROJECT_ROOT_CMD="cd \$(find /home/ain-blockchain* -maxdepth 0 -type d)" fi -if [[ $KEEP_DATA_OPTION = "--no-keep-data" ]]; then - # restart after removing chains, snapshots, and log files (but keep the keys) - CHAINS_DIR=/home/ain_blockchain_data/chains - SNAPSHOTS_DIR=/home/ain_blockchain_data/snapshots - LOGS_DIR=/home/ain_blockchain_data/logs - START_TRACKER_CMD_BASE="sudo rm -rf /home/ain_blockchain_data/ && $GO_TO_PROJECT_ROOT_CMD && . start_tracker_incremental_gcp.sh" - START_NODE_CMD_BASE="sudo rm -rf $CHAINS_DIR $SNAPSHOTS_DIR $LOGS_DIR && $GO_TO_PROJECT_ROOT_CMD && . start_node_incremental_gcp.sh" -else - # restart with existing chains, snapshots, and log files - START_TRACKER_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_tracker_incremental_gcp.sh" - START_NODE_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_node_incremental_gcp.sh" -fi +START_TRACKER_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_tracker_incremental_gcp.sh" +START_NODE_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_node_incremental_gcp.sh" # Tracker server is deployed with PARENT_NODE_INDEX_BEGIN = -1 if [[ $PARENT_NODE_INDEX_BEGIN = -1 ]]; then @@ -379,7 +395,9 @@ fi if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do deploy_node "$node_index" - sleep 40 + if [[ ! $KILL_OPTION = "--kill-only" ]]; then + sleep 40 + fi done fi diff --git a/deploy_blockchain_incremental_onprem.sh b/deploy_blockchain_incremental_onprem.sh new file mode 100644 index 000000000..fa335cb40 --- /dev/null +++ b/deploy_blockchain_incremental_onprem.sh @@ -0,0 +1,384 @@ +#!/bin/bash + +if [[ $# -lt 4 ]] || [[ $# -gt 12 ]]; then + printf "Usage: bash deploy_blockchain_incremental_onprem.sh [staging|spring|mainnet] <# of Shards> [--setup] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--kill-job|--kill-only]\n" + printf "Example: bash deploy_blockchain_incremental_onprem.sh staging 0 0 4 --keystore --no-keep-code\n" + printf "Example: bash deploy_blockchain_incremental_onprem.sh staging 0 0 0 --keystore --keep-code\n" + #printf "Example: bash deploy_blockchain_incremental_onprem.sh staging 0 -1 -1 --setup --keystore --no-keep-code\n" + printf "Example: bash deploy_blockchain_incremental_onprem.sh staging 0 0 0 --setup --keystore --no-keep-code\n" + #printf "Note: = -1 is for tracker\n" + printf "Note: is inclusive\n" + printf "\n" + exit +fi +printf "\n[[[[[ deploy_blockchain_incremental_onprem.sh ]]]]]\n\n" + +if [[ "$1" = 'staging' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid argument: $1\n" + exit +fi +printf "SEASON=$SEASON\n" + +ONPREM_USER="nvidia" +printf "ONPREM_USER=$ONPREM_USER\n" + +number_re='^[0-9]+$' +if [[ ! $2 =~ $number_re ]] ; then + printf "Invalid <# of Shards> argument: $2\n" + exit +fi +if [[ $3 -lt 0 ]] || [[ $3 -gt 4 ]]; then + printf "Invalid argument: $3\n" + exit +fi +PARENT_NODE_INDEX_BEGIN=$3 +printf "PARENT_NODE_INDEX_BEGIN=$PARENT_NODE_INDEX_BEGIN\n" +if [[ $4 -lt 0 ]] || [[ $4 -gt 4 ]]; then + printf "Invalid argument: $4\n" + exit +fi +PARENT_NODE_INDEX_END=$4 +printf "PARENT_NODE_INDEX_END=$PARENT_NODE_INDEX_END\n" +printf "\n" + +function parse_options() { + local option="$1" + if [[ $option = '--setup' ]]; then + SETUP_OPTION="$option" + elif [[ $option = '--private-key' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--keystore' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--mnemonic' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--keep-code' ]]; then + KEEP_CODE_OPTION="$option" + elif [[ $option = '--no-keep-code' ]]; then + KEEP_CODE_OPTION="$option" + elif [[ $option = '--keep-data' ]]; then + KEEP_DATA_OPTION="$option" + elif [[ $option = '--no-keep-data' ]]; then + KEEP_DATA_OPTION="$option" + elif [[ $option = '--full-sync' ]]; then + SYNC_MODE_OPTION="$option" + elif [[ $option = '--fast-sync' ]]; then + SYNC_MODE_OPTION="$option" + elif [[ $option = '--chown-data' ]]; then + CHOWN_DATA_OPTION="$option" + elif [[ $option = '--no-chown-data' ]]; then + CHOWN_DATA_OPTION="$option" + elif [[ $option = '--kill-job' ]]; then + KILL_OPTION="$option" + elif [[ $option = '--kill-only' ]]; then + KILL_OPTION="$option" + else + printf "Invalid option: $option\n" + exit + fi +} + +# Parse options. +SETUP_OPTION="" +ACCOUNT_INJECTION_OPTION="--private-key" +KEEP_CODE_OPTION="--keep-code" +KEEP_DATA_OPTION="--keep-data" +SYNC_MODE_OPTION="--fast-sync" +CHOWN_DATA_OPTION="--no-chown-data" +KILL_OPTION="--kill-job" + +ARG_INDEX=5 +while [ $ARG_INDEX -le $# ]; do + parse_options "${!ARG_INDEX}" + ((ARG_INDEX++)) +done + +if [[ $SETUP_OPTION = "--setup" ]] && [[ ! $KEEP_CODE_OPTION = "--no-keep-code" ]]; then + printf "You cannot use --setup without --no-keep-code\n" + exit +fi + +if [[ $PARENT_NODE_INDEX_BEGIN -lt 0 ]]; then + printf "Please use deploy_blockchain_incremental_gcp.sh instead for the tracker job.\n" + exit +fi + +printf "SETUP_OPTION=$SETUP_OPTION\n" +printf "ACCOUNT_INJECTION_OPTION=$ACCOUNT_INJECTION_OPTION\n" +printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" +printf "KEEP_DATA_OPTION=$KEEP_DATA_OPTION\n" +printf "SYNC_MODE_OPTION=$SYNC_MODE_OPTION\n" +printf "CHOWN_DATA_OPTION=$CHOWN_DATA_OPTION\n" +printf "KILL_OPTION=$KILL_OPTION\n" + +# Json-RPC-enabled blockchain nodes +JSON_RPC_NODE_INDEX_GE=0 +JSON_RPC_NODE_INDEX_LE=4 +# Rest-Function-enabled blockchain nodes +REST_FUNC_NODE_INDEX_GE=0 +REST_FUNC_NODE_INDEX_LE=2 +# Event-Handler-enabled blockchain nodes +EVENT_HANDLER_NODE_INDEX_GE=0 +EVENT_HANDLER_NODE_INDEX_LE=4 + +printf "\n" +printf "JSON_RPC_NODE_INDEX_GE=$JSON_RPC_NODE_INDEX_GE\n" +printf "JSON_RPC_NODE_INDEX_LE=$JSON_RPC_NODE_INDEX_LE\n" +printf "REST_FUNC_NODE_INDEX_GE=$REST_FUNC_NODE_INDEX_GE\n" +printf "REST_FUNC_NODE_INDEX_LE=$REST_FUNC_NODE_INDEX_LE\n" +printf "EVENT_HANDLER_NODE_INDEX_GE=$EVENT_HANDLER_NODE_INDEX_GE\n" +printf "EVENT_HANDLER_NODE_INDEX_LE=$EVENT_HANDLER_NODE_INDEX_LE\n" + +if [[ "$ACCOUNT_INJECTION_OPTION" = "" ]]; then + printf "Must provide an ACCOUNT_INJECTION_OPTION\n" + exit +fi + +# Get confirmation. +if [[ "$SEASON" = "mainnet" ]]; then + printf "\n" + printf "Do you want to proceed for $SEASON? Enter [mainnet]: " + read CONFIRM + printf "\n" + if [[ ! $CONFIRM = "mainnet" ]] + then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +elif [[ "$SEASON" = "spring" ]] || [[ "$SEASON" = "summer" ]]; then + printf "\n" + printf "Do you want to proceed for $SEASON? Enter [testnet]: " + read CONFIRM + printf "\n" + if [[ ! $CONFIRM = "testnet" ]]; then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +else + printf "\n" + read -p "Do you want to proceed for $SEASON? [y/N]: " -n 1 -r + printf "\n\n" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell + fi +fi + +# Read node ip addresses and passwords +IFS=$'\n' read -d '' -r -a NODE_IP_LIST < ./ip_addresses/${SEASON}_onprem_ip.txt +IFS=$'\n' read -d '' -r -a NODE_PW_LIST < ./ip_addresses/${SEASON}_onprem_pw.txt + +if [[ ! $KILL_OPTION = '--kill-only' ]]; then + # Read node urls + IFS=$'\n' read -d '' -r -a NODE_URL_LIST < ./ip_addresses/${SEASON}_onprem.txt + if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then + # Get keystore password + printf "Enter keystore password: " + read -s KEYSTORE_PW + printf "\n\n" + if [[ $SEASON = "mainnet" ]]; then + KEYSTORE_DIR="mainnet_prod_keys" + elif [[ $SEASON = "spring" ]] || [[ $SEASON = "summer" ]]; then + KEYSTORE_DIR="testnet_prod_keys" + else + KEYSTORE_DIR="testnet_dev_staging_keys" + fi + elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then + IFS=$'\n' read -d '' -r -a MNEMONIC_LIST < ./testnet_mnemonics/$SEASON.txt + fi +fi + +#FILES_FOR_TRACKER="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ logger/ tracker-server/ traffic/ package.json setup_blockchain_ubuntu_onprem.sh start_tracker_genesis_onprem.sh start_tracker_incremental_onprem.sh" +FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_onprem.sh start_node_genesis_onprem.sh start_node_incremental_onprem.sh wait_until_node_sync.sh stop_local_blockchain.sh" + +#function deploy_tracker() { +# printf "\n* >> Deploying files for tracker ********************************************************\n\n" +# +# printf "TRACKER_TARGET_ADDR='$TRACKER_TARGET_ADDR'\n" +# printf "TRACKER_ZONE='$TRACKER_ZONE'\n" +# +# # 0. Kill jobs for tracker (if necessary) +# if [[ $KILL_OPTION = "--kill-only" ]]; then +# printf "\n\n<<< Killing tracker job (${TRACKER_TARGET_ADDR}) *********************************************************\n\n" +# +# KILL_CMD="gcloud compute ssh $TRACKER_TARGET_ADDR --command 'sudo killall node' --project $PROJECT_ID --zone $TRACKER_ZONE" +# printf "KILL_CMD=$KILL_CMD\n\n" +# eval $KILL_CMD +# +# return 0 +# fi +# +# # 1. Copy files for tracker (if necessary) +# if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then +# printf "\n\n[[[ Copying files for tracker ]]]\n\n" +# gcloud compute ssh $TRACKER_TARGET_ADDR --command "sudo rm -rf ~/ain-blockchain; sudo mkdir ~/ain-blockchain; sudo chmod -R 777 ~/ain-blockchain" --project $PROJECT_ID --zone $TRACKER_ZONE +# SCP_CMD="gcloud compute scp --recurse $FILES_FOR_TRACKER ${TRACKER_TARGET_ADDR}:~/ain-blockchain --project $PROJECT_ID --zone $TRACKER_ZONE" +# printf "SCP_CMD=$SCP_CMD\n\n" +# eval $SCP_CMD +# fi +# +# # 2. Set up tracker (if necessary) +# # ssh into each instance, set up the ubuntu VM instance (ONLY NEEDED FOR THE FIRST TIME) +# if [[ $SETUP_OPTION = "--setup" ]]; then +# printf "\n\n[[[ Setting up tracker ]]]\n\n" +# SETUP_CMD="gcloud compute ssh $TRACKER_TARGET_ADDR --command 'cd ./ain-blockchain; . setup_blockchain_ubuntu_onprem.sh' --project $PROJECT_ID --zone $TRACKER_ZONE" +# printf "SETUP_CMD=$SETUP_CMD\n\n" +# eval $SETUP_CMD +# fi +# +# # 3. Start tracker +# printf "\n\n[[[ Starting tracker ]]]\n\n" +# +# printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" +# +# printf "\n" +# START_TRACKER_CMD="gcloud compute ssh $TRACKER_TARGET_ADDR --command '$START_TRACKER_CMD_BASE $ONPREM_USER $KEEP_CODE_OPTION' --project $PROJECT_ID --zone $TRACKER_ZONE" +# printf "START_TRACKER_CMD=$START_TRACKER_CMD\n\n" +# eval $START_TRACKER_CMD +#} + +function deploy_node() { + local node_index="$1" + local node_target_addr="${ONPREM_USER}@${NODE_IP_LIST[${node_index}]}" + local node_login_pw="${NODE_PW_LIST[${node_index}]}" + + printf "\n\n* >> Deploying files for node $node_index ($node_target_addr) *********************************************************\n\n" + + # 0. Kill jobs for node (if necessary) + if [[ $KILL_OPTION = "--kill-only" ]]; then + printf "\n\n<<< Killing node $node_index job (${node_target_addr}) *********************************************************\n\n" + + KILL_CMD="ssh $node_target_addr 'sudo -S pkill -f client/${SEASON}-ain-blockchain-index.js'" + printf "\n\nKILL_CMD=$KILL_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${KILL_CMD}" + + return 0 + fi + + # 1. Copy files for node (if necessary) + if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then + printf "\n\n<<< Copying files for node $node_index ($node_target_addr) >>>\n\n" + + echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ssh $node_target_addr "sudo -S rm -rf ~/ain-blockchain; mkdir ~/ain-blockchain; chmod -R 777 ~/ain-blockchain" + SCP_CMD="scp -r $FILES_FOR_NODE ${node_target_addr}:~/ain-blockchain" + printf "\n\nSCP_CMD=$SCP_CMD\n\n" + eval "sshpass -f <(printf '%s\n' ${node_login_pw}) ${SCP_CMD}" + fi + + # 2. Set up node (if necessary) + # ssh into each instance, set up the ubuntu VM instance (ONLY NEEDED FOR THE FIRST TIME) + if [[ $SETUP_OPTION = "--setup" ]]; then + printf "\n\n<<< Setting up node $node_index ($node_target_addr) >>>\n\n" + + SETUP_CMD="ssh $node_target_addr 'cd ./ain-blockchain; . setup_blockchain_ubuntu_onprem.sh'" + printf "\n\nSETUP_CMD=$SETUP_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${SETUP_CMD}" + fi + + # 3. Start node + printf "\n\n<<< Starting node $node_index ($node_target_addr) >>>\n\n" + + if [[ $node_index -ge $JSON_RPC_NODE_INDEX_GE ]] && [[ $node_index -le $JSON_RPC_NODE_INDEX_LE ]]; then + JSON_RPC_OPTION="--json-rpc" + else + JSON_RPC_OPTION="" + fi + UPDATE_FRONT_DB_OPTION="--update-front-db" + if [[ $node_index -ge $REST_FUNC_NODE_INDEX_GE ]] && [[ $node_index -le $REST_FUNC_NODE_INDEX_LE ]]; then + REST_FUNC_OPTION="--rest-func" + else + REST_FUNC_OPTION="" + fi + if [[ $node_index -ge $EVENT_HANDLER_NODE_INDEX_GE ]] && [[ $node_index -le $EVENT_HANDLER_NODE_INDEX_LE ]]; then + EVENT_HANDLER_OPTION="--event-handler" + else + EVENT_HANDLER_OPTION="" + fi + + printf "ACCOUNT_INJECTION_OPTION=$ACCOUNT_INJECTION_OPTION\n" + printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" + printf "KEEP_DATA_OPTION=$KEEP_DATA_OPTION\n" + printf "SYNC_MODE_OPTION=$SYNC_MODE_OPTION\n" + printf "CHOWN_DATA_OPTION=$CHOWN_DATA_OPTION\n" + printf "JSON_RPC_OPTION=$JSON_RPC_OPTION\n" + printf "UPDATE_FRONT_DB_OPTION=$UPDATE_FRONT_DB_OPTION\n" + printf "REST_FUNC_OPTION=$REST_FUNC_OPTION\n" + printf "EVENT_HANDLER_OPTION=$EVENT_HANDLER_OPTION\n" + + START_NODE_CMD="ssh $node_target_addr '$START_NODE_CMD_BASE $SEASON $ONPREM_USER 0 $node_index $KEEP_CODE_OPTION $KEEP_DATA_OPTION $SYNC_MODE_OPTION $CHOWN_DATA_OPTION $ACCOUNT_INJECTION_OPTION $JSON_RPC_OPTION $UPDATE_FRONT_DB_OPTION $REST_FUNC_OPTION $EVENT_HANDLER_OPTION'" + printf "\n\nSTART_NODE_CMD=$START_NODE_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${START_NODE_CMD}" + + # 4. Inject node account + sleep 5 + if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then + local node_url=${NODE_URL_LIST[${node_index}]} + printf "\n\n* >> Initializing account for node $node_index ($node_target_addr) ********************\n\n" + printf "node_url='$node_url'\n" + + KEYSTORE_FILE_PATH="$KEYSTORE_DIR/keystore_node_$node_index.json" + { + echo $KEYSTORE_FILE_PATH + sleep 1 + echo $KEYSTORE_PW + } | node inject_node_account.js $node_url $ACCOUNT_INJECTION_OPTION + elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then + local node_url=${NODE_URL_LIST[${node_index}]} + local MNEMONIC=${MNEMONIC_LIST[${node_index}]} + printf "\n\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" + printf "node_url='$node_url'\n" + + { + echo $MNEMONIC + sleep 1 + echo 0 + } | node inject_node_account.js $node_url $ACCOUNT_INJECTION_OPTION + else + local node_url=${NODE_URL_LIST[${node_index}]} + printf "\n\n* >> Injecting an account for node $node_index ($node_target_addr) ********************\n\n" + printf "node_url='$node_url'\n" + + local GENESIS_ACCOUNTS_PATH="blockchain-configs/base/genesis_accounts.json" + if [[ "$SEASON" = "spring" ]] || [[ "$SEASON" = "summer" ]]; then + GENESIS_ACCOUNTS_PATH="blockchain-configs/testnet-prod/genesis_accounts.json" + fi + PRIVATE_KEY=$(cat $GENESIS_ACCOUNTS_PATH | jq -r '.others['$node_index'].private_key') + echo $PRIVATE_KEY | node inject_node_account.js $node_url $ACCOUNT_INJECTION_OPTION + fi + + # 5. Wait until node is synced + printf "\n\n<<< Waiting until node $node_index ($node_target_addr) is synced >>>\n\n" + + WAIT_CMD="ssh $node_target_addr 'cd \$(find /home/${SEASON}/ain-blockchain* -maxdepth 0 -type d); . wait_until_node_sync.sh $node_url'" + printf "\n\nWAIT_CMD=$WAIT_CMD\n\n" + eval "echo ${node_login_pw} | sshpass -f <(printf '%s\n' ${node_login_pw}) ${WAIT_CMD}" +} + +printf "###############################################################################\n" +printf "# Deploying parent blockchain #\n" +printf "###############################################################################\n\n" + +printf "\nStarting blockchain servers...\n\n" +if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then + GO_TO_PROJECT_ROOT_CMD="cd ./ain-blockchain" +else + GO_TO_PROJECT_ROOT_CMD="cd \$(find /home/${SEASON}/ain-blockchain* -maxdepth 0 -type d)" +fi + +#START_TRACKER_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_tracker_incremental_onprem.sh" +START_NODE_CMD_BASE="$GO_TO_PROJECT_ROOT_CMD && . start_node_incremental_onprem.sh" + +## Tracker server is deployed with PARENT_NODE_INDEX_BEGIN = -1 +#if [[ $PARENT_NODE_INDEX_BEGIN = -1 ]]; then +# deploy_tracker +#fi +begin_index=$PARENT_NODE_INDEX_BEGIN +if [[ $begin_index -lt 0 ]]; then + begin_index=0 +fi +if [[ $begin_index -le $PARENT_NODE_INDEX_END ]] && [[ $PARENT_NODE_INDEX_END -ge 0 ]]; then + for node_index in `seq $(( $begin_index )) $(( $PARENT_NODE_INDEX_END ))`; do + deploy_node "$node_index" + if [[ ! $KILL_OPTION = "--kill-only" ]]; then + sleep 40 + fi + done +fi diff --git a/deploy_blockchain_sandbox_gcp.sh b/deploy_blockchain_sandbox_gcp.sh index 3bec26142..bf66bb5ef 100644 --- a/deploy_blockchain_sandbox_gcp.sh +++ b/deploy_blockchain_sandbox_gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash if [[ $# -lt 2 ]] || [[ $# -gt 7 ]]; then - printf "Usage: bash deploy_blockchain_sandbox_gcp.sh <# start node> <# end node> [--setup] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--chown-data|--no-chown-data] [--kill-only|--skip-kill]\n" + printf "Usage: bash deploy_blockchain_sandbox_gcp.sh <# start node> <# end node> [--setup] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--chown-data|--no-chown-data] [--kill-job|--kill-only]\n" printf "Example: bash deploy_blockchain_sandbox_gcp.sh 10 99 --setup --no-chown-data\n" printf "\n" exit @@ -44,17 +44,9 @@ function parse_options() { CHOWN_DATA_OPTION="$option" elif [[ $option = '--no-chown-data' ]]; then CHOWN_DATA_OPTION="$option" - elif [[ $option = '--kill-only' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-job' ]]; then KILL_OPTION="$option" - elif [[ $option = '--skip-kill' ]]; then - if [[ "$KILL_OPTION" ]]; then - printf "You cannot use both --skip-kill and --kill-only\n" - exit - fi + elif [[ $option = '--kill-only' ]]; then KILL_OPTION="$option" else printf "Invalid options: $option\n" @@ -67,7 +59,7 @@ SETUP_OPTION="" KEEP_CODE_OPTION="--no-keep-code" KEEP_DATA_OPTION="--no-keep-data" CHOWN_DATA_OPTION="--no-chown-data" -KILL_OPTION="" +KILL_OPTION="--kill-job" ARG_INDEX=3 while [ $ARG_INDEX -le $# ]; do @@ -314,7 +306,7 @@ NODE_98_ZONE="us-central1-a" NODE_99_ZONE="europe-west4-a" # deploy files -FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync_gcp.sh" +FILES_FOR_NODE="blockchain/ blockchain-configs/ block-pool/ client/ common/ consensus/ db/ event-handler/ json_rpc/ logger/ node/ p2p/ tools/ traffic/ tx-pool/ package.json setup_blockchain_ubuntu_gcp.sh start_node_genesis_gcp.sh start_node_incremental_gcp.sh wait_until_node_sync.sh" # Work in progress spinner spin="-\|/" @@ -326,29 +318,25 @@ spinner() { sleep .1 } -if [[ $KILL_OPTION = "--skip-kill" ]]; then - printf "\nSkipping process kill...\n" -else - # kill any processes still alive - printf "\nKilling all blockchain nodes...\n" - index=$START_NODE_IDX - while [ $index -le $END_NODE_IDX ]; do - NODE_TARGET_ADDR=NODE_${index}_TARGET_ADDR - NODE_ZONE=NODE_${index}_ZONE +# kill any processes still alive +printf "\nKilling all blockchain nodes...\n" +index=$START_NODE_IDX +while [ $index -le $END_NODE_IDX ]; do + NODE_TARGET_ADDR=NODE_${index}_TARGET_ADDR + NODE_ZONE=NODE_${index}_ZONE - KILL_NODE_CMD="gcloud compute ssh ${!NODE_TARGET_ADDR} --command 'sudo killall node' --project $PROJECT_ID --zone ${!NODE_ZONE}" - # NOTE(minsulee2): Keep printf for extensibility experiment debugging purpose - # printf "KILL_NODE_CMD=$KILL_NODE_CMD\n" - if [[ $index < "$(($NUM_NODES - 1))" ]]; then - eval $KILL_NODE_CMD &> /dev/null & - else - eval $KILL_NODE_CMD &> /dev/null - fi - ((index++)) - spinner - done - printf "Kill all processes done.\n\n"; -fi + KILL_NODE_CMD="gcloud compute ssh ${!NODE_TARGET_ADDR} --command 'sudo killall node' --project $PROJECT_ID --zone ${!NODE_ZONE}" + # NOTE(minsulee2): Keep printf for extensibility experiment debugging purpose + # printf "KILL_NODE_CMD=$KILL_NODE_CMD\n" + if [[ $index < "$(($NUM_NODES - 1))" ]]; then + eval $KILL_NODE_CMD &> /dev/null & + else + eval $KILL_NODE_CMD &> /dev/null + fi + ((index++)) + spinner +done +printf "Kill all processes done.\n\n"; # If --kill-only, do not proceed any further if [[ $KILL_OPTION = "--kill-only" ]]; then diff --git a/node/index.js b/node/index.js index 8c4a53860..4efe3202e 100644 --- a/node/index.js +++ b/node/index.js @@ -361,43 +361,49 @@ class BlockchainNode { startNode(isFirstNode) { const LOG_HEADER = 'startNode'; - // 1. Initialize DB (with the latest snapshot, if it exists) - logger.info( - `[${LOG_HEADER}] Initializing DB with bootstrap snapshot from ${this.bootstrapSnapshotSource}..`); - const startingDb = DB.create( - StateVersions.EMPTY, StateVersions.START, this.bc, true, this.bootstrapSnapshotBlockNumber, - this.stateManager); - startingDb.initDb(this.bootstrapSnapshot); + try { + // 1. Initialize DB (with the latest snapshot, if it exists) + logger.info( + `[${LOG_HEADER}] Initializing DB with bootstrap snapshot from ${this.bootstrapSnapshotSource}..`); + const startingDb = DB.create( + StateVersions.EMPTY, StateVersions.START, this.bc, true, this.bootstrapSnapshotBlockNumber, + this.stateManager); + startingDb.initDb(this.bootstrapSnapshot); - // 2. Initialize the blockchain, starting from `bootstrapSnapshotBlockNumber`. - logger.info( - `[${LOG_HEADER}] Initializing blockchain with bootstrap snapshot from ${this.bootstrapSnapshotSource}..`); - const snapshotChunkSize = this.getBlockchainParam('resource/snapshot_chunk_size'); - const wasBlockDirEmpty = this.bc.initBlockchain( - isFirstNode, this.bootstrapSnapshot, this.snapshotDir, snapshotChunkSize); - - // 3. Execute the chain on the DB and finalize it. - logger.info(`[${LOG_HEADER}] Executing chains on DB if needed..`); - const isGenesisStart = (isFirstNode && wasBlockDirEmpty); - if (!wasBlockDirEmpty || isGenesisStart || NodeConfigs.SYNC_MODE === SyncModeOptions.PEER) { - if (!this.loadAndExecuteChainOnDb( - this.bootstrapSnapshotBlockNumber, startingDb.stateVersion, isGenesisStart)) { - return false; + // 2. Initialize the blockchain, starting from `bootstrapSnapshotBlockNumber`. + logger.info( + `[${LOG_HEADER}] Initializing blockchain with bootstrap snapshot from ${this.bootstrapSnapshotSource}..`); + const snapshotChunkSize = this.getBlockchainParam('resource/snapshot_chunk_size'); + const wasBlockDirEmpty = this.bc.initBlockchain( + isFirstNode, this.bootstrapSnapshot, this.snapshotDir, snapshotChunkSize); + + // 3. Execute the chain on the DB and finalize it. + logger.info(`[${LOG_HEADER}] Executing chains on DB if needed..`); + const isGenesisStart = (isFirstNode && wasBlockDirEmpty); + if (!wasBlockDirEmpty || isGenesisStart || NodeConfigs.SYNC_MODE === SyncModeOptions.PEER) { + if (!this.loadAndExecuteChainOnDb( + this.bootstrapSnapshotBlockNumber, startingDb.stateVersion, isGenesisStart)) { + return false; + } } - } - // 4. Execute transactions from the pool. - logger.info(`[${LOG_HEADER}] Executing the transaction from the tx pool..`); - this.db.executeTransactionList( - this.tp.getValidTransactions(null, this.stateManager.getFinalVersion()), false, true, - this.bc.lastBlockNumber() + 1, this.bc.lastBlockTimestamp()); + // 4. Execute transactions from the pool. + logger.info(`[${LOG_HEADER}] Executing the transaction from the tx pool..`); + this.db.executeTransactionList( + this.tp.getValidTransactions(null, this.stateManager.getFinalVersion()), false, true, + this.bc.lastBlockNumber() + 1, this.bc.lastBlockTimestamp()); - // 5. Node status changed: READY_TO_START -> CHAIN_SYNCING. - this.state = BlockchainNodeStates.CHAIN_SYNCING; - logger.info(`[${LOG_HEADER}] Now node in CHAIN_SYNCING state!`); + // 5. Node status changed: READY_TO_START -> CHAIN_SYNCING. + this.state = BlockchainNodeStates.CHAIN_SYNCING; + logger.info(`[${LOG_HEADER}] Now node in CHAIN_SYNCING state!`); - // 6. Reset bootstrap snapshot. - this.resetBootstrapSnapshot(); + // 6. Reset bootstrap snapshot. + this.resetBootstrapSnapshot(); + } catch (err) { + logger.error( + `[${LOG_HEADER}] Failed to start node with error: ${err} (${err.stack})`); + return false; + } return true; } diff --git a/package.json b/package.json index c08c392e5..6ffa76166 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ain-blockchain", "description": "AI Network Blockchain", - "version": "1.4.2", + "version": "1.5.0", "private": true, "license": "MIT", "author": "dev@ainetwork.ai", diff --git a/setup_blockchain_ubuntu_onprem.sh b/setup_blockchain_ubuntu_onprem.sh index 7943e579b..3da0a312e 100644 --- a/setup_blockchain_ubuntu_onprem.sh +++ b/setup_blockchain_ubuntu_onprem.sh @@ -65,4 +65,8 @@ sudo apt update sudo apt install -y jq printf 'jq --version\n' -jq --version \ No newline at end of file +jq --version + +printf '\n[[ Installing killall.. ]]\n' +sudo apt update +sudo apt install -y psmisc \ No newline at end of file diff --git a/start_node_genesis_gcp.sh b/start_node_genesis_gcp.sh index 12a2294a1..cd8dd26ae 100644 --- a/start_node_genesis_gcp.sh +++ b/start_node_genesis_gcp.sh @@ -48,7 +48,12 @@ function parse_options() { } # Parse options. -SEASON="$1" +if [[ "$1" = 'dev' ]] || [[ "$1" = 'staging' ]] || [[ "$1" = 'sandbox' ]] || [[ "$1" = 'exp' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'summer' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid project/season argument: $1\n" + exit +fi GCP_USER="$2" number_re='^[0-9]+$' @@ -163,9 +168,6 @@ else export ENABLE_EVENT_HANDLER=false fi -printf '\n' -printf 'Killing old jobs..\n' -sudo killall node if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then printf '\n' printf 'Setting up new working directory..\n' diff --git a/start_node_genesis_onprem.sh b/start_node_genesis_onprem.sh index f0c9c41f3..8249e5503 100644 --- a/start_node_genesis_onprem.sh +++ b/start_node_genesis_onprem.sh @@ -2,8 +2,9 @@ # NOTE(minsulee2): Since exit really exits terminals, those are replaced to return 1. if [[ $# -lt 4 ]] || [[ $# -gt 13 ]]; then - printf "Usage: bash start_node_genesis_onprem.sh [dev|staging|sandbox|exp|spring|summer|mainnet] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--json-rpc] [--update-front-db] [--rest-func] [--event-handler]\n" - printf "Example: bash start_node_genesis_onprem.sh spring gcp_user 0 0 --keystore --no-keep-code --full-sync --no-chown-data\n" + printf "Usage: bash start_node_genesis_onprem.sh [staging|spring|mainnet] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--json-rpc] [--update-front-db] [--rest-func] [--event-handler]\n" + printf "Example: bash start_node_genesis_onprem.sh staging nvidia 0 4 --keystore --no-keep-code --full-sync --no-chown-data\n" + printf "Example: bash start_node_genesis_onprem.sh staging nvidia 0 0 --keystore --no-keep-code --full-sync --no-chown-data\n" printf "\n" return 1 fi @@ -57,8 +58,13 @@ function parse_options() { } # Parse options. -SEASON="$1" -GCP_USER="$2" +if [[ "$1" = 'staging' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid project/season argument: $1\n" + exit +fi +ONPREM_USER="$2" number_re='^[0-9]+$' if ! [[ $3 =~ $number_re ]] ; then @@ -95,7 +101,7 @@ while [ $ARG_INDEX -le $# ]; do done printf "SEASON=$SEASON\n" -printf "GCP_USER=$GCP_USER\n" +printf "ONPREM_USER=$ONPREM_USER\n" printf "SHARD_INDEX=$SHARD_INDEX\n" printf "NODE_INDEX=$NODE_INDEX\n" printf "\n" @@ -144,7 +150,7 @@ if [[ $SYNC_MODE_OPTION = "--full-sync" ]]; then else export SYNC_MODE=fast fi -if [[ $SEASON = "staging" ]] || [[ $SEASON = "exp" ]]; then +if [[ $SEASON = "staging" ]]; then # staging: for performance test pipeline # exp: for performance test export ENABLE_EXPRESS_RATE_LIMIT=false @@ -172,140 +178,67 @@ else export ENABLE_EVENT_HANDLER=false fi -printf '\n' -printf 'Killing old jobs..\n' -sudo killall node if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then printf '\n' printf 'Setting up new working directory..\n' - sudo rm -rf /home/ain-blockchain* - CODE_CMD="cd ~; sudo mv ain-blockchain /home; sudo chmod -R 777 /home/ain-blockchain; sudo chown -R $GCP_USER:$GCP_USER /home/ain-blockchain; cd /home/ain-blockchain" + sudo rm -rf /home/${SEASON}/ain-blockchain* + # NOTE(platfowner): Add $SEASON to the node job name to be selectively killed in restarts. + CODE_CMD="sudo mkdir -p /home/${SEASON}; sudo chmod -R 777 /home/${SEASON}; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}; cd ~; sudo mv ./ain-blockchain /home/${SEASON}; sudo chmod -R 777 /home/${SEASON}/ain-blockchain; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain-blockchain; cd /home/${SEASON}/ain-blockchain; mv client/index.js client/${SEASON}-ain-blockchain-index.js" printf "\nCODE_CMD=$CODE_CMD\n" eval $CODE_CMD else printf '\n' printf 'Reusing existing working directory..\n' - OLD_DIR_PATH=$(find /home/ain-blockchain* -maxdepth 0 -type d) + OLD_DIR_PATH=$(find /home/${SEASON}/ain-blockchain* -maxdepth 0 -type d) printf "OLD_DIR_PATH=$OLD_DIR_PATH\n" - CODE_CMD="sudo chmod -R 777 $OLD_DIR_PATH; sudo chown -R $GCP_USER:$GCP_USER $OLD_DIR_PATH" + CODE_CMD="sudo chmod -R 777 $OLD_DIR_PATH; sudo chown -R $ONPREM_USER:$ONPREM_USER $OLD_DIR_PATH" printf "\nCODE_CMD=$CODE_CMD\n" eval $CODE_CMD fi if [[ $KEEP_DATA_OPTION = "--no-keep-data" ]]; then printf '\n' printf 'Setting up new data directory..\n' - sudo rm -rf /home/ain_blockchain_data/chains - sudo rm -rf /home/ain_blockchain_data/snapshots - sudo rm -rf /home/ain_blockchain_data/logs - DATA_CMD="sudo mkdir -p /home/ain_blockchain_data; sudo chmod -R 777 /home/ain_blockchain_data; sudo chown -R $GCP_USER:$GCP_USER /home/ain_blockchain_data" + sudo rm -rf /home/${SEASON}/ain_blockchain_data/chains + sudo rm -rf /home/${SEASON}/ain_blockchain_data/snapshots + sudo rm -rf /home/${SEASON}/ain_blockchain_data/logs + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod -R 777 /home/${SEASON}/ain_blockchain_data; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" printf "\nDATA_CMD=$DATA_CMD\n" eval $DATA_CMD else printf 'Reusing existing data directory..\n' if [[ $CHOWN_DATA_OPTION = "--no-chown-data" ]]; then - DATA_CMD="sudo mkdir -p /home/ain_blockchain_data; sudo chmod 777 /home/ain_blockchain_data; sudo chown $GCP_USER:$GCP_USER /home/ain_blockchain_data" + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod 777 /home/${SEASON}/ain_blockchain_data; sudo chown $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" else - DATA_CMD="sudo mkdir -p /home/ain_blockchain_data; sudo chmod -R 777 /home/ain_blockchain_data; sudo chown -R $GCP_USER:$GCP_USER /home/ain_blockchain_data" + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod -R 777 /home/${SEASON}/ain_blockchain_data; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" fi printf "\nDATA_CMD=$DATA_CMD\n" eval $DATA_CMD fi - if [[ $SEASON = 'mainnet' ]]; then export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/mainnet-prod - export PEER_CANDIDATE_JSON_RPC_URL="http://104.199.237.250:8080/json-rpc" + export PORT=8077 + export EVENT_HANDLER_PORT=5097 + export PEER_CANDIDATE_JSON_RPC_URL="http://101.202.37.2:8077/json-rpc" if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then export PEER_WHITELIST="0x000C63907F7Aeca56A72F5a4F7cd00EfFCF11c3A,0x001C3C9C4a5669eCD8b78946f6fa5549b33362F8,0x002C76f0aeA9Ba615428d9dF7fedEC6f8ed5369f,0x003C9d091584fEC96bC3bD8423c884680BEAaf4E,0x004C4328B6c2ABF7c4Df897a8124b36E3f00a2FC,0x005C99Db64845e5BF24cd152b22c932989479907,0x006C672861e9DBb09232307c17Be6554BC90687c,0x007C36bf5D0F77836eE138EEAc8df7051b43209b,0x008C287187a5626D0a25DbD67327B36AC55B998E,0x009C66DBce144003f8C4B859fFFce78F80fDD639" fi -elif [[ $SEASON = 'summer' ]]; then - export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-prod - export TRACKER_UPDATE_JSON_RPC_URL="http://35.194.172.106:8080/json-rpc" - export PEER_CANDIDATE_JSON_RPC_URL="http://35.194.169.78:8080/json-rpc" - if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then - export PEER_WHITELIST="0x000AF024FEDb636294867bEff390bCE6ef9C5fc4,0x001Ac309EFFFF6d307CbC2d09C811aCD7dD8A35d,0x002A273ECd3aAEc4d8748f4E06eAdE3b34d83211,0x003AD6FdB06684175e7D95EcC36758B014517E4b,0x004A2550661c8a306207C9dabb279d5701fFD66e,0x005A3c55EcE1A593b761D408B6E6BC778E0a638B,0x006Af719E197bC81BBb75d2fec7Ea217D1750bAe,0x007Ac58EAc5F0D0bDd10Af8b90799BcF849c2E74,0x008AeBc041B7ceABc53A4cf393ccF16c10c29dba,0x009A97c0cF07fdbbcdA1197aE11792258b6EcedD" - fi elif [[ $SEASON = 'spring' ]]; then export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-prod - export PEER_CANDIDATE_JSON_RPC_URL="http://35.221.184.48:8080/json-rpc" + export PORT=8078 + export EVENT_HANDLER_PORT=5098 + export PEER_CANDIDATE_JSON_RPC_URL="http://101.202.37.2:8078/json-rpc" if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then export PEER_WHITELIST="0x000AF024FEDb636294867bEff390bCE6ef9C5fc4,0x001Ac309EFFFF6d307CbC2d09C811aCD7dD8A35d,0x002A273ECd3aAEc4d8748f4E06eAdE3b34d83211,0x003AD6FdB06684175e7D95EcC36758B014517E4b,0x004A2550661c8a306207C9dabb279d5701fFD66e,0x005A3c55EcE1A593b761D408B6E6BC778E0a638B,0x006Af719E197bC81BBb75d2fec7Ea217D1750bAe,0x007Ac58EAc5F0D0bDd10Af8b90799BcF849c2E74,0x008AeBc041B7ceABc53A4cf393ccF16c10c29dba,0x009A97c0cF07fdbbcdA1197aE11792258b6EcedD" fi -elif [[ "$SEASON" = "sandbox" ]]; then - export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-sandbox - if [[ $NODE_INDEX -lt 10 ]]; then - # Nodes 10 & above will use https://sandbox-api.ainetwork.ai/json-rpc. - export PEER_CANDIDATE_JSON_RPC_URL="http://130.211.244.169:8080/json-rpc" - fi - if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then - export PEER_WHITELIST="0x00ADEc28B6a845a085e03591bE7550dd68673C1C,0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204,0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00,0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A,0x04A456C92A880cd59D7145C457475515a6f6E0f2,0x05A1247A7400f0C2A893611adD1505743552c631,0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A,0x07A43138CC760C85A5B1F115aa60eADEaa0bf417,0x08Aed7AF9354435c38d52143EE50ac839D20696b,0x09A0d53FDf1c36A131938eb379b98910e55EEfe1" - fi elif [[ $SEASON = 'staging' ]]; then export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-staging - export PEER_CANDIDATE_JSON_RPC_URL="http://101.202.37.2:8080/json-rpc" - if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then - export PEER_WHITELIST="0x00ADEc28B6a845a085e03591bE7550dd68673C1C,0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204,0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00,0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A,0x04A456C92A880cd59D7145C457475515a6f6E0f2,0x05A1247A7400f0C2A893611adD1505743552c631,0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A,0x07A43138CC760C85A5B1F115aa60eADEaa0bf417,0x08Aed7AF9354435c38d52143EE50ac839D20696b,0x09A0d53FDf1c36A131938eb379b98910e55EEfe1" - fi -elif [[ $SEASON = 'exp' ]]; then - export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-exp - export PEER_CANDIDATE_JSON_RPC_URL="http://34.81.178.195:8080/json-rpc" + export PORT=8079 + export EVENT_HANDLER_PORT=5099 + export PEER_CANDIDATE_JSON_RPC_URL="http://101.202.37.2:8079/json-rpc" if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then export PEER_WHITELIST="0x00ADEc28B6a845a085e03591bE7550dd68673C1C,0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204,0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00,0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A,0x04A456C92A880cd59D7145C457475515a6f6E0f2,0x05A1247A7400f0C2A893611adD1505743552c631,0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A,0x07A43138CC760C85A5B1F115aa60eADEaa0bf417,0x08Aed7AF9354435c38d52143EE50ac839D20696b,0x09A0d53FDf1c36A131938eb379b98910e55EEfe1" fi -elif [[ $SEASON = 'dev' ]]; then - export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-dev - if [[ $SHARD_INDEX = 0 ]]; then - export PEER_CANDIDATE_JSON_RPC_URL="http://35.194.235.180:8080/json-rpc" - if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then - export PEER_WHITELIST="0x00ADEc28B6a845a085e03591bE7550dd68673C1C,0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204,0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00,0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A,0x04A456C92A880cd59D7145C457475515a6f6E0f2,0x05A1247A7400f0C2A893611adD1505743552c631,0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A,0x07A43138CC760C85A5B1F115aa60eADEaa0bf417,0x08Aed7AF9354435c38d52143EE50ac839D20696b,0x09A0d53FDf1c36A131938eb379b98910e55EEfe1" - fi - elif [[ $SHARD_INDEX = 1 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.187.153.22:8080/json-rpc" # dev-shard-1-tracker-ip - elif [[ $SHARD_INDEX = 2 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.203.104:8080/json-rpc" # dev-shard-2-tracker-ip - elif [[ $SHARD_INDEX = 3 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.189.174.17:8080/json-rpc" # dev-shard-3-tracker-ip - elif [[ $SHARD_INDEX = 4 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.221.164.158:8080/json-rpc" # dev-shard-4-tracker-ip - elif [[ $SHARD_INDEX = 5 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.234.46.65:8080/json-rpc" # dev-shard-5-tracker-ip - elif [[ $SHARD_INDEX = 6 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.221.210.171:8080/json-rpc" # dev-shard-6-tracker-ip - elif [[ $SHARD_INDEX = 7 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.222.121:8080/json-rpc" # dev-shard-7-tracker-ip - elif [[ $SHARD_INDEX = 8 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.221.200.95:8080/json-rpc" # dev-shard-8-tracker-ip - elif [[ $SHARD_INDEX = 9 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.216.199:8080/json-rpc" # dev-shard-9-tracker-ip - elif [[ $SHARD_INDEX = 10 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.161.85:8080/json-rpc" # dev-shard-10-tracker-ip - elif [[ $SHARD_INDEX = 11 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.194.239.169:8080/json-rpc" # dev-shard-11-tracker-ip - elif [[ $SHARD_INDEX = 12 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.185.156.22:8080/json-rpc" # dev-shard-12-tracker-ip - elif [[ $SHARD_INDEX = 13 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.229.247.143:8080/json-rpc" # dev-shard-13-tracker-ip - elif [[ $SHARD_INDEX = 14 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.229.226.47:8080/json-rpc" # dev-shard-14-tracker-ip - elif [[ $SHARD_INDEX = 15 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.234.61.23:8080/json-rpc" # dev-shard-15-tracker-ip - elif [[ $SHARD_INDEX = 16 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.66.41:8080/json-rpc" # dev-shard-16-tracker-ip - elif [[ $SHARD_INDEX = 17 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.229.143.18:8080/json-rpc" # dev-shard-17-tracker-ip - elif [[ $SHARD_INDEX = 18 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.234.58.137:8080/json-rpc" # dev-shard-18-tracker-ip - elif [[ $SHARD_INDEX = 19 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://34.80.249.104:8080/json-rpc" # dev-shard-19-tracker-ip - elif [[ $SHARD_INDEX = 20 ]]; then - export TRACKER_UPDATE_JSON_RPC_URL="http://35.201.248.92:8080/json-rpc" # dev-shard-20-tracker-ip - else - printf "Invalid shard ID argument: $SHARD_INDEX\n" - return 1 - fi - if [[ $SHARD_INDEX -gt 0 ]]; then - export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/sim-shard - fi else printf "Invalid season argument: $SEASON\n" return 1 @@ -327,6 +260,9 @@ printf "LOG_BANDAGE_INFO=$LOG_BANDAGE_INFO\n" # on-premise nodes run with "comcom" hosting env export HOSTING_ENV="comcom" printf "HOSTING_ENV=$HOSTING_ENV\n" +# on-premise nodes run with a blockchain data directory prefixed by ${SEASON}_ +export BLOCKCHAIN_DATA_DIR="/home/${SEASON}/ain_blockchain_data" +printf "BLOCKCHAIN_DATA_DIR=$BLOCKCHAIN_DATA_DIR\n" if [[ "$SEASON" = "sandbox" ]]; then MAX_OLD_SPACE_SIZE_MB=11000 @@ -335,7 +271,7 @@ else fi printf "\nStarting up Blockchain Node server..\n\n" -START_CMD="nohup node --async-stack-traces --max-old-space-size=$MAX_OLD_SPACE_SIZE_MB client/index.js >/dev/null 2>error_logs.txt &" +START_CMD="nohup node --async-stack-traces --max-old-space-size=$MAX_OLD_SPACE_SIZE_MB client/${SEASON}-ain-blockchain-index.js >/dev/null 2>error_logs.txt &" printf "\nSTART_CMD=$START_CMD\n" printf "START_CMD=$START_CMD\n" >> start_commands.txt eval $START_CMD diff --git a/start_node_incremental_gcp.sh b/start_node_incremental_gcp.sh index 9cb9767c8..e70433c57 100644 --- a/start_node_incremental_gcp.sh +++ b/start_node_incremental_gcp.sh @@ -47,7 +47,12 @@ function parse_options() { } # Parse options. -SEASON="$1" +if [[ "$1" = 'dev' ]] || [[ "$1" = 'staging' ]] || [[ "$1" = 'sandbox' ]] || [[ "$1" = 'exp' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'summer' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid project/season argument: $1\n" + exit +fi GCP_USER="$2" number_re='^[0-9]+$' diff --git a/start_node_incremental_onprem.sh b/start_node_incremental_onprem.sh new file mode 100644 index 000000000..e76aacbc6 --- /dev/null +++ b/start_node_incremental_onprem.sh @@ -0,0 +1,332 @@ +#!/bin/bash + +if [[ $# -lt 4 ]] || [[ $# -gt 13 ]]; then + printf "Usage: bash start_node_incremental_onprem.sh [staging|spring|mainnet] [--keystore|--mnemonic|--private-key] [--keep-code|--no-keep-code] [--keep-data|--no-keep-data] [--full-sync|--fast-sync] [--chown-data|--no-chown-data] [--json-rpc] [--update-front-db] [--rest-func] [--event-handler]\n" + printf "Example: bash start_node_incremental_onprem.sh staging nvidia 0 4 --keystore --no-keep-code --full-sync --no-chown-data\n" + printf "Example: bash start_node_incremental_onprem.sh staging nvidia 0 0 --keystore --no-keep-code --full-sync --no-chown-data\n" + printf "\n" + exit +fi + +# needed for on-premise nvidia machines +# Get node login password +printf "Enter node login password: " +read -s NODE_LOGIN_PW +printf "\n\n" +# do sudo once with a dummy command +echo $NODE_LOGIN_PW | sudo -S ls -la + +printf "\n[[[[[ start_node_incremental_onprem.sh ]]]]]\n\n" + +function parse_options() { + local option="$1" + if [[ $option = '--private-key' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--keystore' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--mnemonic' ]]; then + ACCOUNT_INJECTION_OPTION="$option" + elif [[ $option = '--keep-code' ]]; then + KEEP_CODE_OPTION="$option" + elif [[ $option = '--no-keep-code' ]]; then + KEEP_CODE_OPTION="$option" + elif [[ $option = '--keep-data' ]]; then + KEEP_DATA_OPTION="$option" + elif [[ $option = '--no-keep-data' ]]; then + KEEP_DATA_OPTION="$option" + elif [[ $option = '--full-sync' ]]; then + SYNC_MODE_OPTION="$option" + elif [[ $option = '--fast-sync' ]]; then + SYNC_MODE_OPTION="$option" + elif [[ $option = '--chown-data' ]]; then + CHOWN_DATA_OPTION="$option" + elif [[ $option = '--no-chown-data' ]]; then + CHOWN_DATA_OPTION="$option" + elif [[ $option = '--json-rpc' ]]; then + JSON_RPC_OPTION="$option" + elif [[ $option = '--update-front-db' ]]; then + UPDATE_FRONT_DB_OPTION="$option" + elif [[ $option = '--rest-func' ]]; then + REST_FUNC_OPTION="$option" + elif [[ $option = '--event-handler' ]]; then + EVENT_HANDLER_OPTION="$option" + else + printf "Invalid option: $option\n" + exit + fi +} + +# Parse options. +if [[ "$1" = 'staging' ]] || [[ "$1" = 'spring' ]] || [[ "$1" = 'mainnet' ]]; then + SEASON="$1" +else + printf "Invalid project/season argument: $1\n" + exit +fi +ONPREM_USER="$2" + +number_re='^[0-9]+$' +if ! [[ $3 =~ $number_re ]] ; then + printf "Invalid argument: $3\n" + exit +fi +SHARD_INDEX="$3" + +if ! [[ $4 =~ $number_re ]] ; then + printf "Invalid argument: $4\n" + exit +fi +if [[ "$4" -lt 0 ]] || [[ "$4" -gt 9 ]]; then + printf "Invalid argument: $4\n" + exit +fi +NODE_INDEX="$4" + +ACCOUNT_INJECTION_OPTION="--private-key" +KEEP_CODE_OPTION="--keep-code" +KEEP_DATA_OPTION="--keep-data" +SYNC_MODE_OPTION="--fast-sync" +CHOWN_DATA_OPTION="--no-chown-data" +JSON_RPC_OPTION="" +UPDATE_FRONT_DB_OPTION="" +REST_FUNC_OPTION="" +EVENT_HANDLER_OPTION="" + +ARG_INDEX=5 +while [ $ARG_INDEX -le $# ]; do + parse_options "${!ARG_INDEX}" + ((ARG_INDEX++)) +done + +printf "SEASON=$SEASON\n" +printf "ONPREM_USER=$ONPREM_USER\n" +printf "SHARD_INDEX=$SHARD_INDEX\n" +printf "NODE_INDEX=$NODE_INDEX\n" +printf "\n" + +printf "ACCOUNT_INJECTION_OPTION=$ACCOUNT_INJECTION_OPTION\n" +printf "KEEP_CODE_OPTION=$KEEP_CODE_OPTION\n" +printf "KEEP_DATA_OPTION=$KEEP_DATA_OPTION\n" +printf "SYNC_MODE_OPTION=$SYNC_MODE_OPTION\n" +printf "CHOWN_DATA_OPTION=$CHOWN_DATA_OPTION\n" +printf "JSON_RPC_OPTION=$JSON_RPC_OPTION\n" +printf "UPDATE_FRONT_DB_OPTION=$UPDATE_FRONT_DB_OPTION\n" +printf "REST_FUNC_OPTION=$REST_FUNC_OPTION\n" +printf "EVENT_HANDLER_OPTION=$EVENT_HANDLER_OPTION\n" + +# Peer-whitelisting-enabled blockchain nodes +# Peer whitelisting is disabled now with 5 core blockchain nodes on GCP. +#PEER_WHITELIST_NODE_INDEX_GE=0 +#PEER_WHITELIST_NODE_INDEX_LE=4 +PEER_WHITELIST_NODE_INDEX_GE=-1 +PEER_WHITELIST_NODE_INDEX_LE=-1 + +printf "\n" +printf "PEER_WHITELIST_NODE_INDEX_GE=$PEER_WHITELIST_NODE_INDEX_GE\n" +printf "PEER_WHITELIST_NODE_INDEX_LE=$PEER_WHITELIST_NODE_INDEX_LE\n" + +if [[ "$ACCOUNT_INJECTION_OPTION" = "" ]]; then + printf "Must provide an ACCOUNT_INJECTION_OPTION\n" + exit +fi + +# 1. Configure env vars (BLOCKCHAIN_CONFIGS_DIR, TRACKER_UPDATE_JSON_RPC_URL, ...) +printf "\n#### [Step 1] Configure env vars ####\n\n" + +if [[ $SEASON = 'mainnet' ]]; then + export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/mainnet-prod + export PORT=8077 + export EVENT_HANDLER_PORT=5097 + if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then + export PEER_WHITELIST="0x000C63907F7Aeca56A72F5a4F7cd00EfFCF11c3A,0x001C3C9C4a5669eCD8b78946f6fa5549b33362F8,0x002C76f0aeA9Ba615428d9dF7fedEC6f8ed5369f,0x003C9d091584fEC96bC3bD8423c884680BEAaf4E,0x004C4328B6c2ABF7c4Df897a8124b36E3f00a2FC,0x005C99Db64845e5BF24cd152b22c932989479907,0x006C672861e9DBb09232307c17Be6554BC90687c,0x007C36bf5D0F77836eE138EEAc8df7051b43209b,0x008C287187a5626D0a25DbD67327B36AC55B998E,0x009C66DBce144003f8C4B859fFFce78F80fDD639" + fi +elif [[ $SEASON = 'spring' ]]; then + export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-prod + export PORT=8078 + export EVENT_HANDLER_PORT=5098 + if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then + export PEER_WHITELIST="0x000AF024FEDb636294867bEff390bCE6ef9C5fc4,0x001Ac309EFFFF6d307CbC2d09C811aCD7dD8A35d,0x002A273ECd3aAEc4d8748f4E06eAdE3b34d83211,0x003AD6FdB06684175e7D95EcC36758B014517E4b,0x004A2550661c8a306207C9dabb279d5701fFD66e,0x005A3c55EcE1A593b761D408B6E6BC778E0a638B,0x006Af719E197bC81BBb75d2fec7Ea217D1750bAe,0x007Ac58EAc5F0D0bDd10Af8b90799BcF849c2E74,0x008AeBc041B7ceABc53A4cf393ccF16c10c29dba,0x009A97c0cF07fdbbcdA1197aE11792258b6EcedD" + fi +elif [[ $SEASON = 'staging' ]]; then + export BLOCKCHAIN_CONFIGS_DIR=blockchain-configs/testnet-staging + export PORT=8079 + export EVENT_HANDLER_PORT=5099 + if [[ $NODE_INDEX -ge $PEER_WHITELIST_NODE_INDEX_GE ]] && [[ $NODE_INDEX -le $PEER_WHITELIST_NODE_INDEX_LE ]]; then + export PEER_WHITELIST="0x00ADEc28B6a845a085e03591bE7550dd68673C1C,0x01A0980d2D4e418c7F27e1ef539d01A5b5E93204,0x02A2A1DF4f630d760c82BE07F18e5065d103Fa00,0x03AAb7b6f16A92A1dfe018Fe34ee420eb098B98A,0x04A456C92A880cd59D7145C457475515a6f6E0f2,0x05A1247A7400f0C2A893611adD1505743552c631,0x06AD9C8F611f1e9d9CACD4738167A51aA2e80a1A,0x07A43138CC760C85A5B1F115aa60eADEaa0bf417,0x08Aed7AF9354435c38d52143EE50ac839D20696b,0x09A0d53FDf1c36A131938eb379b98910e55EEfe1" + fi + # NOTE(platfowner): For non-api-servers, the value in the blockchain configs + # (https://staging-api.ainetwork.ai/json-rpc) is used. +else + printf "Invalid argument: $SEASON\n" + exit +fi + +printf "TRACKER_UPDATE_JSON_RPC_URL=$TRACKER_UPDATE_JSON_RPC_URL\n" +printf "BLOCKCHAIN_CONFIGS_DIR=$BLOCKCHAIN_CONFIGS_DIR\n" +printf "PEER_CANDIDATE_JSON_RPC_URL=$PEER_CANDIDATE_JSON_RPC_URL\n" +printf "PEER_WHITELIST=$PEER_WHITELIST\n" + +# NOTE(liayoo): Currently this script supports [--keystore|--mnemonic] option only for the parent chain. +if [[ $ACCOUNT_INJECTION_OPTION != "--private_key" ]] && [[ "$SHARD_INDEX" -gt 0 ]]; then + printf 'Invalid account injection option\n' + return 1 +fi + +if [[ "$ACCOUNT_INJECTION_OPTION" = "" ]]; then + printf "Must provide an ACCOUNT_INJECTION_OPTION\n" + return 1 +fi + +if [[ $ACCOUNT_INJECTION_OPTION = "--keystore" ]]; then + export ACCOUNT_INJECTION_OPTION=keystore +elif [[ $ACCOUNT_INJECTION_OPTION = "--mnemonic" ]]; then + export ACCOUNT_INJECTION_OPTION=mnemonic +else + export ACCOUNT_INJECTION_OPTION=private_key +fi +if [[ $SYNC_MODE_OPTION = "--full-sync" ]]; then + export SYNC_MODE=full +else + export SYNC_MODE=fast +fi +if [[ $SEASON = "staging" ]]; then + # for performance test pipeline + export ENABLE_EXPRESS_RATE_LIMIT=false +else + export ENABLE_EXPRESS_RATE_LIMIT=true +fi +if [[ $JSON_RPC_OPTION ]]; then + export ENABLE_JSON_RPC_API=true +else + export ENABLE_JSON_RPC_API=false +fi +if [[ $UPDATE_FRONT_DB_OPTION ]]; then + export UPDATE_NEW_FINAL_FRONT_DB_WITH_TX_POOL=true +else + export UPDATE_NEW_FINAL_FRONT_DB_WITH_TX_POOL=false +fi +if [[ $REST_FUNC_OPTION ]]; then + export ENABLE_REST_FUNCTION_CALL=true +else + export ENABLE_REST_FUNCTION_CALL=false +fi +if [[ $EVENT_HANDLER_OPTION ]]; then + export ENABLE_EVENT_HANDLER=true +else + export ENABLE_EVENT_HANDLER=false +fi + +# NOTE(liayoo): Currently this script supports [--keystore|--mnemonic] option only for the parent chain. +if [[ $ACCOUNT_INJECTION_OPTION != "private_key" ]] && [[ "$SHARD_INDEX" -gt 0 ]]; then + printf 'Invalid account injection option\n' + exit +fi + +# 2. Get currently used directory & new directory +printf "\n#### [Step 2] Get currently used directory & new directory ####\n\n" + +OLD_DIR_PATH=$(find /home/${SEASON}/ain-blockchain* -maxdepth 0 -type d) +printf "OLD_DIR_PATH=$OLD_DIR_PATH\n" + +date=$(date '+%Y-%m-%dT%H-%M') +printf "date=$date\n" +NEW_DIR_NAME="ain-blockchain-$date" +printf "NEW_DIR_NAME=$NEW_DIR_NAME\n" +NEW_DIR_PATH="/home/${SEASON}/$NEW_DIR_NAME" +printf "NEW_DIR_PATH=$NEW_DIR_PATH\n" + +# 3. Set up working directory & install modules +printf "\n#### [Step 3] Set up working directory & install modules ####\n\n" +if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then + printf '\n' + printf 'Installing node modules..\n' + INSTALL_CMD="yarn install --ignore-engines" + printf "\nINSTALL_CMD=$INSTALL_CMD\n" + eval $INSTALL_CMD + + printf '\n' + printf 'Setting up new working directory..\n' + # NOTE(platfowner): Add $SEASON to the node job name to be selectively killed in restarts. + CODE_CMD="sudo mkdir -p /home/${SEASON}; sudo chmod -R 777 /home/${SEASON}; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}; cd ~; sudo mv ain-blockchain $NEW_DIR_PATH; sudo chmod -R 777 $NEW_DIR_PATH; sudo chown -R $ONPREM_USER:$ONPREM_USER $NEW_DIR_PATH; cd $NEW_DIR_PATH; mv client/index.js client/${SEASON}-ain-blockchain-index.js" + printf "\nCODE_CMD=$CODE_CMD\n" + eval $CODE_CMD +else + printf '\n' + printf 'Reusing existing working directory..\n' + CODE_CMD="sudo chmod -R 777 $OLD_DIR_PATH; sudo chown -R $ONPREM_USER:$ONPREM_USER $OLD_DIR_PATH" + printf "\nCODE_CMD=$CODE_CMD\n" + eval $CODE_CMD +fi + +# 4. Kill old node server +printf "\n#### [Step 4] Kill old node server ####\n\n" + +KILL_CMD="sudo pkill -f client/${SEASON}-ain-blockchain-index.js" +printf "KILL_CMD=$KILL_CMD\n\n" +eval $KILL_CMD +sleep 10 + +# 5. Set up data directory +printf "\n#### [Step 5] Set up data directory ####\n\n" +if [[ $KEEP_DATA_OPTION = "--no-keep-data" ]]; then + printf '\n' + printf 'Setting up new data directory..\n' + sudo rm -rf /home/${SEASON}/ain_blockchain_data/chains + sudo rm -rf /home/${SEASON}/ain_blockchain_data/snapshots + sudo rm -rf /home/${SEASON}/ain_blockchain_data/logs + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod -R 777 /home/${SEASON}/ain_blockchain_data; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" + printf "\nDATA_CMD=$DATA_CMD\n" + eval $DATA_CMD +else + printf '\n' + printf 'Reusing existing data directory..\n' + if [[ $CHOWN_DATA_OPTION = "--no-chown-data" ]]; then + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod 777 /home/${SEASON}/ain_blockchain_data; sudo chown $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" + else + DATA_CMD="sudo mkdir -p /home/${SEASON}/ain_blockchain_data; sudo chmod -R 777 /home/${SEASON}/ain_blockchain_data; sudo chown -R $ONPREM_USER:$ONPREM_USER /home/${SEASON}/ain_blockchain_data" + fi + printf "\nDATA_CMD=$DATA_CMD\n" + eval $DATA_CMD +fi + +# 6. Remove old working directory keeping the chain data +printf "\n#### [Step 6] Remove old working directory if necessary ####\n\n" +if [[ $KEEP_CODE_OPTION = "--no-keep-code" ]]; then + printf '\n' + printf 'Removing old working directory..\n' + RM_CMD="sudo rm -rf $OLD_DIR_PATH" + printf "\nRM_CMD=$RM_CMD\n" + eval $RM_CMD +else + printf '\n' + printf 'Keeping existing working directory..\n' +fi + +# 7. Start a new node server +printf "\n#### [Step 7] Start new node server ####\n\n" + +export STAKE=100000 +printf "STAKE=$STAKE\n" +export LOG_BANDAGE_INFO=true +printf "LOG_BANDAGE_INFO=$LOG_BANDAGE_INFO\n" +# on-premise nodes run with "comcom" hosting env +export HOSTING_ENV="comcom" +printf "HOSTING_ENV=$HOSTING_ENV\n" +# on-premise nodes run with a blockchain data directory prefixed by ${SEASON}_ +export BLOCKCHAIN_DATA_DIR="/home/${SEASON}/ain_blockchain_data" +printf "BLOCKCHAIN_DATA_DIR=$BLOCKCHAIN_DATA_DIR\n" + +if [[ "$SEASON" = "sandbox" ]]; then + MAX_OLD_SPACE_SIZE_MB=11000 +else + MAX_OLD_SPACE_SIZE_MB=55000 +fi + +START_CMD="nohup node --async-stack-traces --max-old-space-size=$MAX_OLD_SPACE_SIZE_MB client/${SEASON}-ain-blockchain-index.js >/dev/null 2>error_logs.txt &" +printf "\nSTART_CMD=$START_CMD\n" +printf "START_CMD=$START_CMD\n" >> start_commands.txt +eval $START_CMD + +# NOTE(platfowner): deploy_blockchain_incremental_onprem.sh waits until the new server gets healthy. + +printf "\n* << Node server [$SEASON $SHARD_INDEX $NODE_INDEX] successfully deployed! ***************************************\n\n" diff --git a/test/unit/p2p.test.js b/test/unit/p2p.test.js index bef1b3b83..e19027102 100644 --- a/test/unit/p2p.test.js +++ b/test/unit/p2p.test.js @@ -227,13 +227,13 @@ describe("P2p", () => { isEnabled: false, networkInfo: { url: "", - maxNumEventChannels: 20, + maxNumEventChannels: 30, numEventChannels: 0, maxNumEventFilters: 40, numEventFilters: 0, }, channelStatus: { - maxNumEventChannels: 20, + maxNumEventChannels: 30, numEventChannels: 0, channelLifeTimeLimitSecs: 3600, channelIdleTimeLimitSecs: 600, diff --git a/wait_until_node_sync_gcp.sh b/wait_until_node_sync.sh similarity index 62% rename from wait_until_node_sync_gcp.sh rename to wait_until_node_sync.sh index 2aae02f9d..582ccd4f8 100644 --- a/wait_until_node_sync_gcp.sh +++ b/wait_until_node_sync.sh @@ -1,6 +1,17 @@ #!/bin/bash -printf "\n[[[[[ wait_until_node_sync_gcp.sh ]]]]]\n\n" +if [[ $# -lt 1 ]] || [[ $# -gt 1 ]]; then + printf "Usage: bash wait_until_node_sync.sh \n" + printf "Example: bash wait_until_node_sync.sh http://123.456.789.1:8079\n" + printf "\n" + return 1 +fi + +printf "\n[[[[[ wait_until_node_sync.sh ]]]]]\n\n" + +# Parse options. +NODE_URL="$1" +printf "NODE_URL=$NODE_URL\n" printf "\n#### Wait until the new node server catches up ####\n\n" @@ -16,11 +27,11 @@ EOF while : do - healthCheck=$(curl -m 20 -X GET -H "Content-Type: application/json" "http://localhost:8080/health_check") + healthCheck=$(curl -m 20 -X GET -H "Content-Type: application/json" "${NODE_URL}/health_check") printf "\nhealthCheck = ${healthCheck}\n" if [[ "$healthCheck" = "true" ]]; then printf "\nBlockchain Node server is synced & running!\n" - lastBlockNumber=$(curl -m 20 -X GET -H "Content-Type: application/json" "http://localhost:8080/last_block_number" | jq -r '.result') + lastBlockNumber=$(curl -m 20 -X GET -H "Content-Type: application/json" "${NODE_URL}/last_block_number" | jq -r '.result') printf "\nlastBlockNumber = ${lastBlockNumber}\n" printf "\nTime it took to sync in seconds: $SECONDS\n" break