From e596d158f3222a3dab91d6c0d863b1a5df905a3d Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Wed, 24 Apr 2024 18:05:11 -0300 Subject: [PATCH 01/15] v0.11.0-SNAPSHOT --- euclid.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/euclid.json b/euclid.json index 4503546..42227a3 100644 --- a/euclid.json +++ b/euclid.json @@ -1,6 +1,6 @@ { "github_token": "", - "version": "0.10.0", + "version": "0.11.0-SNAPSHOT", "tessellation_version": "2.3.3", "project_name": "custom-project", "framework": { From 17e515ff3a3fff5e580d22caf57aeb1b9ada9b79 Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Fri, 26 Apr 2024 17:01:14 -0300 Subject: [PATCH 02/15] Adding remote logs --- scripts/hydra | 18 ++++++++++ scripts/hydra-operations/logs.sh | 6 ++-- scripts/hydra-operations/remote-logs.sh | 48 +++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100755 scripts/hydra-operations/remote-logs.sh diff --git a/scripts/hydra b/scripts/hydra index bd041e9..04f3d7c 100755 --- a/scripts/hydra +++ b/scripts/hydra @@ -41,6 +41,7 @@ function load_scripts() { source ./hydra-operations/remote-deploy.sh source ./hydra-operations/remote-start.sh source ./hydra-operations/remote-status.sh + source ./hydra-operations/remote-logs.sh source ./hydra-operations/logs.sh source ./hydra-operations/install-monitoring-service.sh source ./hydra-operations/remote-deploy-monitoring-service.sh @@ -268,6 +269,23 @@ function remote-status() { exit 0 } + +# @cmd Get the logs from the remote hosts +# @arg host_name! Host name (node-1, node-2, node-3, monitoring) +# @arg layer! Layer name (global-l0, dag-l1, metagraph-l0, currency-l1, data-l1, monitoring) +# @option -n Retroactive rows from logs file (default 10) +function remote-logs() { + check_if_should_run_update + + build_paths + ensure_permissions_on_files_and_directories + load_scripts + + remote_logs + + exit 0 +} + # @cmd Update Euclid # @option --version! The Euclid version (https://github.com/Constellation-Labs/euclid-development-environment/releases) function update() { diff --git a/scripts/hydra-operations/logs.sh b/scripts/hydra-operations/logs.sh index 8212ea2..7de6356 100755 --- a/scripts/hydra-operations/logs.sh +++ b/scripts/hydra-operations/logs.sh @@ -19,8 +19,8 @@ function logs_containers() { exit 1 fi - if [ -z "$argc_retroactive_rows" ]; then - argc_retroactive_rows=10 + if [ -z "$argc_n" ]; then + argc_n=10 fi # Run the command inside the container to check if the directory exists @@ -29,7 +29,7 @@ function logs_containers() { # Run the command inside the container to check if the log file exists docker exec "$container_name" bash -c "[ -f '$layer/$layer.log' ]" if [ $? -eq 0 ]; then - docker exec -it $container_name bash -c "cd $layer && tail -f $layer.log -n $argc_retroactive_rows" + docker exec -it $container_name bash -c "cd $layer && tail -f $layer.log -n $argc_n" else echo_red "Layer $layer does not exists in $container_name" fi diff --git a/scripts/hydra-operations/remote-logs.sh b/scripts/hydra-operations/remote-logs.sh new file mode 100755 index 0000000..dda0f93 --- /dev/null +++ b/scripts/hydra-operations/remote-logs.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +function remote_logs() { + echo_title "################################## REMOTE LOGS ##################################" + echo_white + + valid_hosts=("node-1" "node-2" "node-3" "monitoring") + host_name=$argc_host_name + if [[ ! " ${valid_hosts[*]} " =~ " $host_name " ]]; then + echo_red "Invalid host" + exit 1 + fi + + valid_layers=("metagraph-l0" "currency-l1" "data-l1" "monitoring") + layer=$argc_layer + if [[ ! " ${valid_layers[*]} " =~ " $layer " ]]; then + echo_red "Invalid layer" + exit 1 + fi + + if [ -z "$argc_n" ]; then + argc_n=10 + fi + + if [[ "${host_name}" == "monitoring" ]]; then + check_monitoring_host_file + host=$(yq eval '.monitoring.hosts.monitoring-1.ansible_host' $ANSIBLE_HOSTS_FILE) + user=$(yq eval '.monitoring.hosts.monitoring-1.ansible_user' $ANSIBLE_HOSTS_FILE) + private_key=$(yq eval '.monitoring.hosts.monitoring-1.ansible_ssh_private_key_file' $ANSIBLE_HOSTS_FILE) + + echo_yellow "NOTE: TO STOP LOGGING PRESS CTRL + C" + echo_white + echo "SSH to the node..." + ssh -i "$private_key" $user@$host "echo 'Node connected'; cd code/dor-metagraph-integrationnet-monitoring-service/logs; latest_file=\$(ls -t application-* 2>/dev/null | head -n 1); if [[ -n \$latest_file ]]; then tail -f \"\$latest_file\" -n $argc_n; else echo 'No application log file found'; fi" + + else + check_nodes_host_file + host=$(yq eval ".nodes.hosts.$host_name.ansible_host" $ANSIBLE_HOSTS_FILE) + user=$(yq eval ".nodes.hosts.$host_name.ansible_user" $ANSIBLE_HOSTS_FILE) + private_key=$(yq eval ".nodes.hosts.$host_name.ansible_ssh_private_key_file" $ANSIBLE_HOSTS_FILE) + + echo_yellow "NOTE: TO STOP LOGGING PRESS CTRL + C" + echo_white + echo "SSH to the node..." + ssh -i "$private_key" $user@$host "echo 'Node connected'; cd code/$layer && if [ -f logs/app.log ]; then tail -f logs/app.log -n $argc_n; else echo 'Log file not found'; fi" + + fi +} From aab2c965ba33a6d88d3a9928a6b6d1ec544c91c3 Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Tue, 14 May 2024 20:25:19 -0300 Subject: [PATCH 03/15] Fixed build_paths being called twice --- scripts/hydra | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/scripts/hydra b/scripts/hydra index bd041e9..9935a2f 100755 --- a/scripts/hydra +++ b/scripts/hydra @@ -60,9 +60,9 @@ function load_scripts() { # @cmd Installs a local framework and detaches project function install() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -77,9 +77,9 @@ function install() { # @option --path Template path on repository (default examples) # @flag --list List available templates function install-template() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -100,9 +100,9 @@ function install-template() { # @flag --no_cache Build docker containers with no cache # @flag --run Run containers after build function build() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -117,9 +117,9 @@ function build() { # @cmd Start containers from the genesis snapshot (erasing history) # @alias start_genesis function start-genesis() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -134,9 +134,9 @@ function start-genesis() { # @cmd Start containers from the last snapshot (maintaining history) # @alias start_rollback function start-rollback() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -150,9 +150,9 @@ function start-rollback() { # @cmd Stop containers function stop() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -169,12 +169,12 @@ function stop() { # @cmd Destroy containers # @flag --delete_project Delete custom project function destroy() { + build_paths check_if_should_run_update export FORCE_ROLLBACK="" export METAGRAPH_ID="" - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -189,12 +189,12 @@ function destroy() { # @cmd Destroy containers and images # @flag --delete_project Delete custom project function purge() { + build_paths check_if_should_run_update export FORCE_ROLLBACK="" export METAGRAPH_ID="" - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -208,9 +208,9 @@ function purge() { # @cmd Check the status of the containers function status() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -228,9 +228,9 @@ function status() { # -> DEFAULT_ANSIBLE_NODES_DEPLOY_PLAYBOOK_FILE: infra/ansible/remote/nodes/playbooks/deploy.ansible.yml # @flag --force_genesis Force metagraph to deploy as genesis function remote-deploy() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -245,9 +245,9 @@ function remote-deploy() { # -> DEFAULT_ANSIBLE_START_PLAYBOOK_FILE: infra/ansible/remote/nodes/playbooks/start/start.ansible.yml # @flag --force_genesis Force metagraph to run as genesis function remote-start() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -258,9 +258,9 @@ function remote-start() { # @cmd Check the status of the remote nodes function remote-status() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -289,9 +289,9 @@ function update() { # @arg layer! Layer name (global-l0, dag-l1, metagraph-l0, currency-l1, data-l1) # @option -n Retroactive rows from logs file (default 10) function logs() { + build_paths check_if_should_run_update - build_paths ensure_permissions_on_files_and_directories load_scripts @@ -303,9 +303,9 @@ function logs() { # @cmd Download the metagraph-monitoring-service (https://github.com/Constellation-Labs/metagraph-monitoring-service) # @alias install_monitoring_service function install-monitoring-service() { + build_paths check_if_should_run_update - build_paths load_scripts ensure_permissions_on_files_and_directories @@ -319,9 +319,9 @@ function install-monitoring-service() { # -> DEFAULT_ANSIBLE_HOSTS_FILE: infra/ansible/remote/hosts.ansible.yml # -> DEFAULT_ANSIBLE_DEPLOY_PLAYBOOK_FILE: infra/ansible/remote/monitoring/playbooks/deploy/deploy.ansible.yml function remote-deploy-monitoring-service() { + build_paths check_if_should_run_update - build_paths load_scripts ensure_permissions_on_files_and_directories @@ -336,9 +336,9 @@ function remote-deploy-monitoring-service() { # -> DEFAULT_ANSIBLE_START_PLAYBOOK_FILE: infra/ansible/remote/monitoring/playbooks/start/start.ansible.yml # @flag --force_restart Force metagraph restart function remote-start-monitoring-service() { + build_paths check_if_should_run_update - build_paths load_scripts ensure_permissions_on_files_and_directories @@ -349,7 +349,6 @@ function remote-start-monitoring-service() { # The functions below are made for backward compatibility, they will be moved to scripts/hydra-operations/update in the next releases function check_if_should_run_update() { - build_paths ensure_permissions_on_files_and_directories DIRECTORY="$SCRIPTS_PATH/hydra-operations" From 2a1dd287e2ec11d417b5ef62f6ee2a0ad9b341d5 Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Tue, 14 May 2024 20:46:20 -0300 Subject: [PATCH 04/15] Updating check_if_docker_is_running to throws error in versions lower than 26.0.0 --- scripts/docker/operations.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/docker/operations.sh b/scripts/docker/operations.sh index 965dab5..90da4de 100755 --- a/scripts/docker/operations.sh +++ b/scripts/docker/operations.sh @@ -28,6 +28,19 @@ function check_if_docker_is_running() { exit 1 fi + required_version="26.0.0" + installed_version=$(docker --version | awk '{print $3}' | sed 's/,//') + + if [[ -z "$installed_version" ]]; then + echo_red "Docker is not installed or not found in the PATH." + exit 1 + fi + + if [[ "$(printf '%s\n' "$required_version" "$installed_version" | sort -V | head -n1)" != "$required_version" ]]; then + echo_red "Docker version is lower than $required_version. Installed version: $installed_version" + exit 1 + fi + if [[ $(uname) == "Darwin" ]]; then DOCKER_SOCKET="/var/run/docker.sock" if [ ! -e "$DOCKER_SOCKET" ]; then From 0f484a0fe07579aab5114b6abc7a4f0e16c1c1bd Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Fri, 17 May 2024 12:23:21 -0300 Subject: [PATCH 05/15] Fixing action --- .github/workflows/actions.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 407592e..6c2fe7f 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -5,15 +5,41 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + + - name: Uninstall current Docker version + run: | + sudo apt-get remove -y docker docker-engine docker.io containerd runc + + - name: Install Docker 26.1.1 + run: | + sudo apt-get update + sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce=5:26.1.1~3-0~ubuntu-focal docker-ce-cli=5:26.1.1~3-0~ubuntu-focal containerd.io + + - name: Verify Docker installation + run: docker --version + - name: Install jq shell: bash run: | sudo apt install -y jq + - name: Load config file shell: bash run: | contents="$(jq --arg GITHUB_TOKEN "${{ secrets.TOKEN }}" '.github_token = $GITHUB_TOKEN' euclid.json)" echo -E "${contents}" > euclid.json + - name: Run steps shell: bash run: | From c772806a8ecd693a89f28232ad7ccd8d67ef0665 Mon Sep 17 00:00:00 2001 From: Marcus Girolneto Date: Fri, 17 May 2024 12:25:14 -0300 Subject: [PATCH 06/15] Fixing action --- .github/workflows/actions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 6c2fe7f..f3adc8e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -10,7 +10,7 @@ jobs: run: | sudo apt-get remove -y docker docker-engine docker.io containerd runc - - name: Install Docker 26.1.1 + - name: Install Docker run: | sudo apt-get update sudo apt-get install -y \ @@ -24,7 +24,7 @@ jobs: "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install -y docker-ce=5:26.1.1~3-0~ubuntu-focal docker-ce-cli=5:26.1.1~3-0~ubuntu-focal containerd.io + sudo apt-get install -y docker-ce docker-ce-cli containerd.io - name: Verify Docker installation run: docker --version From 0fd86c72a1bd5ddc11999b5c08e32c974917a11d Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Fri, 28 Jun 2024 12:37:18 -0300 Subject: [PATCH 07/15] Adding messages fees --- .gitignore | 8 +- README.md | 39 +++ euclid.json | 18 +- .../nodes/playbooks/deploy/deploy.ansible.yml | 35 ++- .../start/metagraph-l0/genesis.ansible.yml | 248 ++++++++++++++++-- scripts/hydra | 57 +++- scripts/hydra-operations/remote-deploy.sh | 12 +- .../remote-snapshot-fee-config.sh | 51 ++++ scripts/hydra-operations/remote-start.sh | 42 ++- scripts/migrations/migrations.sh | 18 +- scripts/migrations/v0.11.0.sh | 23 ++ scripts/utils/get-information.sh | 8 + scripts/utils/validations.sh | 75 ++++++ 13 files changed, 597 insertions(+), 37 deletions(-) create mode 100755 scripts/hydra-operations/remote-snapshot-fee-config.sh create mode 100755 scripts/migrations/v0.11.0.sh diff --git a/.gitignore b/.gitignore index 7d2b8c9..cab9dec 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,10 @@ infra/docker/shared/jars/**.jar source/project/* source/metagraph-l0/genesis/genesis.address source/metagraph-l0/genesis/genesis.snapshot -source/*-monitoring-service \ No newline at end of file +source/*-monitoring-service +source/p12-files/* +infra/docker/shared/genesis/* + +!source/p12-files/token-key-1.p12 +!source/p12-files/token-key-2.p12 +!source/p12-files/token-key-3.p12 \ No newline at end of file diff --git a/README.md b/README.md index c71f43f..5958291 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,8 @@ COMMANDS: remote-deploy Remotely deploy to cloud instances using Ansible [aliases: remote_deploy] remote-start Remotely start the metagraph on cloud instances using Ansible [aliases: remote_start] remote-status Check the status of the remote nodes + remote-logs Get the logs from the remote hosts + remote-snapshot-fee-config Get the remote snapshot fee config update Update Euclid logs Get the logs from containers install-monitoring-service Download the metagraph-monitoring-service (https://github.com/Constellation-Labs/metagraph-monitoring-service) [aliases: install_monitoring_service] @@ -305,7 +307,24 @@ Each directory will be created with `cl-keytool.jar`, `cl-wallet.jar`, and a P12 **In `code/data-l1`:** - data-l1.jar // The executable for the dL1 layer +#### Parameters +If you run the command `./hydra remote-deploy -h` you should see something like +``` +Remotely deploy to cloud instances using Ansible + +USAGE: hydra remote-deploy [OPTIONS] + +OPTIONS: + --force_genesis Force metagraph to deploy as genesis + -h, --help +``` +* --force_genesis: Use this flag when you want your metagraph to start from genesis. If you have already started a metagraph and need to execute from genesis again, use this command. + +* --owner_p12_file_name: This parameter specifies the wallet to charge fees on snapshots. Ensure this file is in the `p12-files` directory. We will validate if the file exists in the directory and, if so, send it to the remote `metagraph-l0` directory. + +* --staking_p12_file_name: This parameter specifies the wallet to check the balance to reduce fees according to the amount of DAG in the balance. Ensure this file is in the `p12-files` directory. We will validate if the file exists in the directory and, if so, send it to the remote `metagraph-l0` directory. + ### `hydra remote-start` This method initiates the remote startup of your metagraph in one of the available networks: integrationnet or mainnet. The network should be set in `euclid.json` under `deploy` -> `network` @@ -340,6 +359,26 @@ Each layer directory on every node contains a folder named `logs`. You can monit **NOTE:** Don't forget to add your hosts' information, such as host, user, and SSH key file, to your `infra/ansible/remote/hosts.ansible.yml` file. +#### Parameters +If you run the command `./hydra remote-start -h` you should see something like + +``` +Remotely start the metagraph on cloud instances using Ansible + +USAGE: hydra remote-start [OPTIONS] + +OPTIONS: + --force_genesis Force metagraph to run as genesis + --force_owner_message Force to send owner message + --force_staking_message Force to send staking message + -h, --help +``` +* --force_genesis: Use this flag when you want your metagraph to start from genesis. If you have already started a metagraph and need to execute from genesis again, use this command. +* --force_owner_message: Use this flag when you want your metagraph to force sending the owner message. For example, if you want to change the owner address, provide this flag. +* --force_staking_message: Use this flag when you want your metagraph to force sending the staking message. For example, if you want to change the staking address, provide this flag. + +**NOTE:** When you provide any of the owner or staking parameters, you must provide all the parameters, otherwise, it will fail. + ### `hydra remote-status` This method will return the status of your remote hosts. You should see the following: ``` diff --git a/euclid.json b/euclid.json index 42227a3..b7286a0 100644 --- a/euclid.json +++ b/euclid.json @@ -1,6 +1,6 @@ { "github_token": "", - "version": "0.11.0-SNAPSHOT", + "version": "0.11.0", "tessellation_version": "2.3.3", "project_name": "custom-project", "framework": { @@ -46,6 +46,22 @@ "docker": { "start_grafana_container": false }, + "snapshot_fees": { + "owner": { + "key_file": { + "name": "token-key.p12", + "alias": "token-key", + "password": "password" + } + }, + "staking": { + "key_file": { + "name": "token-key-1.p12", + "alias": "token-key-1", + "password": "password" + } + } + }, "deploy": { "network": { "name": "integrationnet|mainnet", diff --git a/infra/ansible/remote/nodes/playbooks/deploy/deploy.ansible.yml b/infra/ansible/remote/nodes/playbooks/deploy/deploy.ansible.yml index 6a24d35..a2b0da7 100755 --- a/infra/ansible/remote/nodes/playbooks/deploy/deploy.ansible.yml +++ b/infra/ansible/remote/nodes/playbooks/deploy/deploy.ansible.yml @@ -195,4 +195,37 @@ - name: Cleaning file shell: | - rm -f /home/{{ ansible_user }}/code/{{ all_nodes[0].key_file.name }} \ No newline at end of file + rm -f /home/{{ ansible_user }}/code/{{ all_nodes[0].key_file.name }} + +- name: Send fees p12 files to nodes + hosts: nodes + gather_facts: false + vars: + owner_p12_file_name: "{{ owner_p12_file_name }}" + staking_p12_file_name: "{{ staking_p12_file_name }}" + tasks: + - name: Check if owner file exists + stat: + path: "{{ lookup('env', 'SOURCE_PATH') }}/p12-files/{{ owner_p12_file_name }}" + register: owner_p12_file + delegate_to: localhost + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") + + - name: Copy owner file to remote node + copy: + src: "{{ lookup('env', 'SOURCE_PATH') }}/p12-files/{{ owner_p12_file_name }}" + dest: "/home/{{ ansible_user }}/code/metagraph-l0/{{ owner_p12_file_name }}" + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") and owner_p12_file.stat.exists + + - name: Check if staking file exists + stat: + path: "{{ lookup('env', 'SOURCE_PATH') }}/p12-files/{{ staking_p12_file_name }}" + register: staking_p12_file + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") + delegate_to: localhost + + - name: Copy staking file to remote node if it exists + copy: + src: "{{ lookup('env', 'SOURCE_PATH') }}/p12-files/{{ staking_p12_file_name }}" + dest: "/home/{{ ansible_user }}/code/metagraph-l0/{{ staking_p12_file_name }}" + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") and staking_p12_file.stat.exists \ No newline at end of file diff --git a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml index 9bf90b6..9aee213 100755 --- a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml +++ b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml @@ -1,25 +1,42 @@ --- -- name: Check execution type +- name: Check if we already have incremental snapshots + stat: + path: "/home/{{ ansible_user }}/code/metagraph-l0/data/incremental_snapshot" + register: folder_exists + +- name: Set default values for variables if not defined + set_fact: + force_genesis: "{{ force_genesis | default(false) }}" + force_owner_message: "{{ force_owner_message | default(false) }}" + force_staking_message: "{{ force_staking_message | default(false) }}" + +- name: Convert execution type variables to boolean set_fact: - force_genesis_bool: "{{ force_genesis | default(true) | bool }}" + force_genesis_bool: "{{ force_genesis | bool }}" + force_owner_message_bool: "{{ force_owner_message | bool }}" + force_staking_message_bool: "{{ force_staking_message | bool }}" + +- name: Determine if should run genesis + set_fact: + should_run_genesis: "{{ not folder_exists.stat.exists or force_genesis_bool }}" - name: Get current timestamp set_fact: current_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S%z') }}" - when: force_genesis_bool + when: should_run_genesis - name: Ensure archived-data directory exists ansible.builtin.file: path: /home/{{ ansible_user }}/code/metagraph-l0/archived-data state: directory - when: force_genesis_bool + when: should_run_genesis - name: Save previous data directory shell: | cd "/home/{{ ansible_user }}/code/metagraph-l0" mv data archived-data/data_{{ current_time }} ignore_errors: true - when: force_genesis_bool + when: should_run_genesis - name: Check if metagraph-l0.jar exists stat: @@ -41,12 +58,143 @@ msg: "File /home/{{ ansible_user }}/code/metagraph-l0/{{ cl_keystore }} does not exist" when: not token_file.stat.exists -- name: Check if we already have incremental snapshots +- name: Check if owner file exists stat: - path: "/home/{{ ansible_user }}/code/metagraph-l0/data/incremental_snapshot" - register: folder_exists + path: "/home/{{ ansible_user }}/code/metagraph-l0/{{ owner_p12_file_name }}" + register: owner_file + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") -- name: Start as rollback +- name: Throw an error if the owner file doesn't exist + fail: + msg: "File /home/{{ ansible_user }}/code/metagraph-l0/{{ owner_p12_file_name }} does not exist" + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") and not owner_file.stat.exists + +- name: Check if staking file exists + stat: + path: "/home/{{ ansible_user }}/code/metagraph-l0/{{ staking_p12_file_name }}" + register: staking_file + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Throw an error if the staking file doesn't exist + fail: + msg: "File /home/{{ ansible_user }}/code/metagraph-l0/{{ staking_p12_file_name }} does not exist" + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") and not staking_file.stat.exists + +- name: Fetch the latest combined snapshot + uri: + url: "http://{{ gl0_ip }}:{{ gl0_port }}/global-snapshots/latest/combined" + method: GET + return_content: yes + headers: + Accept: "application/json" + register: latest_combined_snapshot_response + when: should_run_genesis or force_owner_message_bool or force_staking_message_bool + +- name: Check if the fetch was successful + fail: + msg: "Error fetching the latest combined snapshot." + when: (should_run_genesis or force_owner_message_bool or force_staking_message_bool) and latest_combined_snapshot_response.status != 200 + +- name: Convert JSON response content to a fact + set_fact: + json_response: "{{ latest_combined_snapshot_response.content | from_json }}" + when: should_run_genesis or force_owner_message_bool or force_staking_message_bool + +- name: Parse the response to extract the last messages + set_fact: + snapshot_fees_messages: "{{ json_response | json_query('[1].lastCurrencySnapshots.' ~ metagraph_id ~ '.Right[1].lastMessages') }}" + when: should_run_genesis or force_owner_message_bool or force_staking_message_bool + +- name: Ensure snapshot_fees_messages is defined + set_fact: + snapshot_fees_messages: {} + when: snapshot_fees_messages is not defined or snapshot_fees_messages is none + +- name: Extract Owner parentOrdinal or default to none + set_fact: + owner_parent_ordinal_raw: "{{ snapshot_fees_messages['Owner']['value']['parentOrdinal'] | default(None) }}" + when: (should_run_genesis or force_owner_message_bool) and not (owner_p12_file_name is undefined or owner_p12_file_name == "") + +- name: Extract Staking parentOrdinal or default to none + set_fact: + staking_parent_ordinal_raw: "{{ snapshot_fees_messages['Staking']['value']['parentOrdinal'] | default(None) }}" + when: (should_run_genesis or force_staking_message_bool) and not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Calculate owner_parent_ordinal + set_fact: + owner_parent_ordinal: >- + {% if owner_parent_ordinal_raw is not none and owner_parent_ordinal_raw != "" %} + {{ (owner_parent_ordinal_raw | int) + 1 }} + {% else %} + 0 + {% endif %} + when: (should_run_genesis or force_owner_message_bool) and not (owner_p12_file_name is undefined or owner_p12_file_name == "") + +- name: Calculate staking_parent_ordinal + set_fact: + staking_parent_ordinal: >- + {% if staking_parent_ordinal_raw is not none and staking_parent_ordinal_raw != "" %} + {{ (staking_parent_ordinal_raw | int) + 1 }} + {% else %} + 0 + {% endif %} + when: (should_run_genesis or force_staking_message_bool) and not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Trim owner_parent_ordinal + set_fact: + owner_parent_ordinal: "{{ owner_parent_ordinal | trim }}" + when: (should_run_genesis or force_owner_message_bool) and not (owner_p12_file_name is undefined or owner_p12_file_name == "") + +- name: Trim staking_parent_ordinal + set_fact: + staking_parent_ordinal: "{{ staking_parent_ordinal | trim }}" + when: (should_run_genesis or force_staking_message_bool) and not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Get owner address + environment: + CL_KEYSTORE: "{{ owner_p12_file_name }}" + CL_KEYALIAS: "{{ owner_p12_alias }}" + CL_PASSWORD: "{{ owner_p12_password }}" + shell: | + cd "/home/{{ ansible_user }}/code/metagraph-l0" + java -jar cl-wallet.jar show-address + register: owner_address_output + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") + +- name: Get owner message + environment: + CL_KEYSTORE: "{{ owner_p12_file_name }}" + CL_KEYALIAS: "{{ owner_p12_alias }}" + CL_PASSWORD: "{{ owner_p12_password }}" + shell: | + cd "/home/{{ ansible_user }}/code/metagraph-l0" + java -jar cl-wallet.jar create-owner-signing-message --address {{ owner_address_output.stdout }} --parentOrdinal {{ owner_parent_ordinal }} --metagraphId {{ metagraph_id }} + register: owner_message_output + when: not (owner_p12_file_name is undefined or owner_p12_file_name == "") + +- name: Get staking address + environment: + CL_KEYSTORE: "{{ staking_p12_file_name }}" + CL_KEYALIAS: "{{ staking_p12_alias }}" + CL_PASSWORD: "{{ staking_p12_password }}" + shell: | + cd "/home/{{ ansible_user }}/code/metagraph-l0" + java -jar cl-wallet.jar show-address + register: staking_address_output + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Get staking message + environment: + CL_KEYSTORE: "{{ staking_p12_file_name }}" + CL_KEYALIAS: "{{ staking_p12_alias }}" + CL_PASSWORD: "{{ staking_p12_password }}" + shell: | + cd "/home/{{ ansible_user }}/code/metagraph-l0" + java -jar cl-wallet.jar create-staking-signing-message --address {{ staking_address_output.stdout }} --parentOrdinal {{ staking_parent_ordinal }} --metagraphId {{ metagraph_id }} + register: staking_message_output + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") + +- name: Start as genesis environment: CL_PUBLIC_HTTP_PORT: "{{ base_metagraph_l0_public_port }}" CL_P2P_HTTP_PORT: "{{ base_metagraph_l0_p2p_port }}" @@ -62,13 +210,44 @@ CL_APP_ENV: "{{ network }}" CL_COLLATERAL: 0 - CL_L0_TOKEN_IDENTIFIER: "{{ metagraph_id }}" shell: | - cd "/home/{{ ansible_user }}/code/metagraph-l0" - nohup java -jar metagraph-l0.jar run-rollback --ip {{ ansible_host }} > metagraph-l0.log 2>&1 & - when: folder_exists.stat.exists and not force_genesis_bool + cd /home/{{ ansible_user }}/code/metagraph-l0 + nohup java -jar metagraph-l0.jar run-genesis genesis.snapshot --ip {{ ansible_host }} > metagraph-l0.log 2>&1 & + when: should_run_genesis -- name: Start as genesis +- name: Check if node is Ready + uri: + url: "http://localhost:{{ base_metagraph_l0_public_port }}/node/info" + method: GET + return_content: yes + register: response + until: response.status == 200 and ("Ready" in response.content | string or retries >= 100) + retries: 120 + delay: 1 + vars: + retries: 0 + when: should_run_genesis + +- name: Find metagraph-l0 process ID by port + shell: "lsof -t -i:{{ base_metagraph_l0_public_port }}" + register: l0_process_id + ignore_errors: true + when: should_run_genesis + +- name: Kill metagraph-l0 process + shell: "sudo kill -9 {{ l0_process_id.stdout }}" + ignore_errors: true + when: should_run_genesis + +- name: Wait 2 minutes before starting the metagraph + pause: + minutes: 2 + when: > + owner_p12_file_name is defined and + owner_p12_file_name != "" and + should_run_genesis or force_owner_message_bool + +- name: Start as rollback environment: CL_PUBLIC_HTTP_PORT: "{{ base_metagraph_l0_public_port }}" CL_P2P_HTTP_PORT: "{{ base_metagraph_l0_p2p_port }}" @@ -84,11 +263,46 @@ CL_APP_ENV: "{{ network }}" CL_COLLATERAL: 0 + CL_L0_TOKEN_IDENTIFIER: "{{ metagraph_id }}" shell: | - cd /home/{{ ansible_user }}/code/metagraph-l0 - nohup java -jar metagraph-l0.jar run-genesis genesis.snapshot --ip {{ ansible_host }} > metagraph-l0.log 2>&1 & - when: not folder_exists.stat.exists or force_genesis_bool + cd "/home/{{ ansible_user }}/code/metagraph-l0" + nohup java -jar metagraph-l0.jar run-rollback --ip {{ ansible_host }} > metagraph-l0.log 2>&1 & +- name: Check if node is online + uri: + url: "http://localhost:{{ base_metagraph_l0_public_port }}/node/info" + method: GET + return_content: yes + register: response + until: response.status == 200 + retries: 120 + delay: 1 + vars: + retries: 0 + +- name: Send owner message + uri: + url: "http://localhost:{{ base_metagraph_l0_public_port }}/currency/message" + method: POST + body: "{{ owner_message_output.stdout }}" + body_format: json + status_code: 204, 200 + register: owner_response + when: > + owner_p12_file_name is defined and + owner_p12_file_name != "" and + should_run_genesis or force_owner_message_bool + +- name: Send staking message + uri: + url: "http://localhost:{{ base_metagraph_l0_public_port }}/currency/message" + method: POST + body: "{{ staking_message_output.stdout }}" + body_format: json + status_code: 204, 200 + register: staking_response + when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") or force_staking_message_bool + - name: Check if node is Ready uri: url: "http://localhost:{{ base_metagraph_l0_public_port }}/node/info" diff --git a/scripts/hydra b/scripts/hydra index 7ab61c5..7361323 100755 --- a/scripts/hydra +++ b/scripts/hydra @@ -1,5 +1,5 @@ #!/usr/bin/env bash -function ensure_permissions_on_files_and_directories(){ +function ensure_permissions_on_files_and_directories() { chmod -R a+rwx $INFRA_PATH 2>/dev/null chmod -R a+rwx $SCRIPTS_PATH 2>/dev/null chmod -R a+rwx $SOURCE_PATH 2>/dev/null @@ -13,7 +13,7 @@ function build_paths() { [[ $SCRIPT_PATH != /* ]] && SCRIPT_PATH="$DIR/$SCRIPT_PATH" done SCRIPT_PATH="$(cd -P "$(dirname "$SCRIPT_PATH")" && pwd)" - + cd "$SCRIPT_PATH" cd .. export ROOT_PATH=$(pwd) @@ -46,6 +46,7 @@ function load_scripts() { source ./hydra-operations/install-monitoring-service.sh source ./hydra-operations/remote-deploy-monitoring-service.sh source ./hydra-operations/remote-start-monitoring-service.sh + source ./hydra-operations/remote-snapshot-fee-config.sh source ./template/custom-template.sh @@ -227,7 +228,7 @@ function status() { # @alias remote_deploy # -> DEFAULT_ANSIBLE_HOSTS_FILE: infra/ansible/remote/hosts.ansible.yml # -> DEFAULT_ANSIBLE_NODES_DEPLOY_PLAYBOOK_FILE: infra/ansible/remote/nodes/playbooks/deploy.ansible.yml -# @flag --force_genesis Force metagraph to deploy as genesis +# @flag --force_genesis Force metagraph to deploy as genesis function remote-deploy() { build_paths check_if_should_run_update @@ -235,6 +236,12 @@ function remote-deploy() { ensure_permissions_on_files_and_directories load_scripts + check_if_owner_and_staking_address_are_equal + confirm_force_genesis + + check_metagraph_owner_fees_file_exists $SNAPSHOT_FEES_OWNER_FILE_NAME + check_metagraph_staking_fees_file_exists $SNAPSHOT_FEES_STAKING_FILE_NAME + remote_deploy_metagraph exit 0 @@ -244,13 +251,21 @@ function remote-deploy() { # @alias remote_start # -> DEFAULT_ANSIBLE_HOSTS_FILE: infra/ansible/remote/hosts.ansible.yml # -> DEFAULT_ANSIBLE_START_PLAYBOOK_FILE: infra/ansible/remote/nodes/playbooks/start/start.ansible.yml -# @flag --force_genesis Force metagraph to run as genesis +# @flag --force_genesis Force metagraph to run as genesis +# @flag --force_owner_message Force to send owner message +# @flag --force_staking_message Force to send owner message function remote-start() { build_paths check_if_should_run_update ensure_permissions_on_files_and_directories load_scripts + + check_if_owner_and_staking_address_are_equal + confirm_force_genesis + + check_metagraph_owner_fees_information $SNAPSHOT_FEES_OWNER_FILE_NAME $SNAPSHOT_FEES_OWNER_ALIAS $SNAPSHOT_FEES_OWNER_PASSWORD + check_metagraph_staking_fees_information $SNAPSHOT_FEES_STAKING_FILE_NAME $SNAPSHOT_FEES_STAKING_ALIAS $SNAPSHOT_FEES_STAKING_PASSWORD remote_start_metagraph @@ -275,13 +290,26 @@ function remote-status() { # @arg layer! Layer name (global-l0, dag-l1, metagraph-l0, currency-l1, data-l1, monitoring) # @option -n Retroactive rows from logs file (default 10) function remote-logs() { + build_paths + check_if_should_run_update + ensure_permissions_on_files_and_directories + load_scripts + + remote_logs + + exit 0 +} +# @cmd Get the remote snapshot fee config +function remote-snapshot-fee-config() { build_paths + + check_if_should_run_update ensure_permissions_on_files_and_directories load_scripts - remote_logs + remote_snapshot_fee_config exit 0 } @@ -294,9 +322,9 @@ function update() { if [ -d "$SCRIPTS_PATH/hydra-operations" ]; then load_scripts fi - + update_euclid - + ensure_permissions_on_files_and_directories exit 0 @@ -326,7 +354,7 @@ function install-monitoring-service() { load_scripts ensure_permissions_on_files_and_directories - + install_monitoring_service exit 0 @@ -342,7 +370,7 @@ function remote-deploy-monitoring-service() { load_scripts ensure_permissions_on_files_and_directories - + remote_deploy_monitoring_service exit 0 @@ -359,7 +387,7 @@ function remote-start-monitoring-service() { load_scripts ensure_permissions_on_files_and_directories - + remote_start_monitoring_service exit 0 @@ -388,6 +416,13 @@ function check_if_should_run_update() { load_scripts run_migrations fi + + local snapshot_fees=$(jq -r '.snapshot_fees // empty' "$ROOT_PATH/euclid.json") + if [ -z "$snapshot_fees" ]; then + echo "$(tput setaf 3) Could not find the snapshot_fees field on euclid.json file, the file is probably outdated. Running migrations..." + load_scripts + run_migrations + fi } function check_if_the_container_is_running() { @@ -543,7 +578,7 @@ function update_euclid() { update_scripts update_remote_ansible_files update_local_ansible_files - + if command -v run_migrations &>/dev/null; then run_migrations fi diff --git a/scripts/hydra-operations/remote-deploy.sh b/scripts/hydra-operations/remote-deploy.sh index bc70678..39517b1 100755 --- a/scripts/hydra-operations/remote-deploy.sh +++ b/scripts/hydra-operations/remote-deploy.sh @@ -24,7 +24,13 @@ function remote_deploy_metagraph() { else force_genesis=false fi - - ansible-playbook -e "force_genesis=$force_genesis" -e "deploy_cl1=$deploy_cl1" -e "deploy_dl1=$deploy_dl1" -i $ANSIBLE_HOSTS_FILE $ANSIBLE_NODES_DEPLOY_PLAYBOOK_FILE - + + ansible-playbook \ + -e "force_genesis=$force_genesis" \ + -e "deploy_cl1=$deploy_cl1" \ + -e "deploy_dl1=$deploy_dl1" \ + -e "owner_p12_file_name=$SNAPSHOT_FEES_OWNER_FILE_NAME" \ + -e "staking_p12_file_name=$SNAPSHOT_FEES_STAKING_FILE_NAME" \ + -i $ANSIBLE_HOSTS_FILE $ANSIBLE_NODES_DEPLOY_PLAYBOOK_FILE + } diff --git a/scripts/hydra-operations/remote-snapshot-fee-config.sh b/scripts/hydra-operations/remote-snapshot-fee-config.sh new file mode 100755 index 0000000..086651b --- /dev/null +++ b/scripts/hydra-operations/remote-snapshot-fee-config.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +function fetch_latest_global_snapshot_metagraph_messages() { + local metagraph_id=$1 + URL="http://$DEPLOY_NETWORK_HOST_IP:$DEPLOY_NETWORK_HOST_PUBLIC_PORT/global-snapshots/latest/combined" # Replace with your actual URL + HEADER="Content-Type: application/json" + + echo_title "Fetching latest global snapshot from $URL" + response=$(curl -s -H "$HEADER" "$URL") + http_code=$(curl -s -o /dev/null -w "%{http_code}" -H "$HEADER" "$URL") + + if [ "$http_code" -ne 200 ]; then + echo_red "Failed to fetch data. HTTP Status Code: $http_code" + exit 1 + else + last_messages=$(echo "$response" | jq -r ".[1].lastCurrencySnapshots.\"$metagraph_id\".Right[1].lastMessages") + + if [ -z "$last_messages" ]; then + echo_red "Failed when extracting the fee configuration from global snapshot" + exit 1 + else + echo_green "Last messages extracted successfully:" + echo "$last_messages" | jq . + fi + fi + +} +function remote_snapshot_fee_config() { + echo_title "################################## REMOTE SNAPSHOT FEE CONFIG ##################################" + echo_white + + check_nodes_host_file + host=$(yq eval ".nodes.hosts.node-1.ansible_host" $ANSIBLE_HOSTS_FILE) + user=$(yq eval ".nodes.hosts.node-1.ansible_user" $ANSIBLE_HOSTS_FILE) + private_key=$(yq eval ".nodes.hosts.node-1.ansible_ssh_private_key_file" $ANSIBLE_HOSTS_FILE) + + echo + echo_title "Fetching the metagraph-id in node $host" + echo_yellow "SSH to the node..." + local metagraph_id=$(ssh -i "$private_key" $user@$host "cd code/metagraph-l0; cat genesis.address") + + if [ $? -ne 0 ]; then + echo_red "SSH command failed. Please check the connection and try again." + return 1 + else + echo_green "Metagraph ID: $metagraph_id" + echo + fetch_latest_global_snapshot_metagraph_messages $metagraph_id + fi + +} diff --git a/scripts/hydra-operations/remote-start.sh b/scripts/hydra-operations/remote-start.sh index 3f21cef..4d59c1d 100755 --- a/scripts/hydra-operations/remote-start.sh +++ b/scripts/hydra-operations/remote-start.sh @@ -15,6 +15,44 @@ function remote_start_metagraph() { force_genesis=false fi - ansible-playbook -e "force_genesis=$force_genesis" -i $ANSIBLE_HOSTS_FILE $ANSIBLE_NODES_START_PLAYBOOK_FILE - + if [ ! -z "$argc_force_owner_message" ]; then + force_owner_message=true + else + force_owner_message=false + fi + + if [ ! -z "$argc_force_staking_message" ]; then + force_staking_message=true + else + force_staking_message=false + fi + + # Validate owner parameters + if [ "$force_owner_message" = "true" ]; then + if [ -z "$SNAPSHOT_FEES_OWNER_FILE_NAME" ] || [ -z "$SNAPSHOT_FEES_OWNER_ALIAS" ] || [ -z "$SNAPSHOT_FEES_OWNER_PASSWORD" ]; then + echo "Error: When force_owner_message is set, you must provide owner_p12_file_name, owner_p12_alias, and owner_p12_password." + exit 1 + fi + fi + + # Validate staking parameters + if [ "$force_staking_message" = "true" ]; then + if [ -z "$SNAPSHOT_FEES_STAKING_FILE_NAME" ] || [ -z "$SNAPSHOT_FEES_STAKING_ALIAS" ] || [ -z "$SNAPSHOT_FEES_STAKING_PASSWORD" ]; then + echo "Error: When force_staking_message is set, you must provide staking_p12_file_name, staking_p12_alias, and staking_p12_password." + exit 1 + fi + fi + + + ansible-playbook \ + -e "force_genesis=$force_genesis" \ + -e "force_owner_message=$force_owner_message" \ + -e "force_staking_message=$force_staking_message" \ + -e "owner_p12_file_name=$SNAPSHOT_FEES_OWNER_FILE_NAME" \ + -e "owner_p12_alias=$SNAPSHOT_FEES_OWNER_ALIAS" \ + -e "owner_p12_password=$SNAPSHOT_FEES_OWNER_PASSWORD" \ + -e "staking_p12_file_name=$SNAPSHOT_FEES_STAKING_FILE_NAME" \ + -e "staking_p12_alias=$SNAPSHOT_FEES_STAKING_ALIAS" \ + -e "staking_p12_password=$SNAPSHOT_FEES_STAKING_PASSWORD" \ + -i $ANSIBLE_HOSTS_FILE $ANSIBLE_NODES_START_PLAYBOOK_FILE } diff --git a/scripts/migrations/migrations.sh b/scripts/migrations/migrations.sh index 3ce78d5..06f985f 100755 --- a/scripts/migrations/migrations.sh +++ b/scripts/migrations/migrations.sh @@ -1,5 +1,11 @@ function version_greater_than() { - [[ "$1" = "$(echo -e "$1\n$2" | sort -V | tail -n 1)" ]] && [[ "$1" != "$2" ]] + local version1="$1" + local version2="$2" + if [[ "$version1" = "$(echo -e "$version1\n$version2" | sort -V | tail -n 1)" ]] && [[ "$version1" != "$version2" ]]; then + return 0 + else + return 1 + fi } function run_migrations() { @@ -9,6 +15,7 @@ function run_migrations() { CURRENT_VERSION=$(jq -r '.version // "0.0.0"' "$ROOT_PATH/euclid.json") CURRENT_VERSION_WITHOUT_V="${CURRENT_VERSION#v}" + echo_yellow "Current version: $CURRENT_VERSION_WITHOUT_V" if version_greater_than "0.9.0" $CURRENT_VERSION_WITHOUT_V; then echo "Running migration v0.9.0" @@ -28,6 +35,15 @@ function run_migrations() { jq --arg current_version "$current_version" '.version = $current_version' $ROOT_PATH/euclid.json > $ROOT_PATH/temp.json && mv $ROOT_PATH/temp.json $ROOT_PATH/euclid.json fi + if version_greater_than "0.11.0" $CURRENT_VERSION_WITHOUT_V; then + echo "Running migration v0.11.0" + cd $SCRIPTS_PATH + source ./migrations/v0.11.0.sh + migrate_v_0_11_0 + current_version="0.11.0" + jq --arg current_version "$current_version" '.version = $current_version' $ROOT_PATH/euclid.json > $ROOT_PATH/temp.json && mv $ROOT_PATH/temp.json $ROOT_PATH/euclid.json + fi + echo "migrations finished..." echo } \ No newline at end of file diff --git a/scripts/migrations/v0.11.0.sh b/scripts/migrations/v0.11.0.sh new file mode 100755 index 0000000..ed8c017 --- /dev/null +++ b/scripts/migrations/v0.11.0.sh @@ -0,0 +1,23 @@ +function migrate_v_0_11_0() { + jq '.version = "0.11.0" | + . + { + "snapshot_fees": { + "owner": { + "key_file": { + "name": "token-key.p12", + "alias": "token-key", + "password": "password" + } + }, + "staking": { + "key_file": { + "name": "token-key-1.p12", + "alias": "token-key-1", + "password": "password" + } + } + } + }' "$ROOT_PATH/euclid.json" > "$ROOT_PATH/temp.json" && mv "$ROOT_PATH/temp.json" "$ROOT_PATH/euclid.json" + + echo_green "Updated" +} diff --git a/scripts/utils/get-information.sh b/scripts/utils/get-information.sh index 6be6014..ad55958 100755 --- a/scripts/utils/get-information.sh +++ b/scripts/utils/get-information.sh @@ -32,6 +32,14 @@ function get_env_variables_from_json_config_file() { export ANSIBLE_MONITORING_DEPLOY_PLAYBOOK_FILE=$(jq -r .deploy.ansible.monitoring.playbooks.deploy $ROOT_PATH/euclid.json) export ANSIBLE_MONITORING_START_PLAYBOOK_FILE=$(jq -r .deploy.ansible.monitoring.playbooks.start $ROOT_PATH/euclid.json) + export SNAPSHOT_FEES_OWNER_FILE_NAME=$(jq -r .snapshot_fees.owner.key_file.name $ROOT_PATH/euclid.json) + export SNAPSHOT_FEES_OWNER_ALIAS=$(jq -r .snapshot_fees.owner.key_file.alias $ROOT_PATH/euclid.json) + export SNAPSHOT_FEES_OWNER_PASSWORD=$(jq -r .snapshot_fees.owner.key_file.password $ROOT_PATH/euclid.json) + + export SNAPSHOT_FEES_STAKING_FILE_NAME=$(jq -r .snapshot_fees.staking.key_file.name $ROOT_PATH/euclid.json) + export SNAPSHOT_FEES_STAKING_ALIAS=$(jq -r .snapshot_fees.staking.key_file.alias $ROOT_PATH/euclid.json) + export SNAPSHOT_FEES_STAKING_PASSWORD=$(jq -r .snapshot_fees.staking.key_file.password $ROOT_PATH/euclid.json) + ## Colors export OUTPUT_RED=$(tput setaf 1) export OUTPUT_GREEN=$(tput setaf 2) diff --git a/scripts/utils/validations.sh b/scripts/utils/validations.sh index b0b7cca..c954401 100755 --- a/scripts/utils/validations.sh +++ b/scripts/utils/validations.sh @@ -205,3 +205,78 @@ function check_if_genesis_files_exists() { exit 1 fi } + +function check_metagraph_owner_fees_file_exists() { + if [ "$#" -eq 0 ]; then + return + fi + local owner_file_path=$SOURCE_PATH/p12-files/$1 + if [ -f "$owner_file_path" ]; then + if [[ "$owner_file_path" == *.p12 ]]; then + echo_green "Owner file exists" + else + echo_red "Invalid owner file extension, it should be .p12" + exit 1 + fi + else + echo_red "Owner file does not exist." + exit 1 + fi +} + +function check_metagraph_staking_fees_file_exists() { + if [ "$#" -eq 0 ]; then + return + fi + local staking_file_path=$SOURCE_PATH/p12-files/$1 + if [ -f "$staking_file_path" ]; then + if [[ "$staking_file_path" == *.p12 ]]; then + echo_green "Staking file exists" + else + echo_red "Invalid staking file extension, it should be .p12" + exit 1 + fi + else + echo_red "Staking file does not exist." + exit 1 + fi +} + +function check_metagraph_owner_fees_information() { + if [ "$#" -eq 0 ]; then + return + elif [ "$#" -ne 3 ]; then + echo_red "You should provide all 3 parameters to owner wallet validation" + exit 1 + fi +} + +function check_metagraph_staking_fees_information() { + if [ "$#" -eq 0 ]; then + return + elif [ "$#" -ne 3 ]; then + echo_red "You should provide all 3 parameters to staking wallet validation" + exit 1 + fi +} + +function check_if_owner_and_staking_address_are_equal() { + if [[ "$SNAPSHOT_FEES_OWNER_FILE_NAME" == "$SNAPSHOT_FEES_STAKING_FILE_NAME" ]]; then + echo_red "Owner and Staking address should be different" + exit 1 + fi +} + +function confirm_force_genesis() { + if [[ -z "$argc_force_genesis" ]]; then + return + fi + echo_yellow "This operation will delete your metagraph and snapshots" + read -p "Do you want to continue? (Y/y to confirm): " -r response + + echo_white "" + if [[ ! "$response" =~ ^[Yy]$ ]]; then + echo "Aborting" + exit 1 + fi +} From e551e0ef7f12aaa6ea054aec397903f6b71c3e39 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Wed, 3 Jul 2024 14:03:02 -0300 Subject: [PATCH 08/15] Fixing README --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 5958291..7b0f7ca 100644 --- a/README.md +++ b/README.md @@ -320,10 +320,6 @@ OPTIONS: -h, --help ``` * --force_genesis: Use this flag when you want your metagraph to start from genesis. If you have already started a metagraph and need to execute from genesis again, use this command. - -* --owner_p12_file_name: This parameter specifies the wallet to charge fees on snapshots. Ensure this file is in the `p12-files` directory. We will validate if the file exists in the directory and, if so, send it to the remote `metagraph-l0` directory. - -* --staking_p12_file_name: This parameter specifies the wallet to check the balance to reduce fees according to the amount of DAG in the balance. Ensure this file is in the `p12-files` directory. We will validate if the file exists in the directory and, if so, send it to the remote `metagraph-l0` directory. ### `hydra remote-start` From 1410988f8256a40377c5ecf24d2f021f20f04075 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Wed, 3 Jul 2024 14:38:30 -0300 Subject: [PATCH 09/15] Formatting --- .../start/metagraph-l0/genesis.ansible.yml | 8 ++--- .../remote-snapshot-fee-config.sh | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml index 9aee213..1e32b83 100755 --- a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml +++ b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml @@ -80,7 +80,7 @@ msg: "File /home/{{ ansible_user }}/code/metagraph-l0/{{ staking_p12_file_name }} does not exist" when: not (staking_p12_file_name is undefined or staking_p12_file_name == "") and not staking_file.stat.exists -- name: Fetch the latest combined snapshot +- name: Fetch the latest combined snapshot from global network uri: url: "http://{{ gl0_ip }}:{{ gl0_port }}/global-snapshots/latest/combined" method: GET @@ -90,17 +90,17 @@ register: latest_combined_snapshot_response when: should_run_genesis or force_owner_message_bool or force_staking_message_bool -- name: Check if the fetch was successful +- name: Check if the snapshot fetch was successful fail: msg: "Error fetching the latest combined snapshot." when: (should_run_genesis or force_owner_message_bool or force_staking_message_bool) and latest_combined_snapshot_response.status != 200 -- name: Convert JSON response content to a fact +- name: Convert response to JSON set_fact: json_response: "{{ latest_combined_snapshot_response.content | from_json }}" when: should_run_genesis or force_owner_message_bool or force_staking_message_bool -- name: Parse the response to extract the last messages +- name: Parse the response to extract the metagraph last messages set_fact: snapshot_fees_messages: "{{ json_response | json_query('[1].lastCurrencySnapshots.' ~ metagraph_id ~ '.Right[1].lastMessages') }}" when: should_run_genesis or force_owner_message_bool or force_staking_message_bool diff --git a/scripts/hydra-operations/remote-snapshot-fee-config.sh b/scripts/hydra-operations/remote-snapshot-fee-config.sh index 086651b..d4df252 100755 --- a/scripts/hydra-operations/remote-snapshot-fee-config.sh +++ b/scripts/hydra-operations/remote-snapshot-fee-config.sh @@ -12,15 +12,41 @@ function fetch_latest_global_snapshot_metagraph_messages() { if [ "$http_code" -ne 200 ]; then echo_red "Failed to fetch data. HTTP Status Code: $http_code" exit 1 - else + else last_messages=$(echo "$response" | jq -r ".[1].lastCurrencySnapshots.\"$metagraph_id\".Right[1].lastMessages") - if [ -z "$last_messages" ]; then - echo_red "Failed when extracting the fee configuration from global snapshot" + if [ -z "$last_messages" ] || [ "$last_messages" = "null" ]; then + echo_red "Failed when extracting the fee configuration from global snapshot. Be sure your metagraph have the fees messages configured" exit 1 else echo_green "Last messages extracted successfully:" - echo "$last_messages" | jq . + + owner_address=$(echo "$last_messages" | jq -r .Owner.value.address) + owner_parent_ordinal=$(echo "$last_messages" | jq -r .Owner.value.parentOrdinal) + staking_address=$(echo "$last_messages" | jq -r .Staking.value.address) + staking_parent_ordinal=$(echo "$last_messages" | jq -r .Staking.value.parentOrdinal) + + # Check if fields are empty or null and handle accordingly + if [ -z "$owner_address" ] || [ "$owner_address" = "null" ]; then + owner_address="N/A" + fi + if [ -z "$owner_parent_ordinal" ] || [ "$owner_parent_ordinal" = "null" ]; then + owner_parent_ordinal="N/A" + fi + if [ -z "$staking_address" ] || [ "$staking_address" = "null" ]; then + staking_address="N/A" + fi + if [ -z "$staking_parent_ordinal" ] || [ "$staking_parent_ordinal" = "null" ]; then + staking_parent_ordinal="N/A" + fi + + echo_white "OWNER" + echo_url "Owner Address" "$owner_address" + echo_url "Owner Parent Ordinal" "$owner_parent_ordinal" + echo + echo_white "STAKING" + echo_url "Staking Address" "$staking_address" + echo_url "Staking Parent Ordinal" "$staking_parent_ordinal" fi fi From b3212e989d99e6d9eaac9e6d8911a744fd6f3087 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Wed, 3 Jul 2024 16:10:57 -0300 Subject: [PATCH 10/15] Adding extra delay --- .../playbooks/start/metagraph-l0/genesis.ansible.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml index 1e32b83..c98f682 100755 --- a/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml +++ b/infra/ansible/remote/nodes/playbooks/start/metagraph-l0/genesis.ansible.yml @@ -228,6 +228,14 @@ retries: 0 when: should_run_genesis +- name: Wait 2 minutes before stopping current execution + pause: + minutes: 2 + when: > + owner_p12_file_name is defined and + owner_p12_file_name != "" and + should_run_genesis or force_owner_message_bool + - name: Find metagraph-l0 process ID by port shell: "lsof -t -i:{{ base_metagraph_l0_public_port }}" register: l0_process_id From 0d6180deafd3b37df5a49a6745140a95239266df Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Wed, 3 Jul 2024 17:58:08 -0300 Subject: [PATCH 11/15] Updating README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b0f7ca..2e31551 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,7 @@ OPTIONS: ``` * --force_genesis: Use this flag when you want your metagraph to start from genesis. If you have already started a metagraph and need to execute from genesis again, use this command. +**NOTE: You'll be prompted to proceed with force_genesis since it will delete all the previous data** ### `hydra remote-start` This method initiates the remote startup of your metagraph in one of the available networks: integrationnet or mainnet. The network should be set in `euclid.json` under `deploy` -> `network` @@ -373,7 +374,7 @@ OPTIONS: * --force_owner_message: Use this flag when you want your metagraph to force sending the owner message. For example, if you want to change the owner address, provide this flag. * --force_staking_message: Use this flag when you want your metagraph to force sending the staking message. For example, if you want to change the staking address, provide this flag. -**NOTE:** When you provide any of the owner or staking parameters, you must provide all the parameters, otherwise, it will fail. +**NOTE: You'll be prompted to proceed with force_genesis since it will delete all the previous data** ### `hydra remote-status` This method will return the status of your remote hosts. You should see the following: From 4675ad30a081c3e6c72858e4741dd16185e439b8 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Wed, 3 Jul 2024 18:06:13 -0300 Subject: [PATCH 12/15] v0.12.0-SNAPSHOT --- euclid.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/euclid.json b/euclid.json index b7286a0..e7f88fd 100644 --- a/euclid.json +++ b/euclid.json @@ -1,6 +1,6 @@ { "github_token": "", - "version": "0.11.0", + "version": "0.12.0-SNAPSHOT", "tessellation_version": "2.3.3", "project_name": "custom-project", "framework": { From 9d39fefed55d11ae7857bb52cf5264e5edf50ba2 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Fri, 19 Jul 2024 12:17:44 -0300 Subject: [PATCH 13/15] Adding tessellation branch running type --- euclid.json | 5 +- .../start/containers/nodes.ansible.yml | 4 +- infra/docker/metagraph-base-image/Dockerfile | 94 ++++++++++--------- .../metagraph-base-image/docker-compose.yml | 1 - infra/docker/metagraph-ubuntu/Dockerfile | 5 +- .../metagraph-ubuntu/docker-compose.yml | 3 +- scripts/hydra | 8 ++ scripts/hydra-operations/build.sh | 12 ++- scripts/hydra-operations/destroy.sh | 4 +- scripts/hydra-operations/start.sh | 6 ++ scripts/migrations/migrations.sh | 9 ++ scripts/migrations/v0.12.0.sh | 8 ++ scripts/utils/get-information.sh | 37 +++++++- scripts/utils/validations.sh | 45 +++++++-- 14 files changed, 171 insertions(+), 70 deletions(-) create mode 100755 scripts/migrations/v0.12.0.sh diff --git a/euclid.json b/euclid.json index e7f88fd..e92a4a2 100644 --- a/euclid.json +++ b/euclid.json @@ -1,14 +1,15 @@ { "github_token": "", "version": "0.12.0-SNAPSHOT", - "tessellation_version": "2.3.3", + "tessellation_version": "2.8.0", + "ref_type": "tag", "project_name": "custom-project", "framework": { "name": "currency", "modules": [ "data" ], - "version": "v2.3.3", + "version": "v2.8.0", "ref_type": "tag" }, "layers": [ diff --git a/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml b/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml index 3221669..13324fa 100755 --- a/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml +++ b/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml @@ -24,7 +24,6 @@ name: "{{ node_info.name }}" image: metagraph-base-image:latest state: started - platform: linux/amd64 networks: - name: custom-network ipv4_address: "{{ base_prefix_ip }}{{ (index + 1) * offset }}" @@ -54,5 +53,4 @@ loop: "{{ all_nodes }}" loop_control: loop_var: node_info - index_var: index - no_log: true \ No newline at end of file + index_var: index \ No newline at end of file diff --git a/infra/docker/metagraph-base-image/Dockerfile b/infra/docker/metagraph-base-image/Dockerfile index ed75667..625adec 100755 --- a/infra/docker/metagraph-base-image/Dockerfile +++ b/infra/docker/metagraph-base-image/Dockerfile @@ -1,6 +1,6 @@ -ARG TESSELLATION_VERSION +ARG TESSELLATION_VERSION_NAME -FROM --platform=linux/amd64 metagraph-ubuntu-${TESSELLATION_VERSION} +FROM metagraph-ubuntu-${TESSELLATION_VERSION_NAME} ARG SHOULD_BUILD_GLOBAL_L0 ARG SHOULD_BUILD_DAG_L1 @@ -17,59 +17,63 @@ COPY project/$TEMPLATE_NAME $TEMPLATE_NAME COPY global-l0/genesis/genesis.csv global-genesis.csv COPY metagraph-l0/genesis/genesis.csv metagraph-genesis.csv -RUN mkdir shared_jars -RUN mkdir shared_genesis +RUN mkdir shared_jars && mkdir shared_genesis -RUN if [ "$SHOULD_BUILD_GLOBAL_L0" = "true" ]; then \ - mkdir global-l0; \ - cp global-l0.jar global-l0/global-l0.jar; \ - cp cl-wallet.jar global-l0/cl-wallet.jar; \ - cp cl-keytool.jar global-l0/cl-keytool.jar; \ +RUN set -e; \ + if [ "$SHOULD_BUILD_GLOBAL_L0" = "true" ]; then \ + mkdir global-l0 && \ + cp global-l0.jar global-l0/global-l0.jar && \ + cp cl-wallet.jar global-l0/cl-wallet.jar && \ + cp cl-keytool.jar global-l0/cl-keytool.jar && \ mv global-genesis.csv global-l0/genesis.csv; \ fi -RUN if [ "$SHOULD_BUILD_DAG_L1" = "true" ]; then \ - mkdir dag-l1; \ - cp dag-l1.jar dag-l1/dag-l1.jar; \ - cp cl-wallet.jar dag-l1/cl-wallet.jar; \ +RUN set -e; \ + if [ "$SHOULD_BUILD_DAG_L1" = "true" ]; then \ + mkdir dag-l1 && \ + cp dag-l1.jar dag-l1/dag-l1.jar && \ + cp cl-wallet.jar dag-l1/cl-wallet.jar && \ cp cl-keytool.jar dag-l1/cl-keytool.jar; \ fi -RUN if [ "$SHOULD_BUILD_METAGRAPH_L0" = "true" ]; then \ - mkdir metagraph-l0; \ - cp cl-wallet.jar metagraph-l0/cl-wallet.jar; \ - cp cl-keytool.jar metagraph-l0/cl-keytool.jar; \ - rm -r -f $TEMPLATE_NAME/modules/l0/target; \ - cd $TEMPLATE_NAME; \ - sbt currencyL0/assembly; \ - cd ..; \ - mv $TEMPLATE_NAME/modules/l0/target/scala-2.13/*.jar metagraph-l0/metagraph-l0.jar; \ - mv metagraph-genesis.csv metagraph-l0/genesis.csv; \ - cp metagraph-l0/metagraph-l0.jar shared_jars/metagraph-l0.jar; \ +RUN set -e; \ + if [ "$SHOULD_BUILD_METAGRAPH_L0" = "true" ]; then \ + mkdir metagraph-l0 && \ + cp cl-wallet.jar metagraph-l0/cl-wallet.jar && \ + cp cl-keytool.jar metagraph-l0/cl-keytool.jar && \ + rm -r -f $TEMPLATE_NAME/modules/l0/target && \ + cd $TEMPLATE_NAME && \ + sbt currencyL0/assembly && \ + cd .. && \ + mv $TEMPLATE_NAME/modules/l0/target/scala-2.13/*.jar metagraph-l0/metagraph-l0.jar && \ + mv metagraph-genesis.csv metagraph-l0/genesis.csv && \ + cp metagraph-l0/metagraph-l0.jar shared_jars/metagraph-l0.jar && \ cp metagraph-l0/genesis.csv shared_genesis/genesis.csv; \ fi -RUN if [ "$SHOULD_BUILD_CURRENCY_L1" = "true" ]; then \ - mkdir currency-l1; \ - cp cl-wallet.jar currency-l1/cl-wallet.jar; \ - cp cl-keytool.jar currency-l1/cl-keytool.jar; \ - rm -r -f $TEMPLATE_NAME/modules/l1/target; \ - cd $TEMPLATE_NAME; \ - sbt currencyL1/assembly; \ - cd ..; \ - mv $TEMPLATE_NAME/modules/l1/target/scala-2.13/*.jar currency-l1/currency-l1.jar; \ +RUN set -e; \ + if [ "$SHOULD_BUILD_CURRENCY_L1" = "true" ]; then \ + mkdir currency-l1 && \ + cp cl-wallet.jar currency-l1/cl-wallet.jar && \ + cp cl-keytool.jar currency-l1/cl-keytool.jar && \ + rm -r -f $TEMPLATE_NAME/modules/l1/target && \ + cd $TEMPLATE_NAME && \ + sbt currencyL1/assembly && \ + cd .. && \ + mv $TEMPLATE_NAME/modules/l1/target/scala-2.13/*.jar currency-l1/currency-l1.jar && \ cp currency-l1/currency-l1.jar shared_jars/currency-l1.jar; \ fi -RUN if [ "$SHOULD_BUILD_DATA_L1" = "true" ]; then \ - mkdir data-l1; \ - cp cl-wallet.jar data-l1/cl-wallet.jar; \ - cp cl-keytool.jar data-l1/cl-keytool.jar; \ - rm -r -f $TEMPLATE_NAME/modules/data_l1/target; \ - cd $TEMPLATE_NAME; \ - sbt dataL1/assembly; \ - cd ..; \ - mv $TEMPLATE_NAME/modules/data_l1/target/scala-2.13/*.jar data-l1/data-l1.jar; \ +RUN set -e; \ + if [ "$SHOULD_BUILD_DATA_L1" = "true" ]; then \ + mkdir data-l1 && \ + cp cl-wallet.jar data-l1/cl-wallet.jar && \ + cp cl-keytool.jar data-l1/cl-keytool.jar && \ + rm -r -f $TEMPLATE_NAME/modules/data_l1/target && \ + cd $TEMPLATE_NAME && \ + sbt dataL1/assembly && \ + cd .. && \ + mv $TEMPLATE_NAME/modules/data_l1/target/scala-2.13/*.jar data-l1/data-l1.jar && \ cp data-l1/data-l1.jar shared_jars/data-l1.jar; \ fi @@ -80,6 +84,6 @@ RUN rm -r -f cl-keytool.jar && \ rm -r -f global-genesis.csv && \ rm -r -f metagraph-genesis.csv && \ rm -r -f tessellation && \ - rm -r -f $TEMPLATE_NAME - -CMD ["sh", "-c", "while true; do sleep 86400; done"] \ No newline at end of file + rm -r -f $TEMPLATE_NAME + +CMD ["sh", "-c", "while true; do sleep 86400; done"] diff --git a/infra/docker/metagraph-base-image/docker-compose.yml b/infra/docker/metagraph-base-image/docker-compose.yml index 58e22ba..73da785 100755 --- a/infra/docker/metagraph-base-image/docker-compose.yml +++ b/infra/docker/metagraph-base-image/docker-compose.yml @@ -1,7 +1,6 @@ version: '3' services: metagraph-base-image: - platform: linux/amd64 image: "metagraph-base-image" build: context: ../../../source diff --git a/infra/docker/metagraph-ubuntu/Dockerfile b/infra/docker/metagraph-ubuntu/Dockerfile index e6959de..56cb554 100755 --- a/infra/docker/metagraph-ubuntu/Dockerfile +++ b/infra/docker/metagraph-ubuntu/Dockerfile @@ -1,9 +1,10 @@ -FROM --platform=linux/amd64 ubuntu:20.04 +FROM ubuntu:20.04 WORKDIR "/code" ARG GIT_PERSONAL_ACCESS_TOKEN ARG TESSELLATION_VERSION +ARG TESSELLATION_VERSION_SEMVER ARG CHECKOUT_TESSELLATION_VERSION ARG SHOULD_USE_UPDATED_MODULES @@ -27,7 +28,7 @@ RUN git clone https://github.com/Constellation-Labs/tessellation.git && \ cd tessellation && \ git checkout $CHECKOUT_TESSELLATION_VERSION && \ rm -r version.sbt && \ - echo "ThisBuild / version := \"$TESSELLATION_VERSION\"" > version.sbt + echo "ThisBuild / version := \"$TESSELLATION_VERSION_SEMVER\"" > version.sbt RUN cd tessellation && \ if [ "$SHOULD_USE_UPDATED_MODULES" = "true" ]; then \ diff --git a/infra/docker/metagraph-ubuntu/docker-compose.yml b/infra/docker/metagraph-ubuntu/docker-compose.yml index a542d9e..f1a0486 100755 --- a/infra/docker/metagraph-ubuntu/docker-compose.yml +++ b/infra/docker/metagraph-ubuntu/docker-compose.yml @@ -1,6 +1,5 @@ version: '3' services: metagraph-ubuntu: - platform: linux/amd64 build: . - image: "metagraph-ubuntu-${TESSELLATION_VERSION}" \ No newline at end of file + image: "metagraph-ubuntu-${TESSELLATION_VERSION_NAME}" \ No newline at end of file diff --git a/scripts/hydra b/scripts/hydra index 7361323..a657fcb 100755 --- a/scripts/hydra +++ b/scripts/hydra @@ -110,6 +110,7 @@ function build() { set_docker_compose check_if_docker_is_running + check_git_ref_exists $TESSELLATION_VERSION build_containers @@ -423,6 +424,13 @@ function check_if_should_run_update() { load_scripts run_migrations fi + + local ref_type=$(jq -r '.ref_type // empty' "$ROOT_PATH/euclid.json") + if [ -z "$ref_type" ]; then + echo "$(tput setaf 3) Could not find the ref_type field on euclid.json file, the file is probably outdated. Running migrations..." + load_scripts + run_migrations + fi } function check_if_the_container_is_running() { diff --git a/scripts/hydra-operations/build.sh b/scripts/hydra-operations/build.sh index 0de78d7..185620f 100755 --- a/scripts/hydra-operations/build.sh +++ b/scripts/hydra-operations/build.sh @@ -1,18 +1,20 @@ #!/usr/bin/env bash function build_metagraph_ubuntu() { - if [[ -z "$(docker images -q metagraph-ubuntu-${TESSELLATION_VERSION})" ]]; then + if [[ -z "$(docker images -q metagraph-ubuntu-${TESSELLATION_VERSION_NAME})" ]]; then echo echo - echo_white "Building metagraph ubuntu for tessellation $TESSELLATION_VERSION" + echo_white "Building metagraph ubuntu for tessellation $TESSELLATION_VERSION in image name metagraph-ubuntu-$TESSELLATION_VERSION_NAME" cd $INFRA_PATH/docker/metagraph-ubuntu CHECKOUT_TESSELLATION_VERSION=$(get_checkout_tessellation_version "$TESSELLATION_VERSION") SHOULD_USE_UPDATED_MODULES=$(get_should_use_updated_modules "$TESSELLATION_VERSION") + if [ ! -z "$argc_no_cache" ]; then $DOCKER_COMPOSE build \ --build-arg GIT_PERSONAL_ACCESS_TOKEN=$GITHUB_TOKEN \ --build-arg TESSELLATION_VERSION=$TESSELLATION_VERSION \ + --build-arg TESSELLATION_VERSION_SEMVER=$TESSELLATION_VERSION_SEMVER \ --build-arg CHECKOUT_TESSELLATION_VERSION=$CHECKOUT_TESSELLATION_VERSION \ --build-arg SHOULD_USE_UPDATED_MODULES=$SHOULD_USE_UPDATED_MODULES \ --no-cache @@ -20,9 +22,11 @@ function build_metagraph_ubuntu() { $DOCKER_COMPOSE build \ --build-arg GIT_PERSONAL_ACCESS_TOKEN=$GITHUB_TOKEN \ --build-arg TESSELLATION_VERSION=$TESSELLATION_VERSION \ + --build-arg TESSELLATION_VERSION_SEMVER=$TESSELLATION_VERSION_SEMVER \ --build-arg CHECKOUT_TESSELLATION_VERSION=$CHECKOUT_TESSELLATION_VERSION \ --build-arg SHOULD_USE_UPDATED_MODULES=$SHOULD_USE_UPDATED_MODULES fi + echo_green "Ubuntu for tessellation $TESSELLATION_VERSION built" else echo_green "Ubuntu for tessellation $TESSELLATION_VERSION already built, skipping..." @@ -78,7 +82,7 @@ function build_metagraph_base_image() { if [ ! -z "$argc_no_cache" ]; then $DOCKER_COMPOSE build \ - --build-arg TESSELLATION_VERSION=$TESSELLATION_VERSION \ + --build-arg TESSELLATION_VERSION_NAME=$TESSELLATION_VERSION_NAME \ --build-arg TEMPLATE_NAME=$PROJECT_NAME \ --build-arg SHOULD_BUILD_GLOBAL_L0=$should_build_global_l0 \ --build-arg SHOULD_BUILD_DAG_L1=$should_build_dag_l1 \ @@ -88,7 +92,7 @@ function build_metagraph_base_image() { --no-cache else $DOCKER_COMPOSE build \ - --build-arg TESSELLATION_VERSION=$TESSELLATION_VERSION \ + --build-arg TESSELLATION_VERSION_NAME=$TESSELLATION_VERSION_NAME \ --build-arg TEMPLATE_NAME=$PROJECT_NAME \ --build-arg SHOULD_BUILD_GLOBAL_L0=$should_build_global_l0 \ --build-arg SHOULD_BUILD_DAG_L1=$should_build_dag_l1 \ diff --git a/scripts/hydra-operations/destroy.sh b/scripts/hydra-operations/destroy.sh index 9f85867..e441f72 100755 --- a/scripts/hydra-operations/destroy.sh +++ b/scripts/hydra-operations/destroy.sh @@ -18,7 +18,7 @@ function destroy_containers() { echo_white "Trying to destroy container grafana ..." if docker inspect grafana &>/dev/null; then - docker rm -f $name + docker rm -f grafana echo_green "Container grafana removed successfully." else echo_yellow "Container grafana does not exist." @@ -29,7 +29,7 @@ function destroy_containers() { echo_white "Trying to destroy container prometheus ..." if docker inspect prometheus &>/dev/null; then - docker rm -f $name + docker rm -f prometheus echo_green "Container prometheus removed successfully." else echo_yellow "Container prometheus does not exist." diff --git a/scripts/hydra-operations/start.sh b/scripts/hydra-operations/start.sh index f35ba8f..f797525 100755 --- a/scripts/hydra-operations/start.sh +++ b/scripts/hydra-operations/start.sh @@ -184,6 +184,12 @@ function print_nodes_information() { ((index++)) done < <(jq -c '.[]' <<<"$NODES") + if [[ "$START_GRAFANA_CONTAINER" == "true" ]]; then + echo_green "Telemetry" + echo_url "Grafana:" "http://localhost:3000" + echo + fi + echo_green "Clusters URLs" if [[ " ${LAYERS[*]} " =~ "global-l0" ]]; then raw_port=$(yq eval '.base_global_l0_public_port' $ansible_vars_path) diff --git a/scripts/migrations/migrations.sh b/scripts/migrations/migrations.sh index 06f985f..3d1deea 100755 --- a/scripts/migrations/migrations.sh +++ b/scripts/migrations/migrations.sh @@ -44,6 +44,15 @@ function run_migrations() { jq --arg current_version "$current_version" '.version = $current_version' $ROOT_PATH/euclid.json > $ROOT_PATH/temp.json && mv $ROOT_PATH/temp.json $ROOT_PATH/euclid.json fi + if version_greater_than "0.12.0" $CURRENT_VERSION_WITHOUT_V; then + echo "Running migration v0.12.0" + cd $SCRIPTS_PATH + source ./migrations/v0.12.0.sh + migrate_v_0_12_0 + current_version="0.12.0" + jq --arg current_version "$current_version" '.version = $current_version' $ROOT_PATH/euclid.json > $ROOT_PATH/temp.json && mv $ROOT_PATH/temp.json $ROOT_PATH/euclid.json + fi + echo "migrations finished..." echo } \ No newline at end of file diff --git a/scripts/migrations/v0.12.0.sh b/scripts/migrations/v0.12.0.sh new file mode 100755 index 0000000..425a129 --- /dev/null +++ b/scripts/migrations/v0.12.0.sh @@ -0,0 +1,8 @@ +function migrate_v_0_12_0() { + jq '.version = "0.12.0" | + . + { + "ref_type": "tag" + }' "$ROOT_PATH/euclid.json" > "$ROOT_PATH/temp.json" && mv "$ROOT_PATH/temp.json" "$ROOT_PATH/euclid.json" + + echo_green "v0.12.0 Updated" +} diff --git a/scripts/utils/get-information.sh b/scripts/utils/get-information.sh index ad55958..721b91c 100755 --- a/scripts/utils/get-information.sh +++ b/scripts/utils/get-information.sh @@ -7,16 +7,22 @@ function get_env_variables_from_json_config_file() { export GITHUB_TOKEN=$(jq -r .github_token $ROOT_PATH/euclid.json) export METAGRAPH_ID=$(jq -r .metagraph_id $ROOT_PATH/euclid.json) + export TESSELLATION_VERSION=$(jq -r .tessellation_version $ROOT_PATH/euclid.json) + export TESSELLATION_VERSION_IS_TAG_OR_BRANCH=$(jq -r .ref_type $ROOT_PATH/euclid.json) + export TESSELLATION_VERSION_NAME=$(get_tessellation_version_name $TESSELLATION_VERSION) + export TESSELLATION_VERSION_SEMVER=$(get_tessellation_version_semver $TESSELLATION_VERSION) + export TEMPLATE_VERSION=$(jq -r .framework.version $ROOT_PATH/euclid.json) export TEMPLATE_VERSION_IS_TAG_OR_BRANCH=$(jq -r .framework.ref_type $ROOT_PATH/euclid.json) + export PROJECT_NAME=$(jq -r .project_name $ROOT_PATH/euclid.json) export FRAMEWORK_NAME=$(jq -r .framework.name $ROOT_PATH/euclid.json) export FRAMEWORK_MODULES=$(jq -r .framework.modules $ROOT_PATH/euclid.json) export NODES=$(jq -r .nodes $ROOT_PATH/euclid.json) - - export START_GRAFANA_CONTAINER=$(jq -r .docker.start_grafana_container $ROOT_PATH/euclid.json) + + export START_GRAFANA_CONTAINER=$(jq -r .docker.start_grafana_container $ROOT_PATH/euclid.json) export LAYERS=$(jq -r .layers $ROOT_PATH/euclid.json) @@ -48,6 +54,31 @@ function get_env_variables_from_json_config_file() { export OUTPUT_WHITE=$(tput setaf 7) } +function get_tessellation_version_name() { + local input="$1" + if [[ "$TESSELLATION_VERSION_IS_TAG_OR_BRANCH" == "branch" ]]; then + # Remove special characters + local cleaned=$(echo "$input" | tr -cd '[:alnum:][:space:]') + # Convert to lowercase + cleaned=$(echo "$cleaned" | tr '[:upper:]' '[:lower:]') + # Replace spaces with underscores + cleaned=$(echo "$cleaned" | tr ' ' '_') + echo "$cleaned" + else + echo "$input" + fi +} + +function get_tessellation_version_semver() { + local input="$1" + if [[ "$TESSELLATION_VERSION_IS_TAG_OR_BRANCH" == "branch" ]]; then + #Mocked version when it's a branch + echo "99.99.99" + else + echo "$input" + fi +} + function get_metagraph_id_from_metagraph_l0_genesis() { for ((i = 1; i <= 51; i++)); do METAGRAPH_ID=$(cat $SOURCE_PATH/metagraph-l0/genesis/genesis.address) @@ -62,7 +93,7 @@ function get_metagraph_id_from_metagraph_l0_genesis() { echo_url "METAGRAPH_ID: " $METAGRAPH_ID echo_white "Filling the euclid.json file" contents="$(jq --arg METAGRAPH_ID "$METAGRAPH_ID" '.metagraph_id = $METAGRAPH_ID' $ROOT_PATH/euclid.json)" && - echo -E "${contents}" > $ROOT_PATH/euclid.json + echo -E "${contents}" >$ROOT_PATH/euclid.json get_env_variables_from_json_config_file break diff --git a/scripts/utils/validations.sh b/scripts/utils/validations.sh index c954401..dd6808f 100755 --- a/scripts/utils/validations.sh +++ b/scripts/utils/validations.sh @@ -129,12 +129,21 @@ function check_network() { function check_if_tessellation_version_of_project_matches_euclid_json() { echo echo_yellow "Checking the project tessellation version and tessellation version provided on euclid.json" - PROJECT_TESSELLATION_VERSION=$(sed -n 's/.*val tessellation = "\(.*\)".*/\1/p' $SOURCE_PATH/project/$PROJECT_NAME/project/Dependencies.scala) - echo_white "Project tessellation version: $PROJECT_TESSELLATION_VERSION" - echo_white "Tessellation version provided on euclid.json: $TESSELLATION_VERSION" - if [[ "$PROJECT_TESSELLATION_VERSION" != "$TESSELLATION_VERSION" ]]; then - echo_red "Your custom project contains a different version of tessellation than provided on euclid.json. Please use the same version!" - exit 1 + + PROJECT_TESSELLATION_VERSION=$(sed -n 's/.*val tessellation = "\(.*\)".*/\1/p' "$SOURCE_PATH/project/$PROJECT_NAME/project/Dependencies.scala") + + if [[ "$TESSELLATION_VERSION_IS_TAG_OR_BRANCH" == "branch" ]]; then + if [[ "$PROJECT_TESSELLATION_VERSION" != "99.99.99" ]]; then + echo_red "You're running using tessellation branch as base. Please update your Dependencies.scala file tessellation version to 99.99.99" + exit 1 + fi + else + echo_white "Project tessellation version: $PROJECT_TESSELLATION_VERSION" + echo_white "Tessellation version provided on euclid.json: $TESSELLATION_VERSION" + if [[ "$PROJECT_TESSELLATION_VERSION" != "$TESSELLATION_VERSION" ]]; then + echo_red "Your custom project contains a different version of tessellation than provided on euclid.json. Please use the same version!" + exit 1 + fi fi } @@ -267,6 +276,30 @@ function check_if_owner_and_staking_address_are_equal() { fi } +function check_git_ref_exists() { + local ref=$1 + local repo_url="https://github.com/Constellation-Labs/tessellation.git" + if [[ "$TESSELLATION_VERSION_IS_TAG_OR_BRANCH" == "branch" ]]; then + echo_white "Checking if branch exists..." + if git ls-remote --exit-code --heads "$repo_url" "$ref" >/dev/null; then + echo_green "Branch '$ref' exists." + echo + else + echo_red "Branch '$ref' not exists." + exit 1 + fi + else + echo_white "Checking if tag exists..." + if git ls-remote --exit-code --tags "$repo_url" "v$ref" >/dev/null; then + echo_green "Tag '$ref' exists." + echo + else + echo_red "Tag '$ref' not exists." + exit 1 + fi + fi +} + function confirm_force_genesis() { if [[ -z "$argc_force_genesis" ]]; then return From 0f919d745501c3f8ba26446f5577dd09666f3301 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Fri, 19 Jul 2024 12:46:49 -0300 Subject: [PATCH 14/15] Adding no_log --- .../local/playbooks/start/containers/nodes.ansible.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml b/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml index 13324fa..3221669 100755 --- a/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml +++ b/infra/ansible/local/playbooks/start/containers/nodes.ansible.yml @@ -24,6 +24,7 @@ name: "{{ node_info.name }}" image: metagraph-base-image:latest state: started + platform: linux/amd64 networks: - name: custom-network ipv4_address: "{{ base_prefix_ip }}{{ (index + 1) * offset }}" @@ -53,4 +54,5 @@ loop: "{{ all_nodes }}" loop_control: loop_var: node_info - index_var: index \ No newline at end of file + index_var: index + no_log: true \ No newline at end of file From 2e5533d733f517da68b022ce846c14b678812646 Mon Sep 17 00:00:00 2001 From: Marcus Vinicius Girolneto Sousa Date: Fri, 19 Jul 2024 16:13:00 -0300 Subject: [PATCH 15/15] v0.12.0 --- euclid.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/euclid.json b/euclid.json index e92a4a2..b7afabb 100644 --- a/euclid.json +++ b/euclid.json @@ -1,6 +1,6 @@ { "github_token": "", - "version": "0.12.0-SNAPSHOT", + "version": "0.12.0", "tessellation_version": "2.8.0", "ref_type": "tag", "project_name": "custom-project",