diff --git a/infra/opentelemetry/config_docker.toml b/infra/opentelemetry/config_docker.toml new file mode 100644 index 000000000..f046a1955 --- /dev/null +++ b/infra/opentelemetry/config_docker.toml @@ -0,0 +1,93 @@ +bootstrap_address = [ + "12D3KooWPXw2dXBRH1bT4vcNos9f6W2KoFTiarqptBuTzxaXg7zu", + "/ip4/198.51.100.103/tcp/5643", +] +p2p_port = 5643 + +# OpenTelemetry parameters +otlp_collector_endpoint = "http://otel-collector:4317" + +[[nodes]] +api_servers = [ + { port = 4201, enabled_apis = [ "admin", "erigon", "eth", "net", "ots", "trace", "txpool", "web3", "zilliqa" ] }, +] + +# These (public key, peerId, stake, rewardAddress) tuples correspond to the private keys of all four nodes in `docker-compose.yaml`. +consensus.genesis_deposits = [ + [ + "b27aebb3b54effd7af87c4a064a711554ee0f3f5abf56ca910b46422f2b21603bc383d42eb3b927c4c3b0b8381ca30a3", + "12D3KooWESMZ2ttSxDwjfnNe23sHCqsJf6sNEKwgHkdgtCHDsbWU", + "10000000000000000000000000", + "7E5F4552091A69125d5DfCb7b8C2659029395Bdf", + "7E5F4552091A69125d5DfCb7b8C2659029395Bdf", + ], + [ + "b37fd66aef29ca78a82d519a284789d59c2bb3880698b461c6c732d094534707d50e345128db372a1e0a4c5d5c42f49c", + "12D3KooWJc2nBgNiSi14GcYaGmU8FoQsRkmhfMnaB1mHmPiBPZHd", + "10000000000000000000000000", + "2B5AD5c4795c026514f8317c7a215E218DcCD6cF", + "2B5AD5c4795c026514f8317c7a215E218DcCD6cF", + ], + [ + "ab035d6cd3321c3b57d14ea09a4f3860899542d2187b5ec87649b1f40980418a096717a671cf62b73880afac252fc5dc", + "12D3KooWLA4xVjiGszqmYJmt8E1NTurVeCujDi17FoSzSDDDKUjT", + "10000000000000000000000000", + "6813Eb9362372EEF6200f3b1dbC3f819671cBA69", + "6813Eb9362372EEF6200f3b1dbC3f819671cBA69", + ], + [ + "985e3a4d367cbfc966d48710806612cc00f6bfd06aa759340cfe13c3990d26a7ddde63f64468cdba5b2ff132a4639a7f", + "12D3KooWPXw2dXBRH1bT4vcNos9f6W2KoFTiarqptBuTzxaXg7zu", + "10000000000000000000000000", + "1efF47bc3a10a45D4B230B5d10E37751FE6AA718", + "1efF47bc3a10a45D4B230B5d10E37751FE6AA718", + ], +] +consensus.genesis_accounts = [ + # Accounts with private key 0x2, 0x3, 0x4. + # Don't use account with priv key = 0x1 since it's also used for voting rewards + [ + "2B5AD5c4795c026514f8317c7a215E218DcCD6cF", + "5000000000000000000000", + ], + [ + "6813Eb9362372EEF6200f3b1dbC3f819671cBA69", + "5000000000000000000000", + ], + [ + "1efF47bc3a10a45D4B230B5d10E37751FE6AA718", + "5000000000000000000000", + ], + [ + "29e562f73488c8a2bB9Dbc5700b361D54b9B0554", + "5000000000000000000000", + ], + [ + "8D393a22E4476fF8212DE13fE1939De2a236F0A7", + "5000000000000000000000", + ], + [ + "9cB422D2Fabe9622ed706ad5D9d3fFd2cDd1C001", + "5000000000000000000000", + ], + [ + "AcE5F1e883d3e02A1b2C78F6909a8C0430C6Fb12", + "5000000000000000000000", + ], + [ + "1958b2f7b5c476F5e8FeBdEFeba5EC39E2f20288", + "5000000000000000000000", + ], +] + +# Speed up local/docker epochs +consensus.blocks_per_epoch = 36 + +# Reward parameters +consensus.rewards_per_hour = "51_000_000_000_000_000_000_000" +consensus.blocks_per_hour = 3600 +consensus.minimum_stake = "10_000_000_000_000_000_000_000_000" + +# Gas parameters +consensus.eth_block_gas_limit = 84000000 +consensus.gas_price = "4_761_904_800_000" \ No newline at end of file diff --git a/infra/opentelemetry/docker-compose.yaml b/infra/opentelemetry/docker-compose.yaml new file mode 100644 index 000000000..5ae315c2e --- /dev/null +++ b/infra/opentelemetry/docker-compose.yaml @@ -0,0 +1,142 @@ +# docker-compsoe file for testing networking with static IPs +version: "3" +services: + node0_with_otel: + depends_on: + - node3_with_otel + environment: + RUST_BACKTRACE: 1 + RUST_LOG: zilliqa=info,opentelemetry=trace + image: zq2-node0 + build: + context: . + dockerfile: Dockerfile + container_name: zilliqanode0_with_otel + volumes: + - "./config_docker.toml:/config.toml" + command: + - 65d7f4da9bedc8fb79cbf6722342960bbdfb9759bc0d9e3fb4989e831ccbc227 + ports: + - "4211:4201" + networks: + testnet_with_otel: + ipv4_address: 198.52.100.100 + + node1_with_otel: + depends_on: + - node3_with_otel + environment: + RUST_BACKTRACE: 1 + RUST_LOG: zilliqa=info,opentelemetry=trace + container_name: zilliqanode1_with_otel + image: zq2-node0 + volumes: + - "./config_docker.toml:/config.toml" + command: + - 62070b1a3b5b30236e43b4f1bfd617e1af7474635558314d46127a708b9d302e + networks: + testnet_with_otel: + ipv4_address: 198.52.100.101 + + node2_with_otel: + depends_on: + - node3_with_otel + environment: + RUST_BACKTRACE: 1 + RUST_LOG: zilliqa=info,opentelemetry=trace + container_name: zilliqanode2_with_otel + image: zq2-node0 + volumes: + - "./config_docker.toml:/config.toml" + command: + - 56d7a450d75c6ba2706ef71da6ca80143ec4971add9c44d7d129a12fa7d3a364 + networks: + testnet_with_otel: + ipv4_address: 198.52.100.102 + + node3_with_otel: + environment: + RUST_BACKTRACE: 1 + RUST_LOG: zilliqa=info,opentelemetry=trace + container_name: zilliqanode3_with_otel + image: zq2-node0 + volumes: + - "./config_docker.toml:/config.toml" + command: + - db670cbff28f4b15297d03fafdab8f5303d68b7591bd59e31eaef215dd0f246a + networks: + testnet_with_otel: + ipv4_address: 198.52.100.103 + + otterscan_with_otel: + environment: + ERIGON_URL: http://localhost:4201 + image: zilliqa/otterscan:develop + pull_policy: always + ports: + - "5101:80" + networks: + testnet_with_otel: + ipv4_address: 198.52.100.9 + + faucet_with_otel: + depends_on: + - node0_with_otel + image: asia-docker.pkg.dev/prj-p-devops-services-tvwmrf63/zilliqa/eth-spout:v1.3.72 + environment: + RPC_URL: http://node0:4201 + NATIVE_TOKEN_SYMBOL: ZIL + PRIVATE_KEY: 65d7f4da9bedc8fb79cbf6722342960bbdfb9759bc0d9e3fb4989e831ccbc227 + ETH_AMOUNT: 100 + EXPLORER_URL: http://localhost:5200 + MINIMUM_SECONDS_BETWEEN_REQUESTS: 60 + BECH32_HRP: zil + ports: + - "5201:80" + # Fast shutdown - Don't gracefully close open connections. + stop_signal: SIGKILL + networks: + testnet_with_otel: + ipv4_address: 198.52.100.10 + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" + - "9011:8888" + networks: + - testnet_with_otel + + mimir: + image: grafana/mimir:latest + volumes: + - "./mimir-config.yaml:/etc/mimir/config.yaml" + command: + - "./mimir" + - "--config.file" + - "/etc/mimir/config.yaml" + ports: + - "9009:9009" + networks: + - testnet_with_otel + + grafana: + image: grafana/grafana-oss:latest + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "9010:3000" + networks: + - testnet_with_otel + +networks: + testnet_with_otel: + driver: bridge + ipam: + config: + - subnet: 198.52.100.0/24 + gateway: 198.52.100.1 diff --git a/infra/mimir.yaml b/infra/opentelemetry/mimir-config.yaml similarity index 81% rename from infra/mimir.yaml rename to infra/opentelemetry/mimir-config.yaml index 139f73662..318ac7428 100644 --- a/infra/mimir.yaml +++ b/infra/opentelemetry/mimir-config.yaml @@ -3,6 +3,12 @@ usage_stats: multitenancy_enabled: false +memberlist: + bind_port: 7946 + advertise_addr: 0.0.0.0 + join_members: + - 0.0.0.0:7946 + blocks_storage: backend: filesystem bucket_store: @@ -20,13 +26,13 @@ compactor: distributor: ring: - instance_addr: 127.0.0.1 + instance_addr: 0.0.0.0 kvstore: store: memberlist ingester: ring: - instance_addr: 127.0.0.1 + instance_addr: 0.0.0.0 kvstore: store: memberlist replication_factor: 1 diff --git a/infra/otel-collector-config.yaml b/infra/opentelemetry/otel-collector-config.yaml similarity index 100% rename from infra/otel-collector-config.yaml rename to infra/opentelemetry/otel-collector-config.yaml diff --git a/infra/tf/locals.tf b/infra/tf/locals.tf index ed179cc52..ba31e29a4 100644 --- a/infra/tf/locals.tf +++ b/infra/tf/locals.tf @@ -19,5 +19,6 @@ locals { "roles/logging.logWriter=>${var.project_id}", "roles/artifactregistry.reader=>${var.gcp_docker_registry_project_id}", "roles/secretmanager.secretAccessor=>${var.project_id}", + "roles/cloudtrace.agent=>${var.project_id}", ] } diff --git a/z2/resources/config.tera.toml b/z2/resources/config.tera.toml index 73d3d9816..309c84d5f 100644 --- a/z2/resources/config.tera.toml +++ b/z2/resources/config.tera.toml @@ -4,6 +4,8 @@ p2p_port = 3333 bootstrap_address = [ "{{ bootstrap_peer_id }}", "/dns/{{ bootstrap_endpoint }}/tcp/3333" ] {%- endif %} +otlp_collector_endpoint = "http://localhost:4317" + [[nodes]] eth_chain_id = {{ eth_chain_id }} allowed_timestamp_skew = { secs = 60, nanos = 0 } diff --git a/z2/resources/node_provision.tera.py b/z2/resources/node_provision.tera.py index 0cb1ce6a2..31a38d60a 100644 --- a/z2/resources/node_provision.tera.py +++ b/z2/resources/node_provision.tera.py @@ -14,7 +14,6 @@ templatefile() vars: - checkpoint_url, the ZQ2 checkpoint URL used for recover the validator nodes - persistence_url, the ZQ2 persistence URL used for recover the network -- conifg, the ZQ2 validators configuration file - docker_image, the ZQ2 docker image (incl. version) - secret_key, the ZQ2 node secret key - role, the node role: validator or apps @@ -121,7 +120,7 @@ def health(): """ ZQ2_SCRIPT="""#!/bin/bash -echo yes | gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev +echo yes | gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev ZQ2_IMAGE="{{ docker_image }}" @@ -256,11 +255,14 @@ def health(): OPS_AGENT_INSTALL_SCRIPT_URL="https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh" - OPS_AGENT_CONFIG_FILE="/etc/google-cloud-ops-agent/config.yaml" - OPS_AGENT_CONFIG=""" +combined: + receivers: + otlp: + type: otlp + metrics_mode: googlecloudmonitoring logging: receivers: zilliqa: @@ -310,6 +312,13 @@ def health(): default_pipeline: receivers: [hostmetrics] processors: [metrics_filter] + otlp: + receivers: [otlp] +traces: + service: + pipelines: + otlp: + receivers: [otlp] """ LOGROTATE_CONFIG=""" diff --git a/z2/resources/otel-compose.yaml b/z2/resources/otel-compose.yaml index 199d80584..92f019254 100644 --- a/z2/resources/otel-compose.yaml +++ b/z2/resources/otel-compose.yaml @@ -3,7 +3,7 @@ services: image: otel/opentelemetry-collector:latest command: ["--config=/etc/otel-collector-config.yaml"] volumes: - - ./otel-config.yaml:/etc/otel-collector-config.yaml + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml ports: - "4317:4317" - "9011:8888" @@ -25,4 +25,4 @@ services: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin ports: - - "9010:3000" + - "9010:3000" \ No newline at end of file