feat: DEVOPS-1632 opentelemetry local setup (#2222)

Zilliqa · Jan 27, 2025 · f321912 · f321912
1 parent afa7e93
commit f321912
Show file tree

Hide file tree

Showing 8 changed files with 261 additions and 8 deletions.
diff --git a/infra/opentelemetry/config_docker.toml b/infra/opentelemetry/config_docker.toml
@@ -0,0 +1,93 @@
+bootstrap_address = [
+    "12D3KooWPXw2dXBRH1bT4vcNos9f6W2KoFTiarqptBuTzxaXg7zu",
+    "/ip4/198.51.100.103/tcp/5643",
+]
+p2p_port = 5643
+
+# OpenTelemetry parameters
+otlp_collector_endpoint = "http://otel-collector:4317"
+
+[[nodes]]
+api_servers = [
+    { port = 4201, enabled_apis = [ "admin", "erigon", "eth", "net", "ots", "trace", "txpool", "web3", "zilliqa" ] },
+]
+
+# These (public key, peerId, stake, rewardAddress) tuples correspond to the private keys of all four nodes in `docker-compose.yaml`.
+consensus.genesis_deposits = [
+    [
+        "b27aebb3b54effd7af87c4a064a711554ee0f3f5abf56ca910b46422f2b21603bc383d42eb3b927c4c3b0b8381ca30a3",
+        "12D3KooWESMZ2ttSxDwjfnNe23sHCqsJf6sNEKwgHkdgtCHDsbWU",
+        "10000000000000000000000000",
+        "7E5F4552091A69125d5DfCb7b8C2659029395Bdf",
+        "7E5F4552091A69125d5DfCb7b8C2659029395Bdf",
+    ],
+    [
+        "b37fd66aef29ca78a82d519a284789d59c2bb3880698b461c6c732d094534707d50e345128db372a1e0a4c5d5c42f49c",
+        "12D3KooWJc2nBgNiSi14GcYaGmU8FoQsRkmhfMnaB1mHmPiBPZHd",
+        "10000000000000000000000000",
+        "2B5AD5c4795c026514f8317c7a215E218DcCD6cF",
+        "2B5AD5c4795c026514f8317c7a215E218DcCD6cF",
+    ],
+    [
+        "ab035d6cd3321c3b57d14ea09a4f3860899542d2187b5ec87649b1f40980418a096717a671cf62b73880afac252fc5dc",
+        "12D3KooWLA4xVjiGszqmYJmt8E1NTurVeCujDi17FoSzSDDDKUjT",
+        "10000000000000000000000000",
+        "6813Eb9362372EEF6200f3b1dbC3f819671cBA69",
+        "6813Eb9362372EEF6200f3b1dbC3f819671cBA69",
+    ],
+    [
+        "985e3a4d367cbfc966d48710806612cc00f6bfd06aa759340cfe13c3990d26a7ddde63f64468cdba5b2ff132a4639a7f",
+        "12D3KooWPXw2dXBRH1bT4vcNos9f6W2KoFTiarqptBuTzxaXg7zu",
+        "10000000000000000000000000",
+        "1efF47bc3a10a45D4B230B5d10E37751FE6AA718",
+        "1efF47bc3a10a45D4B230B5d10E37751FE6AA718",
+    ],
+]
+consensus.genesis_accounts = [
+    # Accounts with private key 0x2, 0x3, 0x4.
+    # Don't use account with priv key = 0x1 since it's also used for voting rewards
+    [
+        "2B5AD5c4795c026514f8317c7a215E218DcCD6cF",
+        "5000000000000000000000",
+    ],
+    [
+        "6813Eb9362372EEF6200f3b1dbC3f819671cBA69",
+        "5000000000000000000000",
+    ],
+    [
+        "1efF47bc3a10a45D4B230B5d10E37751FE6AA718",
+        "5000000000000000000000",
+    ],
+    [
+        "29e562f73488c8a2bB9Dbc5700b361D54b9B0554",
+        "5000000000000000000000",
+    ],
+    [
+        "8D393a22E4476fF8212DE13fE1939De2a236F0A7",
+        "5000000000000000000000",
+    ],
+    [
+        "9cB422D2Fabe9622ed706ad5D9d3fFd2cDd1C001",
+        "5000000000000000000000",
+    ],
+    [
+        "AcE5F1e883d3e02A1b2C78F6909a8C0430C6Fb12",
+        "5000000000000000000000",
+    ],
+    [
+        "1958b2f7b5c476F5e8FeBdEFeba5EC39E2f20288",
+        "5000000000000000000000",
+    ],
+]
+
+# Speed up local/docker epochs
+consensus.blocks_per_epoch = 36
+
+# Reward parameters
+consensus.rewards_per_hour = "51_000_000_000_000_000_000_000"
+consensus.blocks_per_hour = 3600
+consensus.minimum_stake = "10_000_000_000_000_000_000_000_000"
+
+# Gas parameters
+consensus.eth_block_gas_limit = 84000000
+consensus.gas_price = "4_761_904_800_000"
diff --git a/infra/opentelemetry/docker-compose.yaml b/infra/opentelemetry/docker-compose.yaml
@@ -0,0 +1,142 @@
+# docker-compsoe file for testing networking with static IPs
+version: "3"
+services:
+  node0_with_otel:
+    depends_on:
+      - node3_with_otel
+    environment:
+      RUST_BACKTRACE: 1
+      RUST_LOG: zilliqa=info,opentelemetry=trace
+    image: zq2-node0
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: zilliqanode0_with_otel
+    volumes:
+      - "./config_docker.toml:/config.toml"
+    command:
+      - 65d7f4da9bedc8fb79cbf6722342960bbdfb9759bc0d9e3fb4989e831ccbc227
+    ports:
+      - "4211:4201"
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.100
+
+  node1_with_otel:
+    depends_on:
+      - node3_with_otel
+    environment:
+      RUST_BACKTRACE: 1
+      RUST_LOG: zilliqa=info,opentelemetry=trace
+    container_name: zilliqanode1_with_otel
+    image: zq2-node0
+    volumes:
+      - "./config_docker.toml:/config.toml"
+    command:
+      - 62070b1a3b5b30236e43b4f1bfd617e1af7474635558314d46127a708b9d302e
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.101
+
+  node2_with_otel:
+    depends_on:
+      - node3_with_otel
+    environment:
+      RUST_BACKTRACE: 1
+      RUST_LOG: zilliqa=info,opentelemetry=trace
+    container_name: zilliqanode2_with_otel
+    image: zq2-node0
+    volumes:
+      - "./config_docker.toml:/config.toml"
+    command:
+      - 56d7a450d75c6ba2706ef71da6ca80143ec4971add9c44d7d129a12fa7d3a364
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.102
+
+  node3_with_otel:
+    environment:
+      RUST_BACKTRACE: 1
+      RUST_LOG: zilliqa=info,opentelemetry=trace
+    container_name: zilliqanode3_with_otel
+    image: zq2-node0
+    volumes:
+      - "./config_docker.toml:/config.toml"
+    command:
+      - db670cbff28f4b15297d03fafdab8f5303d68b7591bd59e31eaef215dd0f246a
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.103
+
+  otterscan_with_otel:
+    environment:
+      ERIGON_URL: http://localhost:4201
+    image: zilliqa/otterscan:develop
+    pull_policy: always
+    ports:
+      - "5101:80"
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.9
+
+  faucet_with_otel:
+    depends_on:
+      - node0_with_otel
+    image: asia-docker.pkg.dev/prj-p-devops-services-tvwmrf63/zilliqa/eth-spout:v1.3.72
+    environment:
+      RPC_URL: http://node0:4201
+      NATIVE_TOKEN_SYMBOL: ZIL
+      PRIVATE_KEY: 65d7f4da9bedc8fb79cbf6722342960bbdfb9759bc0d9e3fb4989e831ccbc227
+      ETH_AMOUNT: 100
+      EXPLORER_URL: http://localhost:5200
+      MINIMUM_SECONDS_BETWEEN_REQUESTS: 60
+      BECH32_HRP: zil
+    ports:
+      - "5201:80"
+    # Fast shutdown - Don't gracefully close open connections.
+    stop_signal: SIGKILL
+    networks:
+      testnet_with_otel:
+        ipv4_address: 198.52.100.10
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317"
+      - "9011:8888"
+    networks:
+      - testnet_with_otel
+
+  mimir:
+    image: grafana/mimir:latest
+    volumes:
+      - "./mimir-config.yaml:/etc/mimir/config.yaml"
+    command:
+      - "./mimir"
+      - "--config.file"
+      - "/etc/mimir/config.yaml"
+    ports:
+      - "9009:9009"
+    networks:
+      - testnet_with_otel
+
+  grafana:
+    image: grafana/grafana-oss:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    ports:
+      - "9010:3000"
+    networks:
+      - testnet_with_otel
+
+networks:
+  testnet_with_otel:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 198.52.100.0/24
+          gateway: 198.52.100.1
diff --git a/infra/mimir.yaml → infra/opentelemetry/mimir-config.yaml b/infra/mimir.yaml → infra/opentelemetry/mimir-config.yaml
@@ -3,6 +3,12 @@ usage_stats:
 
 multitenancy_enabled: false
 
+memberlist:
+  bind_port: 7946
+  advertise_addr: 0.0.0.0
+  join_members:
+    - 0.0.0.0:7946
+
 blocks_storage:
   backend: filesystem
   bucket_store:
@@ -20,13 +26,13 @@ compactor:
 
 distributor:
   ring:
-    instance_addr: 127.0.0.1
+    instance_addr: 0.0.0.0
     kvstore:
       store: memberlist
 
 ingester:
   ring:
-    instance_addr: 127.0.0.1
+    instance_addr: 0.0.0.0
     kvstore:
       store: memberlist
     replication_factor: 1

diff --git a/infra/otel-collector-config.yaml → .../opentelemetry/otel-collector-config.yaml b/infra/otel-collector-config.yaml → .../opentelemetry/otel-collector-config.yaml
diff --git a/infra/tf/locals.tf b/infra/tf/locals.tf
@@ -19,5 +19,6 @@ locals {
     "roles/logging.logWriter=>${var.project_id}",
     "roles/artifactregistry.reader=>${var.gcp_docker_registry_project_id}",
     "roles/secretmanager.secretAccessor=>${var.project_id}",
+    "roles/cloudtrace.agent=>${var.project_id}",
   ]
 }
diff --git a/z2/resources/config.tera.toml b/z2/resources/config.tera.toml
@@ -4,6 +4,8 @@ p2p_port = 3333
 bootstrap_address = [ "{{ bootstrap_peer_id }}", "/dns/{{ bootstrap_endpoint }}/tcp/3333" ]
 {%- endif %}
 
+otlp_collector_endpoint = "http://localhost:4317"
+
 [[nodes]]
 eth_chain_id = {{ eth_chain_id }}
 allowed_timestamp_skew = { secs = 60, nanos = 0 }

diff --git a/z2/resources/node_provision.tera.py b/z2/resources/node_provision.tera.py
@@ -14,7 +14,6 @@
 templatefile() vars:
 - checkpoint_url, the ZQ2 checkpoint URL used for recover the validator nodes
 - persistence_url, the ZQ2 persistence URL used for recover the network
-- conifg, the ZQ2 validators configuration file
 - docker_image, the ZQ2 docker image (incl. version)
 - secret_key, the ZQ2 node secret key
 - role, the node role: validator or apps
@@ -121,7 +120,7 @@ def health():
 """
 
 ZQ2_SCRIPT="""#!/bin/bash
-echo yes |  gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev
+echo yes | gcloud auth configure-docker asia-docker.pkg.dev,europe-docker.pkg.dev
 
 ZQ2_IMAGE="{{ docker_image }}"
 
@@ -256,11 +255,14 @@ def health():
 
 OPS_AGENT_INSTALL_SCRIPT_URL="https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh"
 
-
 OPS_AGENT_CONFIG_FILE="/etc/google-cloud-ops-agent/config.yaml"
 
-
 OPS_AGENT_CONFIG="""
+combined:
+  receivers:
+    otlp:
+      type: otlp
+      metrics_mode: googlecloudmonitoring
 logging:
   receivers:
     zilliqa:
@@ -310,6 +312,13 @@ def health():
       default_pipeline:
         receivers: [hostmetrics]
         processors: [metrics_filter]
+      otlp:
+        receivers: [otlp]
+traces:
+  service:
+    pipelines:
+      otlp:
+        receivers: [otlp]
 """
 
 LOGROTATE_CONFIG="""

diff --git a/z2/resources/otel-compose.yaml b/z2/resources/otel-compose.yaml
@@ -3,7 +3,7 @@ services:
     image: otel/opentelemetry-collector:latest
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
-      - ./otel-config.yaml:/etc/otel-collector-config.yaml
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
     ports:
       - "4317:4317"
       - "9011:8888"
@@ -25,4 +25,4 @@ services:
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
     ports:
-      - "9010:3000"
+      - "9010:3000"