From dafdb567df37043f71f12cf9945063adffbe4254 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Mon, 5 Aug 2024 12:12:07 -0400 Subject: [PATCH 01/14] Move relayer to subfolder along with docs update --- README.md | 414 +----------------- relayer/README.md | 412 +++++++++++++++++ {api => relayer/api}/health_check.go | 0 {api => relayer/api}/relay_message.go | 0 {main => relayer/main}/main.go | 2 +- .../resources}/relayer-diagram.png | Bin scripts/build_relayer.sh | 14 +- scripts/constants.sh | 13 +- scripts/e2e_test.sh | 6 +- tests/manual_message.go | 2 +- tests/relay_message_api.go | 2 +- tests/utils/utils.go | 4 +- 12 files changed, 445 insertions(+), 424 deletions(-) create mode 100644 relayer/README.md rename {api => relayer/api}/health_check.go (100%) rename {api => relayer/api}/relay_message.go (100%) rename {main => relayer/main}/main.go (99%) rename {resources => relayer/resources}/relayer-diagram.png (100%) diff --git a/README.md b/README.md index c10e44a2..84c64708 100644 --- a/README.md +++ b/README.md @@ -1,412 +1,8 @@ -# AWM Relayer +# Avalanche Interchain Services -Reference relayer implementation for cross-chain Avalanche Warp Message delivery. +This repository contains services related to interchain messaging systems along with shared code to support them. -AWM Relayer listens for Warp message events on a set of source blockchains, and constructs transactions to relay the Warp message to the intended destination blockchain. The relayer does so by querying the source blockchain validator nodes for their BLS signatures on the Warp message, combining the individual BLS signatures into a single aggregate BLS signature, and packaging the aggregate BLS signature into a transaction according to the destination blockchain VM Warp message verification rules. +Currently implemented applications are -## Installation - -### Dev Container & Codespace - -To get started easily, we provide a Dev Container specification, that can be used using GitHub Codespace or locally using Docker and VS Code. [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) are a concept that utilizes containerization to create consistent and isolated development environment. You can run them directly on Github by clicking **Code**, switching to the **Codespaces** tab and clicking **Create codespace on main**. Alternatively, you can run them locally with the extensions for [VS Code](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) or other code editors. - -### Download Prebuilt Binaries - -Prebuilt binaries are available for download from the [releases page](https://github.com/ava-labs/awm-relayer/releases). - -The following commands demonstrate how to download and install the v0.2.13 release of the relayer on MacOS. The exact commands will vary by platform. - -```bash -# Download the release tarball and checksums -curl -w '%{http_code}' -sL -o ~/Downloads/awm-relayer_0.2.13_darwin_arm64.tar.gz https://github.com/ava-labs/awm-relayer/releases/download/v0.2.13/awm-relayer_0.2.13_darwin_arm64.tar.gz -curl -w '%{http_code}' -sL -o ~/Downloads/awm-relayer_0.2.13_checksums.txt https://github.com/ava-labs/awm-relayer/releases/download/v0.2.13/awm-relayer_0.2.13_checksums.txt - -# (Optional) Verify the checksums -cd ~/Downloads -# Confirm that the following two commands output the same checksum -grep "awm-relayer_0.2.13_darwin_arm64.tar.gz" "awm-relayer_0.2.13_checksums.txt" 2>/dev/null -shasum -a 256 "awm-relayer_0.2.13_darwin_arm64.tar.gz" 2>/dev/null - -# Extract the tarball and install the relayer binary -tar -xzf awm-relayer_0.2.13_darwin_arm64.tar.gz -sudo install awm-relayer /usr/local/bin -``` - -_Note:_ If downloading the binaries through a browser on MacOS, the browser may mark the binary as quarantined since it has not been verified through the App Store. To remove the quarantine, run the following command: - -```bash -xattr -d com.apple.quarantine /usr/local/bin/awm-relayer -``` - -### Download Docker Image - -The published Docker image can be pulled from `avaplatform/awm-relayer:latest` on dockerhub. - -### Build from Source - -See the [Building](#building) section for instructions on how to build the relayer from source. - -## Requirements - -### System Requirements - -- Ubuntu 22.04 or later - - Tested on x86_64/amd64 architecture. -- MacOS 14.3 or later - - Tested on arm64 architecture (Apple silicon). - -### API Requirements - -- AWM Relayer requires access to Avalanche API nodes for the P-Chain as well as any connected Subnets. The API nodes must have the following methods enabled: - - Each Subnet API node must have enabled: - - eth API (RPC and WS) - - The P-Chain API node must have enabled: - - platform.getHeight - - platform.validatedBy - - platform.getValidatorsAt OR platform.getCurrentValidators - - The Info API node must have enabled: - - info.peers - - info.getNetworkID - - If the Info API node is also a Subnet validator, it must have enabled: - - info.getNodeID - - info.getNodeIP - -The Fuji and Mainnet [public API nodes](https://docs.avax.network/tooling/rpc-providers) provided by Avalanche have these methods enabled, and are suitable for use with the relayer. - -### Peer-to-Peer Connections - -- By default, the AWM relayer implementation gathers BLS signatures from the validators of the source Subnet via peer-to-peer `AppRequest` messages. Validator nodes need to be configured to accept incoming peer connections. Otherwise, the relayer will fail to gather Warp message signatures. For example, networking rules may need to be adjusted to allow traffic on the default AvalancheGo P2P port (9651), or the public IP may need to be manually set in the [node configuration](https://docs.avax.network/nodes/configure/avalanchego-config-flags#public-ip). -- If configured to use the Warp API (see `warp-api-endpoint` in [Configuration](#configuration)) then aggregate signatures are fetched via a single RPC request, rather than `AppRequests` to individual validators. Note that the Warp API is disabled on the public API. - -### Private Key Management - -- Each configured destination blockchain requires a private key to sign transactions. This key can be provided as a hex-encoded string in the configuration (see `account-private-key` in [Configuration](#configuration)) or environment variable, or stored in KMS and used to sign transactions remotely (see `kms-key-id` and `kms-aws-region` in [Configuration](#configuration)). -- **Each private key used by the relayer should not be used to sign transactions outside of the relayer**, as this may cause the relayer to fail to sign transactions due to nonce mismatches. - -## Usage - -### Options - -The relayer binary accepts the following command line options. Other configuration options are not supported via the command line and must be provided via the configuration JSON file or environment variable. - -```bash -awm-relayer --config-file path-to-config Specifies the relayer config file and begin relaying messages. -awm-relayer --version Display awm-relayer version and exit. -awm-relayer --help Display awm-relayer usage and exit. -``` - -### Initialize the repository - -- Get all submodules: `git submodule update --init --recursive` - -### Building - -Before building, be sure to install Go, which is required even if you're just building the Docker image. You'll also need to install [buf](github.com/bufbuild/buf/). - -Build the relayer by running the script: - -```bash -./scripts/build.sh -``` - -Build a Docker image by running the script: - -```bash -./scripts/build_local_image.sh -``` - -### Configuration - -The relayer is configured via a JSON file, the path to which is passed in via the `--config-file` command line argument. Top level configuration options are also able to be set via environment variable. To get the environment variable corresponding to a key, upper case the key and change the delimiter from "-" to "_". For example, `LOG_LEVEL` sets the `"log-level"` JSON key. The following configuration options are available: - -`"log-level": "verbo" | "debug" | "info" | "warn" | "error" | "fatal" | "panic"` - -- The log level for the relayer. Defaults to `info`. - -`"p-chain-api": APIConfig` - -- The configuration for the Avalanche P-Chain API node. The `PChainAPI` object has the following configuration: - - `"base-url": string` - - - The URL of the Avalanche P-Chain API node to which the relayer will connect. This API node needs to have the following methods enabled: - - platform.getHeight - - platform.validatedBy - - platform.getValidatorsAt OR platform.getCurrentValidators - - `"query-parameters": map[string]string` - - - Additional query parameters to include in the API requests. - - `"http-headers": map[string]string` - - - Additional HTTP headers to include in the API requests. - -`"info-api": APIConfig` - -- The configuration for the Avalanche Info API node. The `InfoAPI` object has the following configuration: - - `"base-url": string` - - - The URL of the Avalanche Info API node to which the relayer will connect. This API node needs to have the following methods enabled: - - - info.peers - - info.getNetworkID - - - Additionally, if the Info API node is also a validator, it must have enabled: - - info.getNodeID - - info.getNodeIP - - `"query-parameters": map[string]string` - - - Additional query parameters to include in the API requests. - - `"http-headers": map[string]string` - - - Additional HTTP headers to include in the API requests. - -`"storage-location": string` - -- The path to the directory in which the relayer will store its state. Defaults to `./awm-relayer-storage`. - -`"redis-url": string` - -- The URL of the Redis server to use to manage state. This URL should specify the user, password, host, port, DB index, and protocol version. For example, `"redis://user:password@localhost:6379/0?protocol=3"`. Overrides `storage-location` if provided. - -`"process-missed-blocks": boolean` - -- Whether or not to process missed blocks after restarting. Defaults to `true`. If set to false, the relayer will start processing blocks from the chain head. - -`"api-port"`: unsigned integer - -- The port on which the relayer will listen for API requests. Defaults to `8080`. - -`"metrics-port"`: unsigned integer - -- The port on which the relayer will expose Prometheus metrics. Defaults to `9090`. - -`"db-write-interval-seconds": unsigned integer` - -- The interval at which the relayer will write to the database. Defaults to `10`. - -`"manual-warp-messages": []ManualWarpMessage` - -- The list of Warp messages to relay on startup, independent of the catch-up mechanism or normal operation. Each `ManualWarpMessage` has the following configuration: - - `"unsigned-message-bytes": string` - - - The hex-encoded bytes of the unsigned Warp message to relay. - - `"source-blockchain-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded blockchain ID of the source blockchain. - - `"destination-blockchain-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded blockchain ID of the destination blockchain. - - `"source-address": string` - - - The address of the source account that sent the Warp message. - - `"destination-address": string` - - - The address of the destination account that will receive the Warp message. - -`"source-blockchains": []SourceBlockchains` - -- The list of source blockchains to support. Each `SourceBlockchain` has the following configuration: - - `"subnet-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded Subnet ID. - - `"blockchain-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded blockchain ID. - - `"vm": string` - - - The VM type of the source blockchain. - - `"rpc-endpoint": APIConfig` - - - The RPC endpoint configuration of the source blockchain's API node. - - `"ws-endpoint": APIConfig` - - - The WebSocket endpoint configuration of the source blockchain's API node. - - `"message-contracts": map[string]MessageProtocolConfig` - - - Map of contract addresses to the config options of the protocol at that address. Each `MessageProtocolConfig` consists of a unique `message-format` name, and the raw JSON `settings`. - - `"supported-destinations": []SupportedDestination` - - - List of destinations that the source blockchain supports. Each `SupportedDestination` consists of a cb58-encoded destination blockchain ID (`"blockchain-id"`), and a list of hex-encoded addresses (`"addresses"`) on that destination blockchain that the relayer supports delivering Warp messages to. The destination address is defined by the message protocol. For example, it could be the address called from the message protocol contract. If no supported addresses are provided, all addresses are allowed on that blockchain. If `supported-destinations` is empty, then all destination blockchains (and therefore all addresses on those destination blockchains) are supported. - - `"process-historical-blocks-from-height": unsigned integer` - - - The block height at which to back-process transactions from the source blockchain. If the database already contains a later block height for the source blockchain, then that will be used instead. Must be non-zero. Will only be used if `process-missed-blocks` is set to `true`. - - `"allowed-origin-sender-addresses": []string` - - - List of addresses on this source blockchain to relay Warp messages from. The sending address is defined by the message protocol. For example, it could be defined as the EOA that initiates the transaction, or the address that calls the message protocol contract. If empty, then all addresses are allowed. - - `"warp-api-endpoint": APIConfig` - - - The RPC endpoint configuration for the Warp API, which is used to fetch Warp aggregate signatures. If omitted, then signatures are fetched via AppRequest instead. - -`"destination-blockchains": []DestinationBlockchains` - -- The list of destination blockchains to support. Each `DestinationBlockchain` has the following configuration: - - `"subnet-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded Subnet ID. - - `"blockchain-id": string` - - - cb58-encoded or "0x" prefixed hex-encoded blockchain ID. - - `"vm": string` - - - The VM type of the source blockchain. - - `"rpc-endpoint": APIConfig` - - - The RPC endpoint configuration of the destination blockchains's API node. - - `"account-private-key": string` - - - The hex-encoded private key to use for signing transactions on the destination blockchain. May be provided by the environment variable `ACCOUNT_PRIVATE_KEY`. Each `destination-subnet` may use a separate private key by appending the cb58 encoded blockchain ID to the private key environment variable name, for example `ACCOUNT_PRIVATE_KEY_11111111111111111111111111111111LpoYY` - - Please note that the private key should be exclusive to the relayer, see [Private Key Management](#private-key-management). - - `"kms-key-id": string` - - - The ID of the KMS key to use for signing transactions on the destination blockchain. Only one of `account-private-key` or `kms-key-id` should be provided. If `kms-key-id` is provided, then `kms-aws-region` is required. - - Please note that the private key in KMS should be exclusive to the relayer, see [Private Key Management](#private-key-management). - - `"kms-aws-region": string` - - - The AWS region in which the KMS key is located. Required if `kms-key-id` is provided. - -`"decider-url": string` - -- The URL of a service implementing the gRPC service defined by `proto/decider`, which will be queried for each message to determine whether that message should be relayed. - -## Architecture - -### Components - -The relayer consists of the following components: - -- At the global level: - - P2P app network: issues signature `AppRequests` - - P-Chain client: gets the validators for a Subnet - - Relayer database: stores latest processed block for each Application Relayer - - Currently supports Redis and local JSON file storage -- Per Source Blockchain - - Subscriber: listens for logs pertaining to cross-chain message transactions - - Source RPC client: queries for missed blocks on startup -- Per Destination Blockchain - - Destination RPC client: broadcasts transactions to the destination -- Application Relayers - - Relay messages from a specific source blockchain and source address to a specific destination blockchain and destination address - -### Data Flow - -
- -
- -### API - -#### `/relay` -- Used to manually relay a Warp message. The body of the request must contain the following JSON: -```json -{ - "blockchain-id": "", - "message-id": "", - "block-num": "" -} -``` -- If successful, the endpoint will return the following JSON: -```json -{ - "transaction-hash": "" -} -``` - -#### `/relay/message` -- Used to manually relay a warp message. The body of the request must contain the following JSON: -```json -{ - "unsigned-message-bytes": "", - "source-address": "" -} -``` -- If successful, the endpoint will return the following JSON: -```json -{ - "transaction-hash": "", -} -``` - -#### `/health` -- Takes no arguments. Returns a `200` status code if all Application Relayers are healthy. Returns a `503` status if any of the Application Relayers have experienced an unrecoverable error. Here is an example return body: -```json -{ - "status": "down", - "details": { - "relayers-all": { - "status": "down", - "timestamp": "2024-06-01T05:06:07.685522Z", - "error": "" - } - } -} -``` - -## Testing - -### Unit Tests - -Unit tests can be ran locally by running the command in the root of the project: - -```bash -./scripts/test.sh -``` - -If your temporary directory is not writable, the unit tests may fail with messages like `fork/exec /tmp/go-build2296620589/b247/config.test: permission denied`. To fix this, set the `TMPDIR` environment variable to something writable, for example `export TMPDIR=~/tmp`. - -### E2E Tests - -E2E tests are ran as part of CI, but can also be ran locally with the `--local` flag. To run the E2E tests locally, you'll need to install Gingko following the instructions [here](https://onsi.github.io/ginkgo/#installing-ginkgo) - -Next, provide the path to the `subnet-evm` repository and the path to a writeable data directory (this example uses `~/subnet-evm` and `~/tmp/e2e-test`) to use for the tests: - -```bash -./scripts/e2e_test.sh --local --subnet-evm ~/subnet-evm --data-dir ~/tmp/e2e-test -``` - -To run a specific E2E test, specify the environment variable `GINKGO_FOCUS`, which will then look for [test descriptions](./tests/e2e_test.go#L68) that match the provided input. For example, to run the `Basic Relay` test: - -```bash -GINKGO_FOCUS="Basic" ./scripts/e2e_test.sh --local --subnet-evm ~/subnet-evm --data-dir ~/tmp/e2e-test -``` - -The E2E tests use the `TeleporterMessenger` contract deployment transaction specified in the following files: - -- `tests/utils/UniversalTeleporterDeployerAddress.txt` -- `tests/utils/UniversalTeleporterDeployerTransaction.txt` -- `tests/utils/UniversalTeleporterMessagerContractAddress.txt` - To update the version of Teleporter used by the E2E tests, update these values with the latest contract deployment information. For more information on how to deploy the Teleporter contract, see the [Teleporter documentation](https://github.com/ava-labs/teleporter/tree/main/utils/contract-deployment). - -### Generate Mocks - -[Gomock](https://pkg.go.dev/go.uber.org/mock/gomock) is used to generate mocks for testing. To generate mocks, run the following command at the root of the project: - -```bash -go generate ./... -``` +1. [AWM Relayer](relayer/README.md) +2. [Signature Aggregator](signature-aggregator/README.md) \ No newline at end of file diff --git a/relayer/README.md b/relayer/README.md new file mode 100644 index 00000000..c10e44a2 --- /dev/null +++ b/relayer/README.md @@ -0,0 +1,412 @@ +# AWM Relayer + +Reference relayer implementation for cross-chain Avalanche Warp Message delivery. + +AWM Relayer listens for Warp message events on a set of source blockchains, and constructs transactions to relay the Warp message to the intended destination blockchain. The relayer does so by querying the source blockchain validator nodes for their BLS signatures on the Warp message, combining the individual BLS signatures into a single aggregate BLS signature, and packaging the aggregate BLS signature into a transaction according to the destination blockchain VM Warp message verification rules. + +## Installation + +### Dev Container & Codespace + +To get started easily, we provide a Dev Container specification, that can be used using GitHub Codespace or locally using Docker and VS Code. [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) are a concept that utilizes containerization to create consistent and isolated development environment. You can run them directly on Github by clicking **Code**, switching to the **Codespaces** tab and clicking **Create codespace on main**. Alternatively, you can run them locally with the extensions for [VS Code](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) or other code editors. + +### Download Prebuilt Binaries + +Prebuilt binaries are available for download from the [releases page](https://github.com/ava-labs/awm-relayer/releases). + +The following commands demonstrate how to download and install the v0.2.13 release of the relayer on MacOS. The exact commands will vary by platform. + +```bash +# Download the release tarball and checksums +curl -w '%{http_code}' -sL -o ~/Downloads/awm-relayer_0.2.13_darwin_arm64.tar.gz https://github.com/ava-labs/awm-relayer/releases/download/v0.2.13/awm-relayer_0.2.13_darwin_arm64.tar.gz +curl -w '%{http_code}' -sL -o ~/Downloads/awm-relayer_0.2.13_checksums.txt https://github.com/ava-labs/awm-relayer/releases/download/v0.2.13/awm-relayer_0.2.13_checksums.txt + +# (Optional) Verify the checksums +cd ~/Downloads +# Confirm that the following two commands output the same checksum +grep "awm-relayer_0.2.13_darwin_arm64.tar.gz" "awm-relayer_0.2.13_checksums.txt" 2>/dev/null +shasum -a 256 "awm-relayer_0.2.13_darwin_arm64.tar.gz" 2>/dev/null + +# Extract the tarball and install the relayer binary +tar -xzf awm-relayer_0.2.13_darwin_arm64.tar.gz +sudo install awm-relayer /usr/local/bin +``` + +_Note:_ If downloading the binaries through a browser on MacOS, the browser may mark the binary as quarantined since it has not been verified through the App Store. To remove the quarantine, run the following command: + +```bash +xattr -d com.apple.quarantine /usr/local/bin/awm-relayer +``` + +### Download Docker Image + +The published Docker image can be pulled from `avaplatform/awm-relayer:latest` on dockerhub. + +### Build from Source + +See the [Building](#building) section for instructions on how to build the relayer from source. + +## Requirements + +### System Requirements + +- Ubuntu 22.04 or later + - Tested on x86_64/amd64 architecture. +- MacOS 14.3 or later + - Tested on arm64 architecture (Apple silicon). + +### API Requirements + +- AWM Relayer requires access to Avalanche API nodes for the P-Chain as well as any connected Subnets. The API nodes must have the following methods enabled: + - Each Subnet API node must have enabled: + - eth API (RPC and WS) + - The P-Chain API node must have enabled: + - platform.getHeight + - platform.validatedBy + - platform.getValidatorsAt OR platform.getCurrentValidators + - The Info API node must have enabled: + - info.peers + - info.getNetworkID + - If the Info API node is also a Subnet validator, it must have enabled: + - info.getNodeID + - info.getNodeIP + +The Fuji and Mainnet [public API nodes](https://docs.avax.network/tooling/rpc-providers) provided by Avalanche have these methods enabled, and are suitable for use with the relayer. + +### Peer-to-Peer Connections + +- By default, the AWM relayer implementation gathers BLS signatures from the validators of the source Subnet via peer-to-peer `AppRequest` messages. Validator nodes need to be configured to accept incoming peer connections. Otherwise, the relayer will fail to gather Warp message signatures. For example, networking rules may need to be adjusted to allow traffic on the default AvalancheGo P2P port (9651), or the public IP may need to be manually set in the [node configuration](https://docs.avax.network/nodes/configure/avalanchego-config-flags#public-ip). +- If configured to use the Warp API (see `warp-api-endpoint` in [Configuration](#configuration)) then aggregate signatures are fetched via a single RPC request, rather than `AppRequests` to individual validators. Note that the Warp API is disabled on the public API. + +### Private Key Management + +- Each configured destination blockchain requires a private key to sign transactions. This key can be provided as a hex-encoded string in the configuration (see `account-private-key` in [Configuration](#configuration)) or environment variable, or stored in KMS and used to sign transactions remotely (see `kms-key-id` and `kms-aws-region` in [Configuration](#configuration)). +- **Each private key used by the relayer should not be used to sign transactions outside of the relayer**, as this may cause the relayer to fail to sign transactions due to nonce mismatches. + +## Usage + +### Options + +The relayer binary accepts the following command line options. Other configuration options are not supported via the command line and must be provided via the configuration JSON file or environment variable. + +```bash +awm-relayer --config-file path-to-config Specifies the relayer config file and begin relaying messages. +awm-relayer --version Display awm-relayer version and exit. +awm-relayer --help Display awm-relayer usage and exit. +``` + +### Initialize the repository + +- Get all submodules: `git submodule update --init --recursive` + +### Building + +Before building, be sure to install Go, which is required even if you're just building the Docker image. You'll also need to install [buf](github.com/bufbuild/buf/). + +Build the relayer by running the script: + +```bash +./scripts/build.sh +``` + +Build a Docker image by running the script: + +```bash +./scripts/build_local_image.sh +``` + +### Configuration + +The relayer is configured via a JSON file, the path to which is passed in via the `--config-file` command line argument. Top level configuration options are also able to be set via environment variable. To get the environment variable corresponding to a key, upper case the key and change the delimiter from "-" to "_". For example, `LOG_LEVEL` sets the `"log-level"` JSON key. The following configuration options are available: + +`"log-level": "verbo" | "debug" | "info" | "warn" | "error" | "fatal" | "panic"` + +- The log level for the relayer. Defaults to `info`. + +`"p-chain-api": APIConfig` + +- The configuration for the Avalanche P-Chain API node. The `PChainAPI` object has the following configuration: + + `"base-url": string` + + - The URL of the Avalanche P-Chain API node to which the relayer will connect. This API node needs to have the following methods enabled: + - platform.getHeight + - platform.validatedBy + - platform.getValidatorsAt OR platform.getCurrentValidators + + `"query-parameters": map[string]string` + + - Additional query parameters to include in the API requests. + + `"http-headers": map[string]string` + + - Additional HTTP headers to include in the API requests. + +`"info-api": APIConfig` + +- The configuration for the Avalanche Info API node. The `InfoAPI` object has the following configuration: + + `"base-url": string` + + - The URL of the Avalanche Info API node to which the relayer will connect. This API node needs to have the following methods enabled: + + - info.peers + - info.getNetworkID + + - Additionally, if the Info API node is also a validator, it must have enabled: + - info.getNodeID + - info.getNodeIP + + `"query-parameters": map[string]string` + + - Additional query parameters to include in the API requests. + + `"http-headers": map[string]string` + + - Additional HTTP headers to include in the API requests. + +`"storage-location": string` + +- The path to the directory in which the relayer will store its state. Defaults to `./awm-relayer-storage`. + +`"redis-url": string` + +- The URL of the Redis server to use to manage state. This URL should specify the user, password, host, port, DB index, and protocol version. For example, `"redis://user:password@localhost:6379/0?protocol=3"`. Overrides `storage-location` if provided. + +`"process-missed-blocks": boolean` + +- Whether or not to process missed blocks after restarting. Defaults to `true`. If set to false, the relayer will start processing blocks from the chain head. + +`"api-port"`: unsigned integer + +- The port on which the relayer will listen for API requests. Defaults to `8080`. + +`"metrics-port"`: unsigned integer + +- The port on which the relayer will expose Prometheus metrics. Defaults to `9090`. + +`"db-write-interval-seconds": unsigned integer` + +- The interval at which the relayer will write to the database. Defaults to `10`. + +`"manual-warp-messages": []ManualWarpMessage` + +- The list of Warp messages to relay on startup, independent of the catch-up mechanism or normal operation. Each `ManualWarpMessage` has the following configuration: + + `"unsigned-message-bytes": string` + + - The hex-encoded bytes of the unsigned Warp message to relay. + + `"source-blockchain-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded blockchain ID of the source blockchain. + + `"destination-blockchain-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded blockchain ID of the destination blockchain. + + `"source-address": string` + + - The address of the source account that sent the Warp message. + + `"destination-address": string` + + - The address of the destination account that will receive the Warp message. + +`"source-blockchains": []SourceBlockchains` + +- The list of source blockchains to support. Each `SourceBlockchain` has the following configuration: + + `"subnet-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded Subnet ID. + + `"blockchain-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded blockchain ID. + + `"vm": string` + + - The VM type of the source blockchain. + + `"rpc-endpoint": APIConfig` + + - The RPC endpoint configuration of the source blockchain's API node. + + `"ws-endpoint": APIConfig` + + - The WebSocket endpoint configuration of the source blockchain's API node. + + `"message-contracts": map[string]MessageProtocolConfig` + + - Map of contract addresses to the config options of the protocol at that address. Each `MessageProtocolConfig` consists of a unique `message-format` name, and the raw JSON `settings`. + + `"supported-destinations": []SupportedDestination` + + - List of destinations that the source blockchain supports. Each `SupportedDestination` consists of a cb58-encoded destination blockchain ID (`"blockchain-id"`), and a list of hex-encoded addresses (`"addresses"`) on that destination blockchain that the relayer supports delivering Warp messages to. The destination address is defined by the message protocol. For example, it could be the address called from the message protocol contract. If no supported addresses are provided, all addresses are allowed on that blockchain. If `supported-destinations` is empty, then all destination blockchains (and therefore all addresses on those destination blockchains) are supported. + + `"process-historical-blocks-from-height": unsigned integer` + + - The block height at which to back-process transactions from the source blockchain. If the database already contains a later block height for the source blockchain, then that will be used instead. Must be non-zero. Will only be used if `process-missed-blocks` is set to `true`. + + `"allowed-origin-sender-addresses": []string` + + - List of addresses on this source blockchain to relay Warp messages from. The sending address is defined by the message protocol. For example, it could be defined as the EOA that initiates the transaction, or the address that calls the message protocol contract. If empty, then all addresses are allowed. + + `"warp-api-endpoint": APIConfig` + + - The RPC endpoint configuration for the Warp API, which is used to fetch Warp aggregate signatures. If omitted, then signatures are fetched via AppRequest instead. + +`"destination-blockchains": []DestinationBlockchains` + +- The list of destination blockchains to support. Each `DestinationBlockchain` has the following configuration: + + `"subnet-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded Subnet ID. + + `"blockchain-id": string` + + - cb58-encoded or "0x" prefixed hex-encoded blockchain ID. + + `"vm": string` + + - The VM type of the source blockchain. + + `"rpc-endpoint": APIConfig` + + - The RPC endpoint configuration of the destination blockchains's API node. + + `"account-private-key": string` + + - The hex-encoded private key to use for signing transactions on the destination blockchain. May be provided by the environment variable `ACCOUNT_PRIVATE_KEY`. Each `destination-subnet` may use a separate private key by appending the cb58 encoded blockchain ID to the private key environment variable name, for example `ACCOUNT_PRIVATE_KEY_11111111111111111111111111111111LpoYY` + - Please note that the private key should be exclusive to the relayer, see [Private Key Management](#private-key-management). + + `"kms-key-id": string` + + - The ID of the KMS key to use for signing transactions on the destination blockchain. Only one of `account-private-key` or `kms-key-id` should be provided. If `kms-key-id` is provided, then `kms-aws-region` is required. + - Please note that the private key in KMS should be exclusive to the relayer, see [Private Key Management](#private-key-management). + + `"kms-aws-region": string` + + - The AWS region in which the KMS key is located. Required if `kms-key-id` is provided. + +`"decider-url": string` + +- The URL of a service implementing the gRPC service defined by `proto/decider`, which will be queried for each message to determine whether that message should be relayed. + +## Architecture + +### Components + +The relayer consists of the following components: + +- At the global level: + - P2P app network: issues signature `AppRequests` + - P-Chain client: gets the validators for a Subnet + - Relayer database: stores latest processed block for each Application Relayer + - Currently supports Redis and local JSON file storage +- Per Source Blockchain + - Subscriber: listens for logs pertaining to cross-chain message transactions + - Source RPC client: queries for missed blocks on startup +- Per Destination Blockchain + - Destination RPC client: broadcasts transactions to the destination +- Application Relayers + - Relay messages from a specific source blockchain and source address to a specific destination blockchain and destination address + +### Data Flow + +
+ +
+ +### API + +#### `/relay` +- Used to manually relay a Warp message. The body of the request must contain the following JSON: +```json +{ + "blockchain-id": "", + "message-id": "", + "block-num": "" +} +``` +- If successful, the endpoint will return the following JSON: +```json +{ + "transaction-hash": "" +} +``` + +#### `/relay/message` +- Used to manually relay a warp message. The body of the request must contain the following JSON: +```json +{ + "unsigned-message-bytes": "", + "source-address": "" +} +``` +- If successful, the endpoint will return the following JSON: +```json +{ + "transaction-hash": "", +} +``` + +#### `/health` +- Takes no arguments. Returns a `200` status code if all Application Relayers are healthy. Returns a `503` status if any of the Application Relayers have experienced an unrecoverable error. Here is an example return body: +```json +{ + "status": "down", + "details": { + "relayers-all": { + "status": "down", + "timestamp": "2024-06-01T05:06:07.685522Z", + "error": "" + } + } +} +``` + +## Testing + +### Unit Tests + +Unit tests can be ran locally by running the command in the root of the project: + +```bash +./scripts/test.sh +``` + +If your temporary directory is not writable, the unit tests may fail with messages like `fork/exec /tmp/go-build2296620589/b247/config.test: permission denied`. To fix this, set the `TMPDIR` environment variable to something writable, for example `export TMPDIR=~/tmp`. + +### E2E Tests + +E2E tests are ran as part of CI, but can also be ran locally with the `--local` flag. To run the E2E tests locally, you'll need to install Gingko following the instructions [here](https://onsi.github.io/ginkgo/#installing-ginkgo) + +Next, provide the path to the `subnet-evm` repository and the path to a writeable data directory (this example uses `~/subnet-evm` and `~/tmp/e2e-test`) to use for the tests: + +```bash +./scripts/e2e_test.sh --local --subnet-evm ~/subnet-evm --data-dir ~/tmp/e2e-test +``` + +To run a specific E2E test, specify the environment variable `GINKGO_FOCUS`, which will then look for [test descriptions](./tests/e2e_test.go#L68) that match the provided input. For example, to run the `Basic Relay` test: + +```bash +GINKGO_FOCUS="Basic" ./scripts/e2e_test.sh --local --subnet-evm ~/subnet-evm --data-dir ~/tmp/e2e-test +``` + +The E2E tests use the `TeleporterMessenger` contract deployment transaction specified in the following files: + +- `tests/utils/UniversalTeleporterDeployerAddress.txt` +- `tests/utils/UniversalTeleporterDeployerTransaction.txt` +- `tests/utils/UniversalTeleporterMessagerContractAddress.txt` + To update the version of Teleporter used by the E2E tests, update these values with the latest contract deployment information. For more information on how to deploy the Teleporter contract, see the [Teleporter documentation](https://github.com/ava-labs/teleporter/tree/main/utils/contract-deployment). + +### Generate Mocks + +[Gomock](https://pkg.go.dev/go.uber.org/mock/gomock) is used to generate mocks for testing. To generate mocks, run the following command at the root of the project: + +```bash +go generate ./... +``` diff --git a/api/health_check.go b/relayer/api/health_check.go similarity index 100% rename from api/health_check.go rename to relayer/api/health_check.go diff --git a/api/relay_message.go b/relayer/api/relay_message.go similarity index 100% rename from api/relay_message.go rename to relayer/api/relay_message.go diff --git a/main/main.go b/relayer/main/main.go similarity index 99% rename from main/main.go rename to relayer/main/main.go index 78038857..236eca9c 100644 --- a/main/main.go +++ b/relayer/main/main.go @@ -18,7 +18,6 @@ import ( "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/set" - "github.com/ava-labs/awm-relayer/api" "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/database" "github.com/ava-labs/awm-relayer/messages" @@ -26,6 +25,7 @@ import ( "github.com/ava-labs/awm-relayer/messages/teleporter" "github.com/ava-labs/awm-relayer/peers" "github.com/ava-labs/awm-relayer/relayer" + "github.com/ava-labs/awm-relayer/relayer/api" "github.com/ava-labs/awm-relayer/signature-aggregator/aggregator" "github.com/ava-labs/awm-relayer/utils" "github.com/ava-labs/awm-relayer/vms" diff --git a/resources/relayer-diagram.png b/relayer/resources/relayer-diagram.png similarity index 100% rename from resources/relayer-diagram.png rename to relayer/resources/relayer-diagram.png diff --git a/scripts/build_relayer.sh b/scripts/build_relayer.sh index 8069a874..842c7c06 100755 --- a/scripts/build_relayer.sh +++ b/scripts/build_relayer.sh @@ -22,15 +22,22 @@ version_lt() { fi } -# Root directory + +# Relayer directory RELAYER_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" + cd ../relayer && pwd +) + +# Base directory +BASE_PATH=$( + cd $RELAYER_PATH cd .. && pwd ) # Load the versions and constants -source "$RELAYER_PATH"/scripts/versions.sh -source "$RELAYER_PATH"/scripts/constants.sh +source "$BASE_PATH"/scripts/versions.sh +source "$BASE_PATH"/scripts/constants.sh go_version_minimum=$GO_VERSION @@ -50,6 +57,7 @@ else exit 1 fi +cd $RELAYER_PATH # Build AWM Relayer, which is run as a standalone process last_git_tag=$(git describe --tags --abbrev=0 2>/dev/null) || last_git_tag="v0.0.0-dev" echo "Building AWM Relayer Version: $last_git_tag at $binary_path" diff --git a/scripts/constants.sh b/scripts/constants.sh index ee1a483b..752f2d18 100755 --- a/scripts/constants.sh +++ b/scripts/constants.sh @@ -5,24 +5,29 @@ # Use lower_case variables in the scripts and UPPER_CASE variables for override # Use the constants.sh for env overrides -RELAYER_PATH=$( +BASE_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" cd .. && pwd ) +RELAYER_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" + cd ../relayer && pwd +) + SIGNATURE_AGGREGATOR_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" cd ../signature-aggregator && pwd ) # Where binaries go -relayer_path="$RELAYER_PATH/build/awm-relayer" -signature_aggregator_path="$SIGNATURE_AGGREGATOR_PATH/build/signature-aggregator" +relayer_path="$BASE_PATH/build/awm-relayer" +signature_aggregator_path="$BASE_PATH/build/signature-aggregator" # Set the PATHS GOPATH="$(go env GOPATH)" -TELEPORTER_PATH="$RELAYER_PATH"/tests/contracts/lib/teleporter +TELEPORTER_PATH="$BASE_PATH"/tests/contracts/lib/teleporter source $TELEPORTER_PATH/scripts/constants.sh # Avalabs docker hub repo is avaplatform/awm-relayer. diff --git a/scripts/e2e_test.sh b/scripts/e2e_test.sh index 84f6718d..028d2547 100755 --- a/scripts/e2e_test.sh +++ b/scripts/e2e_test.sh @@ -52,14 +52,14 @@ if [ "$LOCAL" = true ]; then export DATA_DIR=$DATA_DIRECTORY/data fi -RELAYER_PATH=$( +BASE_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" cd .. && pwd ) -source "$RELAYER_PATH"/scripts/constants.sh +source "$BASE_PATH"/scripts/constants.sh -source "$RELAYER_PATH"/scripts/versions.sh +source "$BASE_PATH"/scripts/versions.sh # Build ginkgo # to install the ginkgo binary (required for test build and run) diff --git a/tests/manual_message.go b/tests/manual_message.go index 03d0ce9f..9ec03fd5 100644 --- a/tests/manual_message.go +++ b/tests/manual_message.go @@ -12,8 +12,8 @@ import ( "net/http" "time" - "github.com/ava-labs/awm-relayer/api" offchainregistry "github.com/ava-labs/awm-relayer/messages/off-chain-registry" + "github.com/ava-labs/awm-relayer/relayer/api" testUtils "github.com/ava-labs/awm-relayer/tests/utils" "github.com/ava-labs/subnet-evm/accounts/abi/bind" "github.com/ava-labs/teleporter/tests/interfaces" diff --git a/tests/relay_message_api.go b/tests/relay_message_api.go index b737e4c7..1dcef874 100644 --- a/tests/relay_message_api.go +++ b/tests/relay_message_api.go @@ -14,7 +14,7 @@ import ( "github.com/ava-labs/avalanchego/ids" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/api" + "github.com/ava-labs/awm-relayer/relayer/api" testUtils "github.com/ava-labs/awm-relayer/tests/utils" "github.com/ava-labs/subnet-evm/core/types" subnetEvmInterfaces "github.com/ava-labs/subnet-evm/interfaces" diff --git a/tests/utils/utils.go b/tests/utils/utils.go index 270706ad..a79c08bb 100644 --- a/tests/utils/utils.go +++ b/tests/utils/utils.go @@ -117,10 +117,10 @@ func BuildAndRunSignatureAggregatorExecutable(ctx context.Context, configPath st var signatureAggregatorCtx context.Context signatureAggregatorCtx, signatureAggregatorCancelFunc := context.WithCancel(ctx) log.Info("Instantiating the signature-aggregator executable command") - log.Info(fmt.Sprintf("./signature-aggregator/build/signature-aggregator --config-file %s ", configPath)) + log.Info(fmt.Sprintf("./build/signature-aggregator --config-file %s ", configPath)) signatureAggregatorCmd := exec.CommandContext( signatureAggregatorCtx, - "./signature-aggregator/build/signature-aggregator", + "./build/signature-aggregator", "--config-file", configPath, ) From 53972c75ed0bbc82f69b6fc3dd57426fa171116a Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 09:22:01 -0400 Subject: [PATCH 02/14] move relayer configs --- {config => relayer/config}/config.go | 10 ++++++---- {config => relayer/config}/config_test.go | 0 .../config}/destination_blockchain.go | 15 ++++++++------- {config => relayer/config}/flags.go | 0 {config => relayer/config}/keys.go | 0 {config => relayer/config}/source_blockchain.go | 7 ++++--- {config => relayer/config}/test_utils.go | 0 {config => relayer/config}/types.go | 0 {config => relayer/config}/viper.go | 0 {config => relayer/config}/viper_test.go | 12 +++++++----- 10 files changed, 25 insertions(+), 19 deletions(-) rename {config => relayer/config}/config.go (96%) rename {config => relayer/config}/config_test.go (100%) rename {config => relayer/config}/destination_blockchain.go (81%) rename {config => relayer/config}/flags.go (100%) rename {config => relayer/config}/keys.go (100%) rename {config => relayer/config}/source_blockchain.go (96%) rename {config => relayer/config}/test_utils.go (100%) rename {config => relayer/config}/types.go (100%) rename {config => relayer/config}/viper.go (100%) rename {config => relayer/config}/viper_test.go (89%) diff --git a/config/config.go b/relayer/config/config.go similarity index 96% rename from config/config.go rename to relayer/config/config.go index 231c0795..72632ac2 100644 --- a/config/config.go +++ b/relayer/config/config.go @@ -9,6 +9,8 @@ import ( "fmt" "net/url" + baseCfg "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" @@ -55,8 +57,8 @@ type Config struct { APIPort uint16 `mapstructure:"api-port" json:"api-port"` MetricsPort uint16 `mapstructure:"metrics-port" json:"metrics-port"` DBWriteIntervalSeconds uint64 `mapstructure:"db-write-interval-seconds" json:"db-write-interval-seconds"` //nolint:lll - PChainAPI *APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` - InfoAPI *APIConfig `mapstructure:"info-api" json:"info-api"` + PChainAPI *baseCfg.APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` + InfoAPI *baseCfg.APIConfig `mapstructure:"info-api" json:"info-api"` SourceBlockchains []*SourceBlockchain `mapstructure:"source-blockchains" json:"source-blockchains"` DestinationBlockchains []*DestinationBlockchain `mapstructure:"destination-blockchains" json:"destination-blockchains"` ProcessMissedBlocks bool `mapstructure:"process-missed-blocks" json:"process-missed-blocks"` @@ -248,11 +250,11 @@ func (c *Config) GetWarpQuorum(blockchainID ids.ID) (WarpQuorum, error) { } // Config implements the peers.Config interface -func (c *Config) GetPChainAPI() *APIConfig { +func (c *Config) GetPChainAPI() *baseCfg.APIConfig { return c.PChainAPI } // Config implements the peers.Config interface -func (c *Config) GetInfoAPI() *APIConfig { +func (c *Config) GetInfoAPI() *baseCfg.APIConfig { return c.InfoAPI } diff --git a/config/config_test.go b/relayer/config/config_test.go similarity index 100% rename from config/config_test.go rename to relayer/config/config_test.go diff --git a/config/destination_blockchain.go b/relayer/config/destination_blockchain.go similarity index 81% rename from config/destination_blockchain.go rename to relayer/config/destination_blockchain.go index 353c6c0c..095cd3e7 100644 --- a/config/destination_blockchain.go +++ b/relayer/config/destination_blockchain.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/ava-labs/avalanchego/ids" + baseCfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/crypto" ) @@ -13,13 +14,13 @@ import ( // Destination blockchain configuration. Specifies how to connect to and issue // transactions on the destination blockchain. type DestinationBlockchain struct { - SubnetID string `mapstructure:"subnet-id" json:"subnet-id"` - BlockchainID string `mapstructure:"blockchain-id" json:"blockchain-id"` - VM string `mapstructure:"vm" json:"vm"` - RPCEndpoint APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` - KMSKeyID string `mapstructure:"kms-key-id" json:"kms-key-id"` - KMSAWSRegion string `mapstructure:"kms-aws-region" json:"kms-aws-region"` - AccountPrivateKey string `mapstructure:"account-private-key" json:"account-private-key"` + SubnetID string `mapstructure:"subnet-id" json:"subnet-id"` + BlockchainID string `mapstructure:"blockchain-id" json:"blockchain-id"` + VM string `mapstructure:"vm" json:"vm"` + RPCEndpoint baseCfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` + KMSKeyID string `mapstructure:"kms-key-id" json:"kms-key-id"` + KMSAWSRegion string `mapstructure:"kms-aws-region" json:"kms-aws-region"` + AccountPrivateKey string `mapstructure:"account-private-key" json:"account-private-key"` // Fetched from the chain after startup warpQuorum WarpQuorum diff --git a/config/flags.go b/relayer/config/flags.go similarity index 100% rename from config/flags.go rename to relayer/config/flags.go diff --git a/config/keys.go b/relayer/config/keys.go similarity index 100% rename from config/keys.go rename to relayer/config/keys.go diff --git a/config/source_blockchain.go b/relayer/config/source_blockchain.go similarity index 96% rename from config/source_blockchain.go rename to relayer/config/source_blockchain.go index 1786d280..7d418cb5 100644 --- a/config/source_blockchain.go +++ b/relayer/config/source_blockchain.go @@ -5,6 +5,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/set" + baseCfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/common" ) @@ -18,13 +19,13 @@ type SourceBlockchain struct { SubnetID string `mapstructure:"subnet-id" json:"subnet-id"` BlockchainID string `mapstructure:"blockchain-id" json:"blockchain-id"` //nolint:lll VM string `mapstructure:"vm" json:"vm"` - RPCEndpoint APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` //nolint:lll - WSEndpoint APIConfig `mapstructure:"ws-endpoint" json:"ws-endpoint"` //nolint:lll + RPCEndpoint baseCfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` //nolint:lll + WSEndpoint baseCfg.APIConfig `mapstructure:"ws-endpoint" json:"ws-endpoint"` //nolint:lll MessageContracts map[string]MessageProtocolConfig `mapstructure:"message-contracts" json:"message-contracts"` //nolint:lll SupportedDestinations []*SupportedDestination `mapstructure:"supported-destinations" json:"supported-destinations"` //nolint:lll ProcessHistoricalBlocksFromHeight uint64 `mapstructure:"process-historical-blocks-from-height" json:"process-historical-blocks-from-height"` //nolint:lll AllowedOriginSenderAddresses []string `mapstructure:"allowed-origin-sender-addresses" json:"allowed-origin-sender-addresses"` //nolint:lll - WarpAPIEndpoint APIConfig `mapstructure:"warp-api-endpoint" json:"warp-api-endpoint"` //nolint:lll + WarpAPIEndpoint baseCfg.APIConfig `mapstructure:"warp-api-endpoint" json:"warp-api-endpoint"` //nolint:lll // convenience fields to access parsed data after initialization subnetID ids.ID diff --git a/config/test_utils.go b/relayer/config/test_utils.go similarity index 100% rename from config/test_utils.go rename to relayer/config/test_utils.go diff --git a/config/types.go b/relayer/config/types.go similarity index 100% rename from config/types.go rename to relayer/config/types.go diff --git a/config/viper.go b/relayer/config/viper.go similarity index 100% rename from config/viper.go rename to relayer/config/viper.go diff --git a/config/viper_test.go b/relayer/config/viper_test.go similarity index 89% rename from config/viper_test.go rename to relayer/config/viper_test.go index 9fd45569..fec8879e 100644 --- a/config/viper_test.go +++ b/relayer/config/viper_test.go @@ -5,6 +5,8 @@ import ( "os" "testing" + baseCfg "github.com/ava-labs/awm-relayer/config" + "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -24,10 +26,10 @@ func TestBuildConfig(t *testing.T) { require.Equal(t, defaultAPIPort, cfg.APIPort) require.Equal(t, defaultMetricsPort, cfg.MetricsPort) require.Equal(t, defaultIntervalSeconds, cfg.DBWriteIntervalSeconds) - require.Equal(t, &APIConfig{ + require.Equal(t, &baseCfg.APIConfig{ BaseURL: "https://api.avax-test.network", }, cfg.PChainAPI) - require.Equal(t, &APIConfig{ + require.Equal(t, &baseCfg.APIConfig{ BaseURL: "https://api.avax-test.network", }, cfg.InfoAPI) @@ -36,10 +38,10 @@ func TestBuildConfig(t *testing.T) { SubnetID: "11111111111111111111111111111111LpoYY", BlockchainID: "yH8D7ThNJkxmtkuv2jgBa4P1Rn3Qpr4pPr7QYNfcdoS6k6HWp", VM: "evm", - RPCEndpoint: APIConfig{ + RPCEndpoint: baseCfg.APIConfig{ BaseURL: "https://api.avax-test.network/ext/bc/C/rpc", }, - WSEndpoint: APIConfig{ + WSEndpoint: baseCfg.APIConfig{ BaseURL: "wss://api.avax-test.network/ext/bc/C/ws", }, MessageContracts: map[string]MessageProtocolConfig{ @@ -56,7 +58,7 @@ func TestBuildConfig(t *testing.T) { require.Equal(t, &DestinationBlockchain{ SubnetID: "7WtoAMPhrmh5KosDUsFL9yTcvw7YSxiKHPpdfs4JsgW47oZT5", BlockchainID: "2D8RG4UpSXbPbvPCAWppNJyqTG2i2CAXSkTgmTBBvs7GKNZjsY", - RPCEndpoint: APIConfig{ + RPCEndpoint: baseCfg.APIConfig{ BaseURL: "https://subnets.avax.network/dispatch/testnet/rpc", }, VM: "evm", From 03ec5dc7aec71ac81cc14a6fd3ad704263127648 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 10:04:02 -0400 Subject: [PATCH 03/14] update all references to configs --- README.md | 4 +-- database/database.go | 2 +- database/database_test.go | 2 +- database/mocks/mock_database.go | 1 + database/relayer_id.go | 2 +- .../off-chain-registry/message_handler.go | 2 +- .../message_handler_test.go | 2 +- messages/teleporter/message_handler.go | 2 +- messages/teleporter/message_handler_test.go | 2 +- peers/app_request_network.go | 16 +++++------ relayer/README.md | 28 ++++++++++++++++--- relayer/application_relayer.go | 2 +- relayer/config/config_test.go | 5 ++-- relayer/config/test_utils.go | 22 +++++++++------ relayer/listener.go | 2 +- relayer/main/main.go | 2 +- tests/allowed_addresses.go | 2 +- tests/utils/utils.go | 27 +++++++++--------- vms/contract_message.go | 2 +- vms/destination_client.go | 2 +- vms/evm/contract_message.go | 2 +- vms/evm/contract_message_test.go | 2 +- vms/evm/destination_client.go | 2 +- vms/evm/destination_client_test.go | 5 ++-- vms/evm/signer/signer.go | 2 +- vms/evm/signer/tx_signer_test.go | 2 +- vms/evm/subscriber_test.go | 5 ++-- vms/subscriber.go | 2 +- 28 files changed, 90 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 4c6dbc7a..0bcec4ca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Avalanche Interchain Services +# Avalanche ICM Off-chain Services -This repository contains services related to interchain messaging systems along with shared code to support them. +This repository contains services related to Avalanche ICM (Interchain Messaging) systems along with shared code to support them. Currently implemented applications are diff --git a/database/database.go b/database/database.go index 3df4be43..9af053ab 100644 --- a/database/database.go +++ b/database/database.go @@ -7,7 +7,7 @@ package database import ( "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" "go.uber.org/zap" diff --git a/database/database_test.go b/database/database_test.go index d07d0359..d645603f 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -6,7 +6,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/set" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/database/mocks/mock_database.go b/database/mocks/mock_database.go index 0dd5aaaa..ae635b1e 100644 --- a/database/mocks/mock_database.go +++ b/database/mocks/mock_database.go @@ -5,6 +5,7 @@ // // mockgen -source=database.go -destination=./mocks/mock_database.go -package=mocks // + // Package mocks is a generated GoMock package. package mocks diff --git a/database/relayer_id.go b/database/relayer_id.go index e1c0d6b1..2b8fee17 100644 --- a/database/relayer_id.go +++ b/database/relayer_id.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" diff --git a/messages/off-chain-registry/message_handler.go b/messages/off-chain-registry/message_handler.go index e76e56ee..090c1a23 100644 --- a/messages/off-chain-registry/message_handler.go +++ b/messages/off-chain-registry/message_handler.go @@ -12,8 +12,8 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" warpPayload "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/messages" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/vms" "github.com/ava-labs/subnet-evm/accounts/abi/bind" "github.com/ava-labs/subnet-evm/ethclient" diff --git a/messages/off-chain-registry/message_handler_test.go b/messages/off-chain-registry/message_handler_test.go index df745c13..258df47c 100644 --- a/messages/off-chain-registry/message_handler_test.go +++ b/messages/off-chain-registry/message_handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" mock_evm "github.com/ava-labs/awm-relayer/vms/evm/mocks" mock_vms "github.com/ava-labs/awm-relayer/vms/mocks" teleporterregistry "github.com/ava-labs/teleporter/abi-bindings/go/teleporter/registry/TeleporterRegistry" diff --git a/messages/teleporter/message_handler.go b/messages/teleporter/message_handler.go index dd7d8b58..98b04d48 100644 --- a/messages/teleporter/message_handler.go +++ b/messages/teleporter/message_handler.go @@ -13,9 +13,9 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" warpPayload "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/messages" pbDecider "github.com/ava-labs/awm-relayer/proto/pb/decider" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ava-labs/awm-relayer/vms" "github.com/ava-labs/subnet-evm/accounts/abi/bind" diff --git a/messages/teleporter/message_handler_test.go b/messages/teleporter/message_handler_test.go index 677735be..afe6fa0f 100644 --- a/messages/teleporter/message_handler_test.go +++ b/messages/teleporter/message_handler_test.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" warpPayload "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" mock_evm "github.com/ava-labs/awm-relayer/vms/evm/mocks" mock_vms "github.com/ava-labs/awm-relayer/vms/mocks" "github.com/ava-labs/subnet-evm/accounts/abi/bind" diff --git a/peers/app_request_network.go b/peers/app_request_network.go index 651d0adb..7ac1f05c 100644 --- a/peers/app_request_network.go +++ b/peers/app_request_network.go @@ -21,8 +21,8 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/set" "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/peers/validators" + relayercfg "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -112,7 +112,7 @@ func NewNetwork( } // TODO: remove dependence on Relayer specific config since this is meant to be a generic AppRequestNetwork file -func (n *AppRequestNetwork) InitializeConnectionsAndCheckStake(cfg *config.Config) error { +func (n *AppRequestNetwork) InitializeConnectionsAndCheckStake(cfg *relayercfg.Config) error { // Manually connect to the validators of each of the source subnets. // We return an error if we are unable to connect to sufficient stake on any of the subnets. // Sufficient stake is determined by the Warp quora of the configured supported destinations, @@ -278,8 +278,8 @@ func (n *AppRequestNetwork) GetSubnetID(blockchainID ids.ID) (ids.ID, error) { // Connect to the validators of the source blockchain. For each destination blockchain, // verify that we have connected to a threshold of stake. func (n *AppRequestNetwork) connectToNonPrimaryNetworkPeers( - cfg *config.Config, - sourceBlockchain *config.SourceBlockchain, + cfg *relayercfg.Config, + sourceBlockchain *relayercfg.SourceBlockchain, ) error { subnetID := sourceBlockchain.GetSubnetID() connectedValidators, err := n.ConnectToCanonicalValidators(subnetID) @@ -310,8 +310,8 @@ func (n *AppRequestNetwork) connectToNonPrimaryNetworkPeers( // Connect to the validators of the destination blockchains. Verify that we have connected // to a threshold of stake for each blockchain. func (n *AppRequestNetwork) connectToPrimaryNetworkPeers( - cfg *config.Config, - sourceBlockchain *config.SourceBlockchain, + cfg *relayercfg.Config, + sourceBlockchain *relayercfg.SourceBlockchain, ) error { for _, destination := range sourceBlockchain.SupportedDestinations { blockchainID := destination.GetBlockchainID() @@ -342,10 +342,10 @@ func (n *AppRequestNetwork) connectToPrimaryNetworkPeers( // Fetch the warp quorum from the config and check if the connected stake exceeds the threshold func (n *AppRequestNetwork) checkForSufficientConnectedStake( - cfg *config.Config, + cfg *relayercfg.Config, connectedValidators *ConnectedCanonicalValidators, destinationBlockchainID ids.ID, -) (bool, *config.WarpQuorum, error) { +) (bool, *relayercfg.WarpQuorum, error) { quorum, err := cfg.GetWarpQuorum(destinationBlockchainID) if err != nil { n.logger.Error( diff --git a/relayer/README.md b/relayer/README.md index c10e44a2..72c0eb07 100644 --- a/relayer/README.md +++ b/relayer/README.md @@ -316,9 +316,29 @@ The relayer consists of the following components: ### Data Flow -
- -
+
+ +
Figure 1: Relayer Data Flow
+
+ +### Processing Missed Blocks + +On startup, the relayer will process any blocks that it missed while offline if `process-missed-blocks` is set to `true` in the configuration. For each configured `source-blockchain`, the starting block height is set as the _minimum_ block height that is stored in the relayer's database across keys that pertain to that blockchain. These keys correspond to distinct sending addresses (as specified in `source-blockchain.allowed-origin-sender-addresses`), meaning that on startup, the relayer will begin processing from the _minimum_ block height across all configured sending addresses for each `source-blockchain`. Note that an empty `source-blockchain.allowed-origin-sender-addresses` list is treated as its own distinct key. If no keys are found, then the relayer begins processing from the current chain tip. + +Once the starting block height is calculated, all blocks between it and the current tip of the `source-blockchain` are processed according to the _current_ configuration rules. + +_Note:_ Given these semantics for computing the starting block height, it's possible for blocks that have previously been ignored under different configuration options to be relayed on a subsequent run. For example, consider the following scenario consisting of three subsequent runs (see Figure 2 below): + +- **Run 1**: Suppose that on Blockchain A we set `allowed-origin-sender-addresses=[0x1234]`, meaning that the relayer should _ignore_ messages sent by any other address. The relayer processes through block `100`, and that height is marked as processed for `0x1234`'s key. + +- **Run 2**: The relayer is then restarted with `allowed-origin-sender-addresses=[0xabcd]`, replacing `0x1234`. A message is sent from address `0x1234` at block height `200`. The relayer will decide to ignore this message, and will mark block `200` as processed for `0xabcd`'s key. + +- **Run 3**: The relayer is then restarted again with the original configuration of `allowed-origin-sender-addresses=[0x1234]`. The relayer will calculate the starting block height as `100`, and process blocks `100` through the current chain tip, reprocessing block `200` along the way. Instead of ignoring the message in this block, however, the relayer will relay it to the destination. + +
+ +
Figure 2: Processing Missed Blocks Example
+
### API @@ -409,4 +429,4 @@ The E2E tests use the `TeleporterMessenger` contract deployment transaction spec ```bash go generate ./... -``` +``` \ No newline at end of file diff --git a/relayer/application_relayer.go b/relayer/application_relayer.go index 2ae439c3..d1b8b2a5 100644 --- a/relayer/application_relayer.go +++ b/relayer/application_relayer.go @@ -12,10 +12,10 @@ import ( "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/database" "github.com/ava-labs/awm-relayer/messages" "github.com/ava-labs/awm-relayer/peers" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/signature-aggregator/aggregator" "github.com/ava-labs/awm-relayer/utils" "github.com/ava-labs/awm-relayer/vms" diff --git a/relayer/config/config_test.go b/relayer/config/config_test.go index 58991841..06f54248 100644 --- a/relayer/config/config_test.go +++ b/relayer/config/config_test.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/set" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/utils" mock_ethclient "github.com/ava-labs/awm-relayer/vms/evm/mocks" "github.com/ava-labs/subnet-evm/params" @@ -373,10 +374,10 @@ func TestGetWarpQuorum(t *testing.T) { func TestValidateSourceBlockchain(t *testing.T) { validSourceCfg := SourceBlockchain{ BlockchainID: testBlockchainID, - RPCEndpoint: APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: fmt.Sprintf("http://test.avax.network/ext/bc/%s/rpc", testBlockchainID), }, - WSEndpoint: APIConfig{ + WSEndpoint: basecfg.APIConfig{ BaseURL: fmt.Sprintf("ws://test.avax.network/ext/bc/%s/ws", testBlockchainID), }, SubnetID: testSubnetID, diff --git a/relayer/config/test_utils.go b/relayer/config/test_utils.go index da2908bb..db9955dd 100644 --- a/relayer/config/test_utils.go +++ b/relayer/config/test_utils.go @@ -2,7 +2,11 @@ package config -import "fmt" +import ( + "fmt" + + basecfg "github.com/ava-labs/awm-relayer/config" +) var ( testSubnetID string = "2TGBXcnwx5PqiXWiqxAKUaNSqDguXNh1mxnp82jui68hxJSZAx" @@ -21,7 +25,7 @@ var ( var ( TestValidConfig = Config{ LogLevel: "info", - PChainAPI: &APIConfig{ + PChainAPI: &basecfg.APIConfig{ BaseURL: "http://test.avax.network", QueryParams: map[string]string{ queryParamKey1: queryParamVal1, @@ -30,16 +34,16 @@ var ( httpHeaderKey1: httpHeaderVal1, }, }, - InfoAPI: &APIConfig{ + InfoAPI: &basecfg.APIConfig{ BaseURL: "http://test.avax.network", }, DBWriteIntervalSeconds: 1, SourceBlockchains: []*SourceBlockchain{ { - RPCEndpoint: APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: fmt.Sprintf("http://test.avax.network/ext/bc/%s/rpc", testBlockchainID), }, - WSEndpoint: APIConfig{ + WSEndpoint: basecfg.APIConfig{ BaseURL: fmt.Sprintf("ws://test.avax.network/ext/bc/%s/ws", testBlockchainID), }, BlockchainID: testBlockchainID, @@ -54,7 +58,7 @@ var ( }, DestinationBlockchains: []*DestinationBlockchain{ { - RPCEndpoint: APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: fmt.Sprintf("http://test.avax.network/ext/bc/%s/rpc", testBlockchainID), }, BlockchainID: testBlockchainID, @@ -65,10 +69,10 @@ var ( }, } TestValidSourceBlockchainConfig = SourceBlockchain{ - RPCEndpoint: APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "http://test.avax.network/ext/bc/C/rpc", }, - WSEndpoint: APIConfig{ + WSEndpoint: basecfg.APIConfig{ BaseURL: "ws://test.avax.network/ext/bc/C/ws", }, BlockchainID: "S4mMqUXe7vHsGiRAma6bv3CKnyaLssyAxmQ2KvFpX1KEvfFCD", @@ -84,7 +88,7 @@ var ( SubnetID: "2TGBXcnwx5PqiXWiqxAKUaNSqDguXNh1mxnp82jui68hxJSZAx", BlockchainID: "S4mMqUXe7vHsGiRAma6bv3CKnyaLssyAxmQ2KvFpX1KEvfFCD", VM: "evm", - RPCEndpoint: APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "http://test.avax.network/ext/bc/C/rpc", }, AccountPrivateKey: "1234567890123456789012345678901234567890123456789012345678901234", diff --git a/relayer/listener.go b/relayer/listener.go index bef9fa22..2dc81456 100644 --- a/relayer/listener.go +++ b/relayer/listener.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ava-labs/awm-relayer/vms" "github.com/ava-labs/subnet-evm/ethclient" diff --git a/relayer/main/main.go b/relayer/main/main.go index 8a9af9f2..157b76e8 100644 --- a/relayer/main/main.go +++ b/relayer/main/main.go @@ -18,7 +18,6 @@ import ( "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/set" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/database" "github.com/ava-labs/awm-relayer/messages" offchainregistry "github.com/ava-labs/awm-relayer/messages/off-chain-registry" @@ -27,6 +26,7 @@ import ( "github.com/ava-labs/awm-relayer/relayer" "github.com/ava-labs/awm-relayer/relayer/api" "github.com/ava-labs/awm-relayer/relayer/checkpoint" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/signature-aggregator/aggregator" sigAggMetrics "github.com/ava-labs/awm-relayer/signature-aggregator/metrics" "github.com/ava-labs/awm-relayer/utils" diff --git a/tests/allowed_addresses.go b/tests/allowed_addresses.go index dbaa809a..e31086f7 100644 --- a/tests/allowed_addresses.go +++ b/tests/allowed_addresses.go @@ -8,8 +8,8 @@ import ( "time" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/database" + "github.com/ava-labs/awm-relayer/relayer/config" testUtils "github.com/ava-labs/awm-relayer/tests/utils" "github.com/ava-labs/subnet-evm/accounts/abi/bind" "github.com/ava-labs/teleporter/tests/interfaces" diff --git a/tests/utils/utils.go b/tests/utils/utils.go index 0875b99f..3da233fa 100644 --- a/tests/utils/utils.go +++ b/tests/utils/utils.go @@ -21,6 +21,7 @@ import ( warpPayload "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" "github.com/ava-labs/awm-relayer/config" offchainregistry "github.com/ava-labs/awm-relayer/messages/off-chain-registry" + relayercfg "github.com/ava-labs/awm-relayer/relayer/config" signatureaggregatorcfg "github.com/ava-labs/awm-relayer/signature-aggregator/config" batchcrosschainmessenger "github.com/ava-labs/awm-relayer/tests/abi-bindings/go/BatchCrossChainMessenger" relayerUtils "github.com/ava-labs/awm-relayer/utils" @@ -178,7 +179,7 @@ func CreateDefaultRelayerConfig( teleporterContractAddress common.Address, fundedAddress common.Address, relayerKey *ecdsa.PrivateKey, -) config.Config { +) relayercfg.Config { logLevel, err := logging.ToLevel(os.Getenv("LOG_LEVEL")) if err != nil { logLevel = logging.Info @@ -189,16 +190,16 @@ func CreateDefaultRelayerConfig( "logLevel", logLevel.LowerString(), ) // Construct the config values for each subnet - sources := make([]*config.SourceBlockchain, len(sourceSubnetsInfo)) - destinations := make([]*config.DestinationBlockchain, len(destinationSubnetsInfo)) + sources := make([]*relayercfg.SourceBlockchain, len(sourceSubnetsInfo)) + destinations := make([]*relayercfg.DestinationBlockchain, len(destinationSubnetsInfo)) for i, subnetInfo := range sourceSubnetsInfo { host, port, err := teleporterTestUtils.GetURIHostAndPort(subnetInfo.NodeURIs[0]) Expect(err).Should(BeNil()) - sources[i] = &config.SourceBlockchain{ + sources[i] = &relayercfg.SourceBlockchain{ SubnetID: subnetInfo.SubnetID.String(), BlockchainID: subnetInfo.BlockchainID.String(), - VM: config.EVM.String(), + VM: relayercfg.EVM.String(), RPCEndpoint: config.APIConfig{ BaseURL: fmt.Sprintf("http://%s:%d/ext/bc/%s/rpc", host, port, subnetInfo.BlockchainID.String()), }, @@ -206,15 +207,15 @@ func CreateDefaultRelayerConfig( BaseURL: fmt.Sprintf("ws://%s:%d/ext/bc/%s/ws", host, port, subnetInfo.BlockchainID.String()), }, - MessageContracts: map[string]config.MessageProtocolConfig{ + MessageContracts: map[string]relayercfg.MessageProtocolConfig{ teleporterContractAddress.Hex(): { - MessageFormat: config.TELEPORTER.String(), + MessageFormat: relayercfg.TELEPORTER.String(), Settings: map[string]interface{}{ "reward-address": fundedAddress.Hex(), }, }, offchainregistry.OffChainRegistrySourceAddress.Hex(): { - MessageFormat: config.OFF_CHAIN_REGISTRY.String(), + MessageFormat: relayercfg.OFF_CHAIN_REGISTRY.String(), Settings: map[string]interface{}{ "teleporter-registry-address": subnetInfo.TeleporterRegistryAddress.Hex(), }, @@ -235,10 +236,10 @@ func CreateDefaultRelayerConfig( host, port, err := teleporterTestUtils.GetURIHostAndPort(subnetInfo.NodeURIs[0]) Expect(err).Should(BeNil()) - destinations[i] = &config.DestinationBlockchain{ + destinations[i] = &relayercfg.DestinationBlockchain{ SubnetID: subnetInfo.SubnetID.String(), BlockchainID: subnetInfo.BlockchainID.String(), - VM: config.EVM.String(), + VM: relayercfg.EVM.String(), RPCEndpoint: config.APIConfig{ BaseURL: fmt.Sprintf("http://%s:%d/ext/bc/%s/rpc", host, port, subnetInfo.BlockchainID.String()), }, @@ -254,7 +255,7 @@ func CreateDefaultRelayerConfig( ) } - return config.Config{ + return relayercfg.Config{ LogLevel: logging.Info.LowerString(), PChainAPI: &config.APIConfig{ BaseURL: sourceSubnetsInfo[0].NodeURIs[0], @@ -494,7 +495,7 @@ func RelayBasicMessage( Expect(receivedTeleporterMessage.Message).Should(Equal(teleporterMessage.Message)) } -func WriteRelayerConfig(relayerConfig config.Config, fname string) string { +func WriteRelayerConfig(relayerConfig relayercfg.Config, fname string) string { data, err := json.MarshalIndent(relayerConfig, "", "\t") Expect(err).Should(BeNil()) @@ -530,7 +531,7 @@ func TriggerProcessMissedBlocks( sourceSubnetInfo interfaces.SubnetTestInfo, destinationSubnetInfo interfaces.SubnetTestInfo, currRelayerCleanup context.CancelFunc, - currrentRelayerConfig config.Config, + currrentRelayerConfig relayercfg.Config, fundedAddress common.Address, fundedKey *ecdsa.PrivateKey, ) { diff --git a/vms/contract_message.go b/vms/contract_message.go index f7331171..a78d6a3a 100644 --- a/vms/contract_message.go +++ b/vms/contract_message.go @@ -6,7 +6,7 @@ package vms import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/vms/evm" ) diff --git a/vms/destination_client.go b/vms/destination_client.go index e1fbda02..fd1cf021 100644 --- a/vms/destination_client.go +++ b/vms/destination_client.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/vms/evm" "github.com/ethereum/go-ethereum/common" "go.uber.org/zap" diff --git a/vms/evm/contract_message.go b/vms/evm/contract_message.go index 5221d8a3..1da24b26 100644 --- a/vms/evm/contract_message.go +++ b/vms/evm/contract_message.go @@ -8,7 +8,7 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/subnet-evm/precompile/contracts/warp" "go.uber.org/zap" ) diff --git a/vms/evm/contract_message_test.go b/vms/evm/contract_message_test.go index 241220ac..9820f83f 100644 --- a/vms/evm/contract_message_test.go +++ b/vms/evm/contract_message_test.go @@ -13,7 +13,7 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/vms/platformvm/warp" warpPayload "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" ) diff --git a/vms/evm/destination_client.go b/vms/evm/destination_client.go index aea6ab90..e7f7c124 100644 --- a/vms/evm/destination_client.go +++ b/vms/evm/destination_client.go @@ -13,7 +13,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ava-labs/awm-relayer/vms/evm/signer" "github.com/ava-labs/subnet-evm/core/types" diff --git a/vms/evm/destination_client_test.go b/vms/evm/destination_client_test.go index b23576b9..b1c067dc 100644 --- a/vms/evm/destination_client_test.go +++ b/vms/evm/destination_client_test.go @@ -11,7 +11,8 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" - "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" mock_ethclient "github.com/ava-labs/awm-relayer/vms/evm/mocks" "github.com/ava-labs/awm-relayer/vms/evm/signer" "github.com/stretchr/testify/require" @@ -22,7 +23,7 @@ var destinationSubnet = config.DestinationBlockchain{ SubnetID: "2TGBXcnwx5PqiXWiqxAKUaNSqDguXNh1mxnp82jui68hxJSZAx", BlockchainID: "S4mMqUXe7vHsGiRAma6bv3CKnyaLssyAxmQ2KvFpX1KEvfFCD", VM: config.EVM.String(), - RPCEndpoint: config.APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "https://subnets.avax.network/mysubnet/rpc", }, AccountPrivateKey: "56289e99c94b6912bfc12adc093c9b51124f0dc54ac7a766b2bc5ccf558d8027", diff --git a/vms/evm/signer/signer.go b/vms/evm/signer/signer.go index 3c27ec19..cd1a5a14 100644 --- a/vms/evm/signer/signer.go +++ b/vms/evm/signer/signer.go @@ -6,7 +6,7 @@ package signer import ( "math/big" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/subnet-evm/core/types" "github.com/ethereum/go-ethereum/common" ) diff --git a/vms/evm/signer/tx_signer_test.go b/vms/evm/signer/tx_signer_test.go index 48e3fb4d..cba0c03d 100644 --- a/vms/evm/signer/tx_signer_test.go +++ b/vms/evm/signer/tx_signer_test.go @@ -8,7 +8,7 @@ import ( "math/big" "testing" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" diff --git a/vms/evm/subscriber_test.go b/vms/evm/subscriber_test.go index 8d3d0c7b..f9661257 100644 --- a/vms/evm/subscriber_test.go +++ b/vms/evm/subscriber_test.go @@ -9,7 +9,8 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" mock_ethclient "github.com/ava-labs/awm-relayer/vms/evm/mocks" "github.com/ava-labs/subnet-evm/core/types" "github.com/stretchr/testify/require" @@ -21,7 +22,7 @@ func makeSubscriberWithMockEthClient(t *testing.T) (*subscriber, *mock_ethclient SubnetID: "2TGBXcnwx5PqiXWiqxAKUaNSqDguXNh1mxnp82jui68hxJSZAx", BlockchainID: "S4mMqUXe7vHsGiRAma6bv3CKnyaLssyAxmQ2KvFpX1KEvfFCD", VM: config.EVM.String(), - RPCEndpoint: config.APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "https://subnets.avax.network/mysubnet/rpc", }, } diff --git a/vms/subscriber.go b/vms/subscriber.go index 83cc8f5e..860caf08 100644 --- a/vms/subscriber.go +++ b/vms/subscriber.go @@ -8,7 +8,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/logging" - "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/relayer/config" "github.com/ava-labs/awm-relayer/vms/evm" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/ethclient" From 82a9bf260d33ba4cc18000cfd2e619ade8ff2c0d Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 10:06:55 -0400 Subject: [PATCH 04/14] move resources back to top level and update links --- relayer/README.md | 4 ++-- .../resources => resources}/catch-up-example.png | Bin .../resources => resources}/relayer-diagram.png | Bin 3 files changed, 2 insertions(+), 2 deletions(-) rename {relayer/resources => resources}/catch-up-example.png (100%) rename {relayer/resources => resources}/relayer-diagram.png (100%) diff --git a/relayer/README.md b/relayer/README.md index 72c0eb07..107ec425 100644 --- a/relayer/README.md +++ b/relayer/README.md @@ -317,7 +317,7 @@ The relayer consists of the following components: ### Data Flow
- +
Figure 1: Relayer Data Flow
@@ -336,7 +336,7 @@ _Note:_ Given these semantics for computing the starting block height, it's poss - **Run 3**: The relayer is then restarted again with the original configuration of `allowed-origin-sender-addresses=[0x1234]`. The relayer will calculate the starting block height as `100`, and process blocks `100` through the current chain tip, reprocessing block `200` along the way. Instead of ignoring the message in this block, however, the relayer will relay it to the destination.
- +
Figure 2: Processing Missed Blocks Example
diff --git a/relayer/resources/catch-up-example.png b/resources/catch-up-example.png similarity index 100% rename from relayer/resources/catch-up-example.png rename to resources/catch-up-example.png diff --git a/relayer/resources/relayer-diagram.png b/resources/relayer-diagram.png similarity index 100% rename from relayer/resources/relayer-diagram.png rename to resources/relayer-diagram.png From 1ae2d0251959c9a867e36457fe2b367ee13905a6 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 10:12:36 -0400 Subject: [PATCH 05/14] update path to sample-relayer-config.json --- relayer/config/viper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relayer/config/viper_test.go b/relayer/config/viper_test.go index fec8879e..708a51e7 100644 --- a/relayer/config/viper_test.go +++ b/relayer/config/viper_test.go @@ -13,7 +13,7 @@ import ( func TestBuildConfig(t *testing.T) { v := viper.New() - cfgBytes, err := os.ReadFile("../sample-relayer-config.json") + cfgBytes, err := os.ReadFile("../../sample-relayer-config.json") require.NoError(t, err) configFile := string(cfgBytes) buf := bytes.NewBufferString(configFile) From 0fb6427b9808fa63af27f14755670d6c133528a9 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 10:36:07 -0400 Subject: [PATCH 06/14] move relayer specific network initialization code to its own main --- peers/app_request_network.go | 121 ------------------------- relayer/config/config.go | 5 +- relayer/main/main.go | 125 +++++++++++++++++++++++++- signature-aggregator/config/config.go | 5 +- 4 files changed, 130 insertions(+), 126 deletions(-) diff --git a/peers/app_request_network.go b/peers/app_request_network.go index 7ac1f05c..fa2fb8e0 100644 --- a/peers/app_request_network.go +++ b/peers/app_request_network.go @@ -5,8 +5,6 @@ package peers import ( "context" - "fmt" - "math/big" "os" "sync" "time" @@ -17,13 +15,10 @@ import ( avagoCommon "github.com/ava-labs/avalanchego/snow/engine/common" snowVdrs "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/subnets" - "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/set" "github.com/ava-labs/avalanchego/vms/platformvm/warp" "github.com/ava-labs/awm-relayer/peers/validators" - relayercfg "github.com/ava-labs/awm-relayer/relayer/config" - "github.com/ava-labs/awm-relayer/utils" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) @@ -111,32 +106,6 @@ func NewNetwork( return arNetwork, nil } -// TODO: remove dependence on Relayer specific config since this is meant to be a generic AppRequestNetwork file -func (n *AppRequestNetwork) InitializeConnectionsAndCheckStake(cfg *relayercfg.Config) error { - // Manually connect to the validators of each of the source subnets. - // We return an error if we are unable to connect to sufficient stake on any of the subnets. - // Sufficient stake is determined by the Warp quora of the configured supported destinations, - // or if the subnet supports all destinations, by the quora of all configured destinations. - for _, sourceBlockchain := range cfg.SourceBlockchains { - if sourceBlockchain.GetSubnetID() == constants.PrimaryNetworkID { - if err := n.connectToPrimaryNetworkPeers(cfg, sourceBlockchain); err != nil { - return fmt.Errorf( - "failed to connect to primary network peers: %w", - err, - ) - } - } else { - if err := n.connectToNonPrimaryNetworkPeers(cfg, sourceBlockchain); err != nil { - return fmt.Errorf( - "failed to connect to non-primary network peers: %w", - err, - ) - } - } - } - return nil -} - // ConnectPeers connects the network to peers with the given nodeIDs. // Returns the set of nodeIDs that were successfully connected to. func (n *AppRequestNetwork) ConnectPeers(nodeIDs set.Set[ids.NodeID]) set.Set[ids.NodeID] { @@ -272,93 +241,3 @@ func (n *AppRequestNetwork) RegisterRequestID(requestID uint32, numExpectedRespo func (n *AppRequestNetwork) GetSubnetID(blockchainID ids.ID) (ids.ID, error) { return n.validatorClient.GetSubnetID(context.Background(), blockchainID) } - -// Private helpers - -// Connect to the validators of the source blockchain. For each destination blockchain, -// verify that we have connected to a threshold of stake. -func (n *AppRequestNetwork) connectToNonPrimaryNetworkPeers( - cfg *relayercfg.Config, - sourceBlockchain *relayercfg.SourceBlockchain, -) error { - subnetID := sourceBlockchain.GetSubnetID() - connectedValidators, err := n.ConnectToCanonicalValidators(subnetID) - if err != nil { - n.logger.Error( - "Failed to connect to canonical validators", - zap.String("subnetID", subnetID.String()), - zap.Error(err), - ) - return err - } - for _, destination := range sourceBlockchain.SupportedDestinations { - blockchainID := destination.GetBlockchainID() - if ok, quorum, err := n.checkForSufficientConnectedStake(cfg, connectedValidators, blockchainID); !ok { - n.logger.Error( - "Failed to connect to a threshold of stake", - zap.String("destinationBlockchainID", blockchainID.String()), - zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), - zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), - zap.Any("warpQuorum", quorum), - ) - return err - } - } - return nil -} - -// Connect to the validators of the destination blockchains. Verify that we have connected -// to a threshold of stake for each blockchain. -func (n *AppRequestNetwork) connectToPrimaryNetworkPeers( - cfg *relayercfg.Config, - sourceBlockchain *relayercfg.SourceBlockchain, -) error { - for _, destination := range sourceBlockchain.SupportedDestinations { - blockchainID := destination.GetBlockchainID() - subnetID := cfg.GetSubnetID(blockchainID) - connectedValidators, err := n.ConnectToCanonicalValidators(subnetID) - if err != nil { - n.logger.Error( - "Failed to connect to canonical validators", - zap.String("subnetID", subnetID.String()), - zap.Error(err), - ) - return err - } - - if ok, quorum, err := n.checkForSufficientConnectedStake(cfg, connectedValidators, blockchainID); !ok { - n.logger.Error( - "Failed to connect to a threshold of stake", - zap.String("destinationBlockchainID", blockchainID.String()), - zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), - zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), - zap.Any("warpQuorum", quorum), - ) - return err - } - } - return nil -} - -// Fetch the warp quorum from the config and check if the connected stake exceeds the threshold -func (n *AppRequestNetwork) checkForSufficientConnectedStake( - cfg *relayercfg.Config, - connectedValidators *ConnectedCanonicalValidators, - destinationBlockchainID ids.ID, -) (bool, *relayercfg.WarpQuorum, error) { - quorum, err := cfg.GetWarpQuorum(destinationBlockchainID) - if err != nil { - n.logger.Error( - "Failed to get warp quorum from config", - zap.String("destinationBlockchainID", destinationBlockchainID.String()), - zap.Error(err), - ) - return false, nil, err - } - return utils.CheckStakeWeightExceedsThreshold( - big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight), - connectedValidators.TotalValidatorWeight, - quorum.QuorumNumerator, - quorum.QuorumDenominator, - ), &quorum, nil -} diff --git a/relayer/config/config.go b/relayer/config/config.go index 72632ac2..f1cfaa96 100644 --- a/relayer/config/config.go +++ b/relayer/config/config.go @@ -10,6 +10,7 @@ import ( "net/url" baseCfg "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/peers" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/constants" @@ -249,12 +250,12 @@ func (c *Config) GetWarpQuorum(blockchainID ids.ID) (WarpQuorum, error) { return WarpQuorum{}, errFailedToGetWarpQuorum } -// Config implements the peers.Config interface +var _ peers.Config = &Config{} + func (c *Config) GetPChainAPI() *baseCfg.APIConfig { return c.PChainAPI } -// Config implements the peers.Config interface func (c *Config) GetInfoAPI() *baseCfg.APIConfig { return c.InfoAPI } diff --git a/relayer/main/main.go b/relayer/main/main.go index 157b76e8..12ff8b11 100644 --- a/relayer/main/main.go +++ b/relayer/main/main.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "log" + "math/big" "net/http" "os" "runtime" @@ -152,7 +153,7 @@ func main() { trackedSubnets, &cfg, ) - network.InitializeConnectionsAndCheckStake(&cfg) + initializeConnectionsAndCheckStake(logger, network, &cfg) if err != nil { logger.Fatal("Failed to create app request network", zap.Error(err)) @@ -534,3 +535,125 @@ func initializeMetrics() (prometheus.Gatherer, prometheus.Registerer, error) { } return gatherer, registry, nil } + +func initializeConnectionsAndCheckStake( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, +) error { + // Manually connect to the validators of each of the source subnets. + // We return an error if we are unable to connect to sufficient stake on any of the subnets. + // Sufficient stake is determined by the Warp quora of the configured supported destinations, + // or if the subnet supports all destinations, by the quora of all configured destinations. + for _, sourceBlockchain := range cfg.SourceBlockchains { + if sourceBlockchain.GetSubnetID() == constants.PrimaryNetworkID { + if err := connectToPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { + return fmt.Errorf( + "failed to connect to primary network peers: %w", + err, + ) + } + } else { + if err := connectToNonPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { + return fmt.Errorf( + "failed to connect to non-primary network peers: %w", + err, + ) + } + } + } + return nil +} + +// Connect to the validators of the source blockchain. For each destination blockchain, +// verify that we have connected to a threshold of stake. +func connectToNonPrimaryNetworkPeers( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, + sourceBlockchain *config.SourceBlockchain, +) error { + subnetID := sourceBlockchain.GetSubnetID() + connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) + if err != nil { + logger.Error( + "Failed to connect to canonical validators", + zap.String("subnetID", subnetID.String()), + zap.Error(err), + ) + return err + } + for _, destination := range sourceBlockchain.SupportedDestinations { + blockchainID := destination.GetBlockchainID() + if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { + logger.Error( + "Failed to connect to a threshold of stake", + zap.String("destinationBlockchainID", blockchainID.String()), + zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), + zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), + zap.Any("warpQuorum", quorum), + ) + return err + } + } + return nil +} + +// Connect to the validators of the destination blockchains. Verify that we have connected +// to a threshold of stake for each blockchain. +func connectToPrimaryNetworkPeers( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, + sourceBlockchain *config.SourceBlockchain, +) error { + for _, destination := range sourceBlockchain.SupportedDestinations { + blockchainID := destination.GetBlockchainID() + subnetID := cfg.GetSubnetID(blockchainID) + connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) + if err != nil { + logger.Error( + "Failed to connect to canonical validators", + zap.String("subnetID", subnetID.String()), + zap.Error(err), + ) + return err + } + + if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { + logger.Error( + "Failed to connect to a threshold of stake", + zap.String("destinationBlockchainID", blockchainID.String()), + zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), + zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), + zap.Any("warpQuorum", quorum), + ) + return err + } + } + return nil +} + +// Fetch the warp quorum from the config and check if the connected stake exceeds the threshold +func checkForSufficientConnectedStake( + logger logging.Logger, + cfg *config.Config, + connectedValidators *peers.ConnectedCanonicalValidators, + destinationBlockchainID ids.ID, +) (bool, *config.WarpQuorum, error) { + quorum, err := cfg.GetWarpQuorum(destinationBlockchainID) + if err != nil { + logger.Error( + "Failed to get warp quorum from config", + zap.String("destinationBlockchainID", destinationBlockchainID.String()), + zap.Error(err), + ) + return false, nil, err + } + return utils.CheckStakeWeightExceedsThreshold( + big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight), + connectedValidators.TotalValidatorWeight, + quorum.QuorumNumerator, + quorum.QuorumDenominator, + ), &quorum, nil +} diff --git a/signature-aggregator/config/config.go b/signature-aggregator/config/config.go index 868994e7..a451d36b 100644 --- a/signature-aggregator/config/config.go +++ b/signature-aggregator/config/config.go @@ -8,6 +8,7 @@ import ( "github.com/ava-labs/avalanchego/utils/logging" baseCfg "github.com/ava-labs/awm-relayer/config" + "github.com/ava-labs/awm-relayer/peers" ) const ( @@ -51,12 +52,12 @@ func (c *Config) Validate() error { return nil } -// Config implements the peers.Config interface +var _ peers.Config = &Config{} + func (c *Config) GetPChainAPI() *baseCfg.APIConfig { return c.PChainAPI } -// Config implements the peers.Config interface func (c *Config) GetInfoAPI() *baseCfg.APIConfig { return c.InfoAPI } From 70daa3dda9ef2ed9c908ed01e0685f7309d59ed1 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 12:33:22 -0400 Subject: [PATCH 07/14] standardize on basecfg as import alias --- relayer/config/config.go | 10 +++++----- relayer/config/destination_blockchain.go | 4 ++-- relayer/config/source_blockchain.go | 8 ++++---- relayer/config/viper_test.go | 12 ++++++------ signature-aggregator/config/config.go | 10 +++++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/relayer/config/config.go b/relayer/config/config.go index f1cfaa96..6264279c 100644 --- a/relayer/config/config.go +++ b/relayer/config/config.go @@ -9,7 +9,7 @@ import ( "fmt" "net/url" - baseCfg "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/peers" "github.com/ava-labs/avalanchego/ids" @@ -58,8 +58,8 @@ type Config struct { APIPort uint16 `mapstructure:"api-port" json:"api-port"` MetricsPort uint16 `mapstructure:"metrics-port" json:"metrics-port"` DBWriteIntervalSeconds uint64 `mapstructure:"db-write-interval-seconds" json:"db-write-interval-seconds"` //nolint:lll - PChainAPI *baseCfg.APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` - InfoAPI *baseCfg.APIConfig `mapstructure:"info-api" json:"info-api"` + PChainAPI *basecfg.APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` + InfoAPI *basecfg.APIConfig `mapstructure:"info-api" json:"info-api"` SourceBlockchains []*SourceBlockchain `mapstructure:"source-blockchains" json:"source-blockchains"` DestinationBlockchains []*DestinationBlockchain `mapstructure:"destination-blockchains" json:"destination-blockchains"` ProcessMissedBlocks bool `mapstructure:"process-missed-blocks" json:"process-missed-blocks"` @@ -252,10 +252,10 @@ func (c *Config) GetWarpQuorum(blockchainID ids.ID) (WarpQuorum, error) { var _ peers.Config = &Config{} -func (c *Config) GetPChainAPI() *baseCfg.APIConfig { +func (c *Config) GetPChainAPI() *basecfg.APIConfig { return c.PChainAPI } -func (c *Config) GetInfoAPI() *baseCfg.APIConfig { +func (c *Config) GetInfoAPI() *basecfg.APIConfig { return c.InfoAPI } diff --git a/relayer/config/destination_blockchain.go b/relayer/config/destination_blockchain.go index 095cd3e7..2e0c9eb1 100644 --- a/relayer/config/destination_blockchain.go +++ b/relayer/config/destination_blockchain.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/ava-labs/avalanchego/ids" - baseCfg "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/crypto" ) @@ -17,7 +17,7 @@ type DestinationBlockchain struct { SubnetID string `mapstructure:"subnet-id" json:"subnet-id"` BlockchainID string `mapstructure:"blockchain-id" json:"blockchain-id"` VM string `mapstructure:"vm" json:"vm"` - RPCEndpoint baseCfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` + RPCEndpoint basecfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` KMSKeyID string `mapstructure:"kms-key-id" json:"kms-key-id"` KMSAWSRegion string `mapstructure:"kms-aws-region" json:"kms-aws-region"` AccountPrivateKey string `mapstructure:"account-private-key" json:"account-private-key"` diff --git a/relayer/config/source_blockchain.go b/relayer/config/source_blockchain.go index 7d418cb5..22d42319 100644 --- a/relayer/config/source_blockchain.go +++ b/relayer/config/source_blockchain.go @@ -5,7 +5,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/set" - baseCfg "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/utils" "github.com/ethereum/go-ethereum/common" ) @@ -19,13 +19,13 @@ type SourceBlockchain struct { SubnetID string `mapstructure:"subnet-id" json:"subnet-id"` BlockchainID string `mapstructure:"blockchain-id" json:"blockchain-id"` //nolint:lll VM string `mapstructure:"vm" json:"vm"` - RPCEndpoint baseCfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` //nolint:lll - WSEndpoint baseCfg.APIConfig `mapstructure:"ws-endpoint" json:"ws-endpoint"` //nolint:lll + RPCEndpoint basecfg.APIConfig `mapstructure:"rpc-endpoint" json:"rpc-endpoint"` //nolint:lll + WSEndpoint basecfg.APIConfig `mapstructure:"ws-endpoint" json:"ws-endpoint"` //nolint:lll MessageContracts map[string]MessageProtocolConfig `mapstructure:"message-contracts" json:"message-contracts"` //nolint:lll SupportedDestinations []*SupportedDestination `mapstructure:"supported-destinations" json:"supported-destinations"` //nolint:lll ProcessHistoricalBlocksFromHeight uint64 `mapstructure:"process-historical-blocks-from-height" json:"process-historical-blocks-from-height"` //nolint:lll AllowedOriginSenderAddresses []string `mapstructure:"allowed-origin-sender-addresses" json:"allowed-origin-sender-addresses"` //nolint:lll - WarpAPIEndpoint baseCfg.APIConfig `mapstructure:"warp-api-endpoint" json:"warp-api-endpoint"` //nolint:lll + WarpAPIEndpoint basecfg.APIConfig `mapstructure:"warp-api-endpoint" json:"warp-api-endpoint"` //nolint:lll // convenience fields to access parsed data after initialization subnetID ids.ID diff --git a/relayer/config/viper_test.go b/relayer/config/viper_test.go index 708a51e7..ae45067d 100644 --- a/relayer/config/viper_test.go +++ b/relayer/config/viper_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - baseCfg "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -26,10 +26,10 @@ func TestBuildConfig(t *testing.T) { require.Equal(t, defaultAPIPort, cfg.APIPort) require.Equal(t, defaultMetricsPort, cfg.MetricsPort) require.Equal(t, defaultIntervalSeconds, cfg.DBWriteIntervalSeconds) - require.Equal(t, &baseCfg.APIConfig{ + require.Equal(t, &basecfg.APIConfig{ BaseURL: "https://api.avax-test.network", }, cfg.PChainAPI) - require.Equal(t, &baseCfg.APIConfig{ + require.Equal(t, &basecfg.APIConfig{ BaseURL: "https://api.avax-test.network", }, cfg.InfoAPI) @@ -38,10 +38,10 @@ func TestBuildConfig(t *testing.T) { SubnetID: "11111111111111111111111111111111LpoYY", BlockchainID: "yH8D7ThNJkxmtkuv2jgBa4P1Rn3Qpr4pPr7QYNfcdoS6k6HWp", VM: "evm", - RPCEndpoint: baseCfg.APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "https://api.avax-test.network/ext/bc/C/rpc", }, - WSEndpoint: baseCfg.APIConfig{ + WSEndpoint: basecfg.APIConfig{ BaseURL: "wss://api.avax-test.network/ext/bc/C/ws", }, MessageContracts: map[string]MessageProtocolConfig{ @@ -58,7 +58,7 @@ func TestBuildConfig(t *testing.T) { require.Equal(t, &DestinationBlockchain{ SubnetID: "7WtoAMPhrmh5KosDUsFL9yTcvw7YSxiKHPpdfs4JsgW47oZT5", BlockchainID: "2D8RG4UpSXbPbvPCAWppNJyqTG2i2CAXSkTgmTBBvs7GKNZjsY", - RPCEndpoint: baseCfg.APIConfig{ + RPCEndpoint: basecfg.APIConfig{ BaseURL: "https://subnets.avax.network/dispatch/testnet/rpc", }, VM: "evm", diff --git a/signature-aggregator/config/config.go b/signature-aggregator/config/config.go index a451d36b..0cb125ad 100644 --- a/signature-aggregator/config/config.go +++ b/signature-aggregator/config/config.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/ava-labs/avalanchego/utils/logging" - baseCfg "github.com/ava-labs/awm-relayer/config" + basecfg "github.com/ava-labs/awm-relayer/config" "github.com/ava-labs/awm-relayer/peers" ) @@ -27,8 +27,8 @@ signature-aggregator --help Display signature-a type Config struct { LogLevel string `mapstructure:"log-level" json:"log-level"` - PChainAPI *baseCfg.APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` - InfoAPI *baseCfg.APIConfig `mapstructure:"info-api" json:"info-api"` + PChainAPI *basecfg.APIConfig `mapstructure:"p-chain-api" json:"p-chain-api"` + InfoAPI *basecfg.APIConfig `mapstructure:"info-api" json:"info-api"` APIPort uint16 `mapstructure:"api-port" json:"api-port"` MetricsPort uint16 `mapstructure:"metrics-port" json:"metrics-port"` @@ -54,10 +54,10 @@ func (c *Config) Validate() error { var _ peers.Config = &Config{} -func (c *Config) GetPChainAPI() *baseCfg.APIConfig { +func (c *Config) GetPChainAPI() *basecfg.APIConfig { return c.PChainAPI } -func (c *Config) GetInfoAPI() *baseCfg.APIConfig { +func (c *Config) GetInfoAPI() *basecfg.APIConfig { return c.InfoAPI } From 07e43af8f585bacf6016d6b34a2a52a0d4945533 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Tue, 13 Aug 2024 13:09:34 -0400 Subject: [PATCH 08/14] Update scripts --- scripts/build_relayer.sh | 11 ++++------- scripts/build_signature_aggregator.sh | 11 +++++------ scripts/versions.sh | 4 ++-- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/scripts/build_relayer.sh b/scripts/build_relayer.sh index 842c7c06..b0e5521a 100755 --- a/scripts/build_relayer.sh +++ b/scripts/build_relayer.sh @@ -23,16 +23,13 @@ version_lt() { } -# Relayer directory -RELAYER_PATH=$( +BASE_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" - cd ../relayer && pwd + cd .. && pwd ) -# Base directory -BASE_PATH=$( - cd $RELAYER_PATH - cd .. && pwd +RELAYER_PATH=$( + cd $BASE_PATH/relayer && pwd ) # Load the versions and constants diff --git a/scripts/build_signature_aggregator.sh b/scripts/build_signature_aggregator.sh index 09e58ae4..8e698e8f 100755 --- a/scripts/build_signature_aggregator.sh +++ b/scripts/build_signature_aggregator.sh @@ -22,17 +22,16 @@ version_lt() { fi } -# Signature Aggregator root directory -SIGNATURE_AGGREGATOR_PATH=$( - cd "$(dirname "${BASH_SOURCE[0]}")" - cd ../signature-aggregator && pwd -) BASE_PATH=$( - cd $SIGNATURE_AGGREGATOR_PATH + cd "$(dirname "${BASH_SOURCE[0]}")" cd .. && pwd ) +SIGNATURE_AGGREGATOR_PATH=$( + cd $BASE_PATH/signature-aggregator && pwd +) + source $BASE_PATH/scripts/constants.sh source $BASE_PATH/scripts/versions.sh diff --git a/scripts/versions.sh b/scripts/versions.sh index 4c00db33..91986e36 100755 --- a/scripts/versions.sh +++ b/scripts/versions.sh @@ -2,7 +2,7 @@ # Copyright (C) 2023, Ava Labs, Inc. All rights reserved. # See the file LICENSE for licensing terms. -RELAYER_PATH=$( +BASE_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" cd .. && pwd ) @@ -10,7 +10,7 @@ RELAYER_PATH=$( # Pass in the full name of the dependency. # Parses go.mod for a matching entry and extracts the version number. function getDepVersion() { - grep -m1 "^\s*$1" $RELAYER_PATH/go.mod | cut -d ' ' -f2 + grep -m1 "^\s*$1" $BASE_PATH/go.mod | cut -d ' ' -f2 } # This needs to be exported to be picked up by the dockerfile. From 4742355ae973a90917160912490e413670b0791e Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Thu, 15 Aug 2024 11:56:17 -0400 Subject: [PATCH 09/14] Update README.md Co-authored-by: Michael Kaplan <55204436+michaelkaplan13@users.noreply.github.com> Signed-off-by: Ian Suvak --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bcec4ca..150d563a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Avalanche ICM Off-chain Services -This repository contains services related to Avalanche ICM (Interchain Messaging) systems along with shared code to support them. +This repository contains off-chain services that help support Avalanche Interchain Messaging (ICM). Currently implemented applications are From 5210e3b16aaf4e11d6adbe6a0cff8221ada6f642 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Thu, 15 Aug 2024 12:14:58 -0400 Subject: [PATCH 10/14] Add descriptions for top level services --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 150d563a..8ca2e4af 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,6 @@ This repository contains off-chain services that help support Avalanche Intercha Currently implemented applications are 1. [AWM Relayer](relayer/README.md) + - Full-service cross-chain message delivery application that is configurable to listen to specific source and destination chain pairs and relay messages according to its configured rules. 2. [Signature Aggregator](signature-aggregator/README.md) + - Lightweight API that requests and aggregates signatures from validators for any ICM message, and returns a valid signed message that the user can then self-deliver to the intended destination chain. From a4302df02c177ff80cac8f9c3ff92ee2b0148b5e Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Thu, 15 Aug 2024 14:31:16 -0400 Subject: [PATCH 11/14] move network util functions from main.go to their own file in relayer --- relayer/main/main.go | 131 ++---------------------------------- relayer/network_utils.go | 139 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 125 deletions(-) create mode 100644 relayer/network_utils.go diff --git a/relayer/main/main.go b/relayer/main/main.go index 0c45b737..3615536a 100644 --- a/relayer/main/main.go +++ b/relayer/main/main.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "log" - "math/big" "net/http" "os" "runtime" @@ -153,13 +152,17 @@ func main() { trackedSubnets, &cfg, ) - initializeConnectionsAndCheckStake(logger, network, &cfg) - if err != nil { logger.Fatal("Failed to create app request network", zap.Error(err)) panic(err) } + err = relayer.InitializeConnectionsAndCheckStake(logger, network, &cfg) + if err != nil { + logger.Fatal("Failed to initialize connections and check stake", zap.Error(err)) + panic(err) + } + startMetricsServer(logger, gatherer, cfg.MetricsPort) relayerMetrics, err := relayer.NewApplicationRelayerMetrics(registerer) @@ -549,125 +552,3 @@ func initializeMetrics() (prometheus.Gatherer, prometheus.Registerer, error) { } return gatherer, registry, nil } - -func initializeConnectionsAndCheckStake( - logger logging.Logger, - network *peers.AppRequestNetwork, - cfg *config.Config, -) error { - // Manually connect to the validators of each of the source subnets. - // We return an error if we are unable to connect to sufficient stake on any of the subnets. - // Sufficient stake is determined by the Warp quora of the configured supported destinations, - // or if the subnet supports all destinations, by the quora of all configured destinations. - for _, sourceBlockchain := range cfg.SourceBlockchains { - if sourceBlockchain.GetSubnetID() == constants.PrimaryNetworkID { - if err := connectToPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { - return fmt.Errorf( - "failed to connect to primary network peers: %w", - err, - ) - } - } else { - if err := connectToNonPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { - return fmt.Errorf( - "failed to connect to non-primary network peers: %w", - err, - ) - } - } - } - return nil -} - -// Connect to the validators of the source blockchain. For each destination blockchain, -// verify that we have connected to a threshold of stake. -func connectToNonPrimaryNetworkPeers( - logger logging.Logger, - network *peers.AppRequestNetwork, - cfg *config.Config, - sourceBlockchain *config.SourceBlockchain, -) error { - subnetID := sourceBlockchain.GetSubnetID() - connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) - if err != nil { - logger.Error( - "Failed to connect to canonical validators", - zap.String("subnetID", subnetID.String()), - zap.Error(err), - ) - return err - } - for _, destination := range sourceBlockchain.SupportedDestinations { - blockchainID := destination.GetBlockchainID() - if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { - logger.Error( - "Failed to connect to a threshold of stake", - zap.String("destinationBlockchainID", blockchainID.String()), - zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), - zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), - zap.Any("warpQuorum", quorum), - ) - return err - } - } - return nil -} - -// Connect to the validators of the destination blockchains. Verify that we have connected -// to a threshold of stake for each blockchain. -func connectToPrimaryNetworkPeers( - logger logging.Logger, - network *peers.AppRequestNetwork, - cfg *config.Config, - sourceBlockchain *config.SourceBlockchain, -) error { - for _, destination := range sourceBlockchain.SupportedDestinations { - blockchainID := destination.GetBlockchainID() - subnetID := cfg.GetSubnetID(blockchainID) - connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) - if err != nil { - logger.Error( - "Failed to connect to canonical validators", - zap.String("subnetID", subnetID.String()), - zap.Error(err), - ) - return err - } - - if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { - logger.Error( - "Failed to connect to a threshold of stake", - zap.String("destinationBlockchainID", blockchainID.String()), - zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), - zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), - zap.Any("warpQuorum", quorum), - ) - return err - } - } - return nil -} - -// Fetch the warp quorum from the config and check if the connected stake exceeds the threshold -func checkForSufficientConnectedStake( - logger logging.Logger, - cfg *config.Config, - connectedValidators *peers.ConnectedCanonicalValidators, - destinationBlockchainID ids.ID, -) (bool, *config.WarpQuorum, error) { - quorum, err := cfg.GetWarpQuorum(destinationBlockchainID) - if err != nil { - logger.Error( - "Failed to get warp quorum from config", - zap.String("destinationBlockchainID", destinationBlockchainID.String()), - zap.Error(err), - ) - return false, nil, err - } - return utils.CheckStakeWeightExceedsThreshold( - big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight), - connectedValidators.TotalValidatorWeight, - quorum.QuorumNumerator, - quorum.QuorumDenominator, - ), &quorum, nil -} diff --git a/relayer/network_utils.go b/relayer/network_utils.go new file mode 100644 index 00000000..a76334bb --- /dev/null +++ b/relayer/network_utils.go @@ -0,0 +1,139 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package relayer + +import ( + "fmt" + "math/big" + + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/awm-relayer/peers" + "github.com/ava-labs/awm-relayer/relayer/config" + "github.com/ava-labs/awm-relayer/utils" + "go.uber.org/zap" +) + +func InitializeConnectionsAndCheckStake( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, +) error { + // Manually connect to the validators of each of the source subnets. + // We return an error if we are unable to connect to sufficient stake on any of the subnets. + // Sufficient stake is determined by the Warp quora of the configured supported destinations, + // or if the subnet supports all destinations, by the quora of all configured destinations. + for _, sourceBlockchain := range cfg.SourceBlockchains { + if sourceBlockchain.GetSubnetID() == constants.PrimaryNetworkID { + if err := connectToPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { + return fmt.Errorf( + "failed to connect to primary network peers: %w", + err, + ) + } + } else { + if err := connectToNonPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { + return fmt.Errorf( + "failed to connect to non-primary network peers: %w", + err, + ) + } + } + } + return nil +} + +// Connect to the validators of the source blockchain. For each destination blockchain, +// verify that we have connected to a threshold of stake. +func connectToNonPrimaryNetworkPeers( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, + sourceBlockchain *config.SourceBlockchain, +) error { + subnetID := sourceBlockchain.GetSubnetID() + connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) + if err != nil { + logger.Error( + "Failed to connect to canonical validators", + zap.String("subnetID", subnetID.String()), + zap.Error(err), + ) + return err + } + for _, destination := range sourceBlockchain.SupportedDestinations { + blockchainID := destination.GetBlockchainID() + if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { + logger.Error( + "Failed to connect to a threshold of stake", + zap.String("destinationBlockchainID", blockchainID.String()), + zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), + zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), + zap.Any("warpQuorum", quorum), + ) + return err + } + } + return nil +} + +// Connect to the validators of the destination blockchains. Verify that we have connected +// to a threshold of stake for each blockchain. +func connectToPrimaryNetworkPeers( + logger logging.Logger, + network *peers.AppRequestNetwork, + cfg *config.Config, + sourceBlockchain *config.SourceBlockchain, +) error { + for _, destination := range sourceBlockchain.SupportedDestinations { + blockchainID := destination.GetBlockchainID() + subnetID := cfg.GetSubnetID(blockchainID) + connectedValidators, err := network.ConnectToCanonicalValidators(subnetID) + if err != nil { + logger.Error( + "Failed to connect to canonical validators", + zap.String("subnetID", subnetID.String()), + zap.Error(err), + ) + return err + } + + if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { + logger.Error( + "Failed to connect to a threshold of stake", + zap.String("destinationBlockchainID", blockchainID.String()), + zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), + zap.Uint64("totalValidatorWeight", connectedValidators.TotalValidatorWeight), + zap.Any("warpQuorum", quorum), + ) + return err + } + } + return nil +} + +// Fetch the warp quorum from the config and check if the connected stake exceeds the threshold +func checkForSufficientConnectedStake( + logger logging.Logger, + cfg *config.Config, + connectedValidators *peers.ConnectedCanonicalValidators, + destinationBlockchainID ids.ID, +) (bool, *config.WarpQuorum, error) { + quorum, err := cfg.GetWarpQuorum(destinationBlockchainID) + if err != nil { + logger.Error( + "Failed to get warp quorum from config", + zap.String("destinationBlockchainID", destinationBlockchainID.String()), + zap.Error(err), + ) + return false, nil, err + } + return utils.CheckStakeWeightExceedsThreshold( + big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight), + connectedValidators.TotalValidatorWeight, + quorum.QuorumNumerator, + quorum.QuorumDenominator, + ), &quorum, nil +} From 5f631d1713654b6ffb5f696636f324a2b0ddad5d Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Fri, 16 Aug 2024 09:56:38 -0400 Subject: [PATCH 12/14] Update comment in network initialization --- relayer/network_utils.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/relayer/network_utils.go b/relayer/network_utils.go index a76334bb..d6e5947e 100644 --- a/relayer/network_utils.go +++ b/relayer/network_utils.go @@ -16,15 +16,19 @@ import ( "go.uber.org/zap" ) +// Convenience function to initialize connections and check stake for all source blockchains. +// Only returns an error if it fails to get a list of canonical validator or a valid warp config. +// +// Failing a sufficient stake check will only log an error but still return successfully +// since each attempted relay will make an attempt at reconnecting to any missing validators. +// +// Sufficient stake is determined by the Warp quora of the configured supported destinations, +// or if the subnet supports all destinations, by the quora of all configured destinations. func InitializeConnectionsAndCheckStake( logger logging.Logger, network *peers.AppRequestNetwork, cfg *config.Config, ) error { - // Manually connect to the validators of each of the source subnets. - // We return an error if we are unable to connect to sufficient stake on any of the subnets. - // Sufficient stake is determined by the Warp quora of the configured supported destinations, - // or if the subnet supports all destinations, by the quora of all configured destinations. for _, sourceBlockchain := range cfg.SourceBlockchains { if sourceBlockchain.GetSubnetID() == constants.PrimaryNetworkID { if err := connectToPrimaryNetworkPeers(logger, network, cfg, sourceBlockchain); err != nil { From cb15259000d0b6b624887a0015e2551b2f4537bc Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Fri, 16 Aug 2024 14:53:07 -0400 Subject: [PATCH 13/14] downgrade error logs to warnings --- relayer/network_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relayer/network_utils.go b/relayer/network_utils.go index d6e5947e..4931ee6a 100644 --- a/relayer/network_utils.go +++ b/relayer/network_utils.go @@ -70,7 +70,7 @@ func connectToNonPrimaryNetworkPeers( for _, destination := range sourceBlockchain.SupportedDestinations { blockchainID := destination.GetBlockchainID() if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { - logger.Error( + logger.Warn( "Failed to connect to a threshold of stake", zap.String("destinationBlockchainID", blockchainID.String()), zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), @@ -105,7 +105,7 @@ func connectToPrimaryNetworkPeers( } if ok, quorum, err := checkForSufficientConnectedStake(logger, cfg, connectedValidators, blockchainID); !ok { - logger.Error( + logger.Warn( "Failed to connect to a threshold of stake", zap.String("destinationBlockchainID", blockchainID.String()), zap.Uint64("connectedWeight", connectedValidators.ConnectedWeight), From aae5c4b0e708d5c6bfe5aed20a405fd9a897a962 Mon Sep 17 00:00:00 2001 From: Ian Suvak Date: Fri, 16 Aug 2024 15:26:08 -0400 Subject: [PATCH 14/14] update .goreleaser.yml --- .goreleaser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 9c38ac88..2b4e4d2a 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,7 +1,7 @@ # ref. https://goreleaser.com/customization/build/ builds: - id: awm-relayer - main: ./main/main.go + main: ./relayer/main/main.go binary: awm-relayer flags: - -v