From 68529307673322f314e027ecc19f12cd6ee5ac85 Mon Sep 17 00:00:00 2001 From: Caner Candan Date: Tue, 19 Apr 2022 15:27:32 +0200 Subject: [PATCH 01/70] =?UTF-8?q?fix:=20=F0=9F=90=9B=20rewards=20script=20?= =?UTF-8?q?missing=20decimals=20for=20allocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/rewards/rewards-period.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rewards/rewards-period.js b/scripts/rewards/rewards-period.js index 13b845d882..531e0e54af 100755 --- a/scripts/rewards/rewards-period.js +++ b/scripts/rewards/rewards-period.js @@ -6,10 +6,10 @@ async function createAssetRewardsFile() { const csv = fs.readFileSync("./pools.csv", "utf-8"); const entries = JSON.parse(fs.readFileSync("./entries.json", "utf-8")).result .registry.entries; - const lines = csv.split("\r\n"); + const lines = csv.split("\r\n").filter((line) => line.split(",")[1] !== ""); let [, allocation] = lines[0].split('"'); - allocation = parseInt(allocation.trim().split(",").join("")); + allocation = `${allocation.trim().split(",").join("")}${"0".repeat(18)}`; const multipliers = lines.slice(1).map((line) => { const [, poolName, multiplier] = line.split(","); From 9ac74dbaf6b0d2d24ff870a133f3b178167b7553 Mon Sep 17 00:00:00 2001 From: Caner Candan Date: Thu, 21 Apr 2022 18:18:49 +0200 Subject: [PATCH 02/70] =?UTF-8?q?test:=20=F0=9F=92=8D=20add=20scripts=20fo?= =?UTF-8?q?r=20manual=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/pmtp/.gitignore | 3 ++- .../4-add-liquidity-symmetry.sh | 15 +++++++++++++++ scripts/pmtp/send.sh | 14 ++++++++++++++ scripts/pmtp/set-rewards-params.sh | 15 +++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100755 scripts/pmtp/scenarios/no-policy-liquity-changes/4-add-liquidity-symmetry.sh create mode 100755 scripts/pmtp/send.sh create mode 100755 scripts/pmtp/set-rewards-params.sh diff --git a/scripts/pmtp/.gitignore b/scripts/pmtp/.gitignore index e6905a2395..9959f86ca3 100644 --- a/scripts/pmtp/.gitignore +++ b/scripts/pmtp/.gitignore @@ -1 +1,2 @@ -.env* \ No newline at end of file +.env* +rewards.json-* \ No newline at end of file diff --git a/scripts/pmtp/scenarios/no-policy-liquity-changes/4-add-liquidity-symmetry.sh b/scripts/pmtp/scenarios/no-policy-liquity-changes/4-add-liquidity-symmetry.sh new file mode 100755 index 0000000000..9f14a3444b --- /dev/null +++ b/scripts/pmtp/scenarios/no-policy-liquity-changes/4-add-liquidity-symmetry.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -x + +sifnoded tx clp add-liquidity \ + --from $SIF_ACT \ + --keyring-backend test \ + --symbol cusdt \ + --nativeAmount 1000000000000000000000000 \ + --externalAmount 25378853317 \ + --fees 100000000000000000rowan \ + --node ${SIFNODE_NODE} \ + --chain-id $SIFNODE_CHAIN_ID \ + --broadcast-mode block \ + -y \ No newline at end of file diff --git a/scripts/pmtp/send.sh b/scripts/pmtp/send.sh new file mode 100755 index 0000000000..f7d7c63bc5 --- /dev/null +++ b/scripts/pmtp/send.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -x + +sifnoded tx bank send \ + $SIF_ACT \ + sif144w8cpva2xkly74xrms8djg69y3mljzplx3fjt \ + 9299999999750930000rowan \ + --keyring-backend test \ + --node ${SIFNODE_NODE} \ + --chain-id $SIFNODE_CHAIN_ID \ + --fees 100000000000000000rowan \ + --broadcast-mode block \ + -y \ No newline at end of file diff --git a/scripts/pmtp/set-rewards-params.sh b/scripts/pmtp/set-rewards-params.sh new file mode 100755 index 0000000000..3ab5aa6af1 --- /dev/null +++ b/scripts/pmtp/set-rewards-params.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -x + +sifnoded tx clp reward-params \ + --cancelPeriod 60 \ + --lockPeriod 20 \ + --from=$SIF_ACT \ + --keyring-backend=test \ + --fees 100000000000000000rowan \ + --gas 500000 \ + --node ${SIFNODE_NODE} \ + --chain-id=$SIFNODE_CHAIN_ID \ + --broadcast-mode=block \ + -y \ No newline at end of file From 07b9ad58d76ffcc532ed60a6b62b841d55212cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 12 Jan 2022 09:39:58 +0100 Subject: [PATCH 03/70] Removal of '--relayerdb-path' option for ebrelayer in Peggy2.0 --- test/integration/framework/main.py | 7 ------- test/integration/framework/sifchain.py | 5 ++--- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index e9e19e5da2..1c7b3184a3 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -972,7 +972,6 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, validator_power, denom_whitelist_file), "home": validator0_home, - "db_path": project_dir("smart-contracts", "witnessdb"), } for name in [f"witness-{i}" for i in range(witness_count)]] tcp_url = "tcp://{}:{}".format(ANY_ADDR, tendermint_port) @@ -1030,9 +1029,6 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, sifnode_witness0_mnemonic = sifnode_witness0["name"] sifnode_witness0_address = sifnode_witness0["address"] sifnode_witness0_home = sifnode_witness0["home"] - sifnode_witness0_db_path = sifnode_witness0["db_path"] - self.cmd.rmdir(sifnode_witness0_db_path) - self.cmd.mkdir(sifnode_witness0_db_path) bridge_registry_contract_addr = peggy_sc_addrs.bridge_registry # bridge_bank_contract_addr = peggy_sc_addrs.bridge_bank @@ -1089,7 +1085,6 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, # --node tcp://0.0.0.0:26657 # --from sif1l7025ps7lt24effpduwxhk45sd977djvu38lhr # --symbol-translator-file ../test/integration/config/symbol_translator.json - # --relayerdb-path ./witnessdb # --log_format json # --keyring-backend test # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded @@ -1104,7 +1099,6 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, node=tcp_url, sign_with=sifnode_witness0_address, symbol_translator_file=symbol_translator_file, - relayerdb_path=sifnode_witness0_db_path, ethereum_address=evm_validator0_addr, ethereum_private_key=evm_validator0_key, keyring_backend="test", @@ -1306,7 +1300,6 @@ def format_sif_account(sif_account): "--keyring-backend", "test", "--from", witness["address"], "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", - "--relayerdb-path", witness["db_path"], "--home", witness["home"] ] } for i, witness in enumerate(sifnode_witnesses)], { diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index 0ab246677c..f0d82d5c45 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -197,8 +197,8 @@ def __init__(self, cmd): def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_node, web3_provider, bridge_registry_contract_address, validator_mnemonic, chain_id, node=None, keyring_backend=None, - sign_with=None, symbol_translator_file=None, relayerdb_path=None, log_format=None, extra_args=None, - ethereum_private_key=None, ethereum_address=None, home=None, cwd=None + sign_with=None, symbol_translator_file=None, log_format=None, extra_args=None, ethereum_private_key=None, + ethereum_address=None, home=None, cwd=None ): env = _env_for_ethereum_address_and_key(ethereum_address, ethereum_private_key) args = [ @@ -215,7 +215,6 @@ def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_n (["--node", node] if node else []) + \ (["--keyring-backend", keyring_backend] if keyring_backend else []) + \ (["--from", sign_with] if sign_with else []) + \ - (["--relayerdb-path", relayerdb_path] if relayerdb_path else []) + \ (["--home", home] if home else []) + \ (["--symbol-translator-file", symbol_translator_file] if symbol_translator_file else []) + \ (["--log_format", log_format] if log_format else []) From b50a55b4cc102901e773a82ac81da9cb2e925e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 13 Jan 2022 05:21:26 +0100 Subject: [PATCH 04/70] Bugfix --- test/integration/framework/inflate_tokens.py | 31 +++++-------------- .../integration/src/py/test_inflate_tokens.py | 5 +-- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index 0b5ef47d51..93191e8c0e 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -78,17 +78,10 @@ def build_list_of_tokens_to_create(self, existing_tokens, requested_tokens): token_symbol = token["symbol"] if (token_symbol == test_utils.CETH) or (token_symbol == test_utils.ROWAN): assert False, f"Token {token_symbol} cannot be used by this procedure, please remove it from list of requested assets" - if not token_symbol.startswith("c"): - assert False, f"Token {token_symbol} is invalid - should start with 'c'" - eth_token_symbol = token_symbol[1:] # Strip "c", e.g. "cusdt" -> "usdt" - existing_token = zero_or_one(find_by_value(existing_tokens, "symbol", eth_token_symbol)) + existing_token = zero_or_one(find_by_value(existing_tokens, "symbol", token_symbol)) if existing_token is None: - tokens_to_create.append({ - "name": token["name"], - "symbol": eth_token_symbol, - "decimals": token["decimals"], - }) + tokens_to_create.append(token) else: if not all([existing_token[f] == token[f] for f in ["name", "decimals"]]): assert False, "Existing token's name/decimals does not match requested for token: " \ @@ -100,23 +93,12 @@ def build_list_of_tokens_to_create(self, existing_tokens, requested_tokens): return tokens_to_create def create_new_tokens(self, tokens_to_create): - amount_in_token_units = 0 pending_txs = [] for token in tokens_to_create: token_name = token["name"] token_symbol = token["symbol"] token_decimals = token["decimals"] log.info(f"Creating token {token_symbol}...") - amount = amount_in_token_units * (10**token_decimals) - # Deploy a SifchainTestToken - # call BridgeBank.updateEthWhiteList with its address - # Mint amount_in_token_units to operator_address - # Approve entire minted amount to BridgeBank - # TODO We don't really need create_new_currency here, we only need to deploy the smart contract - # since we do the minting and approval in next step (token_refresh). - - # token_addr = self.ctx.create_new_currency(token_symbol, token_name, token_decimals, amount, minted_tokens_recipient) - txhash = self.ctx.tx_deploy_new_generic_erc20_token(self.ctx.operator, token_name, token_symbol, token_decimals) pending_txs.append(txhash) @@ -140,8 +122,9 @@ def create_new_tokens(self, tokens_to_create): "is_whitelisted": True, "sif_denom": self.ctx.eth_symbol_to_sif_symbol(token_symbol), }) - txhash = self.ctx.tx_update_bridge_bank_whitelist(token_sc.address, True) - pending_txs.append(txhash) + if not on_peggy2_branch: + txhash = self.ctx.tx_update_bridge_bank_whitelist(token_sc.address, True) + pending_txs.append(txhash) self.wait_for_all(pending_txs) return new_tokens @@ -226,7 +209,9 @@ def transfer(self, requested_tokens, amount, target_sif_accounts): new_tokens = self.create_new_tokens(tokens_to_create) existing_tokens.extend(new_tokens) - tokens_to_transfer = [exactly_one(find_by_value(existing_tokens, "sif_denom", t["symbol"])) + # At this point, all tokens that we want to transfer should exist both on Ethereum blockchain as well as in + # existing_tokens. + tokens_to_transfer = [exactly_one(find_by_value(existing_tokens, "symbol", t["symbol"])) for t in requested_tokens] self.mint([t["address"] for t in tokens_to_transfer], amount_per_token, eth_broker_account) diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index 576cbbdf46..77b57a5e54 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -79,12 +79,9 @@ def test_inflate_tokens_short(ctx): # TODO Read tokens from file requested_tokens = [{ - "symbol": ctx.eth_symbol_to_sif_symbol(t.symbol), + "symbol": t.symbol, "name": t.name, "decimals": t.decimals, - # Those are ignored - # "imageUrl": None, - # "network": None, } for t in [ctx.generate_random_erc20_token_data() for _ in range(3)]] script = InflateTokens(ctx) From 6db42c90a7b3745c0b8a68a88f912ce52df89d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 14 Jan 2022 12:54:04 +0100 Subject: [PATCH 05/70] #2354: inflate_tokens.py improvements --- test/integration/framework/inflate_tokens.py | 65 ++++++++++++------- test/integration/framework/test_utils.py | 11 ++-- .../integration/src/py/test_inflate_tokens.py | 13 +++- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index 93191e8c0e..ccbde540dd 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -6,6 +6,7 @@ import logging import re +import eth import test_utils from common import * @@ -65,11 +66,6 @@ def build_list_of_tokens_to_create(self, existing_tokens, requested_tokens): # their addresses appear in BridgeBank's past events implies that the corresponding ERC20 smart contracts have # been deployed, hence there is no need to deploy them. - # This assumes that requested token symbols are in sifchain format (c-prefixed, i.e. "cusdt", "csushi" etc.). - # There can also be "ceth" and "rowan" in this list, which we ignore as they represent special cases. - # To compare it to entries on existing_whitelist, we need to prefix entries on existing_whitelist with "c". - # TODO It would be better if the requested tokens didn't have "c" prefixes. For now we keep it for - # compatibility. Ask people who use this script. token_symbols_to_skip = set() token_symbols_to_skip.add(test_utils.CETH) # ceth is special since we can't just mint it or create an ERC20 contract for it token_symbols_to_skip.add(test_utils.ROWAN) @@ -139,31 +135,38 @@ def mint(self, list_of_tokens_addrs, amount_in_tokens, mint_recipient): pending_txs.append(txhash) self.wait_for_all(pending_txs) - def approve_and_lock(self, token_addr_list, eth_addr, to_sif_addr, amount): + def transfer_from_eth_to_sifnode(self, from_eth_addr, to_sif_addr, tokens_to_transfer, amount_in_tokens, amount_eth_gwei): + sif_balances_before = self.ctx.get_sifchain_balance(to_sif_addr) + sent_amounts = [] pending_txs = [] - for token_addr in token_addr_list: + for token in tokens_to_transfer: + token_addr = token["address"] + decimals = token["decimals"] token_sc = self.ctx.get_generic_erc20_sc(token_addr) - pending_txs.extend(self.ctx.tx_approve_and_lock(token_sc, eth_addr, to_sif_addr, amount)) - return self.wait_for_all(pending_txs) - - def transfer_from_eth_to_sifnode(self, eth_addr, sif_addr, tokens_to_transfer, amount): - sif_balances_before = self.ctx.get_sifchain_balance(sif_addr) - self.approve_and_lock([t["address"] for t in tokens_to_transfer], eth_addr, sif_addr, amount) + amount = amount_in_tokens * 10**decimals + pending_txs.extend(self.ctx.tx_approve_and_lock(token_sc, from_eth_addr, to_sif_addr, amount)) + sent_amounts.append([amount, token["sif_denom"]]) + if amount_eth_gwei > 0: + amount = amount_eth_gwei * eth.GWEI + pending_txs.append(self.ctx.tx_bridge_bank_lock_eth(from_eth_addr, to_sif_addr, amount)) + sent_amounts.append([amount, self.ctx.ceth_symbol]) + self.wait_for_all(pending_txs) # Wait for intermediate_sif_account to receive all funds across the bridge self.ctx.advance_blocks() - send_amounts = [[amount, t["sif_denom"]] for t in tokens_to_transfer] - self.ctx.wait_for_sif_balance_change(sif_addr, sif_balances_before, - min_changes=send_amounts, timeout=self.wait_for_account_change_timeout) + self.ctx.wait_for_sif_balance_change(to_sif_addr, sif_balances_before, + min_changes=sent_amounts, timeout=self.wait_for_account_change_timeout) - def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amount, target_sif_accounts): + def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amount_in_tokens, target_sif_accounts, amount_eth_gwei): # Distribute from intermediate_sif_account to each individual account # Note: firing transactions with "sifnoded tx bank send" in rapid succession does not work. This is currently a # known limitation of Cosmos SDK, see https://github.com/cosmos/cosmos-sdk/issues/4186 # Instead, we take advantage of batching multiple denoms to single account with single send command (amounts # separated by by comma: "sifnoded tx bank send ... 100denoma,100denomb,100denomc") and wait for destination # account to show changes for all denoms after each send. - send_amounts = [[amount, t["sif_denom"]] for t in tokens_to_transfer] + send_amounts = [[amount_in_tokens * 10**t["decimals"], t["sif_denom"]] for t in tokens_to_transfer] + if amount_eth_gwei > 0: + send_amounts.append([amount_eth_gwei * eth.GWEI, self.ctx.ceth_symbol]) for sif_acct in target_sif_accounts: sif_balance_before = self.ctx.get_sifchain_balance(sif_acct) self.ctx.send_from_sifchain_to_sifchain(from_sif_account, sif_acct, send_amounts) @@ -177,7 +180,7 @@ def export(self): "decimals": token["decimals"] } for token in self.get_whitelisted_tokens() if ("ibc" not in token) and (token["symbol"] not in excluded)] - def transfer(self, requested_tokens, amount, target_sif_accounts): + def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amount_gwei): """ It goes like this: 1. Starting with assets.json of your choice, It will first compare the list of tokens to existing whitelist and deploy any new tokens (ones that have not yet been whitelisted) @@ -190,11 +193,16 @@ def transfer(self, requested_tokens, amount, target_sif_accounts): # TODO Add support for "ceth" and "rowan" - amount_per_token = amount * len(target_sif_accounts) + total_token_amount = token_amount * len(target_sif_accounts) + total_eth_amount_gwei = eth_amount_gwei * len(target_sif_accounts) fund_rowan = [5 * test_utils.sifnode_funds_for_transfer_peggy1, "rowan"] + ether_faucet_account = self.ctx.operator sif_broker_account = self.ctx.create_sifchain_addr(fund_amounts=[fund_rowan]) eth_broker_account = self.ctx.operator + if (total_eth_amount_gwei > 0) and (ether_faucet_account != eth_broker_account): + self.ctx.eth.send_eth(ether_faucet_account, eth_broker_account, total_eth_amount_gwei) + log.info("Using eth_broker_account {}".format(eth_broker_account)) log.info("Using sif_broker_account {}".format(sif_broker_account)) @@ -214,9 +222,16 @@ def transfer(self, requested_tokens, amount, target_sif_accounts): tokens_to_transfer = [exactly_one(find_by_value(existing_tokens, "symbol", t["symbol"])) for t in requested_tokens] - self.mint([t["address"] for t in tokens_to_transfer], amount_per_token, eth_broker_account) - self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, tokens_to_transfer, amount_per_token) - self.distribute_tokens_to_wallets(sif_broker_account, tokens_to_transfer, amount, target_sif_accounts) + self.mint([t["address"] for t in tokens_to_transfer], total_token_amount, eth_broker_account) + self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, tokens_to_transfer, total_token_amount, total_eth_amount_gwei) + self.distribute_tokens_to_wallets(sif_broker_account, tokens_to_transfer, token_amount, target_sif_accounts, eth_amount_gwei) + + def transfer_eth(self, from_eth_addr, amount_gewi, target_sif_accounts): + pending_txs = [] + for sif_acct in target_sif_accounts: + txrcpt = self.ctx.eth.tx_bridge_bank_lock_eth(from_eth_addr, sif_acct, amount_gewi * eth.GWEI) + pending_txs.append(txrcpt) + self.wait_for_all(pending_txs) def run(*args): @@ -230,10 +245,10 @@ def run(*args): ctx.cmd.write_text_file(args[0], json.dumps(script.export(), indent=4)) elif cmd == "transfer": # Usage: inflate_tokens.py transfer assets.json amount accounts.json - assets_json_file, amount, accounts_json_file = args + assets_json_file, token_amount, accounts_json_file, amount_eth_gwei = args tokens = json.loads(ctx.cmd.read_text_file(assets_json_file)) accounts = json.loads(ctx.cmd.read_text_file(accounts_json_file)) - script.transfer(tokens, int(amount), accounts) + script.transfer(tokens, int(token_amount), accounts, int(amount_eth_gwei)) else: raise Exception("Invalid usage") diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 37c246967d..8b145988f4 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -90,10 +90,12 @@ def get_env_ctx_peggy2(): eth_node_is_local = True generic_erc20_contract = "BridgeToken" + ceth_symbol = sifchain.sifchain_denom_hash(ethereum_network_descriptor, eth.NULL_ADDRESS) + assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" ctx_eth = eth.EthereumTxWrapper(w3_conn, eth_node_is_local) ctx = EnvCtx(cmd, w3_conn, ctx_eth, abi_provider, owner_address, sifnoded_home, sifnode_url, sifnode_chain_id, - rowan_source, generic_erc20_contract) + rowan_source, ceth_symbol, generic_erc20_contract) if owner_private_key: ctx.eth.set_private_key(owner_address, owner_private_key) @@ -114,8 +116,6 @@ def get_env_ctx_peggy2(): ctx.cross_chain_lock_fee = 1 ctx.cross_chain_burn_fee = 1 ctx.ethereum_network_descriptor = ethereum_network_descriptor - ctx.ceth_symbol = sifchain.sifchain_denom_hash(ctx.ethereum_network_descriptor, eth.NULL_ADDRESS) - assert ctx.ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" return ctx @@ -209,7 +209,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): ctx_eth = eth.EthereumTxWrapper(w3_conn, eth_node_is_local) abi_provider = GanacheAbiProvider(cmd, artifacts_dir, ethereum_network_id) ctx = EnvCtx(cmd, w3_conn, ctx_eth, abi_provider, operator_address, sifnoded_home, sifnode_url, sifnode_chain_id, - rowan_source, generic_erc20_contract_name) + rowan_source, CETH, generic_erc20_contract_name) if operator_private_key: ctx.eth.set_private_key(operator_address, operator_private_key) @@ -286,7 +286,7 @@ def get_descriptor(self, sc_name): class EnvCtx: def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, sifnode_url, sifnode_chain_id, - rowan_source, generic_erc20_contract + rowan_source, ceth_symbol, generic_erc20_contract ): self.cmd = cmd self.w3_conn = w3_conn @@ -297,6 +297,7 @@ def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, self.sifnode_url = sifnode_url self.sifnode_chain_id = sifnode_chain_id self.rowan_source = rowan_source + self.ceth_symbol = ceth_symbol self.generic_erc20_contract = generic_erc20_contract self.available_test_eth_accounts = None diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index 77b57a5e54..0b8f2e6a91 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -74,7 +74,8 @@ @pytest.mark.skipif("on_peggy2_branch") def test_inflate_tokens_short(ctx): - amount = 12 * 10**10 + amount_in_tokens = 123 + amount_gwei = 456 wallets = test_wallets[:2] # TODO Read tokens from file @@ -85,7 +86,15 @@ def test_inflate_tokens_short(ctx): } for t in [ctx.generate_random_erc20_token_data() for _ in range(3)]] script = InflateTokens(ctx) - script.transfer(requested_tokens, amount, wallets) + + balances_before = [ctx.get_sifchain_balance(w) for w in wallets] + script.transfer(requested_tokens, amount_in_tokens, wallets, amount_gwei) + balances_delta = [ctx.sif_balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] + + for balances_delta in balances_delta: + for t in requested_tokens: + assert balances_delta[ctx.eth_symbol_to_sif_symbol(t["symbol"])] == amount_in_tokens * 10**t["decimals"] + assert balances_delta.get(ctx.ceth_symbol, 0) == amount_gwei * eth.GWEI @pytest.mark.skipif("on_peggy2_branch") From 7bdaa62ae16c0130ffc9c3ae00ec6a6ae9a0e95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 13 Jan 2022 20:29:19 +0100 Subject: [PATCH 06/70] Flexible timeouts for wait_for_sif_balance_change --- test/integration/framework/inflate_tokens.py | 27 +++++++++---------- test/integration/framework/test_utils.py | 22 +++++++++++---- .../integration/src/py/test_inflate_tokens.py | 25 +++++++++++++++++ 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index ccbde540dd..341e193af4 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -17,6 +17,7 @@ class InflateTokens: def __init__(self, ctx): self.ctx = ctx self.wait_for_account_change_timeout = 1800 # For Ropsten we need to wait for 50 blocks i.e. ~20 mins + self.excluded_token_symbols = ["erowan"] def get_whitelisted_tokens(self): whitelist = self.ctx.get_whitelisted_tokens_from_bridge_bank_past_events() @@ -40,7 +41,7 @@ def get_whitelisted_tokens(self): assert token_symbol not in result, f"Symbol {token_symbol} is being used by more than one whitelisted token" result.append(token) erowan_token = [t for t in result if t["symbol"] == "erowan"] - assert len(erowan_token) == 1, "erowan is not whitelisted" + assert len(erowan_token) == 1, "erowan is not whitelisted, probably bad/incomplete deployment" assert erowan_token[0]["is_whitelisted"], "erowan is un-whitelisted" return result @@ -66,13 +67,10 @@ def build_list_of_tokens_to_create(self, existing_tokens, requested_tokens): # their addresses appear in BridgeBank's past events implies that the corresponding ERC20 smart contracts have # been deployed, hence there is no need to deploy them. - token_symbols_to_skip = set() - token_symbols_to_skip.add(test_utils.CETH) # ceth is special since we can't just mint it or create an ERC20 contract for it - token_symbols_to_skip.add(test_utils.ROWAN) - tokens_to_create = [] # = requested - existing - {rowan, ceth} + tokens_to_create = [] for token in requested_tokens: token_symbol = token["symbol"] - if (token_symbol == test_utils.CETH) or (token_symbol == test_utils.ROWAN): + if token_symbol in self.excluded_token_symbols: assert False, f"Token {token_symbol} cannot be used by this procedure, please remove it from list of requested assets" existing_token = zero_or_one(find_by_value(existing_tokens, "symbol", token_symbol)) @@ -154,8 +152,8 @@ def transfer_from_eth_to_sifnode(self, from_eth_addr, to_sif_addr, tokens_to_tra # Wait for intermediate_sif_account to receive all funds across the bridge self.ctx.advance_blocks() - self.ctx.wait_for_sif_balance_change(to_sif_addr, sif_balances_before, - min_changes=sent_amounts, timeout=self.wait_for_account_change_timeout) + self.ctx.wait_for_sif_balance_change(to_sif_addr, sif_balances_before, min_changes=sent_amounts, + polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amount_in_tokens, target_sif_accounts, amount_eth_gwei): # Distribute from intermediate_sif_account to each individual account @@ -170,15 +168,15 @@ def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amo for sif_acct in target_sif_accounts: sif_balance_before = self.ctx.get_sifchain_balance(sif_acct) self.ctx.send_from_sifchain_to_sifchain(from_sif_account, sif_acct, send_amounts) - self.ctx.wait_for_sif_balance_change(sif_acct, sif_balance_before, min_changes=send_amounts) + self.ctx.wait_for_sif_balance_change(sif_acct, sif_balance_before, min_changes=send_amounts, + polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) def export(self): - excluded = ["erowan"] return [{ "symbol": token["symbol"], "name": token["name"], "decimals": token["decimals"] - } for token in self.get_whitelisted_tokens() if ("ibc" not in token) and (token["symbol"] not in excluded)] + } for token in self.get_whitelisted_tokens() if ("ibc" not in token) and (token["symbol"] not in self.excluded_token_symbols)] def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amount_gwei): """ @@ -193,9 +191,10 @@ def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amou # TODO Add support for "ceth" and "rowan" - total_token_amount = token_amount * len(target_sif_accounts) - total_eth_amount_gwei = eth_amount_gwei * len(target_sif_accounts) - fund_rowan = [5 * test_utils.sifnode_funds_for_transfer_peggy1, "rowan"] + n_accounts = len(target_sif_accounts) + total_token_amount = token_amount * n_accounts + total_eth_amount_gwei = eth_amount_gwei * n_accounts + fund_rowan = [5 * test_utils.sifnode_funds_for_transfer_peggy1 * n_accounts, "rowan"] ether_faucet_account = self.ctx.operator sif_broker_account = self.ctx.create_sifchain_addr(fund_amounts=[fund_rowan]) eth_broker_account = self.ctx.operator diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 8b145988f4..24bc9a2dd7 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -631,10 +631,11 @@ def sif_balance_delta(self, balances1, balances2): result[denom] = change return result - def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, polling_time=1, timeout=90): + def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, polling_time=1, timeout=90, change_timeout=None): start_time = time.time() - result = None - while result is None: + last_change_time = None + last_change_state = None + while True: new_balances = self.get_sifchain_balance(sif_addr) if min_changes is not None: have_all = True @@ -646,10 +647,21 @@ def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, else: if not self.sif_balances_equal(old_balances, new_balances): return new_balances - time.sleep(polling_time) now = time.time() - if now - start_time > timeout: + if (timeout is not None) and (now - start_time > timeout): raise Exception("Timeout waiting for sif balance to change") + if last_change_time is None: + last_change_state = new_balances + last_change_time = now + else: + delta = self.sif_balance_delta(new_balances, last_change_state) + if delta: + last_change_state = new_balances + last_change_time = now + log.debug("New state detected: {}".format(delta)) + if (change_timeout is not None) and (now - last_change_time > change_timeout): + raise Exception("Timeout waiting for sif balance to change") + time.sleep(polling_time) def eth_symbol_to_sif_symbol(self, eth_token_symbol): # TODO sifchain.use sifchain_denom_hash() if on_peggy2_branch diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index 0b8f2e6a91..1c749ad77a 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -97,6 +97,31 @@ def test_inflate_tokens_short(ctx): assert balances_delta.get(ctx.ceth_symbol, 0) == amount_gwei * eth.GWEI +@pytest.mark.skipif("on_peggy2_branch") +def test_inflate_tokens_long(ctx): + amount_in_tokens = 123 + amount_gwei = 456 + wallets = test_wallets[:2] + + # TODO Read tokens from file + requested_tokens = [{ + "symbol": t.symbol, + "name": t.name, + "decimals": t.decimals, + } for t in [ctx.generate_random_erc20_token_data() for _ in range(300)]] + + script = InflateTokens(ctx) + + balances_before = [ctx.get_sifchain_balance(w) for w in wallets] + script.transfer(requested_tokens, amount_in_tokens, wallets, amount_gwei) + balances_delta = [ctx.sif_balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] + + for balances_delta in balances_delta: + for t in requested_tokens: + assert balances_delta[ctx.eth_symbol_to_sif_symbol(t["symbol"])] == amount_in_tokens * 10**t["decimals"] + assert balances_delta.get(ctx.ceth_symbol, 0) == amount_gwei * eth.GWEI + + @pytest.mark.skipif("on_peggy2_branch") def disabled_test_inflate_tokens_full(ctx): amount = 12 * 10**10 From 4e4507fb3a6120bc9d5e9f69c2af0040dec7ff84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 15 Jan 2022 07:13:12 +0100 Subject: [PATCH 07/70] Refactoring --- test/integration/framework/project.py | 5 ++- test/integration/framework/sifchain.py | 13 ++++++ test/integration/framework/test_utils.py | 45 ++++--------------- .../integration/src/py/test_inflate_tokens.py | 6 +-- 4 files changed, 29 insertions(+), 40 deletions(-) diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index 98c5efc908..174ee443ca 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -49,10 +49,13 @@ def rebuild(self): self.install_smart_contracts_dependencies() self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) + def __rm_files_develop(self): + self.__rm(self.project_dir("test", "integration", "sifchainrelayerdb")) # TODO move to /tmp + def __rm_files(self, level): if level >= 0: # rm -rvf /tmp/tmp.xxxx (ganache DB, unique for every run) - self.__rm(self.project_dir("test", "integration", "sifchainrelayerdb")) # TODO move to /tmp + self.__rm_files_develop() self.__rm(self.project_dir("smart-contracts", "build")) # truffle deploy self.__rm(self.project_dir("test", "integration", "vagrant", "data")) self.__rm(self.cmd.get_user_home(".sifnoded")) # Probably needed for "--keyring-backend test" diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index f0d82d5c45..c63e7023c7 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -11,6 +11,19 @@ def sifchain_denom_hash(network_descriptor, token_contract_address): s = str(network_descriptor) + token_contract_address.lower() return "sif" + hashlib.sha256(s.encode("UTF-8")).digest().hex() +def balance_delta(balances1, balances2): + all_denoms = set(balances1.keys()) + all_denoms.update(balances2.keys()) + result = {} + for denom in all_denoms: + change = balances2.get(denom, 0) - balances1.get(denom, 0) + if change != 0: + result[denom] = change + return result + +def balance_zero(balances): + return len(balances) == 0 + class Sifnoded: def __init__(self, cmd, home=None): diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 24bc9a2dd7..1d9d832529 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -315,10 +315,12 @@ def advance_block(self, number): else: self.advance_block_truffle(number) # TODO Probably calls the same, check and remove - def advance_blocks(self): + def advance_blocks(self, number=50): # TODO Move to eth (it should be per-w3_conn) if self.eth.is_local_node: - self.advance_block(50) + previous_block = self.eth.w3_conn.eth.block_number + self.advance_block(number) + assert self.eth.w3_conn.eth.block_number - previous_block >= number # Otherwise just wait def get_blocklist_sc(self): @@ -599,13 +601,6 @@ def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): raise Exception(raw_log) return retval - # TODO - # def generate_test_account(self, target_ceth_balance=10**18, target_rowan_balance=10**18): - # sifchain_addr = self.create_sifchain_addr() - # self.send_eth_from_ethereum_to_sifchain(self.operator, sifchain_addr, target_ceth_balance) - # self.send_from_sifchain_to_sifchain(self.rowan_source, sifchain_addr, target_rowan_balance) - # return sifchain_addr - def get_sifchain_balance(self, sif_addr): args = ["query", "bank", "balances", sif_addr, "--limit", str(100000000), "--output", "json"] + \ self._sifnoded_chain_id_and_node_arg() @@ -613,40 +608,18 @@ def get_sifchain_balance(self, sif_addr): res = json.loads(stdout(res))["balances"] return dict(((x["denom"], int(x["amount"])) for x in res)) - def sif_balances_equal(self, dict1, dict2): - d2k = set(dict2.keys()) - for k in dict1.keys(): - if (k not in dict2) or (dict1[k] != dict2[k]): - return False - d2k.remove(k) - return len(d2k) == 0 - - def sif_balance_delta(self, balances1, balances2): - all_denoms = set(balances1.keys()) - all_denoms.update(balances2.keys()) - result = {} - for denom in all_denoms: - change = balances2.get(denom, 0) - balances1.get(denom, 0) - if change != 0: - result[denom] = change - return result - def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, polling_time=1, timeout=90, change_timeout=None): start_time = time.time() last_change_time = None last_change_state = None while True: new_balances = self.get_sifchain_balance(sif_addr) + delta = sifchain.balance_delta(old_balances, new_balances) if min_changes is not None: - have_all = True - for amount, denom in min_changes: - change = new_balances.get(denom, 0) - old_balances.get(denom, 0) - have_all = have_all and change >= amount - if have_all: - return new_balances - else: - if not self.sif_balances_equal(old_balances, new_balances): + if all([delta.get(denom, 0) >= amount for amount, denom in min_changes]): return new_balances + elif not sifchain.balance_zero(delta): + return new_balances now = time.time() if (timeout is not None) and (now - start_time > timeout): raise Exception("Timeout waiting for sif balance to change") @@ -654,7 +627,7 @@ def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, last_change_state = new_balances last_change_time = now else: - delta = self.sif_balance_delta(new_balances, last_change_state) + delta = sifchain.balance_delta(new_balances, last_change_state) if delta: last_change_state = new_balances last_change_time = now diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index 1c749ad77a..b71246fe3e 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -1,6 +1,6 @@ import pytest -from integration_framework import main, common, eth, test_utils, inflate_tokens +from integration_framework import main, common, eth, test_utils, sifchain, inflate_tokens from inflate_tokens import InflateTokens from common import * @@ -89,7 +89,7 @@ def test_inflate_tokens_short(ctx): balances_before = [ctx.get_sifchain_balance(w) for w in wallets] script.transfer(requested_tokens, amount_in_tokens, wallets, amount_gwei) - balances_delta = [ctx.sif_balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] + balances_delta = [sifchain.balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] for balances_delta in balances_delta: for t in requested_tokens: @@ -114,7 +114,7 @@ def test_inflate_tokens_long(ctx): balances_before = [ctx.get_sifchain_balance(w) for w in wallets] script.transfer(requested_tokens, amount_in_tokens, wallets, amount_gwei) - balances_delta = [ctx.sif_balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] + balances_delta = [sifchain.balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] for balances_delta in balances_delta: for t in requested_tokens: From eaa7ffc3a0b7ae9921d0b47eb9099531aed76d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 15 Jan 2022 18:35:17 +0100 Subject: [PATCH 08/70] #2318: refactoring, add 'cosmosBridge.setBridgeBank()' and 'update-consensus-needed' --- test/integration/framework/eth.py | 17 +++- test/integration/framework/hardhat.py | 112 ++++++++++++++--------- test/integration/framework/main.py | 90 ++++++++++-------- test/integration/framework/sifchain.py | 11 +++ test/integration/framework/test_utils.py | 48 ++-------- test/integration/framework/truffle.py | 22 +++++ 6 files changed, 177 insertions(+), 123 deletions(-) diff --git a/test/integration/framework/eth.py b/test/integration/framework/eth.py index a552e88b59..0befa9070b 100644 --- a/test/integration/framework/eth.py +++ b/test/integration/framework/eth.py @@ -25,6 +25,19 @@ def web3_connect(url, websocket_timeout=None): kwargs["websocket_timeout"] = websocket_timeout return web3.Web3(web3.Web3.WebsocketProvider(url, **kwargs)) +def web3_wait_for_connection_up(url, polling_time=1, timeout=90): + start_time = time.time() + w3_conn = web3_connect(url) + while True: + try: + w3_conn.eth.block_number + return w3_conn + except OSError: + pass + now = time.time() + if now - start_time > timeout: + raise Exception(f"Timeout when trying to connect to {url}") + time.sleep(polling_time) class EthereumTxWrapper: """ @@ -52,13 +65,13 @@ def __init__(self, w3_conn, is_local_node): self.used_tx_nonces = {} def _get_private_key(self, addr): - addr = self.w3_conn.toChecksumAddress(addr) + addr = web3.Web3.toChecksumAddress(addr) if not addr in self.private_keys: raise Exception(f"No private key set for address {addr}") return self.private_keys[addr] def set_private_key(self, addr, private_key): - addr = self.w3_conn.toChecksumAddress(addr) + addr = web3.Web3.toChecksumAddress(addr) if private_key is None: self.private_keys.pop(addr) # Remove else: diff --git a/test/integration/framework/hardhat.py b/test/integration/framework/hardhat.py index 9cbcf1ffce..9f89e3eaf0 100644 --- a/test/integration/framework/hardhat.py +++ b/test/integration/framework/hardhat.py @@ -1,4 +1,5 @@ import json +import web3 from dataclasses import dataclass from common import * from command import buildcmd @@ -18,46 +19,6 @@ def __init__(self, cmd): self.cmd = cmd self.project = cmd.project - @staticmethod - def default_accounts(): - # Hardhat doesn't provide a way to get the private keys of its default accounts, so just hardcode them for now. - # TODO hardhat prints 20 accounts upon startup - # Keep synced to smart-contracts/src/devenv/hardhatNode.ts:defaultHardhatAccounts - # Format: [address, private_key] - # Note: for compatibility with ganache, private keys should be stripped of "0x" prefix - # (when you pass a private key to ebrelayer via ETHEREUM_PRIVATE_KEY, the key is treated as invalid) - return [[address, private_key[2:]] for address, private_key in [[ - "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - ], [ - "0x70997970c51812dc3a010c7d01b50e0d17dc79c8", - "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", - ], [ - "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", - "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", - ], [ - "0x90f79bf6eb2c4f870365e785982e1f101e93b906", - "0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", - ], [ - "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", - "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", - ], [ - "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", - "0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", - ], [ - "0x976ea74026e726554db657fa54763abd0c3a0aa9", - "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", - ], [ - "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", - "0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", - ], [ - "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", - "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", - ], [ - "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", - "0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", - ]]] - def build_start_args(self, hostname=None, port=None, fork=None, fork_block_number=None): # TODO We need to manaege smart-contracts/hardhat.config.ts + it also reads smart-contracts/.env via dotenv # TODO Handle failures, e.g. if the process is already running we get exit value 1 and @@ -72,7 +33,7 @@ def build_start_args(self, hostname=None, port=None, fork=None, fork_block_numbe def compile_smart_contracts(self): self.project.npx(["hardhat", "compile"], cwd=project_dir("smart-contracts"), pipe=False) - def deploy_smart_contracts(self) -> Peggy2SmartContractAddresses: + def deploy_smart_contracts(self): # If this fails with tsyringe complaining about missing "../../build" directory, do this: # rm -rf smart-contracts/artifacts. res = self.project.npx(["hardhat", "run", "scripts/deploy_contracts.ts", "--network", "localhost"], @@ -91,5 +52,70 @@ def deploy_smart_contracts(self) -> Peggy2SmartContractAddresses: assert len(stdout_lines) == 2 assert stdout_lines[0] == "No need to generate any newer typings." tmp = json.loads(stdout_lines[1]) - return Peggy2SmartContractAddresses(cosmos_bridge=tmp["cosmosBridge"], bridge_bank=tmp["bridgeBank"], - bridge_registry=tmp["bridgeRegistry"], rowan=tmp["rowanContract"]) + return { + "BridgeBank": tmp["bridgeBank"], + "BridgeRegistry": tmp["bridgeRegistry"], + "CosmosBridge": tmp["cosmosBridge"], + "Rowan": tmp["rowanContract"], + } + + +class HardhatAbiProvider: + def __init__(self, cmd, deployed_contract_addresses): + self.cmd = cmd + self.deployed_contract_addresses = deployed_contract_addresses + + def get_descriptor(self, sc_name): + relpath = { + "BridgeBank": ["BridgeBank"], + "BridgeToken": ["BridgeBank"], + "CosmosBridge": [], + "Rowan": ["BridgeBank"], + "TrollToken": ["Mocks"], + }.get(sc_name, []) + [f"{sc_name}.sol", f"{sc_name}.json"] + path = os.path.join(self.cmd.project.project_dir("smart-contracts/artifacts/contracts"), *relpath) + tmp = json.loads(self.cmd.read_text_file(path)) + abi = tmp["abi"] + bytecode = tmp["bytecode"] + deployed_address = self.deployed_contract_addresses.get(sc_name) + return abi, bytecode, deployed_address + + +def default_accounts(): + # Hardhat doesn't provide a way to get the private keys of its default accounts, so just hardcode them for now. + # TODO hardhat prints 20 accounts upon startup + # Keep synced to smart-contracts/src/devenv/hardhatNode.ts:defaultHardhatAccounts + # Format: [address, private_key] + # Note: for compatibility with ganache, private keys should be stripped of "0x" prefix + # (when you pass a private key to ebrelayer via ETHEREUM_PRIVATE_KEY, the key is treated as invalid) + return [[web3.Web3.toChecksumAddress(address), private_key[2:]] for address, private_key in [[ + "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + ], [ + "0x70997970c51812dc3a010c7d01b50e0d17dc79c8", + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + ], [ + "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", + "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + ], [ + "0x90f79bf6eb2c4f870365e785982e1f101e93b906", + "0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + ], [ + "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", + "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + ], [ + "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", + "0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + ], [ + "0x976ea74026e726554db657fa54763abd0c3a0aa9", + "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + ], [ + "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", + "0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + ], [ + "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", + "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + ], [ + "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", + "0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", + ]]] diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index 1c7b3184a3..3ecb405e3e 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -2,13 +2,14 @@ import re import sys import time -from eth import NULL_ADDRESS + +import eth +import hardhat from truffle import Ganache from command import Command -from hardhat import Hardhat from sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash from project import Project, killall, force_kill_processes -from test_utils import get_env_ctx +import test_utils # TODO Circular reference, currently relying on partial module initialization in Python from common import * @@ -356,7 +357,7 @@ def stack_save_snapshot(self): # NOTE: this probably doesn't work anymore since setTokenLockBurnLimit.js was replaced burn_limits = [ - [NULL_ADDRESS, 31 * 10 ** 18], + [eth.NULL_ADDRESS, 31 * 10 ** 18], [bridge_token_address, 10 ** 25], [atk_address, 10 ** 25], [btk_address, 10 ** 25], @@ -540,7 +541,7 @@ def run(self): # # TODO This should be last (after return from setup_sifchain.sh) # burn_limits = [ - # [NULL_ADDRESS, 31*10**18], + # [eth.NULL_ADDRESS, 31*10**18], # [bridge_token_sc_addr, 10**25], # ] # env_file_vars = self.cmd.primitive_parse_env_file(env_file) @@ -763,7 +764,7 @@ def restart_processes(self): class Peggy2Environment(IntegrationTestsEnvironment): def __init__(self, cmd): super().__init__(cmd) - self.hardhat = Hardhat(cmd) + self.hardhat = hardhat.Hardhat(cmd) # Destuctures a linear array of EVM accounts into: # [operator, owner, pauser, [validator-0, validator-1, ...], [...available...]] @@ -809,7 +810,7 @@ def run(self): hardhat_exec_args = self.hardhat.build_start_args(hostname=hardhat_bind_hostname, port=hardhat_port) hardhat_proc = self.cmd.spawn_asynchronous_process(hardhat_exec_args, log_file=hardhat_log_file) - # This determines how much EVM accounts we want to allocate for validators. + # This determines how many EVM accounts we want to allocate for validators. # Since every validator needs on EVM account, this should be equal to the number of validators (possibly more). hardhat_validator_count = 1 hardhat_network_id = 1 # Not used in smart-contracts/src/devenv/hardhatNode.ts @@ -818,13 +819,20 @@ def run(self): # See https://hardhat.org/advanced/hardhat-runtime-environment.html # The value is not used; instead a hardcoded constant 31337 is passed to ebrelayerWitnessBuilder. # Ask juniuszhou for details. - hardhat_chain_id = 1 hardhat_chain_id = 31337 - hardhat_accounts = self.signer_array_to_ethereum_accounts(Hardhat.default_accounts(), hardhat_validator_count) + hardhat_accounts = self.signer_array_to_ethereum_accounts(hardhat.default_accounts(), hardhat_validator_count) + + self.hardhat.compile_smart_contracts() + peggy_sc_addrs = self.hardhat.deploy_smart_contracts() + + # Initialization of smart contracts (technically this is part of deployment) + operator_acct = hardhat_accounts["operator"] + w3_websocket_address = eth.web3_host_port_url("localhost", hardhat_port) + self.init_smart_contracts(w3_websocket_address, operator_acct, peggy_sc_addrs) admin_account_name = "sifnodeadmin" chain_id = "localnet" - ceth_symbol = sifchain_denom_hash(hardhat_chain_id, NULL_ADDRESS) + ceth_symbol = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" # Mint goes to validator mint_amount = [ @@ -850,10 +858,6 @@ def run(self): validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, admin_account_name) - self.hardhat.compile_smart_contracts() - peggy_sc_addrs = self.hardhat.deploy_smart_contracts() - - w3_websocket_address = "ws://localhost:{}/".format(hardhat_port) symbol_translator_file = os.path.join(self.test_integration_dir, "config", "symbol_translator.json") [relayer0_exec_args], [witness0_exec_args] = \ self.start_witnesses_and_relayers(w3_websocket_address, hardhat_chain_id, tcp_url, @@ -878,22 +882,33 @@ def run(self): "chain_id": chain_id, "validators": sifnode_validators, # From yaml file generated by sifgen "relayers": sifnode_relayers, - "smart_contracts": { - "BridgeBank": peggy_sc_addrs.bridge_bank, - "BridgeRegistry": peggy_sc_addrs.bridge_registry, - "CosmosBridge": peggy_sc_addrs.cosmos_bridge, - "RowanContract": peggy_sc_addrs.rowan, - } + "smart_contracts": peggy_sc_addrs } self.write_env_files(self.project.project_dir(), self.project.go_bin_dir, peggy_sc_addrs, hardhat_accounts, admin_account_name, admin_account_address, sifnode_validator0_home, sifnode_validators, sifnode_relayers, sifnode_witnesses, tcp_url, hardhat_bind_hostname, hardhat_port, hardhat_chain_id, chain_dir, sifnoded_exec_args, relayer0_exec_args, witness0_exec_args - ) + ) return hardhat_proc, sifnoded_proc, relayer0_proc, witness0_proc + def init_smart_contracts(self, w3_url, operator_account, deployed_contract_addresses): + # Looks like this is already done somewhere else... + # operator_addr, operator_private_key = operator_account + # w3_conn = eth.web3_wait_for_connection_up(w3_url) + # eth_tx = eth.EthereumTxWrapper(w3_conn, True) + # eth_tx.set_private_key(operator_addr, operator_private_key) + # + # # CosmosBridge doesn't have BridgeBank in its init and expects a separate setBridgeBank call. CosmosBridge + # # doesn't really work without BridgeBank. + # abi_provider = hardhat.HardhatAbiProvider(self.cmd, deployed_contract_addresses) + # abi, _, deployed_address = abi_provider.get_descriptor("CosmosBridge") + # cosmos_bridge = w3_conn.eth.contract(abi=abi, address=deployed_address) + # bridge_bank_addr = deployed_contract_addresses["BridgeBank"] + # txrcpt = eth_tx.transact_sync(cosmos_bridge.functions.setBridgeBank, operator_addr)(bridge_bank_addr) + return + def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, admin_account_name @@ -991,6 +1006,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh # TODO This command exits with status 0, but looks like there are some errros. # The same happens also in devenv. + # TODO Try whitelister account instead of admin res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_address, chain_id) log.debug("Result from token registry: {}".format(repr(res))) @@ -1001,13 +1017,15 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 - ethereum_cross_chain_fee_token = sifchain_denom_hash(hardhat_chain_id, NULL_ADDRESS) + ethereum_cross_chain_fee_token = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) gas_prices = [0.5, "rowan"] gas_adjustment = 1.5 sifnode.peggy2_set_cross_chain_fee(admin_account_address, hardhat_chain_id, ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, admin_account_name, chain_id, gas_prices, gas_adjustment) + sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) + return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ relayers, witnesses, validator0_home, chain_dir @@ -1114,7 +1132,7 @@ def write_env_files(self, project_dir, go_bin_dir, evm_smart_contract_addrs, eth relayer0_exec_args, witness0_exec_args ): eth_chain_id = hardhat_chain_id - w3_url = f"ws://{hardhat_bind_hostname}:{hardhat_port}/" + w3_url = eth.web3_host_port_url(hardhat_bind_hostname, hardhat_port) # @TODO At the moment, values are fed from one rendered template into the next. # We should use values directly from parameters instead. @@ -1131,10 +1149,10 @@ def format_sif_account(sif_account): # "completed": True, # "output": "...", "contractAddresses": { - "cosmosBridge": evm_smart_contract_addrs.cosmos_bridge, - "bridgeBank": evm_smart_contract_addrs.bridge_bank, - "bridgeRegistry": evm_smart_contract_addrs.bridge_registry, - "rowanContract": evm_smart_contract_addrs.rowan, + "cosmosBridge": evm_smart_contract_addrs["CosmosBridge"], + "bridgeBank": evm_smart_contract_addrs["BridgeBank"], + "bridgeRegistry": evm_smart_contract_addrs["BridgeRegistry"], + "rowanContract": evm_smart_contract_addrs["Rowan"], } }, "ethResults": { @@ -1186,11 +1204,11 @@ def format_sif_account(sif_account): "ETH_PORT": str(hardhat_port), "ROWAN_SOURCE": admin_account_address, "BRIDGE_BANK_ADDRESS": evm_smart_contract_addrs.bridge_bank, - # "BRIDGE_REGISTRY_ADDRESS": evm_smart_contract_addrs.bridge_registry, - "BRIDGE_REGISTERY_ADDRESS": evm_smart_contract_addrs.bridge_registry, # TODO Typo, remove, keeping it for now for compatibility - "COSMOS_BRIDGE_ADDRESS": evm_smart_contract_addrs.cosmos_bridge, - "ROWANTOKEN_ADDRESS": evm_smart_contract_addrs.rowan, - "BRIDGE_TOKEN_ADDRESS": evm_smart_contract_addrs.rowan, + # "BRIDGE_REGISTRY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], + "BRIDGE_REGISTERY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], # TODO Typo, remove, keeping it for now for compatibility + "COSMOS_BRIDGE_ADDRESS": evm_smart_contract_addrs["CosmosBridge"], + "ROWANTOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], + "BRIDGE_TOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], "GOBIN": go_bin_dir, "TCP_URL": tcp_url, "VALIDATOR_ADDRESS": sifnode_validators[0]["address"], @@ -1264,7 +1282,7 @@ def format_sif_account(sif_account): "--network-descriptor", str(eth_chain_id), "--tendermint-node", tcp_url, "--web3-provider", w3_url, - "--bridge-registry-contract-address", evm_smart_contract_addrs.bridge_registry, + "--bridge-registry-contract-address", evm_smart_contract_addrs["BridgeRegistry"], "--validator-mnemonic", relayer["name"], "--chain-id", "localnet", "--node", tcp_url, @@ -1293,7 +1311,7 @@ def format_sif_account(sif_account): str(eth_chain_id), tcp_url, w3_url, - evm_smart_contract_addrs.bridge_registry, + evm_smart_contract_addrs["BridgeRegistry"], witness["name"], "--chain-id", "localnet", "--node", tcp_url, @@ -1517,7 +1535,7 @@ def main(argv): force_kill_processes(cmd) # Some processes are restarted during integration tests so we don't own them log.info("Everything OK") elif what == "check-env": - ctx = get_env_ctx() + ctx = test_utils.get_env_ctx() ctx.sanity_check() elif what == "test-logging": ls_cmd = mkcmd(["ls", "-al", "."], cwd="/tmp") @@ -1532,7 +1550,7 @@ def main(argv): args = g.geth_cmd__test_integration_geth_branch(datadir=datadir_for_running) geth_proc = cmd.popen(args, log_file=geth_log_file) import hardhat - for expected_addr, private_key in hardhat.Hardhat(cmd).default_accounts(): + for expected_addr, private_key in hardhat.default_accounts(): addr = g.create_account("password", private_key, datadir=datadir_for_keys) assert addr == expected_addr input("Press ENTER to exit...") diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index c63e7023c7..e13b22b353 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -152,6 +152,17 @@ def peggy2_set_cross_chain_fee(self, admin_account_address, network_id, ethereum res = self.sifnoded_exec(args, keyring_backend=self.keyring_backend, sifnoded_home=self.home) return res + def peggy2_update_consensus_needed(self, admin_account_address, hardhat_chain_id, chain_id): + consensus_needed = "49" + args = ["tx", "ethbridge", "update-consensus-needed", admin_account_address, str(hardhat_chain_id), + consensus_needed, "--from", admin_account_address, "--chain-id", chain_id, "--gas-prices", + "0.5rowan", "--gas-adjustment", "1.5", "-y"] + # TODO Currently "sifnoded tx ethbridge" does not have a "update-consensus-needed" subcommand so this command + # fails by printing help and exiting with errorcode 0. Wait for PR to be merged: + # https://github.com/Sifchain/sifnode/pull/2263 + res = self.sifnoded_exec(args, keyring_backend=self.keyring_backend, sifnoded_home=self.home) + return res + def sifnoded_start(self, tcp_url=None, minimum_gas_prices=None, log_format_json=False, log_file=None): sifnoded_exec_args = self.build_start_cmd(tcp_url=tcp_url, minimum_gas_prices=minimum_gas_prices, log_format_json=log_format_json) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 1d9d832529..5e1bb98191 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -6,6 +6,8 @@ import main import eth +import truffle +import hardhat import sifchain from common import * @@ -67,7 +69,7 @@ def get_env_ctx_peggy2(): "BridgeRegistry": tmp["bridgeRegistry"], "Rowan": tmp["rowanContract"], } - abi_provider = HardhatAbiProvider(cmd, deployed_contract_addresses) + abi_provider = hardhat.HardhatAbiProvider(cmd, deployed_contract_addresses) # TODO We're mixing "OPERATOR" vs. "OWNER" # TODO Addressses from dot_env_vars are not in correct EIP55 "checksum" format @@ -207,7 +209,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): eth_node_is_local = deployment_name is None ctx_eth = eth.EthereumTxWrapper(w3_conn, eth_node_is_local) - abi_provider = GanacheAbiProvider(cmd, artifacts_dir, ethereum_network_id) + abi_provider = truffle.GanacheAbiProvider(cmd, artifacts_dir, ethereum_network_id) ctx = EnvCtx(cmd, w3_conn, ctx_eth, abi_provider, operator_address, sifnoded_home, sifnode_url, sifnode_chain_id, rowan_source, CETH, generic_erc20_contract_name) if operator_private_key: @@ -246,44 +248,6 @@ def sif_addr_to_evm_arg(sif_address): return sif_address.encode("UTF-8") -class GanacheAbiProvider: - def __init__(self, cmd, artifacts_dir, ethereum_network_id): - self.cmd = cmd - self.artifacts_dir = artifacts_dir - self.ethereum_default_network_id = ethereum_network_id - - def get_descriptor(self, sc_name): - path = self.cmd.project.project_dir(self.artifacts_dir, "contracts/{}.json".format(sc_name)) - tmp = json.loads(self.cmd.read_text_file(path)) - abi = tmp["abi"] - bytecode = tmp["bytecode"] - deployed_address = None - if ("networks" in tmp) and (self.ethereum_default_network_id is not None): - str_network_id = str(self.ethereum_default_network_id) - if str_network_id in tmp["networks"]: - deployed_address = tmp["networks"][str_network_id]["address"] - return abi, bytecode, deployed_address - - -class HardhatAbiProvider: - def __init__(self, cmd, deployed_contract_addresses): - self.cmd = cmd - self.deployed_contract_addresses = deployed_contract_addresses - - def get_descriptor(self, sc_name): - relpath = { - "BridgeBank": ["BridgeBank"], - "BridgeToken": ["BridgeBank"], - "TrollToken": ["Mocks"], - }.get(sc_name, []) + [f"{sc_name}.sol", f"{sc_name}.json"] - path = os.path.join(self.cmd.project.project_dir("smart-contracts/artifacts/contracts"), *relpath) - tmp = json.loads(self.cmd.read_text_file(path)) - abi = tmp["abi"] - bytecode = tmp["bytecode"] - deployed_address = self.deployed_contract_addresses.get(sc_name) - return abi, bytecode, deployed_address - - class EnvCtx: def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, sifnode_url, sifnode_chain_id, rowan_source, ceth_symbol, generic_erc20_contract @@ -462,7 +426,7 @@ def get_whitelisted_tokens_from_bridge_bank_past_events(self): for e in past_events: token_addr = e.args["_token"] value = e.args["_value"] - assert self.eth.w3_conn.toChecksumAddress(token_addr) == token_addr + assert web3.Web3.toChecksumAddress(token_addr) == token_addr # Logically the whitelist only consists of entries that have the last value of True. # If the data is clean, then for each token_addr we should first see a True event, possibly # followed by alternating False and True. The last value is the active one. @@ -770,7 +734,7 @@ def bridge_bank_lock_erc20(self, token_addr, from_eth_acct, to_sif_acct, amount) # Peggy1-specific def set_ofac_blocklist_to(self, addrs): blocklist_sc = self.get_blocklist_sc() - addrs = [self.eth.w3_conn.toChecksumAddress(addr) for addr in addrs] + addrs = [web3.Web3.toChecksumAddress(addr) for addr in addrs] existing_entries = blocklist_sc.functions.getFullList().call() to_add = [addr for addr in addrs if addr not in existing_entries] to_remove = [addr for addr in existing_entries if addr not in addrs] diff --git a/test/integration/framework/truffle.py b/test/integration/framework/truffle.py index 52c08016fd..1a24822384 100644 --- a/test/integration/framework/truffle.py +++ b/test/integration/framework/truffle.py @@ -1,3 +1,6 @@ +import json + + class Ganache: @staticmethod def start_ganache_cli(env, mnemonic=None, db=None, port=None, host=None, network_id=None, gas_price=None, @@ -15,3 +18,22 @@ def start_ganache_cli(env, mnemonic=None, db=None, port=None, host=None, network (["--blockTime", str(block_time)] if block_time is not None else []) + \ (["--account_keys_path", account_keys_path] if account_keys_path is not None else []) return env.popen(args, log_file=log_file) + + +class GanacheAbiProvider: + def __init__(self, cmd, artifacts_dir, ethereum_network_id): + self.cmd = cmd + self.artifacts_dir = artifacts_dir + self.ethereum_default_network_id = ethereum_network_id + + def get_descriptor(self, sc_name): + path = self.cmd.project.project_dir(self.artifacts_dir, "contracts/{}.json".format(sc_name)) + tmp = json.loads(self.cmd.read_text_file(path)) + abi = tmp["abi"] + bytecode = tmp["bytecode"] + deployed_address = None + if ("networks" in tmp) and (self.ethereum_default_network_id is not None): + str_network_id = str(self.ethereum_default_network_id) + if str_network_id in tmp["networks"]: + deployed_address = tmp["networks"][str_network_id]["address"] + return abi, bytecode, deployed_address From 3193c7fb1f6820ad272b8f743ccf8d0897a6e739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 17 Jan 2022 12:11:11 +0100 Subject: [PATCH 09/70] Refactoring --- test/integration/framework/main.py | 1451 +-------------------- test/integration/framework/run_env.py | 1453 ++++++++++++++++++++++ test/integration/framework/test_utils.py | 10 +- 3 files changed, 1460 insertions(+), 1454 deletions(-) create mode 100644 test/integration/framework/run_env.py diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index 3ecb405e3e..b8584e71e3 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -1,1459 +1,12 @@ -import json -import re import sys import time -import eth -import hardhat -from truffle import Ganache -from command import Command -from sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash +from run_env import Integrator, UIStackEnvironment, Peggy2Environment, IBCEnvironment, IntegrationTestsEnvironment from project import Project, killall, force_kill_processes -import test_utils # TODO Circular reference, currently relying on partial module initialization in Python +import test_utils from common import * -class Integrator(Ganache, Command): - def __init__(self): - super().__init__() # TODO Which super is this? All of them? - self.project = Project(self, project_dir()) - - def primitive_parse_env_file(self, path): - def split(lines): - result = dict() - for line in lines: - m = patt.match(line) - result[m[1]] = m[2] - return result - - tmp1 = self.mktempfile() - tmp2 = self.mktempfile() - try: - self.execst(["env", "-i", "bash", "-c", "set -o posix; IFS=' '; set > {}; source {}; set > {}".format(tmp1, path, tmp2)]) - t1 = self.read_text_file(tmp1).splitlines() - t2 = self.read_text_file(tmp2).splitlines() - finally: - self.rm(tmp1) - self.rm(tmp2) - patt = re.compile("^(.*?)=(.*)$") - d1 = split(t1) - d2 = split(t2) - result = dict() - for k, v in d2.items(): - if (k in d1) and (d1[k] == d2[k]): - continue - if k in ["_", "BASH_ARGC"]: - continue - result[k] = v - return result - - def _check_env_vs_file(self, env, env_path): - if (not self.exists(env_path)) or (env is None): - return - fenv = self.primitive_parse_env_file(env_path) - for k, v in env.items(): - if k in fenv: - if env[k] == fenv[k]: - log.warning(f"Variable '{k}' specified both as a parameter and in '{env_path}'") - else: - log.warning(f"Variable '{k}' has different values, parameter: {env[k]}, in '{env_path}': " - f"{fenv[k]}. According to observation, value from parameter will be used.") - - def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_threshold=None, operator=None, - owner=None, initial_validator_addresses=None, initial_validator_powers=None, pauser=None, - mainnet_gas_price=None, env_file=None - ): - env = {} - if consensus_threshold is not None: # required - env["CONSENSUS_THRESHOLD"] = str(consensus_threshold) - if operator is not None: # required - env["OPERATOR"] = operator - if owner is not None: # required - env["OWNER"] = owner - if initial_validator_addresses is not None: - env["INITIAL_VALIDATOR_ADDRESSES"] = ",".join(initial_validator_addresses) - if initial_validator_powers is not None: - env["INITIAL_VALIDATOR_POWERS"] = ",".join([str(x) for x in initial_validator_powers]) - if pauser is not None: - env["PAUSER"] = pauser - if mainnet_gas_price is not None: - env["MAINNET_GAS_PRICE"] = mainnet_gas_price - - env_path = os.path.join(self.project.smart_contracts_dir, ".env") - if env_file is not None: - self.copy_file(env_file, env_path) - - self._check_env_vs_file(env, env_path) - - # TODO ui scripts use just "yarn; yarn migrate" alias "npx truffle migrate --reset", - self.project.npx(["truffle", "deploy", "--network", network_name, "--reset"], env=env, - cwd=self.project.smart_contracts_dir, pipe=False) - - def deploy_smart_contracts_for_ui_stack(self): - self.copy_file(os.path.join(self.project.smart_contracts_dir, ".env.ui.example"), - os.path.join(self.project.smart_contracts_dir, ".env")) - # TODO Might not be neccessary - self.project.yarn([], cwd=self.project.smart_contracts_dir) - self.project.yarn(["migrate"], cwd=self.project.smart_contracts_dir) - - # truffle - def get_smart_contract_address(self, compiled_json_path, network_id): - return json.loads(self.read_text_file(compiled_json_path))["networks"][str(network_id)]["address"] - - # truffle - def get_bridge_smart_contract_addresses(self, network_id): - return [self.get_smart_contract_address(os.path.join( - self.project.smart_contracts_dir, f"build/contracts/{x}.json"), network_id) - for x in ["BridgeToken", "BridgeRegistry", "BridgeBank"]] - - def truffle_exec(self, script_name, *script_args, env=None): - self._check_env_vs_file(env, os.path.join(self.project.smart_contracts_dir, ".env")) - script_path = os.path.join(self.project.smart_contracts_dir, f"scripts/{script_name}.js") - # Hint: call web3 directly, avoid npx + truffle + script - # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) - self.project.npx(["truffle", "exec", script_path] + list(script_args), env=env, cwd=self.project.smart_contracts_dir, pipe=False) - - # TODO setTokenLockBurnLimit is gone, possibly replaced by bulkSetTokenLockBurnLimit - def set_token_lock_burn_limit(self, update_address, amount, ethereum_private_key, infura_project_id, local_provider): - env = { - "ETHEREUM_PRIVATE_KEY": ethereum_private_key, - "UPDATE_ADDRESS": update_address, - "INFURA_PROJECT_ID": infura_project_id, - "LOCAL_PROVIDER": local_provider, - } - # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS - # TODO script is no longer there! - self.truffle_exec("setTokenLockBurnLimit", str(amount), env=env) - - # @TODO Merge - def sifchain_init_integration(self, sifnode, validator_moniker, validator_mnemonic, denom_whitelist_file): - # now we have to add the validator key to the test keyring so the tests can send rowan from validator1 - sifnode0 = Sifnoded(self) - sifnode0.keys_add(validator_moniker, validator_mnemonic) - valoper = sifnode.keys_show(validator_moniker, bech="val")[0]["address"] - assert valoper == sifnode0.get_val_address(validator_moniker) # This does not use "home"; if it the assertion holds it could be grouped with sifchain_init_peggy - - # This was deleted in commit f00242302dd226bc9c3060fb78b3de771e3ff429 from sifchain_start_daemon.sh because - # it was not working. But we assume that we want to keep it. - sifnode.sifnoded_exec(["add-genesis-validators", valoper], sifnoded_home=sifnode.home) - - adminuser_addr = self.sifchain_init_common(sifnode, denom_whitelist_file) - return adminuser_addr - - def sifnoded_peggy2_init_validator(self, sifnode, validator_moniker, validator_mnemonic, evm_network_descriptor, validator_power, chain_dir_base): - # Add validator key to test keyring - # This effectively copies key for validator_moniker from what sifgen creates in /tmp/sifnodedNetwork/validators - # to ~/.sifnoded (note absence of explicit sifnoded_home, therefore it's ~/.sifnoded) - sifnode0 = Sifnoded(self) - sifnode0.keys_add(validator_moniker, validator_mnemonic) - - # Read valoper key - # (Since we now copied the key to main keyring we could also read it from there) - valoper = sifnode.get_val_address(validator_moniker) - - # Add genesis validator - sifnode.add_genesis_validators_peggy(evm_network_descriptor, valoper, validator_power) - - # Get whitelisted validator - # TODO Value is not being used - # TODO We're using default home here instead of sifnoded_home above. Does this even work? - _whitelisted_validator = sifnode.get_val_address(validator_moniker) - assert valoper == _whitelisted_validator - - # TODO Not any longer shared between IntegrationEnvironment and PeggyEnvironment - # Peggy2Environment calls sifnoded_peggy2_add_account - def sifchain_init_common(self, sifnode, denom_whitelist_file): - # Add sifnodeadmin to ~/.sifnoded - sifnode0 = Sifnoded(self) - sifnodeadmin_addr = sifnode0.keys_add_1("sifnodeadmin")["address"] - tokens = [[10**20, "rowan"]] - # Original from peggy: - # self.cmd.execst(["sifnoded", "add-genesis-account", sifnoded_admin_address, "100000000000000000000rowan", "--home", sifnoded_home]) - sifnode.add_genesis_account(sifnodeadmin_addr, tokens) - sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) - sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) - sifnode.set_gen_denom_whitelist(denom_whitelist_file) - return sifnodeadmin_addr - - # @TODO Move to Sifgen class - def sifgen_create_network(self, chain_id, validator_count, networks_dir, network_definition_file, seed_ip_address, mint_amount=None): - # Old call (no longer works either): - # sifgen network create localnet 1 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks \ - # 192.168.1.2 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks/network-definition.yml \ - # --keyring-backend file - # self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, - # os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file"]) - # TODO Most likely, this should be "--keyring-backend file" - args = ["sifgen", "network", "create", chain_id, str(validator_count), networks_dir, seed_ip_address, - network_definition_file, "--keyring-backend", "test"] + \ - (["--mint-amount", ",".join([sif_format_amount(*x) for x in mint_amount])] if mint_amount else []) - self.execst(args) - - def wait_for_sif_account(self, netdef_json, validator1_address): - # TODO Replace with test_utilities.wait_for_sif_account / wait_for_sif_account_up - return self.execst(["python3", os.path.join(self.project.test_integration_dir, "src/py/wait_for_sif_account.py"), - netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) - - def wait_for_sif_account_up(self, address, tcp_url=None): - # TODO Deduplicate: this is also in run_ebrelayer() - # netdef_json is path to file containing json_dump(netdef) - # while not self.cmd.tcp_probe_connect("localhost", tendermint_port): - # time.sleep(1) - # self.wait_for_sif_account(netdef_json, validator1_address) - - # Peggy2 - # How this works: by default, the command below will try to do a POST to http://localhost:26657. - # So the port has to be up first, but this query will fail anyway if it is not. - args = ["sifnoded", "query", "account", address] + \ - (["--node", tcp_url] if tcp_url else []) - while True: - try: - self.execst(args) - break - except Exception as e: - log.debug(f"Waiting for sif account {address}... ({repr(e)})") - time.sleep(1) - - -class UIStackEnvironment: - def __init__(self, cmd): - self.cmd = cmd - self.project = cmd.project - self.chain_id = "sifchain-local" - self.network_name = "develop" - self.network_id = 5777 - self.keyring_backend = "test" - self.ganache_db_path = self.cmd.get_user_home(".ganachedb") - self.sifnoded_path = self.cmd.get_user_home(".sifnoded") - self.sifnode = Sifnoded(cmd) - - # From ui/chains/credentials.sh - self.shadowfiend_name = "shadowfiend" - self.shadowfiend_mnemonic = ["race", "draft", "rival", "universe", "maid", "cheese", "steel", "logic", "crowd", - "fork", "comic", "easy", "truth", "drift", "tomorrow", "eye", "buddy", "head", "time", "cash", "swing", - "swift", "midnight", "borrow"] - self.kasha_name = "akasha" - self.akasha_mnemonic = ["hand", "inmate", "canvas", "head", "lunar", "naive", "increase", "recycle", "dog", - "ecology", "inhale", "december", "wide", "bubble", "hockey", "dice", "worth", "gravity", "ketchup", "feed", - "balance", "parent", "secret", "orchard"] - self.juniper_name = "juniper" - self.juniper_mnemonic = ["clump", "genre", "baby", "drum", "canvas", "uncover", "firm", "liberty", "verb", - "moment", "access", "draft", "erupt", "fog", "alter", "gadget", "elder", "elephant", "divide", "biology", - "choice", "sentence", "oppose", "avoid"] - self.ethereum_root_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", - "crumble", "sweet", "treat"] - - def stack_save_snapshot(self): - # ui-stack.yml - # cd .; go get -v -t -d ./... - # cd ui; yarn install --frozen-lockfile --silent - # Compile smart contracts: - # cd ui; yarn build - - # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh - # rm ui/node_modules/.migrate-complete - - # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh => ui/scripts/stack-launch.sh - # ui/scripts/stack-launch.sh -> ui/scripts/_sif-build.sh -> ui/chains/sif/build.sh - # killall sifnoded - # rm $(which sifnoded) - self.cmd.rmdir(self.sifnoded_path) - self.project.make_go_binaries_2() - - # ui/scripts/stack-launch.sh -> ui/scripts/_eth.sh -> ui/chains/etc/launch.sh - self.cmd.rmdir(self.ganache_db_path) - self.project.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules - # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth - ganache_proc = Ganache.start_ganache_cli(self.cmd, mnemonic=self.ethereum_root_mnemonic, db=self.ganache_db_path, - port=7545, network_id=self.network_id, gas_price=20000000000, gas_limit=6721975, host=ANY_ADDR) - - sifnode = Sifnoded(self.cmd) - # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh - sifnode.sifnoded_init("test", self.chain_id) - self.cmd.copy_file(project_dir("ui/chains/sif/app.toml"), os.path.join(self.sifnoded_path, "config/app.toml")) - log.info(f"Generating deterministic account - {self.shadowfiend_name}...") - shadowfiend_account = self.cmd.sifnoded_keys_add(self.shadowfiend_name, self.shadowfiend_mnemonic) - log.info(f"Generating deterministic account - {self.akasha_name}...") - akasha_account = self.sifnode.keys_add(self.akasha_name, self.akasha_mnemonic) - log.info(f"Generating deterministic account - {self.juniper_name}...") - juniper_account = self.cmd.sifnoded_keys_add(self.juniper_name, self.juniper_mnemonic) - shadowfiend_address = shadowfiend_account["address"] - akasha_address = akasha_account["address"] - juniper_address = juniper_account["address"] - assert shadowfiend_address == self.sifnode.keys_show(self.shadowfiend_name)[0]["address"] - assert akasha_address == self.sifnode.keys_show(self.akasha_name)[0]["address"] - assert juniper_address == self.sifnode.keys_show(self.juniper_name)[0]["address"] - - tokens_shadowfiend = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] - tokens_akasha = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] - tokens_juniper = [[10**22, "rowan"], [10**22, "cusdc"], [10**20, "clink"], [10**20, "ceth"]] - sifnode.add_genesis_account(shadowfiend_address, tokens_shadowfiend) - sifnode.add_genesis_account(akasha_address, tokens_akasha) - sifnode.add_genesis_account(juniper_address, tokens_juniper) - - shadowfiend_address_bech_val = sifnode.keys_show(self.shadowfiend_name, bech="val")[0]["address"] - self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) - - amount = sif_format_amount(10**24, "stake") - self.cmd.execst(["sifnoded", "gentx", self.shadowfiend_name, amount, f"--chain-id={self.chain_id}", - f"--keyring-backend={self.keyring_backend}"]) - - log.info("Collecting genesis txs...") - self.cmd.execst(["sifnoded", "collect-gentxs"]) - log.info("Validating genesis file...") - self.cmd.execst(["sifnoded", "validate-genesis"]) - - log.info("Starting test chain...") - sifnoded_proc = self.cmd.sifnoded_start(minimum_gas_prices=[0.5, "rowan"]) # TODO sifnoded_home=??? - - # sifnoded must be up before continuing - self.cmd.sif_wait_up("localhost", 1317) - - # ui/scripts/_migrate.sh -> ui/chains/peggy/migrate.sh - self.cmd.deploy_smart_contracts_for_ui_stack() - - # ui/scripts/_migrate.sh -> ui/chains/eth/migrate.sh - # send through atk and btk tokens to eth chain - self.project.yarn(["migrate"], cwd=project_dir("ui/chains/eth")) - - # ui/scripts/_migrate.sh -> ui/chains/sif/migrate.sh - # Original scripts say "if we don't sleep there are issues" - time.sleep(10) - log.info("Creating liquidity pool from catk:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) - time.sleep(5) - log.info("Creating liquidity pool from cbtk:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cbtk", [10**5, "rowan"], 10**25, 10**25) - # should now be able to swap from catk:cbtk - time.sleep(5) - log.info("Creating liquidity pool from ceth:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ceth", [10**5, "rowan"], 10**25, 83*10**20) - # should now be able to swap from x:ceth - time.sleep(5) - log.info("Creating liquidity pool from cusdc:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cusdc", [10**5, "rowan"], 10**25, 10**25) - time.sleep(5) - log.info("Creating liquidity pool from clink:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "clink", [10**5, "rowan"], 10**25, 588235*10**18) - time.sleep(5) - log.info("Creating liquidity pool from ctest:rowan...") - sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) - - # ui/scripts/_migrate.sh -> ui/chains/post_migrate.sh - - atk_address, btk_address, usdc_address, link_address = [ - self.cmd.get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json"), self.network_id) - for x in ["AliceToken", "BobToken", "UsdCoin", "LinkCoin"] - ] - - bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) - - # From smart-contracts/.env.ui.example - smart_contracts_env_ui_example_vars = { - "ETHEREUM_PRIVATE_KEY": "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3", - "INFURA_PROJECT_ID": "JFSH7439sjsdtqTM23Dz", - "LOCAL_PROVIDER": "http://localhost:7545", - } - - # NOTE: this probably doesn't work anymore since setTokenLockBurnLimit.js was replaced - burn_limits = [ - [eth.NULL_ADDRESS, 31 * 10 ** 18], - [bridge_token_address, 10 ** 25], - [atk_address, 10 ** 25], - [btk_address, 10 ** 25], - [usdc_address, 10 ** 25], - [link_address, 10 ** 25], - ] - for address, amount in burn_limits: - self.cmd.set_token_lock_burn_limit( - address, - amount, - smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"], - smart_contracts_env_ui_example_vars["INFURA_PROJECT_ID"], - smart_contracts_env_ui_example_vars["LOCAL_PROVIDER"] - ) - - # signal migrate-complete - - # Whitelist test tokens - for addr in [atk_address, btk_address, usdc_address, link_address]: - self.project.yarn(["peggy:whiteList", addr, "true"], cwd=self.project.smart_contracts_dir) - - # ui/scripts/stack-launch.sh -> ui/scripts/_peggy.sh -> ui/chains/peggy/launch.sh - # rm -rf ui/chains/peggy/relayerdb - # ebrelayer is in $GOBIN, gets installed by "make install" - ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] - ebrelayer_proc = Ebrelayer(self.cmd).init("tcp://localhost:26657", "ws://localhost:7545/", - bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, self.chain_id, - ethereum_private_key=ethereum_private_key, gas=5*10**12, gas_prices=[0.5, "rowan"]) - - # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc - # await sif-node-up and migrate-complete - - time.sleep(30) - # ui/scripts/_snapshot.sh - - # ui/scripts/stack-pause.sh: - # killall sifnoded sifnoded ebrelayer ganache-cli - sifnoded_proc.kill() - ebrelayer_proc.kill() - ganache_proc.kill() - time.sleep(10) - - snapshots_dir = project_dir("ui/chains/snapshots") - self.cmd.mkdir(snapshots_dir) # TODO self.cmd.rmdir(snapshots_dir) - # ui/chains/peggy/snapshot.sh: - # mkdir -p ui/chains/peggy/relayerdb - self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz")) - # mkdir -p smart-contracts/build - self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(snapshots_dir, "peggy_build.tar.gz")) - - # ui/chains/sif/snapshot.sh: - self.cmd.tar_create(self.sifnoded_path, os.path.join(snapshots_dir, "sif.tar.gz")) - - # ui/chains/etc/snapshot.sh: - self.cmd.tar_create(self.ganache_db_path, os.path.join(snapshots_dir, "eth.tar.gz")) - - def stack_push(self): - # ui/scripts/stack-push.sh - # $PWD=ui - - # User must be logged in to Docker hub: - # ~/.docker/config.json must exist and .auths['ghcr.io'].auth != null - log.info("Github Registry Login found.") - - commit = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "HEAD"], cwd=project_dir()))) - branch = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=project_dir()))) - - image_root = "ghcr.io/sifchain/sifnode/ui-stack" - image_name = "{}:{}".format(image_root, commit) - stable_tag = "{}:{}".format(image_root, branch.replace("/", "__")) - - running_in_ci = bool(os.environ.get("CI")) - - if running_in_ci: - res = self.cmd.execst(["git", "status", "--porcelain", "--untracked-files=no"], cwd=project_dir()) - # # reverse grep for go.mod because on CI this can be altered by installing go dependencies - # if [[ -z "$CI" && ! -z "$(git status --porcelain --untracked-files=no)" ]]; then - # echo "Git workspace must be clean to save git commit hash" - # exit 1 - # fi - pass - - log.info("Building new container...") - log.info(f"New image name: {image_name}") - - self.cmd.execst(["docker", "build", "-f", project_dir("ui/scripts/stack.Dockerfile"), "-t", image_name, "."], - cwd=project_dir(), env={"DOCKER_BUILDKIT": "1"}, pipe=False) - - if running_in_ci: - log.info(f"Tagging image as {stable_tag}...") - self.cmd.execst(["docker", "tag", image_name, stable_tag]) - self.cmd.execst(["docker", "push", image_name]) - self.cmd.execst(["docker", "push", stable_tag]) - - -class IntegrationTestsEnvironment: - def __init__(self, cmd): - self.cmd = cmd - self.project = cmd.project - # Fixed, set in start-integration-env.sh - self.ethereum_private_key = "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3" - self.owner = "0x627306090abaB3A6e1400e9345bC60c78a8BEf57" - # we may eventually switch things so PAUSER and OWNER aren't the same account, but for now they're the same - self.pauser = self.owner - # set_persistant_env_var BASEDIR $(fullpath $BASEDIR) $envexportfile - # set_persistant_env_var SIFCHAIN_BIN $BASEDIR/cmd $envexportfile - # set_persistant_env_var envexportfile $(fullpath $envexportfile) $envexportfile - # set_persistant_env_var TEST_INTEGRATION_DIR ${BASEDIR}/test/integration $envexportfile - # set_persistant_env_var TEST_INTEGRATION_PY_DIR ${BASEDIR}/test/integration/src/py $envexportfile - # set_persistant_env_var SMART_CONTRACTS_DIR ${BASEDIR}/smart-contracts $envexportfile - # set_persistant_env_var datadir ${TEST_INTEGRATION_DIR}/vagrant/data $envexportfile - # set_persistant_env_var CONTAINER_NAME integration_sifnode1_1 $envexportfile - # set_persistant_env_var NETWORKDIR $BASEDIR/deploy/networks $envexportfile - # set_persistant_env_var GANACHE_DB_DIR $(mktemp -d /tmp/ganachedb.XXXX) $envexportfile - # set_persistant_env_var ETHEREUM_WEBSOCKET_ADDRESS ws://localhost:7545/ $envexportfile - # set_persistant_env_var CHAINNET localnet $envexportfile - self.network_name = "develop" - self.network_id = 5777 - self.peruser_storage_dir = self.cmd.get_user_home(".sifnode-integration") - self.state_vars = {} - self.test_integration_dir = project_dir("test/integration") - self.data_dir = project_dir("test/integration/vagrant/data") - self.chainnet = "localnet" - self.tcp_url = f"tcp://{ANY_ADDR}:26657" - self.ethereum_websocket_address = "ws://localhost:7545/" - self.ganache_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", - "crumble", "sweet", "treat"] - - def prepare(self): - self.project.make_go_binaries() - self.project.install_smart_contracts_dependencies() - - def run(self): - self.cmd.mkdir(self.data_dir) - - self.prepare() - - log_dir = "/tmp/sifnode" - self.cmd.mkdir(log_dir) - ganache_log_file = open(os.path.join(log_dir, "ganache.log"), "w") # TODO close - sifnoded_log_file = open(os.path.join(log_dir, "sifnoded.log"), "w") # TODO close - ebrelayer_log_file = open(os.path.join(log_dir, "ebrelayer.log"), "w") # TODO close - - # test/integration/ganache-start.sh: - # 1. pkill -9 -f ganache-cli || true - # 2. while nc -z localhost 7545; do sleep 1; done - # 3. nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic \ - # 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' \ - # --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON \ - # > $GANACHE_LOG 2>&1" - # 4. sleep 5 - # 5. while ! nc -z localhost 4545; do sleep 5; done - # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt - block_time = None # TODO - account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") - ganache_db_path = self.cmd.mktempdir() - ganache_proc = Ganache.start_ganache_cli(self.cmd, block_time=block_time, host=ANY_ADDR, - mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, - account_keys_path=account_keys_path, log_file=ganache_log_file) - - self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli - time.sleep(2) - - ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) - ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] - ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] - # TODO Check for possible non-determinism of dict().keys() ordering (c.f. test/integration/vagrantenv.sh) - # TODO ebrelayer_ethereum_private_key is NOT the same as in test/integration/.env.ciExample - assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" - assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" - - env_file = project_dir("test/integration/.env.ciExample") - env_vars = self.cmd.primitive_parse_env_file(env_file) - self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, - operator=env_vars["OPERATOR"], consensus_threshold=int(env_vars["CONSENSUS_THRESHOLD"]), - initial_validator_powers=[int(x) for x in env_vars["INITIAL_VALIDATOR_POWERS"].split(",")], - initial_validator_addresses=[ebrelayer_ethereum_addr], env_file=env_file) - - bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ - self.cmd.get_bridge_smart_contract_addresses(self.network_id) - - # # TODO This should be last (after return from setup_sifchain.sh) - # burn_limits = [ - # [eth.NULL_ADDRESS, 31*10**18], - # [bridge_token_sc_addr, 10**25], - # ] - # env_file_vars = self.cmd.primitive_parse_env_file(env_file) - # for address, amount in burn_limits: - # self.cmd.set_token_lock_burn_limit( - # address, - # amount, - # env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key - # env_file_vars["INFURA_PROJECT_ID"], - # env_file_vars["LOCAL_PROVIDER"], # for web3.js to connect to ganache - # ) - - # test/integration/setup_sifchain.sh: - networks_dir = project_dir("deploy/networks") - self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it - self.cmd.mkdir(networks_dir) - # Old: - # self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) - # New: - # sifgen network create localnet 1 $NETWORKDIR 192.168.1.2 $NETWORKDIR/network-definition.yml --keyring-backend test \ - # --mint-amount 999999000000000000000000000rowan,1370000000000000000ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE - chain_id = "localnet" - validator_count = 1 - network_definition_file = os.path.join(networks_dir, "network-definition.yml") - seed_ip_address = "192.168.1.2" - mint_amount = [[999999 * 10**21, "rowan"], [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"]] - - self.cmd.sifgen_create_network(chain_id, validator_count, networks_dir, network_definition_file, seed_ip_address, mint_amount=mint_amount) - - netdef, netdef_json = self.process_netdef(network_definition_file) - - validator1_moniker = netdef["moniker"] - validator1_address = netdef["address"] - validator1_password = netdef["password"] - validator1_mnemonic = netdef["mnemonic"].split(" ") - chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator1_moniker}") - sifnoded_home = os.path.join(chaindir, ".sifnoded") - denom_whitelist_file = os.path.join(self.test_integration_dir, "whitelisted-denoms.json") - # SIFNODED_LOG=$datadir/logs/sifnoded.log - - sifnode = Sifnoded(self.cmd, home=sifnoded_home) - - adminuser_addr = self.cmd.sifchain_init_integration(sifnode, validator1_moniker, validator1_mnemonic, - denom_whitelist_file) - - # Start sifnoded - sifnoded_proc = sifnode.sifnoded_start(tcp_url=self.tcp_url, minimum_gas_prices=[0.5, "rowan"], - log_file=sifnoded_log_file) - - # TODO: wait for sifnoded to come up before continuing - # in sifchain_start_daemon.sh: "sleep 10" - # in sifchain_run_ebrelayer.sh (also run_ebrelayer here) we already wait for connection to port 26657 and sif account validator1_addr - - # Removed - # # TODO Process exits immediately with returncode 1 - # # TODO Why does it not stop start-integration-env.sh? - # # rest_server_proc = self.cmd.popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd - - # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh - # This script is also called from tests - - relayer_db_path = os.path.join(self.test_integration_dir, "sifchainrelayerdb") - ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator1_moniker, validator1_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path, log_file=ebrelayer_log_file) - - vagrantenv_path = project_dir("test/integration/vagrantenv.sh") - self.state_vars = { - "ETHEREUM_PRIVATE_KEY": self.ethereum_private_key, - "OWNER": self.owner, - "PAUSER": self.pauser, - "BASEDIR": project_dir(), - # export SIFCHAIN_BIN="/home/jurez/work/projects/sif/sifnode/local/cmd" - "envexportfile": vagrantenv_path, - # export TEST_INTEGRATION_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration" - # export TEST_INTEGRATION_PY_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration/src/py" - "SMART_CONTRACTS_DIR": self.project.smart_contracts_dir, - # export datadir="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data" - # export CONTAINER_NAME="integration_sifnode1_1" - "NETWORKDIR": networks_dir, - # export ETHEREUM_WEBSOCKET_ADDRESS="ws://localhost:7545/" - # export CHAINNET="localnet" - "GANACHE_DB_DIR": ganache_db_path, - # export GANACHE_KEYS_JSON="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data/ganachekeys.json" - "EBRELAYER_ETHEREUM_ADDR": ebrelayer_ethereum_addr, - "EBRELAYER_ETHEREUM_PRIVATE_KEY": ebrelayer_ethereum_private_key, # Needed by sifchain_run_ebrelayer.sh - # # BRIDGE_REGISTRY_ADDRESS and ETHEREUM_CONTRACT_ADDRESS are synonyms - "BRIDGE_REGISTRY_ADDRESS": bridge_registry_sc_addr, - "BRIDGE_TOKEN_ADDRESS": bridge_token_sc_addr, - "BRIDGE_BANK_ADDRESS": bridge_bank_sc_addr, - "NETDEF": os.path.join(networks_dir, "network-definition.yml"), - "NETDEF_JSON": project_dir("test/integration/vagrant/data/netdef.json"), - "MONIKER": validator1_moniker, - "VALIDATOR1_PASSWORD": validator1_password, - "VALIDATOR1_ADDR": validator1_address, - "MNEMONIC": " ".join(validator1_mnemonic), - "CHAINDIR": os.path.join(networks_dir, "validators", self.chainnet, validator1_moniker), - "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) - "EBRELAYER_DB": relayer_db_path, # Created by sifchain_run_ebrelayer.sh, does not appear to be used anywhere at the moment - } - self.project.write_vagrantenv_sh(self.state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) - - return ganache_proc, sifnoded_proc, ebrelayer_proc - - def remove_and_add_sifnoded_keys(self, moniker, mnemonic): - # Error: The specified item could not be found in the keyring - # This is not neccessary during start-integration-env.sh (as the key does not exist yet), but is neccessary - # during tests that restart ebrelayer - # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) - sifnode = Sifnoded(self.cmd) - sifnode.keys_delete(moniker) - sifnode.keys_add(moniker, mnemonic) - - def process_netdef(self, network_definition_file): - # networks_dir = deploy/networks - # File deploy/networks/network-definition.yml is created by "rake genesis:network:scaffold", specifically by - # "sifgen network create" - # We read it and convert to test/integration/vagrant/data/netdef.json - netdef = exactly_one(yaml_load(self.cmd.read_text_file(network_definition_file))) - netdef_json = os.path.join(self.data_dir, "netdef.json") - self.cmd.write_text_file(netdef_json, json.dumps(netdef, indent=4)) - return netdef, netdef_json - - def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, validator_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path, log_file=None - ): - # TODO Deduplicate - while not self.cmd.tcp_probe_connect("localhost", 26657): - time.sleep(1) - self.cmd.wait_for_sif_account(netdef_json, validator1_address) - time.sleep(10) - self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) # Creates ~/.sifnoded/keyring-tests/xxxx.address - ebrelayer_proc = Ebrelayer(self.cmd).init(self.tcp_url, self.ethereum_websocket_address, bridge_registry_sc_addr, - validator_moniker, validator_mnemonic, self.chainnet, ethereum_private_key=ebrelayer_ethereum_private_key, - node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker, - symbol_translator_file=os.path.join(self.test_integration_dir, "config/symbol_translator.json"), - relayerdb_path=relayer_db_path, cwd=self.test_integration_dir, log_file=log_file) - return ebrelayer_proc - - def create_own_dirs(self): - self.cmd.mkdir(self.peruser_storage_dir) - self.cmd.mkdir(os.path.join(self.peruser_storage_dir, "snapshots")) - - def create_snapshot(self, snapshot_name): - self.create_own_dirs() - named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) - if self.cmd.exists(named_snapshot_dir): - raise Exception(f"Directory '{named_snapshot_dir}' already exists") - self.cmd.mkdir(named_snapshot_dir) - self.cmd.tar_create(self.state_vars["GANACHE_DB_DIR"], os.path.join(named_snapshot_dir, "ganache.tar.gz")) - self.cmd.tar_create(self.state_vars["EBRELAYER_DB"], os.path.join(named_snapshot_dir, "sifchainrelayerdb.tar.gz")) - self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz")) - self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz")) - self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), os.path.join(named_snapshot_dir, "sifnoded.tar.gz")) - self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) - - def restore_snapshot(self, snapshot_name): - named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) - state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) - - def extract(tarfile, path): - self.cmd.rmdir(path) - self.cmd.mkdir(path) - self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) - - ganache_db_dir = self.cmd.mktempdir() - relayer_db_path = state_vars["EBRELAYER_DB"] # TODO use /tmp - assert os.path.realpath(relayer_db_path) == os.path.realpath(os.path.join(self.test_integration_dir, "sifchainrelayerdb")) - extract("ganache.tar.gz", ganache_db_dir) - extract("sifchainrelayerdb.tar.gz", relayer_db_path) - deploy_networks_dir = project_dir("deploy/networks") - extract("networks.tar.gz", deploy_networks_dir) - smart_contracts_build_dir = project_dir("smart-contracts/build") - extract("smart-contracts.tar.gz", smart_contracts_build_dir) - extract("sifnoded.tar.gz", self.cmd.get_user_home(".sifnoded")) # Needed for "--keyring-backend test" - - state_vars["GANACHE_DB_DIR"] = ganache_db_dir - state_vars["EBRELAYER_DB"] = relayer_db_path - self.state_vars = state_vars - self.project.write_vagrantenv_sh(state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) - self.cmd.mkdir(self.data_dir) - - def restart_processes(self): - block_time = None - ganache_db_path = self.state_vars["GANACHE_DB_DIR"] - account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") # TODO this is in test/integration/vagrant/data, which is supposed to be cleared - - ganache_proc = Ganache.start_ganache_cli(self.cmd, block_time=block_time, host=ANY_ADDR, - mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, - account_keys_path=account_keys_path) # TODO log_file - - self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli - time.sleep(2) - - validator_moniker = self.state_vars["MONIKER"] - networks_dir = project_dir("deploy/networks") - chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") - sifnoded_home = os.path.join(chaindir, ".sifnoded") - sifnoded_proc = self.cmd.sifnoded_start(tcp_url=self.tcp_url, minimum_gas_prices=[0.5, "rowan"], sifnoded_home=sifnoded_home) - - bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ - self.cmd.get_bridge_smart_contract_addresses(self.network_id) - - validator_mnemonic = self.state_vars["MNEMONIC"].split(" ") - account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") - ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) - ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] - ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] - network_definition_file = project_dir(networks_dir, "network-definition.yml") - - netdef, netdef_json = self.process_netdef(network_definition_file) - validator1_address = netdef["address"] - assert validator1_address == self.state_vars["VALIDATOR1_ADDR"] - relayer_db_path = self.state_vars["EBRELAYER_DB"] - ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, - ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path) - - return ganache_proc, sifnoded_proc, ebrelayer_proc - - -class Peggy2Environment(IntegrationTestsEnvironment): - def __init__(self, cmd): - super().__init__(cmd) - self.hardhat = hardhat.Hardhat(cmd) - - # Destuctures a linear array of EVM accounts into: - # [operator, owner, pauser, [validator-0, validator-1, ...], [...available...]] - # proxy_admin is the same as operator. - def signer_array_to_ethereum_accounts(self, accounts, n_validators): - assert len(accounts) >= n_validators + 3 - operator, owner, pauser, *rest = accounts # Take 3 and store remaining in rest - validators, available = rest[:n_validators], rest[n_validators:] # Take n_validators for validators the remaining for available - return { - "proxy_admin": operator, - "operator": operator, - "owner": owner, - "pauser": pauser, - "validators": validators, - "available": available, - } - - # Override - def run(self): - # self.project._make_go_binaries() - - # Ordering (for possible parallelisation): - # - build_golang_binaries before start_sifchain - # - start_hardhat before deploy_smart_contracts - # - start_sifchain before start_witnesses_and_relayers - # - deploy_smart_contracts before start_witnesses_and_relayers - # - start_witnesses_and_relayers before return - # - write_env_file before return - - # TODO: where is log watcher? - - log_dir = "/tmp/sifnode" - self.cmd.mkdir(log_dir) - hardhat_log_file = open(os.path.join(log_dir, "hardhat.log"), "w") # TODO close + use a different name - sifnoded_log_file = open(os.path.join(log_dir, "sifnoded.log"), "w") # TODO close - relayer_log_file = open(os.path.join(log_dir, "relayer.log"), "w") # TODO close - witness_log_file = open(os.path.join(log_dir, "witness.log"), "w") # TODO close; will be empty on non-peggy2 branch - - self.cmd.rmdir(self.cmd.get_user_home(".sifnoded")) # Purge test keyring backend - - hardhat_bind_hostname = "localhost" # The host to which to bind to for new connections (Defaults to 127.0.0.1 running locally, and 0.0.0.0 in Docker) - hardhat_port = 8545 # The port on which to listen for new connections (default: 8545) - hardhat_exec_args = self.hardhat.build_start_args(hostname=hardhat_bind_hostname, port=hardhat_port) - hardhat_proc = self.cmd.spawn_asynchronous_process(hardhat_exec_args, log_file=hardhat_log_file) - - # This determines how many EVM accounts we want to allocate for validators. - # Since every validator needs on EVM account, this should be equal to the number of validators (possibly more). - hardhat_validator_count = 1 - hardhat_network_id = 1 # Not used in smart-contracts/src/devenv/hardhatNode.ts - # This value is actually returned from HardhatNodeRunner. It comes from smart-contracts/hardhat.config.ts. - # In Typescript, its value is obtained by 'require("hardhat").hre.network.config.chainId'. - # See https://hardhat.org/advanced/hardhat-runtime-environment.html - # The value is not used; instead a hardcoded constant 31337 is passed to ebrelayerWitnessBuilder. - # Ask juniuszhou for details. - hardhat_chain_id = 31337 - hardhat_accounts = self.signer_array_to_ethereum_accounts(hardhat.default_accounts(), hardhat_validator_count) - - self.hardhat.compile_smart_contracts() - peggy_sc_addrs = self.hardhat.deploy_smart_contracts() - - # Initialization of smart contracts (technically this is part of deployment) - operator_acct = hardhat_accounts["operator"] - w3_websocket_address = eth.web3_host_port_url("localhost", hardhat_port) - self.init_smart_contracts(w3_websocket_address, operator_acct, peggy_sc_addrs) - - admin_account_name = "sifnodeadmin" - chain_id = "localnet" - ceth_symbol = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) - assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" - # Mint goes to validator - mint_amount = [ - [999999 * 10**21, "rowan"], - [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"], - [999999 * 10**21, ceth_symbol], - ] - validator_power = 100 - seed_ip_address = "10.10.1.1" - tendermint_port = 26657 - denom_whitelist_file = project_dir("test", "integration", "whitelisted-denoms.json") - tokens = [ - [10**20, "rowan"], - [2 * 10**19, "ceth"] - ] - registry_json = project_dir("smart-contracts", "src", "devenv", "registry.json") - sifnoded_network_dir = "/tmp/sifnodedNetwork" # Gets written to .vscode/launch.json - self.cmd.rmdir(sifnoded_network_dir) - self.cmd.mkdir(sifnoded_network_dir) - network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, sifnode_validators, \ - sifnode_relayers, sifnode_witnesses, sifnode_validator0_home, chain_dir = \ - self.init_sifchain(sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, - validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, - admin_account_name) - - symbol_translator_file = os.path.join(self.test_integration_dir, "config", "symbol_translator.json") - [relayer0_exec_args], [witness0_exec_args] = \ - self.start_witnesses_and_relayers(w3_websocket_address, hardhat_chain_id, tcp_url, - chain_id, peggy_sc_addrs, hardhat_accounts["validators"], sifnode_validators, sifnode_relayers, - sifnode_witnesses, symbol_translator_file) - - relayer0_proc = self.cmd.spawn_asynchronous_process(relayer0_exec_args, log_file=relayer_log_file) - witness0_proc = self.cmd.spawn_asynchronous_process(witness0_exec_args, log_file=witness_log_file) - - # In the future, we want to have one descriptor for entire environment. - # It should be able to support multiple EVM and multiple Cosmos chains, including all neccessary bridges and - # relayers. For now this is just a prototype which is not used yet. - _unused_peggy2_environment = { - "admin": { - "name": admin_account_name, - "address": admin_account_address, - "home": sifnode_validator0_home, - }, - "symbol_translator_file": symbol_translator_file, - "w3_websocket_address": w3_websocket_address, - "evm_chain_id": hardhat_chain_id, - "chain_id": chain_id, - "validators": sifnode_validators, # From yaml file generated by sifgen - "relayers": sifnode_relayers, - "smart_contracts": peggy_sc_addrs - } - - self.write_env_files(self.project.project_dir(), self.project.go_bin_dir, peggy_sc_addrs, hardhat_accounts, - admin_account_name, admin_account_address, sifnode_validator0_home, sifnode_validators, sifnode_relayers, - sifnode_witnesses, tcp_url, hardhat_bind_hostname, hardhat_port, hardhat_chain_id, chain_dir, - sifnoded_exec_args, relayer0_exec_args, witness0_exec_args - ) - - return hardhat_proc, sifnoded_proc, relayer0_proc, witness0_proc - - def init_smart_contracts(self, w3_url, operator_account, deployed_contract_addresses): - # Looks like this is already done somewhere else... - # operator_addr, operator_private_key = operator_account - # w3_conn = eth.web3_wait_for_connection_up(w3_url) - # eth_tx = eth.EthereumTxWrapper(w3_conn, True) - # eth_tx.set_private_key(operator_addr, operator_private_key) - # - # # CosmosBridge doesn't have BridgeBank in its init and expects a separate setBridgeBank call. CosmosBridge - # # doesn't really work without BridgeBank. - # abi_provider = hardhat.HardhatAbiProvider(self.cmd, deployed_contract_addresses) - # abi, _, deployed_address = abi_provider.get_descriptor("CosmosBridge") - # cosmos_bridge = w3_conn.eth.contract(abi=abi, address=deployed_address) - # bridge_bank_addr = deployed_contract_addresses["BridgeBank"] - # txrcpt = eth_tx.transact_sync(cosmos_bridge.functions.setBridgeBank, operator_addr)(bridge_bank_addr) - return - - def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, - validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, - admin_account_name - ): - validator_count = 1 - relayer_count = 1 - witness_count = 1 - # TODO Not used - # rpc_port = 9000 - - network_config_file_path = self.cmd.mktempfile() - try: - self.cmd.sifgen_create_network(chain_id, validator_count, sifnoded_network_dir, network_config_file_path, - seed_ip_address, mint_amount=mint_amount) - network_config_file = self.cmd.read_text_file(network_config_file_path) - finally: - self.cmd.rm(network_config_file_path) - validators = yaml_load(network_config_file) - - # netdef_yml is a list of generated validators like below. - # Each one has its unique IP (starting with base IP + 1), the first one also has is_seed=True. - # - # class ValidatorValues: - # chain_id: str - # node_id: str - # ipv4_address: str - # moniker: str - # password: str - # address: str - # pub_key: str - # mnemonic: str - # validator_address: str - # validator_consensus_address: str - # is_seed: bool - assert len(validators) == validator_count - - chain_dir_base = os.path.join(sifnoded_network_dir, "validators", chain_id) - - for validator in validators: - validator_moniker = validator["moniker"] - validator_mnemonic = validator["mnemonic"].split(" ") - # TODO Not used - # validator_password = validator["password"] - evm_network_descriptor = 1 # TODO Why not hardhat_chain_id? - sifnoded_home = os.path.join(chain_dir_base, validator_moniker, ".sifnoded") - sifnode = Sifnoded(self.cmd, home=sifnoded_home) - self.cmd.sifnoded_peggy2_init_validator(sifnode, validator_moniker, validator_mnemonic, evm_network_descriptor, validator_power, chain_dir_base) - - # TODO Needs to be fixed when we support more than 1 validator - validator0 = exactly_one(validators) - validator0_home = os.path.join(chain_dir_base, validator0["moniker"], ".sifnoded") - validator0_address = validator0["address"] - chain_dir = os.path.join(chain_dir_base, validator0["moniker"]) - - sifnode = Sifnoded(self.cmd, home=validator0_home) - - # Create an ADMIN account on sifnode with name admin_account_name (e.g. "sifnodeadmin") - admin_account_address = sifnode.peggy2_add_account(admin_account_name, tokens, is_admin=True) - - # TODO Check if sifnoded_peggy2_add_relayer_witness_account can be executed offline (without sifnoded running) - # TODO Check if sifnoded_peggy2_set_cross_chain_fee can be executed offline (without sifnoded running) - - # Create an account for each relayer - # Note: "--home" is shared with sifnoded's "--home" - relayers = [{ - "name": name, - "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, - validator_power, denom_whitelist_file), - "home": validator0_home, - } for name in [f"relayer-{i}" for i in range(relayer_count)]] - - # Create an account for each witness - # Note: "--home" is shared with sifnoded's "--home" - witnesses = [{ - "name": name, - "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, - validator_power, denom_whitelist_file), - "home": validator0_home, - } for name in [f"witness-{i}" for i in range(witness_count)]] - - tcp_url = "tcp://{}:{}".format(ANY_ADDR, tendermint_port) - # sifnoded - # start - # --log_level debug - # --log_format json - # --minimum-gas-prices 0.5rowan - # --rpc.laddr tcp://0.0.0.0:26657 - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded - # @TODO Detect if sifnoded is already running, for now it fails silently and we wait forever in wait_for_sif_account_up - sifnoded_exec_args = sifnode.build_start_cmd(tcp_url=tcp_url, minimum_gas_prices=[0.5, "rowan"], - log_format_json=True) - sifnoded_proc = self.cmd.spawn_asynchronous_process(sifnoded_exec_args, log_file=sifnoded_log_file) - - self.cmd.wait_for_sif_account_up(validator0_address, tcp_url) - - # TODO This command exits with status 0, but looks like there are some errros. - # The same happens also in devenv. - # TODO Try whitelister account instead of admin - res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_address, - chain_id) - log.debug("Result from token registry: {}".format(repr(res))) - assert len(res) == 2 - assert res[0]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" - assert res[1]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" - - cross_chain_fee_base = 1 - cross_chain_lock_fee = 1 - cross_chain_burn_fee = 1 - ethereum_cross_chain_fee_token = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) - gas_prices = [0.5, "rowan"] - gas_adjustment = 1.5 - sifnode.peggy2_set_cross_chain_fee(admin_account_address, hardhat_chain_id, - ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, - admin_account_name, chain_id, gas_prices, gas_adjustment) - - sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) - - return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ - relayers, witnesses, validator0_home, chain_dir - - def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, tcp_url, chain_id, peggy_sc_addrs, - evm_validator_accounts, sifnode_validators, sifnode_relayers, sifnode_witnesses, symbol_translator_file - ): - # For now we assume a single validator - evm_validator0_addr, evm_validator0_key = exactly_one(evm_validator_accounts) - - sifnode_validator0 = exactly_one(sifnode_validators) - sifnode_validator0_address = sifnode_validator0["address"] - - sifnode_relayer0 = exactly_one(sifnode_relayers) - sifnode_relayer0_mnemonic = sifnode_relayer0["name"] - sifnode_relayer0_address = sifnode_relayer0["address"] - sifnode_relayer0_home = sifnode_relayer0["home"] - - sifnode_witness0 = exactly_one(sifnode_witnesses) - sifnode_witness0_mnemonic = sifnode_witness0["name"] - sifnode_witness0_address = sifnode_witness0["address"] - sifnode_witness0_home = sifnode_witness0["home"] - - bridge_registry_contract_addr = peggy_sc_addrs.bridge_registry - # bridge_bank_contract_addr = peggy_sc_addrs.bridge_bank - # cosmos_bridge_contract_addr = peggy_sc_addrs.cosmos_bridge - # rowan_contract_addr = peggy_sc_addrs.rowan - - self.cmd.wait_for_sif_account_up(sifnode_validator0_address, tcp_url=tcp_url) # Required for both relayer and witness - - ebrelayer = Ebrelayer(self.cmd) - - # Example: - # ebrelayer - # init-relayer - # --network-descriptor 31337 - # --tendermint-node tcp://0.0.0.0:26657 - # --web3-provider ws://localhost:8545/ - # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e - # --validator-mnemonic relayer-0 - # --chain-id localnet - # --node tcp://0.0.0.0:26657 - # --from sif1a44w20496lgyv5asx4d4fnekdpy9xg8ymy9k3s - # --symbol-translator-file ../test/integration/config/symbol_translator.json - # --keyring-backend test - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded - # env: - # "ETHEREUM_ADDRESS": evm_accounts["validators"][0] - # "ETHEREUM_PRIVATE_KEY": evm_account["validators"][1] - relayer0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( - "init-relayer", - hardhat_chain_id, - tcp_url, - web3_websocket_address, - bridge_registry_contract_addr, - sifnode_relayer0_mnemonic, - chain_id=chain_id, - node=tcp_url, - sign_with=sifnode_relayer0_address, - symbol_translator_file=symbol_translator_file, - ethereum_address=evm_validator0_addr, - ethereum_private_key=evm_validator0_key, - keyring_backend="test", - home=sifnode_relayer0_home, - ) - - # Example from devenv: - # ebrelayer - # init-witness - # --network-descriptor 31337 - # --tendermint-node tcp://0.0.0.0:26657 - # --web3-provider ws://localhost:8545/ - # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e - # --validator-mnemonic witness-0 - # --chain-id localnet - # --node tcp://0.0.0.0:26657 - # --from sif1l7025ps7lt24effpduwxhk45sd977djvu38lhr - # --symbol-translator-file ../test/integration/config/symbol_translator.json - # --log_format json - # --keyring-backend test - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded - witness0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( - "init-witness", - hardhat_chain_id, - tcp_url, - web3_websocket_address, - bridge_registry_contract_addr, - sifnode_witness0_mnemonic, - chain_id=chain_id, - node=tcp_url, - sign_with=sifnode_witness0_address, - symbol_translator_file=symbol_translator_file, - ethereum_address=evm_validator0_addr, - ethereum_private_key=evm_validator0_key, - keyring_backend="test", - log_format="json", - home=sifnode_witness0_home, - ) - - return [relayer0_exec_args], [witness0_exec_args] - - def write_env_files(self, project_dir, go_bin_dir, evm_smart_contract_addrs, eth_accounts, admin_account_name, - admin_account_address, sifnode_validator0_home, sifnode_validators, sifnode_relayers, sifnode_witnesses, - tcp_url, hardhat_bind_hostname, hardhat_port, hardhat_chain_id, chain_dir, sifnoded_exec_args, - relayer0_exec_args, witness0_exec_args - ): - eth_chain_id = hardhat_chain_id - w3_url = eth.web3_host_port_url(hardhat_bind_hostname, hardhat_port) - - # @TODO At the moment, values are fed from one rendered template into the next. - # We should use values directly from parameters instead. - - def format_eth_account(eth_account): - assert eth_account[0].startswith("0x") and not eth_account[1].startswith("0x") - return {"address": eth_account[0], "privateKey": "0x" + eth_account[1]} - - def format_sif_account(sif_account): - return {"account": sif_account["address"], "name": sif_account["name"], "homeDir": sif_account["home"]} - - environment_json = { - "contractResults": { - # "completed": True, - # "output": "...", - "contractAddresses": { - "cosmosBridge": evm_smart_contract_addrs["CosmosBridge"], - "bridgeBank": evm_smart_contract_addrs["BridgeBank"], - "bridgeRegistry": evm_smart_contract_addrs["BridgeRegistry"], - "rowanContract": evm_smart_contract_addrs["Rowan"], - } - }, - "ethResults": { - # "process": { ... }, - "accounts": { - "proxyAdmin": format_eth_account(eth_accounts["proxy_admin"]), - "operator": format_eth_account(eth_accounts["operator"]), - "owner": format_eth_account(eth_accounts["owner"]), - "pauser": format_eth_account(eth_accounts["pauser"]), - "validators": [format_eth_account(a) for a in eth_accounts["validators"]], - "available": [format_eth_account(a) for a in eth_accounts["available"]], - }, - "httpHost": hardhat_bind_hostname, - "httpPort": hardhat_port, - "chainId": eth_chain_id, - }, - "goResults": { - # "completed": True, - # "output": "...", - "goBin": go_bin_dir - }, - "sifResults": { - "validatorValues": sifnode_validators, - "adminAddress": format_sif_account({"address": admin_account_address, "name": admin_account_name, "home": sifnode_validator0_home}), - "relayerAddresses": [format_sif_account(a) for a in sifnode_relayers], - "witnessAddresses": [format_sif_account(a) for a in sifnode_witnesses], - # "process": { ... }, - "tcpurl": tcp_url, - } - } - - # TODO Do we want "0x" prefixes here for private keys? - dot_env = dict_merge({ - "BASEDIR": project_dir, - "ETHEREUM_ADDRESS": eth_accounts["available"][0][0], - "ETHEREUM_PRIVATE_KEY": "0x" + eth_accounts["available"][0][1], - "ETH_ACCOUNT_OPERATOR_ADDRESS": eth_accounts["operator"][0], - "ETH_ACCOUNT_OPERATOR_PRIVATEKEY": "0x" + eth_accounts["operator"][1], - "ETH_ACCOUNT_OWNER_ADDRESS": eth_accounts["owner"][0], - "ETH_ACCOUNT_OWNER_PRIVATEKEY": "0x" + eth_accounts["owner"][1], - "ETH_ACCOUNT_PAUSER_ADDRESS": eth_accounts["pauser"][0], - "ETH_ACCOUNT_PAUSER_PRIVATEKEY": "0x" + eth_accounts["pauser"][1], - "ETH_ACCOUNT_PROXYADMIN_ADDRESS": eth_accounts["proxy_admin"][0], - "ETH_ACCOUNT_PROXYADMIN_PRIVATEKEY": "0x" + eth_accounts["proxy_admin"][1], - "ETH_ACCOUNT_VALIDATOR_ADDRESS": eth_accounts["validators"][0][0], - "ETH_ACCOUNT_VALIDATOR_PRIVATEKEY": "0x" + eth_accounts["validators"][0][1], - "ETH_CHAIN_ID": str(eth_chain_id), - "ETH_HOST": hardhat_bind_hostname, - "ETH_PORT": str(hardhat_port), - "ROWAN_SOURCE": admin_account_address, - "BRIDGE_BANK_ADDRESS": evm_smart_contract_addrs.bridge_bank, - # "BRIDGE_REGISTRY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], - "BRIDGE_REGISTERY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], # TODO Typo, remove, keeping it for now for compatibility - "COSMOS_BRIDGE_ADDRESS": evm_smart_contract_addrs["CosmosBridge"], - "ROWANTOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], - "BRIDGE_TOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], - "GOBIN": go_bin_dir, - "TCP_URL": tcp_url, - "VALIDATOR_ADDRESS": sifnode_validators[0]["address"], - "VALIDATOR_CONSENSUS_ADDRESS": sifnode_validators[0]["validator_consensus_address"], - "VALIDATOR_MENOMONIC": sifnode_validators[0]["mnemonic"], - "VALIDATOR_MONIKER": sifnode_validators[0]["moniker"], - "VALIDATOR_PASSWORD": sifnode_validators[0]["password"], - "VALIDATOR_PUB_KEY": sifnode_validators[0]["pub_key"], - "ADMIN_ADDRESS": admin_account_address, - "ADMIN_NAME": admin_account_name, - "CHAINDIR": chain_dir, - "HOME": chain_dir, - }, *[{ - f"ETHEREUM_ADDRESS_{i}": account[0], - f"ETHEREUM_PRIVATE_KEY_{i}": "0x" + account[1], - } for i, account in enumerate(eth_accounts["available"])], *[{ - f"ETH_ACCOUNT_VALIDATOR_{i}_ADDRESS": account[0], - f"ETH_ACCOUNT_VALIDATOR_{i}_PRIVATEKEY": "0x" + account[1], - } for i, account in enumerate(eth_accounts["validators"])], *[{ - f"VALIDATOR_ADDRESS_{i}": validator["address"], - f"VALIDATOR_CONSENSUS_ADDRESS_{i}": validator["validator_consensus_address"], - f"VALIDATOR_MENOMONIC_{i}": validator["mnemonic"], - f"VALIDATOR_MONIKER_{i}": validator["moniker"], - f"VALIDATOR_PASSWORD_{i}": validator["password"], - f"VALIDATOR_PUB_KEY_{i}": validator["pub_key"], - } for i, validator in enumerate(sifnode_validators)], *[{ - f"RELAYER_ADDRESS_{i}": relayer["address"], - f"RELAYER_NAME_{i}": relayer["name"], - } for i, relayer in enumerate(sifnode_relayers)], *[{ - f"WITNESS_ADDRESS_{i}": witness["address"], - f"WITNESS_NAME_{i}": witness["name"], - } for i, witness in enumerate(sifnode_witnesses)]) - - # launch.json for VS Code - launch_json = { - "version": "0.2.0", - "configurations": [ - { - "runtimeArgs": ["node_modules/.bin/hardhat", "run"], - "cwd": "${workspaceFolder}/smart-contracts", - "type": "node", - "request": "launch", - "name": "Debug DevENV scripts", - "skipFiles": ["/**"], - "program": "${workspaceFolder}/smart-contracts/scripts/devenv.ts", - }, { - "runtimeArgs": ["node_modules/.bin/hardhat", "test"], - "args": ["--network", "localhost"], - "cwd": "${workspaceFolder}/smart-contracts", - "type": "node", - "request": "launch", - "name": "Typescript Tests", - "skipFiles": ["/**"], - "program": "${workspaceFolder}/smart-contracts/test/watcher/watcher.ts", - }, *[{ - "name": f"Debug Relayer-{i}", - "type": "go", - "request": "launch", - "mode": "debug", - "program": "cmd/ebrelayer", - "envFile": "${workspaceFolder}/smart-contracts/.env", - # Generally we want to use relayer0_exec_args, except for: - # - here we don't have the initial "ebrelayer" - # - here we are using "${workspaceFolder} for" "--symbol-translator-file" - # - here we don't have ETHEREUM_ADDRESS env - # TODO Probable bug, should be "eth_accounts["validators"][0][1]}" - "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["available"][i][1]}, - # "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["validators"][0][1]}, - "args": [ - "init-relayer", - "--network-descriptor", str(eth_chain_id), - "--tendermint-node", tcp_url, - "--web3-provider", w3_url, - "--bridge-registry-contract-address", evm_smart_contract_addrs["BridgeRegistry"], - "--validator-mnemonic", relayer["name"], - "--chain-id", "localnet", - "--node", tcp_url, - "--keyring-backend", "test", - "--from", relayer["address"], - "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", - "--home", relayer["home"] - ] - } for i, relayer in enumerate(sifnode_relayers)], *[{ - "name": f"Debug Witness-{i}", - "type": "go", - "request": "launch", - "mode": "debug", - "program": "cmd/ebrelayer", - "envFile": "${workspaceFolder}/smart-contracts/.env", - # Generally we want to use witness0_exec_args, except for: - # - here we don't have the initial "ebrelayer" - # - here we are using "${workspaceFolder} for" "--symbol-translator-file" - # - here we don't have ETHEREUM_ADDRESS env - # TODO Probable bug, should be "eth_accounts["validators"][0][1]}" - "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["available"][i][1]}, - # "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["validators"][0][1]}, - "args": [ - "init-witness", - # TODO This is probably obsolete, need "--network-descriptor" etc. - str(eth_chain_id), - tcp_url, - w3_url, - evm_smart_contract_addrs["BridgeRegistry"], - witness["name"], - "--chain-id", "localnet", - "--node", tcp_url, - "--keyring-backend", "test", - "--from", witness["address"], - "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", - "--home", witness["home"] - ] - } for i, witness in enumerate(sifnode_witnesses)], { - "name": "Debug Sifnoded", - "type": "go", - "request": "launch", - "mode": "debug", - "program": "cmd/sifnoded", - # Generally we want to use sifnoded_exec_args, except for: - # - here we don't have the initial "sifnoded" - # TODO Do not use .env file here - "envFile": "${workspaceFolder}/smart-contracts/.env", - "args": [ - "start", - "--log_format", "json", - "--log_level", "debug", - "--minimum-gas-prices", "0.5rowan", - "--rpc.laddr", tcp_url, - "--home", sifnode_validator0_home - ] - }, { - "name": "Pytest", - "type": "python", - "request": "launch", - "stopOnEntry": False, - "python": "${command:python.interpreterPath}", - "module": "pytest", - "args": [ - "-olog_cli=false", - "-olog_level=DEBUG", - "-olog_file=/tmp/pytest.out", - "-v", - "test/integration/src/py/test_eth_transfers.py::test_eth_to_ceth_and_back_to_eth" - ], - "cwd": "${workspaceRoot}", - "env": dot_env, - "debugOptions": ["WaitOnAbnormalExit", "WaitOnNormalExit", "RedirectOutput"] - } - ] - } - - # IntelliJ - def render_intellij_run_xml(name, joined_args, package, filepath, envs): - def q(s): return s # TODO Qoute/escape - - # since the contents is being fed from launch_json, we have ${workspaceFolder} in joined_args - joined_args = re.sub("\\${workspaceFolder}/", "", joined_args) - - return [ - "", - " ", - " ", - " ", - " ", - ] + (( - [" "] + - [" " for k, v in envs.items()] + - [" "] - ) if envs else []) + [ - " ", - " ", - " ", - " ", - " ", - " ", - "", - ] - - intellij_sifnoded_configs = [] - intellij_ebrelayer_configs = [] - intellij_witness_configs = [] - for config in launch_json["configurations"]: - if config["name"].startswith("Debug Relayer"): - intellij_ebrelayer_configs.append(render_intellij_run_xml( - "ebrelayer devenv", - " ".join(config["args"]), - "github.com/Sifchain/sifnode/cmd/ebrelayer", - "$PROJECT_DIR$/cmd/ebrelayer/main.go", - {"ETHEREUM_PRIVATE_KEY": dot_env["ETHEREUM_PRIVATE_KEY"]})) - elif config["name"].startswith("Debug Witness"): - intellij_witness_configs.append(render_intellij_run_xml( - "witness devenv", - " ".join(config["args"]), - "github.com/Sifchain/sifnode/cmd/ebrelayer", - "$PROJECT_DIR$/cmd/ebrelayer/main.go", - {"ETHEREUM_PRIVATE_KEY": dot_env["ETHEREUM_PRIVATE_KEY"]})) - elif config["name"].startswith("Debug Sifnoded"): - intellij_sifnoded_configs.append(render_intellij_run_xml( - "sifnoded devenv", - " ".join(config["args"]), - "github.com/Sifchain/sifnode/cmd/sifnoded", - "$PROJECT_DIR$/cmd/sifnoded/main.go", - {})) - - intellij_ebrelayer_config = exactly_one(intellij_ebrelayer_configs) - intellij_witness_config = exactly_one(intellij_witness_configs) - intellij_sifnoded_config = exactly_one(intellij_sifnoded_configs) - - run_dir = self.project.project_dir(".run") - self.cmd.mkdir(run_dir) - vscode_dir = self.project.project_dir(".vscode") - self.cmd.mkdir(vscode_dir) - - self.cmd.write_text_file(self.project.project_dir("smart-contracts/environment.json"), json.dumps(environment_json, indent=2)) - self.cmd.write_text_file(self.project.project_dir("smart-contracts/env.json"), json.dumps(dot_env, indent=2)) - self.cmd.write_text_file(self.project.project_dir("smart-contracts/.env"), joinlines(format_as_shell_env_vars(dot_env, export=True))) - self.cmd.write_text_file(os.path.join(vscode_dir, "launch.json"), json.dumps(launch_json, indent=2)) - self.cmd.write_text_file(os.path.join(run_dir, "ebrelayer.run.xml"), joinlines(intellij_ebrelayer_config)) - self.cmd.write_text_file(os.path.join(run_dir, "witness.run.xml"), joinlines(intellij_witness_config)) - self.cmd.write_text_file(os.path.join(run_dir, "sifnoded.run.xml"), joinlines(intellij_sifnoded_config)) - - return environment_json, dot_env, launch_json, intellij_ebrelayer_config, intellij_witness_config, intellij_sifnoded_config - -class IBCEnvironment(IntegrationTestsEnvironment): - def __init__(self, cmd): - super().__init__(cmd) - - def run(self): - chainnet0 = "sifchain-ibc-0" - chainnet1 = "sifchain-ibc-1" - ipaddr0 = "192.168.65.2" - ipaddr1 = "192.168.65.3" - subnet = "192.168.65.1/24" - # Mnemonics can be generated with "sifgen key generate" or "sifnoded keys mnemonic", but that gives us 24 words - # and here are only 12. - # A mnemonic contains both public and private key. Public key is the address, there can only be one such entry - # in the keyring. - mnemonic = "toddler spike waste purpose neutral beach science dawn joke stock help beyond" - - sifgen = Sifgen(self.cmd) - # This does not work - "--keyring-backend" is not supported - x = sifgen.create_standalone(chainnet0, "chain1", mnemonic, ipaddr0, keyring_backend=None) - - print() - - def main(argv): # tmux usage: # tmux new-session -d -s env1 diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py new file mode 100644 index 0000000000..e7ce379651 --- /dev/null +++ b/test/integration/framework/run_env.py @@ -0,0 +1,1453 @@ +import json +import re +import time + +import eth +import hardhat +from truffle import Ganache +from command import Command +from sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash +from project import Project, killall, force_kill_processes +from common import * + + +class Integrator(Ganache, Command): + def __init__(self): + super().__init__() # TODO Which super is this? All of them? + self.project = Project(self, project_dir()) + + def primitive_parse_env_file(self, path): + def split(lines): + result = dict() + for line in lines: + m = patt.match(line) + result[m[1]] = m[2] + return result + + tmp1 = self.mktempfile() + tmp2 = self.mktempfile() + try: + self.execst(["env", "-i", "bash", "-c", "set -o posix; IFS=' '; set > {}; source {}; set > {}".format(tmp1, path, tmp2)]) + t1 = self.read_text_file(tmp1).splitlines() + t2 = self.read_text_file(tmp2).splitlines() + finally: + self.rm(tmp1) + self.rm(tmp2) + patt = re.compile("^(.*?)=(.*)$") + d1 = split(t1) + d2 = split(t2) + result = dict() + for k, v in d2.items(): + if (k in d1) and (d1[k] == d2[k]): + continue + if k in ["_", "BASH_ARGC"]: + continue + result[k] = v + return result + + def _check_env_vs_file(self, env, env_path): + if (not self.exists(env_path)) or (env is None): + return + fenv = self.primitive_parse_env_file(env_path) + for k, v in env.items(): + if k in fenv: + if env[k] == fenv[k]: + log.warning(f"Variable '{k}' specified both as a parameter and in '{env_path}'") + else: + log.warning(f"Variable '{k}' has different values, parameter: {env[k]}, in '{env_path}': " + f"{fenv[k]}. According to observation, value from parameter will be used.") + + def deploy_smart_contracts_for_integration_tests(self, network_name, consensus_threshold=None, operator=None, + owner=None, initial_validator_addresses=None, initial_validator_powers=None, pauser=None, + mainnet_gas_price=None, env_file=None + ): + env = {} + if consensus_threshold is not None: # required + env["CONSENSUS_THRESHOLD"] = str(consensus_threshold) + if operator is not None: # required + env["OPERATOR"] = operator + if owner is not None: # required + env["OWNER"] = owner + if initial_validator_addresses is not None: + env["INITIAL_VALIDATOR_ADDRESSES"] = ",".join(initial_validator_addresses) + if initial_validator_powers is not None: + env["INITIAL_VALIDATOR_POWERS"] = ",".join([str(x) for x in initial_validator_powers]) + if pauser is not None: + env["PAUSER"] = pauser + if mainnet_gas_price is not None: + env["MAINNET_GAS_PRICE"] = mainnet_gas_price + + env_path = os.path.join(self.project.smart_contracts_dir, ".env") + if env_file is not None: + self.copy_file(env_file, env_path) + + self._check_env_vs_file(env, env_path) + + # TODO ui scripts use just "yarn; yarn migrate" alias "npx truffle migrate --reset", + self.project.npx(["truffle", "deploy", "--network", network_name, "--reset"], env=env, + cwd=self.project.smart_contracts_dir, pipe=False) + + def deploy_smart_contracts_for_ui_stack(self): + self.copy_file(os.path.join(self.project.smart_contracts_dir, ".env.ui.example"), + os.path.join(self.project.smart_contracts_dir, ".env")) + # TODO Might not be neccessary + self.project.yarn([], cwd=self.project.smart_contracts_dir) + self.project.yarn(["migrate"], cwd=self.project.smart_contracts_dir) + + # truffle + def get_smart_contract_address(self, compiled_json_path, network_id): + return json.loads(self.read_text_file(compiled_json_path))["networks"][str(network_id)]["address"] + + # truffle + def get_bridge_smart_contract_addresses(self, network_id): + return [self.get_smart_contract_address(os.path.join( + self.project.smart_contracts_dir, f"build/contracts/{x}.json"), network_id) + for x in ["BridgeToken", "BridgeRegistry", "BridgeBank"]] + + def truffle_exec(self, script_name, *script_args, env=None): + self._check_env_vs_file(env, os.path.join(self.project.smart_contracts_dir, ".env")) + script_path = os.path.join(self.project.smart_contracts_dir, f"scripts/{script_name}.js") + # Hint: call web3 directly, avoid npx + truffle + script + # Maybe: self.cmd.yarn(["integrationtest:setTokenLockBurnLimit", str(amount)]) + self.project.npx(["truffle", "exec", script_path] + list(script_args), env=env, cwd=self.project.smart_contracts_dir, pipe=False) + + # TODO setTokenLockBurnLimit is gone, possibly replaced by bulkSetTokenLockBurnLimit + def set_token_lock_burn_limit(self, update_address, amount, ethereum_private_key, infura_project_id, local_provider): + env = { + "ETHEREUM_PRIVATE_KEY": ethereum_private_key, + "UPDATE_ADDRESS": update_address, + "INFURA_PROJECT_ID": infura_project_id, + "LOCAL_PROVIDER": local_provider, + } + # Needs: ETHEREUM_PRIVATE_KEY, INFURA_PROJECT_ID, LOCAL_PROVIDER, UPDATE_ADDRESS + # TODO script is no longer there! + self.truffle_exec("setTokenLockBurnLimit", str(amount), env=env) + + # @TODO Merge + def sifchain_init_integration(self, sifnode, validator_moniker, validator_mnemonic, denom_whitelist_file): + # now we have to add the validator key to the test keyring so the tests can send rowan from validator1 + sifnode0 = Sifnoded(self) + sifnode0.keys_add(validator_moniker, validator_mnemonic) + valoper = sifnode.keys_show(validator_moniker, bech="val")[0]["address"] + assert valoper == sifnode0.get_val_address(validator_moniker) # This does not use "home"; if it the assertion holds it could be grouped with sifchain_init_peggy + + # This was deleted in commit f00242302dd226bc9c3060fb78b3de771e3ff429 from sifchain_start_daemon.sh because + # it was not working. But we assume that we want to keep it. + sifnode.sifnoded_exec(["add-genesis-validators", valoper], sifnoded_home=sifnode.home) + + adminuser_addr = self.sifchain_init_common(sifnode, denom_whitelist_file) + return adminuser_addr + + def sifnoded_peggy2_init_validator(self, sifnode, validator_moniker, validator_mnemonic, evm_network_descriptor, validator_power, chain_dir_base): + # Add validator key to test keyring + # This effectively copies key for validator_moniker from what sifgen creates in /tmp/sifnodedNetwork/validators + # to ~/.sifnoded (note absence of explicit sifnoded_home, therefore it's ~/.sifnoded) + sifnode0 = Sifnoded(self) + sifnode0.keys_add(validator_moniker, validator_mnemonic) + + # Read valoper key + # (Since we now copied the key to main keyring we could also read it from there) + valoper = sifnode.get_val_address(validator_moniker) + + # Add genesis validator + sifnode.add_genesis_validators_peggy(evm_network_descriptor, valoper, validator_power) + + # Get whitelisted validator + # TODO Value is not being used + # TODO We're using default home here instead of sifnoded_home above. Does this even work? + _whitelisted_validator = sifnode.get_val_address(validator_moniker) + assert valoper == _whitelisted_validator + + # TODO Not any longer shared between IntegrationEnvironment and PeggyEnvironment + # Peggy2Environment calls sifnoded_peggy2_add_account + def sifchain_init_common(self, sifnode, denom_whitelist_file): + # Add sifnodeadmin to ~/.sifnoded + sifnode0 = Sifnoded(self) + sifnodeadmin_addr = sifnode0.keys_add_1("sifnodeadmin")["address"] + tokens = [[10**20, "rowan"]] + # Original from peggy: + # self.cmd.execst(["sifnoded", "add-genesis-account", sifnoded_admin_address, "100000000000000000000rowan", "--home", sifnoded_home]) + sifnode.add_genesis_account(sifnodeadmin_addr, tokens) + sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) + sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) + sifnode.set_gen_denom_whitelist(denom_whitelist_file) + return sifnodeadmin_addr + + # @TODO Move to Sifgen class + def sifgen_create_network(self, chain_id, validator_count, networks_dir, network_definition_file, seed_ip_address, mint_amount=None): + # Old call (no longer works either): + # sifgen network create localnet 1 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks \ + # 192.168.1.2 /mnt/shared/work/projects/sif/sifnode/local-tmp/my/deploy/rake/../networks/network-definition.yml \ + # --keyring-backend file + # self.cmd.execst(["sifgen", "network", "create", "localnet", str(validator_count), networks_dir, seed_ip_address, + # os.path.join(networks_dir, "network-definition.yml"), "--keyring-backend", "file"]) + # TODO Most likely, this should be "--keyring-backend file" + args = ["sifgen", "network", "create", chain_id, str(validator_count), networks_dir, seed_ip_address, + network_definition_file, "--keyring-backend", "test"] + \ + (["--mint-amount", ",".join([sif_format_amount(*x) for x in mint_amount])] if mint_amount else []) + self.execst(args) + + def wait_for_sif_account(self, netdef_json, validator1_address): + # TODO Replace with test_utilities.wait_for_sif_account / wait_for_sif_account_up + return self.execst(["python3", os.path.join(self.project.test_integration_dir, "src/py/wait_for_sif_account.py"), + netdef_json, validator1_address], env={"USER1ADDR": "nothing"}) + + def wait_for_sif_account_up(self, address, tcp_url=None): + # TODO Deduplicate: this is also in run_ebrelayer() + # netdef_json is path to file containing json_dump(netdef) + # while not self.cmd.tcp_probe_connect("localhost", tendermint_port): + # time.sleep(1) + # self.wait_for_sif_account(netdef_json, validator1_address) + + # Peggy2 + # How this works: by default, the command below will try to do a POST to http://localhost:26657. + # So the port has to be up first, but this query will fail anyway if it is not. + args = ["sifnoded", "query", "account", address] + \ + (["--node", tcp_url] if tcp_url else []) + while True: + try: + self.execst(args) + break + except Exception as e: + log.debug(f"Waiting for sif account {address}... ({repr(e)})") + time.sleep(1) + + +class UIStackEnvironment: + def __init__(self, cmd): + self.cmd = cmd + self.project = cmd.project + self.chain_id = "sifchain-local" + self.network_name = "develop" + self.network_id = 5777 + self.keyring_backend = "test" + self.ganache_db_path = self.cmd.get_user_home(".ganachedb") + self.sifnoded_path = self.cmd.get_user_home(".sifnoded") + self.sifnode = Sifnoded(cmd) + + # From ui/chains/credentials.sh + self.shadowfiend_name = "shadowfiend" + self.shadowfiend_mnemonic = ["race", "draft", "rival", "universe", "maid", "cheese", "steel", "logic", "crowd", + "fork", "comic", "easy", "truth", "drift", "tomorrow", "eye", "buddy", "head", "time", "cash", "swing", + "swift", "midnight", "borrow"] + self.kasha_name = "akasha" + self.akasha_mnemonic = ["hand", "inmate", "canvas", "head", "lunar", "naive", "increase", "recycle", "dog", + "ecology", "inhale", "december", "wide", "bubble", "hockey", "dice", "worth", "gravity", "ketchup", "feed", + "balance", "parent", "secret", "orchard"] + self.juniper_name = "juniper" + self.juniper_mnemonic = ["clump", "genre", "baby", "drum", "canvas", "uncover", "firm", "liberty", "verb", + "moment", "access", "draft", "erupt", "fog", "alter", "gadget", "elder", "elephant", "divide", "biology", + "choice", "sentence", "oppose", "avoid"] + self.ethereum_root_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", + "crumble", "sweet", "treat"] + + def stack_save_snapshot(self): + # ui-stack.yml + # cd .; go get -v -t -d ./... + # cd ui; yarn install --frozen-lockfile --silent + # Compile smart contracts: + # cd ui; yarn build + + # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh + # rm ui/node_modules/.migrate-complete + + # yarn stack --save-snapshot -> ui/scripts/stack.sh -> ui/scripts/stack-save-snapshot.sh => ui/scripts/stack-launch.sh + # ui/scripts/stack-launch.sh -> ui/scripts/_sif-build.sh -> ui/chains/sif/build.sh + # killall sifnoded + # rm $(which sifnoded) + self.cmd.rmdir(self.sifnoded_path) + self.project.make_go_binaries_2() + + # ui/scripts/stack-launch.sh -> ui/scripts/_eth.sh -> ui/chains/etc/launch.sh + self.cmd.rmdir(self.ganache_db_path) + self.project.yarn([], cwd=project_dir("ui/chains/eth")) # Installs ui/chains/eth/node_modules + # Note that this runs ganache-cli from $PATH whereas scripts start it with yarn in ui/chains/eth + ganache_proc = Ganache.start_ganache_cli(self.cmd, mnemonic=self.ethereum_root_mnemonic, db=self.ganache_db_path, + port=7545, network_id=self.network_id, gas_price=20000000000, gas_limit=6721975, host=ANY_ADDR) + + sifnode = Sifnoded(self.cmd) + # ui/scripts/stack-launch.sh -> ui/scripts/_sif.sh -> ui/chains/sif/launch.sh + sifnode.sifnoded_init("test", self.chain_id) + self.cmd.copy_file(project_dir("ui/chains/sif/app.toml"), os.path.join(self.sifnoded_path, "config/app.toml")) + log.info(f"Generating deterministic account - {self.shadowfiend_name}...") + shadowfiend_account = self.cmd.sifnoded_keys_add(self.shadowfiend_name, self.shadowfiend_mnemonic) + log.info(f"Generating deterministic account - {self.akasha_name}...") + akasha_account = self.sifnode.keys_add(self.akasha_name, self.akasha_mnemonic) + log.info(f"Generating deterministic account - {self.juniper_name}...") + juniper_account = self.cmd.sifnoded_keys_add(self.juniper_name, self.juniper_mnemonic) + shadowfiend_address = shadowfiend_account["address"] + akasha_address = akasha_account["address"] + juniper_address = juniper_account["address"] + assert shadowfiend_address == self.sifnode.keys_show(self.shadowfiend_name)[0]["address"] + assert akasha_address == self.sifnode.keys_show(self.akasha_name)[0]["address"] + assert juniper_address == self.sifnode.keys_show(self.juniper_name)[0]["address"] + + tokens_shadowfiend = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] + tokens_akasha = [[10**29, "rowan"], [10**29, "catk"], [10**29, "cbtk"], [10**29, "ceth"], [10**29, "cusdc"], [10**29, "clink"], [10**26, "stake"]] + tokens_juniper = [[10**22, "rowan"], [10**22, "cusdc"], [10**20, "clink"], [10**20, "ceth"]] + sifnode.add_genesis_account(shadowfiend_address, tokens_shadowfiend) + sifnode.add_genesis_account(akasha_address, tokens_akasha) + sifnode.add_genesis_account(juniper_address, tokens_juniper) + + shadowfiend_address_bech_val = sifnode.keys_show(self.shadowfiend_name, bech="val")[0]["address"] + self.cmd.sifnoded_add_genesis_validators(shadowfiend_address_bech_val) + + amount = sif_format_amount(10**24, "stake") + self.cmd.execst(["sifnoded", "gentx", self.shadowfiend_name, amount, f"--chain-id={self.chain_id}", + f"--keyring-backend={self.keyring_backend}"]) + + log.info("Collecting genesis txs...") + self.cmd.execst(["sifnoded", "collect-gentxs"]) + log.info("Validating genesis file...") + self.cmd.execst(["sifnoded", "validate-genesis"]) + + log.info("Starting test chain...") + sifnoded_proc = self.cmd.sifnoded_start(minimum_gas_prices=[0.5, "rowan"]) # TODO sifnoded_home=??? + + # sifnoded must be up before continuing + self.cmd.sif_wait_up("localhost", 1317) + + # ui/scripts/_migrate.sh -> ui/chains/peggy/migrate.sh + self.cmd.deploy_smart_contracts_for_ui_stack() + + # ui/scripts/_migrate.sh -> ui/chains/eth/migrate.sh + # send through atk and btk tokens to eth chain + self.project.yarn(["migrate"], cwd=project_dir("ui/chains/eth")) + + # ui/scripts/_migrate.sh -> ui/chains/sif/migrate.sh + # Original scripts say "if we don't sleep there are issues" + time.sleep(10) + log.info("Creating liquidity pool from catk:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "catk", [10**5, "rowan"], 10**25, 10**25) + time.sleep(5) + log.info("Creating liquidity pool from cbtk:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cbtk", [10**5, "rowan"], 10**25, 10**25) + # should now be able to swap from catk:cbtk + time.sleep(5) + log.info("Creating liquidity pool from ceth:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ceth", [10**5, "rowan"], 10**25, 83*10**20) + # should now be able to swap from x:ceth + time.sleep(5) + log.info("Creating liquidity pool from cusdc:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "cusdc", [10**5, "rowan"], 10**25, 10**25) + time.sleep(5) + log.info("Creating liquidity pool from clink:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "clink", [10**5, "rowan"], 10**25, 588235*10**18) + time.sleep(5) + log.info("Creating liquidity pool from ctest:rowan...") + sifnode.tx_clp_create_pool(self.chain_id, self.keyring_backend, "akasha", "ctest", [10**5, "rowan"], 10**25, 10**13) + + # ui/scripts/_migrate.sh -> ui/chains/post_migrate.sh + + atk_address, btk_address, usdc_address, link_address = [ + self.cmd.get_smart_contract_address(project_dir(f"ui/chains/eth/build/contracts/{x}.json"), self.network_id) + for x in ["AliceToken", "BobToken", "UsdCoin", "LinkCoin"] + ] + + bridge_token_address, bridge_registry_address, bridge_bank = self.cmd.get_bridge_smart_contract_addresses(self.network_id) + + # From smart-contracts/.env.ui.example + smart_contracts_env_ui_example_vars = { + "ETHEREUM_PRIVATE_KEY": "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3", + "INFURA_PROJECT_ID": "JFSH7439sjsdtqTM23Dz", + "LOCAL_PROVIDER": "http://localhost:7545", + } + + # NOTE: this probably doesn't work anymore since setTokenLockBurnLimit.js was replaced + burn_limits = [ + [eth.NULL_ADDRESS, 31 * 10 ** 18], + [bridge_token_address, 10 ** 25], + [atk_address, 10 ** 25], + [btk_address, 10 ** 25], + [usdc_address, 10 ** 25], + [link_address, 10 ** 25], + ] + for address, amount in burn_limits: + self.cmd.set_token_lock_burn_limit( + address, + amount, + smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"], + smart_contracts_env_ui_example_vars["INFURA_PROJECT_ID"], + smart_contracts_env_ui_example_vars["LOCAL_PROVIDER"] + ) + + # signal migrate-complete + + # Whitelist test tokens + for addr in [atk_address, btk_address, usdc_address, link_address]: + self.project.yarn(["peggy:whiteList", addr, "true"], cwd=self.project.smart_contracts_dir) + + # ui/scripts/stack-launch.sh -> ui/scripts/_peggy.sh -> ui/chains/peggy/launch.sh + # rm -rf ui/chains/peggy/relayerdb + # ebrelayer is in $GOBIN, gets installed by "make install" + ethereum_private_key = smart_contracts_env_ui_example_vars["ETHEREUM_PRIVATE_KEY"] + ebrelayer_proc = Ebrelayer(self.cmd).init("tcp://localhost:26657", "ws://localhost:7545/", + bridge_registry_address, self.shadowfiend_name, self.shadowfiend_mnemonic, self.chain_id, + ethereum_private_key=ethereum_private_key, gas=5*10**12, gas_prices=[0.5, "rowan"]) + + # At this point we have 3 running processes - ganache_proc, sifnoded_proc and ebrelayer_proc + # await sif-node-up and migrate-complete + + time.sleep(30) + # ui/scripts/_snapshot.sh + + # ui/scripts/stack-pause.sh: + # killall sifnoded sifnoded ebrelayer ganache-cli + sifnoded_proc.kill() + ebrelayer_proc.kill() + ganache_proc.kill() + time.sleep(10) + + snapshots_dir = project_dir("ui/chains/snapshots") + self.cmd.mkdir(snapshots_dir) # TODO self.cmd.rmdir(snapshots_dir) + # ui/chains/peggy/snapshot.sh: + # mkdir -p ui/chains/peggy/relayerdb + self.cmd.tar_create(project_dir("ui/chains/peggy/relayerdb"), os.path.join(snapshots_dir, "peggy.tar.gz")) + # mkdir -p smart-contracts/build + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(snapshots_dir, "peggy_build.tar.gz")) + + # ui/chains/sif/snapshot.sh: + self.cmd.tar_create(self.sifnoded_path, os.path.join(snapshots_dir, "sif.tar.gz")) + + # ui/chains/etc/snapshot.sh: + self.cmd.tar_create(self.ganache_db_path, os.path.join(snapshots_dir, "eth.tar.gz")) + + def stack_push(self): + # ui/scripts/stack-push.sh + # $PWD=ui + + # User must be logged in to Docker hub: + # ~/.docker/config.json must exist and .auths['ghcr.io'].auth != null + log.info("Github Registry Login found.") + + commit = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "HEAD"], cwd=project_dir()))) + branch = exactly_one(stdout_lines(self.cmd.execst(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=project_dir()))) + + image_root = "ghcr.io/sifchain/sifnode/ui-stack" + image_name = "{}:{}".format(image_root, commit) + stable_tag = "{}:{}".format(image_root, branch.replace("/", "__")) + + running_in_ci = bool(os.environ.get("CI")) + + if running_in_ci: + res = self.cmd.execst(["git", "status", "--porcelain", "--untracked-files=no"], cwd=project_dir()) + # # reverse grep for go.mod because on CI this can be altered by installing go dependencies + # if [[ -z "$CI" && ! -z "$(git status --porcelain --untracked-files=no)" ]]; then + # echo "Git workspace must be clean to save git commit hash" + # exit 1 + # fi + pass + + log.info("Building new container...") + log.info(f"New image name: {image_name}") + + self.cmd.execst(["docker", "build", "-f", project_dir("ui/scripts/stack.Dockerfile"), "-t", image_name, "."], + cwd=project_dir(), env={"DOCKER_BUILDKIT": "1"}, pipe=False) + + if running_in_ci: + log.info(f"Tagging image as {stable_tag}...") + self.cmd.execst(["docker", "tag", image_name, stable_tag]) + self.cmd.execst(["docker", "push", image_name]) + self.cmd.execst(["docker", "push", stable_tag]) + + +class IntegrationTestsEnvironment: + def __init__(self, cmd): + self.cmd = cmd + self.project = cmd.project + # Fixed, set in start-integration-env.sh + self.ethereum_private_key = "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3" + self.owner = "0x627306090abaB3A6e1400e9345bC60c78a8BEf57" + # we may eventually switch things so PAUSER and OWNER aren't the same account, but for now they're the same + self.pauser = self.owner + # set_persistant_env_var BASEDIR $(fullpath $BASEDIR) $envexportfile + # set_persistant_env_var SIFCHAIN_BIN $BASEDIR/cmd $envexportfile + # set_persistant_env_var envexportfile $(fullpath $envexportfile) $envexportfile + # set_persistant_env_var TEST_INTEGRATION_DIR ${BASEDIR}/test/integration $envexportfile + # set_persistant_env_var TEST_INTEGRATION_PY_DIR ${BASEDIR}/test/integration/src/py $envexportfile + # set_persistant_env_var SMART_CONTRACTS_DIR ${BASEDIR}/smart-contracts $envexportfile + # set_persistant_env_var datadir ${TEST_INTEGRATION_DIR}/vagrant/data $envexportfile + # set_persistant_env_var CONTAINER_NAME integration_sifnode1_1 $envexportfile + # set_persistant_env_var NETWORKDIR $BASEDIR/deploy/networks $envexportfile + # set_persistant_env_var GANACHE_DB_DIR $(mktemp -d /tmp/ganachedb.XXXX) $envexportfile + # set_persistant_env_var ETHEREUM_WEBSOCKET_ADDRESS ws://localhost:7545/ $envexportfile + # set_persistant_env_var CHAINNET localnet $envexportfile + self.network_name = "develop" + self.network_id = 5777 + self.peruser_storage_dir = self.cmd.get_user_home(".sifnode-integration") + self.state_vars = {} + self.test_integration_dir = project_dir("test/integration") + self.data_dir = project_dir("test/integration/vagrant/data") + self.chainnet = "localnet" + self.tcp_url = f"tcp://{ANY_ADDR}:26657" + self.ethereum_websocket_address = "ws://localhost:7545/" + self.ganache_mnemonic = ["candy", "maple", "cake", "sugar", "pudding", "cream", "honey", "rich", "smooth", + "crumble", "sweet", "treat"] + + def prepare(self): + self.project.make_go_binaries() + self.project.install_smart_contracts_dependencies() + + def run(self): + self.cmd.mkdir(self.data_dir) + + self.prepare() + + log_dir = "/tmp/sifnode" + self.cmd.mkdir(log_dir) + ganache_log_file = open(os.path.join(log_dir, "ganache.log"), "w") # TODO close + sifnoded_log_file = open(os.path.join(log_dir, "sifnoded.log"), "w") # TODO close + ebrelayer_log_file = open(os.path.join(log_dir, "ebrelayer.log"), "w") # TODO close + + # test/integration/ganache-start.sh: + # 1. pkill -9 -f ganache-cli || true + # 2. while nc -z localhost 7545; do sleep 1; done + # 3. nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic \ + # 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' \ + # --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON \ + # > $GANACHE_LOG 2>&1" + # 4. sleep 5 + # 5. while ! nc -z localhost 4545; do sleep 5; done + # GANACHE_LOG=ui/test/integration/vagrant/data/logs/ganache.$(filenamedate).txt + block_time = None # TODO + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") + ganache_db_path = self.cmd.mktempdir() + ganache_proc = Ganache.start_ganache_cli(self.cmd, block_time=block_time, host=ANY_ADDR, + mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, + account_keys_path=account_keys_path, log_file=ganache_log_file) + + self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli + time.sleep(2) + + ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) + ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] + ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] + # TODO Check for possible non-determinism of dict().keys() ordering (c.f. test/integration/vagrantenv.sh) + # TODO ebrelayer_ethereum_private_key is NOT the same as in test/integration/.env.ciExample + assert ebrelayer_ethereum_addr == "0x5aeda56215b167893e80b4fe645ba6d5bab767de" + assert ebrelayer_ethereum_private_key == "8d5366123cb560bb606379f90a0bfd4769eecc0557f1b362dcae9012b548b1e5" + + env_file = project_dir("test/integration/.env.ciExample") + env_vars = self.cmd.primitive_parse_env_file(env_file) + self.cmd.deploy_smart_contracts_for_integration_tests(self.network_name, owner=self.owner, pauser=self.pauser, + operator=env_vars["OPERATOR"], consensus_threshold=int(env_vars["CONSENSUS_THRESHOLD"]), + initial_validator_powers=[int(x) for x in env_vars["INITIAL_VALIDATOR_POWERS"].split(",")], + initial_validator_addresses=[ebrelayer_ethereum_addr], env_file=env_file) + + bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ + self.cmd.get_bridge_smart_contract_addresses(self.network_id) + + # # TODO This should be last (after return from setup_sifchain.sh) + # burn_limits = [ + # [eth.NULL_ADDRESS, 31*10**18], + # [bridge_token_sc_addr, 10**25], + # ] + # env_file_vars = self.cmd.primitive_parse_env_file(env_file) + # for address, amount in burn_limits: + # self.cmd.set_token_lock_burn_limit( + # address, + # amount, + # env_file_vars["ETHEREUM_PRIVATE_KEY"], # != ebrelayer_ethereum_private_key + # env_file_vars["INFURA_PROJECT_ID"], + # env_file_vars["LOCAL_PROVIDER"], # for web3.js to connect to ganache + # ) + + # test/integration/setup_sifchain.sh: + networks_dir = project_dir("deploy/networks") + self.cmd.rmdir(networks_dir) # networks_dir has many directories without write permission, so change those before deleting it + self.cmd.mkdir(networks_dir) + # Old: + # self.cmd.execst(["rake", f"genesis:network:scaffold[{self.chainnet}]"], env={"BASEDIR": project_dir()}, pipe=False) + # New: + # sifgen network create localnet 1 $NETWORKDIR 192.168.1.2 $NETWORKDIR/network-definition.yml --keyring-backend test \ + # --mint-amount 999999000000000000000000000rowan,1370000000000000000ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE + chain_id = "localnet" + validator_count = 1 + network_definition_file = os.path.join(networks_dir, "network-definition.yml") + seed_ip_address = "192.168.1.2" + mint_amount = [[999999 * 10**21, "rowan"], [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"]] + + self.cmd.sifgen_create_network(chain_id, validator_count, networks_dir, network_definition_file, seed_ip_address, mint_amount=mint_amount) + + netdef, netdef_json = self.process_netdef(network_definition_file) + + validator1_moniker = netdef["moniker"] + validator1_address = netdef["address"] + validator1_password = netdef["password"] + validator1_mnemonic = netdef["mnemonic"].split(" ") + chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator1_moniker}") + sifnoded_home = os.path.join(chaindir, ".sifnoded") + denom_whitelist_file = os.path.join(self.test_integration_dir, "whitelisted-denoms.json") + # SIFNODED_LOG=$datadir/logs/sifnoded.log + + sifnode = Sifnoded(self.cmd, home=sifnoded_home) + + adminuser_addr = self.cmd.sifchain_init_integration(sifnode, validator1_moniker, validator1_mnemonic, + denom_whitelist_file) + + # Start sifnoded + sifnoded_proc = sifnode.sifnoded_start(tcp_url=self.tcp_url, minimum_gas_prices=[0.5, "rowan"], + log_file=sifnoded_log_file) + + # TODO: wait for sifnoded to come up before continuing + # in sifchain_start_daemon.sh: "sleep 10" + # in sifchain_run_ebrelayer.sh (also run_ebrelayer here) we already wait for connection to port 26657 and sif account validator1_addr + + # Removed + # # TODO Process exits immediately with returncode 1 + # # TODO Why does it not stop start-integration-env.sh? + # # rest_server_proc = self.cmd.popen(["sifnoded", "rest-server", "--laddr", "tcp://0.0.0.0:1317"]) # TODO cwd + + # test/integration/sifchain_start_ebrelayer.sh -> test/integration/sifchain_run_ebrelayer.sh + # This script is also called from tests + + relayer_db_path = os.path.join(self.test_integration_dir, "sifchainrelayerdb") + ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator1_moniker, validator1_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path, log_file=ebrelayer_log_file) + + vagrantenv_path = project_dir("test/integration/vagrantenv.sh") + self.state_vars = { + "ETHEREUM_PRIVATE_KEY": self.ethereum_private_key, + "OWNER": self.owner, + "PAUSER": self.pauser, + "BASEDIR": project_dir(), + # export SIFCHAIN_BIN="/home/jurez/work/projects/sif/sifnode/local/cmd" + "envexportfile": vagrantenv_path, + # export TEST_INTEGRATION_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration" + # export TEST_INTEGRATION_PY_DIR="/home/jurez/work/projects/sif/sifnode/local/test/integration/src/py" + "SMART_CONTRACTS_DIR": self.project.smart_contracts_dir, + # export datadir="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data" + # export CONTAINER_NAME="integration_sifnode1_1" + "NETWORKDIR": networks_dir, + # export ETHEREUM_WEBSOCKET_ADDRESS="ws://localhost:7545/" + # export CHAINNET="localnet" + "GANACHE_DB_DIR": ganache_db_path, + # export GANACHE_KEYS_JSON="/home/jurez/work/projects/sif/sifnode/local/test/integration/vagrant/data/ganachekeys.json" + "EBRELAYER_ETHEREUM_ADDR": ebrelayer_ethereum_addr, + "EBRELAYER_ETHEREUM_PRIVATE_KEY": ebrelayer_ethereum_private_key, # Needed by sifchain_run_ebrelayer.sh + # # BRIDGE_REGISTRY_ADDRESS and ETHEREUM_CONTRACT_ADDRESS are synonyms + "BRIDGE_REGISTRY_ADDRESS": bridge_registry_sc_addr, + "BRIDGE_TOKEN_ADDRESS": bridge_token_sc_addr, + "BRIDGE_BANK_ADDRESS": bridge_bank_sc_addr, + "NETDEF": os.path.join(networks_dir, "network-definition.yml"), + "NETDEF_JSON": project_dir("test/integration/vagrant/data/netdef.json"), + "MONIKER": validator1_moniker, + "VALIDATOR1_PASSWORD": validator1_password, + "VALIDATOR1_ADDR": validator1_address, + "MNEMONIC": " ".join(validator1_mnemonic), + "CHAINDIR": os.path.join(networks_dir, "validators", self.chainnet, validator1_moniker), + "SIFCHAIN_ADMIN_ACCOUNT": adminuser_addr, # Needed by test_peggy_fees.py (via conftest.py) + "EBRELAYER_DB": relayer_db_path, # Created by sifchain_run_ebrelayer.sh, does not appear to be used anywhere at the moment + } + self.project.write_vagrantenv_sh(self.state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) + + return ganache_proc, sifnoded_proc, ebrelayer_proc + + def remove_and_add_sifnoded_keys(self, moniker, mnemonic): + # Error: The specified item could not be found in the keyring + # This is not neccessary during start-integration-env.sh (as the key does not exist yet), but is neccessary + # during tests that restart ebrelayer + # res = self.cmd.execst(["sifnoded", "keys", "delete", moniker, "--keyring-backend", "test"], stdin=["y"]) + sifnode = Sifnoded(self.cmd) + sifnode.keys_delete(moniker) + sifnode.keys_add(moniker, mnemonic) + + def process_netdef(self, network_definition_file): + # networks_dir = deploy/networks + # File deploy/networks/network-definition.yml is created by "rake genesis:network:scaffold", specifically by + # "sifgen network create" + # We read it and convert to test/integration/vagrant/data/netdef.json + netdef = exactly_one(yaml_load(self.cmd.read_text_file(network_definition_file))) + netdef_json = os.path.join(self.data_dir, "netdef.json") + self.cmd.write_text_file(netdef_json, json.dumps(netdef, indent=4)) + return netdef, netdef_json + + def run_ebrelayer(self, netdef_json, validator1_address, validator_moniker, validator_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path, log_file=None + ): + # TODO Deduplicate + while not self.cmd.tcp_probe_connect("localhost", 26657): + time.sleep(1) + self.cmd.wait_for_sif_account(netdef_json, validator1_address) + time.sleep(10) + self.remove_and_add_sifnoded_keys(validator_moniker, validator_mnemonic) # Creates ~/.sifnoded/keyring-tests/xxxx.address + ebrelayer_proc = Ebrelayer(self.cmd).init(self.tcp_url, self.ethereum_websocket_address, bridge_registry_sc_addr, + validator_moniker, validator_mnemonic, self.chainnet, ethereum_private_key=ebrelayer_ethereum_private_key, + node=self.tcp_url, keyring_backend="test", sign_with=validator_moniker, + symbol_translator_file=os.path.join(self.test_integration_dir, "config/symbol_translator.json"), + relayerdb_path=relayer_db_path, cwd=self.test_integration_dir, log_file=log_file) + return ebrelayer_proc + + def create_own_dirs(self): + self.cmd.mkdir(self.peruser_storage_dir) + self.cmd.mkdir(os.path.join(self.peruser_storage_dir, "snapshots")) + + def create_snapshot(self, snapshot_name): + self.create_own_dirs() + named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) + if self.cmd.exists(named_snapshot_dir): + raise Exception(f"Directory '{named_snapshot_dir}' already exists") + self.cmd.mkdir(named_snapshot_dir) + self.cmd.tar_create(self.state_vars["GANACHE_DB_DIR"], os.path.join(named_snapshot_dir, "ganache.tar.gz")) + self.cmd.tar_create(self.state_vars["EBRELAYER_DB"], os.path.join(named_snapshot_dir, "sifchainrelayerdb.tar.gz")) + self.cmd.tar_create(project_dir("deploy/networks"), os.path.join(named_snapshot_dir, "networks.tar.gz")) + self.cmd.tar_create(project_dir("smart-contracts/build"), os.path.join(named_snapshot_dir, "smart-contracts.tar.gz")) + self.cmd.tar_create(self.cmd.get_user_home(".sifnoded"), os.path.join(named_snapshot_dir, "sifnoded.tar.gz")) + self.cmd.write_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"), json.dumps(self.state_vars, indent=4)) + + def restore_snapshot(self, snapshot_name): + named_snapshot_dir = os.path.join(self.peruser_storage_dir, "snapshots", snapshot_name) + state_vars = json.loads(self.cmd.read_text_file(os.path.join(named_snapshot_dir, "vagrantenv.json"))) + + def extract(tarfile, path): + self.cmd.rmdir(path) + self.cmd.mkdir(path) + self.cmd.tar_extract(os.path.join(named_snapshot_dir, tarfile), path) + + ganache_db_dir = self.cmd.mktempdir() + relayer_db_path = state_vars["EBRELAYER_DB"] # TODO use /tmp + assert os.path.realpath(relayer_db_path) == os.path.realpath(os.path.join(self.test_integration_dir, "sifchainrelayerdb")) + extract("ganache.tar.gz", ganache_db_dir) + extract("sifchainrelayerdb.tar.gz", relayer_db_path) + deploy_networks_dir = project_dir("deploy/networks") + extract("networks.tar.gz", deploy_networks_dir) + smart_contracts_build_dir = project_dir("smart-contracts/build") + extract("smart-contracts.tar.gz", smart_contracts_build_dir) + extract("sifnoded.tar.gz", self.cmd.get_user_home(".sifnoded")) # Needed for "--keyring-backend test" + + state_vars["GANACHE_DB_DIR"] = ganache_db_dir + state_vars["EBRELAYER_DB"] = relayer_db_path + self.state_vars = state_vars + self.project.write_vagrantenv_sh(state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) + self.cmd.mkdir(self.data_dir) + + def restart_processes(self): + block_time = None + ganache_db_path = self.state_vars["GANACHE_DB_DIR"] + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") # TODO this is in test/integration/vagrant/data, which is supposed to be cleared + + ganache_proc = Ganache.start_ganache_cli(self.cmd, block_time=block_time, host=ANY_ADDR, + mnemonic=self.ganache_mnemonic, network_id=self.network_id, port=7545, db=ganache_db_path, + account_keys_path=account_keys_path) # TODO log_file + + self.cmd.wait_for_file(account_keys_path) # Created by ganache-cli + time.sleep(2) + + validator_moniker = self.state_vars["MONIKER"] + networks_dir = project_dir("deploy/networks") + chaindir = os.path.join(networks_dir, f"validators/{self.chainnet}/{validator_moniker}") + sifnoded_home = os.path.join(chaindir, ".sifnoded") + sifnoded_proc = self.cmd.sifnoded_start(tcp_url=self.tcp_url, minimum_gas_prices=[0.5, "rowan"], sifnoded_home=sifnoded_home) + + bridge_token_sc_addr, bridge_registry_sc_addr, bridge_bank_sc_addr = \ + self.cmd.get_bridge_smart_contract_addresses(self.network_id) + + validator_mnemonic = self.state_vars["MNEMONIC"].split(" ") + account_keys_path = os.path.join(self.data_dir, "ganachekeys.json") + ganache_keys = json.loads(self.cmd.read_text_file(account_keys_path)) + ebrelayer_ethereum_addr = list(ganache_keys["private_keys"].keys())[9] + ebrelayer_ethereum_private_key = ganache_keys["private_keys"][ebrelayer_ethereum_addr] + network_definition_file = project_dir(networks_dir, "network-definition.yml") + + netdef, netdef_json = self.process_netdef(network_definition_file) + validator1_address = netdef["address"] + assert validator1_address == self.state_vars["VALIDATOR1_ADDR"] + relayer_db_path = self.state_vars["EBRELAYER_DB"] + ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator_moniker, validator_mnemonic, + ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path) + + return ganache_proc, sifnoded_proc, ebrelayer_proc + + +class Peggy2Environment(IntegrationTestsEnvironment): + def __init__(self, cmd): + super().__init__(cmd) + self.hardhat = hardhat.Hardhat(cmd) + + # Destuctures a linear array of EVM accounts into: + # [operator, owner, pauser, [validator-0, validator-1, ...], [...available...]] + # proxy_admin is the same as operator. + def signer_array_to_ethereum_accounts(self, accounts, n_validators): + assert len(accounts) >= n_validators + 3 + operator, owner, pauser, *rest = accounts # Take 3 and store remaining in rest + validators, available = rest[:n_validators], rest[n_validators:] # Take n_validators for validators the remaining for available + return { + "proxy_admin": operator, + "operator": operator, + "owner": owner, + "pauser": pauser, + "validators": validators, + "available": available, + } + + # Override + def run(self): + # self.project._make_go_binaries() + + # Ordering (for possible parallelisation): + # - build_golang_binaries before start_sifchain + # - start_hardhat before deploy_smart_contracts + # - start_sifchain before start_witnesses_and_relayers + # - deploy_smart_contracts before start_witnesses_and_relayers + # - start_witnesses_and_relayers before return + # - write_env_file before return + + # TODO: where is log watcher? + + log_dir = "/tmp/sifnode" + self.cmd.mkdir(log_dir) + hardhat_log_file = open(os.path.join(log_dir, "hardhat.log"), "w") # TODO close + use a different name + sifnoded_log_file = open(os.path.join(log_dir, "sifnoded.log"), "w") # TODO close + relayer_log_file = open(os.path.join(log_dir, "relayer.log"), "w") # TODO close + witness_log_file = open(os.path.join(log_dir, "witness.log"), "w") # TODO close; will be empty on non-peggy2 branch + + self.cmd.rmdir(self.cmd.get_user_home(".sifnoded")) # Purge test keyring backend + + hardhat_bind_hostname = "localhost" # The host to which to bind to for new connections (Defaults to 127.0.0.1 running locally, and 0.0.0.0 in Docker) + hardhat_port = 8545 # The port on which to listen for new connections (default: 8545) + hardhat_exec_args = self.hardhat.build_start_args(hostname=hardhat_bind_hostname, port=hardhat_port) + hardhat_proc = self.cmd.spawn_asynchronous_process(hardhat_exec_args, log_file=hardhat_log_file) + + # This determines how many EVM accounts we want to allocate for validators. + # Since every validator needs on EVM account, this should be equal to the number of validators (possibly more). + hardhat_validator_count = 1 + hardhat_network_id = 1 # Not used in smart-contracts/src/devenv/hardhatNode.ts + # This value is actually returned from HardhatNodeRunner. It comes from smart-contracts/hardhat.config.ts. + # In Typescript, its value is obtained by 'require("hardhat").hre.network.config.chainId'. + # See https://hardhat.org/advanced/hardhat-runtime-environment.html + # The value is not used; instead a hardcoded constant 31337 is passed to ebrelayerWitnessBuilder. + # Ask juniuszhou for details. + hardhat_chain_id = 31337 + hardhat_accounts = self.signer_array_to_ethereum_accounts(hardhat.default_accounts(), hardhat_validator_count) + + self.hardhat.compile_smart_contracts() + peggy_sc_addrs = self.hardhat.deploy_smart_contracts() + + # Initialization of smart contracts (technically this is part of deployment) + operator_acct = hardhat_accounts["operator"] + w3_websocket_address = eth.web3_host_port_url("localhost", hardhat_port) + self.init_smart_contracts(w3_websocket_address, operator_acct, peggy_sc_addrs) + + admin_account_name = "sifnodeadmin" + chain_id = "localnet" + ceth_symbol = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) + assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" + # Mint goes to validator + mint_amount = [ + [999999 * 10**21, "rowan"], + [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"], + [999999 * 10**21, ceth_symbol], + ] + validator_power = 100 + seed_ip_address = "10.10.1.1" + tendermint_port = 26657 + denom_whitelist_file = project_dir("test", "integration", "whitelisted-denoms.json") + tokens = [ + [10**20, "rowan"], + [2 * 10**19, "ceth"] + ] + registry_json = project_dir("smart-contracts", "src", "devenv", "registry.json") + sifnoded_network_dir = "/tmp/sifnodedNetwork" # Gets written to .vscode/launch.json + self.cmd.rmdir(sifnoded_network_dir) + self.cmd.mkdir(sifnoded_network_dir) + network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, sifnode_validators, \ + sifnode_relayers, sifnode_witnesses, sifnode_validator0_home, chain_dir = \ + self.init_sifchain(sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, + validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, + admin_account_name) + + symbol_translator_file = os.path.join(self.test_integration_dir, "config", "symbol_translator.json") + [relayer0_exec_args], [witness0_exec_args] = \ + self.start_witnesses_and_relayers(w3_websocket_address, hardhat_chain_id, tcp_url, + chain_id, peggy_sc_addrs, hardhat_accounts["validators"], sifnode_validators, sifnode_relayers, + sifnode_witnesses, symbol_translator_file) + + relayer0_proc = self.cmd.spawn_asynchronous_process(relayer0_exec_args, log_file=relayer_log_file) + witness0_proc = self.cmd.spawn_asynchronous_process(witness0_exec_args, log_file=witness_log_file) + + # In the future, we want to have one descriptor for entire environment. + # It should be able to support multiple EVM and multiple Cosmos chains, including all neccessary bridges and + # relayers. For now this is just a prototype which is not used yet. + _unused_peggy2_environment = { + "admin": { + "name": admin_account_name, + "address": admin_account_address, + "home": sifnode_validator0_home, + }, + "symbol_translator_file": symbol_translator_file, + "w3_websocket_address": w3_websocket_address, + "evm_chain_id": hardhat_chain_id, + "chain_id": chain_id, + "validators": sifnode_validators, # From yaml file generated by sifgen + "relayers": sifnode_relayers, + "smart_contracts": peggy_sc_addrs + } + + self.write_env_files(self.project.project_dir(), self.project.go_bin_dir, peggy_sc_addrs, hardhat_accounts, + admin_account_name, admin_account_address, sifnode_validator0_home, sifnode_validators, sifnode_relayers, + sifnode_witnesses, tcp_url, hardhat_bind_hostname, hardhat_port, hardhat_chain_id, chain_dir, + sifnoded_exec_args, relayer0_exec_args, witness0_exec_args + ) + + return hardhat_proc, sifnoded_proc, relayer0_proc, witness0_proc + + def init_smart_contracts(self, w3_url, operator_account, deployed_contract_addresses): + # Looks like this is already done somewhere else... + # operator_addr, operator_private_key = operator_account + # w3_conn = eth.web3_wait_for_connection_up(w3_url) + # eth_tx = eth.EthereumTxWrapper(w3_conn, True) + # eth_tx.set_private_key(operator_addr, operator_private_key) + # + # # CosmosBridge doesn't have BridgeBank in its init and expects a separate setBridgeBank call. CosmosBridge + # # doesn't really work without BridgeBank. + # abi_provider = hardhat.HardhatAbiProvider(self.cmd, deployed_contract_addresses) + # abi, _, deployed_address = abi_provider.get_descriptor("CosmosBridge") + # cosmos_bridge = w3_conn.eth.contract(abi=abi, address=deployed_address) + # bridge_bank_addr = deployed_contract_addresses["BridgeBank"] + # txrcpt = eth_tx.transact_sync(cosmos_bridge.functions.setBridgeBank, operator_addr)(bridge_bank_addr) + return + + def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, + validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, + admin_account_name + ): + validator_count = 1 + relayer_count = 1 + witness_count = 1 + # TODO Not used + # rpc_port = 9000 + + network_config_file_path = self.cmd.mktempfile() + try: + self.cmd.sifgen_create_network(chain_id, validator_count, sifnoded_network_dir, network_config_file_path, + seed_ip_address, mint_amount=mint_amount) + network_config_file = self.cmd.read_text_file(network_config_file_path) + finally: + self.cmd.rm(network_config_file_path) + validators = yaml_load(network_config_file) + + # netdef_yml is a list of generated validators like below. + # Each one has its unique IP (starting with base IP + 1), the first one also has is_seed=True. + # + # class ValidatorValues: + # chain_id: str + # node_id: str + # ipv4_address: str + # moniker: str + # password: str + # address: str + # pub_key: str + # mnemonic: str + # validator_address: str + # validator_consensus_address: str + # is_seed: bool + assert len(validators) == validator_count + + chain_dir_base = os.path.join(sifnoded_network_dir, "validators", chain_id) + + for validator in validators: + validator_moniker = validator["moniker"] + validator_mnemonic = validator["mnemonic"].split(" ") + # TODO Not used + # validator_password = validator["password"] + evm_network_descriptor = 1 # TODO Why not hardhat_chain_id? + sifnoded_home = os.path.join(chain_dir_base, validator_moniker, ".sifnoded") + sifnode = Sifnoded(self.cmd, home=sifnoded_home) + self.cmd.sifnoded_peggy2_init_validator(sifnode, validator_moniker, validator_mnemonic, evm_network_descriptor, validator_power, chain_dir_base) + + # TODO Needs to be fixed when we support more than 1 validator + validator0 = exactly_one(validators) + validator0_home = os.path.join(chain_dir_base, validator0["moniker"], ".sifnoded") + validator0_address = validator0["address"] + chain_dir = os.path.join(chain_dir_base, validator0["moniker"]) + + sifnode = Sifnoded(self.cmd, home=validator0_home) + + # Create an ADMIN account on sifnode with name admin_account_name (e.g. "sifnodeadmin") + admin_account_address = sifnode.peggy2_add_account(admin_account_name, tokens, is_admin=True) + + # TODO Check if sifnoded_peggy2_add_relayer_witness_account can be executed offline (without sifnoded running) + # TODO Check if sifnoded_peggy2_set_cross_chain_fee can be executed offline (without sifnoded running) + + # Create an account for each relayer + # Note: "--home" is shared with sifnoded's "--home" + relayers = [{ + "name": name, + "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, + validator_power, denom_whitelist_file), + "home": validator0_home, + } for name in [f"relayer-{i}" for i in range(relayer_count)]] + + # Create an account for each witness + # Note: "--home" is shared with sifnoded's "--home" + witnesses = [{ + "name": name, + "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, + validator_power, denom_whitelist_file), + "home": validator0_home, + } for name in [f"witness-{i}" for i in range(witness_count)]] + + tcp_url = "tcp://{}:{}".format(ANY_ADDR, tendermint_port) + # sifnoded + # start + # --log_level debug + # --log_format json + # --minimum-gas-prices 0.5rowan + # --rpc.laddr tcp://0.0.0.0:26657 + # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded + # @TODO Detect if sifnoded is already running, for now it fails silently and we wait forever in wait_for_sif_account_up + sifnoded_exec_args = sifnode.build_start_cmd(tcp_url=tcp_url, minimum_gas_prices=[0.5, "rowan"], + log_format_json=True) + sifnoded_proc = self.cmd.spawn_asynchronous_process(sifnoded_exec_args, log_file=sifnoded_log_file) + + self.cmd.wait_for_sif_account_up(validator0_address, tcp_url) + + # TODO This command exits with status 0, but looks like there are some errros. + # The same happens also in devenv. + # TODO Try whitelister account instead of admin + res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_address, + chain_id) + log.debug("Result from token registry: {}".format(repr(res))) + assert len(res) == 2 + assert res[0]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" + assert res[1]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" + + cross_chain_fee_base = 1 + cross_chain_lock_fee = 1 + cross_chain_burn_fee = 1 + ethereum_cross_chain_fee_token = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) + gas_prices = [0.5, "rowan"] + gas_adjustment = 1.5 + sifnode.peggy2_set_cross_chain_fee(admin_account_address, hardhat_chain_id, + ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, + admin_account_name, chain_id, gas_prices, gas_adjustment) + + sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) + + return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ + relayers, witnesses, validator0_home, chain_dir + + def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, tcp_url, chain_id, peggy_sc_addrs, + evm_validator_accounts, sifnode_validators, sifnode_relayers, sifnode_witnesses, symbol_translator_file + ): + # For now we assume a single validator + evm_validator0_addr, evm_validator0_key = exactly_one(evm_validator_accounts) + + sifnode_validator0 = exactly_one(sifnode_validators) + sifnode_validator0_address = sifnode_validator0["address"] + + sifnode_relayer0 = exactly_one(sifnode_relayers) + sifnode_relayer0_mnemonic = sifnode_relayer0["name"] + sifnode_relayer0_address = sifnode_relayer0["address"] + sifnode_relayer0_home = sifnode_relayer0["home"] + + sifnode_witness0 = exactly_one(sifnode_witnesses) + sifnode_witness0_mnemonic = sifnode_witness0["name"] + sifnode_witness0_address = sifnode_witness0["address"] + sifnode_witness0_home = sifnode_witness0["home"] + + bridge_registry_contract_addr = peggy_sc_addrs["BridgeRegistry"] + # bridge_bank_contract_addr = peggy_sc_addrs["BridgeBank"] + # cosmos_bridge_contract_addr = peggy_sc_addrs["CosmosBridge"] + # rowan_contract_addr = peggy_sc_addrs["Rowan"] + + self.cmd.wait_for_sif_account_up(sifnode_validator0_address, tcp_url=tcp_url) # Required for both relayer and witness + + ebrelayer = Ebrelayer(self.cmd) + + # Example: + # ebrelayer + # init-relayer + # --network-descriptor 31337 + # --tendermint-node tcp://0.0.0.0:26657 + # --web3-provider ws://localhost:8545/ + # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e + # --validator-mnemonic relayer-0 + # --chain-id localnet + # --node tcp://0.0.0.0:26657 + # --from sif1a44w20496lgyv5asx4d4fnekdpy9xg8ymy9k3s + # --symbol-translator-file ../test/integration/config/symbol_translator.json + # --keyring-backend test + # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded + # env: + # "ETHEREUM_ADDRESS": evm_accounts["validators"][0] + # "ETHEREUM_PRIVATE_KEY": evm_account["validators"][1] + relayer0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( + "init-relayer", + hardhat_chain_id, + tcp_url, + web3_websocket_address, + bridge_registry_contract_addr, + sifnode_relayer0_mnemonic, + chain_id=chain_id, + node=tcp_url, + sign_with=sifnode_relayer0_address, + symbol_translator_file=symbol_translator_file, + ethereum_address=evm_validator0_addr, + ethereum_private_key=evm_validator0_key, + keyring_backend="test", + home=sifnode_relayer0_home, + ) + + # Example from devenv: + # ebrelayer + # init-witness + # --network-descriptor 31337 + # --tendermint-node tcp://0.0.0.0:26657 + # --web3-provider ws://localhost:8545/ + # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e + # --validator-mnemonic witness-0 + # --chain-id localnet + # --node tcp://0.0.0.0:26657 + # --from sif1l7025ps7lt24effpduwxhk45sd977djvu38lhr + # --symbol-translator-file ../test/integration/config/symbol_translator.json + # --log_format json + # --keyring-backend test + # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded + witness0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( + "init-witness", + hardhat_chain_id, + tcp_url, + web3_websocket_address, + bridge_registry_contract_addr, + sifnode_witness0_mnemonic, + chain_id=chain_id, + node=tcp_url, + sign_with=sifnode_witness0_address, + symbol_translator_file=symbol_translator_file, + ethereum_address=evm_validator0_addr, + ethereum_private_key=evm_validator0_key, + keyring_backend="test", + log_format="json", + home=sifnode_witness0_home, + ) + + return [relayer0_exec_args], [witness0_exec_args] + + def write_env_files(self, project_dir, go_bin_dir, evm_smart_contract_addrs, eth_accounts, admin_account_name, + admin_account_address, sifnode_validator0_home, sifnode_validators, sifnode_relayers, sifnode_witnesses, + tcp_url, hardhat_bind_hostname, hardhat_port, hardhat_chain_id, chain_dir, sifnoded_exec_args, + relayer0_exec_args, witness0_exec_args + ): + eth_chain_id = hardhat_chain_id + w3_url = eth.web3_host_port_url(hardhat_bind_hostname, hardhat_port) + + # @TODO At the moment, values are fed from one rendered template into the next. + # We should use values directly from parameters instead. + + def format_eth_account(eth_account): + assert eth_account[0].startswith("0x") and not eth_account[1].startswith("0x") + return {"address": eth_account[0], "privateKey": "0x" + eth_account[1]} + + def format_sif_account(sif_account): + return {"account": sif_account["address"], "name": sif_account["name"], "homeDir": sif_account["home"]} + + environment_json = { + "contractResults": { + # "completed": True, + # "output": "...", + "contractAddresses": { + "cosmosBridge": evm_smart_contract_addrs["CosmosBridge"], + "bridgeBank": evm_smart_contract_addrs["BridgeBank"], + "bridgeRegistry": evm_smart_contract_addrs["BridgeRegistry"], + "rowanContract": evm_smart_contract_addrs["Rowan"], + } + }, + "ethResults": { + # "process": { ... }, + "accounts": { + "proxyAdmin": format_eth_account(eth_accounts["proxy_admin"]), + "operator": format_eth_account(eth_accounts["operator"]), + "owner": format_eth_account(eth_accounts["owner"]), + "pauser": format_eth_account(eth_accounts["pauser"]), + "validators": [format_eth_account(a) for a in eth_accounts["validators"]], + "available": [format_eth_account(a) for a in eth_accounts["available"]], + }, + "httpHost": hardhat_bind_hostname, + "httpPort": hardhat_port, + "chainId": eth_chain_id, + }, + "goResults": { + # "completed": True, + # "output": "...", + "goBin": go_bin_dir + }, + "sifResults": { + "validatorValues": sifnode_validators, + "adminAddress": format_sif_account({"address": admin_account_address, "name": admin_account_name, "home": sifnode_validator0_home}), + "relayerAddresses": [format_sif_account(a) for a in sifnode_relayers], + "witnessAddresses": [format_sif_account(a) for a in sifnode_witnesses], + # "process": { ... }, + "tcpurl": tcp_url, + } + } + + # TODO Do we want "0x" prefixes here for private keys? + dot_env = dict_merge({ + "BASEDIR": project_dir, + "ETHEREUM_ADDRESS": eth_accounts["available"][0][0], + "ETHEREUM_PRIVATE_KEY": "0x" + eth_accounts["available"][0][1], + "ETH_ACCOUNT_OPERATOR_ADDRESS": eth_accounts["operator"][0], + "ETH_ACCOUNT_OPERATOR_PRIVATEKEY": "0x" + eth_accounts["operator"][1], + "ETH_ACCOUNT_OWNER_ADDRESS": eth_accounts["owner"][0], + "ETH_ACCOUNT_OWNER_PRIVATEKEY": "0x" + eth_accounts["owner"][1], + "ETH_ACCOUNT_PAUSER_ADDRESS": eth_accounts["pauser"][0], + "ETH_ACCOUNT_PAUSER_PRIVATEKEY": "0x" + eth_accounts["pauser"][1], + "ETH_ACCOUNT_PROXYADMIN_ADDRESS": eth_accounts["proxy_admin"][0], + "ETH_ACCOUNT_PROXYADMIN_PRIVATEKEY": "0x" + eth_accounts["proxy_admin"][1], + "ETH_ACCOUNT_VALIDATOR_ADDRESS": eth_accounts["validators"][0][0], + "ETH_ACCOUNT_VALIDATOR_PRIVATEKEY": "0x" + eth_accounts["validators"][0][1], + "ETH_CHAIN_ID": str(eth_chain_id), + "ETH_HOST": hardhat_bind_hostname, + "ETH_PORT": str(hardhat_port), + "ROWAN_SOURCE": admin_account_address, + "BRIDGE_BANK_ADDRESS": evm_smart_contract_addrs["BridgeBank"], + # "BRIDGE_REGISTRY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], + "BRIDGE_REGISTERY_ADDRESS": evm_smart_contract_addrs["BridgeRegistry"], # TODO Typo, remove, keeping it for now for compatibility + "COSMOS_BRIDGE_ADDRESS": evm_smart_contract_addrs["CosmosBridge"], + "ROWANTOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], + "BRIDGE_TOKEN_ADDRESS": evm_smart_contract_addrs["Rowan"], + "GOBIN": go_bin_dir, + "TCP_URL": tcp_url, + "VALIDATOR_ADDRESS": sifnode_validators[0]["address"], + "VALIDATOR_CONSENSUS_ADDRESS": sifnode_validators[0]["validator_consensus_address"], + "VALIDATOR_MENOMONIC": sifnode_validators[0]["mnemonic"], + "VALIDATOR_MONIKER": sifnode_validators[0]["moniker"], + "VALIDATOR_PASSWORD": sifnode_validators[0]["password"], + "VALIDATOR_PUB_KEY": sifnode_validators[0]["pub_key"], + "ADMIN_ADDRESS": admin_account_address, + "ADMIN_NAME": admin_account_name, + "CHAINDIR": chain_dir, + "HOME": chain_dir, + }, *[{ + f"ETHEREUM_ADDRESS_{i}": account[0], + f"ETHEREUM_PRIVATE_KEY_{i}": "0x" + account[1], + } for i, account in enumerate(eth_accounts["available"])], *[{ + f"ETH_ACCOUNT_VALIDATOR_{i}_ADDRESS": account[0], + f"ETH_ACCOUNT_VALIDATOR_{i}_PRIVATEKEY": "0x" + account[1], + } for i, account in enumerate(eth_accounts["validators"])], *[{ + f"VALIDATOR_ADDRESS_{i}": validator["address"], + f"VALIDATOR_CONSENSUS_ADDRESS_{i}": validator["validator_consensus_address"], + f"VALIDATOR_MENOMONIC_{i}": validator["mnemonic"], + f"VALIDATOR_MONIKER_{i}": validator["moniker"], + f"VALIDATOR_PASSWORD_{i}": validator["password"], + f"VALIDATOR_PUB_KEY_{i}": validator["pub_key"], + } for i, validator in enumerate(sifnode_validators)], *[{ + f"RELAYER_ADDRESS_{i}": relayer["address"], + f"RELAYER_NAME_{i}": relayer["name"], + } for i, relayer in enumerate(sifnode_relayers)], *[{ + f"WITNESS_ADDRESS_{i}": witness["address"], + f"WITNESS_NAME_{i}": witness["name"], + } for i, witness in enumerate(sifnode_witnesses)]) + + # launch.json for VS Code + launch_json = { + "version": "0.2.0", + "configurations": [ + { + "runtimeArgs": ["node_modules/.bin/hardhat", "run"], + "cwd": "${workspaceFolder}/smart-contracts", + "type": "node", + "request": "launch", + "name": "Debug DevENV scripts", + "skipFiles": ["/**"], + "program": "${workspaceFolder}/smart-contracts/scripts/devenv.ts", + }, { + "runtimeArgs": ["node_modules/.bin/hardhat", "test"], + "args": ["--network", "localhost"], + "cwd": "${workspaceFolder}/smart-contracts", + "type": "node", + "request": "launch", + "name": "Typescript Tests", + "skipFiles": ["/**"], + "program": "${workspaceFolder}/smart-contracts/test/watcher/watcher.ts", + }, *[{ + "name": f"Debug Relayer-{i}", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/ebrelayer", + "envFile": "${workspaceFolder}/smart-contracts/.env", + # Generally we want to use relayer0_exec_args, except for: + # - here we don't have the initial "ebrelayer" + # - here we are using "${workspaceFolder} for" "--symbol-translator-file" + # - here we don't have ETHEREUM_ADDRESS env + # TODO Probable bug, should be "eth_accounts["validators"][0][1]}" + "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["available"][i][1]}, + # "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["validators"][0][1]}, + "args": [ + "init-relayer", + "--network-descriptor", str(eth_chain_id), + "--tendermint-node", tcp_url, + "--web3-provider", w3_url, + "--bridge-registry-contract-address", evm_smart_contract_addrs["BridgeRegistry"], + "--validator-mnemonic", relayer["name"], + "--chain-id", "localnet", + "--node", tcp_url, + "--keyring-backend", "test", + "--from", relayer["address"], + "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", + "--home", relayer["home"] + ] + } for i, relayer in enumerate(sifnode_relayers)], *[{ + "name": f"Debug Witness-{i}", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/ebrelayer", + "envFile": "${workspaceFolder}/smart-contracts/.env", + # Generally we want to use witness0_exec_args, except for: + # - here we don't have the initial "ebrelayer" + # - here we are using "${workspaceFolder} for" "--symbol-translator-file" + # - here we don't have ETHEREUM_ADDRESS env + # TODO Probable bug, should be "eth_accounts["validators"][0][1]}" + "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["available"][i][1]}, + # "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["validators"][0][1]}, + "args": [ + "init-witness", + # TODO This is probably obsolete, need "--network-descriptor" etc. + str(eth_chain_id), + tcp_url, + w3_url, + evm_smart_contract_addrs["BridgeRegistry"], + witness["name"], + "--chain-id", "localnet", + "--node", tcp_url, + "--keyring-backend", "test", + "--from", witness["address"], + "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", + "--home", witness["home"] + ] + } for i, witness in enumerate(sifnode_witnesses)], { + "name": "Debug Sifnoded", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/sifnoded", + # Generally we want to use sifnoded_exec_args, except for: + # - here we don't have the initial "sifnoded" + # TODO Do not use .env file here + "envFile": "${workspaceFolder}/smart-contracts/.env", + "args": [ + "start", + "--log_format", "json", + "--log_level", "debug", + "--minimum-gas-prices", "0.5rowan", + "--rpc.laddr", tcp_url, + "--home", sifnode_validator0_home + ] + }, { + "name": "Pytest", + "type": "python", + "request": "launch", + "stopOnEntry": False, + "python": "${command:python.interpreterPath}", + "module": "pytest", + "args": [ + "-olog_cli=false", + "-olog_level=DEBUG", + "-olog_file=/tmp/pytest.out", + "-v", + "test/integration/src/py/test_eth_transfers.py::test_eth_to_ceth_and_back_to_eth" + ], + "cwd": "${workspaceRoot}", + "env": dot_env, + "debugOptions": ["WaitOnAbnormalExit", "WaitOnNormalExit", "RedirectOutput"] + } + ] + } + + # IntelliJ + def render_intellij_run_xml(name, joined_args, package, filepath, envs): + def q(s): return s # TODO Qoute/escape + + # since the contents is being fed from launch_json, we have ${workspaceFolder} in joined_args + joined_args = re.sub("\\${workspaceFolder}/", "", joined_args) + + return [ + "", + " ", + " ", + " ", + " ", + ] + (( + [" "] + + [" " for k, v in envs.items()] + + [" "] + ) if envs else []) + [ + " ", + " ", + " ", + " ", + " ", + " ", + "", + ] + + intellij_sifnoded_configs = [] + intellij_ebrelayer_configs = [] + intellij_witness_configs = [] + for config in launch_json["configurations"]: + if config["name"].startswith("Debug Relayer"): + intellij_ebrelayer_configs.append(render_intellij_run_xml( + "ebrelayer devenv", + " ".join(config["args"]), + "github.com/Sifchain/sifnode/cmd/ebrelayer", + "$PROJECT_DIR$/cmd/ebrelayer/main.go", + {"ETHEREUM_PRIVATE_KEY": dot_env["ETHEREUM_PRIVATE_KEY"]})) + elif config["name"].startswith("Debug Witness"): + intellij_witness_configs.append(render_intellij_run_xml( + "witness devenv", + " ".join(config["args"]), + "github.com/Sifchain/sifnode/cmd/ebrelayer", + "$PROJECT_DIR$/cmd/ebrelayer/main.go", + {"ETHEREUM_PRIVATE_KEY": dot_env["ETHEREUM_PRIVATE_KEY"]})) + elif config["name"].startswith("Debug Sifnoded"): + intellij_sifnoded_configs.append(render_intellij_run_xml( + "sifnoded devenv", + " ".join(config["args"]), + "github.com/Sifchain/sifnode/cmd/sifnoded", + "$PROJECT_DIR$/cmd/sifnoded/main.go", + {})) + + intellij_ebrelayer_config = exactly_one(intellij_ebrelayer_configs) + intellij_witness_config = exactly_one(intellij_witness_configs) + intellij_sifnoded_config = exactly_one(intellij_sifnoded_configs) + + run_dir = self.project.project_dir(".run") + self.cmd.mkdir(run_dir) + vscode_dir = self.project.project_dir(".vscode") + self.cmd.mkdir(vscode_dir) + + self.cmd.write_text_file(self.project.project_dir("smart-contracts/environment.json"), json.dumps(environment_json, indent=2)) + self.cmd.write_text_file(self.project.project_dir("smart-contracts/env.json"), json.dumps(dot_env, indent=2)) + self.cmd.write_text_file(self.project.project_dir("smart-contracts/.env"), joinlines(format_as_shell_env_vars(dot_env, export=True))) + self.cmd.write_text_file(os.path.join(vscode_dir, "launch.json"), json.dumps(launch_json, indent=2)) + self.cmd.write_text_file(os.path.join(run_dir, "ebrelayer.run.xml"), joinlines(intellij_ebrelayer_config)) + self.cmd.write_text_file(os.path.join(run_dir, "witness.run.xml"), joinlines(intellij_witness_config)) + self.cmd.write_text_file(os.path.join(run_dir, "sifnoded.run.xml"), joinlines(intellij_sifnoded_config)) + + return environment_json, dot_env, launch_json, intellij_ebrelayer_config, intellij_witness_config, intellij_sifnoded_config + + +class IBCEnvironment(IntegrationTestsEnvironment): + def __init__(self, cmd): + super().__init__(cmd) + + def run(self): + chainnet0 = "sifchain-ibc-0" + chainnet1 = "sifchain-ibc-1" + ipaddr0 = "192.168.65.2" + ipaddr1 = "192.168.65.3" + subnet = "192.168.65.1/24" + # Mnemonics can be generated with "sifgen key generate" or "sifnoded keys mnemonic", but that gives us 24 words + # and here are only 12. + # A mnemonic contains both public and private key. Public key is the address, there can only be one such entry + # in the keyring. + mnemonic = "toddler spike waste purpose neutral beach science dawn joke stock help beyond" + + sifgen = Sifgen(self.cmd) + # This does not work - "--keyring-backend" is not supported + x = sifgen.create_standalone(chainnet0, "chain1", mnemonic, ipaddr0, keyring_backend=None) + + print() diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 5e1bb98191..0f8b27748d 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -4,10 +4,10 @@ import time import web3 -import main import eth import truffle import hardhat +import run_env import sifchain from common import * @@ -58,7 +58,7 @@ def get_env_ctx(cmd=None, env_file=None, env_vars=None): return ctx def get_env_ctx_peggy2(): - cmd = main.Integrator() + cmd = run_env.Integrator() dot_env_vars = json.loads(cmd.read_text_file(cmd.project.project_dir("smart-contracts/env.json"))) environment_vars = json.loads(cmd.read_text_file(cmd.project.project_dir("smart-contracts/environment.json"))) @@ -112,7 +112,7 @@ def get_env_ctx_peggy2(): assert ctx.eth.fixed_gas_args["gasPrice"] == 1 * eth.GWEI + 7 # Monkeypatching for peggy2 extras - # TODO These are set in main.py:Peggy2Environment.init_sifchain(), specifically "sifnoded tx ethbridge set-cross-chain-fee" + # TODO These are set in run_env.py:Peggy2Environment.init_sifchain(), specifically "sifnoded tx ethbridge set-cross-chain-fee" # Consider passing them via environment ctx.cross_chain_fee_base = 1 ctx.cross_chain_lock_fee = 1 @@ -122,7 +122,7 @@ def get_env_ctx_peggy2(): return ctx def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): - cmd = cmd or main.Integrator() + cmd = cmd or run_env.Integrator() if "ENV_FILE" in os.environ: env_file = os.environ["ENV_FILE"] @@ -271,7 +271,7 @@ def advance_block_w3(self, number): def advance_block_truffle(self, number): args = ["npx", "truffle", "exec", "scripts/advanceBlock.js", str(number)] - self.cmd.execst(args, cwd=main.project_dir("smart-contracts")) + self.cmd.execst(args, cwd=run_env.project_dir("smart-contracts")) def advance_block(self, number): if on_peggy2_branch: From 9d07790ce6af1d06dabb96a710374e557da87c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 18 Jan 2022 12:17:31 +0100 Subject: [PATCH 10/70] Add project shortcut commands --- test/integration/framework/hardhat.py | 36 ++++++------- test/integration/framework/main.py | 7 +++ test/integration/framework/run_env.py | 74 +++++++++++++-------------- 3 files changed, 58 insertions(+), 59 deletions(-) diff --git a/test/integration/framework/hardhat.py b/test/integration/framework/hardhat.py index 9f89e3eaf0..df744c463d 100644 --- a/test/integration/framework/hardhat.py +++ b/test/integration/framework/hardhat.py @@ -1,19 +1,9 @@ import json import web3 -from dataclasses import dataclass from common import * from command import buildcmd -# Peggy uses different smart contracts (e.g. in Peggy2.0 there is no BridgeToken, there is CosmosBridge etc.) -@dataclass -class Peggy2SmartContractAddresses: - bridge_bank: str - bridge_registry: str - cosmos_bridge: str - rowan: str - - class Hardhat: def __init__(self, cmd): self.cmd = cmd @@ -31,6 +21,10 @@ def build_start_args(self, hostname=None, port=None, fork=None, fork_block_numbe return buildcmd(args, cwd=self.project.smart_contracts_dir) def compile_smart_contracts(self): + # Creates: + # smart-contracts/artifacts + # smart-contracts/build + # smart-contracts/cache self.project.npx(["hardhat", "compile"], cwd=project_dir("smart-contracts"), pipe=False) def deploy_smart_contracts(self): @@ -88,34 +82,34 @@ def default_accounts(): # Format: [address, private_key] # Note: for compatibility with ganache, private keys should be stripped of "0x" prefix # (when you pass a private key to ebrelayer via ETHEREUM_PRIVATE_KEY, the key is treated as invalid) - return [[web3.Web3.toChecksumAddress(address), private_key[2:]] for address, private_key in [[ + return [[web3.Web3.toChecksumAddress(address), private_key] for address, private_key in [[ "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", ], [ "0x70997970c51812dc3a010c7d01b50e0d17dc79c8", - "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", ], [ "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", - "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + "5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", ], [ "0x90f79bf6eb2c4f870365e785982e1f101e93b906", - "0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + "7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", ], [ "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", - "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", ], [ "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", - "0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + "8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", ], [ "0x976ea74026e726554db657fa54763abd0c3a0aa9", - "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + "92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", ], [ "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", - "0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + "4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", ], [ "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", - "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + "dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", ], [ "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", - "0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", + "2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", ]]] diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index b8584e71e3..8be2f054f2 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -25,6 +25,13 @@ def main(argv): project.clean(*argv[1:]) elif what == "rebuild": cmd.project.rebuild() + elif what == "dev-clean": + cmd.dev_clean() + elif what == "dev-build": + cmd.dev_build() + elif what == "dev-rebuild": + cmd.dev_clean() + cmd.dev_build() elif what == "run-ui-env": e = UIStackEnvironment(cmd) e.stack_save_snapshot() diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index e7ce379651..f70ca25d89 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -212,6 +212,41 @@ def wait_for_sif_account_up(self, address, tcp_url=None): log.debug(f"Waiting for sif account {address}... ({repr(e)})") time.sleep(1) + def dev_clean(self): + if on_peggy2_branch: + assert False, "Not implemented yet" + else: + self._dev_clean_peggy1() + + def dev_build(self): + if on_peggy2_branch: + assert False, "Not implemented yet" + else: + self._dev_build_peggy1() + + def _dev_clean_peggy1(self): + self.rmf(self.project.project_dir("smart-contracts", "node_modules")) + + # Output from "truffle compile" + self.rmf(self.project.project_dir("smart-contracts", "build")) + + for filename in ["sifnoded", "ebrelayer", "sifgen"]: + self.rmf(os.path.join(self.project.go_bin_dir, filename)) + + def _dev_build_peggy1(self): + self._npm_install() + self.execst(["make", "install"], cwd=self.project.project_dir(), pipe=False) + self.execst(["npx", "hardhat", "compile"], cwd=self.project.project_dir("smart-contracts"), pipe=False) + self.execst([self.project.project_dir("smart-contracts", "node_modules", ".bin", "truffle"), "compile"], + cwd=self.project.project_dir("smart-contracts"), pipe=False) + + def _dev_clean_peggy2(self): + for file in [".proto-gen", ".run", "cmd/ebrelayer/contract/generated/artifacts", "smart-contracts/.hardhat-compile"]: + self.rmf(self.project.project_dir(file)) + + def _npm_install(self): + self.execst(["npm", "install"], cwd=self.project.project_dir("smart-contracts"), pipe=False) + class UIStackEnvironment: def __init__(self, cmd): @@ -892,7 +927,7 @@ def run(self): return hardhat_proc, sifnoded_proc, relayer0_proc, witness0_proc def init_smart_contracts(self, w3_url, operator_account, deployed_contract_addresses): - # Looks like this is already done somewhere else... + # TODO Looks like this is already done somewhere else... # operator_addr, operator_private_key = operator_account # w3_conn = eth.web3_wait_for_connection_up(w3_url) # eth_tx = eth.EthereumTxWrapper(w3_conn, True) @@ -1047,31 +1082,11 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, sifnode_witness0_home = sifnode_witness0["home"] bridge_registry_contract_addr = peggy_sc_addrs["BridgeRegistry"] - # bridge_bank_contract_addr = peggy_sc_addrs["BridgeBank"] - # cosmos_bridge_contract_addr = peggy_sc_addrs["CosmosBridge"] - # rowan_contract_addr = peggy_sc_addrs["Rowan"] self.cmd.wait_for_sif_account_up(sifnode_validator0_address, tcp_url=tcp_url) # Required for both relayer and witness ebrelayer = Ebrelayer(self.cmd) - # Example: - # ebrelayer - # init-relayer - # --network-descriptor 31337 - # --tendermint-node tcp://0.0.0.0:26657 - # --web3-provider ws://localhost:8545/ - # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e - # --validator-mnemonic relayer-0 - # --chain-id localnet - # --node tcp://0.0.0.0:26657 - # --from sif1a44w20496lgyv5asx4d4fnekdpy9xg8ymy9k3s - # --symbol-translator-file ../test/integration/config/symbol_translator.json - # --keyring-backend test - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded - # env: - # "ETHEREUM_ADDRESS": evm_accounts["validators"][0] - # "ETHEREUM_PRIVATE_KEY": evm_account["validators"][1] relayer0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( "init-relayer", hardhat_chain_id, @@ -1089,21 +1104,6 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, home=sifnode_relayer0_home, ) - # Example from devenv: - # ebrelayer - # init-witness - # --network-descriptor 31337 - # --tendermint-node tcp://0.0.0.0:26657 - # --web3-provider ws://localhost:8545/ - # --bridge-registry-contract-address 0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e - # --validator-mnemonic witness-0 - # --chain-id localnet - # --node tcp://0.0.0.0:26657 - # --from sif1l7025ps7lt24effpduwxhk45sd977djvu38lhr - # --symbol-translator-file ../test/integration/config/symbol_translator.json - # --log_format json - # --keyring-backend test - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded witness0_exec_args = ebrelayer.peggy2_build_ebrelayer_cmd( "init-witness", hardhat_chain_id, @@ -1449,5 +1449,3 @@ def run(self): sifgen = Sifgen(self.cmd) # This does not work - "--keyring-backend" is not supported x = sifgen.create_standalone(chainnet0, "chain1", mnemonic, ipaddr0, keyring_backend=None) - - print() From b4a8ebd1fde4a807cf413c54e436d039249a7518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 26 Jan 2022 20:26:32 +0100 Subject: [PATCH 11/70] Added diagnostic assertions for inflate_tokens --- test/integration/framework/test_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 0f8b27748d..a99c439e10 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -166,6 +166,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): else: operator_address = env_vars["OPERATOR_ADDRESS"] operator_private_key = env_vars.get("OPERATOR_PRIVATE_KEY") + operator_address = web3.Web3.toChecksumAddress(operator_address) # Already added below # collected_private_keys[operator_address] = operator_private_key @@ -294,6 +295,7 @@ def get_blocklist_sc(self): def get_bridge_bank_sc(self): abi, _, address = self.abi_provider.get_descriptor("BridgeBank") + assert address, "No address for BridgeBank" result = self.w3_conn.eth.contract(address=address, abi=abi) return result @@ -542,6 +544,11 @@ def create_sifchain_addr(self, moniker=None, fund_amounts=None): acct = self.sifnode.keys_add_1(moniker) sif_address = acct["address"] if fund_amounts: + rowan_source_balances = self.get_sifchain_balance(self.rowan_source) + for required_amount, denom in fund_amounts: + available_amount = rowan_source_balances.get(denom, 0) + assert available_amount >= required_amount, "Rowan source {} would need {}, but only has {}".format( + self.rowan_source, sif_format_amount(required_amount, denom), sif_format_amount(available_amount, denom)) old_balances = self.get_sifchain_balance(sif_address) self.send_from_sifchain_to_sifchain(self.rowan_source, sif_address, fund_amounts) self.wait_for_sif_balance_change(sif_address, old_balances, min_changes=fund_amounts) @@ -561,8 +568,9 @@ def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): res = self.sifnode.sifnoded_exec(args, sifnoded_home=self.sifnode.home, keyring_backend=self.sifnode.keyring_backend) retval = json.loads(stdout(res)) raw_log = retval["raw_log"] - if "insufficient funds" in raw_log: - raise Exception(raw_log) + for bad_thing in ["insufficient funds", "signature verification failed"]: + if bad_thing in raw_log: + raise Exception(raw_log) return retval def get_sifchain_balance(self, sif_addr): From a3a1ac84a28ffbbdde840e5b178166adc2964f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 06:15:57 +0100 Subject: [PATCH 12/70] Update method of advancing blocks on local nodes --- test/integration/framework/test_utils.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index a99c439e10..9551cdb4d9 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -268,25 +268,16 @@ def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, def advance_block_w3(self, number): for _ in range(number): + # See smart-contracts/node_modules/@openzeppelin/test-helpers/src/time.js:advanceBlockTo() self.w3_conn.provider.make_request("evm_mine", []) - def advance_block_truffle(self, number): - args = ["npx", "truffle", "exec", "scripts/advanceBlock.js", str(number)] - self.cmd.execst(args, cwd=run_env.project_dir("smart-contracts")) - - def advance_block(self, number): - if on_peggy2_branch: - self.advance_block_w3(number) - else: - self.advance_block_truffle(number) # TODO Probably calls the same, check and remove - def advance_blocks(self, number=50): # TODO Move to eth (it should be per-w3_conn) if self.eth.is_local_node: previous_block = self.eth.w3_conn.eth.block_number - self.advance_block(number) + self.advance_block_w3(number) assert self.eth.w3_conn.eth.block_number - previous_block >= number - # Otherwise just wait + # Otherwise do nothing (e.g. wait for balance change takes longer) def get_blocklist_sc(self): abi, _, address = self.abi_provider.get_descriptor("Blocklist") From 5b55447217767b0d9d53eeda0c48592b385ebcf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 06:20:56 +0100 Subject: [PATCH 13/70] Ability to override smart contract addresses in ENV_FILE --- test/integration/framework/test_utils.py | 23 ++++++++++++++++++++--- test/integration/framework/truffle.py | 14 +++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 9551cdb4d9..8820206b1d 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -62,13 +62,14 @@ def get_env_ctx_peggy2(): dot_env_vars = json.loads(cmd.read_text_file(cmd.project.project_dir("smart-contracts/env.json"))) environment_vars = json.loads(cmd.read_text_file(cmd.project.project_dir("smart-contracts/environment.json"))) + deployed_contract_address_overrides = get_overrides_for_smart_contract_addresses(dot_env_vars) tmp = environment_vars["contractResults"]["contractAddresses"] - deployed_contract_addresses = { + deployed_contract_addresses = dict_merge({ "BridgeBank": tmp["bridgeBank"], "CosmosBridge": tmp["cosmosBridge"], "BridgeRegistry": tmp["bridgeRegistry"], "Rowan": tmp["rowanContract"], - } + }, deployed_contract_address_overrides) abi_provider = hardhat.HardhatAbiProvider(cmd, deployed_contract_addresses) # TODO We're mixing "OPERATOR" vs. "OWNER" @@ -198,6 +199,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): sifnode_url = env_vars.get("SIFNODE") # Defaults to "tcp://localhost:26657" sifnoded_home = None # Implies default ~/.sifnoded + deployed_smart_contract_address_overrides = get_overrides_for_smart_contract_addresses(env_vars) w3_conn = eth.web3_connect(w3_url, websocket_timeout=90) @@ -210,7 +212,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): eth_node_is_local = deployment_name is None ctx_eth = eth.EthereumTxWrapper(w3_conn, eth_node_is_local) - abi_provider = truffle.GanacheAbiProvider(cmd, artifacts_dir, ethereum_network_id) + abi_provider = truffle.GanacheAbiProvider(cmd, artifacts_dir, ethereum_network_id, deployed_smart_contract_address_overrides) ctx = EnvCtx(cmd, w3_conn, ctx_eth, abi_provider, operator_address, sifnoded_home, sifnode_url, sifnode_chain_id, rowan_source, CETH, generic_erc20_contract_name) if operator_private_key: @@ -245,6 +247,21 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): return ctx +def get_overrides_for_smart_contract_addresses(env_vars): + mappings = { + "BridgeBank": "BRIDGE_BANK_ADDRESS", + "BridgeRegistry": "BRIDGE_REGISTRY_ADDRESS", + "CosmosBridge": "COSMOS_BRIDGE_ADDRESS", # Peggy2 only? + "Rowan": "ROWAN_ADDRESS", # Peggy2 only? + "BridgeToken": "BRIDGE_TOKEN_ADDRESS", # Peggy1 only + } + tmp = {} + for k, v in mappings.items(): + if v in env_vars: + tmp[k] = web3.Web3.toChecksumAddress(env_vars[v]) + return tmp + + def sif_addr_to_evm_arg(sif_address): return sif_address.encode("UTF-8") diff --git a/test/integration/framework/truffle.py b/test/integration/framework/truffle.py index 1a24822384..4aaf4769de 100644 --- a/test/integration/framework/truffle.py +++ b/test/integration/framework/truffle.py @@ -21,10 +21,11 @@ def start_ganache_cli(env, mnemonic=None, db=None, port=None, host=None, network class GanacheAbiProvider: - def __init__(self, cmd, artifacts_dir, ethereum_network_id): + def __init__(self, cmd, artifacts_dir, ethereum_network_id, deployed_smart_contract_address_overrides): self.cmd = cmd self.artifacts_dir = artifacts_dir self.ethereum_default_network_id = ethereum_network_id + self.deployed_smart_contract_address_overrides = deployed_smart_contract_address_overrides def get_descriptor(self, sc_name): path = self.cmd.project.project_dir(self.artifacts_dir, "contracts/{}.json".format(sc_name)) @@ -32,8 +33,11 @@ def get_descriptor(self, sc_name): abi = tmp["abi"] bytecode = tmp["bytecode"] deployed_address = None - if ("networks" in tmp) and (self.ethereum_default_network_id is not None): - str_network_id = str(self.ethereum_default_network_id) - if str_network_id in tmp["networks"]: - deployed_address = tmp["networks"][str_network_id]["address"] + if (self.deployed_smart_contract_address_overrides is not None) and (sc_name in self.deployed_smart_contract_address_overrides): + deployed_address = self.deployed_smart_contract_address_overrides[sc_name] + else: + if ("networks" in tmp) and (self.ethereum_default_network_id is not None): + str_network_id = str(self.ethereum_default_network_id) + if str_network_id in tmp["networks"]: + deployed_address = tmp["networks"][str_network_id]["address"] return abi, bytecode, deployed_address From e836b19dd26421bb453f325d6e13cea583601fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 06:22:57 +0100 Subject: [PATCH 14/70] Add diagnostic logging, cleanup --- test/integration/framework/inflate_tokens.py | 3 +++ test/integration/framework/main.py | 1 - test/integration/framework/test_utils.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index 341e193af4..fea6d74d25 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -149,9 +149,12 @@ def transfer_from_eth_to_sifnode(self, from_eth_addr, to_sif_addr, tokens_to_tra pending_txs.append(self.ctx.tx_bridge_bank_lock_eth(from_eth_addr, to_sif_addr, amount)) sent_amounts.append([amount, self.ctx.ceth_symbol]) self.wait_for_all(pending_txs) + log.info("{} Ethereum transactions commited: {}".format(len(pending_txs), repr(sent_amounts))) # Wait for intermediate_sif_account to receive all funds across the bridge + previous_block = self.ctx.eth.w3_conn.eth.block_number self.ctx.advance_blocks() + log.info("Ethereum blocks advanced by {}".format(self.ctx.eth.w3_conn.eth.block_number - previous_block)) self.ctx.wait_for_sif_balance_change(to_sif_addr, sif_balances_before, min_changes=sent_amounts, polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index 8be2f054f2..50143de2fe 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -119,7 +119,6 @@ def main(argv): import inflate_tokens inflate_tokens.run(*argv[1:]) elif what == "recover-eth": - import test_utils test_utils.recover_eth_from_test_accounts() elif what == "run-peggy2-tests": cmd.execst(["yarn", "test"], cwd=project.smart_contracts_dir) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 8820206b1d..3abcae52f0 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -389,6 +389,8 @@ def tx_approve_and_lock(self, token_sc, from_eth_acct, to_sif_acct, amount): bridge_bank_sc = self.get_bridge_bank_sc() txhash1 = self.tx_approve(token_sc, self.operator, bridge_bank_sc.address, amount) txhash2 = self.tx_bridge_bank_lock_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) + log.debug("tx_approve_and_lock: {} '{}' ({}) from {} to {}".format(amount, token_sc.functions.name().call(), + token_sc.functions.symbol().call(), from_eth_acct, to_sif_acct)) return txhash1, txhash2 # @@ -772,7 +774,8 @@ def sanity_check(self): assert (self.sifnode_chain_id != "sifchain-testnet-1") or (bridge_bank_sc.address == "0x6CfD69783E3fFb44CBaaFF7F509a4fcF0d8e2835") assert (self.sifnode_chain_id != "sifchain-devnet-1") or (bridge_bank_sc.address == "0x96DC6f02C66Bbf2dfbA934b8DafE7B2c08715A73") assert (self.sifnode_chain_id != "localnet") or (bridge_bank_sc.address == "0x30753E4A8aad7F8597332E813735Def5dD395028") - assert bridge_bank_sc.functions.owner().call() == self.operator + assert bridge_bank_sc.functions.owner().call() == self.operator, \ + "BridgeBank owner is {}, but OPERATOR is {}".format(bridge_bank_sc.functions.owner().call(), self.operator) operator_balance = self.eth.get_eth_balance(self.operator) / eth.ETH assert operator_balance >= 1, "Insufficient operator balance, should be at least 1 ETH" From 3d9299b6ec3ac87a1b5d93e444de888b42793684 Mon Sep 17 00:00:00 2001 From: Michael Ho Date: Mon, 24 Jan 2022 15:55:05 -0600 Subject: [PATCH 15/70] Update siftool to find correct python3 path --- test/integration/framework/siftool | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/siftool b/test/integration/framework/siftool index cefa5dd715..94d49b4bd9 100755 --- a/test/integration/framework/siftool +++ b/test/integration/framework/siftool @@ -1,4 +1,4 @@ -#!/bin/python3 +#!/usr/bin/env python3 # This is an executable command-line frontend that makes sure we are running with a suitable Python virtual environment. # If the virtual environment does not exist yet, it is created on first use. From 222d7a6130fcaf1bdea944ff9036f9b05d1e53d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 07:34:06 +0100 Subject: [PATCH 16/70] Import Ethereum private key from mnemonic if necessary --- test/integration/framework/eth.py | 14 ++++++++++++++ test/integration/framework/test_utils.py | 6 +----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/test/integration/framework/eth.py b/test/integration/framework/eth.py index 0befa9070b..de4eeb8d53 100644 --- a/test/integration/framework/eth.py +++ b/test/integration/framework/eth.py @@ -75,6 +75,8 @@ def set_private_key(self, addr, private_key): if private_key is None: self.private_keys.pop(addr) # Remove else: + is_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key) + private_key = private_key if is_hex else _get_account_from_mnemonic(private_key) # Convert from mnemonic if necessary assert (not private_key.startswith("0x")) and (private_key == private_key.lower()), "Private key must be in lowercase hex without '0x' prefix" check_addr = self.w3_conn.eth.account.from_key(private_key).address assert check_addr == addr, f"Private key does not correspond to given address {addr}" @@ -370,3 +372,15 @@ def estimate_fees(self, tx): @staticmethod def estimate_gas_price(): return 0 + + +__web3_enabled_unaudited_hdwallet_features = False + +# https://stackoverflow.com/questions/68050645/how-to-create-a-web3py-account-using-mnemonic-phrase +def _get_account_from_mnemonic(mnemonic): + a = web3.Web3().eth.account + global __web3_enabled_unaudited_hdwallet_features + if not __web3_enabled_unaudited_hdwallet_features: + a.enable_unaudited_hdwallet_features() + __web3_enabled_unaudited_hdwallet_features = True + return a.from_mnemonic(mnemonic, account_path="m/44'/60'/0'/0/0").privateKey.hex()[2:] diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 3abcae52f0..846f4b985a 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -255,11 +255,7 @@ def get_overrides_for_smart_contract_addresses(env_vars): "Rowan": "ROWAN_ADDRESS", # Peggy2 only? "BridgeToken": "BRIDGE_TOKEN_ADDRESS", # Peggy1 only } - tmp = {} - for k, v in mappings.items(): - if v in env_vars: - tmp[k] = web3.Web3.toChecksumAddress(env_vars[v]) - return tmp + return dict(((k, web3.Web3.toChecksumAddress(env_vars[v])) for k, v in mappings.items() if v in env_vars)) def sif_addr_to_evm_arg(sif_address): From 5b1cfa035de6ea221746e1991bb007658bf0a6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 09:01:10 +0100 Subject: [PATCH 17/70] Caching of node_modules --- test/integration/framework/command.py | 6 +- test/integration/framework/main.py | 13 +-- test/integration/framework/project.py | 113 +++++++++++++++++++++----- test/integration/framework/run_env.py | 34 +------- 4 files changed, 103 insertions(+), 63 deletions(-) diff --git a/test/integration/framework/command.py b/test/integration/framework/command.py index ece1f2fed7..555503dbc1 100644 --- a/test/integration/framework/command.py +++ b/test/integration/framework/command.py @@ -91,7 +91,7 @@ def __tar_compression_option(self, tarfile): def tar_create(self, path, tarfile): comp = self.__tar_compression_option(tarfile) # tar on 9p filesystem reports "file shrank by ... bytes" and exits with errorcode 1 - tar_quirks = True + tar_quirks = False if tar_quirks: tmpdir = self.mktempdir() try: @@ -115,3 +115,7 @@ def wait_for_file(self, path): def tcp_probe_connect(self, host, port): res = self.execst(["nc", "-z", host, str(port)], check_exit=False) return res[0] == 0 + + def sha1_of_file(self, path): + res = self.execst(["sha1sum", "-b", path]) + return stdout_lines(res)[0][:40] diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index 50143de2fe..3bd5c9d3fc 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -22,16 +22,11 @@ def main(argv): if what == "project-init": project.init() elif what == "clean": - project.clean(*argv[1:]) + project.clean() + elif what == "build": + project.build() elif what == "rebuild": - cmd.project.rebuild() - elif what == "dev-clean": - cmd.dev_clean() - elif what == "dev-build": - cmd.dev_build() - elif what == "dev-rebuild": - cmd.dev_clean() - cmd.dev_build() + project.rebuild() elif what == "run-ui-env": e = UIStackEnvironment(cmd) e.stack_save_snapshot() diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index 174ee443ca..c04a70742f 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -42,13 +42,6 @@ def __rm(self, path): else: log.debug("Nothing to delete for '{}'".format(path)) - def rebuild(self): - # Use this after switching branches (i.e. develop vs. future/peggy2) - self.clean(1) - # self.cmd.execst(["npm", "install", "-g", "ganache-cli", "dotenv", "yarn"], cwd=self.smart_contracts_dir) - self.install_smart_contracts_dependencies() - self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) - def __rm_files_develop(self): self.__rm(self.project_dir("test", "integration", "sifchainrelayerdb")) # TODO move to /tmp @@ -102,7 +95,6 @@ def __rm_files(self, level): self.__rm(self.cmd.get_user_home(".cache/yarn")) self.__rm(self.cmd.get_user_home(".sifnoded")) self.__rm(self.cmd.get_user_home(".sifnode-integration")) - self.__rm(project_dir("smart-contracts/node_modules")) # Peggy2 # Generated Go stubs (by smart-contracts/Makefile) @@ -123,12 +115,6 @@ def __rm_files(self, level): # smart-contracts/env.json # smart-contracts/environment.json - # Use this between run-env. - def clean(self, level=None): - level = 0 if level is None else int(level) - force_kill_processes(self.cmd) - self.__rm_files(level) - def yarn(self, args, cwd=None, env=None): return self.cmd.execst(["yarn"] + args, cwd=cwd, env=env, pipe=False) @@ -215,12 +201,6 @@ def write_vagrantenv_sh(self, state_vars, data_dir, ethereum_websocket_address, self.cmd.write_text_file(vagrantenv_path, joinlines(format_as_shell_env_vars(env))) self.cmd.write_text_file(project_dir("test/integration/vagrantenv.json"), json.dumps(env)) - def init(self): - self.clean() - self.cmd.rmdir(project_dir("smart-contracts/node_modules")) - self.make_go_binaries_2() - self.install_smart_contracts_dependencies() - def get_peruser_config_dir(self): return self.cmd.get_user_home(".config", "siftool") @@ -234,3 +214,96 @@ def read_peruser_config_file(self, name): return json.loads(self.cmd.read_text_file(path)) else: return None + + def init(self): + self.clean() + # self.cmd.rmdir(project_dir("smart-contracts/node_modules")) + self.make_go_binaries_2() + self.install_smart_contracts_dependencies() + + def clean(self): + if on_peggy2_branch: + for file in [".proto-gen", ".run", "cmd/ebrelayer/contract/generated/artifacts", "smart-contracts/.hardhat-compile"]: + self.cmd.rmf(self.project_dir(file)) + else: + self.cmd.rmf(self.project_dir("smart-contracts", "node_modules")) + + # Output from "truffle compile" + self.cmd.rmf(self.project_dir("smart-contracts", "build")) + + for filename in ["sifnoded", "ebrelayer", "sifgen"]: + self.cmd.rmf(os.path.join(self.go_bin_dir, filename)) + + # Use this between run-env. + def old_clean(self, level=None): + level = 0 if level is None else int(level) + force_kill_processes(self.cmd) + self.__rm_files(level) + + def build(self): + if on_peggy2_branch: + assert False, "Not implemented yet" + # self.cmd.execst(["npx", "hardhat", "compile"], cwd=self.project_dir("smart-contracts"), pipe=False) + else: + self.npm_install(self.project_dir("smart-contracts")) + self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) + self.cmd.execst([self.project_dir("smart-contracts", "node_modules", ".bin", "truffle"), "compile"], + cwd=self.project_dir("smart-contracts"), pipe=False) + + def rebuild(self): + self.clean() + self.build() + + def old_rebuild(self): + # Use this after switching branches (i.e. develop vs. future/peggy2) + self.clean(1) + # self.cmd.execst(["npm", "install", "-g", "ganache-cli", "dotenv", "yarn"], cwd=self.smart_contracts_dir) + self.install_smart_contracts_dependencies() + self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) + + def npm_install(self, path): + package_lock_json = os.path.join(path, "package-lock.json") + sha1 = self.cmd.sha1_of_file(package_lock_json) + node_modules = os.path.join(path, "node_modules") + + if self.cmd.exists(node_modules): + cache_tag_file = os.path.join(node_modules, ".siftool-cache-tag") + cache_tag = self.cmd.read_text_file(cache_tag_file) if self.cmd.exists(cache_tag_file) else None + if (cache_tag is None) or (cache_tag != sha1): + self.cmd.rmdir(node_modules) + else: + return + + assert not self.cmd.exists(node_modules) + cache_dir = os.path.join(self.get_peruser_config_dir(), "npm-cache") + cache_index = os.path.join(cache_dir, "index.json") + cache = [] + if not self.cmd.exists(cache_dir): + self.cmd.mkdir(cache_dir) + else: + cache = json.loads(self.cmd.read_text_file(cache_index)) + idx = None + for i, s in enumerate(cache): + if s == sha1: + idx = i + break + tarfile = os.path.join(cache_dir, "{}.tar".format(sha1)) + if idx is None: + saved_package_lock = self.cmd.read_text_file(os.path.join(path, "package-lock.json")) + saved_yarn_lock = self.cmd.read_text_file(os.path.join(path, "yarn.lock")) + self.cmd.execst(["npm", "install"], cwd=path, pipe=False) + cache_tag_file = os.path.join(node_modules, ".siftool-cache-tag") + self.cmd.write_text_file(cache_tag_file, sha1) + self.cmd.write_text_file(os.path.join(path, "package-lock.json"), saved_package_lock) + self.cmd.write_text_file(os.path.join(path, "yarn.lock"), saved_yarn_lock) + self.cmd.tar_create(node_modules, tarfile) + else: + cache.pop(idx) + self.cmd.tar_extract(tarfile, node_modules) + cache.insert(0, sha1) + max_cache_items = 5 + if len(cache) > max_cache_items: + for s in cache[max_cache_items:]: + self.cmd.rm(os.path.join(cache_dir, "{}.tar".format(s))) + cache = cache[:max_cache_items] + self.cmd.write_text_file(cache_index, json.dumps(cache)) diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index f70ca25d89..287633819f 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -212,40 +212,8 @@ def wait_for_sif_account_up(self, address, tcp_url=None): log.debug(f"Waiting for sif account {address}... ({repr(e)})") time.sleep(1) - def dev_clean(self): - if on_peggy2_branch: - assert False, "Not implemented yet" - else: - self._dev_clean_peggy1() - - def dev_build(self): - if on_peggy2_branch: - assert False, "Not implemented yet" - else: - self._dev_build_peggy1() - - def _dev_clean_peggy1(self): - self.rmf(self.project.project_dir("smart-contracts", "node_modules")) - - # Output from "truffle compile" - self.rmf(self.project.project_dir("smart-contracts", "build")) - - for filename in ["sifnoded", "ebrelayer", "sifgen"]: - self.rmf(os.path.join(self.project.go_bin_dir, filename)) - - def _dev_build_peggy1(self): - self._npm_install() - self.execst(["make", "install"], cwd=self.project.project_dir(), pipe=False) - self.execst(["npx", "hardhat", "compile"], cwd=self.project.project_dir("smart-contracts"), pipe=False) - self.execst([self.project.project_dir("smart-contracts", "node_modules", ".bin", "truffle"), "compile"], - cwd=self.project.project_dir("smart-contracts"), pipe=False) - - def _dev_clean_peggy2(self): - for file in [".proto-gen", ".run", "cmd/ebrelayer/contract/generated/artifacts", "smart-contracts/.hardhat-compile"]: - self.rmf(self.project.project_dir(file)) - def _npm_install(self): - self.execst(["npm", "install"], cwd=self.project.project_dir("smart-contracts"), pipe=False) + self.project.npm_install(self.project.project_dir("smart-contracts")) class UIStackEnvironment: From 5d80f7a92adcd1aec507cc68568bcb40550a6bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 13:56:15 +0100 Subject: [PATCH 18/70] Caching of node_modules for peggy2 branch --- test/integration/framework/main.py | 2 ++ test/integration/framework/project.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index 3bd5c9d3fc..ea65cf75c2 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -27,6 +27,8 @@ def main(argv): project.build() elif what == "rebuild": project.rebuild() + elif what == "project": + return getattr(project, argv[1])(*argv[2:]) elif what == "run-ui-env": e = UIStackEnvironment(cmd) e.stack_save_snapshot() diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index c04a70742f..1dca0e8a73 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -242,8 +242,8 @@ def old_clean(self, level=None): def build(self): if on_peggy2_branch: - assert False, "Not implemented yet" - # self.cmd.execst(["npx", "hardhat", "compile"], cwd=self.project_dir("smart-contracts"), pipe=False) + self.npm_install(self.project_dir("smart-contracts")) + self.npx(["hardhat", "compile"], cwd=self.project_dir("smart-contracts"), pipe=False) else: self.npm_install(self.project_dir("smart-contracts")) self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) @@ -289,13 +289,13 @@ def npm_install(self, path): break tarfile = os.path.join(cache_dir, "{}.tar".format(sha1)) if idx is None: - saved_package_lock = self.cmd.read_text_file(os.path.join(path, "package-lock.json")) - saved_yarn_lock = self.cmd.read_text_file(os.path.join(path, "yarn.lock")) + saved = dict(((f, self.cmd.read_text_file(f)) + for f in [os.path.join(path, x) for x in ["package-lock.json", "yarn.lock"]] if self.cmd.exists(f))) self.cmd.execst(["npm", "install"], cwd=path, pipe=False) cache_tag_file = os.path.join(node_modules, ".siftool-cache-tag") self.cmd.write_text_file(cache_tag_file, sha1) - self.cmd.write_text_file(os.path.join(path, "package-lock.json"), saved_package_lock) - self.cmd.write_text_file(os.path.join(path, "yarn.lock"), saved_yarn_lock) + for file, contents in saved.items(): + self.cmd.write_text_file(file, contents) self.cmd.tar_create(node_modules, tarfile) else: cache.pop(idx) From 18db43b8f897cfd527b66b4f65f706d0fea09571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 14:29:47 +0100 Subject: [PATCH 19/70] Bugfix --- test/integration/framework/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index 1dca0e8a73..8a4715cb8b 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -262,7 +262,8 @@ def old_rebuild(self): self.cmd.execst(["make", "install"], cwd=self.project_dir(), pipe=False) def npm_install(self, path): - package_lock_json = os.path.join(path, "package-lock.json") + # TODO Add package-lock.json also on future/peggy2 branch? + package_lock_json = os.path.join(path, "package.json" if on_peggy2_branch else "package-lock.json") sha1 = self.cmd.sha1_of_file(package_lock_json) node_modules = os.path.join(path, "node_modules") From 3e92c8df7332990c2edd5d33c6c5093b5f689d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 28 Jan 2022 15:57:18 +0100 Subject: [PATCH 20/70] Parse output of 'sifnoded tx ethbridge' --- test/integration/framework/test_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 846f4b985a..5cec3c72cf 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -536,9 +536,9 @@ def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, den self._sifnoded_home_arg() + \ self._sifnoded_chain_id_and_node_arg() res = self.sifnode.sifnoded_exec(args, keyring_backend=self.sifnode.keyring_backend) - result = json.loads(stdout(res)) + result = sifnoded_parse_output_lines(stdout(res)) assert "failed to execute message" not in result["raw_log"] - return json.loads(stdout(res)) + return result def create_sifchain_addr(self, moniker=None, fund_amounts=None): """ @@ -809,3 +809,12 @@ def recover_eth_from_test_accounts(): ctx.eth.send_eth(addr, ctx.operator, to_recover) total_recovered += to_recover log.info("Total recovered: {} ETH".format(total_recovered/eth.ETH)) + + +def sifnoded_parse_output_lines(stdout): + pat = re.compile("^(.*?): (.*)$") + result = {} + for line in stdout.splitlines(): + m = pat.match(line) + result[m[1]] = m[2] + return result From 9cefcbc8734fcb7316825b0edde378642018fc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 30 Jan 2022 14:22:59 +0100 Subject: [PATCH 21/70] Change sifnoded output to json --- test/integration/framework/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 5cec3c72cf..695722aec0 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -531,12 +531,13 @@ def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, den "--from", from_sif_addr, # Mandatory, either name from keyring or address "--gas-prices", "0.5rowan", "--gas-adjustment", "1.5", + "--output", "json", "-y" ] + \ self._sifnoded_home_arg() + \ self._sifnoded_chain_id_and_node_arg() res = self.sifnode.sifnoded_exec(args, keyring_backend=self.sifnode.keyring_backend) - result = sifnoded_parse_output_lines(stdout(res)) + result = json.loads(stdout(res)) assert "failed to execute message" not in result["raw_log"] return result From 64efe1085b1a3f8bc565b605324bb36c7c6da774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 31 Jan 2022 21:52:09 +0100 Subject: [PATCH 22/70] Add delays around set_cross_chain_fee to work aroung the wrong sequence --- test/integration/framework/run_env.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 287633819f..11eb201cd3 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -1015,6 +1015,10 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh assert res[0]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" assert res[1]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" + # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid + # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) + # TODO Can we make it more robust? + time.sleep(10) cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 @@ -1025,6 +1029,9 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, admin_account_name, chain_id, gas_prices, gas_adjustment) + # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid + # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) + time.sleep(10) sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ From 1ef181edb70a59dcde3d0e1619e2568120845718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 31 Jan 2022 22:29:31 +0100 Subject: [PATCH 23/70] Change output of sifnoded to json --- test/integration/framework/sifchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index e13b22b353..7d74de78cc 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -136,7 +136,8 @@ def peggy2_token_registry_register_all(self, registry_path, gas_prices, gas_adju chain_id ): args = ["tx", "tokenregistry", "register-all", registry_path, "--gas-prices", sif_format_amount(*gas_prices), - "--gas-adjustment", str(gas_adjustment), "--from", from_account, "--chain-id", chain_id, "--yes"] + "--gas-adjustment", str(gas_adjustment), "--from", from_account, "--chain-id", chain_id, "--output", "json", + "--yes"] res = self.sifnoded_exec(args, keyring_backend=self.keyring_backend, sifnoded_home=self.home) return [json.loads(x) for x in stdout(res).splitlines()] From 3670fc5cf4e68533af76636b4fc69d12b722291b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 13 Feb 2022 11:09:11 +0100 Subject: [PATCH 24/70] WIP: add tokens for Peggy2 IBC integration tests --- test/integration/framework/run_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 11eb201cd3..5087f65422 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -840,7 +840,7 @@ def run(self): [999999 * 10**21, "rowan"], [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"], [999999 * 10**21, ceth_symbol], - ] + ] + [[10**18, "test{}".format(i)] for i in range(1, 6)] validator_power = 100 seed_ip_address = "10.10.1.1" tendermint_port = 26657 @@ -848,7 +848,7 @@ def run(self): tokens = [ [10**20, "rowan"], [2 * 10**19, "ceth"] - ] + ] + [[10**18, "xtest{}".format(i)] for i in range(1, 6)] registry_json = project_dir("smart-contracts", "src", "devenv", "registry.json") sifnoded_network_dir = "/tmp/sifnodedNetwork" # Gets written to .vscode/launch.json self.cmd.rmdir(sifnoded_network_dir) From 9841e6f937b168a5a4ad438fa457553c6b143d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 3 Mar 2022 09:39:45 +0100 Subject: [PATCH 25/70] Peggy2.0 changes to siftool --- test/integration/framework/run_env.py | 20 ++++++++++---------- test/integration/framework/sifchain.py | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 5087f65422..7381846118 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -1011,14 +1011,10 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_address, chain_id) log.debug("Result from token registry: {}".format(repr(res))) - assert len(res) == 2 - assert res[0]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" - assert res[1]["raw_log"] == "failed to execute message; message index: 0: unauthorised signer: invalid address" + assert len(res) == 1 + assert res[0]["code"] == 0 - # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid - # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) - # TODO Can we make it more robust? - time.sleep(10) + self.wait_for_last_transaction_to_be_mined() cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 @@ -1029,14 +1025,18 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, admin_account_name, chain_id, gas_prices, gas_adjustment) - # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid - # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) - time.sleep(10) + self.wait_for_last_transaction_to_be_mined() sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ relayers, witnesses, validator0_home, chain_dir + def wait_for_last_transaction_to_be_mined(self): + # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid + # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) + # TODO Can we make it more robust? + time.sleep(10) + def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, tcp_url, chain_id, peggy_sc_addrs, evm_validator_accounts, sifnode_validators, sifnode_relayers, sifnode_witnesses, symbol_translator_file ): diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index 7d74de78cc..03e4545030 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -135,7 +135,7 @@ def tx_clp_create_pool(self, chain_id, from_name, symbol, fees, native_amount, e def peggy2_token_registry_register_all(self, registry_path, gas_prices, gas_adjustment, from_account, chain_id ): - args = ["tx", "tokenregistry", "register-all", registry_path, "--gas-prices", sif_format_amount(*gas_prices), + args = ["tx", "tokenregistry", "set-registry", registry_path, "--gas-prices", sif_format_amount(*gas_prices), "--gas-adjustment", str(gas_adjustment), "--from", from_account, "--chain-id", chain_id, "--output", "json", "--yes"] res = self.sifnoded_exec(args, keyring_backend=self.keyring_backend, sifnoded_home=self.home) @@ -241,6 +241,7 @@ def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_n (["--keyring-backend", keyring_backend] if keyring_backend else []) + \ (["--from", sign_with] if sign_with else []) + \ (["--home", home] if home else []) + \ + (["--keyring-dir", home] if (home and on_peggy2_branch) else []) + \ (["--symbol-translator-file", symbol_translator_file] if symbol_translator_file else []) + \ (["--log_format", log_format] if log_format else []) return buildcmd(args, env=env, cwd=cwd) From 7a6103c6deea20730efacbc6d68b23fb8721a8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 17 Feb 2022 20:03:37 +0100 Subject: [PATCH 26/70] Compatibility changes for #2470 --- test/integration/framework/eth.py | 27 ++++++++++++++++++------ test/integration/framework/project.py | 3 ++- test/integration/framework/test_utils.py | 13 +++++++----- test/integration/framework/truffle.py | 2 +- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/test/integration/framework/eth.py b/test/integration/framework/eth.py index de4eeb8d53..6f7caf6d71 100644 --- a/test/integration/framework/eth.py +++ b/test/integration/framework/eth.py @@ -39,6 +39,21 @@ def web3_wait_for_connection_up(url, polling_time=1, timeout=90): raise Exception(f"Timeout when trying to connect to {url}") time.sleep(polling_time) +def validate_address_and_private_key(addr, private_key): + a = web3.Web3().eth.account + addr = web3.Web3.toChecksumAddress(addr) if addr else None + if private_key: + private_key_is_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key) + private_key = private_key_is_hex[2] if private_key_is_hex else _mnemonic_to_private_key(private_key) + account = a.from_key(private_key) + addr = addr or account.address + assert addr == account.address, "Address does not correspond to private key" + assert (not private_key.startswith("0x")) and (private_key == private_key.lower()), "Private key must be in lowercase hex without '0x' prefix" + else: + private_key = None + assert addr + return addr, private_key + class EthereumTxWrapper: """ This class wraps a Web3 connection in a way that makes calling web3 functions and sending @@ -71,15 +86,13 @@ def _get_private_key(self, addr): return self.private_keys[addr] def set_private_key(self, addr, private_key): + a = web3.Web3().eth.account addr = web3.Web3.toChecksumAddress(addr) if private_key is None: self.private_keys.pop(addr) # Remove else: - is_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key) - private_key = private_key if is_hex else _get_account_from_mnemonic(private_key) # Convert from mnemonic if necessary - assert (not private_key.startswith("0x")) and (private_key == private_key.lower()), "Private key must be in lowercase hex without '0x' prefix" - check_addr = self.w3_conn.eth.account.from_key(private_key).address - assert check_addr == addr, f"Private key does not correspond to given address {addr}" + assert re.match("^([0-9a-f]{64})$", private_key) + assert addr == a.from_key(private_key).address, f"Private key does not correspond to given address {addr}" self.private_keys[addr] = private_key if self.is_local_node: # existing_accounts = self.w3_conn.geth.personal.list_accounts() @@ -377,10 +390,10 @@ def estimate_gas_price(): __web3_enabled_unaudited_hdwallet_features = False # https://stackoverflow.com/questions/68050645/how-to-create-a-web3py-account-using-mnemonic-phrase -def _get_account_from_mnemonic(mnemonic): +def _mnemonic_to_private_key(mnemonic, account_path="m/44'/60'/0'/0/0"): a = web3.Web3().eth.account global __web3_enabled_unaudited_hdwallet_features if not __web3_enabled_unaudited_hdwallet_features: a.enable_unaudited_hdwallet_features() __web3_enabled_unaudited_hdwallet_features = True - return a.from_mnemonic(mnemonic, account_path="m/44'/60'/0'/0/0").privateKey.hex()[2:] + return a.from_mnemonic(mnemonic, account_path=account_path).privateKey.hex()[2:] diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index 8a4715cb8b..309e8ec7e3 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -153,7 +153,8 @@ def install_smart_contracts_dependencies(self): self.cmd.execst(["make", "clean-smartcontracts"], cwd=self.smart_contracts_dir) # = rm -rf build .openzeppelin # According to peggy2, the plan is to move from npm install to yarn, but there are some issues with yarn atm. # self.yarn(["install"], cwd=self.smart_contracts_dir) - self.cmd.execst(["npm", "install"], cwd=self.smart_contracts_dir, pipe=False) + # self.cmd.execst(["npm", "install"], cwd=self.smart_contracts_dir, pipe=False) + self.npm_install(self.smart_contracts_dir) def write_vagrantenv_sh(self, state_vars, data_dir, ethereum_websocket_address, chainnet): # Trace of test_utilities.py get_required_env_var/get_optional_env_var: diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 695722aec0..74b9c242e7 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -42,6 +42,7 @@ def get_env_ctx(cmd=None, env_file=None, env_vars=None): if eth_user_private_keys: available_test_accounts = [] for address, key in [[e["address"], e["key"]] for e in eth_user_private_keys]: + address, key = eth.validate_address_and_private_key(address, key) available_test_accounts.append(address) ctx.eth.set_private_key(address, key) ctx.available_test_eth_accounts = available_test_accounts @@ -79,7 +80,9 @@ def get_env_ctx_peggy2(): owner_address = web3.Web3.toChecksumAddress(dot_env_vars["ETH_ACCOUNT_OWNER_ADDRESS"]) owner_private_key = dot_env_vars.get("ETH_ACCOUNT_OWNER_PRIVATEKEY") if (owner_private_key is not None) and (owner_private_key.startswith("0x")): - owner_private_key = owner_private_key[2:] + owner_private_key = owner_private_key[2:] # TODO Remove + owner_address, owner_private_key = eth.validate_address_and_private_key(owner_address, owner_private_key) + rowan_source = dot_env_vars["ROWAN_SOURCE"] w3_url = eth.web3_host_port_url(dot_env_vars["ETH_HOST"], int(dot_env_vars["ETH_PORT"])) @@ -167,7 +170,7 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): else: operator_address = env_vars["OPERATOR_ADDRESS"] operator_private_key = env_vars.get("OPERATOR_PRIVATE_KEY") - operator_address = web3.Web3.toChecksumAddress(operator_address) + operator_address, operator_private_key = eth.validate_address_and_private_key(operator_address, operator_private_key) # Already added below # collected_private_keys[operator_address] = operator_private_key @@ -188,14 +191,14 @@ def get_env_ctx_peggy1(cmd=None, env_file=None, env_vars=None): if "SMART_CONTRACT_ARTIFACT_DIR" in env_vars: artifacts_dir = env_vars["SMART_CONTRACT_ARTIFACT_DIR"] elif deployment_name: - artifacts_dir = cmd.project.project_dir("smart-contracts/deployments/{}/build".format(deployment_name)) + artifacts_dir = cmd.project.project_dir("smart-contracts/deployments/{}/build/contracts".format(deployment_name)) if deployment_name == "sifchain-1": # Special case for Betanet because SifchainTestToken is not deployed there. # It's only available on Testnet, Devnet and in local environment. # However, BridgeToken will work on Betanet meaning that name(), symbol() and decimals() return meaningful values. generic_erc20_contract_name = "BridgeToken" else: - artifacts_dir = cmd.project.project_dir("smart-contracts/build") + artifacts_dir = cmd.project.project_dir("smart-contracts/build/contracts") sifnode_url = env_vars.get("SIFNODE") # Defaults to "tcp://localhost:26657" sifnoded_home = None # Implies default ~/.sifnoded @@ -607,7 +610,7 @@ def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, last_change_time = now else: delta = sifchain.balance_delta(new_balances, last_change_state) - if delta: + if not sifchain.balance_zero(delta): last_change_state = new_balances last_change_time = now log.debug("New state detected: {}".format(delta)) diff --git a/test/integration/framework/truffle.py b/test/integration/framework/truffle.py index 4aaf4769de..7c197bbd96 100644 --- a/test/integration/framework/truffle.py +++ b/test/integration/framework/truffle.py @@ -28,7 +28,7 @@ def __init__(self, cmd, artifacts_dir, ethereum_network_id, deployed_smart_contr self.deployed_smart_contract_address_overrides = deployed_smart_contract_address_overrides def get_descriptor(self, sc_name): - path = self.cmd.project.project_dir(self.artifacts_dir, "contracts/{}.json".format(sc_name)) + path = self.cmd.project.project_dir(self.artifacts_dir, "{}.json".format(sc_name)) tmp = json.loads(self.cmd.read_text_file(path)) abi = tmp["abi"] bytecode = tmp["bytecode"] From 0de4341f314f506edc5cba70499cc41271f5ba33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Fri, 18 Feb 2022 14:06:31 +0100 Subject: [PATCH 27/70] Readme --- test/integration/framework/README.md | 73 ++++++++++++++++++++++++++++ test/integration/framework/eth.py | 4 +- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/README.md b/test/integration/framework/README.md index 1b53acc6a3..4e9b0ac679 100644 --- a/test/integration/framework/README.md +++ b/test/integration/framework/README.md @@ -59,3 +59,76 @@ e.g. SIFNODE="https://api-testnet.sifchain.finance" ``` - Integration test to be targeted for PoC: test_eth_transfers.py - Dependency diagram: https://files.slack.com/files-pri/T0187TWB4V8-F02BC477N79/sifchaindevenv.jpg + + +# Standardized environment setup + +## Peggy1 - Tempnet on AWS + +chain_id = "mychain" // Parameter + +// Generate account with name 'sif' in the local keyring +mnemonic = generate_mnemonic() +exec("echo $mnemonic | sifnoded keys add --recover --keyring-backend test") +sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx... + +// Init the chain. This command creates files: +// ~/.sifnoded/config/node_key.json +// ~/.sifnoded/config/genesis.json +// ~/.sifnoded/config/priv_validator_key.json +// ~/.sifnoded/data/priv_validator_state.json +// and prints some JSON (what?) +exec("sifnoded init test --chain-id {chain_id}") + +// Add Genesis Accounts +exec("sifnoded add-genesis-account {sif_admin} --keyring-backend test 999999000000000000000000000rowan,500000000000000000000000catk,500000000000000000000000cbtk,500000000000000000000000ceth,990000000000000000000000000stake,500000000000000000000000cdash,500000000000000000000000clink") + +// Add Genesis CLP ADMIN sif +exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") + +// Add Genesis CLP ADMIN sif +exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") + +// Set Genesis whitelist admin ${SIF_WALLET} +exec("sifnoded set-genesis-whitelister-admin {sif_admin} --keyring-backend test") + +// Fund account (Genesis TX stake) +exec("sifnoded gentx {sif_admin} 1000000000000000000000000stake --keyring-backend test --chain-id {chain_id}") + +// Generate token json +sifnoded q tokenregistry generate -o json \ + --token_base_denom=cosmos \ + --token_ibc_counterparty_chain_id=${GAIA_CHAIN_ID} \ + --token_ibc_channel_id=$GAIA_CHANNEL_ID \ + --token_ibc_counterparty_channel_id=$GAIA_COUNTERPARTY_CHANNEL_ID \ + --token_ibc_counterparty_denom="" \ + --token_unit_denom="" \ + --token_decimals=6 \ + --token_display_name="COSMOS" \ + --token_external_symbol="cosmos" \ + --token_permission_clp=true \ + --token_permission_ibc_export=true \ + --token_permission_ibc_import=true | jq > gaia.json + +// Whitelist tokens +// printf "registering cosmos... \n" +sifnoded tx tokenregistry register gaia.json \ + --node tcp://${SIFNODE_P2P_HOSTNAME}:26657 \ + --chain-id $SIFCHAIN_ID \ + --from $SIF_WALLET \ + --keyring-backend test \ + --gas=500000 \ + --gas-prices=0.5rowan \ + -y + +// Deploy token registry +// Registering Tokens... +// Set Whitelist from denoms.json... +sifnoded set-gen-denom-whitelist DENOM.json + + +## Peggy1 - integration tests + +// Parameters: validator moniker, validator mnemonic +sifnoded_keys_add(validator_moniker, validator_password) // Test keyring +valoper = get_val_address(validator_moniker) diff --git a/test/integration/framework/eth.py b/test/integration/framework/eth.py index 6f7caf6d71..9405d1366c 100644 --- a/test/integration/framework/eth.py +++ b/test/integration/framework/eth.py @@ -43,8 +43,8 @@ def validate_address_and_private_key(addr, private_key): a = web3.Web3().eth.account addr = web3.Web3.toChecksumAddress(addr) if addr else None if private_key: - private_key_is_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key) - private_key = private_key_is_hex[2] if private_key_is_hex else _mnemonic_to_private_key(private_key) + match_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key) + private_key = match_hex[2].lower() if match_hex else _mnemonic_to_private_key(private_key) account = a.from_key(private_key) addr = addr or account.address assert addr == account.address, "Address does not correspond to private key" From 2210051ba924982d44499b586b52fb334652629e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 2 Mar 2022 14:19:52 +0100 Subject: [PATCH 28/70] Readme, cleanup --- test/integration/framework/README.md | 13 ++++++++++--- test/integration/framework/inflate_tokens.py | 2 ++ test/integration/framework/run_env.py | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/test/integration/framework/README.md b/test/integration/framework/README.md index 4e9b0ac679..695e1fa23d 100644 --- a/test/integration/framework/README.md +++ b/test/integration/framework/README.md @@ -78,7 +78,7 @@ sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx. // ~/.sifnoded/config/priv_validator_key.json // ~/.sifnoded/data/priv_validator_state.json // and prints some JSON (what?) -exec("sifnoded init test --chain-id {chain_id}") +exec("sifnoded init {moniker} --chain-id {chain_id}") // Add Genesis Accounts exec("sifnoded add-genesis-account {sif_admin} --keyring-backend test 999999000000000000000000000rowan,500000000000000000000000catk,500000000000000000000000cbtk,500000000000000000000000ceth,990000000000000000000000000stake,500000000000000000000000cdash,500000000000000000000000clink") @@ -130,5 +130,12 @@ sifnoded set-gen-denom-whitelist DENOM.json ## Peggy1 - integration tests // Parameters: validator moniker, validator mnemonic -sifnoded_keys_add(validator_moniker, validator_password) // Test keyring -valoper = get_val_address(validator_moniker) +valicator1_moniker, validator1_address, validator1_password, validator1_mnemonic = exec("sifgen create network ...") + +sifnoded_keys_add(validator1_moniker, validator1_password) // Test keyring +valoper = get_val_address(validator1_moniker) + +exec("sifnoded add-genesis-validators {valoper}") +exec("sifnoded add-geneeis-account {}") +exec("sifnoded set-genesis-oracle-admin {}") +exec("sifnoded set-denom-whitelist {}") diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index fea6d74d25..74592cfc4f 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -41,6 +41,8 @@ def get_whitelisted_tokens(self): assert token_symbol not in result, f"Symbol {token_symbol} is being used by more than one whitelisted token" result.append(token) erowan_token = [t for t in result if t["symbol"] == "erowan"] + # These assertions are broken in Tempnet, possibly indicating missing/incomplete chain init, see README.md + # for comparision of steps assert len(erowan_token) == 1, "erowan is not whitelisted, probably bad/incomplete deployment" assert erowan_token[0]["is_whitelisted"], "erowan is un-whitelisted" return result diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 7381846118..1ef4a73e93 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -169,7 +169,6 @@ def sifchain_init_common(self, sifnode, denom_whitelist_file): # self.cmd.execst(["sifnoded", "add-genesis-account", sifnoded_admin_address, "100000000000000000000rowan", "--home", sifnoded_home]) sifnode.add_genesis_account(sifnodeadmin_addr, tokens) sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) - sifnode.set_genesis_oracle_admin(sifnodeadmin_addr) sifnode.set_gen_denom_whitelist(denom_whitelist_file) return sifnodeadmin_addr From b141d49e7f53011ca4ad8deff2f9805bec96dd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 5 Mar 2022 16:17:01 +0100 Subject: [PATCH 29/70] Improve robustness --- test/integration/framework/command.py | 3 +++ test/integration/framework/project.py | 2 +- test/integration/framework/run_env.py | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/command.py b/test/integration/framework/command.py index 555503dbc1..d67f825453 100644 --- a/test/integration/framework/command.py +++ b/test/integration/framework/command.py @@ -38,6 +38,9 @@ def popen(self, args, log_file=None, **kwargs): def spawn_asynchronous_process(self, exec_args, log_file=None): return self.popen(**exec_args, log_file=log_file) + def ls(self, path): + return os.listdir(path) + def rm(self, path): if os.path.exists(path): os.remove(path) diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index 309e8ec7e3..f5d8a36789 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -282,7 +282,7 @@ def npm_install(self, path): cache = [] if not self.cmd.exists(cache_dir): self.cmd.mkdir(cache_dir) - else: + if self.cmd.exists(cache_index): cache = json.loads(self.cmd.read_text_file(cache_index)) idx = None for i, s in enumerate(cache): diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 1ef4a73e93..6ab4095989 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -604,6 +604,11 @@ def run(self): # This script is also called from tests relayer_db_path = os.path.join(self.test_integration_dir, "sifchainrelayerdb") + + # Prevent starting over dirty/existing relayer_db_path + if self.cmd.exists(relayer_db_path): + assert not self.cmd.ls(relayer_db_path), "relayer_db_path {} not empty".format(relayer_db_path) + ebrelayer_proc = self.run_ebrelayer(netdef_json, validator1_address, validator1_moniker, validator1_mnemonic, ebrelayer_ethereum_private_key, bridge_registry_sc_addr, relayer_db_path, log_file=ebrelayer_log_file) From 650b8a0eaf7241bd31789ddf1aba3eda1911c060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 6 Mar 2022 14:46:38 +0100 Subject: [PATCH 30/70] Test deduplication --- .../integration/src/py/test_inflate_tokens.py | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index b71246fe3e..ae3132c773 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -74,31 +74,15 @@ @pytest.mark.skipif("on_peggy2_branch") def test_inflate_tokens_short(ctx): - amount_in_tokens = 123 - amount_gwei = 456 - wallets = test_wallets[:2] - - # TODO Read tokens from file - requested_tokens = [{ - "symbol": t.symbol, - "name": t.name, - "decimals": t.decimals, - } for t in [ctx.generate_random_erc20_token_data() for _ in range(3)]] - - script = InflateTokens(ctx) - - balances_before = [ctx.get_sifchain_balance(w) for w in wallets] - script.transfer(requested_tokens, amount_in_tokens, wallets, amount_gwei) - balances_delta = [sifchain.balance_delta(balances_before[i], ctx.get_sifchain_balance(w)) for i, w in enumerate(wallets)] - - for balances_delta in balances_delta: - for t in requested_tokens: - assert balances_delta[ctx.eth_symbol_to_sif_symbol(t["symbol"])] == amount_in_tokens * 10**t["decimals"] - assert balances_delta.get(ctx.ceth_symbol, 0) == amount_gwei * eth.GWEI + _test_inflate_tokens_parametrized(ctx, 3) @pytest.mark.skipif("on_peggy2_branch") def test_inflate_tokens_long(ctx): + _test_inflate_tokens_parametrized(ctx, 300) + + +def _test_inflate_tokens_parametrized(ctx, number_of_tokens): amount_in_tokens = 123 amount_gwei = 456 wallets = test_wallets[:2] @@ -108,7 +92,7 @@ def test_inflate_tokens_long(ctx): "symbol": t.symbol, "name": t.name, "decimals": t.decimals, - } for t in [ctx.generate_random_erc20_token_data() for _ in range(300)]] + } for t in [ctx.generate_random_erc20_token_data() for _ in range(number_of_tokens)]] script = InflateTokens(ctx) From 37b261b7cb601d153571b76daef3f5b0d74524a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 6 Mar 2022 14:49:09 +0100 Subject: [PATCH 31/70] Improve robustness of token minting --- test/integration/framework/inflate_tokens.py | 62 ++++++++++++++++---- test/integration/framework/run_env.py | 14 ++--- test/integration/framework/sifchain.py | 17 +++++- test/integration/framework/test_utils.py | 7 ++- 4 files changed, 77 insertions(+), 23 deletions(-) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index 74592cfc4f..3004e8c9ac 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -19,6 +19,21 @@ def __init__(self, ctx): self.wait_for_account_change_timeout = 1800 # For Ropsten we need to wait for 50 blocks i.e. ~20 mins self.excluded_token_symbols = ["erowan"] + # Only transfer this tokens in a batch for Peggy1. See #2397. You would need to adjust this if + # test_inflate_tokens_short is passing, but test_inflate_tokens_long is timing out. It only applies to Peggy 1. + # The value of 3 is experimental; if tokens are still not getting across the bridge reliably, reduce the value + # down to 1 (minimum). The lower the value the more time the transfers will take as there will be more + # sequential transfers instead of parallel. + self.max_ethereum_batch_size = 0 + + # Firing transactions with "sifnoded tx bank send" in rapid succession does not work. This is currently a + # known limitation of Cosmos SDK, see https://github.com/cosmos/cosmos-sdk/issues/4186 + # Instead, we take advantage of batching multiple denoms to single account with single send command (amounts + # separated by by comma: "sifnoded tx bank send ... 100denoma,100denomb,100denomc") and wait for destination + # account to show changes for all denoms after each send. But also batches don't work reliably if they are too + # big, so we limit the maximum batch size here. + self.max_sifnoded_batch_size = 5 + def get_whitelisted_tokens(self): whitelist = self.ctx.get_whitelisted_tokens_from_bridge_bank_past_events() ibc_pattern = re.compile("^ibc\/([0-9a-fA-F]{64})$") @@ -160,21 +175,23 @@ def transfer_from_eth_to_sifnode(self, from_eth_addr, to_sif_addr, tokens_to_tra self.ctx.wait_for_sif_balance_change(to_sif_addr, sif_balances_before, min_changes=sent_amounts, polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) + # Distributes from intermediate_sif_account to each individual account def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amount_in_tokens, target_sif_accounts, amount_eth_gwei): - # Distribute from intermediate_sif_account to each individual account - # Note: firing transactions with "sifnoded tx bank send" in rapid succession does not work. This is currently a - # known limitation of Cosmos SDK, see https://github.com/cosmos/cosmos-sdk/issues/4186 - # Instead, we take advantage of batching multiple denoms to single account with single send command (amounts - # separated by by comma: "sifnoded tx bank send ... 100denoma,100denomb,100denomc") and wait for destination - # account to show changes for all denoms after each send. send_amounts = [[amount_in_tokens * 10**t["decimals"], t["sif_denom"]] for t in tokens_to_transfer] if amount_eth_gwei > 0: send_amounts.append([amount_eth_gwei * eth.GWEI, self.ctx.ceth_symbol]) for sif_acct in target_sif_accounts: - sif_balance_before = self.ctx.get_sifchain_balance(sif_acct) - self.ctx.send_from_sifchain_to_sifchain(from_sif_account, sif_acct, send_amounts) - self.ctx.wait_for_sif_balance_change(sif_acct, sif_balance_before, min_changes=send_amounts, - polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) + remaining = send_amounts + while remaining: + batch_size = len(remaining) + if (self.max_sifnoded_batch_size > 0) and (batch_size > self.max_sifnoded_batch_size): + batch_size = self.max_sifnoded_batch_size + batch = remaining[:batch_size] + remaining = remaining[batch_size:] + sif_balance_before = self.ctx.get_sifchain_balance(sif_acct) + self.ctx.send_from_sifchain_to_sifchain(from_sif_account, sif_acct, batch) + self.ctx.wait_for_sif_balance_change(sif_acct, sif_balance_before, min_changes=batch, + polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) def export(self): return [{ @@ -199,7 +216,14 @@ def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amou n_accounts = len(target_sif_accounts) total_token_amount = token_amount * n_accounts total_eth_amount_gwei = eth_amount_gwei * n_accounts - fund_rowan = [5 * test_utils.sifnode_funds_for_transfer_peggy1 * n_accounts, "rowan"] + + # Calculate how much rowan we need to fund intermediate account with. This is only an estimation at this point. + # We need to take into account that we might need to break transfers in batches. The number of tokens is the + # number of ERC20 tokens plus one for ETH, rounded up. 5 is a safety factor + number_of_batches = 1 if self.max_sifnoded_batch_size == 0 else (len(requested_tokens) + 1) // self.max_sifnoded_batch_size + 1 + fund_rowan = [5 * test_utils.sifnode_funds_for_transfer_peggy1 * n_accounts * number_of_batches, "rowan"] + log.debug("Estimated number of batches needed to transfer tokens from intermediate sif account to target sif wallet: {}".format(number_of_batches)) + log.debug("Estimated rowan funding needed for intermediate account: {}".format(fund_rowan)) ether_faucet_account = self.ctx.operator sif_broker_account = self.ctx.create_sifchain_addr(fund_amounts=[fund_rowan]) eth_broker_account = self.ctx.operator @@ -227,7 +251,21 @@ def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amou for t in requested_tokens] self.mint([t["address"] for t in tokens_to_transfer], total_token_amount, eth_broker_account) - self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, tokens_to_transfer, total_token_amount, total_eth_amount_gwei) + + if (self.max_ethereum_batch_size > 0) and (len(tokens_to_transfer) > self.max_ethereum_batch_size): + log.debug(f"Transferring {len(tokens_to_transfer)} tokens from ethereum to sifndde in batches of {self.max_ethereum_batch_size}...") + remaining = tokens_to_transfer + while remaining: + batch = remaining[:self.max_ethereum_batch_size] + remaining = remaining[self.max_ethereum_batch_size:] + self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, batch, total_token_amount, 0) + log.debug(f"Batch completed, {len(remaining)} tokens remaining") + # Transfer ETH separately + log.debug("Thansfering ETH from ethereum to sifnode...") + self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, [], 0, total_eth_amount_gwei) + else: + log.debug(f"Transferring {len(tokens_to_transfer)} tokens from ethereum to sifnode in single batch...") + self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, tokens_to_transfer, total_token_amount, total_eth_amount_gwei) self.distribute_tokens_to_wallets(sif_broker_account, tokens_to_transfer, token_amount, target_sif_accounts, eth_amount_gwei) def transfer_eth(self, from_eth_addr, amount_gewi, target_sif_accounts): diff --git a/test/integration/framework/run_env.py b/test/integration/framework/run_env.py index 6ab4095989..c8fc864890 100644 --- a/test/integration/framework/run_env.py +++ b/test/integration/framework/run_env.py @@ -1018,7 +1018,9 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh assert len(res) == 1 assert res[0]["code"] == 0 - self.wait_for_last_transaction_to_be_mined() + # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid + # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) + sifnode.wait_for_last_transaction_to_be_mined() cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 @@ -1029,18 +1031,14 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh ethereum_cross_chain_fee_token, cross_chain_fee_base, cross_chain_lock_fee, cross_chain_burn_fee, admin_account_name, chain_id, gas_prices, gas_adjustment) - self.wait_for_last_transaction_to_be_mined() + # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid + # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) + sifnode.wait_for_last_transaction_to_be_mined() sifnode.peggy2_update_consensus_needed(admin_account_address, hardhat_chain_id, chain_id) return network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, validators, \ relayers, witnesses, validator0_home, chain_dir - def wait_for_last_transaction_to_be_mined(self): - # We need wait for last tx wrapped up in block, otherwise we could get a wrong sequence, resulting in invalid - # signatures. This delay waits for block production. (See commit 5854d8b6f3970c1254cac0eca0e3817354151853) - # TODO Can we make it more robust? - time.sleep(10) - def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, tcp_url, chain_id, peggy_sc_addrs, evm_validator_accounts, sifnode_validators, sifnode_relayers, sifnode_witnesses, symbol_translator_file ): diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index 03e4545030..bccf388e22 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -185,10 +185,23 @@ def sifnoded_exec(self, args, sifnoded_home=None, keyring_backend=None, stdin=No res = self.cmd.execst(args, stdin=stdin, cwd=cwd) return res - def get_status(self, host, port): - url = "http://{}:{}/node_info".format(host, port) + def _rpc_get(self, host, port, relative_url): + url = "http://{}:{}/{}".format(host, port, relative_url) return json.loads(http_get(url).decode("UTF-8")) + def get_status(self, host, port): + return self._rpc_get(host, port, "node_info") + + def wait_for_last_transaction_to_be_mined(self, count=1): + # TODO return int(self._rpc_get(host, port, abci_info)["response"]["last_block_height"]) + def latest_block_height(): + args = ["satus"] # TODO --node + return int(json.loads(stderr(self.sifnoded_exec(args)))["SyncInfo"]["latest_block_height"]) + initial_block = latest_block_height() + while initial_block + count < latest_block_height(): + time.sleep(2) + time.sleep(10) + def wait_up(self, host, port): while True: from urllib.error import URLError diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/test_utils.py index 74b9c242e7..3939c5f543 100644 --- a/test/integration/framework/test_utils.py +++ b/test/integration/framework/test_utils.py @@ -609,7 +609,7 @@ def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, last_change_state = new_balances last_change_time = now else: - delta = sifchain.balance_delta(new_balances, last_change_state) + delta = sifchain.balance_delta(last_change_state, new_balances) if not sifchain.balance_zero(delta): last_change_state = new_balances last_change_time = now @@ -749,6 +749,11 @@ def bridge_bank_lock_erc20(self, token_addr, from_eth_acct, to_sif_acct, amount) txhash = self.tx_bridge_bank_lock_erc20(token_addr, from_eth_acct, to_sif_acct, amount) return self.eth.wait_for_transaction_receipt(txhash) + # def bridge_bank_lock_erc20(self, token_sc, from_eth_acct, to_sif_acct, amount): + # self.approve_erc20_token(token_sc, from_eth_acct, amount) + # txhash = self.tx_bridge_bank_lock_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) + # return self.eth.wait_for_transaction_receipt(txhash) + # Peggy1-specific def set_ofac_blocklist_to(self, addrs): blocklist_sc = self.get_blocklist_sc() From e80d3a47d6992ebfd6eeca80468d406380bc035f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 6 Mar 2022 15:55:11 +0100 Subject: [PATCH 32/70] Add progress for debugging --- test/integration/framework/inflate_tokens.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/inflate_tokens.py index 3004e8c9ac..072e4176ea 100644 --- a/test/integration/framework/inflate_tokens.py +++ b/test/integration/framework/inflate_tokens.py @@ -180,6 +180,8 @@ def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amo send_amounts = [[amount_in_tokens * 10**t["decimals"], t["sif_denom"]] for t in tokens_to_transfer] if amount_eth_gwei > 0: send_amounts.append([amount_eth_gwei * eth.GWEI, self.ctx.ceth_symbol]) + progress_total = len(target_sif_accounts) * len(send_amounts) + progress_current = 0 for sif_acct in target_sif_accounts: remaining = send_amounts while remaining: @@ -192,6 +194,8 @@ def distribute_tokens_to_wallets(self, from_sif_account, tokens_to_transfer, amo self.ctx.send_from_sifchain_to_sifchain(from_sif_account, sif_acct, batch) self.ctx.wait_for_sif_balance_change(sif_acct, sif_balance_before, min_changes=batch, polling_time=2, timeout=None, change_timeout=self.wait_for_account_change_timeout) + progress_current += batch_size + log.debug("Distributing tokens to wallets: {:0.0f}% done".format((progress_current/progress_total) * 100)) def export(self): return [{ From 840a3e36b65cfca363429e55e6a52c622fba1d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 6 Mar 2022 18:41:27 +0100 Subject: [PATCH 33/70] Add utility functions --- test/integration/framework/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/framework/common.py b/test/integration/framework/common.py index c34f5f67fa..2ec33f83a7 100644 --- a/test/integration/framework/common.py +++ b/test/integration/framework/common.py @@ -117,3 +117,5 @@ def template_transform(s, d): on_peggy2_branch = not os.path.exists(project_dir("smart-contracts", "truffle-config.js")) + +in_github_ci = (os.environ.get("CI") == "true") and os.environ.get("GITHUB_REPOSITORY") and os.environ.get("GITHUB_RUN_ID") From 09397c4d9a12769b54a60b4864b58dd0d1707094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 12 Mar 2022 14:19:33 +0100 Subject: [PATCH 34/70] Bugfix --- test/integration/framework/sifchain.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index bccf388e22..a499d1237c 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -195,12 +195,11 @@ def get_status(self, host, port): def wait_for_last_transaction_to_be_mined(self, count=1): # TODO return int(self._rpc_get(host, port, abci_info)["response"]["last_block_height"]) def latest_block_height(): - args = ["satus"] # TODO --node + args = ["status"] # TODO --node return int(json.loads(stderr(self.sifnoded_exec(args)))["SyncInfo"]["latest_block_height"]) initial_block = latest_block_height() - while initial_block + count < latest_block_height(): - time.sleep(2) - time.sleep(10) + while latest_block_height() < initial_block + count: + time.sleep(1) def wait_up(self, host, port): while True: From d89897adec4d41c5c485f467bea8dad97777d14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 14 Mar 2022 08:36:11 +0100 Subject: [PATCH 35/70] Generate gRPC stubs --- test/integration/framework/command.py | 3 +++ test/integration/framework/main.py | 5 +++++ test/integration/framework/project.py | 23 +++++++++++++++++++++ test/integration/framework/requirements.txt | 3 ++- test/integration/framework/sifchain.py | 21 +++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/test/integration/framework/command.py b/test/integration/framework/command.py index d67f825453..df578ff533 100644 --- a/test/integration/framework/command.py +++ b/test/integration/framework/command.py @@ -82,6 +82,9 @@ def mktempdir(self): def mktempfile(self): return exactly_one(stdout_lines(self.execst(["mktemp"]))) + def pwd(self): + return exactly_one(stdout_lines(self.execst(["pwd"]))) + def __tar_compression_option(self, tarfile): filename = os.path.basename(tarfile).lower() if filename.endswith(".tar"): diff --git a/test/integration/framework/main.py b/test/integration/framework/main.py index ea65cf75c2..7b38f7cf3c 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/main.py @@ -119,6 +119,11 @@ def main(argv): test_utils.recover_eth_from_test_accounts() elif what == "run-peggy2-tests": cmd.execst(["yarn", "test"], cwd=project.smart_contracts_dir) + elif what == "generate-python-grpc-stubs": + project.generate_python_protobuf_stubs() + elif what == "grpc-poc": + import sifchain + sifchain.grpc_poc() else: raise Exception("Missing/unknown command") diff --git a/test/integration/framework/project.py b/test/integration/framework/project.py index f5d8a36789..0a87dd3ca5 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/project.py @@ -309,3 +309,26 @@ def npm_install(self, path): self.cmd.rm(os.path.join(cache_dir, "{}.tar".format(s))) cache = cache[:max_cache_items] self.cmd.write_text_file(cache_index, json.dumps(cache)) + + def project_python(self): + project_venv_dir = project_dir("test", "integration", "framework", "venv") + return os.path.join(project_venv_dir, "bin", "python3") + + def generate_python_protobuf_stubs(self, path=None): + # https://grpc.io/ + # https://grpc.github.io/grpc/python/grpc_asyncio.html + path = path or self.cmd.pwd() + workdir = path + project_proto_dir = self.project_dir("proto") + gogo_proto_dir = os.path.join(workdir, "gogoproto") + generated_dir = self.project_dir("test", "integration", "framework") + # self.cmd.rmf(generated_dir) + # self.cmd.mkdir(generated_dir) + self.cmd.execst(["git", "clone", "--depth", "1", "https://github.com/gogo/protobuf", gogo_proto_dir], pipe=False) + args = [self.project_python(), "-m", "grpc_tools.protoc", "-I", project_proto_dir, "-I", gogo_proto_dir, + "--python_out", generated_dir, "--grpc_python_out", generated_dir, + os.path.join(project_proto_dir, "sifnode/ethbridge/v1/tx.proto"), + os.path.join(project_proto_dir, "sifnode/ethbridge/v1/types.proto"), + os.path.join(project_proto_dir, "sifnode/oracle/v1/network_descriptor.proto"), + os.path.join(gogo_proto_dir, "gogoproto/gogo.proto")] + self.cmd.execst(args, pipe=False) diff --git a/test/integration/framework/requirements.txt b/test/integration/framework/requirements.txt index d7f820d918..955f18c0fe 100644 --- a/test/integration/framework/requirements.txt +++ b/test/integration/framework/requirements.txt @@ -1,4 +1,5 @@ -PyYAML==5.4.1 +grpcio-tools==1.44.0 pytest==6.2.5 +PyYAML==5.4.1 rusty-rlp==0.2.1 web3==5.25.0 diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/sifchain.py index a499d1237c..9cd75e578f 100644 --- a/test/integration/framework/sifchain.py +++ b/test/integration/framework/sifchain.py @@ -209,6 +209,27 @@ def wait_up(self, host, port): except URLError: time.sleep(1) +# See https://docs.cosmos.network/v0.42/core/grpc_rest.html +# See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 +# See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml +class SifnodeGrpc: + def __init__(self): + pass + + def ethbridge_lock(self): + pass + + def ethbridge_burn(self): + pass + + +def grpc_poc(): + log.debug("Hello gRPC") + import grpc + import sifnode.ethbridge.v1.tx_pb2_grpc as tx_pb2_grpc + # import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 + c = grpc.insecure_channel("localhost:9090") + class Sifgen: def __init__(self, cmd): From a83b89a6be49ecff949b6524664ced3179642006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 14 Mar 2022 10:00:16 +0100 Subject: [PATCH 36/70] Refactoring: move source files to src/ --- test/integration/framework/.gitignore | 1 + test/integration/framework/siftool | 4 +++ .../framework/{ => src}/command.py | 0 .../integration/framework/{ => src}/common.py | 2 +- .../integration/framework/{ => src}/cosmos.py | 0 test/integration/framework/{ => src}/eth.py | 0 test/integration/framework/{ => src}/geth.py | 0 .../framework/{ => src}/hardhat.py | 0 .../{ => src}/ibc_transfer_test_tool.py | 0 .../framework/{ => src}/inflate_tokens.py | 0 .../framework/{ => src}/ip_addr_pool.py | 0 test/integration/framework/{ => src}/main.py | 2 +- .../framework/{ => src}/project.py | 25 ++++++++++++------- .../framework/{ => src}/run_env.py | 0 .../framework/{ => src}/sifchain.py | 0 .../framework/{ => src}/test_geth.py | 0 .../framework/{ => src}/test_utils.py | 0 .../framework/{ => src}/truffle.py | 0 18 files changed, 23 insertions(+), 11 deletions(-) rename test/integration/framework/{ => src}/command.py (100%) rename test/integration/framework/{ => src}/common.py (98%) rename test/integration/framework/{ => src}/cosmos.py (100%) rename test/integration/framework/{ => src}/eth.py (100%) rename test/integration/framework/{ => src}/geth.py (100%) rename test/integration/framework/{ => src}/hardhat.py (100%) rename test/integration/framework/{ => src}/ibc_transfer_test_tool.py (100%) rename test/integration/framework/{ => src}/inflate_tokens.py (100%) rename test/integration/framework/{ => src}/ip_addr_pool.py (100%) rename test/integration/framework/{ => src}/main.py (99%) rename test/integration/framework/{ => src}/project.py (95%) rename test/integration/framework/{ => src}/run_env.py (100%) rename test/integration/framework/{ => src}/sifchain.py (100%) rename test/integration/framework/{ => src}/test_geth.py (100%) rename test/integration/framework/{ => src}/test_utils.py (100%) rename test/integration/framework/{ => src}/truffle.py (100%) diff --git a/test/integration/framework/.gitignore b/test/integration/framework/.gitignore index 92e506e338..edd47152bd 100644 --- a/test/integration/framework/.gitignore +++ b/test/integration/framework/.gitignore @@ -1,3 +1,4 @@ /venv/ +/build/ /__pycache__/ /.lock diff --git a/test/integration/framework/siftool b/test/integration/framework/siftool index 94d49b4bd9..421479176f 100755 --- a/test/integration/framework/siftool +++ b/test/integration/framework/siftool @@ -38,8 +38,10 @@ def ensure_venv(venv_dir, requirements_txt, lock_file=None): else: wrapped() + def load_main_module(): base_dir = get_basedir() + src_dir = os.path.join(base_dir, "src") project_root = os.path.abspath(os.path.join(os.path.normpath(os.path.join(base_dir, *([os.path.pardir] * 3))))) venv_dir = os.path.join(base_dir, "venv") requirements_txt = os.path.join(base_dir, "requirements.txt") @@ -48,6 +50,8 @@ def load_main_module(): ensure_venv(venv_dir, requirements_txt, lock_file=lock_file) venv_lib_dir = glob.glob(os.path.join(venv_dir, "lib", "python3.*"))[0] sys.path = sys.path + [ + src_dir, + os.path.join(base_dir, "build/generated"), os.path.join(venv_lib_dir, "site-packages"), os.path.join(project_root, "test", "integration"), # For running integration tests in-process ] diff --git a/test/integration/framework/command.py b/test/integration/framework/src/command.py similarity index 100% rename from test/integration/framework/command.py rename to test/integration/framework/src/command.py diff --git a/test/integration/framework/common.py b/test/integration/framework/src/common.py similarity index 98% rename from test/integration/framework/common.py rename to test/integration/framework/src/common.py index 2ec33f83a7..4700cbc66d 100644 --- a/test/integration/framework/common.py +++ b/test/integration/framework/src/common.py @@ -49,7 +49,7 @@ def random_string(length): return "".join([chars[random.randrange(len(chars))] for _ in range(length)]) def project_dir(*paths): - return os.path.abspath(os.path.join(os.path.normpath(os.path.join(os.path.dirname(__file__), *([os.path.pardir]*3))), *paths)) + return os.path.abspath(os.path.join(os.path.normpath(os.path.join(os.path.dirname(__file__), *([os.path.pardir]*4))), *paths)) def yaml_load(s): return yaml.load(s, Loader=yaml.SafeLoader) diff --git a/test/integration/framework/cosmos.py b/test/integration/framework/src/cosmos.py similarity index 100% rename from test/integration/framework/cosmos.py rename to test/integration/framework/src/cosmos.py diff --git a/test/integration/framework/eth.py b/test/integration/framework/src/eth.py similarity index 100% rename from test/integration/framework/eth.py rename to test/integration/framework/src/eth.py diff --git a/test/integration/framework/geth.py b/test/integration/framework/src/geth.py similarity index 100% rename from test/integration/framework/geth.py rename to test/integration/framework/src/geth.py diff --git a/test/integration/framework/hardhat.py b/test/integration/framework/src/hardhat.py similarity index 100% rename from test/integration/framework/hardhat.py rename to test/integration/framework/src/hardhat.py diff --git a/test/integration/framework/ibc_transfer_test_tool.py b/test/integration/framework/src/ibc_transfer_test_tool.py similarity index 100% rename from test/integration/framework/ibc_transfer_test_tool.py rename to test/integration/framework/src/ibc_transfer_test_tool.py diff --git a/test/integration/framework/inflate_tokens.py b/test/integration/framework/src/inflate_tokens.py similarity index 100% rename from test/integration/framework/inflate_tokens.py rename to test/integration/framework/src/inflate_tokens.py diff --git a/test/integration/framework/ip_addr_pool.py b/test/integration/framework/src/ip_addr_pool.py similarity index 100% rename from test/integration/framework/ip_addr_pool.py rename to test/integration/framework/src/ip_addr_pool.py diff --git a/test/integration/framework/main.py b/test/integration/framework/src/main.py similarity index 99% rename from test/integration/framework/main.py rename to test/integration/framework/src/main.py index 7b38f7cf3c..2c784107b7 100755 --- a/test/integration/framework/main.py +++ b/test/integration/framework/src/main.py @@ -120,7 +120,7 @@ def main(argv): elif what == "run-peggy2-tests": cmd.execst(["yarn", "test"], cwd=project.smart_contracts_dir) elif what == "generate-python-grpc-stubs": - project.generate_python_protobuf_stubs() + project.generate_python_grpc_stubs() elif what == "grpc-poc": import sifchain sifchain.grpc_poc() diff --git a/test/integration/framework/project.py b/test/integration/framework/src/project.py similarity index 95% rename from test/integration/framework/project.py rename to test/integration/framework/src/project.py index 0a87dd3ca5..4e62db5b48 100644 --- a/test/integration/framework/project.py +++ b/test/integration/framework/src/project.py @@ -24,6 +24,7 @@ def __init__(self, cmd, base_dir): self.base_dir = base_dir self.smart_contracts_dir = project_dir("smart-contracts") self.test_integration_dir = project_dir("test", "integration") + self.siftool_dir = project_dir("test", "integration", "framework") self.go_path = os.environ.get("GOPATH") if self.go_path is None: # https://pkg.go.dev/cmd/go#hdr-GOPATH_and_Modules @@ -223,12 +224,12 @@ def init(self): self.install_smart_contracts_dependencies() def clean(self): + self.cmd.rmf(self.project_dir("smart-contracts", "node_modules")) + self.cmd.rmf(os.path.join(self.siftool_dir, "build")) if on_peggy2_branch: for file in [".proto-gen", ".run", "cmd/ebrelayer/contract/generated/artifacts", "smart-contracts/.hardhat-compile"]: self.cmd.rmf(self.project_dir(file)) else: - self.cmd.rmf(self.project_dir("smart-contracts", "node_modules")) - # Output from "truffle compile" self.cmd.rmf(self.project_dir("smart-contracts", "build")) @@ -314,16 +315,22 @@ def project_python(self): project_venv_dir = project_dir("test", "integration", "framework", "venv") return os.path.join(project_venv_dir, "bin", "python3") - def generate_python_protobuf_stubs(self, path=None): + def _ensure_build_dirs(self): + for d in ["build", "build/repos", "build/generated"]: + self.cmd.mkdir(os.path.join(self.siftool_dir, d)) + + def generate_python_grpc_stubs(self, path=None): # https://grpc.io/ # https://grpc.github.io/grpc/python/grpc_asyncio.html - path = path or self.cmd.pwd() - workdir = path + + self._ensure_build_dirs() project_proto_dir = self.project_dir("proto") - gogo_proto_dir = os.path.join(workdir, "gogoproto") - generated_dir = self.project_dir("test", "integration", "framework") - # self.cmd.rmf(generated_dir) - # self.cmd.mkdir(generated_dir) + gogo_proto_dir = os.path.join(self.siftool_dir, "build/repos/gogoproto") + generated_dir = os.path.join(self.siftool_dir, "build/generated") + self.cmd.rmf(generated_dir) + self.cmd.mkdir(generated_dir) + self.cmd.rmf(gogo_proto_dir) + self.cmd.mkdir(gogo_proto_dir) self.cmd.execst(["git", "clone", "--depth", "1", "https://github.com/gogo/protobuf", gogo_proto_dir], pipe=False) args = [self.project_python(), "-m", "grpc_tools.protoc", "-I", project_proto_dir, "-I", gogo_proto_dir, "--python_out", generated_dir, "--grpc_python_out", generated_dir, diff --git a/test/integration/framework/run_env.py b/test/integration/framework/src/run_env.py similarity index 100% rename from test/integration/framework/run_env.py rename to test/integration/framework/src/run_env.py diff --git a/test/integration/framework/sifchain.py b/test/integration/framework/src/sifchain.py similarity index 100% rename from test/integration/framework/sifchain.py rename to test/integration/framework/src/sifchain.py diff --git a/test/integration/framework/test_geth.py b/test/integration/framework/src/test_geth.py similarity index 100% rename from test/integration/framework/test_geth.py rename to test/integration/framework/src/test_geth.py diff --git a/test/integration/framework/test_utils.py b/test/integration/framework/src/test_utils.py similarity index 100% rename from test/integration/framework/test_utils.py rename to test/integration/framework/src/test_utils.py diff --git a/test/integration/framework/truffle.py b/test/integration/framework/src/truffle.py similarity index 100% rename from test/integration/framework/truffle.py rename to test/integration/framework/src/truffle.py From 5ae361cfc368a18ff7725e6038e8f8b38bb1f5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 14 Mar 2022 12:19:46 +0100 Subject: [PATCH 37/70] Proof of concept: gRPC call --- test/integration/framework/src/project.py | 2 ++ test/integration/framework/src/sifchain.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/src/project.py b/test/integration/framework/src/project.py index 4e62db5b48..a723be72af 100644 --- a/test/integration/framework/src/project.py +++ b/test/integration/framework/src/project.py @@ -335,7 +335,9 @@ def generate_python_grpc_stubs(self, path=None): args = [self.project_python(), "-m", "grpc_tools.protoc", "-I", project_proto_dir, "-I", gogo_proto_dir, "--python_out", generated_dir, "--grpc_python_out", generated_dir, os.path.join(project_proto_dir, "sifnode/ethbridge/v1/tx.proto"), + os.path.join(project_proto_dir, "sifnode/ethbridge/v1/query.proto"), os.path.join(project_proto_dir, "sifnode/ethbridge/v1/types.proto"), os.path.join(project_proto_dir, "sifnode/oracle/v1/network_descriptor.proto"), + os.path.join(project_proto_dir, "sifnode/oracle/v1/types.proto"), os.path.join(gogo_proto_dir, "gogoproto/gogo.proto")] self.cmd.execst(args, pipe=False) diff --git a/test/integration/framework/src/sifchain.py b/test/integration/framework/src/sifchain.py index 9cd75e578f..67d36e6d45 100644 --- a/test/integration/framework/src/sifchain.py +++ b/test/integration/framework/src/sifchain.py @@ -209,6 +209,7 @@ def wait_up(self, host, port): except URLError: time.sleep(1) + # See https://docs.cosmos.network/v0.42/core/grpc_rest.html # See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 # See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml @@ -227,8 +228,24 @@ def grpc_poc(): log.debug("Hello gRPC") import grpc import sifnode.ethbridge.v1.tx_pb2_grpc as tx_pb2_grpc - # import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 - c = grpc.insecure_channel("localhost:9090") + import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 + import sifnode.oracle.v1.network_descriptor_pb2 as network_descriptor_pb2 + import sifnode.ethbridge.v1.query_pb2 as query_pb2 + import sifnode.ethbridge.v1.query_pb2_grpc as query_pb2_grpc + + channel = grpc.insecure_channel("127.0.0.1:9090") + # client = tx_pb2_grpc.MsgStub(channel) + # msg_lock = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", + # ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) + # msg_loc_res = client.Lock(msg_lock) + + client1 = query_pb2_grpc.QueryStub(channel) + req = query_pb2.QueryBlacklistRequest() + respoonse = client1.GetBlacklist(req) + + req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=31337) + response2 = client1.CrosschainFeeConfig(req2) + print() class Sifgen: From 3e98b8011b4350bee28128089531ae617b7717a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 14 Mar 2022 14:12:56 +0100 Subject: [PATCH 38/70] Add localnet integration --- test/integration/framework/.gitignore | 2 +- test/integration/framework/README.md | 59 +++++---- test/integration/framework/src/command.py | 6 + test/integration/framework/src/localnet.py | 133 +++++++++++++++++++++ test/integration/framework/src/main.py | 6 + test/integration/framework/src/run_env.py | 4 + 6 files changed, 187 insertions(+), 23 deletions(-) create mode 100644 test/integration/framework/src/localnet.py diff --git a/test/integration/framework/.gitignore b/test/integration/framework/.gitignore index edd47152bd..6654aceeef 100644 --- a/test/integration/framework/.gitignore +++ b/test/integration/framework/.gitignore @@ -1,4 +1,4 @@ /venv/ /build/ -/__pycache__/ +/src/__pycache__/ /.lock diff --git a/test/integration/framework/README.md b/test/integration/framework/README.md index 695e1fa23d..d0006e2baf 100644 --- a/test/integration/framework/README.md +++ b/test/integration/framework/README.md @@ -1,11 +1,13 @@ # Resources 1. Docker setup in docker/ (currently only on future/peggy2 branch, Tim Lind): + - setups two sifnode instances running independent chains + IBC relayer (ts-relayer) 2. Brent's PoC (docker): https://github.com/Sifchain/sifchain-deploy/tree/feature/ibc-poc/docker/localnet/ibc 3. Test environment for testing the new Sifchain public SDK (Caner): + - https://docs.google.com/document/d/1MAlg-I0xMnUvbavAZdAN---WuqbyuRyKw-6Lfgfe130/edit - https://github.com/sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/index.ts#L1 - https://github.com/Sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/cosmoshub/index.ts @@ -14,7 +16,7 @@ 5. https://github.com/Sifchain/sifnode/commit/9ab620e148be8f4850eef59d39b0e869956f87a4 -6. sifchain-devops script to deploy TestNet (by _IM): https://github.com/Sifchain/sifchain-devops/blob/main/scripts/testnet/launch.sh#L19 +6. sifchain-devops script to deploy TestNet (by \_IM): https://github.com/Sifchain/sifchain-devops/blob/main/scripts/testnet/launch.sh#L19 7. Tempnet scripts by chainops @@ -24,14 +26,18 @@ 9. erowan should be deployed and whitelisted (assumption) # RPC endpoints: + e.g. SIFNODE="https://api-testnet.sifchain.finance" + - $SIFNODE/node_info - $SIFNODE/tokenregistry/entries # Peggy2 devenv + - Directory: smart-contracts/scripts/src/devenv - Init: cd smart-contracts; rm -rf node_modules; npm install (plan is to move to yarn eventually) - Run: GOBIN=/home/anderson/go/bin npx hardhat run scripts/devenv.ts + ``` { // vscode launch.json file to debug the Dev Environment Scripts @@ -57,10 +63,10 @@ e.g. SIFNODE="https://api-testnet.sifchain.finance" ] } ``` + - Integration test to be targeted for PoC: test_eth_transfers.py - Dependency diagram: https://files.slack.com/files-pri/T0187TWB4V8-F02BC477N79/sifchaindevenv.jpg - # Standardized environment setup ## Peggy1 - Tempnet on AWS @@ -70,7 +76,7 @@ chain_id = "mychain" // Parameter // Generate account with name 'sif' in the local keyring mnemonic = generate_mnemonic() exec("echo $mnemonic | sifnoded keys add --recover --keyring-backend test") -sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx... +sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx... // Init the chain. This command creates files: // ~/.sifnoded/config/node_key.json @@ -97,36 +103,35 @@ exec("sifnoded gentx {sif_admin} 1000000000000000000000000stake --keyring-backen // Generate token json sifnoded q tokenregistry generate -o json \ - --token_base_denom=cosmos \ - --token_ibc_counterparty_chain_id=${GAIA_CHAIN_ID} \ + --token_base_denom=cosmos \ + --token_ibc_counterparty_chain_id=${GAIA_CHAIN_ID} \ --token_ibc_channel_id=$GAIA_CHANNEL_ID \ - --token_ibc_counterparty_channel_id=$GAIA_COUNTERPARTY_CHANNEL_ID \ - --token_ibc_counterparty_denom="" \ - --token_unit_denom="" \ - --token_decimals=6 \ - --token_display_name="COSMOS" \ - --token_external_symbol="cosmos" \ - --token_permission_clp=true \ - --token_permission_ibc_export=true \ - --token_permission_ibc_import=true | jq > gaia.json + --token_ibc_counterparty_channel_id=$GAIA_COUNTERPARTY_CHANNEL_ID \ + --token_ibc_counterparty_denom="" \ + --token_unit_denom="" \ + --token_decimals=6 \ + --token_display_name="COSMOS" \ + --token_external_symbol="cosmos" \ + --token_permission_clp=true \ + --token_permission_ibc_export=true \ + --token_permission_ibc_import=true | jq > gaia.json // Whitelist tokens // printf "registering cosmos... \n" sifnoded tx tokenregistry register gaia.json \ - --node tcp://${SIFNODE_P2P_HOSTNAME}:26657 \ - --chain-id $SIFCHAIN_ID \ - --from $SIF_WALLET \ - --keyring-backend test \ - --gas=500000 \ - --gas-prices=0.5rowan \ - -y + --node tcp://${SIFNODE_P2P_HOSTNAME}:26657 \ + --chain-id $SIFCHAIN_ID \ + --from $SIF_WALLET \ + --keyring-backend test \ + --gas=500000 \ + --gas-prices=0.5rowan \ + -y // Deploy token registry // Registering Tokens... // Set Whitelist from denoms.json... sifnoded set-gen-denom-whitelist DENOM.json - ## Peggy1 - integration tests // Parameters: validator moniker, validator mnemonic @@ -139,3 +144,13 @@ exec("sifnoded add-genesis-validators {valoper}") exec("sifnoded add-geneeis-account {}") exec("sifnoded set-genesis-oracle-admin {}") exec("sifnoded set-denom-whitelist {}") + +## Coupled with the localnet framework + +The localnet test framework is located under `./test/localnet` within the same repository and offers some interesting features such as spinning up a bunch of IBC chains along with relayers and storing the states of the chains for later use for deterministic testing against various IBC flows. + +The `localnet` framework is supported by `siftool` and can be enabled by using the following environment variable `LOCALNET` set to `true` as follow: + +``` +LOCALNET=true siftool run-env +``` diff --git a/test/integration/framework/src/command.py b/test/integration/framework/src/command.py index df578ff533..ef1f7d3c4b 100644 --- a/test/integration/framework/src/command.py +++ b/test/integration/framework/src/command.py @@ -125,3 +125,9 @@ def tcp_probe_connect(self, host, port): def sha1_of_file(self, path): res = self.execst(["sha1sum", "-b", path]) return stdout_lines(res)[0][:40] + + def download_url(self, url, output_file=None, output_dir=None): + args = ["curl", "--location", "--silent", "--show-error", url] + \ + (["-O"] if not (output_dir or output_file) else []) + \ + (["-o", output_file] if (output_file and not output_dir) else []) + self.execst(args, cwd=output_dir) diff --git a/test/integration/framework/src/localnet.py b/test/integration/framework/src/localnet.py new file mode 100644 index 0000000000..cf1d2925f0 --- /dev/null +++ b/test/integration/framework/src/localnet.py @@ -0,0 +1,133 @@ +import os +import json +from command import Command +from common import project_dir +import logging + + +log = logging.getLogger(__name__) + + +# This is called from run_env as a hook to run additional IBC chains defined in LOCALNET variable. +def run_localnet_hook(): + localnet_env_var = os.environ.get("LOCALNET") + if not localnet_env_var: + return + + localnet = Localnet() + if not os.path.exists(localnet.node_module_dir): + log.info("Installing localnet dependencies on first use in '{}'...".format(localnet.node_module_dir)) + localnet.install_deps() + if not os.path.exists(localnet.bin_dir): + log.info("Downloading localnet binaries on first use in '{}'...".format(localnet.bin_dir)) + localnet.download_binaries() + + if not os.path.exists(localnet.config_dir): + log.info("Init all chains on first use in '{}'...".format(localnet.config_dir)) + localnet.init_all_chains() + + # rm -rf /tmp/localnet/config/cosmos/cosmoshub-testnet + # mkdir -p /tmp/localnet/config/cosmos/cosmoshub-testnet + # /tmp/localnet/bin/gaiad init cosmoshub-testnet --chain-id cosmoshub-testnet --home /tmp/localnet/config/cosmos/cosmoshub-testnet + # /tmp/localnet/bin/gaiad keys add cosmos-validator --keyring-backend test --home /tmp/localnet/config/cosmos/cosmoshub-testnet + # /tmp/localnet/bin/gaiad keys add cosmos-source --keyring-backend test --home /tmp/localnet/config/cosmos/cosmoshub-testnet + # /tmp/localnet/bin/gaiad add-genesis-account cosmos-validator 10000000000000000000uphoton --keyring-backend test --home /tmp/localnet/config/cosmos/cosmoshub-testnet + # /tmp/localnet/bin/gaiad add-genesis-account cosmos-source 10000000000000000000uphoton --keyring-backend test --home /tmp/localnet/config/cosmos/cosmoshub-testnet + + # rm -rf /tmp/localnet/config/sifchain/sifchain-testnet-1 + # mkdir -p /tmp/localnet/config/sifchain/sifchain-testnet-1 + # /tmp/localnet/bin/sifnoded init sifchain-testnet-1 --chain-id sifchain-testnet-1 --home /tmp/localnet/config/sifchain/sifchain-testnet-1 + # /tmp/localnet/bin/sifnoded keys add sifchain-validator --keyring-backend test --home /tmp/localnet/config/sifchain/sifchain-testnet-1 + # /tmp/localnet/bin/sifnoded keys add sifchain-source --keyring-backend test --home /tmp/localnet/config/sifchain/sifchain-testnet-1 + # /tmp/localnet/bin/sifnoded add-genesis-account sifchain-validator 10000000000000000000rowan --keyring-backend test --home /tmp/localnet/config/sifchain/sifchain-testnet-1 + # /tmp/localnet/bin/sifnoded add-genesis-account sifchain-source 10000000000000000000rowan --keyring-backend test --home /tmp/localnet/config/sifchain/sifchain-testnet-1 + + # For each chain: + # defaultGenesis = what was created in ${home}/config/genesis.json + # remoteGenesis = curl (${node from config}/genesis).data e.g. https://rpc.testnet.cosmos.network:443/genesis + # cleanedUpGenesis = cleanUpGenesisState(defaultGenesis, remoteGenesis) + # + # writeFile(genesis, "${home}/config/genesis.json") + # + # if sifchain: ${binPath}/${binary} set-gen-denom-whitelist ${home}/config/denoms.json --home ${home} + # + # ${binPath}/${binary} gentx ${validatorAccountName} ${amount}${denom} --chain-id ${chainId} --keyring-backend test --home ${home} + # ${binPath}/${binary} collect-gentxs --home ${home} + + localnet.start_all_chains() # Runs sifnoded and gaiad + + + +def run(cmd, argv): + log.debug(repr(argv)) + config = json.loads(cmd.read_text_file(cmd.project.project_dir("test/localnet/config/chains.json"))) + + # Filter out items with "disabled": true + config = {k: v for k, v in config.items() if not v.get("disabled", False)} + + tmpdir = cmd.mktempdir() + log.debug(tmpdir) + + localnet = Localnet() + localnet.init_all_chains() + # localnet.start_all_chains() + + return + + +def download_ibc_binaries(cmd, output_path=None): + if not output_path: + output_path = os.environ.get("GOBIN") + if not output_path: + output_path = cmd.pwd() + print(repr(output_path)) + + +def fetch_genesis(base_url): + pass + + +def init_chain(cmd): + pass + + +class Localnet(Command): + def __init__(self, script_dir=None, config_dir=None, bin_dir=None): + self.script_dir = script_dir if script_dir else project_dir("test/localnet") + self.config_dir = config_dir if config_dir else os.path.join("/tmp/localnet", "./config") + self.bin_dir = bin_dir if bin_dir else os.path.join("/tmp/localnet", "./bin") + self.node_module_dir = os.path.join(self.script_dir, "./node_modules") + + def install_deps(self): + self.execst(["yarn"], cwd=self.script_dir, pipe=False) + + def download_binaries(self): + self.execst(["yarn", "downloadBinaries"], cwd=self.script_dir, pipe=False) + + def init_all_chains(self): + self.execst(["yarn", "initAllChains"], cwd=self.script_dir, pipe=False) + + def start_all_chains(self): + self.execst(['yarn', 'startAllChains'], cmd=self.script_dir, pipe=False) + + def init_all_relayers(self): + self.execst(['yarn', 'initAllRelayers'], cmd=self.script_dir, pipe=False) + + def start_all_relayers(self): + self.execst(['yarn', 'startAllRelayers'], cmd=self.script_dir, pipe=False) + + def build_local_net(self): + self.execst(['yarn', 'buildLocalNet'], cmd=self.script_dir, pipe=False) + + def load_local_net(self): + self.execst(['yarn', 'loadLocalNet'], cmd=self.script_dir, pipe=False) + + def take_snapshot(self): + self.execst(['yarn', 'takeSnapshot'], cmd=self.script_dir, pipe=False) + + def create_snapshot(self): + self.execst(['yarn', 'createSnapshot'], cmd=self.script_dir, pipe=False) + + def test(self): + self.execst(['yarn', 'test'], cmd=self.script_dir, pipe=False) + \ No newline at end of file diff --git a/test/integration/framework/src/main.py b/test/integration/framework/src/main.py index 2c784107b7..0a9f4bb53c 100755 --- a/test/integration/framework/src/main.py +++ b/test/integration/framework/src/main.py @@ -124,6 +124,12 @@ def main(argv): elif what == "grpc-poc": import sifchain sifchain.grpc_poc() + elif what == "localnet": + import localnet + localnet.run(cmd, argv[1:]) + elif what == "download-ibc-binaries": + import localnet + localnet.download_ibc_binaries(cmd, *argv[1:]) else: raise Exception("Missing/unknown command") diff --git a/test/integration/framework/src/run_env.py b/test/integration/framework/src/run_env.py index c8fc864890..20e426186e 100644 --- a/test/integration/framework/src/run_env.py +++ b/test/integration/framework/src/run_env.py @@ -5,6 +5,7 @@ import eth import hardhat from truffle import Ganache +from localnet import Localnet from command import Command from sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash from project import Project, killall, force_kill_processes @@ -648,6 +649,9 @@ def run(self): } self.project.write_vagrantenv_sh(self.state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) + import localnet + localnet.run_localnet_hook() + return ganache_proc, sifnoded_proc, ebrelayer_proc def remove_and_add_sifnoded_keys(self, moniker, mnemonic): From 58bad411a798170a222b7b04e0d233c9636621d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 14 Mar 2022 18:41:49 +0100 Subject: [PATCH 39/70] Siftool: download IBC binaries --- test/integration/framework/src/command.py | 14 +++-- test/integration/framework/src/localnet.py | 63 +++++++++++++++++--- test/integration/framework/src/test_utils.py | 2 + 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/src/command.py b/test/integration/framework/src/command.py index ef1f7d3c4b..61d27b1a83 100644 --- a/test/integration/framework/src/command.py +++ b/test/integration/framework/src/command.py @@ -76,11 +76,17 @@ def exists(self, path): def get_user_home(self, *paths): return os.path.join(os.environ["HOME"], *paths) - def mktempdir(self): - return exactly_one(stdout_lines(self.execst(["mktemp", "-d"]))) + def mktempdir(self, parent_dir=None): + args = ["mktemp", "-d"] + (["-p", parent_dir] if parent_dir else []) + return exactly_one(stdout_lines(self.execst(args))) - def mktempfile(self): - return exactly_one(stdout_lines(self.execst(["mktemp"]))) + def mktempfile(self, parent_dir=None): + args = ["mktemp"] + (["-p", parent_dir] if parent_dir else []) + return exactly_one(stdout_lines(self.execst(args))) + + def chmod(self, path, mode_str, recursive=False): + args = ["chmod"] + (["-r"] if recursive else []) + [mode_str, path] + self.execst(args) def pwd(self): return exactly_one(stdout_lines(self.execst(["pwd"]))) diff --git a/test/integration/framework/src/localnet.py b/test/integration/framework/src/localnet.py index cf1d2925f0..c6c79145d6 100644 --- a/test/integration/framework/src/localnet.py +++ b/test/integration/framework/src/localnet.py @@ -58,13 +58,16 @@ def run_localnet_hook(): -def run(cmd, argv): - log.debug(repr(argv)) +def get_localnet_config(cmd): config = json.loads(cmd.read_text_file(cmd.project.project_dir("test/localnet/config/chains.json"))) + return config + +def run(cmd, argv): + log.debug(repr(argv)) + config = get_localnet_config(cmd) # Filter out items with "disabled": true config = {k: v for k, v in config.items() if not v.get("disabled", False)} - tmpdir = cmd.mktempdir() log.debug(tmpdir) @@ -75,12 +78,58 @@ def run(cmd, argv): return -def download_ibc_binaries(cmd, output_path=None): - if not output_path: - output_path = os.environ.get("GOBIN") +def download_ibc_binaries(cmd, chains_to_download=None, output_path=None): if not output_path: output_path = cmd.pwd() - print(repr(output_path)) + config = get_localnet_config(cmd) + tmpdir = cmd.mktempdir() + all_supported_chains = set(config.keys()).difference({"sifchain"}) + chains_to_download = chains_to_download or "all" + if chains_to_download == "all": + chains_to_download = all_supported_chains + else: + chains_to_download = ",".split(chains_to_download) + try: + tmp_gobin = os.path.join(tmpdir, "bin") + cmd.mkdir(tmp_gobin) + for chain_name in chains_to_download: + if chain_name not in config: + raise Exception("Chain {} not supported yet".format(chain_name)) + values = config[chain_name] + binary = values["binary"] + binary_url = values.get("binaryUrl") + source_url = values.get("sourceUrl") + binary_relative_path = values.get("binaryRelativePath") + source_relative_path = values.get("sourceRelativePath") + assert bool(source_url) ^ bool(binary_url) + url = binary_url or source_url + dlfile = os.path.join(tmpdir, "{}-download.tmp".format(chain_name)) + log.info("Downloading {} from '{}' to '{}'...".format(chain_name, url, dlfile)) + cmd.download_url(url, output_file=dlfile) + extract_dir = os.path.join(tmpdir, chain_name) + src_file = None + cmd.mkdir(extract_dir) + if url.endswith(".zip"): + cmd.execst(["unzip", dlfile], cwd=extract_dir) + elif url.endswith(".tar.gz"): + cmd.execst(["tar", "xfz", dlfile], cwd=extract_dir) + elif binary_url: + # We have binaryUrl but it is not an archive => must be binary itself + assert not source_url and not binary_relative_path + src_file = dlfile + if not src_file: + if binary_url: + src_file = os.path.join(extract_dir, binary_relative_path if binary_relative_path else binary) + if source_url: + src_dir = extract_dir if not source_relative_path else os.path.join(extract_dir, source_relative_path) + cmd.execst(["make", "install"], cwd=src_dir, env={"GOBIN": tmp_gobin}) + src_file = os.path.join(tmp_gobin, binary) + assert src_file + dst_file = os.path.join(output_path, binary) + cmd.copy_file(src_file, dst_file) + cmd.chmod(dst_file, "+x") + finally: + cmd.rmf(tmpdir) def fetch_genesis(base_url): diff --git a/test/integration/framework/src/test_utils.py b/test/integration/framework/src/test_utils.py index 3939c5f543..b960aa14bf 100644 --- a/test/integration/framework/src/test_utils.py +++ b/test/integration/framework/src/test_utils.py @@ -430,6 +430,8 @@ def update_bridge_bank_whitelist(self, token_addr, value): # Call of updateEthWhiteList will fail if we try to remove an item from whitelist which is not on the whitelist. return self.eth.wait_for_transaction_receipt(self.tx_update_bridge_bank_whitelist(token_addr, value)) + # This function walks through all historical events LogWhiteListUpdate of a BridgeBanksmart contract and builds the + # current whitelist from live on-chain data. def get_whitelisted_tokens_from_bridge_bank_past_events(self): bridge_bank = self.get_bridge_bank_sc() past_events = self.smart_contract_get_past_events(bridge_bank, "LogWhiteListUpdate") From 1030ba65cb73ad24a0b18cc5f75b48b35c63c274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 15 Mar 2022 08:38:13 +0100 Subject: [PATCH 40/70] Cleanup, refactoring, minor fixes --- test/integration/framework/siftool | 3 +-- test/integration/framework/src/localnet.py | 6 +++++- test/integration/framework/src/project.py | 4 +++- test/integration/framework/{src => test}/test_geth.py | 0 test/localnet/config/chains.json | 3 ++- 5 files changed, 11 insertions(+), 5 deletions(-) rename test/integration/framework/{src => test}/test_geth.py (100%) diff --git a/test/integration/framework/siftool b/test/integration/framework/siftool index 421479176f..0da6cfbe74 100755 --- a/test/integration/framework/siftool +++ b/test/integration/framework/siftool @@ -41,7 +41,6 @@ def ensure_venv(venv_dir, requirements_txt, lock_file=None): def load_main_module(): base_dir = get_basedir() - src_dir = os.path.join(base_dir, "src") project_root = os.path.abspath(os.path.join(os.path.normpath(os.path.join(base_dir, *([os.path.pardir] * 3))))) venv_dir = os.path.join(base_dir, "venv") requirements_txt = os.path.join(base_dir, "requirements.txt") @@ -50,7 +49,7 @@ def load_main_module(): ensure_venv(venv_dir, requirements_txt, lock_file=lock_file) venv_lib_dir = glob.glob(os.path.join(venv_dir, "lib", "python3.*"))[0] sys.path = sys.path + [ - src_dir, + os.path.join(base_dir, "src"), os.path.join(base_dir, "build/generated"), os.path.join(venv_lib_dir, "site-packages"), os.path.join(project_root, "test", "integration"), # For running integration tests in-process diff --git a/test/integration/framework/src/localnet.py b/test/integration/framework/src/localnet.py index c6c79145d6..5b79f32e45 100644 --- a/test/integration/framework/src/localnet.py +++ b/test/integration/framework/src/localnet.py @@ -81,9 +81,13 @@ def run(cmd, argv): def download_ibc_binaries(cmd, chains_to_download=None, output_path=None): if not output_path: output_path = cmd.pwd() + else: + if not cmd.exists(output_path): + cmd.mkdir(output_path) config = get_localnet_config(cmd) tmpdir = cmd.mktempdir() - all_supported_chains = set(config.keys()).difference({"sifchain"}) + # We prefer to compile sifchain. Sentinel uses sourceUrl, but there is no Makefile. + all_supported_chains = set(config.keys()).difference({"sifchain", "sentinel"}) chains_to_download = chains_to_download or "all" if chains_to_download == "all": chains_to_download = all_supported_chains diff --git a/test/integration/framework/src/project.py b/test/integration/framework/src/project.py index a723be72af..26a7596c58 100644 --- a/test/integration/framework/src/project.py +++ b/test/integration/framework/src/project.py @@ -52,6 +52,8 @@ def __rm_files(self, level): self.__rm_files_develop() self.__rm(self.project_dir("smart-contracts", "build")) # truffle deploy self.__rm(self.project_dir("test", "integration", "vagrant", "data")) + self.__rm(self.project_dir("test", "integration", "src", ".pytest_cache")) + self.__rm(self.project_dir("test", "integration", "src", "py", ".pytest_cache")) self.__rm(self.cmd.get_user_home(".sifnoded")) # Probably needed for "--keyring-backend test" self.__rm(self.project_dir("deploy", "networks")) # from running integration tests @@ -85,6 +87,7 @@ def __rm_files(self, level): for file in ["sifnoded", "ebrelayer", "sifgen"]: self.__rm(os.path.join(self.go_bin_dir, file)) self.__rm(self.project_dir("smart-contracts", "node_modules")) + self.__rm(self.project_dir("test", "localnet", "node_modules")) if level >= 2: if self.cmd.exists(self.go_path): @@ -145,7 +148,6 @@ def make_go_binaries(self): # TODO Merge # Main Makefile requires GOBIN to be set to an absolute path. Compiled executables ebrelayer, sifgen and # sifnoded will be written there. The directory will be created if it doesn't exist yet. - # def make_go_binaries_2(self): # Original: cd smart-contracts; make -C .. install self.cmd.execst(["make", "install"], cwd=project_dir(), pipe=False) diff --git a/test/integration/framework/src/test_geth.py b/test/integration/framework/test/test_geth.py similarity index 100% rename from test/integration/framework/src/test_geth.py rename to test/integration/framework/test/test_geth.py diff --git a/test/localnet/config/chains.json b/test/localnet/config/chains.json index 3c5db7fede..64d9919344 100644 --- a/test/localnet/config/chains.json +++ b/test/localnet/config/chains.json @@ -86,7 +86,8 @@ "node": "http://rpc.sentinel.co:26657", "chainId": "sentinelhub-2", "binary": "sentinelhub", - "binaryUrl": "https://github.com/sentinel-official/sentinel/archive/refs/tags/v0.1.4.zip", + "sourceUrl": "https://github.com/sentinel-official/sentinel/archive/refs/tags/v0.1.4.zip", + "sourceRelativePath": "sentinel-0.1.4", "fees": 20000, "denom": "udvpn", "prefix": "sent", From 6a28624e80e570bf8fc8f2c94247cb5913b7e590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 15 Mar 2022 09:47:22 +0100 Subject: [PATCH 41/70] Incorporate chanes from future/peggy2 --- test/integration/framework/src/run_env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/src/run_env.py b/test/integration/framework/src/run_env.py index 20e426186e..0c2f7dac07 100644 --- a/test/integration/framework/src/run_env.py +++ b/test/integration/framework/src/run_env.py @@ -828,7 +828,7 @@ def run(self): # See https://hardhat.org/advanced/hardhat-runtime-environment.html # The value is not used; instead a hardcoded constant 31337 is passed to ebrelayerWitnessBuilder. # Ask juniuszhou for details. - hardhat_chain_id = 31337 + hardhat_chain_id = 9999 hardhat_accounts = self.signer_array_to_ethereum_accounts(hardhat.default_accounts(), hardhat_validator_count) self.hardhat.compile_smart_contracts() @@ -1028,7 +1028,8 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 - ethereum_cross_chain_fee_token = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) + ethereum_cross_chain_fee_token = "sifBridge99990x0000000000000000000000000000000000000000" + assert hardhat_chain_id == int(ethereum_cross_chain_fee_token[9:13]) # Assume they should match gas_prices = [0.5, "rowan"] gas_adjustment = 1.5 sifnode.peggy2_set_cross_chain_fee(admin_account_address, hardhat_chain_id, From 2d68748a7505f20b98b43e9a331da0f3c210a34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 15 Mar 2022 09:58:54 +0100 Subject: [PATCH 42/70] Incorporate chanes from future/peggy2 --- test/integration/framework/src/run_env.py | 2 ++ test/integration/framework/src/sifchain.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/src/run_env.py b/test/integration/framework/src/run_env.py index 0c2f7dac07..e2cc10dc7b 100644 --- a/test/integration/framework/src/run_env.py +++ b/test/integration/framework/src/run_env.py @@ -1083,6 +1083,7 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, ethereum_address=evm_validator0_addr, ethereum_private_key=evm_validator0_key, keyring_backend="test", + keyring_dir=sifnode_relayer0_home, home=sifnode_relayer0_home, ) @@ -1100,6 +1101,7 @@ def start_witnesses_and_relayers(self, web3_websocket_address, hardhat_chain_id, ethereum_address=evm_validator0_addr, ethereum_private_key=evm_validator0_key, keyring_backend="test", + keyring_dir=sifnode_relayer0_home, log_format="json", home=sifnode_witness0_home, ) diff --git a/test/integration/framework/src/sifchain.py b/test/integration/framework/src/sifchain.py index 67d36e6d45..a0ce163116 100644 --- a/test/integration/framework/src/sifchain.py +++ b/test/integration/framework/src/sifchain.py @@ -272,8 +272,8 @@ def __init__(self, cmd): def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_node, web3_provider, bridge_registry_contract_address, validator_mnemonic, chain_id, node=None, keyring_backend=None, - sign_with=None, symbol_translator_file=None, log_format=None, extra_args=None, ethereum_private_key=None, - ethereum_address=None, home=None, cwd=None + keyring_dir=None, sign_with=None, symbol_translator_file=None, log_format=None, extra_args=None, + ethereum_private_key=None, ethereum_address=None, home=None, cwd=None ): env = _env_for_ethereum_address_and_key(ethereum_address, ethereum_private_key) args = [ @@ -291,7 +291,7 @@ def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_n (["--keyring-backend", keyring_backend] if keyring_backend else []) + \ (["--from", sign_with] if sign_with else []) + \ (["--home", home] if home else []) + \ - (["--keyring-dir", home] if (home and on_peggy2_branch) else []) + \ + (["--keyring-dir", keyring_dir] if keyring_dir else []) + \ (["--symbol-translator-file", symbol_translator_file] if symbol_translator_file else []) + \ (["--log_format", log_format] if log_format else []) return buildcmd(args, env=env, cwd=cwd) From 92579b44f61ed78e1054401703974a987190795e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 15 Mar 2022 19:00:39 +0100 Subject: [PATCH 43/70] Incorporate changes from future/peggy2 --- test/integration/framework/src/run_env.py | 5 +++-- test/integration/framework/src/sifchain.py | 12 ++++++++---- test/integration/framework/src/test_utils.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/integration/framework/src/run_env.py b/test/integration/framework/src/run_env.py index e2cc10dc7b..d77dbaa6b6 100644 --- a/test/integration/framework/src/run_env.py +++ b/test/integration/framework/src/run_env.py @@ -826,7 +826,7 @@ def run(self): # This value is actually returned from HardhatNodeRunner. It comes from smart-contracts/hardhat.config.ts. # In Typescript, its value is obtained by 'require("hardhat").hre.network.config.chainId'. # See https://hardhat.org/advanced/hardhat-runtime-environment.html - # The value is not used; instead a hardcoded constant 31337 is passed to ebrelayerWitnessBuilder. + # The value is not used; instead a hardcoded constant 9999 is passed to ebrelayerWitnessBuilder. # Ask juniuszhou for details. hardhat_chain_id = 9999 hardhat_accounts = self.signer_array_to_ethereum_accounts(hardhat.default_accounts(), hardhat_validator_count) @@ -842,7 +842,8 @@ def run(self): admin_account_name = "sifnodeadmin" chain_id = "localnet" ceth_symbol = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) - assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" + print("ceth symbol is: {0}".format(ceth_symbol)) + assert ceth_symbol == "sifBridge99990x0000000000000000000000000000000000000000" # Mint goes to validator mint_amount = [ [999999 * 10**21, "rowan"], diff --git a/test/integration/framework/src/sifchain.py b/test/integration/framework/src/sifchain.py index a0ce163116..e112749948 100644 --- a/test/integration/framework/src/sifchain.py +++ b/test/integration/framework/src/sifchain.py @@ -3,13 +3,17 @@ import time from command import buildcmd from common import * +import typing -def sifchain_denom_hash(network_descriptor, token_contract_address): +def sifchain_denom_hash(network_descriptor_raw: typing.Union[int, str], token_contract_address: str) -> str: assert on_peggy2_branch assert token_contract_address.startswith("0x") - s = str(network_descriptor) + token_contract_address.lower() - return "sif" + hashlib.sha256(s.encode("UTF-8")).digest().hex() + network_descriptor = int(network_descriptor_raw) + assert network_descriptor > 0 + assert network_descriptor <= 9999 + denom = f"sifBridge{network_descriptor:04d}{token_contract_address.lower()}" + return denom def balance_delta(balances1, balances2): all_denoms = set(balances1.keys()) @@ -279,7 +283,7 @@ def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_n args = [ self.binary, init_what, - "--network-descriptor", str(network_descriptor), # Network descriptor for the chain (31337) + "--network-descriptor", str(network_descriptor), # Network descriptor for the chain (9999) "--tendermint-node", tendermint_node, # URL to tendermint node "--web3-provider", web3_provider, # Ethereum web3 service address (ws://localhost:8545/) "--bridge-registry-contract-address", bridge_registry_contract_address, diff --git a/test/integration/framework/src/test_utils.py b/test/integration/framework/src/test_utils.py index b960aa14bf..f0be907c2a 100644 --- a/test/integration/framework/src/test_utils.py +++ b/test/integration/framework/src/test_utils.py @@ -97,7 +97,7 @@ def get_env_ctx_peggy2(): eth_node_is_local = True generic_erc20_contract = "BridgeToken" ceth_symbol = sifchain.sifchain_denom_hash(ethereum_network_descriptor, eth.NULL_ADDRESS) - assert ceth_symbol == "sif5ebfaf95495ceb5a3efbd0b0c63150676ec71e023b1043c40bcaaf91c00e15b2" + assert ceth_symbol == "sifBridge99990x0000000000000000000000000000000000000000" ctx_eth = eth.EthereumTxWrapper(w3_conn, eth_node_is_local) ctx = EnvCtx(cmd, w3_conn, ctx_eth, abi_provider, owner_address, sifnoded_home, sifnode_url, sifnode_chain_id, From 17633c837ec9217b5c6c97ef4d10848ebf3100da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Tue, 15 Mar 2022 19:03:00 +0100 Subject: [PATCH 44/70] gRPC proof of concept --- test/integration/framework/src/sifchain.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test/integration/framework/src/sifchain.py b/test/integration/framework/src/sifchain.py index e112749948..38cecf3a64 100644 --- a/test/integration/framework/src/sifchain.py +++ b/test/integration/framework/src/sifchain.py @@ -237,18 +237,23 @@ def grpc_poc(): import sifnode.ethbridge.v1.query_pb2 as query_pb2 import sifnode.ethbridge.v1.query_pb2_grpc as query_pb2_grpc + network_descriptor = 9999 # Set in run_env::Peggy2Environment.run() + channel = grpc.insecure_channel("127.0.0.1:9090") - # client = tx_pb2_grpc.MsgStub(channel) - # msg_lock = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", - # ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) - # msg_loc_res = client.Lock(msg_lock) client1 = query_pb2_grpc.QueryStub(channel) - req = query_pb2.QueryBlacklistRequest() - respoonse = client1.GetBlacklist(req) + client2 = tx_pb2_grpc.MsgStub(channel) + + req1 = query_pb2.QueryBlacklistRequest() + res1 = client1.GetBlacklist(req1) + + req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=network_descriptor) + res2 = client1.CrosschainFeeConfig(req2) + + req3 = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", + ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) + res3 = client2.Lock(req3) - req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=31337) - response2 = client1.CrosschainFeeConfig(req2) print() From 895fb5e9517aae6d39a847af05c0faa35d511fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 16 Mar 2022 14:05:39 +0100 Subject: [PATCH 45/70] Add shortcut method for approve + lock --- test/integration/framework/src/test_utils.py | 25 +++++++++++-------- .../integration/src/py/test_ofac_blocklist.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/test/integration/framework/src/test_utils.py b/test/integration/framework/src/test_utils.py index f0be907c2a..95fb0db1ae 100644 --- a/test/integration/framework/src/test_utils.py +++ b/test/integration/framework/src/test_utils.py @@ -566,11 +566,6 @@ def create_sifchain_addr(self, moniker=None, fund_amounts=None): self.wait_for_sif_balance_change(sif_address, old_balances, min_changes=fund_amounts) return sif_address - # smart-contracts/scripts/test/{sendLockTx.js OR sendBurnTx.js} - # sendBurnTx is called when sifchain_symbol == "rowan", sendLockTx otherwise - def send_from_ethereum_to_sifchain(self): - assert False,"Not implemented yet" # TODO - def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): amounts_string = ",".join([sif_format_amount(*a) for a in amounts]) args = ["tx", "bank", "send", from_sif_addr, to_sif_addr, amounts_string] + \ @@ -747,14 +742,22 @@ def bridge_bank_lock_eth(self, from_eth_acct, to_sif_acct, amount): txhash = self.tx_bridge_bank_lock_eth(from_eth_acct, to_sif_acct, amount) return self.eth.wait_for_transaction_receipt(txhash) - def bridge_bank_lock_erc20(self, token_addr, from_eth_acct, to_sif_acct, amount): - txhash = self.tx_bridge_bank_lock_erc20(token_addr, from_eth_acct, to_sif_acct, amount) + def bridge_bank_lock_erc20(self, token_sc, from_eth_acct, to_sif_acct, amount): + txhash = self.tx_bridge_bank_lock_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) return self.eth.wait_for_transaction_receipt(txhash) - # def bridge_bank_lock_erc20(self, token_sc, from_eth_acct, to_sif_acct, amount): - # self.approve_erc20_token(token_sc, from_eth_acct, amount) - # txhash = self.tx_bridge_bank_lock_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) - # return self.eth.wait_for_transaction_receipt(txhash) + # TODO At the moment this is only for Ethereum-native assets (ETH and ERC20 tokens) which always use "lock". + # For Sifchain-native assets (rowan) we need to use "burn". + # Compare: smart-contracts/scripts/test/{sendLockTx.js OR sendBurnTx.js} + # sendBurnTx is called when sifchain_symbol == "rowan", sendLockTx otherwise + def send_from_ethereum_to_sifchain(self, from_eth_acct, to_sif_acct, amount, token_sc=None): + if token_sc is None: + # ETH transfer + self.bridge_bank_lock_eth(from_eth_acct, to_sif_acct, amount) + else: + # ERC20 token transfer + self.approve_erc20_token(token_sc, from_eth_acct, amount) + self.bridge_bank_lock_erc20(token_sc, from_eth_acct, to_sif_acct, amount) # Peggy1-specific def set_ofac_blocklist_to(self, addrs): diff --git a/test/integration/src/py/test_ofac_blocklist.py b/test/integration/src/py/test_ofac_blocklist.py index af6a81c3d5..a3f07a32de 100644 --- a/test/integration/src/py/test_ofac_blocklist.py +++ b/test/integration/src/py/test_ofac_blocklist.py @@ -23,7 +23,7 @@ def bridge_bank_lock_eth(ctx, from_eth_acct, to_sif_acct, amount): def bridge_bank_lock_erc20(ctx, bridge_token, from_eth_acct, to_sif_acct, amount): assert ctx.eth.get_eth_balance(from_eth_acct) > max_gas_required * max_gas_price, "Not enough gas for test" assert ctx.get_erc20_token_balance(bridge_token.address, from_eth_acct) >= amount, "Not enough tokens for test" - return ctx.bridge_bank_lock_erc20(bridge_token.address, from_eth_acct, to_sif_acct, amount) + return ctx.bridge_bank_lock_erc20(bridge_token, from_eth_acct, to_sif_acct, amount) def is_blocklisted_exception(ctx, exception): return ctx.eth.is_contract_logic_error(exception, "Address is blocklisted") From 0d40775583ad5dda184d541ca493c527fc528d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 16 Mar 2022 14:13:34 +0100 Subject: [PATCH 46/70] Do not wait for ENTER if running in GitHub CI --- test/integration/framework/src/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/src/main.py b/test/integration/framework/src/main.py index 0a9f4bb53c..01a7a99226 100755 --- a/test/integration/framework/src/main.py +++ b/test/integration/framework/src/main.py @@ -49,8 +49,9 @@ def main(argv): # - rm -rf networks/validators/localnet/$moniker/.sifnoded # - If you ran the execute_integration_test_*.sh you need to kill ganache-cli for proper cleanup # as it might have been killed and started outside of our control - input("Press ENTER to exit...") - killall(processes) + if not in_github_ci: + input("Press ENTER to exit...") + killall(processes) elif what == "devenv": project.npx(["hardhat", "run", "scripts/devenv.ts"], cwd=project.smart_contracts_dir, pipe=False) elif what == "create_snapshot": From 2096b499865e563e61942a0e0e3f98274473deb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 16 Mar 2022 16:20:09 +0100 Subject: [PATCH 47/70] Upgrade dependencies --- test/integration/framework/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/requirements.txt b/test/integration/framework/requirements.txt index 955f18c0fe..aac8206e52 100644 --- a/test/integration/framework/requirements.txt +++ b/test/integration/framework/requirements.txt @@ -1,5 +1,5 @@ grpcio-tools==1.44.0 pytest==6.2.5 -PyYAML==5.4.1 +PyYAML==6.0 rusty-rlp==0.2.1 -web3==5.25.0 +web3==5.28.0 From f83a15d7854dfc1ea7b6bc0cea3490ff85a646b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 08:17:06 +0100 Subject: [PATCH 48/70] Initial implementation of gRPC client --- test/integration/framework/.gitignore | 2 +- test/integration/framework/requirements.txt | 2 +- test/integration/framework/siftool | 4 +- .../framework/src/{ => siftool}/command.py | 19 +- .../framework/src/{ => siftool}/common.py | 5 +- .../framework/src/{ => siftool}/cosmos.py | 2 +- .../framework/src/{ => siftool}/eth.py | 8 +- .../framework/src/{ => siftool}/geth.py | 2 +- .../framework/src/{ => siftool}/hardhat.py | 4 +- .../{ => siftool}/ibc_transfer_test_tool.py | 2 +- .../src/{ => siftool}/inflate_tokens.py | 6 +- .../src/{ => siftool}/ip_addr_pool.py | 0 .../framework/src/{ => siftool}/localnet.py | 4 +- .../framework/src/{ => siftool}/main.py | 15 +- .../framework/src/{ => siftool}/project.py | 81 ++++- .../framework/src/{ => siftool}/run_env.py | 16 +- .../framework/src/{ => siftool}/sifchain.py | 279 +++++++++++++++--- .../framework/src/{ => siftool}/test_utils.py | 47 +-- .../framework/src/{ => siftool}/truffle.py | 0 test/integration/framework/test/test_geth.py | 6 +- .../src/py/integration_test_context.py | 3 +- ...tegration_framework.py => siftool_path.py} | 15 +- 22 files changed, 380 insertions(+), 142 deletions(-) rename test/integration/framework/src/{ => siftool}/command.py (90%) rename test/integration/framework/src/{ => siftool}/common.py (96%) rename test/integration/framework/src/{ => siftool}/cosmos.py (98%) rename test/integration/framework/src/{ => siftool}/eth.py (98%) rename test/integration/framework/src/{ => siftool}/geth.py (99%) rename test/integration/framework/src/{ => siftool}/hardhat.py (98%) rename test/integration/framework/src/{ => siftool}/ibc_transfer_test_tool.py (99%) rename test/integration/framework/src/{ => siftool}/inflate_tokens.py (99%) rename test/integration/framework/src/{ => siftool}/ip_addr_pool.py (100%) rename test/integration/framework/src/{ => siftool}/localnet.py (99%) rename test/integration/framework/src/{ => siftool}/main.py (93%) rename test/integration/framework/src/{ => siftool}/project.py (79%) rename test/integration/framework/src/{ => siftool}/run_env.py (99%) rename test/integration/framework/src/{ => siftool}/sifchain.py (60%) rename test/integration/framework/src/{ => siftool}/test_utils.py (96%) rename test/integration/framework/src/{ => siftool}/truffle.py (100%) rename test/integration/src/py/{integration_framework.py => siftool_path.py} (60%) diff --git a/test/integration/framework/.gitignore b/test/integration/framework/.gitignore index 6654aceeef..5cc945694d 100644 --- a/test/integration/framework/.gitignore +++ b/test/integration/framework/.gitignore @@ -1,4 +1,4 @@ /venv/ /build/ -/src/__pycache__/ +/src/siftool/__pycache__/ /.lock diff --git a/test/integration/framework/requirements.txt b/test/integration/framework/requirements.txt index aac8206e52..4fc508ceb3 100644 --- a/test/integration/framework/requirements.txt +++ b/test/integration/framework/requirements.txt @@ -2,4 +2,4 @@ grpcio-tools==1.44.0 pytest==6.2.5 PyYAML==6.0 rusty-rlp==0.2.1 -web3==5.28.0 +web3==5.25.0 diff --git a/test/integration/framework/siftool b/test/integration/framework/siftool index 0da6cfbe74..3d0bc28ae1 100755 --- a/test/integration/framework/siftool +++ b/test/integration/framework/siftool @@ -54,8 +54,8 @@ def load_main_module(): os.path.join(venv_lib_dir, "site-packages"), os.path.join(project_root, "test", "integration"), # For running integration tests in-process ] - import main as main_module - return main_module + import siftool.main as siftool_main + return siftool_main def main(argv): main_module = load_main_module() diff --git a/test/integration/framework/src/command.py b/test/integration/framework/src/siftool/command.py similarity index 90% rename from test/integration/framework/src/command.py rename to test/integration/framework/src/siftool/command.py index 61d27b1a83..3bb147a018 100644 --- a/test/integration/framework/src/command.py +++ b/test/integration/framework/src/siftool/command.py @@ -1,6 +1,6 @@ import shutil import time -from common import * +from siftool.common import * def buildcmd(args, cwd=None, env=None): @@ -45,6 +45,9 @@ def rm(self, path): if os.path.exists(path): os.remove(path) + def mv(self, src, dst): + os.rename(src, dst) + def read_text_file(self, path): with open(path, "rt") as f: return f.read() # TODO Convert to exec @@ -73,6 +76,20 @@ def copy_file(self, src, dst): def exists(self, path): return os.path.exists(path) + def is_dir(self, path): + return os.path.isdir(path) if self.exists(path) else False + + def find_files(self, path, filter=None): + items = [os.path.join(path, name) for name in self.ls(path)] + result = [] + for i in items: + if self.is_dir(i): + result.extend(self.find_files(i)) + else: + if (filter is None) or filter(i): + result.append(i) + return result + def get_user_home(self, *paths): return os.path.join(os.environ["HOME"], *paths) diff --git a/test/integration/framework/src/common.py b/test/integration/framework/src/siftool/common.py similarity index 96% rename from test/integration/framework/src/common.py rename to test/integration/framework/src/siftool/common.py index 4700cbc66d..0433e4afc2 100644 --- a/test/integration/framework/src/common.py +++ b/test/integration/framework/src/siftool/common.py @@ -49,7 +49,7 @@ def random_string(length): return "".join([chars[random.randrange(len(chars))] for _ in range(length)]) def project_dir(*paths): - return os.path.abspath(os.path.join(os.path.normpath(os.path.join(os.path.dirname(__file__), *([os.path.pardir]*4))), *paths)) + return os.path.abspath(os.path.join(os.path.normpath(os.path.join(os.path.dirname(__file__), *([os.path.pardir]*5))), *paths)) def yaml_load(s): return yaml.load(s, Loader=yaml.SafeLoader) @@ -90,6 +90,9 @@ def dict_merge(*dicts, override=True): result[k] = v return result +def flatten_list(l): + return [item for sublist in l for item in sublist] + def format_as_shell_env_vars(env, export=True): # TODO escaping/quoting, e.g. shlex.quote(v) return ["{}{}=\"{}\"".format("export " if export else "", k, v) for k, v in env.items()] diff --git a/test/integration/framework/src/cosmos.py b/test/integration/framework/src/siftool/cosmos.py similarity index 98% rename from test/integration/framework/src/cosmos.py rename to test/integration/framework/src/siftool/cosmos.py index b78eb29966..f82cee6ba8 100644 --- a/test/integration/framework/src/cosmos.py +++ b/test/integration/framework/src/siftool/cosmos.py @@ -1,4 +1,4 @@ -from common import * +from siftool.common import * akash_binary = "akash" diff --git a/test/integration/framework/src/eth.py b/test/integration/framework/src/siftool/eth.py similarity index 98% rename from test/integration/framework/src/eth.py rename to test/integration/framework/src/siftool/eth.py index 9405d1366c..ff61a4cef1 100644 --- a/test/integration/framework/src/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -2,7 +2,7 @@ import time import web3 -from common import * +from siftool.common import * ETH = 10**18 @@ -79,6 +79,12 @@ def __init__(self, w3_conn, is_local_node): self.gas_estimate_fn = None self.used_tx_nonces = {} + # These are only set in get_env_ctx_peggy2(), otherwise they are undefined. + # self.cross_chain_fee_base = None + # self.cross_chain_lock_fee = None + # self.cross_chain_burn_fee = None + # self.ethereum_network_descriptor = None + def _get_private_key(self, addr): addr = web3.Web3.toChecksumAddress(addr) if not addr in self.private_keys: diff --git a/test/integration/framework/src/geth.py b/test/integration/framework/src/siftool/geth.py similarity index 99% rename from test/integration/framework/src/geth.py rename to test/integration/framework/src/siftool/geth.py index 6aea4e7216..05917cde8e 100644 --- a/test/integration/framework/src/geth.py +++ b/test/integration/framework/src/siftool/geth.py @@ -1,6 +1,6 @@ import json import re -from common import * +from siftool.common import * def js_fmt(str, *params): diff --git a/test/integration/framework/src/hardhat.py b/test/integration/framework/src/siftool/hardhat.py similarity index 98% rename from test/integration/framework/src/hardhat.py rename to test/integration/framework/src/siftool/hardhat.py index df744c463d..93258a34da 100644 --- a/test/integration/framework/src/hardhat.py +++ b/test/integration/framework/src/siftool/hardhat.py @@ -1,7 +1,7 @@ import json import web3 -from common import * -from command import buildcmd +from siftool.common import * +from siftool.command import buildcmd class Hardhat: diff --git a/test/integration/framework/src/ibc_transfer_test_tool.py b/test/integration/framework/src/siftool/ibc_transfer_test_tool.py similarity index 99% rename from test/integration/framework/src/ibc_transfer_test_tool.py rename to test/integration/framework/src/siftool/ibc_transfer_test_tool.py index fda8539016..7ac7a34f7b 100644 --- a/test/integration/framework/src/ibc_transfer_test_tool.py +++ b/test/integration/framework/src/siftool/ibc_transfer_test_tool.py @@ -8,7 +8,7 @@ import json import sys -from command import Command +from siftool.command import Command chains = { "akash": {"binary": "akash", "relayer": "ibc"}, diff --git a/test/integration/framework/src/inflate_tokens.py b/test/integration/framework/src/siftool/inflate_tokens.py similarity index 99% rename from test/integration/framework/src/inflate_tokens.py rename to test/integration/framework/src/siftool/inflate_tokens.py index 072e4176ea..9831ed6f28 100644 --- a/test/integration/framework/src/inflate_tokens.py +++ b/test/integration/framework/src/siftool/inflate_tokens.py @@ -6,9 +6,9 @@ import logging import re -import eth -import test_utils -from common import * +import siftool.eth +import siftool.test_utils +from siftool.common import * log = logging.getLogger(__name__) diff --git a/test/integration/framework/src/ip_addr_pool.py b/test/integration/framework/src/siftool/ip_addr_pool.py similarity index 100% rename from test/integration/framework/src/ip_addr_pool.py rename to test/integration/framework/src/siftool/ip_addr_pool.py diff --git a/test/integration/framework/src/localnet.py b/test/integration/framework/src/siftool/localnet.py similarity index 99% rename from test/integration/framework/src/localnet.py rename to test/integration/framework/src/siftool/localnet.py index 5b79f32e45..5ef2a00df6 100644 --- a/test/integration/framework/src/localnet.py +++ b/test/integration/framework/src/siftool/localnet.py @@ -1,7 +1,7 @@ import os import json -from command import Command -from common import project_dir +from siftool.command import Command +from siftool.common import project_dir import logging diff --git a/test/integration/framework/src/main.py b/test/integration/framework/src/siftool/main.py similarity index 93% rename from test/integration/framework/src/main.py rename to test/integration/framework/src/siftool/main.py index 01a7a99226..a4f2fd04f0 100755 --- a/test/integration/framework/src/main.py +++ b/test/integration/framework/src/siftool/main.py @@ -1,10 +1,10 @@ import sys import time -from run_env import Integrator, UIStackEnvironment, Peggy2Environment, IBCEnvironment, IntegrationTestsEnvironment -from project import Project, killall, force_kill_processes -import test_utils -from common import * +from siftool import test_utils +from siftool.run_env import Integrator, UIStackEnvironment, Peggy2Environment, IBCEnvironment, IntegrationTestsEnvironment +from siftool.project import Project, killall, force_kill_processes +from siftool.common import * def main(argv): @@ -120,11 +120,8 @@ def main(argv): test_utils.recover_eth_from_test_accounts() elif what == "run-peggy2-tests": cmd.execst(["yarn", "test"], cwd=project.smart_contracts_dir) - elif what == "generate-python-grpc-stubs": - project.generate_python_grpc_stubs() - elif what == "grpc-poc": - import sifchain - sifchain.grpc_poc() + elif what == "generate-python-protobuf-stubs": + project.generate_python_protobuf_stubs() elif what == "localnet": import localnet localnet.run(cmd, argv[1:]) diff --git a/test/integration/framework/src/project.py b/test/integration/framework/src/siftool/project.py similarity index 79% rename from test/integration/framework/src/project.py rename to test/integration/framework/src/siftool/project.py index 26a7596c58..854ad722fe 100644 --- a/test/integration/framework/src/project.py +++ b/test/integration/framework/src/siftool/project.py @@ -1,6 +1,6 @@ import os import json -from common import * +from siftool.common import * def force_kill_processes(cmd): @@ -321,25 +321,84 @@ def _ensure_build_dirs(self): for d in ["build", "build/repos", "build/generated"]: self.cmd.mkdir(os.path.join(self.siftool_dir, d)) - def generate_python_grpc_stubs(self, path=None): + def generate_python_protobuf_stubs(self): # https://grpc.io/ # https://grpc.github.io/grpc/python/grpc_asyncio.html - self._ensure_build_dirs() project_proto_dir = self.project_dir("proto") - gogo_proto_dir = os.path.join(self.siftool_dir, "build/repos/gogoproto") + third_party_proto_dir = self.project_dir("third_party", "proto") generated_dir = os.path.join(self.siftool_dir, "build/generated") + repos_dir = os.path.join(self.siftool_dir, "build/repos") self.cmd.rmf(generated_dir) self.cmd.mkdir(generated_dir) - self.cmd.rmf(gogo_proto_dir) - self.cmd.mkdir(gogo_proto_dir) - self.cmd.execst(["git", "clone", "--depth", "1", "https://github.com/gogo/protobuf", gogo_proto_dir], pipe=False) - args = [self.project_python(), "-m", "grpc_tools.protoc", "-I", project_proto_dir, "-I", gogo_proto_dir, - "--python_out", generated_dir, "--grpc_python_out", generated_dir, + cosmos_sdk_repo_dir = os.path.join(repos_dir, "cosmos-sdk") + cosmos_proto_repo_dir = os.path.join(repos_dir, "cosmos-proto") + # self.git_clone("https://github.com/gogo/protobuf", gogo_proto_dir, shallow=True) + self.git_clone("https://github.com/cosmos/cosmos-sdk.git", cosmos_sdk_repo_dir, checkout_commit="dd65ef87322baa2023f195635890a2128a03d318") + self.git_clone("https://github.com/cosmos/cosmos-proto.git", cosmos_proto_repo_dir, checkout_commit="213b76899fac883ac122728f7ab258166137be29") + cosmos_sdk_proto_dir = os.path.join(cosmos_sdk_repo_dir, "proto") + cosmos_proto_proto_dir = os.path.join(cosmos_proto_repo_dir, "proto") + includes = [ + project_proto_dir, + third_party_proto_dir, + cosmos_sdk_proto_dir, + cosmos_proto_proto_dir, + ] + + # We cannot compile all proto files due to conflicting/inconsistent definitions (e.g. coin.proto). + # + # def find_proto_files(path, excludes=()): + # import re + # tmp = [os.path.relpath(i, start=path) for i in + # self.cmd.find_files(path, filter=lambda x: re.match(os.path.basename(x), "^(.*)\.proto$")) + # return sorted(list(set(tmp).difference(set(excludes) if excludes else set()))) + # + # project_proto_files = find_proto_files(project_proto_dir) + # third_party_proto_files = find_proto_files(third_party_proto_dir, excludes=[ + # "cosmos/base/coin.proto", + # ]) + # cosmos_sdk_proto_files = find_proto_files(cosmos_sdk_proto_dir, excludes=[ + # "cosmos/base/query/v1beta1/pagination.proto", + # ]) + # cosmos_proto_proto_files = find_proto_files(cosmos_proto_proto_dir) + # proto_files = project_proto_files + third_party_proto_files + cosmos_sdk_proto_files + cosmos_proto_proto_files + + proto_files = [ os.path.join(project_proto_dir, "sifnode/ethbridge/v1/tx.proto"), os.path.join(project_proto_dir, "sifnode/ethbridge/v1/query.proto"), os.path.join(project_proto_dir, "sifnode/ethbridge/v1/types.proto"), os.path.join(project_proto_dir, "sifnode/oracle/v1/network_descriptor.proto"), os.path.join(project_proto_dir, "sifnode/oracle/v1/types.proto"), - os.path.join(gogo_proto_dir, "gogoproto/gogo.proto")] - self.cmd.execst(args, pipe=False) + os.path.join(third_party_proto_dir, "gogoproto/gogo.proto"), + os.path.join(third_party_proto_dir, "google/api/annotations.proto"), + os.path.join(third_party_proto_dir, "google/api/http.proto"), + os.path.join(third_party_proto_dir, "cosmos/base/query/v1beta1/pagination.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/tx/v1beta1/service.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/base/abci/v1beta1/abci.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/tx/v1beta1/tx.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/tx/signing/v1beta1/signing.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/crypto/multisig/v1beta1/multisig.proto"), + os.path.join(cosmos_sdk_proto_dir, "cosmos/base/v1beta1/coin.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/abci/types.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/crypto/proof.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/crypto/keys.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/types/types.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/types/validator.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/types/params.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/types/block.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/types/evidence.proto"), + os.path.join(cosmos_sdk_proto_dir, "tendermint/version/types.proto"), + os.path.join(cosmos_proto_proto_dir, "cosmos_proto/cosmos.proto"), + ] + + args = [self.project_python(), "-m", "grpc_tools.protoc"] + flatten_list([["-I", i] for i in includes]) + [ + "--python_out", generated_dir, "--grpc_python_out", generated_dir] + proto_files + self.cmd.execst(args, pipe=True) + + def git_clone(self, url, path, checkout_commit=None, shallow=False): + if self.cmd.exists(os.path.join(path, ".git")): + return + log.debug("Cloning repository '{}' into '{}',,,".format(url, path)) + self.cmd.execst(["git", "clone", "-q"] + (["--depth", "1"] if shallow else []) + [url, path]) + if checkout_commit: + self.cmd.execst(["git", "checkout", checkout_commit], cwd=path) diff --git a/test/integration/framework/src/run_env.py b/test/integration/framework/src/siftool/run_env.py similarity index 99% rename from test/integration/framework/src/run_env.py rename to test/integration/framework/src/siftool/run_env.py index d77dbaa6b6..f7d851dcb5 100644 --- a/test/integration/framework/src/run_env.py +++ b/test/integration/framework/src/siftool/run_env.py @@ -2,14 +2,14 @@ import re import time -import eth -import hardhat -from truffle import Ganache -from localnet import Localnet -from command import Command -from sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash -from project import Project, killall, force_kill_processes -from common import * +import siftool.eth as eth +import siftool.hardhat as hardhat +from siftool.truffle import Ganache +from siftool.localnet import Localnet +from siftool.command import Command +from siftool.sifchain import Sifgen, Sifnoded, Ebrelayer, sifchain_denom_hash +from siftool.project import Project, killall, force_kill_processes +from siftool.common import * class Integrator(Ganache, Command): diff --git a/test/integration/framework/src/sifchain.py b/test/integration/framework/src/siftool/sifchain.py similarity index 60% rename from test/integration/framework/src/sifchain.py rename to test/integration/framework/src/siftool/sifchain.py index 38cecf3a64..967a841ebf 100644 --- a/test/integration/framework/src/sifchain.py +++ b/test/integration/framework/src/siftool/sifchain.py @@ -1,9 +1,9 @@ -import hashlib +import base64 import json import time -from command import buildcmd -from common import * import typing +from siftool.command import buildcmd +from siftool.common import * def sifchain_denom_hash(network_descriptor_raw: typing.Union[int, str], token_contract_address: str) -> str: @@ -162,9 +162,6 @@ def peggy2_update_consensus_needed(self, admin_account_address, hardhat_chain_id args = ["tx", "ethbridge", "update-consensus-needed", admin_account_address, str(hardhat_chain_id), consensus_needed, "--from", admin_account_address, "--chain-id", chain_id, "--gas-prices", "0.5rowan", "--gas-adjustment", "1.5", "-y"] - # TODO Currently "sifnoded tx ethbridge" does not have a "update-consensus-needed" subcommand so this command - # fails by printing help and exiting with errorcode 0. Wait for PR to be merged: - # https://github.com/Sifchain/sifnode/pull/2263 res = self.sifnoded_exec(args, keyring_backend=self.keyring_backend, sifnoded_home=self.home) return res @@ -214,47 +211,239 @@ def wait_up(self, host, port): time.sleep(1) -# See https://docs.cosmos.network/v0.42/core/grpc_rest.html -# See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 -# See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml -class SifnodeGrpc: - def __init__(self): - pass - - def ethbridge_lock(self): - pass - - def ethbridge_burn(self): - pass - - -def grpc_poc(): - log.debug("Hello gRPC") - import grpc - import sifnode.ethbridge.v1.tx_pb2_grpc as tx_pb2_grpc - import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 - import sifnode.oracle.v1.network_descriptor_pb2 as network_descriptor_pb2 - import sifnode.ethbridge.v1.query_pb2 as query_pb2 - import sifnode.ethbridge.v1.query_pb2_grpc as query_pb2_grpc - - network_descriptor = 9999 # Set in run_env::Peggy2Environment.run() - - channel = grpc.insecure_channel("127.0.0.1:9090") - - client1 = query_pb2_grpc.QueryStub(channel) - client2 = tx_pb2_grpc.MsgStub(channel) - - req1 = query_pb2.QueryBlacklistRequest() - res1 = client1.GetBlacklist(req1) - - req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=network_descriptor) - res2 = client1.CrosschainFeeConfig(req2) +# Refactoring in progress +class SifnodeClient: + def __init__(self, cmd, node=None, home=None, chain_id=None, grpc_port=None): + self.cmd = cmd + self.binary = "sifnoded" + self.node = node + self.home = home + self.chain_id = chain_id + self.grpc_port = grpc_port + + def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, denom, generate_only=False): + """ Sends ETH from Sifchain to Ethereum (burn) """ + assert on_peggy2_branch, "Only for Peggy2.0" + assert self.ctx.eth + eth = self.ctx.eth + + direction = "burn" + cross_chain_ceth_fee = eth.cross_chain_fee_base * eth.cross_chain_burn_fee # TODO + args = ["tx", "ethbridge", direction, from_sif_addr, to_eth_addr, str(amount), denom, str(cross_chain_ceth_fee), + "--network-descriptor", str(eth.ethereum_network_descriptor), # Mandatory + "--from", from_sif_addr, # Mandatory, either name from keyring or address + "--output", "json", + "-y" + ] + \ + (["--generate-only"] if generate_only else []) + \ + self._gas_prices_args() + \ + self._home_args() + \ + self._chain_id_and_node_args() + \ + (self._keyring_backend_args() if not generate_only else []) + res = self.sifnoded_exec(args) + result = json.loads(stdout(res)) + if not generate_only: + assert "failed to execute message" not in result["raw_log"] + return result - req3 = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", - ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) - res3 = client2.Lock(req3) + def send_from_sifchain_to_ethereum_grpc(self, from_sif_addr, to_eth_addr, amount, denom): + tx = self.send_from_sifchain_to_ethereum(from_sif_addr, to_eth_addr, amount, denom, generate_only=True) + signed_tx = self.sign_transaction(tx, from_sif_addr) + encoded_tx = self.encode_transaction(signed_tx) + result = self.broadcast_tx(encoded_tx) + return result - print() + def sign_transaction(self, tx, from_sif_addr, sequence=None): + tmp_tx_file = self.cmd.mktempfile() + account_number = 0 # TODO + try: + self.cmd.write_text_file(tmp_tx_file, json.dumps(tx)) + args = ["tx", "sign", tmp_tx_file, "--from", from_sif_addr] + \ + (["--sequence", str(sequence), "--offline", "--account-number", str(account_number)] if sequence else []) + \ + self._home_args() + \ + self._chain_id_and_node_args() + \ + self._keyring_backend_args() + res = self.sifnoded_exec(args) + signed_tx = json.loads(stderr(res)) + return signed_tx + finally: + self.cmd.rm(tmp_tx_file) + + def encode_transaction(self, tx): + tmp_file = self.cmd.mktempfile() + try: + self.cmd.write_text_file(tmp_file, json.dumps(tx)) + res = self.sifnoded_exec(["tx", "encode", tmp_file]) + encoded_tx = base64.b64decode(stdout(res)) + return encoded_tx + finally: + self.cmd.rm(tmp_file) + + def open_grpc_channel(self): + import grpc + return grpc.insecure_channel("127.0.0.1:9090") + + def broadcast_tx(self, encoded_tx): + import cosmos.tx.v1beta1.service_pb2 + import cosmos.tx.v1beta1.service_pb2_grpc + broadcast_mode = cosmos.tx.v1beta1.service_pb2.BROADCAST_MODE_ASYNC + with self.open_grpc_channel() as channel: + tx_stub = cosmos.tx.v1beta1.service_pb2_grpc.ServiceStub(channel) + req = cosmos.tx.v1beta1.service_pb2.BroadcastTxRequest(tx_bytes=encoded_tx, mode=broadcast_mode) + resp = tx_stub.BroadcastTx(req) + return resp + + def _gas_prices_args(self): + return ["--gas-prices", "0.5rowan", "--gas-adjustment", "1.5"] + + def _chain_id_and_node_args(self): + return \ + (["--node", self.node] if self.node else []) + \ + (["--chain-id", self.chain_id] if self.chain_id else []) + + def _keyring_backend_args(self): + keyring_backend = self.ctx.sifnode.keyring_backend + return ["--keyring-backend", keyring_backend] if keyring_backend else [] + + def _home_args(self): + return ["--home", self.home] if self.home else [] + + def sifnoded_exec(self, *args, **kwargs): + return self.ctx.sifnode.sifnoded_exec(*args, **kwargs) + + +# # See https://docs.cosmos.network/v0.44/core/proto-docs.html +# # See https://docs.cosmos.network/v0.44/core/grpc_rest.html +# # See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 +# # See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml +# class SifnodeGrpc: +# def __init__(self): +# pass +# +# def ethbridge_lock(self): +# pass +# +# def ethbridge_burn(self): +# args = ["tx"] +# pass +# +# +# def grpc_poc(): +# log.debug("Hello gRPC") +# import grpc +# import sifnode.ethbridge.v1.tx_pb2_grpc as tx_pb2_grpc +# import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 +# import sifnode.oracle.v1.network_descriptor_pb2 as network_descriptor_pb2 +# import sifnode.ethbridge.v1.query_pb2 as query_pb2 +# import sifnode.ethbridge.v1.query_pb2_grpc as query_pb2_grpc +# +# import cosmos.tx.v1beta1.service_pb2 as service_pb2 +# +# network_descriptor = 9999 # Set in run_env::Peggy2Environment.run() +# +# channel = grpc.insecure_channel("127.0.0.1:9090") +# +# client1 = query_pb2_grpc.QueryStub(channel) +# client2 = tx_pb2_grpc.MsgStub(channel) +# +# req1 = query_pb2.QueryBlacklistRequest() +# res1 = client1.GetBlacklist(req1) +# +# req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=network_descriptor) +# res2 = client1.CrosschainFeeConfig(req2) +# +# req3 = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", +# ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) +# res3 = client2.Lock(req3) +# +# tx_plain_send = { +# "body": { +# "messages": [ +# { +# "@type": "/cosmos.bank.v1beta1.MsgSend", +# "from_address": "sif1n7ctv0zqyx78203lqssafejvgfhqephpnk5duc", +# "to_address": "sif1c7z93gth8meuls88l9e53mkws8k4c3x4p6kayr", +# "amount": [{"denom": "rowan", "amount": "1"}] +# } +# ], +# "memo": "", +# "timeout_height": "0", +# "extension_options": [], +# "non_critical_extension_options": [] +# }, +# "auth_info": { +# "signer_infos": [], +# "fee": {"amount": [], "gas_limit": "200000", "payer": "", "granter": ""} +# }, +# "signatures": [] +# } +# +# tx_ethbridge_lock = { +# "body": { +# "messages": [ +# { +# "@type": "/sifnode.ethbridge.v1.MsgBurn", +# "cosmos_sender": "sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk", +# "amount": "3000000000000000000", +# "denom_hash": "sifBridge99990xc6ba8c3233ecf65b761049ef63466945c362edd2", +# "network_descriptor": "NETWORK_DESCRIPTOR_HARDHAT", +# "ethereum_receiver": "0xcDE217398B5A290005D5f247137211AEA992B937", +# "crosschain_fee": "1" +# } +# ], +# "memo": "", +# "timeout_height": "0", +# "extension_options": [], +# "non_critical_extension_options": [] +# }, +# "auth_info": { +# "signer_infos": [], +# "fee": {"amount": [{"denom": "rowan", "amount": "100000"}], "gas_limit": "200000", "payer": "", "granter": ""} +# }, +# "signatures": [] +# } +# +# # sifnoded tx sign a.json --chain-id localnet --from sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk --keyring-backend test --home /tmp/sifnodedNetwork/validators/localnet/little-leaf/.sifnoded +# signed_tx = { +# "body": { +# "messages": [ +# { +# "@type": "/sifnode.ethbridge.v1.MsgBurn", +# "cosmos_sender": "sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk", +# "amount": "3000000000000000000", +# "denom_hash": "sifBridge99990xc6ba8c3233ecf65b761049ef63466945c362edd2", +# "network_descriptor": "NETWORK_DESCRIPTOR_HARDHAT", +# "ethereum_receiver": "0xcDE217398B5A290005D5f247137211AEA992B937", +# "crosschain_fee": "1" +# } +# ], +# "memo": "", +# "timeout_height": "0", +# "extension_options": [], +# "non_critical_extension_options": [] +# }, +# "auth_info": { +# "signer_infos": [ +# { +# "public_key": { +# "@type": "/cosmos.crypto.secp256k1.PubKey", +# "key": "AtRIz7t6jZhX+7ZPLMmkqtF2BsZWB76164Kbh2g5KdIK" +# }, +# "mode_info": { +# "single": {"mode": "SIGN_MODE_DIRECT"} +# }, +# "sequence": "0" +# } +# ], +# "fee": {"amount": [{"denom": "rowan", "amount": "100000"}], "gas_limit": "200000", "payer": "", "granter": ""} +# }, +# "signatures": [ +# "x1TmzWEQk2qSDBHbGO5WZQq14MrG6KIu7RXWBNYoFyN8hgIrnrL6k8kBGrIdsUCpRFmAouVYJefYef3YzhkqBg==" +# ] +# } +# # To set sequence, add --sequence 12345 --offline --account-number +# +# print() class Sifgen: diff --git a/test/integration/framework/src/test_utils.py b/test/integration/framework/src/siftool/test_utils.py similarity index 96% rename from test/integration/framework/src/test_utils.py rename to test/integration/framework/src/siftool/test_utils.py index 95fb0db1ae..71f7b8d331 100644 --- a/test/integration/framework/src/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -4,12 +4,12 @@ import time import web3 -import eth -import truffle -import hardhat -import run_env -import sifchain -from common import * +import siftool.eth as eth +import siftool.truffle as truffle +import siftool.hardhat as hardhat +import siftool.run_env as run_env +import siftool.sifchain as sifchain +from siftool.common import * # These are utilities to interact with running environment (running agains local ganache-cli/hardhat/sifnoded). @@ -118,10 +118,10 @@ def get_env_ctx_peggy2(): # Monkeypatching for peggy2 extras # TODO These are set in run_env.py:Peggy2Environment.init_sifchain(), specifically "sifnoded tx ethbridge set-cross-chain-fee" # Consider passing them via environment - ctx.cross_chain_fee_base = 1 - ctx.cross_chain_lock_fee = 1 - ctx.cross_chain_burn_fee = 1 - ctx.ethereum_network_descriptor = ethereum_network_descriptor + ctx.eth.cross_chain_fee_base = 1 + ctx.eth.cross_chain_lock_fee = 1 + ctx.eth.cross_chain_burn_fee = 1 + ctx.eth.ethereum_network_descriptor = ethereum_network_descriptor return ctx @@ -277,6 +277,9 @@ def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, self.sifnode = sifchain.Sifnoded(self.cmd, home=sifnoded_home) self.sifnode_url = sifnode_url self.sifnode_chain_id = sifnode_chain_id + # Refactoring in progress: moving stuff into separate client that encapsulates things like url, home and chain_id + self.sifnode_client = sifchain.SifnodeClient(self.cmd, node=sifnode_url, home=sifnoded_home, chain_id=sifnode_chain_id, grpc_port=9090) + self.sifnode_client.ctx = self # For cross-chain fees for Peggy2 self.rowan_source = rowan_source self.ceth_symbol = ceth_symbol self.generic_erc20_contract = generic_erc20_contract @@ -522,30 +525,6 @@ def send_erc20_from_ethereum_to_sifchain(self, from_eth_addr, dest_sichain_addr, self.approve_erc20_token(token_sc, from_eth_addr, amount) self.bridge_bank_lock_eth(from_eth_addr, dest_sichain_addr, amount) - def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, denom): - """ Sends ETH from Sifchain to Ethereum (burn) """ - - # TODO Move to sifchain.py - - assert on_peggy2_branch, "Only for Peggy2.0" - - direction = "burn" - cross_chain_ceth_fee = self.cross_chain_fee_base * self.cross_chain_burn_fee # TODO - args = ["tx", "ethbridge", direction, from_sif_addr, to_eth_addr, str(amount), denom, str(cross_chain_ceth_fee), - "--network-descriptor", str(self.ethereum_network_descriptor), # Mandatory - "--from", from_sif_addr, # Mandatory, either name from keyring or address - "--gas-prices", "0.5rowan", - "--gas-adjustment", "1.5", - "--output", "json", - "-y" - ] + \ - self._sifnoded_home_arg() + \ - self._sifnoded_chain_id_and_node_arg() - res = self.sifnode.sifnoded_exec(args, keyring_backend=self.sifnode.keyring_backend) - result = json.loads(stdout(res)) - assert "failed to execute message" not in result["raw_log"] - return result - def create_sifchain_addr(self, moniker=None, fund_amounts=None): """ Generates a new sifchain address in test keyring. If moniker is given, uses it, otherwise diff --git a/test/integration/framework/src/truffle.py b/test/integration/framework/src/siftool/truffle.py similarity index 100% rename from test/integration/framework/src/truffle.py rename to test/integration/framework/src/siftool/truffle.py diff --git a/test/integration/framework/test/test_geth.py b/test/integration/framework/test/test_geth.py index ead1d2a442..d252e143e7 100644 --- a/test/integration/framework/test/test_geth.py +++ b/test/integration/framework/test/test_geth.py @@ -1,6 +1,6 @@ -import main as mod_main -import geth as mod_geth -from eth import ETH +import siftool.main as mod_main +import siftool.geth as mod_geth +from siftool.eth import ETH def geth_proof_of_concept(): diff --git a/test/integration/src/py/integration_test_context.py b/test/integration/src/py/integration_test_context.py index 220693a6db..27238c04f8 100644 --- a/test/integration/src/py/integration_test_context.py +++ b/test/integration/src/py/integration_test_context.py @@ -3,7 +3,8 @@ import logging import test_utilities -from integration_framework import main +import siftool_path +from siftool import main # TODO This class is obsolete, transitioning to test_utils.Peggy1EnvCtx diff --git a/test/integration/src/py/integration_framework.py b/test/integration/src/py/siftool_path.py similarity index 60% rename from test/integration/src/py/integration_framework.py rename to test/integration/src/py/siftool_path.py index 357a6dd1d5..bcf17f2722 100644 --- a/test/integration/src/py/integration_framework.py +++ b/test/integration/src/py/siftool_path.py @@ -1,7 +1,7 @@ import os import sys -# Temporary workaround to include integration framework +# Temporary workaround to include siftool project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), *([os.path.pardir] * 4))) base_dir = os.path.join(project_root, "test", "integration", "framework") enabled = False @@ -9,16 +9,3 @@ enabled = enabled or os.path.realpath(p) == os.path.realpath(base_dir) if not enabled: sys.path = sys.path + [base_dir] - -import command -import cosmos -import eth -import main -import common -import project -import geth -import hardhat -import truffle -import test_utils -import inflate_tokens -import sifchain From 0f96d7f37bb05e3f18a051d93ded833aabf38d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 11:49:34 +0100 Subject: [PATCH 49/70] Minor refactoring --- test/integration/framework/src/siftool/eth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index ff61a4cef1..0771c661f2 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -396,10 +396,10 @@ def estimate_gas_price(): __web3_enabled_unaudited_hdwallet_features = False # https://stackoverflow.com/questions/68050645/how-to-create-a-web3py-account-using-mnemonic-phrase -def _mnemonic_to_private_key(mnemonic, account_path="m/44'/60'/0'/0/0"): +def _mnemonic_to_private_key(mnemonic, derivation_path="m/44'/60'/0'/0/0"): a = web3.Web3().eth.account global __web3_enabled_unaudited_hdwallet_features if not __web3_enabled_unaudited_hdwallet_features: a.enable_unaudited_hdwallet_features() __web3_enabled_unaudited_hdwallet_features = True - return a.from_mnemonic(mnemonic, account_path=account_path).privateKey.hex()[2:] + return a.from_mnemonic(mnemonic, account_path=derivation_path).privateKey.hex()[2:] From be74eded360495120fc45091f18a82ab46b9968c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 12:38:44 +0100 Subject: [PATCH 50/70] Cleanup --- .../framework/src/siftool/sifchain.py | 138 +----------------- 1 file changed, 4 insertions(+), 134 deletions(-) diff --git a/test/integration/framework/src/siftool/sifchain.py b/test/integration/framework/src/siftool/sifchain.py index 967a841ebf..b90878b378 100644 --- a/test/integration/framework/src/siftool/sifchain.py +++ b/test/integration/framework/src/siftool/sifchain.py @@ -280,6 +280,10 @@ def encode_transaction(self, tx): self.cmd.rm(tmp_file) def open_grpc_channel(self): + # See https://docs.cosmos.network/v0.44/core/proto-docs.html + # See https://docs.cosmos.network/v0.44/core/grpc_rest.html + # See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 + # See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml import grpc return grpc.insecure_channel("127.0.0.1:9090") @@ -312,140 +316,6 @@ def sifnoded_exec(self, *args, **kwargs): return self.ctx.sifnode.sifnoded_exec(*args, **kwargs) -# # See https://docs.cosmos.network/v0.44/core/proto-docs.html -# # See https://docs.cosmos.network/v0.44/core/grpc_rest.html -# # See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 -# # See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml -# class SifnodeGrpc: -# def __init__(self): -# pass -# -# def ethbridge_lock(self): -# pass -# -# def ethbridge_burn(self): -# args = ["tx"] -# pass -# -# -# def grpc_poc(): -# log.debug("Hello gRPC") -# import grpc -# import sifnode.ethbridge.v1.tx_pb2_grpc as tx_pb2_grpc -# import sifnode.ethbridge.v1.tx_pb2 as tx_pb2 -# import sifnode.oracle.v1.network_descriptor_pb2 as network_descriptor_pb2 -# import sifnode.ethbridge.v1.query_pb2 as query_pb2 -# import sifnode.ethbridge.v1.query_pb2_grpc as query_pb2_grpc -# -# import cosmos.tx.v1beta1.service_pb2 as service_pb2 -# -# network_descriptor = 9999 # Set in run_env::Peggy2Environment.run() -# -# channel = grpc.insecure_channel("127.0.0.1:9090") -# -# client1 = query_pb2_grpc.QueryStub(channel) -# client2 = tx_pb2_grpc.MsgStub(channel) -# -# req1 = query_pb2.QueryBlacklistRequest() -# res1 = client1.GetBlacklist(req1) -# -# req2 = query_pb2.QueryCrosschainFeeConfigRequest(network_descriptor=network_descriptor) -# res2 = client1.CrosschainFeeConfig(req2) -# -# req3 = tx_pb2.MsgLock(amount=str(1000), cosmos_sender="sender", crosschain_fee=str(0), denom_hash="denom_hash", -# ethereum_receiver="ethereum_receiver", network_descriptor=network_descriptor_pb2.NETWORK_DESCRIPTOR_ETHEREUM) -# res3 = client2.Lock(req3) -# -# tx_plain_send = { -# "body": { -# "messages": [ -# { -# "@type": "/cosmos.bank.v1beta1.MsgSend", -# "from_address": "sif1n7ctv0zqyx78203lqssafejvgfhqephpnk5duc", -# "to_address": "sif1c7z93gth8meuls88l9e53mkws8k4c3x4p6kayr", -# "amount": [{"denom": "rowan", "amount": "1"}] -# } -# ], -# "memo": "", -# "timeout_height": "0", -# "extension_options": [], -# "non_critical_extension_options": [] -# }, -# "auth_info": { -# "signer_infos": [], -# "fee": {"amount": [], "gas_limit": "200000", "payer": "", "granter": ""} -# }, -# "signatures": [] -# } -# -# tx_ethbridge_lock = { -# "body": { -# "messages": [ -# { -# "@type": "/sifnode.ethbridge.v1.MsgBurn", -# "cosmos_sender": "sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk", -# "amount": "3000000000000000000", -# "denom_hash": "sifBridge99990xc6ba8c3233ecf65b761049ef63466945c362edd2", -# "network_descriptor": "NETWORK_DESCRIPTOR_HARDHAT", -# "ethereum_receiver": "0xcDE217398B5A290005D5f247137211AEA992B937", -# "crosschain_fee": "1" -# } -# ], -# "memo": "", -# "timeout_height": "0", -# "extension_options": [], -# "non_critical_extension_options": [] -# }, -# "auth_info": { -# "signer_infos": [], -# "fee": {"amount": [{"denom": "rowan", "amount": "100000"}], "gas_limit": "200000", "payer": "", "granter": ""} -# }, -# "signatures": [] -# } -# -# # sifnoded tx sign a.json --chain-id localnet --from sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk --keyring-backend test --home /tmp/sifnodedNetwork/validators/localnet/little-leaf/.sifnoded -# signed_tx = { -# "body": { -# "messages": [ -# { -# "@type": "/sifnode.ethbridge.v1.MsgBurn", -# "cosmos_sender": "sif18hsjhq8guhr5yj7n0q2764a76wmt3la2wykdjk", -# "amount": "3000000000000000000", -# "denom_hash": "sifBridge99990xc6ba8c3233ecf65b761049ef63466945c362edd2", -# "network_descriptor": "NETWORK_DESCRIPTOR_HARDHAT", -# "ethereum_receiver": "0xcDE217398B5A290005D5f247137211AEA992B937", -# "crosschain_fee": "1" -# } -# ], -# "memo": "", -# "timeout_height": "0", -# "extension_options": [], -# "non_critical_extension_options": [] -# }, -# "auth_info": { -# "signer_infos": [ -# { -# "public_key": { -# "@type": "/cosmos.crypto.secp256k1.PubKey", -# "key": "AtRIz7t6jZhX+7ZPLMmkqtF2BsZWB76164Kbh2g5KdIK" -# }, -# "mode_info": { -# "single": {"mode": "SIGN_MODE_DIRECT"} -# }, -# "sequence": "0" -# } -# ], -# "fee": {"amount": [{"denom": "rowan", "amount": "100000"}], "gas_limit": "200000", "payer": "", "granter": ""} -# }, -# "signatures": [ -# "x1TmzWEQk2qSDBHbGO5WZQq14MrG6KIu7RXWBNYoFyN8hgIrnrL6k8kBGrIdsUCpRFmAouVYJefYef3YzhkqBg==" -# ] -# } -# # To set sequence, add --sequence 12345 --offline --account-number -# -# print() - - class Sifgen: def __init__(self, cmd): self.cmd = cmd From 46190eeb9ddb94c46f58f56216aabc70e6f33c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 13:38:31 +0100 Subject: [PATCH 51/70] Bugfix --- test/integration/framework/src/siftool/run_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/src/siftool/run_env.py b/test/integration/framework/src/siftool/run_env.py index f7d851dcb5..7817a774ef 100644 --- a/test/integration/framework/src/siftool/run_env.py +++ b/test/integration/framework/src/siftool/run_env.py @@ -649,7 +649,7 @@ def run(self): } self.project.write_vagrantenv_sh(self.state_vars, self.data_dir, self.ethereum_websocket_address, self.chainnet) - import localnet + from siftool import localnet localnet.run_localnet_hook() return ganache_proc, sifnoded_proc, ebrelayer_proc From 1b3e0342199fedb49a63859bd31011820743d97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 13:52:21 +0100 Subject: [PATCH 52/70] Bugfixes --- .../execute_integration_tests_against_test_chain_peg.sh | 2 +- test/integration/src/py/conftest.py | 2 +- test/integration/src/py/siftool_path.py | 2 +- test/integration/src/py/test_inflate_tokens.py | 7 ++++--- test/integration/src/py/test_ofac_blocklist.py | 5 +++-- ..._integration_framework.py => test_siftool_framework.py} | 7 +++---- 6 files changed, 13 insertions(+), 12 deletions(-) rename test/integration/src/py/{test_integration_framework.py => test_siftool_framework.py} (98%) diff --git a/test/integration/execute_integration_tests_against_test_chain_peg.sh b/test/integration/execute_integration_tests_against_test_chain_peg.sh index 8aff910f9c..68d10377f5 100755 --- a/test/integration/execute_integration_tests_against_test_chain_peg.sh +++ b/test/integration/execute_integration_tests_against_test_chain_peg.sh @@ -20,7 +20,7 @@ python3 -m pytest -olog_level=$loglevel -v -olog_file=/tmp/log.txt -v \ ${TEST_INTEGRATION_PY_DIR}/test_random_currency_roundtrip.py \ ${TEST_INTEGRATION_PY_DIR}/test_rollback_chain.py \ ${TEST_INTEGRATION_PY_DIR}/test_ofac_blocklist.py \ - ${TEST_INTEGRATION_PY_DIR}/test_integration_framework.py \ + ${TEST_INTEGRATION_PY_DIR}/test_siftool_framework.py \ ${TEST_INTEGRATION_PY_DIR}/test_inflate_tokens.py \ # run replay tests after other tests since they interact badly with replaydb diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index 04442d63ff..b66ed8ee7c 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -4,7 +4,7 @@ import threading import pytest -import integration_test_context +import siftool_path import test_utilities from burn_lock_functions import decrease_log_level, force_log_level diff --git a/test/integration/src/py/siftool_path.py b/test/integration/src/py/siftool_path.py index bcf17f2722..94c29bb0ef 100644 --- a/test/integration/src/py/siftool_path.py +++ b/test/integration/src/py/siftool_path.py @@ -3,7 +3,7 @@ # Temporary workaround to include siftool project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), *([os.path.pardir] * 4))) -base_dir = os.path.join(project_root, "test", "integration", "framework") +base_dir = os.path.join(project_root, "test", "integration", "framework", "src") enabled = False for p in sys.path: enabled = enabled or os.path.realpath(p) == os.path.realpath(base_dir) diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index ae3132c773..17009bb9f1 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -1,8 +1,9 @@ import pytest -from integration_framework import main, common, eth, test_utils, sifchain, inflate_tokens -from inflate_tokens import InflateTokens -from common import * +import siftool_path +from siftool import eth, sifchain +from siftool.inflate_tokens import InflateTokens +from siftool.common import * # Sifchain wallets to which we want to distribute diff --git a/test/integration/src/py/test_ofac_blocklist.py b/test/integration/src/py/test_ofac_blocklist.py index a3f07a32de..0253310b49 100644 --- a/test/integration/src/py/test_ofac_blocklist.py +++ b/test/integration/src/py/test_ofac_blocklist.py @@ -1,7 +1,8 @@ import pytest -from integration_framework import main, common, eth, test_utils, inflate_tokens -from common import * +import siftool_path +from siftool import eth, test_utils +from siftool.common import * max_gas_required = 200000 diff --git a/test/integration/src/py/test_integration_framework.py b/test/integration/src/py/test_siftool_framework.py similarity index 98% rename from test/integration/src/py/test_integration_framework.py rename to test/integration/src/py/test_siftool_framework.py index e702c242c3..df7e86df3d 100644 --- a/test/integration/src/py/test_integration_framework.py +++ b/test/integration/src/py/test_siftool_framework.py @@ -1,10 +1,9 @@ import logging import web3 -from integration_framework import main, common, eth, test_utils, inflate_tokens -import eth -import test_utils -from common import * +import siftool_path +from siftool import eth +from siftool.common import * # Note: these tests burn a lot of ether very inefficiently. If you care about that make sure to recover From 34569f0e3e05da306af41843fad5ff84cb032158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 14:45:17 +0100 Subject: [PATCH 53/70] Bugfixes --- test/integration/framework/src/siftool/inflate_tokens.py | 3 +-- test/integration/src/py/conftest.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/src/siftool/inflate_tokens.py b/test/integration/framework/src/siftool/inflate_tokens.py index 9831ed6f28..0503261002 100644 --- a/test/integration/framework/src/siftool/inflate_tokens.py +++ b/test/integration/framework/src/siftool/inflate_tokens.py @@ -6,8 +6,7 @@ import logging import re -import siftool.eth -import siftool.test_utils +from siftool import eth, test_utils from siftool.common import * log = logging.getLogger(__name__) diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index b66ed8ee7c..c2104e086c 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -381,7 +381,7 @@ def ctx(request): snapshot_name = request.node.get_closest_marker("snapshot_name") if snapshot_name is not None: snapshot_name = snapshot_name.args[0] - from integration_framework import test_utils + from siftool import test_utils logging.error("Context setup: snapshot_name={}".format(repr(snapshot_name))) with test_utils.get_test_env_ctx() as ctx: yield ctx From 672b9029206e794cbfa862d20349ad57e7b8396a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 21 Mar 2022 16:40:13 +0100 Subject: [PATCH 54/70] Cleanup --- test/integration/framework/src/siftool/inflate_tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/src/siftool/inflate_tokens.py b/test/integration/framework/src/siftool/inflate_tokens.py index 0503261002..b27e7a2f56 100644 --- a/test/integration/framework/src/siftool/inflate_tokens.py +++ b/test/integration/framework/src/siftool/inflate_tokens.py @@ -35,7 +35,7 @@ def __init__(self, ctx): def get_whitelisted_tokens(self): whitelist = self.ctx.get_whitelisted_tokens_from_bridge_bank_past_events() - ibc_pattern = re.compile("^ibc\/([0-9a-fA-F]{64})$") + ibc_pattern = re.compile("^ibc/([0-9a-fA-F]{64})$") result = [] for token_addr, value in whitelist.items(): token_data = self.ctx.get_generic_erc20_token_data(token_addr) From 25a3ffa339e410e58129f1d391a609c35da95b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 3 Apr 2022 20:00:42 +0200 Subject: [PATCH 55/70] Exclude test_inflate_tokens_long from GitHub CI because it takes too long --- test/integration/src/py/test_inflate_tokens.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/src/py/test_inflate_tokens.py b/test/integration/src/py/test_inflate_tokens.py index 17009bb9f1..2de251d0b6 100644 --- a/test/integration/src/py/test_inflate_tokens.py +++ b/test/integration/src/py/test_inflate_tokens.py @@ -78,7 +78,9 @@ def test_inflate_tokens_short(ctx): _test_inflate_tokens_parametrized(ctx, 3) +# This test takes >1h, times out in GitHub CI @pytest.mark.skipif("on_peggy2_branch") +@pytest.mark.skipif("in_github_ci") def test_inflate_tokens_long(ctx): _test_inflate_tokens_parametrized(ctx, 300) From 0b774a010f35d57f83fe298e859f68bb4a5af8fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 6 Apr 2022 12:45:51 +0200 Subject: [PATCH 56/70] Merge siftool changes from future/peggy2 --- test/integration/framework/README.md | 160 ++--------------- test/integration/framework/notes.md | 161 ++++++++++++++++++ .../framework/src/siftool/command.py | 10 +- .../framework/src/siftool/common.py | 7 +- .../framework/src/siftool/cosmos.py | 59 +++++++ test/integration/framework/src/siftool/eth.py | 4 +- .../framework/src/siftool/hardhat.py | 4 + .../framework/src/siftool/project.py | 5 +- .../framework/src/siftool/run_env.py | 78 ++++----- .../framework/src/siftool/sifchain.py | 90 +++++----- .../framework/src/siftool/test_utils.py | 161 ++++++++++++++---- test/integration/src/py/siftool_path.py | 15 +- 12 files changed, 479 insertions(+), 275 deletions(-) create mode 100644 test/integration/framework/notes.md diff --git a/test/integration/framework/README.md b/test/integration/framework/README.md index d0006e2baf..f579ef0f1d 100644 --- a/test/integration/framework/README.md +++ b/test/integration/framework/README.md @@ -1,156 +1,16 @@ -# Resources +# siftool -1. Docker setup in docker/ (currently only on future/peggy2 branch, Tim Lind): +To start the local environment: -- setups two sifnode instances running independent chains + IBC relayer (ts-relayer) +siftool run-env -2. Brent's PoC (docker): https://github.com/Sifchain/sifchain-deploy/tree/feature/ibc-poc/docker/localnet/ibc +It will automatically install Python dependencies upon first use. This command will detect if you are on Peggy1 or +Peggy2 branch, and will start local processes accordingly: +- For Peggy1, it will run ganache-cli, sifnoded and ebrelayer. +- For Peggy2, it will run hardhat, sifnoded and two instances of ebrelayer. -3. Test environment for testing the new Sifchain public SDK (Caner): +At the moment, the environment consists of Ethereum-compliant local node (ganache/hardhat), one `sifnode` validator and +a Peggy bridge implemented by `ebrelayer` binary. -- https://docs.google.com/document/d/1MAlg-I0xMnUvbavAZdAN---WuqbyuRyKw-6Lfgfe130/edit -- https://github.com/sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/index.ts#L1 -- https://github.com/Sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/cosmoshub/index.ts -4. scripts/init-multichain.sh (on future/peggy2 branch) - -5. https://github.com/Sifchain/sifnode/commit/9ab620e148be8f4850eef59d39b0e869956f87a4 - -6. sifchain-devops script to deploy TestNet (by \_IM): https://github.com/Sifchain/sifchain-devops/blob/main/scripts/testnet/launch.sh#L19 - -7. Tempnet scripts by chainops - -8. In Sifchain/sifnode/scripts there's init.sh which, if you have everything installed, will run a single node. Ping - @Brianosaurus for more info. - -9. erowan should be deployed and whitelisted (assumption) - -# RPC endpoints: - -e.g. SIFNODE="https://api-testnet.sifchain.finance" - -- $SIFNODE/node_info -- $SIFNODE/tokenregistry/entries - -# Peggy2 devenv - -- Directory: smart-contracts/scripts/src/devenv -- Init: cd smart-contracts; rm -rf node_modules; npm install (plan is to move to yarn eventually) -- Run: GOBIN=/home/anderson/go/bin npx hardhat run scripts/devenv.ts - -``` -{ - // vscode launch.json file to debug the Dev Environment Scripts - "version": "0.2.0", - "configurations": [ - { - "runtimeArgs": [ - "node_modules/.bin/hardhat", - "run" - ], - "cwd": "${workspaceFolder}/smart-contracts", - "type": "node", - "request": "launch", - "name": "Dev Environment Debugger", - "env": { - "GOBIN": "/home/anderson/go/bin" - }, - "skipFiles": [ - "/**" - ], - "program": "${workspaceFolder}/smart-contracts/scripts/devenv.ts", - } - ] -} -``` - -- Integration test to be targeted for PoC: test_eth_transfers.py -- Dependency diagram: https://files.slack.com/files-pri/T0187TWB4V8-F02BC477N79/sifchaindevenv.jpg - -# Standardized environment setup - -## Peggy1 - Tempnet on AWS - -chain_id = "mychain" // Parameter - -// Generate account with name 'sif' in the local keyring -mnemonic = generate_mnemonic() -exec("echo $mnemonic | sifnoded keys add --recover --keyring-backend test") -sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx... - -// Init the chain. This command creates files: -// ~/.sifnoded/config/node_key.json -// ~/.sifnoded/config/genesis.json -// ~/.sifnoded/config/priv_validator_key.json -// ~/.sifnoded/data/priv_validator_state.json -// and prints some JSON (what?) -exec("sifnoded init {moniker} --chain-id {chain_id}") - -// Add Genesis Accounts -exec("sifnoded add-genesis-account {sif_admin} --keyring-backend test 999999000000000000000000000rowan,500000000000000000000000catk,500000000000000000000000cbtk,500000000000000000000000ceth,990000000000000000000000000stake,500000000000000000000000cdash,500000000000000000000000clink") - -// Add Genesis CLP ADMIN sif -exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") - -// Add Genesis CLP ADMIN sif -exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") - -// Set Genesis whitelist admin ${SIF_WALLET} -exec("sifnoded set-genesis-whitelister-admin {sif_admin} --keyring-backend test") - -// Fund account (Genesis TX stake) -exec("sifnoded gentx {sif_admin} 1000000000000000000000000stake --keyring-backend test --chain-id {chain_id}") - -// Generate token json -sifnoded q tokenregistry generate -o json \ - --token_base_denom=cosmos \ - --token_ibc_counterparty_chain_id=${GAIA_CHAIN_ID} \ - --token_ibc_channel_id=$GAIA_CHANNEL_ID \ - --token_ibc_counterparty_channel_id=$GAIA_COUNTERPARTY_CHANNEL_ID \ - --token_ibc_counterparty_denom="" \ - --token_unit_denom="" \ - --token_decimals=6 \ - --token_display_name="COSMOS" \ - --token_external_symbol="cosmos" \ - --token_permission_clp=true \ - --token_permission_ibc_export=true \ - --token_permission_ibc_import=true | jq > gaia.json - -// Whitelist tokens -// printf "registering cosmos... \n" -sifnoded tx tokenregistry register gaia.json \ - --node tcp://${SIFNODE_P2P_HOSTNAME}:26657 \ - --chain-id $SIFCHAIN_ID \ - --from $SIF_WALLET \ - --keyring-backend test \ - --gas=500000 \ - --gas-prices=0.5rowan \ - -y - -// Deploy token registry -// Registering Tokens... -// Set Whitelist from denoms.json... -sifnoded set-gen-denom-whitelist DENOM.json - -## Peggy1 - integration tests - -// Parameters: validator moniker, validator mnemonic -valicator1_moniker, validator1_address, validator1_password, validator1_mnemonic = exec("sifgen create network ...") - -sifnoded_keys_add(validator1_moniker, validator1_password) // Test keyring -valoper = get_val_address(validator1_moniker) - -exec("sifnoded add-genesis-validators {valoper}") -exec("sifnoded add-geneeis-account {}") -exec("sifnoded set-genesis-oracle-admin {}") -exec("sifnoded set-denom-whitelist {}") - -## Coupled with the localnet framework - -The localnet test framework is located under `./test/localnet` within the same repository and offers some interesting features such as spinning up a bunch of IBC chains along with relayers and storing the states of the chains for later use for deterministic testing against various IBC flows. - -The `localnet` framework is supported by `siftool` and can be enabled by using the following environment variable `LOCALNET` set to `true` as follow: - -``` -LOCALNET=true siftool run-env -``` +Original design document: https://docs.google.com/document/d/1IhE2Y03Z48ROmTwO9-J_0x_lx2vIOFkyDFG7BkAIqCk/edit# diff --git a/test/integration/framework/notes.md b/test/integration/framework/notes.md new file mode 100644 index 0000000000..00f0ef4d7c --- /dev/null +++ b/test/integration/framework/notes.md @@ -0,0 +1,161 @@ +# siftool + +Original design document: https://docs.google.com/document/d/1IhE2Y03Z48ROmTwO9-J_0x_lx2vIOFkyDFG7BkAIqCk/edit# + + +# Resources + +1. Docker setup in docker/ (currently only on future/peggy2 branch, Tim Lind): + +- setups two sifnode instances running independent chains + IBC relayer (ts-relayer) + +2. PoC (docker): https://github.com/Sifchain/sifchain-deploy/tree/feature/ibc-poc/docker/localnet/ibc + +3. Test environment for testing the new Sifchain public SDK (Caner): + +- https://docs.google.com/document/d/1MAlg-I0xMnUvbavAZdAN---WuqbyuRyKw-6Lfgfe130/edit +- https://github.com/sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/index.ts#L1 +- https://github.com/Sifchain/sifchain-ui/blob/3868ac7138c6c4149dced4ced5b36690e5fc1da7/ui/core/src/config/chains/cosmoshub/index.ts + +4. scripts/init-multichain.sh (on future/peggy2 branch) + +5. https://github.com/Sifchain/sifnode/commit/9ab620e148be8f4850eef59d39b0e869956f87a4 + +6. sifchain-devops script to deploy TestNet (by \_IM): https://github.com/Sifchain/sifchain-devops/blob/main/scripts/testnet/launch.sh#L19 + +7. Tempnet scripts by chainops + +8. In Sifchain/sifnode/scripts there's init.sh which, if you have everything installed, will run a single node. Ping + @Brianosaurus for more info. + +9. erowan should be deployed and whitelisted (assumption) + +# RPC endpoints: + +e.g. SIFNODE="https://api-testnet.sifchain.finance" + +- $SIFNODE/node_info +- $SIFNODE/tokenregistry/entries + +# Peggy2 devenv + +- Directory: smart-contracts/scripts/src/devenv +- Init: cd smart-contracts; rm -rf node_modules; npm install (plan is to move to yarn eventually) +- Run: GOBIN=/home/anderson/go/bin npx hardhat run scripts/devenv.ts + +``` +{ + // vscode launch.json file to debug the Dev Environment Scripts + "version": "0.2.0", + "configurations": [ + { + "runtimeArgs": [ + "node_modules/.bin/hardhat", + "run" + ], + "cwd": "${workspaceFolder}/smart-contracts", + "type": "node", + "request": "launch", + "name": "Dev Environment Debugger", + "env": { + "GOBIN": "/home/anderson/go/bin" + }, + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/smart-contracts/scripts/devenv.ts", + } + ] +} +``` + +- Integration test to be targeted for PoC: test_eth_transfers.py +- Dependency diagram: https://files.slack.com/files-pri/T0187TWB4V8-F02BC477N79/sifchaindevenv.jpg + +# Standardized environment setup + +## Peggy1 - Tempnet on AWS + +chain_id = "mychain" // Parameter + +// Generate account with name 'sif' in the local keyring +mnemonic = generate_mnemonic() +exec("echo $mnemonic | sifnoded keys add --recover --keyring-backend test") +sif_admin = exec("sifnoded keys show sif -a --keyring-backend test") // sif1xxx... + +// Init the chain. This command creates files: +// ~/.sifnoded/config/node_key.json +// ~/.sifnoded/config/genesis.json +// ~/.sifnoded/config/priv_validator_key.json +// ~/.sifnoded/data/priv_validator_state.json +// and prints some JSON (what?) +exec("sifnoded init {moniker} --chain-id {chain_id}") + +// Add Genesis Accounts +exec("sifnoded add-genesis-account {sif_admin} --keyring-backend test 999999000000000000000000000rowan,500000000000000000000000catk,500000000000000000000000cbtk,500000000000000000000000ceth,990000000000000000000000000stake,500000000000000000000000cdash,500000000000000000000000clink") + +// Add Genesis CLP ADMIN sif +exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") + +// Add Genesis CLP ADMIN sif +exec("sifnoded add-genesis-clp-admin ${sif_admin} --keyring-backend test") + +// Set Genesis whitelist admin ${SIF_WALLET} +exec("sifnoded set-genesis-whitelister-admin {sif_admin} --keyring-backend test") + +// Fund account (Genesis TX stake) +exec("sifnoded gentx {sif_admin} 1000000000000000000000000stake --keyring-backend test --chain-id {chain_id}") + +// Generate token json +sifnoded q tokenregistry generate -o json \ + --token_base_denom=cosmos \ + --token_ibc_counterparty_chain_id=${GAIA_CHAIN_ID} \ + --token_ibc_channel_id=$GAIA_CHANNEL_ID \ + --token_ibc_counterparty_channel_id=$GAIA_COUNTERPARTY_CHANNEL_ID \ + --token_ibc_counterparty_denom="" \ + --token_unit_denom="" \ + --token_decimals=6 \ + --token_display_name="COSMOS" \ + --token_external_symbol="cosmos" \ + --token_permission_clp=true \ + --token_permission_ibc_export=true \ + --token_permission_ibc_import=true | jq > gaia.json + +// Whitelist tokens +// printf "registering cosmos... \n" +sifnoded tx tokenregistry register gaia.json \ + --node tcp://${SIFNODE_P2P_HOSTNAME}:26657 \ + --chain-id $SIFCHAIN_ID \ + --from $SIF_WALLET \ + --keyring-backend test \ + --gas=500000 \ + --gas-prices=0.5rowan \ + -y + +// Deploy token registry +// Registering Tokens... +// Set Whitelist from denoms.json... +sifnoded set-gen-denom-whitelist DENOM.json + +## Peggy1 - integration tests + +// Parameters: validator moniker, validator mnemonic +valicator1_moniker, validator1_address, validator1_password, validator1_mnemonic = exec("sifgen create network ...") + +sifnoded_keys_add(validator1_moniker, validator1_password) // Test keyring +valoper = get_val_address(validator1_moniker) + +exec("sifnoded add-genesis-validators {valoper}") +exec("sifnoded add-geneeis-account {}") +exec("sifnoded set-genesis-oracle-admin {}") +exec("sifnoded set-denom-whitelist {}") + +## Coupled with the localnet framework + +The localnet test framework is located under `./test/localnet` within the same repository and offers some interesting features such as spinning up a bunch of IBC chains along with relayers and storing the states of the chains for later use for deterministic testing against various IBC flows. + +The `localnet` framework is supported by `siftool` and can be enabled by using the following environment variable `LOCALNET` set to `true` as follow: + +``` +LOCALNET=true siftool run-env +``` diff --git a/test/integration/framework/src/siftool/command.py b/test/integration/framework/src/siftool/command.py index 3bb147a018..7634676316 100644 --- a/test/integration/framework/src/siftool/command.py +++ b/test/integration/framework/src/siftool/command.py @@ -1,9 +1,13 @@ import shutil import time +from typing import Mapping, List, Union, Optional from siftool.common import * +ExecArgs = Mapping[str, Union[List[str], str, Mapping[str, str]]] -def buildcmd(args, cwd=None, env=None): + +def buildcmd(args: Optional[str] = None, cwd: Optional[str] = None, env: Optional[Mapping[str, Optional[str]]] = None +) -> ExecArgs: return dict((("args", args),) + ((("cwd", cwd),) if cwd is not None else ()) + ((("env", env),) if env is not None else ()) @@ -28,14 +32,14 @@ def execst(self, args, cwd=None, env=None, stdin=None, binary=False, pipe=True, return proc.returncode, stdout_data, stderr_data # Default implementation of popen for environemnts to start long-lived processes - def popen(self, args, log_file=None, **kwargs): + def popen(self, args, log_file=None, **kwargs) -> subprocess.Popen: stdout = log_file or None stderr = log_file or None return popen(args, stdout=stdout, stderr=stderr, **kwargs) # Starts a process asynchronously (for sifnoded, hardhat, ebrelayer etc.) # The arguments should correspond to what buildcmd() returns. - def spawn_asynchronous_process(self, exec_args, log_file=None): + def spawn_asynchronous_process(self, exec_args: ExecArgs, log_file=None) -> subprocess.Popen: return self.popen(**exec_args, log_file=log_file) def ls(self, path): diff --git a/test/integration/framework/src/siftool/common.py b/test/integration/framework/src/siftool/common.py index 0433e4afc2..6a0d992ffe 100644 --- a/test/integration/framework/src/siftool/common.py +++ b/test/integration/framework/src/siftool/common.py @@ -6,7 +6,7 @@ import random import yaml import urllib.request - +from typing import Optional, Mapping, Sequence, IO, Union log = logging.getLogger(__name__) @@ -76,7 +76,10 @@ def mkcmd(args, env=None, cwd=None, stdin=None): # stdin will always be redirected to the returned process' stdin. # If pipe, the stdout and stderr will be redirected and available as stdout and stderr of the returned object. # If not pipe, the stdout and stderr will not be redirected and will inherit sys.stdout and sys.stderr. -def popen(args, cwd=None, env=None, text=None, stdin=None, stdout=None, stderr=None): +def popen(args: Sequence[str], cwd: Optional[str] = None, env: Optional[Mapping[str, str]] = None, + text: Optional[bool] = None, stdin: Union[str, int, IO, None] = None, stdout: Optional[IO] = None, + stderr: Optional[IO] = None +) -> subprocess.Popen: if env: env = dict_merge(os.environ, env) logging.debug(f"popen(): args={repr(args)}, cwd={repr(cwd)}") diff --git a/test/integration/framework/src/siftool/cosmos.py b/test/integration/framework/src/siftool/cosmos.py index f82cee6ba8..157740c242 100644 --- a/test/integration/framework/src/siftool/cosmos.py +++ b/test/integration/framework/src/siftool/cosmos.py @@ -1,7 +1,66 @@ +from typing import Union, Iterable, Mapping, List from siftool.common import * akash_binary = "akash" +LegacyBalance = List[List[Union[int, str]]] # e.g. [[3, "rowan"], [2, "ibc/xxxxx"]] +Balance = Mapping[str, int] +CompatBalance = Union[LegacyBalance, Balance] +Address = str + + +def balance_normalize(bal: CompatBalance = None) -> Balance: + if type(bal) == list: + bal = dict(((k, v) for v, k in bal)) + elif type(bal) == dict: + pass + else: + assert False, "Balances should be either a dict or a list" + return {k: v for k, v in bal.items() if v != 0} + + +def balance_add(bal1: Balance, bal2: Balance) -> Balance: + result = {} + for denom in set(bal1.keys()).union(set(bal2.keys())): + val = bal1.get(denom, 0) + bal2.get(denom, 0) + if val != 0: + result[denom] = val + return result + + +def balance_neg(bal: Balance) -> Balance: + return {k: -v for k, v in bal.items()} + + +def balance_sub(bal1: Balance, bal2: Balance) -> Balance: + return balance_add(bal1, balance_neg(bal2)) + + +def balance_zero(bal: Balance) -> bool: + return len(bal) == 0 + + +def balance_equal(bal1: Balance, bal2: Balance) -> bool: + return balance_zero(balance_sub(bal1, bal2)) + + +def balance_format(bal: Balance) -> str: + return ",".join("{}{}".format(v, k) for k, v in bal.items()) + + +def balance_exceeds(bal: Balance, min_changes: Balance) -> bool: + have_all = True + for denom, required_value in min_changes.items(): + actual_value = bal.get(denom, 0) + if required_value < 0: + have_all &= actual_value <= required_value + elif required_value > 0: + have_all &= actual_value >= required_value + else: + assert False + return have_all + + # # This is for Akash, but might be useful for other cosmos-based chains as well. (If not, it should be moved to separate diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 0771c661f2..4621d0d525 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -65,7 +65,7 @@ class EthereumTxWrapper: """ def __init__(self, w3_conn, is_local_node): - self.w3_conn = w3_conn + self.w3_conn: web3.Web3 = w3_conn self.use_eip_1559 = True self.private_keys = {} self.default_timeout = 600 @@ -87,7 +87,7 @@ def __init__(self, w3_conn, is_local_node): def _get_private_key(self, addr): addr = web3.Web3.toChecksumAddress(addr) - if not addr in self.private_keys: + if addr not in self.private_keys: raise Exception(f"No private key set for address {addr}") return self.private_keys[addr] diff --git a/test/integration/framework/src/siftool/hardhat.py b/test/integration/framework/src/siftool/hardhat.py index 93258a34da..918ada7e03 100644 --- a/test/integration/framework/src/siftool/hardhat.py +++ b/test/integration/framework/src/siftool/hardhat.py @@ -66,6 +66,10 @@ def get_descriptor(self, sc_name): "CosmosBridge": [], "Rowan": ["BridgeBank"], "TrollToken": ["Mocks"], + "FailHardToken": ["Mocks"], + "UnicodeToken": ["Mocks"], + "CommissionToken": ["Mocks"], + "RandomTrollToken": ["Mocks"], }.get(sc_name, []) + [f"{sc_name}.sol", f"{sc_name}.json"] path = os.path.join(self.cmd.project.project_dir("smart-contracts/artifacts/contracts"), *relpath) tmp = json.loads(self.cmd.read_text_file(path)) diff --git a/test/integration/framework/src/siftool/project.py b/test/integration/framework/src/siftool/project.py index 854ad722fe..7b89837a9a 100644 --- a/test/integration/framework/src/siftool/project.py +++ b/test/integration/framework/src/siftool/project.py @@ -232,8 +232,11 @@ def clean(self): for file in [".proto-gen", ".run", "cmd/ebrelayer/contract/generated/artifacts", "smart-contracts/.hardhat-compile"]: self.cmd.rmf(self.project_dir(file)) else: - # Output from "truffle compile" + # Output from "truffle compile" / "npx hardhat compile". + # Wrong contents can cause hardhat to fail compilation after switching branches. self.cmd.rmf(self.project_dir("smart-contracts", "build")) + self.cmd.rmf(self.project_dir("smart-contracts", "cache")) + self.cmd.rmf(self.project_dir("smart-contracts", "artifacts")) for filename in ["sifnoded", "ebrelayer", "sifgen"]: self.cmd.rmf(os.path.join(self.go_bin_dir, filename)) diff --git a/test/integration/framework/src/siftool/run_env.py b/test/integration/framework/src/siftool/run_env.py index 7817a774ef..ad0fc329f3 100644 --- a/test/integration/framework/src/siftool/run_env.py +++ b/test/integration/framework/src/siftool/run_env.py @@ -1,9 +1,9 @@ import json import re import time +from typing import List, Tuple, TextIO, Any -import siftool.eth as eth -import siftool.hardhat as hardhat +from siftool import eth, hardhat, cosmos, command from siftool.truffle import Ganache from siftool.localnet import Localnet from siftool.command import Command @@ -771,7 +771,7 @@ def restart_processes(self): class Peggy2Environment(IntegrationTestsEnvironment): - def __init__(self, cmd): + def __init__(self, cmd: Command): super().__init__(cmd) self.hardhat = hardhat.Hardhat(cmd) @@ -842,31 +842,38 @@ def run(self): admin_account_name = "sifnodeadmin" chain_id = "localnet" ceth_symbol = sifchain_denom_hash(hardhat_chain_id, eth.NULL_ADDRESS) - print("ceth symbol is: {0}".format(ceth_symbol)) assert ceth_symbol == "sifBridge99990x0000000000000000000000000000000000000000" - # Mint goes to validator - mint_amount = [ - [999999 * 10**21, "rowan"], + # This goes to validator0, i.e. sifnode_validators[0]["address"] + validator_mint_amounts: cosmos.LegacyBalance = [ + [999999 * 10**27, "rowan"], [137 * 10**16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"], [999999 * 10**21, ceth_symbol], - ] + [[10**18, "test{}".format(i)] for i in range(1, 6)] + [137 * 10**16, "sifBridge00030x1111111111111111111111111111111111111111"], + ] validator_power = 100 seed_ip_address = "10.10.1.1" tendermint_port = 26657 denom_whitelist_file = project_dir("test", "integration", "whitelisted-denoms.json") - tokens = [ - [10**20, "rowan"], - [2 * 10**19, "ceth"] - ] + [[10**18, "xtest{}".format(i)] for i in range(1, 6)] + # These go to admin account, relayers and witnesses + admin_account_mint_amounts: cosmos.LegacyBalance = [ + [10**27, "rowan"], + [2 * 10**22, ceth_symbol], + [10 ** 16, "ibc/FEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACEFEEDFACE"], + [10 ** 16, "sifBridge00030x1111111111111111111111111111111111111111"], + ] registry_json = project_dir("smart-contracts", "src", "devenv", "registry.json") sifnoded_network_dir = "/tmp/sifnodedNetwork" # Gets written to .vscode/launch.json self.cmd.rmdir(sifnoded_network_dir) self.cmd.mkdir(sifnoded_network_dir) network_config_file, sifnoded_exec_args, sifnoded_proc, tcp_url, admin_account_address, sifnode_validators, \ sifnode_relayers, sifnode_witnesses, sifnode_validator0_home, chain_dir = \ - self.init_sifchain(sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, - validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, - admin_account_name) + self.init_sifchain(sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, + validator_mint_amounts, validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, + admin_account_mint_amounts, registry_json, admin_account_name, ceth_symbol) + + log.debug("ceth symbol is: {}".format(ceth_symbol)) + log.debug("Admin account address: {}".format(admin_account_address)) # tokens + log.debug("Validator 0 address: {}".format(sifnode_validators[0]["address"])) # mint symbol_translator_file = os.path.join(self.test_integration_dir, "config", "symbol_translator.json") [relayer0_exec_args], [witness0_exec_args] = \ @@ -919,10 +926,11 @@ def init_smart_contracts(self, w3_url, operator_account, deployed_contract_addre # txrcpt = eth_tx.transact_sync(cosmos_bridge.functions.setBridgeBank, operator_addr)(bridge_bank_addr) return - def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardhat_chain_id, mint_amount, - validator_power, seed_ip_address, tendermint_port, denom_whitelist_file, tokens, registry_json, - admin_account_name - ): + def init_sifchain(self, sifnoded_network_dir: str, sifnoded_log_file: TextIO, chain_id: str, hardhat_chain_id: int, + validator_mint_amounts: cosmos.LegacyBalance, validator_power: int, seed_ip_address: str, tendermint_port: int, + denom_whitelist_file: str, admin_account_mint_amounts: cosmos.LegacyBalance, registry_json: str, + admin_account_name: str, ceth_symbol: str + ) -> Tuple[str, command.ExecArgs, subprocess.Popen, str, cosmos.Address, List, List, List, str, str]: validator_count = 1 relayer_count = 1 witness_count = 1 @@ -932,7 +940,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh network_config_file_path = self.cmd.mktempfile() try: self.cmd.sifgen_create_network(chain_id, validator_count, sifnoded_network_dir, network_config_file_path, - seed_ip_address, mint_amount=mint_amount) + seed_ip_address, mint_amount=validator_mint_amounts) network_config_file = self.cmd.read_text_file(network_config_file_path) finally: self.cmd.rm(network_config_file_path) @@ -976,7 +984,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh sifnode = Sifnoded(self.cmd, home=validator0_home) # Create an ADMIN account on sifnode with name admin_account_name (e.g. "sifnodeadmin") - admin_account_address = sifnode.peggy2_add_account(admin_account_name, tokens, is_admin=True) + admin_account_address = sifnode.peggy2_add_account(admin_account_name, admin_account_mint_amounts, is_admin=True) # TODO Check if sifnoded_peggy2_add_relayer_witness_account can be executed offline (without sifnoded running) # TODO Check if sifnoded_peggy2_set_cross_chain_fee can be executed offline (without sifnoded running) @@ -985,7 +993,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh # Note: "--home" is shared with sifnoded's "--home" relayers = [{ "name": name, - "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, + "address": sifnode.peggy2_add_relayer_witness_account(name, admin_account_mint_amounts, hardhat_chain_id, validator_power, denom_whitelist_file), "home": validator0_home, } for name in [f"relayer-{i}" for i in range(relayer_count)]] @@ -994,19 +1002,12 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh # Note: "--home" is shared with sifnoded's "--home" witnesses = [{ "name": name, - "address": sifnode.peggy2_add_relayer_witness_account(name, tokens, hardhat_chain_id, + "address": sifnode.peggy2_add_relayer_witness_account(name, admin_account_mint_amounts, hardhat_chain_id, validator_power, denom_whitelist_file), "home": validator0_home, } for name in [f"witness-{i}" for i in range(witness_count)]] tcp_url = "tcp://{}:{}".format(ANY_ADDR, tendermint_port) - # sifnoded - # start - # --log_level debug - # --log_format json - # --minimum-gas-prices 0.5rowan - # --rpc.laddr tcp://0.0.0.0:26657 - # --home /tmp/sifnodedNetwork/validators/localnet/xxx-yyy/.sifnoded # @TODO Detect if sifnoded is already running, for now it fails silently and we wait forever in wait_for_sif_account_up sifnoded_exec_args = sifnode.build_start_cmd(tcp_url=tcp_url, minimum_gas_prices=[0.5, "rowan"], log_format_json=True) @@ -1017,7 +1018,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh # TODO This command exits with status 0, but looks like there are some errros. # The same happens also in devenv. # TODO Try whitelister account instead of admin - res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_address, + res = sifnode.peggy2_token_registry_register_all(registry_json, [0.5, "rowan"], 1.5, admin_account_name, chain_id) log.debug("Result from token registry: {}".format(repr(res))) assert len(res) == 1 @@ -1029,7 +1030,7 @@ def init_sifchain(self, sifnoded_network_dir, sifnoded_log_file, chain_id, hardh cross_chain_fee_base = 1 cross_chain_lock_fee = 1 cross_chain_burn_fee = 1 - ethereum_cross_chain_fee_token = "sifBridge99990x0000000000000000000000000000000000000000" + ethereum_cross_chain_fee_token = ceth_symbol assert hardhat_chain_id == int(ethereum_cross_chain_fee_token[9:13]) # Assume they should match gas_prices = [0.5, "rowan"] gas_adjustment = 1.5 @@ -1290,16 +1291,17 @@ def format_sif_account(sif_account): # "env": {"ETHEREUM_PRIVATE_KEY": eth_accounts["validators"][0][1]}, "args": [ "init-witness", - # TODO This is probably obsolete, need "--network-descriptor" etc. - str(eth_chain_id), - tcp_url, - w3_url, - evm_smart_contract_addrs["BridgeRegistry"], - witness["name"], + "--network-descriptor", str(eth_chain_id), + "--tendermint-node", tcp_url, + "--web3-provider", w3_url, + "--bridge-registry-contract-address", evm_smart_contract_addrs["BridgeRegistry"], + "--validator-mnemonic", witness["name"], "--chain-id", "localnet", "--node", tcp_url, "--keyring-backend", "test", "--from", witness["address"], + # TODO: This shouldnt be needed, it defaults to --home value + "--keyring-dir", witness["home"], "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", "--home", witness["home"] ] diff --git a/test/integration/framework/src/siftool/sifchain.py b/test/integration/framework/src/siftool/sifchain.py index b90878b378..913415dd9d 100644 --- a/test/integration/framework/src/siftool/sifchain.py +++ b/test/integration/framework/src/siftool/sifchain.py @@ -1,36 +1,39 @@ import base64 import json import time -import typing -from siftool.command import buildcmd +import grpc +from typing import Mapping, Any +from siftool import command, cosmos from siftool.common import * - -def sifchain_denom_hash(network_descriptor_raw: typing.Union[int, str], token_contract_address: str) -> str: +def sifchain_denom_hash(network_descriptor: int, token_contract_address: str) -> str: assert on_peggy2_branch assert token_contract_address.startswith("0x") - network_descriptor = int(network_descriptor_raw) - assert network_descriptor > 0 - assert network_descriptor <= 9999 + assert type(network_descriptor) == int + assert network_descriptor in range(1, 10000) denom = f"sifBridge{network_descriptor:04d}{token_contract_address.lower()}" return denom -def balance_delta(balances1, balances2): - all_denoms = set(balances1.keys()) - all_denoms.update(balances2.keys()) - result = {} - for denom in all_denoms: - change = balances2.get(denom, 0) - balances1.get(denom, 0) - if change != 0: - result[denom] = change - return result +# Deprecated +def balance_delta(balances1: cosmos.Balance, balances2: cosmos.Balance) -> cosmos.Balance: + return cosmos.balance_sub(balances2, balances1) + +# Deprecated +def balance_zero(balances: cosmos.Balance) -> bool: + return cosmos.balance_zero(balances) -def balance_zero(balances): - return len(balances) == 0 +def is_cosmos_native_denom(denom: str) -> bool: + """Returns true if denom is a native cosmos token (Rowan, ibc) + that was not imported using Peggy""" + return not str.startswith(denom, "sifBridge") + +def import_generated_protobuf_sources(): + import cosmos.tx.v1beta1.service_pb2 as cosmos_pb + import cosmos.tx.v1beta1.service_pb2_grpc as cosmos_pb_grpc class Sifnoded: - def __init__(self, cmd, home=None): + def __init__(self, cmd, home: str = None): self.cmd = cmd self.binary = "sifnoded" self.home = home @@ -102,6 +105,9 @@ def add_genesis_validators_peggy(self, evm_network_descriptor, valoper, validato def set_genesis_oracle_admin(self, address): self.sifnoded_exec(["set-genesis-oracle-admin", address], sifnoded_home=self.home) + def set_genesis_token_registry_admin(self, address): + self.sifnoded_exec(["set-genesis-token-registry-admin", address], sifnoded_home=self.home) + def set_genesis_whitelister_admin(self, address): self.sifnoded_exec(["set-genesis-whitelister-admin", address], sifnoded_home=self.home) @@ -118,7 +124,7 @@ def peggy2_add_account(self, name, tokens, is_admin=False): self.add_genesis_account(account_address, tokens) if is_admin: self.set_genesis_oracle_admin(account_address) - self.set_genesis_whitelister_admin(account_address) + self.set_genesis_whitelister_admin(account_address) return account_address def peggy2_add_relayer_witness_account(self, name, tokens, evm_network_descriptor, validator_power, denom_whitelist_file): @@ -170,14 +176,14 @@ def sifnoded_start(self, tcp_url=None, minimum_gas_prices=None, log_format_json= log_format_json=log_format_json) return self.cmd.spawn_asynchronous_process(sifnoded_exec_args, log_file=log_file) - def build_start_cmd(self, tcp_url=None, minimum_gas_prices=None, log_format_json=False): - args = [self.binary, "start"] + \ + def build_start_cmd(self, tcp_url: str = None, minimum_gas_prices=None, log_format_json=False): + args = [self.binary, "start", "--trace"] + \ (["--minimum-gas-prices", sif_format_amount(*minimum_gas_prices)] if minimum_gas_prices is not None else []) + \ (["--rpc.laddr", tcp_url] if tcp_url else []) + \ (["--log_level", "debug"] if log_format_json else []) + \ (["--log_format", "json"] if log_format_json else []) + \ (["--home", self.home] if self.home else []) - return buildcmd(args) + return command.buildcmd(args) def sifnoded_exec(self, args, sifnoded_home=None, keyring_backend=None, stdin=None, cwd=None): args = [self.binary] + args + \ @@ -221,13 +227,19 @@ def __init__(self, cmd, node=None, home=None, chain_id=None, grpc_port=None): self.chain_id = chain_id self.grpc_port = grpc_port - def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, denom, generate_only=False): + def query_account(self, sif_addr): + result = json.loads(stdout(self.sifnoded_exec(["query", "account", sif_addr, "--output", "json"]))) + return result + + def send_from_sifchain_to_ethereum(self, from_sif_addr: cosmos.Address, to_eth_addr: str, amount: int, denom: str, + generate_only: bool = False + ) -> Mapping: """ Sends ETH from Sifchain to Ethereum (burn) """ assert on_peggy2_branch, "Only for Peggy2.0" assert self.ctx.eth eth = self.ctx.eth - direction = "burn" + direction = "lock" if is_cosmos_native_denom(denom) else "burn" cross_chain_ceth_fee = eth.cross_chain_fee_base * eth.cross_chain_burn_fee # TODO args = ["tx", "ethbridge", direction, from_sif_addr, to_eth_addr, str(amount), denom, str(cross_chain_ceth_fee), "--network-descriptor", str(eth.ethereum_network_descriptor), # Mandatory @@ -246,16 +258,20 @@ def send_from_sifchain_to_ethereum(self, from_sif_addr, to_eth_addr, amount, den assert "failed to execute message" not in result["raw_log"] return result - def send_from_sifchain_to_ethereum_grpc(self, from_sif_addr, to_eth_addr, amount, denom): + def send_from_sifchain_to_ethereum_grpc(self, from_sif_addr: cosmos.Address, to_eth_addr: str, amount: int, + denom: str + ): tx = self.send_from_sifchain_to_ethereum(from_sif_addr, to_eth_addr, amount, denom, generate_only=True) signed_tx = self.sign_transaction(tx, from_sif_addr) encoded_tx = self.encode_transaction(signed_tx) result = self.broadcast_tx(encoded_tx) return result - def sign_transaction(self, tx, from_sif_addr, sequence=None): + def sign_transaction(self, tx: Mapping, from_sif_addr: cosmos.Address, sequence: int = None, + account_number: int = None + ) -> Mapping: tmp_tx_file = self.cmd.mktempfile() - account_number = 0 # TODO + assert (sequence is not None) == (account_number is not None) # We need either both or none try: self.cmd.write_text_file(tmp_tx_file, json.dumps(tx)) args = ["tx", "sign", tmp_tx_file, "--from", from_sif_addr] + \ @@ -269,7 +285,7 @@ def sign_transaction(self, tx, from_sif_addr, sequence=None): finally: self.cmd.rm(tmp_tx_file) - def encode_transaction(self, tx): + def encode_transaction(self, tx: Mapping[str, Any]) -> bytes: tmp_file = self.cmd.mktempfile() try: self.cmd.write_text_file(tmp_file, json.dumps(tx)) @@ -279,21 +295,19 @@ def encode_transaction(self, tx): finally: self.cmd.rm(tmp_file) - def open_grpc_channel(self): + def open_grpc_channel(self) -> grpc.Channel: # See https://docs.cosmos.network/v0.44/core/proto-docs.html # See https://docs.cosmos.network/v0.44/core/grpc_rest.html # See https://app.swaggerhub.com/apis/Ivan-Verchenko/sifnode-swagger-api/1.1.1 # See https://raw.githubusercontent.com/Sifchain/sifchain-ui/develop/ui/core/swagger.yaml - import grpc return grpc.insecure_channel("127.0.0.1:9090") - def broadcast_tx(self, encoded_tx): - import cosmos.tx.v1beta1.service_pb2 - import cosmos.tx.v1beta1.service_pb2_grpc - broadcast_mode = cosmos.tx.v1beta1.service_pb2.BROADCAST_MODE_ASYNC + def broadcast_tx(self, encoded_tx: bytes): + import_generated_protobuf_sources() + broadcast_mode = cosmos_pb.BROADCAST_MODE_ASYNC with self.open_grpc_channel() as channel: - tx_stub = cosmos.tx.v1beta1.service_pb2_grpc.ServiceStub(channel) - req = cosmos.tx.v1beta1.service_pb2.BroadcastTxRequest(tx_bytes=encoded_tx, mode=broadcast_mode) + tx_stub = cosmos_pb_grpc.ServiceStub(channel) + req = cosmos_pb.BroadcastTxRequest(tx_bytes=encoded_tx, mode=broadcast_mode) resp = tx_stub.BroadcastTx(req) return resp @@ -362,7 +376,7 @@ def peggy2_build_ebrelayer_cmd(self, init_what, network_descriptor, tendermint_n (["--keyring-dir", keyring_dir] if keyring_dir else []) + \ (["--symbol-translator-file", symbol_translator_file] if symbol_translator_file else []) + \ (["--log_format", log_format] if log_format else []) - return buildcmd(args, env=env, cwd=cwd) + return command.buildcmd(args, env=env, cwd=cwd) # Legacy stuff - pre-peggy2 # Called from IntegrationContext diff --git a/test/integration/framework/src/siftool/test_utils.py b/test/integration/framework/src/siftool/test_utils.py index 71f7b8d331..5a7bbf62f1 100644 --- a/test/integration/framework/src/siftool/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -2,16 +2,15 @@ import os import random import time +from typing import Iterable, Mapping, Union, List import web3 +from web3.eth import Contract +from hexbytes import HexBytes +from web3.types import TxReceipt -import siftool.eth as eth -import siftool.truffle as truffle -import siftool.hardhat as hardhat -import siftool.run_env as run_env -import siftool.sifchain as sifchain +from siftool import eth, truffle, hardhat, run_env, sifchain, cosmos from siftool.common import * - # These are utilities to interact with running environment (running agains local ganache-cli/hardhat/sifnoded). # This is to replace test_utilities.py, conftest.py, burn_lock_functions.py and integration_test_context.py. # Also to replace smart-contracts/scripts/... @@ -92,7 +91,7 @@ def get_env_ctx_peggy2(): sifnode_chain_id = "localnet" # TODO Mandatory, but not present either in environment_vars or dot_env_vars assert dot_env_vars["CHAINDIR"] == dot_env_vars["HOME"] sifnoded_home = os.path.join(dot_env_vars["CHAINDIR"], ".sifnoded") - ethereum_network_descriptor = dot_env_vars["ETH_CHAIN_ID"] + ethereum_network_descriptor = int(dot_env_vars["ETH_CHAIN_ID"]) eth_node_is_local = True generic_erc20_contract = "BridgeToken" @@ -266,13 +265,13 @@ def sif_addr_to_evm_arg(sif_address): class EnvCtx: - def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, sifnode_url, sifnode_chain_id, + def __init__(self, cmd, w3_conn: web3.Web3, ctx_eth, abi_provider, operator, sifnoded_home, sifnode_url, sifnode_chain_id, rowan_source, ceth_symbol, generic_erc20_contract ): self.cmd = cmd self.w3_conn = w3_conn - self.eth = ctx_eth - self.abi_provider = abi_provider + self.eth: eth.EthereumTxWrapper = ctx_eth + self.abi_provider: hardhat.HardhatAbiProvider = abi_provider self.operator = operator self.sifnode = sifchain.Sifnoded(self.cmd, home=sifnoded_home) self.sifnode_url = sifnode_url @@ -285,6 +284,9 @@ def __init__(self, cmd, w3_conn, ctx_eth, abi_provider, operator, sifnoded_home, self.generic_erc20_contract = generic_erc20_contract self.available_test_eth_accounts = None + def get_current_block_number(self) -> int: + return self.eth.w3_conn.eth.block_number + def advance_block_w3(self, number): for _ in range(number): # See smart-contracts/node_modules/@openzeppelin/test-helpers/src/time.js:advanceBlockTo() @@ -309,11 +311,17 @@ def get_bridge_bank_sc(self): result = self.w3_conn.eth.contract(address=address, abi=abi) return result + def get_cosmos_bridge_sc(self) -> Contract: + abi, _, address = self.abi_provider.get_descriptor("CosmosBridge") + assert address, "No address for CosmosBridge" + result = self.w3_conn.eth.contract(address=address, abi=abi) + return result + def get_generic_erc20_sc(self, address): abi, _, _ = self.abi_provider.get_descriptor(self.generic_erc20_contract) return self.w3_conn.eth.contract(abi=abi, address=address) - def get_erc20_token_balance(self, token_addr, eth_addr): + def get_erc20_token_balance(self, token_addr, eth_addr) -> int: token_sc = self.get_generic_erc20_sc(token_addr) return token_sc.functions.balanceOf(eth_addr).call() @@ -346,12 +354,14 @@ def smart_contract_get_past_events(self, sc, event_name, from_block=None, to_blo finally: self.w3_conn.eth.uninstall_filter(filter.filter_id) - def tx_deploy_new_generic_erc20_token(self, deployer_addr, name, symbol, decimals): + def tx_deploy_new_generic_erc20_token(self, deployer_addr: str, name: str, symbol: str, decimals: int, cosmosDenom: str = None) -> Contract: # return self.tx_deploy("SifchainTestToken", self.operator, [name, symbol, decimals]) if on_peggy2_branch: # Use BridgeToken assert self.generic_erc20_contract == "BridgeToken" - cosmosDenom = "erc20denom" # TODO Dummy variable since we're using BridgeToken instead of SifchainTestToken + if cosmosDenom is None: + cosmosDenom = "erc20denom" # TODO Dummy variable since we're using BridgeToken instead of SifchainTestToken + constructor_args = [name, symbol, decimals, cosmosDenom] else: # Use SifchainTestToken for TestNet and Devnet, and BridgeToken for Betanet @@ -368,6 +378,16 @@ def tx_update_bridge_bank_whitelist(self, token_addr, value=True): bridge_bank = self.get_bridge_bank_sc() return self.eth.transact(bridge_bank.functions.updateEthWhiteList, self.operator)(token_addr, value) + def tx_grant_minter_role(self, token_sc: Contract, minter_addr: str): + self.get_erc20_token_minter_role(token_sc, minter_addr) + minter_role_hash = token_sc.functions.MINTER_ROLE().call() + self.eth.transact(token_sc.functions.grantRole, self.operator)(minter_role_hash, minter_addr) + assert self.get_erc20_token_minter_role(token_sc, minter_addr) is True + + def get_erc20_token_minter_role(self, token_sc: Contract, minter_addr: str) -> bool: + minter_role_hash = token_sc.functions.MINTER_ROLE().call() + return token_sc.functions.hasRole(minter_role_hash, minter_addr).call() + def tx_approve(self, token_sc, from_addr, to_addr, amount): return self.eth.transact(token_sc.functions.approve, from_addr)(to_addr, amount) @@ -387,6 +407,18 @@ def tx_bridge_bank_lock_erc20(self, token_addr, from_eth_acct, to_sif_acct, amou tx_opts = {"value": 0} return self.eth.transact(bridge_bank.functions.lock, from_eth_acct, tx_opts=tx_opts)(recipient, token_addr, amount) + def tx_bridge_bank_burn_erc20(self, token_addr: str, from_eth_acct: str, to_sif_acct: str, amount: int) -> HexBytes: + recipient = sif_addr_to_evm_arg(to_sif_acct) + bridge_bank = self.get_bridge_bank_sc() + # When transfering ERC20, the amount needs to be passed as argument, and the "message.value" should be 0 + tx_opts = {"value": 0} + return self.eth.transact(bridge_bank.functions.burn, from_eth_acct, tx_opts=tx_opts)(recipient, token_addr, amount) + + def tx_bridge_bank_add_existing_bridge_token(self, token_addr: str) -> HexBytes: + bridge_bank = self.get_bridge_bank_sc() + tx_opts = {"value": 0} + return self.eth.transact(bridge_bank.functions.addExistingBridgeToken, self.operator, tx_opts=tx_opts)(token_addr) + def tx_approve_and_lock(self, token_sc, from_eth_acct, to_sif_acct, amount): bridge_bank_sc = self.get_bridge_bank_sc() txhash1 = self.tx_approve(token_sc, self.operator, bridge_bank_sc.address, amount) @@ -398,9 +430,9 @@ def tx_approve_and_lock(self, token_sc, from_eth_acct, to_sif_acct, amount): # # Used from test_integration_framework.py, test_eth_transfers.py - def deploy_new_generic_erc20_token(self, name, symbol, decimals, owner=None, mint_amount=None, mint_recipient=None): + def deploy_new_generic_erc20_token(self, name: str, symbol: str, decimals: int, owner: str = None, mint_amount: int = None, mint_recipient: str = None, cosmosDenom: str = None) -> Contract: owner = self.operator if owner is None else owner - txhash = self.tx_deploy_new_generic_erc20_token(owner, name, symbol, decimals) + txhash = self.tx_deploy_new_generic_erc20_token(owner, name, symbol, decimals, cosmosDenom) txrcpt = self.eth.wait_for_transaction_receipt(txhash) token_addr = txrcpt.contractAddress token_sc = self.get_generic_erc20_sc(token_addr) @@ -525,7 +557,7 @@ def send_erc20_from_ethereum_to_sifchain(self, from_eth_addr, dest_sichain_addr, self.approve_erc20_token(token_sc, from_eth_addr, amount) self.bridge_bank_lock_eth(from_eth_addr, dest_sichain_addr, amount) - def create_sifchain_addr(self, moniker=None, fund_amounts=None): + def create_sifchain_addr(self, moniker: str = None, fund_amounts: Union[cosmos.Balance, cosmos.LegacyBalance] = None): """ Generates a new sifchain address in test keyring. If moniker is given, uses it, otherwise generates a random one 'test-xxx'. If fund_amounts is given, the sifchain funds are transferred @@ -535,18 +567,22 @@ def create_sifchain_addr(self, moniker=None, fund_amounts=None): acct = self.sifnode.keys_add_1(moniker) sif_address = acct["address"] if fund_amounts: + fund_amounts = cosmos.balance_normalize(fund_amounts) # Convert from old format if neccessary rowan_source_balances = self.get_sifchain_balance(self.rowan_source) - for required_amount, denom in fund_amounts: + for denom, required_amount in fund_amounts.items(): available_amount = rowan_source_balances.get(denom, 0) assert available_amount >= required_amount, "Rowan source {} would need {}, but only has {}".format( self.rowan_source, sif_format_amount(required_amount, denom), sif_format_amount(available_amount, denom)) old_balances = self.get_sifchain_balance(sif_address) self.send_from_sifchain_to_sifchain(self.rowan_source, sif_address, fund_amounts) self.wait_for_sif_balance_change(sif_address, old_balances, min_changes=fund_amounts) + new_balances = self.get_sifchain_balance(sif_address) + assert cosmos.balance_zero(cosmos.balance_sub(new_balances, fund_amounts)) return sif_address def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): - amounts_string = ",".join([sif_format_amount(*a) for a in amounts]) + amounts = cosmos.balance_normalize(amounts) + amounts_string = cosmos.balance_format(amounts) args = ["tx", "bank", "send", from_sif_addr, to_sif_addr, amounts_string] + \ self._sifnoded_chain_id_and_node_arg() + \ self._sifnoded_fees_arg() + \ @@ -559,24 +595,39 @@ def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): raise Exception(raw_log) return retval - def get_sifchain_balance(self, sif_addr): + def get_sifchain_balance(self, sif_addr: cosmos.Address) -> cosmos.Balance: args = ["query", "bank", "balances", sif_addr, "--limit", str(100000000), "--output", "json"] + \ self._sifnoded_chain_id_and_node_arg() res = self.sifnode.sifnoded_exec(args, sifnoded_home=self.sifnode.home) res = json.loads(stdout(res))["balances"] - return dict(((x["denom"], int(x["amount"])) for x in res)) - - def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, polling_time=1, timeout=90, change_timeout=None): + return {denom: amount for denom, amount in ((x["denom"], int(x["amount"])) for x in res) if amount != 0} + + # Unless timed out, this function will exit: + # - if min_changes are given: when changes are greater. + # - if expected_balance is given: when balances are equal to that. + # - if neither min_changes nor expected_balance are given: when anything changes. + # You cannot use min_changes and expected_balance at the same time. + def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balances: cosmos.Balance, + min_changes: cosmos.CompatBalance = None, expected_balance: cosmos.CompatBalance = None, polling_time=1, + timeout=90, change_timeout=None + ) -> cosmos.Balance: + assert (min_changes is None) or (expected_balance is None), "Cannot use both min_changes and expected_balance" + min_changes = None if min_changes is None else cosmos.balance_normalize(min_changes) + expected_balance = None if expected_balance is None else cosmos.balance_normalize(expected_balance) start_time = time.time() last_change_time = None last_change_state = None while True: new_balances = self.get_sifchain_balance(sif_addr) - delta = sifchain.balance_delta(old_balances, new_balances) - if min_changes is not None: - if all([delta.get(denom, 0) >= amount for amount, denom in min_changes]): - return new_balances - elif not sifchain.balance_zero(delta): + delta = cosmos.balance_sub(new_balances, old_balances) + should_return = False + if expected_balance is not None: + should_return |= cosmos.balance_equal(new_balances) + elif min_changes is not None: + should_return |= cosmos.balance_exceeds(delta, min_changes) + else: + should_return |= not cosmos.balance_zero(delta) + if should_return: return new_balances now = time.time() if (timeout is not None) and (now - start_time > timeout): @@ -585,8 +636,8 @@ def wait_for_sif_balance_change(self, sif_addr, old_balances, min_changes=None, last_change_state = new_balances last_change_time = now else: - delta = sifchain.balance_delta(last_change_state, new_balances) - if not sifchain.balance_zero(delta): + delta = cosmos.balance_sub(new_balances, last_change_state) + if not cosmos.balance_zero(delta): last_change_state = new_balances last_change_time = now log.debug("New state detected: {}".format(delta)) @@ -684,7 +735,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # self.scavenge_ether() pass - def wait_for_eth_balance_change(self, eth_addr, old_balance, timeout=90, polling_time=1, token_addr=None): + def wait_for_eth_balance_change(self, eth_addr, old_balance: int, timeout=90, polling_time=1, token_addr=None): start_time = time.time() while True: new_balance = self.get_erc20_token_balance(token_addr, eth_addr) if token_addr \ @@ -696,6 +747,22 @@ def wait_for_eth_balance_change(self, eth_addr, old_balance, timeout=90, polling if now - start_time > timeout: raise Exception("Timeout waiting for Ethereum balance to change") + def wait_for_new_bridge_token_created(self, cosmos_denom: str, timeout: int = 90, polling_time: int = 1) -> str: + start_time = time.time() + while True: + cosmos_bridge_sc = self.get_cosmos_bridge_sc() + events = self.smart_contract_get_past_events(cosmos_bridge_sc, "LogNewBridgeTokenCreated") + + if len(events) > 0: + for e in events: + if e.args["cosmosDenom"] == cosmos_denom: + return e.args["bridgeTokenAddress"] + + time.sleep(polling_time) + now = time.time() + if now - start_time > timeout: + raise Exception("Timeout waiting for Ethereum balance to change") + def create_and_fund_eth_account(self, fund_from=None, fund_amount=None): if self.available_test_eth_accounts is not None: address = self.available_test_eth_accounts.pop(0) @@ -707,7 +774,8 @@ def create_and_fund_eth_account(self, fund_from=None, fund_amount=None): if fund_amount is not None: fund_from = fund_from or self.operator funder_balance_before = self.eth.get_eth_balance(fund_from) - assert funder_balance_before >= fund_amount + assert funder_balance_before >= fund_amount, "Cannot fund created account with ETH: need {}, have {}" \ + .format(fund_amount, funder_balance_before) target_balance_before = self.eth.get_eth_balance(address) difference = fund_amount - target_balance_before if difference > 0: @@ -725,18 +793,39 @@ def bridge_bank_lock_erc20(self, token_sc, from_eth_acct, to_sif_acct, amount): txhash = self.tx_bridge_bank_lock_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) return self.eth.wait_for_transaction_receipt(txhash) + def bridge_bank_burn_erc20(self, token_sc: Contract, from_eth_acct: str, to_sif_acct: str, amount: int) -> TxReceipt: + txhash = self.tx_bridge_bank_burn_erc20(token_sc.address, from_eth_acct, to_sif_acct, amount) + return self.eth.wait_for_transaction_receipt(txhash) + + def bridge_bank_add_existing_bridge_token(self, token_addr: str): + txhash = self.tx_bridge_bank_add_existing_bridge_token(token_addr) + self.eth.wait_for_transaction_receipt(txhash) + final_value = self.get_cosmos_token_in_white_list(token_addr) + assert final_value is True + + def get_cosmos_token_in_white_list(self, token_addr: str) -> bool: + bridge_bank_sc = self.get_bridge_bank_sc() + return bridge_bank_sc.functions.getCosmosTokenInWhiteList(token_addr).call() + + def get_destination_contract_address(self, cosmos_denom: str) -> Contract: + cosmos_bridge_sc = self.get_cosmos_bridge_sc() + return cosmos_bridge_sc.functions.cosmosDenomToDestinationAddress(cosmos_denom).call() + # TODO At the moment this is only for Ethereum-native assets (ETH and ERC20 tokens) which always use "lock". # For Sifchain-native assets (rowan) we need to use "burn". # Compare: smart-contracts/scripts/test/{sendLockTx.js OR sendBurnTx.js} # sendBurnTx is called when sifchain_symbol == "rowan", sendLockTx otherwise - def send_from_ethereum_to_sifchain(self, from_eth_acct, to_sif_acct, amount, token_sc=None): + def send_from_ethereum_to_sifchain(self, from_eth_acct: str, to_sif_acct: str, amount: int, token_sc: Contract = None, isLock: bool = True) -> TxReceipt: if token_sc is None: # ETH transfer self.bridge_bank_lock_eth(from_eth_acct, to_sif_acct, amount) else: # ERC20 token transfer self.approve_erc20_token(token_sc, from_eth_acct, amount) - self.bridge_bank_lock_erc20(token_sc, from_eth_acct, to_sif_acct, amount) + if isLock: + self.bridge_bank_lock_erc20(token_sc, from_eth_acct, to_sif_acct, amount) + else: + self.bridge_bank_burn_erc20(token_sc, from_eth_acct, to_sif_acct, amount) # Peggy1-specific def set_ofac_blocklist_to(self, addrs): @@ -782,9 +871,9 @@ def sanity_check(self): class ERC20TokenData: def __init__(self, symbol, name, decimals): - self.symbol = symbol - self.name = name - self.decimals = decimals + self.symbol: string = symbol + self.name: string = name + self.decimals: int = decimals def recover_eth_from_test_accounts(): diff --git a/test/integration/src/py/siftool_path.py b/test/integration/src/py/siftool_path.py index 94c29bb0ef..2d0c2039f6 100644 --- a/test/integration/src/py/siftool_path.py +++ b/test/integration/src/py/siftool_path.py @@ -3,9 +3,14 @@ # Temporary workaround to include siftool project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), *([os.path.pardir] * 4))) -base_dir = os.path.join(project_root, "test", "integration", "framework", "src") +base_dir = os.path.join(project_root, "test", "integration", "framework") +src_dir = os.path.join(base_dir, "src") +build_generated_dir = os.path.join(base_dir, "build", "generated") +paths = [src_dir, build_generated_dir] enabled = False -for p in sys.path: - enabled = enabled or os.path.realpath(p) == os.path.realpath(base_dir) -if not enabled: - sys.path = sys.path + [base_dir] +paths_to_add = [] +for p in paths: + enabled = any([os.path.realpath(p) == os.path.realpath(s) for s in sys.path]) + if not enabled: + paths_to_add.append(p) +sys.path.extend(paths_to_add) From 22f7c8f57b1d95caee9d0ac935f5cc69f18e349c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 7 Apr 2022 19:57:00 +0200 Subject: [PATCH 57/70] Merge siftool changes from future/peggy2 --- test/integration/framework/src/siftool/eth.py | 2 +- test/integration/framework/src/siftool/test_utils.py | 7 +++---- test/integration/src/py/conftest.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 4621d0d525..0a706f0e5d 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -261,7 +261,7 @@ def wrapped_fn(*args, **kwargs): return txhash return wrapped_fn - def send_eth(self, from_addr, to_addr, amount): + def send_eth(self, from_addr: str, to_addr: str, amount: int): log.info(f"Sending {amount} wei from {from_addr} to {to_addr}...") tx = {"to": to_addr, "value": amount} txhash = self._send_raw_transaction(None, from_addr, tx) diff --git a/test/integration/framework/src/siftool/test_utils.py b/test/integration/framework/src/siftool/test_utils.py index 5a7bbf62f1..1b5d38c1d9 100644 --- a/test/integration/framework/src/siftool/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -620,13 +620,12 @@ def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balances: co while True: new_balances = self.get_sifchain_balance(sif_addr) delta = cosmos.balance_sub(new_balances, old_balances) - should_return = False if expected_balance is not None: - should_return |= cosmos.balance_equal(new_balances) + should_return = cosmos.balance_equal(new_balances) elif min_changes is not None: - should_return |= cosmos.balance_exceeds(delta, min_changes) + should_return = cosmos.balance_exceeds(delta, min_changes) else: - should_return |= not cosmos.balance_zero(delta) + should_return = not cosmos.balance_zero(delta) if should_return: return new_balances now = time.time() diff --git a/test/integration/src/py/conftest.py b/test/integration/src/py/conftest.py index c2104e086c..c1817e6155 100644 --- a/test/integration/src/py/conftest.py +++ b/test/integration/src/py/conftest.py @@ -381,8 +381,8 @@ def ctx(request): snapshot_name = request.node.get_closest_marker("snapshot_name") if snapshot_name is not None: snapshot_name = snapshot_name.args[0] + logging.debug("Context setup: snapshot_name={}".format(repr(snapshot_name))) from siftool import test_utils - logging.error("Context setup: snapshot_name={}".format(repr(snapshot_name))) with test_utils.get_test_env_ctx() as ctx: yield ctx logging.debug("Test context cleanup") From 31573ea4d299fd0c6e400230dad82227dff3f7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 7 Apr 2022 21:03:47 +0200 Subject: [PATCH 58/70] Add missing dependency --- test/integration/setup-linux-environment-user.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/setup-linux-environment-user.sh b/test/integration/setup-linux-environment-user.sh index 1ae9222d3c..c6c6f2d22c 100644 --- a/test/integration/setup-linux-environment-user.sh +++ b/test/integration/setup-linux-environment-user.sh @@ -27,4 +27,5 @@ echo '. ~/.bash_profile' >> ~/.bashrc . ~/.bash_profile curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.33.0 -python3 -m pip install -U pytest web3 +python3 -m pip install -U pytest web3 grpcio-tools + From 729cd5b496316f51b69b87da39a1bdc4c43ee2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 9 Apr 2022 19:03:29 +0200 Subject: [PATCH 59/70] Fix regression with localnet integration tests --- .../localnet/utils/tests/__snapshots__/getChains.test.mjs.snap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/localnet/utils/tests/__snapshots__/getChains.test.mjs.snap b/test/localnet/utils/tests/__snapshots__/getChains.test.mjs.snap index 78de67a25f..4afd282d7a 100644 --- a/test/localnet/utils/tests/__snapshots__/getChains.test.mjs.snap +++ b/test/localnet/utils/tests/__snapshots__/getChains.test.mjs.snap @@ -124,7 +124,6 @@ Object { }, "sentinel": Object { "binary": "sentinelhub", - "binaryUrl": "https://github.com/sentinel-official/sentinel/archive/refs/tags/v0.1.4.zip", "chainId": "sentinelhub-2", "denom": "udvpn", "devnet-1": Object { @@ -139,6 +138,8 @@ Object { "pprofPort": 13004, "prefix": "sent", "rpcPort": 11004, + "sourceRelativePath": "sentinel-0.1.4", + "sourceUrl": "https://github.com/sentinel-official/sentinel/archive/refs/tags/v0.1.4.zip", "testnet-1": Object { "channelId": 39, "counterpartyChannelId": 2, From 90928e24edcfe93b52d9808008c56df279bb332f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sat, 9 Apr 2022 20:07:45 +0200 Subject: [PATCH 60/70] Refactoring: typing hints, typos --- test/integration/framework/src/siftool/eth.py | 2 ++ .../framework/src/siftool/inflate_tokens.py | 4 ++-- .../framework/src/siftool/test_utils.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 0a706f0e5d..71684818be 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -1,6 +1,7 @@ import logging import time import web3 +from typing import NewType from siftool.common import * @@ -9,6 +10,7 @@ GWEI = 10**9 NULL_ADDRESS = "0x0000000000000000000000000000000000000000" MIN_TX_GAS = 21000 +Address = NewType("Address", str) log = logging.getLogger(__name__) diff --git a/test/integration/framework/src/siftool/inflate_tokens.py b/test/integration/framework/src/siftool/inflate_tokens.py index b27e7a2f56..0288d54c11 100644 --- a/test/integration/framework/src/siftool/inflate_tokens.py +++ b/test/integration/framework/src/siftool/inflate_tokens.py @@ -271,10 +271,10 @@ def transfer(self, requested_tokens, token_amount, target_sif_accounts, eth_amou self.transfer_from_eth_to_sifnode(eth_broker_account, sif_broker_account, tokens_to_transfer, total_token_amount, total_eth_amount_gwei) self.distribute_tokens_to_wallets(sif_broker_account, tokens_to_transfer, token_amount, target_sif_accounts, eth_amount_gwei) - def transfer_eth(self, from_eth_addr, amount_gewi, target_sif_accounts): + def transfer_eth(self, from_eth_addr, amount_gwei, target_sif_accounts): pending_txs = [] for sif_acct in target_sif_accounts: - txrcpt = self.ctx.eth.tx_bridge_bank_lock_eth(from_eth_addr, sif_acct, amount_gewi * eth.GWEI) + txrcpt = self.ctx.eth.tx_bridge_bank_lock_eth(from_eth_addr, sif_acct, amount_gwei * eth.GWEI) pending_txs.append(txrcpt) self.wait_for_all(pending_txs) diff --git a/test/integration/framework/src/siftool/test_utils.py b/test/integration/framework/src/siftool/test_utils.py index 1b5d38c1d9..f309ad3a32 100644 --- a/test/integration/framework/src/siftool/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -607,7 +607,7 @@ def get_sifchain_balance(self, sif_addr: cosmos.Address) -> cosmos.Balance: # - if expected_balance is given: when balances are equal to that. # - if neither min_changes nor expected_balance are given: when anything changes. # You cannot use min_changes and expected_balance at the same time. - def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balances: cosmos.Balance, + def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balance: cosmos.Balance, min_changes: cosmos.CompatBalance = None, expected_balance: cosmos.CompatBalance = None, polling_time=1, timeout=90, change_timeout=None ) -> cosmos.Balance: @@ -616,28 +616,28 @@ def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balances: co expected_balance = None if expected_balance is None else cosmos.balance_normalize(expected_balance) start_time = time.time() last_change_time = None - last_change_state = None + last_changed_balance = None while True: - new_balances = self.get_sifchain_balance(sif_addr) - delta = cosmos.balance_sub(new_balances, old_balances) + new_balance = self.get_sifchain_balance(sif_addr) + delta = cosmos.balance_sub(new_balance, old_balance) if expected_balance is not None: - should_return = cosmos.balance_equal(new_balances) + should_return = cosmos.balance_equal(expected_balance, new_balance) elif min_changes is not None: should_return = cosmos.balance_exceeds(delta, min_changes) else: should_return = not cosmos.balance_zero(delta) if should_return: - return new_balances + return new_balance now = time.time() if (timeout is not None) and (now - start_time > timeout): raise Exception("Timeout waiting for sif balance to change") if last_change_time is None: - last_change_state = new_balances + last_changed_balance = new_balance last_change_time = now else: - delta = cosmos.balance_sub(new_balances, last_change_state) + delta = cosmos.balance_sub(new_balance, last_changed_balance) if not cosmos.balance_zero(delta): - last_change_state = new_balances + last_changed_balance = new_balance last_change_time = now log.debug("New state detected: {}".format(delta)) if (change_timeout is not None) and (now - last_change_time > change_timeout): From 4002a0b3bf5c8f614042d61cbc71088ba4a67d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Sun, 10 Apr 2022 08:49:24 +0200 Subject: [PATCH 61/70] Refactoring --- test/integration/framework/src/siftool/eth.py | 13 ++++++++++++- .../integration/framework/src/siftool/test_utils.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 71684818be..20092e046f 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -1,7 +1,9 @@ import logging import time import web3 -from typing import NewType +from hexbytes import HexBytes +from web3.datastructures import AttributeDict +from typing import NewType, Sequence from siftool.common import * @@ -244,6 +246,15 @@ def _send_raw_transaction(self, smart_contract_call_obj, from_addr, tx_opts=None txhash = self.w3_conn.eth.send_raw_transaction(signed_tx.rawTransaction) return txhash + def wait_for_all_transaction_receipts(self, tx_hashes: Sequence[HexBytes], sleep_time: int = 5, + timeout: Union[int, None] = None + ) -> Sequence[AttributeDict]: + result = [] + for txhash in tx_hashes: + txrcpt = self.wait_for_transaction_receipt(txhash, sleep_time=sleep_time, timeout=timeout) + result.append(txrcpt) + return result + def wait_for_transaction_receipt(self, txhash, sleep_time=5, timeout=None): return self.w3_conn.eth.wait_for_transaction_receipt(txhash, timeout=timeout, poll_latency=sleep_time) diff --git a/test/integration/framework/src/siftool/test_utils.py b/test/integration/framework/src/siftool/test_utils.py index f309ad3a32..6808fb19dc 100644 --- a/test/integration/framework/src/siftool/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -608,8 +608,8 @@ def get_sifchain_balance(self, sif_addr: cosmos.Address) -> cosmos.Balance: # - if neither min_changes nor expected_balance are given: when anything changes. # You cannot use min_changes and expected_balance at the same time. def wait_for_sif_balance_change(self, sif_addr: cosmos.Address, old_balance: cosmos.Balance, - min_changes: cosmos.CompatBalance = None, expected_balance: cosmos.CompatBalance = None, polling_time=1, - timeout=90, change_timeout=None + min_changes: cosmos.CompatBalance = None, expected_balance: cosmos.CompatBalance = None, polling_time: int = 1, + timeout: int = 90, change_timeout: int = None ) -> cosmos.Balance: assert (min_changes is None) or (expected_balance is None), "Cannot use both min_changes and expected_balance" min_changes = None if min_changes is None else cosmos.balance_normalize(min_changes) From cdef6f5b243f919002bafdf458e02085879abdad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Mon, 11 Apr 2022 17:51:56 +0200 Subject: [PATCH 62/70] Refactoring --- test/integration/framework/src/siftool/cosmos.py | 11 ++++++----- test/integration/framework/src/siftool/eth.py | 3 ++- .../framework/src/siftool/sifchain.py | 16 +++++++++++++--- .../framework/src/siftool/test_utils.py | 6 ++++-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/test/integration/framework/src/siftool/cosmos.py b/test/integration/framework/src/siftool/cosmos.py index 157740c242..952b58d488 100644 --- a/test/integration/framework/src/siftool/cosmos.py +++ b/test/integration/framework/src/siftool/cosmos.py @@ -19,10 +19,11 @@ def balance_normalize(bal: CompatBalance = None) -> Balance: return {k: v for k, v in bal.items() if v != 0} -def balance_add(bal1: Balance, bal2: Balance) -> Balance: +def balance_add(*bal: Balance) -> Balance: result = {} - for denom in set(bal1.keys()).union(set(bal2.keys())): - val = bal1.get(denom, 0) + bal2.get(denom, 0) + all_denoms = set(flatten_list([[*b.keys()] for b in bal])) + for denom in all_denoms: + val = sum(b.get(denom, 0) for b in bal) if val != 0: result[denom] = val return result @@ -32,8 +33,8 @@ def balance_neg(bal: Balance) -> Balance: return {k: -v for k, v in bal.items()} -def balance_sub(bal1: Balance, bal2: Balance) -> Balance: - return balance_add(bal1, balance_neg(bal2)) +def balance_sub(bal1: Balance, *bal2: Balance) -> Balance: + return balance_add(bal1, *[balance_neg(b) for b in bal2]) def balance_zero(bal: Balance) -> bool: diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 20092e046f..2a03c07fb1 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -1,6 +1,7 @@ import logging import time import web3 +import eth_typing from hexbytes import HexBytes from web3.datastructures import AttributeDict from typing import NewType, Sequence @@ -12,7 +13,7 @@ GWEI = 10**9 NULL_ADDRESS = "0x0000000000000000000000000000000000000000" MIN_TX_GAS = 21000 -Address = NewType("Address", str) +Address = eth_typing.AnyAddress log = logging.getLogger(__name__) diff --git a/test/integration/framework/src/siftool/sifchain.py b/test/integration/framework/src/siftool/sifchain.py index 913415dd9d..7d9a0b0dc5 100644 --- a/test/integration/framework/src/siftool/sifchain.py +++ b/test/integration/framework/src/siftool/sifchain.py @@ -2,11 +2,13 @@ import json import time import grpc -from typing import Mapping, Any -from siftool import command, cosmos +import re +import web3 +from typing import Mapping, Any, Tuple +from siftool import command, cosmos, eth from siftool.common import * -def sifchain_denom_hash(network_descriptor: int, token_contract_address: str) -> str: +def sifchain_denom_hash(network_descriptor: int, token_contract_address: eth.Address) -> str: assert on_peggy2_branch assert token_contract_address.startswith("0x") assert type(network_descriptor) == int @@ -14,6 +16,14 @@ def sifchain_denom_hash(network_descriptor: int, token_contract_address: str) -> denom = f"sifBridge{network_descriptor:04d}{token_contract_address.lower()}" return denom +def sifchain_denom_hash_to_token_contract_address(token_hash: str) -> Tuple[int, eth.Address]: + m = re.match("^sifBridge(\\d{4})0x([0-9a-fA-F]{40})$", token_hash) + if not m: + raise Exception("Invalid sifchain denom '{}'".format(token_hash)) + network_descriptor = int(m[1]) + token_address = web3.Web3.toChecksumAddress(m[2]) + return network_descriptor, token_address + # Deprecated def balance_delta(balances1: cosmos.Balance, balances2: cosmos.Balance) -> cosmos.Balance: return cosmos.balance_sub(balances2, balances1) diff --git a/test/integration/framework/src/siftool/test_utils.py b/test/integration/framework/src/siftool/test_utils.py index 6808fb19dc..abd1f23bf0 100644 --- a/test/integration/framework/src/siftool/test_utils.py +++ b/test/integration/framework/src/siftool/test_utils.py @@ -321,7 +321,7 @@ def get_generic_erc20_sc(self, address): abi, _, _ = self.abi_provider.get_descriptor(self.generic_erc20_contract) return self.w3_conn.eth.contract(abi=abi, address=address) - def get_erc20_token_balance(self, token_addr, eth_addr) -> int: + def get_erc20_token_balance(self, token_addr: eth.Address, eth_addr: eth.Address) -> int: token_sc = self.get_generic_erc20_sc(token_addr) return token_sc.functions.balanceOf(eth_addr).call() @@ -580,7 +580,9 @@ def create_sifchain_addr(self, moniker: str = None, fund_amounts: Union[cosmos.B assert cosmos.balance_zero(cosmos.balance_sub(new_balances, fund_amounts)) return sif_address - def send_from_sifchain_to_sifchain(self, from_sif_addr, to_sif_addr, amounts): + def send_from_sifchain_to_sifchain(self, from_sif_addr: cosmos.Address, to_sif_addr: cosmos.Address, + amounts: cosmos.Balance + ): amounts = cosmos.balance_normalize(amounts) amounts_string = cosmos.balance_format(amounts) args = ["tx", "bank", "send", from_sif_addr, to_sif_addr, amounts_string] + \ From 7b48f6531794b631d4294ec5ce33ac8cab8e783e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Wed, 13 Apr 2022 17:16:33 +0200 Subject: [PATCH 63/70] Refactoring --- test/integration/framework/siftool | 10 ++++++++-- test/integration/framework/src/siftool/common.py | 3 ++- test/integration/framework/src/siftool/cosmos.py | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/siftool b/test/integration/framework/siftool index 3d0bc28ae1..6f57256653 100755 --- a/test/integration/framework/siftool +++ b/test/integration/framework/siftool @@ -13,9 +13,11 @@ import subprocess def execst(args, cwd=None): return subprocess.run(args, cwd=cwd, check=True, capture_output=False) + def get_basedir(): return os.path.abspath(os.path.join(os.path.normpath(os.path.dirname(__file__)))) + def init_venv(venv_dir, requirements_txt): execst(["python3", "-m", "venv", venv_dir]) venv_pip = os.path.join(venv_dir, "bin", "pip3") @@ -23,6 +25,7 @@ def init_venv(venv_dir, requirements_txt): execst([venv_pip, "install", "wheel"]) execst([venv_pip, "install", "-r", requirements_txt]) + def ensure_venv(venv_dir, requirements_txt, lock_file=None): def wrapped(): if not os.path.exists(venv_dir): @@ -48,18 +51,21 @@ def load_main_module(): ensure_venv(venv_dir, requirements_txt, lock_file=lock_file) venv_lib_dir = glob.glob(os.path.join(venv_dir, "lib", "python3.*"))[0] - sys.path = sys.path + [ + paths = [ os.path.join(base_dir, "src"), os.path.join(base_dir, "build/generated"), os.path.join(venv_lib_dir, "site-packages"), - os.path.join(project_root, "test", "integration"), # For running integration tests in-process ] + paths_to_add = [p for p in paths if not any(os.path.realpath(p) == os.path.realpath(s) for s in sys.path)] + sys.path.extend(paths_to_add) import siftool.main as siftool_main return siftool_main + def main(argv): main_module = load_main_module() main_module.main(argv) + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/test/integration/framework/src/siftool/common.py b/test/integration/framework/src/siftool/common.py index 6a0d992ffe..9b2fb76df7 100644 --- a/test/integration/framework/src/siftool/common.py +++ b/test/integration/framework/src/siftool/common.py @@ -82,7 +82,7 @@ def popen(args: Sequence[str], cwd: Optional[str] = None, env: Optional[Mapping[ ) -> subprocess.Popen: if env: env = dict_merge(os.environ, env) - logging.debug(f"popen(): args={repr(args)}, cwd={repr(cwd)}") + log.debug(f"popen(): args={repr(args)}, cwd={repr(cwd)}") return subprocess.Popen(args, cwd=cwd, env=env, stdin=stdin, stdout=stdout, stderr=stderr, text=text) def dict_merge(*dicts, override=True): @@ -108,6 +108,7 @@ def basic_logging_setup(): logging.getLogger("eth").setLevel(logging.WARNING) logging.getLogger("websockets").setLevel(logging.WARNING) logging.getLogger("web3").setLevel(logging.WARNING) + logging.getLogger("asyncio").setLevel(logging.WARNING) # Recursively transforms template strings containing "${VALUE}". Example: # >>> template_transform("You are ${what}!", {"what": "${how} late", "how": "very"}) diff --git a/test/integration/framework/src/siftool/cosmos.py b/test/integration/framework/src/siftool/cosmos.py index 952b58d488..9790205c5c 100644 --- a/test/integration/framework/src/siftool/cosmos.py +++ b/test/integration/framework/src/siftool/cosmos.py @@ -29,6 +29,15 @@ def balance_add(*bal: Balance) -> Balance: return result +def balance_mul(bal: Balance, multiplier: Union[int, float]) -> Balance: + result = {} + for denom, value in bal.items(): + val = value * multiplier + if val != 0: + result[denom] = val + return result + + def balance_neg(bal: Balance) -> Balance: return {k: -v for k, v in bal.items()} From 0c0afd01ad437923b8615de2a602c47d25a6cf07 Mon Sep 17 00:00:00 2001 From: James Moore Date: Mon, 11 Apr 2022 16:54:27 -0700 Subject: [PATCH 64/70] VSCode needs --keyring-dir set to run ebrelayer --- test/integration/framework/src/siftool/run_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/framework/src/siftool/run_env.py b/test/integration/framework/src/siftool/run_env.py index ad0fc329f3..cc1d1ffc83 100644 --- a/test/integration/framework/src/siftool/run_env.py +++ b/test/integration/framework/src/siftool/run_env.py @@ -1273,7 +1273,7 @@ def format_sif_account(sif_account): "--keyring-backend", "test", "--from", relayer["address"], "--symbol-translator-file", "${workspaceFolder}/test/integration/config/symbol_translator.json", - "--home", relayer["home"] + "--keyring-dir", relayer["home"], ] } for i, relayer in enumerate(sifnode_relayers)], *[{ "name": f"Debug Witness-{i}", From 185595cd1ef6f9896cb3f7dd8bb5f467fcc4b97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvikart?= Date: Thu, 21 Apr 2022 11:51:35 +0200 Subject: [PATCH 65/70] Use more specific type hints --- test/integration/framework/src/siftool/eth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/framework/src/siftool/eth.py b/test/integration/framework/src/siftool/eth.py index 2a03c07fb1..70a778d124 100644 --- a/test/integration/framework/src/siftool/eth.py +++ b/test/integration/framework/src/siftool/eth.py @@ -3,7 +3,7 @@ import web3 import eth_typing from hexbytes import HexBytes -from web3.datastructures import AttributeDict +from web3.types import TxReceipt from typing import NewType, Sequence from siftool.common import * @@ -249,14 +249,14 @@ def _send_raw_transaction(self, smart_contract_call_obj, from_addr, tx_opts=None def wait_for_all_transaction_receipts(self, tx_hashes: Sequence[HexBytes], sleep_time: int = 5, timeout: Union[int, None] = None - ) -> Sequence[AttributeDict]: + ) -> Sequence[TxReceipt]: result = [] for txhash in tx_hashes: txrcpt = self.wait_for_transaction_receipt(txhash, sleep_time=sleep_time, timeout=timeout) result.append(txrcpt) return result - def wait_for_transaction_receipt(self, txhash, sleep_time=5, timeout=None): + def wait_for_transaction_receipt(self, txhash, sleep_time=5, timeout=None) -> TxReceipt: return self.w3_conn.eth.wait_for_transaction_receipt(txhash, timeout=timeout, poll_latency=sleep_time) def transact_sync(self, smart_contract_function, eth_addr, tx_opts=None, timeout=None): From 7ed4e4dc87eff2e50e106e5ce819c41adfcaa401 Mon Sep 17 00:00:00 2001 From: Tim Lind Date: Fri, 22 Apr 2022 13:11:00 +0200 Subject: [PATCH 66/70] Fix for amino/ledger support for unbonding liquidity --- x/clp/types/msgs.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x/clp/types/msgs.go b/x/clp/types/msgs.go index 94590250a7..096f3cb24d 100644 --- a/x/clp/types/msgs.go +++ b/x/clp/types/msgs.go @@ -6,6 +6,7 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/x/auth/legacy/legacytx" ) var ( @@ -21,6 +22,19 @@ var ( _ sdk.Msg = &MsgModifyPmtpRates{} _ sdk.Msg = &MsgUpdatePmtpParams{} _ sdk.Msg = &MsgUpdateStakingRewardParams{} + + _ legacytx.LegacyMsg = &MsgRemoveLiquidity{} + _ legacytx.LegacyMsg = &MsgRemoveLiquidityUnits{} + _ legacytx.LegacyMsg = &MsgCreatePool{} + _ legacytx.LegacyMsg = &MsgAddLiquidity{} + _ legacytx.LegacyMsg = &MsgSwap{} + _ legacytx.LegacyMsg = &MsgDecommissionPool{} + _ legacytx.LegacyMsg = &MsgUnlockLiquidityRequest{} + _ legacytx.LegacyMsg = &MsgUpdateRewardsParamsRequest{} + _ legacytx.LegacyMsg = &MsgAddRewardPeriodRequest{} + _ legacytx.LegacyMsg = &MsgModifyPmtpRates{} + _ legacytx.LegacyMsg = &MsgUpdatePmtpParams{} + _ legacytx.LegacyMsg = &MsgUpdateStakingRewardParams{} ) func (m MsgUpdateStakingRewardParams) Route() string { @@ -449,3 +463,15 @@ func (m MsgUnlockLiquidityRequest) GetSigners() []sdk.AccAddress { } return []sdk.AccAddress{addr} } + +func (m MsgUnlockLiquidityRequest) Route() string { + return RouterKey +} + +func (m MsgUnlockLiquidityRequest) Type() string { + return "unlock_liquidity" +} + +func (m MsgUnlockLiquidityRequest) GetSignBytes() []byte { + return sdk.MustSortJSON(ModuleCdc.MustMarshalJSON(&m)) +} From 0e8db43ee0da94f3354d114ffe74601d7375e495 Mon Sep 17 00:00:00 2001 From: austinobombino Date: Mon, 25 Apr 2022 11:47:39 -0700 Subject: [PATCH 67/70] fix tests --- x/clp/keeper/calculations_test.go | 3 ++- x/clp/keeper/msg_server_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x/clp/keeper/calculations_test.go b/x/clp/keeper/calculations_test.go index 1bbe668852..e86cdec038 100644 --- a/x/clp/keeper/calculations_test.go +++ b/x/clp/keeper/calculations_test.go @@ -1074,7 +1074,7 @@ func TestKeeper_CalculatePoolUnits(t *testing.T) { lpunits: sdk.ZeroUint(), }, { - name: "successful", + name: "fail asymmetric", oldPoolUnits: sdk.ZeroUint(), nativeAssetBalance: sdk.NewUint(10000), externalAssetBalance: sdk.NewUint(100), @@ -1084,6 +1084,7 @@ func TestKeeper_CalculatePoolUnits(t *testing.T) { adjustExternalToken: false, poolUnits: sdk.ZeroUint(), lpunits: sdk.ZeroUint(), + errString: errors.New("Cannot add liquidity asymmetrically"), }, } diff --git a/x/clp/keeper/msg_server_test.go b/x/clp/keeper/msg_server_test.go index 09c9b6bf45..54dbdca6da 100644 --- a/x/clp/keeper/msg_server_test.go +++ b/x/clp/keeper/msg_server_test.go @@ -543,7 +543,7 @@ func TestMsgServer_RemoveLiquidity(t *testing.T) { Signer: "sif1syavy2npfyt9tcncdtsdzf7kny9lh777yqc2nd", ExternalAsset: &types.Asset{Symbol: "eth"}, WBasisPoints: sdk.NewInt(1), - Asymmetry: sdk.NewInt(1), + Asymmetry: sdk.NewInt(0), }, }, { @@ -566,6 +566,7 @@ func TestMsgServer_RemoveLiquidity(t *testing.T) { WBasisPoints: sdk.NewInt(1), Asymmetry: sdk.NewInt(1), }, + errString: errors.New("Cannot remove liquidity asymmetrically"), }, } From 0466d90d5a62cd03f26a8064a14a086683a36aeb Mon Sep 17 00:00:00 2001 From: Caner Candan Date: Mon, 25 Apr 2022 23:05:52 +0200 Subject: [PATCH 68/70] =?UTF-8?q?test:=20=F0=9F=92=8D=20fix=20broken=20tes?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- x/clp/handler_test.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/x/clp/handler_test.go b/x/clp/handler_test.go index 43a84b90b5..24c6850943 100644 --- a/x/clp/handler_test.go +++ b/x/clp/handler_test.go @@ -109,8 +109,8 @@ func TestAddLiquidity(t *testing.T) { require.NotNil(t, res) msg = clptypes.NewMsgAddLiquidity(signer, asset, sdk.ZeroUint(), addLiquidityAmount) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res) + require.EqualError(t, err, "Cannot add liquidity asymmetrically") + require.Nil(t, res) // Subtracted twice , during create and add externalCoin = sdk.NewCoin(asset.Symbol, sdk.Int(initialBalance.Sub(addLiquidityAmount).Sub(addLiquidityAmount))) nativeCoin = sdk.NewCoin(clptypes.NativeSymbol, sdk.Int(initialBalance.Sub(addLiquidityAmount).Sub(sdk.ZeroUint()))) @@ -155,8 +155,8 @@ func TestAddLiquidity_LargeValue(t *testing.T) { require.NotNil(t, res) msg := clptypes.NewMsgAddLiquidity(signer, asset, addLiquidityAmountRowan, addLiquidityAmountCaCoin) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res) + require.EqualError(t, err, "Cannot add liquidity asymmetrically") + require.Nil(t, res) } func TestRemoveLiquidity(t *testing.T) { @@ -194,8 +194,8 @@ func TestRemoveLiquidity(t *testing.T) { coins := CalculateWithdraw(t, clpKeeper, ctx, asset, signer.String(), wBasis.String(), asymmetry) msg = clptypes.NewMsgRemoveLiquidity(signer, asset, wBasis, asymmetry) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res) + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res) for _, coin := range coins { ok := clpKeeper.HasBalance(ctx, signer, coin) assert.True(t, ok, "") @@ -205,8 +205,8 @@ func TestRemoveLiquidity(t *testing.T) { coins = CalculateWithdraw(t, clpKeeper, ctx, asset, signer.String(), wBasis.String(), asymmetry) msg = clptypes.NewMsgRemoveLiquidity(signer, asset, wBasis, asymmetry) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res) + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res) for _, coin := range coins { ok := clpKeeper.HasBalance(ctx, signer, coin) assert.True(t, ok, "") @@ -227,8 +227,8 @@ func TestRemoveLiquidity(t *testing.T) { coins = CalculateWithdraw(t, clpKeeper, ctx, asset, signer.String(), wBasis.String(), asymmetry) msg = clptypes.NewMsgRemoveLiquidity(signer, asset, wBasis, asymmetry) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res) + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res) for _, coin := range coins { ok := clpKeeper.HasBalance(ctx, signer, coin) assert.True(t, ok, "") @@ -256,8 +256,8 @@ func TestRemoveLiquidity(t *testing.T) { msg = clptypes.NewMsgRemoveLiquidity(newLP, asset, wBasis, asymmetry) res, err = handler(ctx, &msg) - require.NoError(t, err) - require.NotNil(t, res, "Can withdraw now as new LP has added liquidity") + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res, "Cannot withdraw now as new LP hasnt added liquidity") } func TestSwap(t *testing.T) { @@ -352,6 +352,11 @@ func TestDecommisionPool(t *testing.T) { msgrm := clptypes.NewMsgRemoveLiquidity(signer, asset, sdk.NewInt(5001), sdk.NewInt(1)) res, err = handler(ctx, &msgrm) + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res) + + msgrm = clptypes.NewMsgRemoveLiquidity(signer, asset, sdk.NewInt(5001), sdk.NewInt(0)) + res, err = handler(ctx, &msgrm) require.NoError(t, err) require.NotNil(t, res) @@ -468,8 +473,14 @@ func TestUnlockLiquidity(t *testing.T) { msg = clptypes.NewMsgRemoveLiquidity(signer, asset, wBasis, asymmetry) res, err = handler(ctx, &msg) + require.EqualError(t, err, "Cannot remove liquidity asymmetrically") + require.Nil(t, res) + + msg = clptypes.NewMsgRemoveLiquidity(signer, asset, sdk.NewInt(5001), sdk.NewInt(0)) + res, err = handler(ctx, &msg) require.NoError(t, err) require.NotNil(t, res) + for _, coin := range coins { ok := clpKeeper.HasBalance(ctx, signer, coin) assert.True(t, ok, "") From 7be3b2a545551847e9e665dacab27f4b769ab6c0 Mon Sep 17 00:00:00 2001 From: austinobombino Date: Mon, 25 Apr 2022 14:26:41 -0700 Subject: [PATCH 69/70] version bump --- app/setup_handlers.go | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/setup_handlers.go b/app/setup_handlers.go index 9feb88cde6..a6d6fd61de 100644 --- a/app/setup_handlers.go +++ b/app/setup_handlers.go @@ -7,7 +7,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/upgrade/types" ) -const releaseVersion = "0.13.0" +const releaseVersion = "0.13.1-rc1" func SetupHandlers(app *SifchainApp) { app.UpgradeKeeper.SetUpgradeHandler(releaseVersion, func(ctx sdk.Context, plan types.Plan, vm m.VersionMap) (m.VersionMap, error) { diff --git a/version b/version index 54d1a4f2a4..eccb60dedf 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.13.0 +0.13.1-rc1 From 7503a593a0a752fcd8104d81ec4422992a3113b2 Mon Sep 17 00:00:00 2001 From: Caner Candan Date: Tue, 26 Apr 2022 01:01:48 +0200 Subject: [PATCH 70/70] =?UTF-8?q?chore:=20=F0=9F=A4=96=20update=20script?= =?UTF-8?q?=20to=20include=20past=20periods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/rewards/rewards-period.js | 33 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/scripts/rewards/rewards-period.js b/scripts/rewards/rewards-period.js index 531e0e54af..63deb33942 100755 --- a/scripts/rewards/rewards-period.js +++ b/scripts/rewards/rewards-period.js @@ -2,13 +2,13 @@ const fs = require("fs"); -async function createAssetRewardsFile() { - const csv = fs.readFileSync("./pools.csv", "utf-8"); +async function createAssetRewardsFile(periodId, startBlock, endBlock) { + const csv = fs.readFileSync(`./${periodId}.csv`, "utf-8"); const entries = JSON.parse(fs.readFileSync("./entries.json", "utf-8")).result .registry.entries; const lines = csv.split("\r\n").filter((line) => line.split(",")[1] !== ""); - let [, allocation] = lines[0].split('"'); + let [, allocation] = lines[1].split('"'); allocation = `${allocation.trim().split(",").join("")}${"0".repeat(18)}`; const multipliers = lines.slice(1).map((line) => { @@ -30,22 +30,25 @@ async function createAssetRewardsFile() { }; }); - const rewards = [ - { - reward_period_id: "RP_1", - reward_period_start_block: 1, - reward_period_end_block: 100, - reward_period_allocation: allocation, - reward_period_pool_multipliers: multipliers, - reward_period_default_multiplier: "0.0", - }, - ]; + const rewardPeriod = { + reward_period_id: periodId, + reward_period_start_block: startBlock, + reward_period_end_block: endBlock, + reward_period_allocation: allocation, + reward_period_pool_multipliers: multipliers, + reward_period_default_multiplier: "0.0", + }; - fs.writeFileSync("./rewards.json", JSON.stringify(rewards, null, 2)); + return rewardPeriod; } async function start() { - await createAssetRewardsFile(); + const rewardPeriods = [ + await createAssetRewardsFile("RP_2", 6586931, 6687730), + await createAssetRewardsFile("RP_1", 6486131, 6586930), + ]; + + fs.writeFileSync("./rewards.json", JSON.stringify(rewardPeriods, null, 2)); } start();