From 5fb86ba96389fd8f18b40b4631cfb42a80cb6e25 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:12:59 +0530 Subject: [PATCH] Upgrade dependencies and update referential integrity checks (#17) * Upgrade dependencies and update referential integrity queries * Update GitHub workflow to run tests --- .github/workflows/tests.yml | 55 +++++++++++--------------- .gitignore | 2 + docker-compose.yml | 2 +- go.mod | 4 +- go.sum | 8 ++-- pkg/validator/ref_integrity_queries.go | 2 + test/README.md | 21 +++++++--- 7 files changed, 50 insertions(+), 44 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d227ed..6b3ab58 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,9 +3,9 @@ on: workflow_call: env: - STACK_ORCHESTRATOR_REF: "b3cb26e93b7e387d96417c81f880a3b8699b67db" - GO_ETHEREUM_REF: "v1.10.19-statediff-4.0.2-alpha" # Use the tag, we are going to download the bin not build it. - IPLD_ETH_DB_REF: "4e948c58ce20c20ab633289f986d2ed2a1fe02ec" + STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702" + GO_ETHEREUM_REF: "v1.10.19-statediff-4.1.0-alpha" # Use the tag, we are going to download the bin not build it. + IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9" jobs: integrationtest: @@ -17,33 +17,41 @@ jobs: steps: - name: Create GOPATH run: mkdir -p /tmp/go + - uses: actions/setup-go@v3 with: go-version: ">=1.18.0" check-latest: true + - uses: actions/checkout@v2 with: path: "./ipld-eth-db-validator" + - uses: actions/checkout@v2 with: ref: ${{ env.STACK_ORCHESTRATOR_REF }} path: "./stack-orchestrator/" repository: vulcanize/stack-orchestrator + - uses: actions/checkout@v2 with: ref: ${{ env.IPLD_ETH_DB_REF }} repository: vulcanize/ipld-eth-db path: "./ipld-eth-db/" + - name: Create config file run: | echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh + echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh echo db_write=$DB_WRITE >> ./config.sh cat ./config.sh - - name: Download Geth geth + + - name: Download Geth run: | cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts wget https://github.com/vulcanize/go-ethereum/releases/download/${{env.GO_ETHEREUM_REF}}/geth-linux-amd64 + - name: Run docker compose run: | docker-compose \ @@ -52,6 +60,7 @@ jobs: -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \ --env-file "$GITHUB_WORKSPACE/config.sh" \ up -d --build + - name: Run integration test. run: | cd $GITHUB_WORKSPACE/ipld-eth-db-validator @@ -59,42 +68,24 @@ jobs: unittest: name: Run unit tests - env: - GOPATH: /tmp/go - DB_WRITE: true runs-on: ubuntu-latest steps: - name: Create GOPATH run: mkdir -p /tmp/go + - uses: actions/setup-go@v3 with: go-version: ">=1.18.0" check-latest: true - - uses: actions/checkout@v2 - with: - path: "./ipld-eth-db-validator" - - uses: actions/checkout@v2 - with: - ref: ${{ env.STACK_ORCHESTRATOR_REF }} - path: "./stack-orchestrator/" - repository: vulcanize/stack-orchestrator - - uses: actions/checkout@v2 - with: - ref: ${{ env.IPLD_ETH_DB_REF }} - repository: vulcanize/ipld-eth-db - path: "./ipld-eth-db/" - - name: Create config file - run: | - echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh - echo db_write=$DB_WRITE >> ./config.sh - cat ./config.sh - - name: Run docker compose + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Spin up database run: | - docker-compose \ - -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \ - --env-file "$GITHUB_WORKSPACE/config.sh" \ - up -d --build - - name: Run unit test. + docker-compose up -d + + - name: Run unit tests run: | - cd $GITHUB_WORKSPACE/ipld-eth-db-validator + sleep 30 PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a4a1a6b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +ipld-eth-db-validator +.vscode diff --git a/docker-compose.yml b/docker-compose.yml index 628519b..fbfa0db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: vulcanize/ipld-eth-db:v4.1.1-alpha + image: vulcanize/ipld-eth-db:v4.2.0-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "vulcanize_testing" diff --git a/go.mod b/go.mod index 93556d5..4940512 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.11.0 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha - github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha + github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha ) require ( @@ -275,4 +275,4 @@ require ( lukechampine.com/blake3 v1.1.6 // indirect ) -replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha +replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha diff --git a/go.sum b/go.sum index 0390efd..70632c7 100644 --- a/go.sum +++ b/go.sum @@ -1562,12 +1562,12 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha h1:sDDK4eOdW3JEds+0eP/OGgPr7T7SS4jtje5W4VjhCgM= github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha/go.mod h1:/pHfZd1IWsSTpCtGq6nnzUZBAkLV+zMrRh6Z3Hr3NFc= -github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha h1:xD4fA2khoAnhBEk84JwrIEGvQCndVXpQGv5n7a9cgwc= -github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8= +github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha h1:8ge2ban6t/e53XDwe6s28jcCevT7Ggo51lNJ0Eo1PgA= +github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8= github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha h1:xak1uYmFWqJ2Hz3pM+0jDcqdlwYwRWeSkQV6B8IxD/0= github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha/go.mod h1:pHbLbW4Hk1IFpxrY9yi50IuoPPzmSY7lwOqpFAa369k= -github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha h1:yNpTBjP71ZaUFz2Q8XFUbupvi1yGc4rtniBCTWLtmLk= -github.com/vulcanize/ipld-eth-server/v4 v4.0.5-alpha/go.mod h1:rYP+9ejznXwuN8eKeoFGdzi7wjzm2MOaWcI1XObPdBo= +github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha h1:UYPRvG9fKgUTLQOMgTMxxPPHgST5IpLRI4FT9goAHXY= +github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha/go.mod h1:HbXa8Zz4g0TIS8vEq6XdK9J1Wc2x4Oa1+YaHwzmLMCQ= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= diff --git a/pkg/validator/ref_integrity_queries.go b/pkg/validator/ref_integrity_queries.go index cbc5f20..dba648c 100644 --- a/pkg/validator/ref_integrity_queries.go +++ b/pkg/validator/ref_integrity_queries.go @@ -48,6 +48,7 @@ const ( FROM eth.receipt_cids LEFT JOIN eth.transaction_cids ON ( receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.header_id = transaction_cids.header_id AND receipt_cids.block_number = transaction_cids.block_number ) WHERE @@ -110,6 +111,7 @@ const ( FROM eth.log_cids LEFT JOIN eth.receipt_cids ON ( log_cids.rct_id = receipt_cids.tx_id + AND log_cids.header_id = receipt_cids.header_id AND log_cids.block_number = receipt_cids.block_number ) WHERE diff --git a/test/README.md b/test/README.md index d76d1a6..fdb6ffe 100644 --- a/test/README.md +++ b/test/README.md @@ -4,20 +4,27 @@ - For running integration tests: - - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories. + - Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator), [go-ethereum](https://github.com/vulcanize/go-ethereum) and [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db) repositories. - - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) in go-ethereum repo. + - Checkout [v4 release](https://github.com/vulcanize/ipld-eth-db/releases/tag/v4.2.0-alpha) in ipld-eth-db repo. + + ```bash + # In ipld-eth-db repo. + git checkout v4.2.0-alpha + ``` + + - Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.19-statediff-4.1.0-alpha) in go-ethereum repo. ```bash # In go-ethereum repo. - git checkout v1.10.18-statediff-4.0.2-alpha + git checkout v1.10.19-statediff-4.1.0-alpha ``` - Checkout working commit in stack-orchestrator repo. ```bash # In stack-orchestrator repo. - git checkout 382aca8e42bc5e33f301f77cdd2e09cc80602fc3 + git checkout f2fd766f5400fcb9eb47b50675d2e3b1f2753702 ``` ## Run @@ -48,12 +55,16 @@ ```bash #!/bin/bash + # Path to ipld-eth-server repo. + vulcanize_ipld_eth_db=~/ipld-eth-db/ + # Path to go-ethereum repo. vulcanize_go_ethereum=~/go-ethereum # Path to contract folder. vulcanize_test_contract=~/ipld-eth-db-validator/test/contract + genesis_file_path='start-up-files/go-ethereum/genesis.json' db_write=true ``` @@ -65,7 +76,7 @@ ./wrapper.sh \ -e docker \ - -d ../docker/latest/docker-compose-db-sharding.yml \ + -d ../docker/local/docker-compose-db-sharding.yml \ -d ../docker/local/docker-compose-go-ethereum.yml \ -d ../docker/local/docker-compose-contract.yml \ -v remove \