Merge branch 'main' into sb-stage-upgrade-info

matter-labs · Feb 3, 2025 · 50456ac · 50456ac
2 parents 83f6ee7 + ddc953d
commit 50456ac
Show file tree

Hide file tree

Showing 384 changed files with 11,487 additions and 8,446 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -20,11 +20,12 @@ keys/setup
 !prover/
 !yarn.lock
 !package.json
-!Cargo.lock
-!Cargo.toml
+!core/Cargo.lock
+!core/Cargo.toml
 !contracts/
 !setup_2\^26.key
 !setup_2\^24.key
+!setup_compact.key
 # It's required to remove .git from contracts,
 # otherwise yarn tries to use .git parent directory that
 # doesn't exist.

diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -15,8 +15,6 @@ check_fmt () {
     fi
 }
 
-check_fmt
-
-cd prover/
-
-check_fmt
+( cd core/ && check_fmt )
+( cd prover/ && check_fmt )
+( cd zkstack_cli/ && check_fmt )
diff --git a/.github/release-please/config.json b/.github/release-please/config.json
@@ -5,26 +5,27 @@
   "bump-minor-pre-major": true,
   "bump-patch-for-minor-pre-major": true,
   "include-component-in-tag": true,
+  "release-type": "simple",
   "packages": {
     "core": {
-      "release-type": "simple",
-      "component": "core",
+      "component": "core"
+    },
+    "prover": {
+      "component": "prover",
       "extra-files": [
         {
           "type": "generic",
-          "path": "bin/external_node/Cargo.toml"
+          "path": "Cargo.toml"
         }
       ]
     },
-    "prover": {
-      "release-type": "simple",
-      "component": "prover"
-    },
     "zkstack_cli": {
-      "release-type": "simple",
       "component": "zkstack_cli",
-      "plugins": [
-        "cargo-workspace"
+      "extra-files": [
+        {
+          "type": "generic",
+          "path": "Cargo.toml"
+        }
       ]
     }
   }

diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,5 +1,5 @@
 {
-  "core": "25.4.0",
-  "prover": "17.1.1",
+  "core": "26.2.1",
+  "prover": "18.0.0",
   "zkstack_cli": "0.1.2"
 }
diff --git a/...ri-gpu-gar-and-circuit-prover-gpu-gar.yml → ...orkflows/build-circuit-prover-gpu-gar.yml b/...ri-gpu-gar-and-circuit-prover-gpu-gar.yml → ...orkflows/build-circuit-prover-gpu-gar.yml
@@ -27,7 +27,7 @@ jobs:
 
       - name: Download Setup data
         run: |
-          gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
+          gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/circuit-prover-gpu-gar
 
       - name: Login to us-central1 GAR
         run: |
@@ -47,32 +47,6 @@ jobs:
         run: |
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
 
-      - name: Build and push prover-gpu-fri-gar
-        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
-        with:
-          context: docker/prover-gpu-fri-gar
-          build-args: |
-            PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
-          push: true
-          tags: |
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
-
-      - name: Build and push prover-gpu-fri-gar to Asia GAR
-        run: |
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
-
-      - name: Build and push prover-gpu-fri-gar to Europe GAR
-        run: |
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
-
-      - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar
-        run: |
-          mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
-
       - name: Build and push circuit-prover-gpu-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:

diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml
@@ -110,10 +110,10 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
+  build-circuit-prover-gpu-gar:
     name: Build GAR prover FRI GPU
     needs: [setup, build-push-prover-images]
-    uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
+    uses: ./.github/workflows/build-circuit-prover-gpu-gar.yml
     if: contains(github.ref_name, 'prover')
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}

diff --git a/.github/workflows/build-proof-fri-gpu-compressor-gar.yml b/.github/workflows/build-proof-fri-gpu-compressor-gar.yml
@@ -28,7 +28,7 @@ jobs:
       - name: Download FFLONK key and setup data
         run: |
           gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/proof-fri-gpu-compressor-gar
-          gsutil -m cp -r gs://matterlabs-setup-keys-us/setup-keys/setup_fflonk_compact.key docker/proof-fri-gpu-compressor-gar
+          gsutil -m cp -r gs://matterlabs-setup-keys-us/setup-keys/setup_compact.key docker/proof-fri-gpu-compressor-gar
 
       - name: Login to us-central1 GAR
         run: |
@@ -58,13 +58,13 @@ jobs:
           tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/proof-fri-gpu-compressor-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push prover-gpu-fri-gar to Asia GAR
+      - name: Build and push proof-fri-gpu-compressor-gar to Asia GAR
         run: |
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/proof-fri-gpu-compressor-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/proof-fri-gpu-compressor-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push prover-gpu-fri-gar to Europe GAR
+      - name: Build and push proof-fri-gpu-compressor-gar to Europe GAR
         run: |
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/proof-fri-gpu-compressor-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -98,8 +98,6 @@ jobs:
       matrix:
         components:
           - witness-generator
-          - prover-gpu-fri
-          - witness-vector-generator
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor
@@ -201,42 +199,3 @@ jobs:
           docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
           docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.components }}:latest
           docker push matterlabs/${{ matrix.components }}:latest
-
-  copy-images:
-    name: Copy images between docker registries
-    needs: [build-images, get-protocol-version]
-    env:
-      PROTOCOL_VERSION: ${{ needs.get-protocol-version.outputs.protocol_version }}
-    runs-on: matterlabs-ci-runner
-    if: ${{ inputs.action == 'push' }}
-    strategy:
-      matrix:
-        component:
-          - witness-vector-generator
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
-
-      - name: Login to us-central1 GAR
-        run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
-
-      - name: Login and push to Asia GAR
-        run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }}
-          docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
-
-      - name: Login and push to Europe GAR
-        run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ inputs.image_tag_suffix }}
-          docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
diff --git a/.github/workflows/cargo-license.yaml b/.github/workflows/cargo-license.yaml
@@ -7,5 +7,6 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - uses: EmbarkStudios/cargo-deny-action@8371184bd11e21dcf8ac82ebf8c9c9f74ebf7268 # v2.0.1
       with:
+        manifest-path: "./core/Cargo.toml"
         command: check
         command-arguments: "--hide-inclusion-graph"
diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -72,14 +72,14 @@ jobs:
           ci_run zkstack dev contracts
 
       - name: Download compilers for contract verifier tests
-        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
 
       - name: Rust unit tests
         run: |
           ci_run zkstack dev test rust
           # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
           # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
-          ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch
+          ci_run cargo test --manifest-path ./core/Cargo.toml --release -p vm-benchmark --bench oneshot --bench batch
 
   loadtest:
     runs-on: [ matterlabs-ci-runner-high-performance ]
@@ -431,7 +431,7 @@ jobs:
 
       - name: Initialize Contract verifier
         run: |
-          ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+          ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
           ci_run zkstack contract-verifier wait --chain era --verbose
 
@@ -456,12 +456,21 @@ jobs:
       - name: Set up attester committee for the consensus chain
         run: |
           ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
-          ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
+          ci_run zkstack consensus set-attester-committee --chain consensus --ignore-prerequisites --verbose --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
 
       - name: Run integration tests
         run: |
           ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}
 
+      - name: Repeat integration tests on push to main to check for flakiness
+        if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+        run: |
+          for i in {1..10}; do
+            echo "Iteration $i"
+            mkdir -p ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/$i
+            ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/$i
+          done
+
       - name: Init external nodes
         run: |
           GATEWAY_RPC_URL="${{ matrix.use_gateway_chain == 'WITH_GATEWAY' && '--gateway-rpc-url=http://localhost:3550' || '' }}"
@@ -506,6 +515,15 @@ jobs:
         run: |
           ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites --external-node" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}
 
+      - name: Repeat integration tests en on push to main to check for flakiness
+        if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+        run: |
+          for i in {1..10}; do
+            echo "Iteration $i"
+            mkdir -p ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/$i
+            ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites --external-node" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/$i
+          done
+
       - name: Fee projection tests
         run: |
           ci_run killall -INT zksync_server || true
@@ -519,6 +537,15 @@ jobs:
           # Always run the chain-specific fee tests
           ci_run ./bin/run_on_all_chains.sh "zkstack dev test fees --no-deps --no-kill" ${{ env.CHAINS }} ${{ env.FEES_LOGS_DIR }}
 
+      - name: Repeat fee projection tests on push to main to check for flakiness
+        if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+        run: |
+          for i in {1..10}; do
+            echo "Iteration $i"
+            mkdir -p ${{ env.FEES_LOGS_DIR }}/$i
+            ci_run ./bin/run_on_all_chains.sh "zkstack dev test fees --no-deps --no-kill" ${{ env.CHAINS }} ${{ env.FEES_LOGS_DIR }}/$i
+          done
+
       - name: Run revert tests
         run: |
           ci_run killall -INT zksync_server || true

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -57,8 +57,8 @@ jobs:
               - '.github/workflows/build-contract-verifier-template.yml'
               - '.github/workflows/ci-core-reusable.yml'
               - '.github/workflows/ci-core-lint-reusable.yml'
-              - 'Cargo.toml'
-              - 'Cargo.lock'
+              - './core/Cargo.toml'
+              - './core/Cargo.lock'
               - 'zkstack_cli/**'
               - '!**/*.md'
               - '!**/*.MD'

diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml
@@ -31,7 +31,7 @@ env:
 
 jobs:
   compatibility:
-    runs-on: [ubuntu-22.04-github-hosted-16core]
+    runs-on: [ubuntu-24.04-github-hosted-16core]
     steps:
       - uses: mozilla-actions/sccache-action@89e9040de88b577a072e3760aaf59f585da083af # v0.0.5
 
@@ -46,13 +46,15 @@ jobs:
         run:
           git checkout $(git merge-base $BASE $HEAD) --recurse-submodules
         working-directory: ./before
+
       - name: compile before
-        run: cargo check --all-targets
-        working-directory: ./before/
+        run: cargo check --manifest-path ./core/Cargo.toml --all-targets
+        working-directory: ./before
+
       - name: build before.binpb
         run: >
           perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/'
-          `find ./before/target/debug/build/*/output`
+          `find ./before/core/target/debug/build/*/output`
           | xargs cat > ./before.binpb
 
       # after
@@ -61,20 +63,24 @@ jobs:
           ref: ${{ env.HEAD }}
           path: after
           submodules: recursive
+
       - name: compile after
-        run: cargo check --all-targets
+        run: cargo check --manifest-path ./core/Cargo.toml --all-targets
         working-directory: ./after
+
       - name: build after.binpb
         run: >
           perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/'
-          `find ./after/target/debug/build/*/output`
+          `find ./after/core/target/debug/build/*/output`
           | xargs cat > ./after.binpb
 
       # compare
       - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0
         with:
           github_token: ${{ github.token }}
       - name: buf breaking
-        run: >
-          buf breaking './after.binpb' --against './before.binpb' --exclude-path 'zksync/config/experimental.proto'
-          --config '{"version":"v1","breaking":{"use":["WIRE_JSON","WIRE"]}}' --error-format 'github-actions'
+        run: |
+          pwd
+          ls -la
+          buf breaking './after.binpb' --against './before.binpb' --exclude-path 'zksync/config/experimental.proto' \
+            --config '{"version":"v1","breaking":{"use":["WIRE_JSON","WIRE"]}}' --error-format 'github-actions'