Merge branch 'main' into Odessa

.github/workflows/qa-clean-exit-block-downloading.yml

@@ -1,6 +1,9 @@
 name: QA - Clean exit (block downloading)
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 8 * * 1-6'  # Run every day at 08:00 AM UTC except Sunday
   workflow_dispatch:     # Run manually

.github/workflows/qa-rpc-integration-tests.yml

@@ -5,9 +5,11 @@ on:
   push:
     branches:
       - main
+      - 'release/3.*'
   pull_request:
     branches:
       - main
+      - 'release/3.*'
     types:
       - opened
       - reopened
@@ -52,13 +54,22 @@ jobs:
       - name: Run RpcDaemon
         working-directory: ${{ github.workspace }}/build/bin
         run: |
-          echo "RpcDaemon starting..."
+          echo "Starting RpcDaemon..."
           
-          ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws --verbosity 1 > erigon.log 2>&1 &
+          ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 &
 
           RPC_DAEMON_PID=$!          
+          RPC_DAEMON_EXIT_STATUS=$?
           echo "RPC_DAEMON_PID=$RPC_DAEMON_PID" >> $GITHUB_ENV
           
+          sleep 5
+          tail erigon.log
+          
+          if [ $RPC_DAEMON_EXIT_STATUS -ne 0 ]; then            
+            echo "RpcDaemon failed to start"
+            echo "::error::Error detected during tests: RpcDaemon failed to start"
+            exit 1
+          fi
           echo "RpcDaemon started"
 
       - name: Wait for port 8545 to be opened
@@ -73,6 +84,7 @@ jobs:
           done
           if ! nc -z localhost 8545; then
             echo "Port 8545 did not open in time"
+            echo "::error::Error detected during tests: Port 8545 did not open in time"
             exit 1
           fi
 
@@ -151,6 +163,6 @@ jobs:
       - name: Action for Failure
         if: steps.test_step.outputs.TEST_RESULT != 'success'
         run: |
-          echo "::error::Error detected during tests"
+          echo "::error::Error detected during tests: some tests failed, check the logs or the artifacts for more details"
           exit 1
 

.github/workflows/qa-rpc-performance-tests.yml

@@ -1,6 +1,9 @@
 name: QA - RPC Performance Tests
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   workflow_dispatch:
   schedule:
     - cron: '0 0 * * 0'  # Run on Sunday at 00:00 AM UTC

.github/workflows/qa-snap-download.yml

@@ -1,6 +1,9 @@
 name: QA - Snapshot Download
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 20 * * 1-6'  # Run every night at 20:00 (08:00 PM) UTC except Sunday
   workflow_dispatch:     # Run manually

.github/workflows/qa-sync-from-scratch-minimal-node.yml

@@ -1,6 +1,9 @@
 name: QA - Sync from scratch (minimal node)
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 0 * * *'  # Run every night at 00:00 AM UTC
   workflow_dispatch:     # Run manually
@@ -49,13 +52,6 @@ jobs:
           # Save the subsection reached status
           echo "::set-output name=test_executed::true"
           
-          # Clean up Erigon process if it's still running
-          if kill -0 $ERIGON_PID 2> /dev/null; then
-            echo "Terminating Erigon"
-            kill $ERIGON_PID
-            wait $ERIGON_PID
-          fi
-          
           # Check test runner script exit status
           if [ $test_exit_status -eq 0 ]; then
             echo "Tests completed successfully"

.github/workflows/qa-sync-from-scratch.yml

@@ -53,13 +53,6 @@ jobs:
         # Save the subsection reached status
         echo "::set-output name=test_executed::true"
         
-        # Clean up Erigon process if it's still running
-        if kill -0 $ERIGON_PID 2> /dev/null; then
-          echo "Terminating Erigon"
-          kill $ERIGON_PID
-          wait $ERIGON_PID
-        fi
-        
         # Check test runner script exit status
         if [ $test_exit_status -eq 0 ]; then
           echo "Tests completed successfully"

.github/workflows/qa-sync-with-externalcl.yml

@@ -0,0 +1,129 @@
+name: QA - Sync with external CL
+
+on:
+  schedule:
+    - cron: '0 8 * * 0'  # Run on Sunday at 08:00 AM UTC
+  workflow_dispatch:     # Run manually
+
+jobs:
+  sync-with-externalcl:
+    runs-on: [self-hosted, linux, X64]
+    timeout-minutes: 500 # 8+ hours
+    strategy:
+      matrix:
+        client: [lighthouse, prysm]
+        chain: [mainnet, gnosis]
+        exclude:
+          - client: prysm
+            chain: gnosis
+    env:
+      ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
+      CL_DATA_DIR: ${{ github.workspace }}/consensus
+      ERIGON_QA_PATH: /home/qarunner/erigon-qa
+      TRACKING_TIME_SECONDS: 3600 # 1 hour
+      TOTAL_TIME_SECONDS: 25200 # 7 hours
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Clean Erigon Build & Data Directories
+        run: |
+          make clean
+          rm -rf $ERIGON_DATA_DIR
+
+      - name: Install ${{ matrix.client }} and generate JWT secret
+        run: |
+          mkdir -p $CL_DATA_DIR
+          if [ "${{ matrix.client }}" == "lighthouse" ]; then
+            curl -LO https://github.com/sigp/lighthouse/releases/download/v7.0.0-beta.0/lighthouse-v7.0.0-beta.0-x86_64-unknown-linux-gnu.tar.gz
+            tar -xvf lighthouse-v7.0.0-beta.0-x86_64-unknown-linux-gnu.tar.gz -C $CL_DATA_DIR
+            rm lighthouse-v7.0.0-beta.0-x86_64-unknown-linux-gnu.tar.gz
+          elif [ "${{ matrix.client }}" == "prysm" ]; then
+            curl -L https://raw.githubusercontent.com/prysmaticlabs/prysm/master/prysm.sh -o $CL_DATA_DIR/prysm.sh
+            chmod +x $CL_DATA_DIR/prysm.sh
+          fi
+          openssl rand -hex 32 > $CL_DATA_DIR/jwt.hex
+
+      - name: Build Erigon
+        run: |
+          make erigon
+        working-directory: ${{ github.workspace }}
+
+      - name: Pause the Erigon instance dedicated to db maintenance
+        run: |
+          python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
+
+      - name: Run Erigon and monitor chain sync
+        id: test_step
+        run: |
+          set +e # Disable exit on error
+          
+          # Run Erigon, wait sync and check ability to maintain sync
+          python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
+            ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 ${{ matrix.chain }} minimal_node no_statistics ${{ matrix.client }} $CL_DATA_DIR
+          
+          # Capture monitoring script exit status
+          test_exit_status=$?
+          
+          # Save the subsection reached status
+          echo "::set-output name=test_executed::true"
+          
+          # Check test runner script exit status
+          if [ $test_exit_status -eq 0 ]; then
+            echo "Tests completed successfully"
+            echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
+          else
+            echo "Error detected during tests"
+            echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Save test results
+        if: steps.test_step.outputs.test_executed == 'true'
+        env:
+          TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
+        run: |
+          python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
+            --repo erigon \
+            --commit $(git rev-parse HEAD) \
+            --branch ${{ github.ref_name }} \
+            --test_name sync-from-scratch-${{ matrix.client }}-minimal-node \
+            --chain ${{ matrix.chain }} \
+            --runner ${{ runner.name }} \
+            --outcome $TEST_RESULT \
+            --result_file ${{ github.workspace }}/result-${{ matrix.chain }}.json
+
+      - name: Upload test results
+        if: steps.test_step.outputs.test_executed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.client }}-${{ matrix.chain }}
+          path: |
+            ${{ github.workspace }}/result-${{ matrix.chain }}.json
+            ${{ github.workspace }}/erigon_data/logs/erigon.log
+            ${{ matrix.client == 'lighthouse' && '$CL_DATA_DIR/data/beacon/logs/beacon.log' || '' }}
+            ${{ matrix.client == 'prysm' && '$CL_DATA_DIR/data/beacon.log' || '' }}
+
+      - name: Clean up Erigon data directory
+        if: always()
+        run: |
+          rm -rf $ERIGON_DATA_DIR
+
+      - name: Cleanup consensus runner directory
+        if: always()
+        run: |
+          rm -rf $CL_DATA_DIR
+
+      - name: Resume the Erigon instance dedicated to db maintenance
+        run: |
+          python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
+
+      - name: Action for Success
+        if: steps.test_step.outputs.TEST_RESULT == 'success'
+        run: echo "::notice::Tests completed successfully"
+
+      - name: Action for Not Success
+        if: steps.test_step.outputs.TEST_RESULT != 'success'
+        run: |
+          echo "::error::Error detected during tests"
+          exit 1

.github/workflows/qa-tip-tracking-gnosis.yml

@@ -1,6 +1,9 @@
 name: QA - Tip tracking (Gnosis)
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 0 * * 1-6'  # Run every night at 00:00 AM UTC except Sunday
   workflow_dispatch:     # Run manually

.github/workflows/qa-tip-tracking-polygon.yml

@@ -1,6 +1,9 @@
 name: QA - Tip tracking (Polygon)
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 0 * * 1-6'  # Run every night at 00:00 AM UTC except Sunday
   workflow_dispatch:     # Run manually

.github/workflows/qa-tip-tracking.yml

@@ -1,6 +1,9 @@
 name: QA - Tip tracking
 
 on:
+  push:
+    branches:
+      - 'release/3.*'
   schedule:
     - cron: '0 20 * * 1-6'  # Run every night at 08:00 PM UTC except Sunday
   workflow_dispatch:     # Run manually
@@ -11,6 +14,7 @@ jobs:
     timeout-minutes: 600
     env:
       ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
+      ERIGON_TESTBED_AREA: /opt/erigon-testbed
       ERIGON_QA_PATH: /home/qarunner/erigon-qa
       TRACKING_TIME_SECONDS: 14400 # 4 hours
       TOTAL_TIME_SECONDS: 28800 # 8 hours
@@ -33,9 +37,10 @@ jobs:
       run: |
         python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
 
-    - name: Clean Erigon Chaindata Directory
+    - name: Save Erigon Chaindata Directory
+      id: save_chaindata_step
       run: |
-        rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata
+        mv $ERIGON_REFERENCE_DATA_DIR/chaindata $ERIGON_TESTBED_AREA/chaindata-prev
 
     - name: Run Erigon, wait sync and check ability to maintain sync
       id: test_step
@@ -112,10 +117,11 @@ jobs:
         name: metric-plots
         path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots*
 
-    - name: Clean Erigon Chaindata Directory
-      if: always()
+    - name: Restore Erigon Chaindata Directory
+      if: steps.save_chaindata_step.outcome == 'success'
       run: |
         rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata
+        mv $ERIGON_TESTBED_AREA/chaindata-prev $ERIGON_REFERENCE_DATA_DIR/chaindata
 
     - name: Resume the Erigon instance dedicated to db maintenance
       run: |

.github/workflows/release.yml

@@ -1,14 +1,14 @@
 name: Release
-run-name: Build release ${{ inputs.release_version}} from branch ${{ inputs.checkout_ref }} by @${{ github.actor }}
+run-name: Build release ${{ inputs.release_version}} from branch ${{ inputs.checkout_ref }}, Skip tests=${{ inputs.skip_tests }}
 
 env:
   APPLICATION: "erigon"
   APPLICATION_VERSION: "Erigon3"
   TEST_TRACKING_TIME_SECONDS: 7200 # 2 hours
   TEST_TOTAL_TIME_SECONDS: 432000   # 5 days
   TEST_CHAIN: "mainnet"
-  BUILDER_IMAGE: "golang:1.22-bookworm"
-  DOCKER_BASE_IMAGE: "debian:12.8-slim"
+  BUILDER_IMAGE: "golang:1.23-bookworm"
+  DOCKER_BASE_IMAGE: "debian:12-slim"
   APP_REPO: "erigontech/erigon"
   PACKAGE: "github.com/erigontech/erigon"
   DOCKERHUB_REPOSITORY: "erigontech/erigon"
@@ -37,6 +37,11 @@ on:
         type: boolean
         default: false
         description: 'publish_latest_tag: when set then docker image with tag :latest will be also published'
+      skip_tests:
+        required: false
+        type: boolean
+        default: false
+        description: 'Skip tests during release build (not recommended)'
 
 jobs:
 
@@ -199,6 +204,7 @@ jobs:
 
   test-release:
     name: test on ${{ matrix.id }}
+    if: ${{ ! inputs.skip_tests }}
     runs-on: [ self-hosted, Release, "${{ matrix.runner-arch }}" ]
     timeout-minutes: 7200  # 5 days
     needs: [ build-release ]
@@ -282,6 +288,7 @@ jobs:
   build-debian-pkg:
     name: Debian packages
     needs: [ build-release, test-release ]
+    if: always() && contains(needs.build-release.result, 'success') && !contains(needs.test-release.result, 'failure')
     uses: erigontech/erigon/.github/workflows/reusable-release-build-debian-pkg.yml@main
     with:
       application: ${{ needs.build-release.outputs.application }}
@@ -290,6 +297,7 @@ jobs:
 
   publish-docker-image:
     needs: [ build-release, test-release ]
+    if: always() && contains(needs.build-release.result, 'success') && !contains(needs.test-release.result, 'failure')
     runs-on: ubuntu-latest
     timeout-minutes: 30
     name: Docker image        
@@ -368,6 +376,7 @@ jobs:
 
   publish-release:
     needs: [ build-debian-pkg, publish-docker-image, build-release ]
+    if: always() && contains(needs.build-release.result, 'success') && contains(needs.build-debian-pkg.result, 'success') && contains(needs.publish-docker-image.result, 'success')
     runs-on: ubuntu-latest
     timeout-minutes: 15
     name: Publish release notes
@@ -425,8 +434,8 @@ jobs:
 
   In-case-of-failure:
     name: "In case of failure: remove remote git tag pointing to the new version."
-    needs: [ publish-release, build-release, test-release ]
-    if: always() && !contains(needs.build-release.result, 'success')
+    needs: [ publish-release, build-release, test-release, build-debian-pkg, publish-docker-image ]
+    if: always() && !contains(needs.build-release.result, 'success') && contains(needs.test-release.result, 'failure') && !contains(needs.publish-release.result, 'success') && !contains(needs.build-debian-pkg.result, 'success') && !contains(needs.publish-docker-image.result, 'success')
     runs-on: ubuntu-22.04
 
     steps: