save

.github/workflows/qa-rpc-integration-tests.yml

@@ -54,13 +54,22 @@ jobs:
       - name: Run RpcDaemon
         working-directory: ${{ github.workspace }}/build/bin
         run: |
-          echo "RpcDaemon starting..."
+          echo "Starting RpcDaemon..."
           
-          ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws --verbosity 1 > erigon.log 2>&1 &
+          ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 &
 
           RPC_DAEMON_PID=$!          
+          RPC_DAEMON_EXIT_STATUS=$?
           echo "RPC_DAEMON_PID=$RPC_DAEMON_PID" >> $GITHUB_ENV
           
+          sleep 5
+          tail erigon.log
+          
+          if [ $RPC_DAEMON_EXIT_STATUS -ne 0 ]; then            
+            echo "RpcDaemon failed to start"
+            echo "::error::Error detected during tests: RpcDaemon failed to start"
+            exit 1
+          fi
           echo "RpcDaemon started"
 
       - name: Wait for port 8545 to be opened
@@ -75,6 +84,7 @@ jobs:
           done
           if ! nc -z localhost 8545; then
             echo "Port 8545 did not open in time"
+            echo "::error::Error detected during tests: Port 8545 did not open in time"
             exit 1
           fi
 
@@ -153,6 +163,6 @@ jobs:
       - name: Action for Failure
         if: steps.test_step.outputs.TEST_RESULT != 'success'
         run: |
-          echo "::error::Error detected during tests"
+          echo "::error::Error detected during tests: some tests failed, check the logs or the artifacts for more details"
           exit 1
 

.github/workflows/qa-sync-with-externalcl.yml

@@ -2,7 +2,7 @@ name: QA - Sync with external CL
 
 on:
   schedule:
-    - cron: '0 0 * * *'  # Run every night at 00:00 AM UTC
+    - cron: '0 8 * * 0'  # Run on Sunday at 08:00 AM UTC
   workflow_dispatch:     # Run manually
 
 jobs:

.github/workflows/qa-tip-tracking.yml

@@ -14,6 +14,7 @@ jobs:
     timeout-minutes: 600
     env:
       ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
+      ERIGON_TESTBED_AREA: /opt/erigon-testbed
       ERIGON_QA_PATH: /home/qarunner/erigon-qa
       TRACKING_TIME_SECONDS: 14400 # 4 hours
       TOTAL_TIME_SECONDS: 28800 # 8 hours
@@ -36,9 +37,10 @@ jobs:
       run: |
         python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
 
-    - name: Clean Erigon Chaindata Directory
+    - name: Save Erigon Chaindata Directory
+      id: save_chaindata_step
       run: |
-        rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata
+        mv $ERIGON_REFERENCE_DATA_DIR/chaindata $ERIGON_TESTBED_AREA/chaindata-prev
 
     - name: Run Erigon, wait sync and check ability to maintain sync
       id: test_step
@@ -115,10 +117,11 @@ jobs:
         name: metric-plots
         path: ${{ github.workspace }}/metrics-${{ env.CHAIN }}-plots*
 
-    - name: Clean Erigon Chaindata Directory
-      if: always()
+    - name: Restore Erigon Chaindata Directory
+      if: steps.save_chaindata_step.outcome == 'success'
       run: |
         rm -rf $ERIGON_REFERENCE_DATA_DIR/chaindata
+        mv $ERIGON_TESTBED_AREA/chaindata-prev $ERIGON_REFERENCE_DATA_DIR/chaindata
 
     - name: Resume the Erigon instance dedicated to db maintenance
       run: |

.github/workflows/release.yml

@@ -1,14 +1,14 @@
 name: Release
-run-name: Build release ${{ inputs.release_version}} from branch ${{ inputs.checkout_ref }} by @${{ github.actor }}
+run-name: Build release ${{ inputs.release_version}} from branch ${{ inputs.checkout_ref }}, Skip tests=${{ inputs.skip_tests }}
 
 env:
   APPLICATION: "erigon"
   APPLICATION_VERSION: "Erigon3"
   TEST_TRACKING_TIME_SECONDS: 7200 # 2 hours
   TEST_TOTAL_TIME_SECONDS: 432000   # 5 days
   TEST_CHAIN: "mainnet"
-  BUILDER_IMAGE: "golang:1.22-bookworm"
-  DOCKER_BASE_IMAGE: "debian:12.8-slim"
+  BUILDER_IMAGE: "golang:1.23-bookworm"
+  DOCKER_BASE_IMAGE: "debian:12-slim"
   APP_REPO: "erigontech/erigon"
   PACKAGE: "github.com/erigontech/erigon"
   DOCKERHUB_REPOSITORY: "erigontech/erigon"
@@ -37,6 +37,11 @@ on:
         type: boolean
         default: false
         description: 'publish_latest_tag: when set then docker image with tag :latest will be also published'
+      skip_tests:
+        required: false
+        type: boolean
+        default: false
+        description: 'Skip tests during release build (not recommended)'
 
 jobs:
 
@@ -199,6 +204,7 @@ jobs:
 
   test-release:
     name: test on ${{ matrix.id }}
+    if: ${{ ! inputs.skip_tests }}
     runs-on: [ self-hosted, Release, "${{ matrix.runner-arch }}" ]
     timeout-minutes: 7200  # 5 days
     needs: [ build-release ]
@@ -282,6 +288,7 @@ jobs:
   build-debian-pkg:
     name: Debian packages
     needs: [ build-release, test-release ]
+    if: always() && contains(needs.build-release.result, 'success') && !contains(needs.test-release.result, 'failure')
     uses: erigontech/erigon/.github/workflows/reusable-release-build-debian-pkg.yml@main
     with:
       application: ${{ needs.build-release.outputs.application }}
@@ -290,6 +297,7 @@ jobs:
 
   publish-docker-image:
     needs: [ build-release, test-release ]
+    if: always() && contains(needs.build-release.result, 'success') && !contains(needs.test-release.result, 'failure')
     runs-on: ubuntu-latest
     timeout-minutes: 30
     name: Docker image        
@@ -368,6 +376,7 @@ jobs:
 
   publish-release:
     needs: [ build-debian-pkg, publish-docker-image, build-release ]
+    if: always() && contains(needs.build-release.result, 'success') && contains(needs.build-debian-pkg.result, 'success') && contains(needs.publish-docker-image.result, 'success')
     runs-on: ubuntu-latest
     timeout-minutes: 15
     name: Publish release notes
@@ -425,8 +434,8 @@ jobs:
 
   In-case-of-failure:
     name: "In case of failure: remove remote git tag pointing to the new version."
-    needs: [ publish-release, build-release, test-release ]
-    if: always() && !contains(needs.build-release.result, 'success')
+    needs: [ publish-release, build-release, test-release, build-debian-pkg, publish-docker-image ]
+    if: always() && !contains(needs.build-release.result, 'success') && contains(needs.test-release.result, 'failure') && !contains(needs.publish-release.result, 'success') && !contains(needs.build-debian-pkg.result, 'success') && !contains(needs.publish-docker-image.result, 'success')
     runs-on: ubuntu-22.04
 
     steps:

.github/workflows/scripts/run_rpc_tests.sh

@@ -4,6 +4,8 @@ set +e # Disable exit on error
 
 # Array of disabled tests
 disabled_tests=(
+    # Failing after the PR https://github.com/erigontech/erigon/pull/13903 - diff is only an error message in the result
+    eth_estimateGas/test_14.json
     # Failing after the PR https://github.com/erigontech/erigon/pull/13617 that fixed this incompatibility
     # issues https://hive.pectra-devnet-5.ethpandaops.io/suite.html?suiteid=1738266984-51ae1a2f376e5de5e9ba68f034f80e32.json&suitename=rpc-compat
     net_listening/test_1.json
@@ -15,11 +17,6 @@ disabled_tests=(
     engine_exchangeTransitionConfigurationV1/test_01.json
     engine_getClientVersionV1/test_1.json
     # these tests require Fix on erigon DM on repeipts domain
-    eth_getLogs/test_16
-    eth_getLogs/test_17
-    eth_getLogs/test_18
-    eth_getLogs/test_19
-    eth_getLogs/test_20
     # these tests requires Erigon active
     admin_nodeInfo/test_01.json
     admin_peers/test_01.json
@@ -42,6 +39,6 @@ disabled_tests=(
 # Transform the array into a comma-separated string
 disabled_test_list=$(IFS=,; echo "${disabled_tests[*]}")
 
-python3 ./run_tests.py -p 8545 --continue -f --json-diff -x "$disabled_test_list"
+python3 ./run_tests.py -p 8545 --continue -f --json-diff --serial -x "$disabled_test_list"
 
 exit $?

accounts/abi/bind/backends/simulated.go

@@ -857,6 +857,7 @@ func (m callMsg) Data() []byte                          { return m.CallMsg.Data
 func (m callMsg) AccessList() types.AccessList          { return m.CallMsg.AccessList }
 func (m callMsg) Authorizations() []types.Authorization { return m.CallMsg.Authorizations }
 func (m callMsg) IsFree() bool                          { return false }
+func (m callMsg) SetIsFree(_ bool)                      {}
 
 func (m callMsg) BlobGas() uint64                        { return misc.GetBlobGasUsed(len(m.CallMsg.BlobHashes)) }
 func (m callMsg) MaxFeePerBlobGas() *uint256.Int         { return m.CallMsg.MaxFeePerBlobGas }

cl/aggregation/pool_impl.go

@@ -19,12 +19,12 @@ package aggregation
 import (
 	"context"
 	"errors"
-	"fmt"
 	"sync"
 	"time"
 
 	"github.com/Giulio2002/bls"
 	"github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/cl/clparams"
 	"github.com/erigontech/erigon/cl/cltypes/solid"
 	"github.com/erigontech/erigon/cl/phase1/core/state/lru"
@@ -44,7 +44,7 @@ type aggregationPoolImpl struct {
 	netConfig      *clparams.NetworkConfig
 	ethClock       eth_clock.EthereumClock
 	aggregatesLock sync.RWMutex
-	aggregates     map[common.Hash]*solid.Attestation
+	aggregates     map[common.Hash]*solid.Attestation // don't need this anymore after electra upgrade
 	// aggregationInCommittee is a cache for aggregation in committee, which is used after electra upgrade
 	aggregatesInCommittee *lru.CacheWithTTL[keyAggrInCommittee, *solid.Attestation]
 }
@@ -78,68 +78,48 @@ func (p *aggregationPoolImpl) AddAttestation(inAtt *solid.Attestation) error {
 	if err != nil {
 		return err
 	}
-	p.aggregatesLock.Lock()
-	defer p.aggregatesLock.Unlock()
-	att, ok := p.aggregates[hashRoot]
-	if !ok {
-		p.aggregates[hashRoot] = inAtt.Copy()
-		return nil
-	}
-
-	if utils.IsOverlappingSSZBitlist(att.AggregationBits.Bytes(), inAtt.AggregationBits.Bytes()) {
-		// the on bit is already set, so ignore
-		return ErrIsSuperset
-	}
 
-	// merge signature
-	baseSig := att.Signature
-	inSig := inAtt.Signature
-	merged, err := blsAggregate([][]byte{baseSig[:], inSig[:]})
-	if err != nil {
-		return err
-	}
-	if len(merged) != 96 {
-		return errors.New("merged signature is too long")
-	}
-	var mergedSig [96]byte
-	copy(mergedSig[:], merged)
-
-	epoch := p.ethClock.GetEpochAtSlot(att.Data.Slot)
+	epoch := p.ethClock.GetEpochAtSlot(inAtt.Data.Slot)
 	clversion := p.ethClock.StateVersionByEpoch(epoch)
 	if clversion.BeforeOrEqual(clparams.DenebVersion) {
-		// merge aggregation bits
-		mergedBits, err := att.AggregationBits.Union(inAtt.AggregationBits)
-		if err != nil {
-			return err
+		p.aggregatesLock.Lock()
+		defer p.aggregatesLock.Unlock()
+		att, ok := p.aggregates[hashRoot]
+		if !ok {
+			p.aggregates[hashRoot] = inAtt.Copy()
+			return nil
 		}
-		// update attestation
-		p.aggregates[hashRoot] = &solid.Attestation{
-			AggregationBits: mergedBits,
-			Data:            att.Data,
-			Signature:       mergedSig,
+
+		if utils.IsOverlappingSSZBitlist(att.AggregationBits.Bytes(), inAtt.AggregationBits.Bytes()) {
+			// the on bit is already set, so ignore
+			return ErrIsSuperset
 		}
-	} else {
-		// Electra and after case
-		aggrBitSize := p.beaconConfig.MaxCommitteesPerSlot * p.beaconConfig.MaxValidatorsPerCommittee
-		mergedAggrBits, err := att.AggregationBits.Union(inAtt.AggregationBits)
+		// merge signature
+		baseSig := att.Signature
+		inSig := inAtt.Signature
+		merged, err := blsAggregate([][]byte{baseSig[:], inSig[:]})
 		if err != nil {
 			return err
 		}
-		if mergedAggrBits.Cap() != int(aggrBitSize) {
-			return fmt.Errorf("incorrect aggregation bits size: %d", mergedAggrBits.Cap())
+		if len(merged) != 96 {
+			return errors.New("merged signature is too long")
 		}
-		mergedCommitteeBits, err := att.CommitteeBits.Union(inAtt.CommitteeBits)
+		var mergedSig [96]byte
+		copy(mergedSig[:], merged)
+
+		// merge aggregation bits
+		mergedBits, err := att.AggregationBits.Merge(inAtt.AggregationBits)
 		if err != nil {
 			return err
 		}
+		// update attestation
 		p.aggregates[hashRoot] = &solid.Attestation{
-			AggregationBits: mergedAggrBits,
-			CommitteeBits:   mergedCommitteeBits,
+			AggregationBits: mergedBits,
 			Data:            att.Data,
 			Signature:       mergedSig,
 		}
-
-		// aggregate by committee
+	} else {
+		// Electra and after case, aggregate by committee
 		p.aggregateByCommittee(inAtt)
 	}
 	return nil
@@ -166,9 +146,15 @@ func (p *aggregationPoolImpl) aggregateByCommittee(inAtt *solid.Attestation) err
 		return nil
 	}
 
-	// merge aggregation bits and signature
-	mergedAggrBits, err := att.AggregationBits.Union(inAtt.AggregationBits)
+	if utils.IsOverlappingSSZBitlist(att.AggregationBits.Bytes(), inAtt.AggregationBits.Bytes()) {
+		// the on bit is already set, so ignore
+		return ErrIsSuperset
+	}
+
+	// It's fine to directly merge aggregation bits here, because the attestation is from the same committee
+	mergedAggrBits, err := att.AggregationBits.Merge(inAtt.AggregationBits)
 	if err != nil {
+		log.Debug("failed to merge aggregation bits", "err", err)
 		return err
 	}
 	merged, err := blsAggregate([][]byte{att.Signature[:], inAtt.Signature[:]})