[Cherry-Pick] jsonrpc: eth_estimateGas - stateoverrides (#13772), fix historical (#13903), qa-tests fix (#13909), and optimize for max contract gas used (#13913)

.github/workflows/scripts/run_rpc_tests.sh

@@ -4,6 +4,8 @@ set +e # Disable exit on error
 
 # Array of disabled tests
 disabled_tests=(
+    # Failing after the PR https://github.com/erigontech/erigon/pull/13903 - diff is only an error message in the result
+    eth_estimateGas/test_14.json
     # Failing after the PR https://github.com/erigontech/erigon/pull/13617 that fixed this incompatibility
     # issues https://hive.pectra-devnet-5.ethpandaops.io/suite.html?suiteid=1738266984-51ae1a2f376e5de5e9ba68f034f80e32.json&suitename=rpc-compat
     net_listening/test_1.json

core/state_transition.go

@@ -536,6 +536,8 @@ func (st *StateTransition) TransitionDb(refunds bool, gasBailout bool) (*evmtype
 		SenderInitBalance:   senderInitBalance,
 		CoinbaseInitBalance: coinbaseInitBalance,
 		FeeTipped:           amount,
+		EvmRefund:           st.state.GetRefund(),
+		EvmGasUsed:          st.gasUsed(),
 	}
 
 	if st.evm.Context.PostApplyMessage != nil {

core/vm/evmtypes/evmtypes.go

@@ -72,6 +72,8 @@ type ExecutionResult struct {
 	SenderInitBalance   *uint256.Int
 	CoinbaseInitBalance *uint256.Int
 	FeeTipped           *uint256.Int
+	EvmRefund           uint64 // Gas refunded by EVM without considering refundQuotient
+	EvmGasUsed          uint64 // Gas used by the execution of all instructions only
 }
 
 // Unwrap returns the internal evm error which allows us for further

turbo/jsonrpc/eth_call.go

@@ -49,7 +49,6 @@ import (
 	"github.com/erigontech/erigon/core/state"
 	"github.com/erigontech/erigon/core/types"
 	"github.com/erigontech/erigon/core/vm"
-	"github.com/erigontech/erigon/core/vm/evmtypes"
 	"github.com/erigontech/erigon/eth/stagedsync"
 	"github.com/erigontech/erigon/eth/tracers/logger"
 	"github.com/erigontech/erigon/params"
@@ -151,76 +150,63 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs
 	}
 	defer dbtx.Rollback()
 
+	// Use latest block by default
+	if blockNrOrHash == nil {
+		blockNrOrHash = &latestNumOrHash
+	}
+
 	chainConfig, err := api.chainConfig(ctx, dbtx)
 	if err != nil {
 		return 0, err
 	}
 	engine := api.engine()
 
-	latestCanBlockNumber, latestCanHash, isLatest, err := rpchelper.GetCanonicalBlockNumber(ctx, latestNumOrHash, dbtx, api._blockReader, api.filters) // DoCall cannot be executed on non-canonical blocks
+	blockNum, blockHash, isLatest, err := rpchelper.GetCanonicalBlockNumber(ctx, *blockNrOrHash, dbtx, api._blockReader, api.filters) // DoCall cannot be executed on non-canonical blocks
 	if err != nil {
 		return 0, err
 	}
 
 	// try and get the block from the lru cache first then try DB before failing
-	block := api.tryBlockFromLru(latestCanHash)
+	block := api.tryBlockFromLru(blockHash)
 	if block == nil {
-		block, err = api.blockWithSenders(ctx, dbtx, latestCanHash, latestCanBlockNumber)
+		block, err = api.blockWithSenders(ctx, dbtx, blockHash, blockNum)
 		if err != nil {
 			return 0, err
 		}
 	}
+
 	if block == nil {
-		return 0, errors.New("could not find latest block in cache or db")
+		return 0, errors.New(fmt.Sprintf("could not find the block %s in cache or db", blockNrOrHash.String()))
 	}
+	header := block.HeaderNoCopy()
 
 	txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, api._blockReader))
-	stateReader, err := rpchelper.CreateStateReaderFromBlockNumber(ctx, dbtx, txNumsReader, latestCanBlockNumber, isLatest, 0, api.stateCache, chainConfig.ChainName)
+	stateReader, err := rpchelper.CreateStateReaderFromBlockNumber(ctx, dbtx, txNumsReader, blockNum, isLatest, 0, api.stateCache, chainConfig.ChainName)
 	if err != nil {
 		return 0, err
 	}
 
 	// Binary search the gas requirement, as it may be higher than the amount used
 	var (
-		lo     = params.TxGas - 1
-		hi     uint64
-		gasCap uint64
+		lo uint64
+		hi uint64
 	)
 	// Use zero address if sender unspecified.
 	if args.From == nil {
 		args.From = new(libcommon.Address)
 	}
 
-	bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber)
-	if blockNrOrHash != nil {
-		bNrOrHash = *blockNrOrHash
-	}
-
 	// Determine the highest gas limit can be used during the estimation.
 	if args.Gas != nil && uint64(*args.Gas) >= params.TxGas {
 		hi = uint64(*args.Gas)
 	} else {
 		// Retrieve the block to act as the gas ceiling
-		h, err := headerByNumberOrHash(ctx, dbtx, bNrOrHash, api)
-		if err != nil {
-			return 0, err
-		}
-		if h == nil {
-			// if a block number was supplied and there is no header return 0
-			if blockNrOrHash != nil {
-				return 0, nil
-			}
-
-			// block number not supplied, so we haven't found a pending block, read the latest block instead
-			h, err = headerByNumberOrHash(ctx, dbtx, latestNumOrHash, api)
-			if err != nil {
-				return 0, err
-			}
-			if h == nil {
-				return 0, nil
-			}
-		}
-		hi = h.GasLimit
+		hi = header.GasLimit
+	}
+	// Recap the highest gas allowance with specified gascap.
+	if hi > api.GasCap {
+		log.Warn("Caller gas above allowance, capping", "requested", hi, "cap", api.GasCap)
+		hi = api.GasCap
 	}
 
 	var feeCap *big.Int
@@ -265,68 +251,49 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs
 		}
 	}
 
-	// Recap the highest gas allowance with specified gascap.
-	if hi > api.GasCap {
-		log.Warn("Caller gas above allowance, capping", "requested", hi, "cap", api.GasCap)
-		hi = api.GasCap
-	}
-	gasCap = hi
-
-	header := block.HeaderNoCopy()
-
-	caller, err := transactions.NewReusableCaller(engine, stateReader, overrides, header, args, api.GasCap, latestNumOrHash, dbtx, api._blockReader, chainConfig, api.evmCallTimeout)
+	caller, err := transactions.NewReusableCaller(engine, stateReader, overrides, header, args, api.GasCap, *blockNrOrHash, dbtx, api._blockReader, chainConfig, api.evmCallTimeout)
 	if err != nil {
 		return 0, err
 	}
 
-	// Create a helper to check if a gas allowance results in an executable transaction
-	executable := func(gas uint64) (bool, *evmtypes.ExecutionResult, error) {
-		result, err := caller.DoCallWithNewGas(ctx, gas)
-		if err != nil {
-			if errors.Is(err, core.ErrIntrinsicGas) {
-				// Special case, raise gas limit
-				return true, nil, nil
+	// First try with highest gas possible
+	result, err := caller.DoCallWithNewGas(ctx, hi, engine, overrides)
+	if err != nil || result == nil {
+		return 0, err
+	}
+	if result.Failed() {
+		if !errors.Is(result.Err, vm.ErrOutOfGas) {
+			if len(result.Revert()) > 0 {
+				return 0, ethapi2.NewRevertError(result)
 			}
-
-			// Bail out
-			return true, nil, err
+			return 0, result.Err
 		}
-		return result.Failed(), result, nil
+		// Otherwise, the specified gas cap is too low
+		return 0, fmt.Errorf("gas required exceeds allowance (%d)", hi)
 	}
+	// Assuming a contract can freely run all the instructions, we have
+	// the true amount of gas it wants to consume to execute fully.
+	// We want to ensure that the gas used doesn't fall below this
+	trueGas := result.EvmGasUsed // Must not fall below this
+	lo = min(trueGas+result.EvmRefund-1, params.TxGas-1)
 
+	i := 0
 	// Execute the binary search and hone in on an executable gas limit
 	for lo+1 < hi {
 		mid := (hi + lo) / 2
-		failed, _, err := executable(mid)
+		result, err := caller.DoCallWithNewGas(ctx, mid, engine, overrides)
 		// If the error is not nil(consensus error), it means the provided message
 		// call or transaction will never be accepted no matter how much gas it is
 		// assigened. Return the error directly, don't struggle any more.
 		if err != nil {
 			return 0, err
 		}
-		if failed {
+		if result.Failed() || result.EvmGasUsed < trueGas {
 			lo = mid
 		} else {
 			hi = mid
 		}
-	}
-
-	// Reject the transaction as invalid if it still fails at the highest allowance
-	if hi == gasCap {
-		failed, result, err := executable(hi)
-		if err != nil {
-			return 0, err
-		}
-		if failed {
-			if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) {
-				if len(result.Revert()) > 0 {
-					return 0, ethapi2.NewRevertError(result)
-				}
-				return 0, result.Err
-			}
-			// Otherwise, the specified gas cap is too low
-			return 0, fmt.Errorf("gas required exceeds allowance (%d)", gasCap)
-		}
+		i++
 	}
 	return hexutil.Uint64(hi), nil
 }

turbo/transactions/call.go

@@ -156,6 +156,8 @@ type ReusableCaller struct {
 func (r *ReusableCaller) DoCallWithNewGas(
 	ctx context.Context,
 	newGas uint64,
+	engine consensus.EngineReader,
+	overrides *ethapi2.StateOverrides,
 ) (*evmtypes.ExecutionResult, error) {
 	var cancel context.CancelFunc
 	if r.callTimeout > 0 {
@@ -172,7 +174,10 @@ func (r *ReusableCaller) DoCallWithNewGas(
 
 	// reset the EVM so that we can continue to use it with the new context
 	txCtx := core.NewEVMTxContext(r.message)
-	r.intraBlockState = state.New(r.stateReader)
+	if overrides == nil {
+		r.intraBlockState = state.New(r.stateReader)
+	}
+
 	r.evm.Reset(txCtx, r.intraBlockState)
 
 	timedOut := false