silkworm: add state snapshots (#13766)

Previously the Silkworm library was capable of reading Erigon 2 "blocks" snapshots (.seg and .idx). The code to provide them to Silkworm was a part of RoSnapshots. The old code is moved to be a part of silkworm.SnapshotsRepository, and migrated to the updated Silkworm C API. The new code adds an ability to provide Erigon 3 "state" snapshots for reading by Silkworm during evmone execution mode (`--silkworm.exec`). It is based on the AggregatorRoTx.StaticFilesInRange method that was made public (with the related DTOs) to be accessible from the silkworm package. BtIndex.DataHandle() added to expose the memory mapped file pointer to Silkworm (in the same way as Decompressor and recsplit.Index). A bunch of public methods added to filesItem via FilesItem interface to expose the snapshot file objects to Silkworm.
erigontech · Feb 17, 2025 · 579dc63 · 579dc63
1 parent 10e2e49
commit 579dc63
Show file tree

Hide file tree

Showing 13 changed files with 365 additions and 100 deletions.
diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go
@@ -692,7 +692,7 @@ func (a *Aggregator) mergeLoopStep(ctx context.Context, toTxNum uint64) (somethi
 		return false, nil
 	}
 
-	outs, err := aggTx.staticFilesInRange(r)
+	outs, err := aggTx.StaticFilesInRange(r)
 	defer func() {
 		if closeAll {
 			outs.Close()
@@ -1244,6 +1244,10 @@ type RangesV3 struct {
 	invertedIndex []*MergeRange
 }
 
+func NewRangesV3(domain [kv.DomainLen]DomainRanges, invertedIndex []*MergeRange) RangesV3 {
+	return RangesV3{domain: domain, invertedIndex: invertedIndex}
+}
+
 func (r RangesV3) String() string {
 	ss := []string{}
 	for _, d := range &r.domain {

diff --git a/erigon-lib/state/aggregator_files.go b/erigon-lib/state/aggregator_files.go
@@ -17,6 +17,7 @@
 package state
 
 import (
+	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"
 )
 
@@ -27,7 +28,23 @@ type SelectedStaticFilesV3 struct {
 	ii    [][]*filesItem
 }
 
-func (sf SelectedStaticFilesV3) Close() {
+func (sf *SelectedStaticFilesV3) DomainFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.d[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) DomainHistoryFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.dHist[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) DomainInvertedIndexFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.dIdx[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) InvertedIndexFiles(id int) []FilesItem {
+	return common.SliceMap(sf.ii[id], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) Close() {
 	clist := make([][]*filesItem, 0, int(kv.DomainLen)+len(sf.ii))
 	for id := range sf.d {
 		clist = append(clist, sf.d[id], sf.dIdx[id], sf.dHist[id])
@@ -48,7 +65,7 @@ func (sf SelectedStaticFilesV3) Close() {
 	}
 }
 
-func (ac *AggregatorRoTx) staticFilesInRange(r *RangesV3) (*SelectedStaticFilesV3, error) {
+func (ac *AggregatorRoTx) StaticFilesInRange(r *RangesV3) (*SelectedStaticFilesV3, error) {
 	sf := &SelectedStaticFilesV3{ii: make([][]*filesItem, len(r.invertedIndex))}
 	for id := range ac.d {
 		if !r.domain[id].any() {
@@ -65,6 +82,14 @@ func (ac *AggregatorRoTx) staticFilesInRange(r *RangesV3) (*SelectedStaticFilesV
 	return sf, nil
 }
 
+func (ac *AggregatorRoTx) InvertedIndicesLen() int {
+	return len(ac.iis)
+}
+
+func (ac *AggregatorRoTx) InvertedIndexName(id int) kv.InvertedIdx {
+	return ac.iis[id].name
+}
+
 type MergedFilesV3 struct {
 	d     [kv.DomainLen]*filesItem
 	dHist [kv.DomainLen]*filesItem

diff --git a/erigon-lib/state/btree_index.go b/erigon-lib/state/btree_index.go
@@ -30,6 +30,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unsafe"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/edsrzf/mmap-go"
@@ -994,6 +995,10 @@ func (b *BtIndex) newCursor(k, v []byte, d uint64, g *seg.Reader) *Cursor {
 	return c
 }
 
+func (b *BtIndex) DataHandle() unsafe.Pointer {
+	return unsafe.Pointer(&b.data[0])
+}
+
 func (b *BtIndex) Size() int64 { return b.size }
 
 func (b *BtIndex) ModTime() time.Time { return b.modTime }

diff --git a/erigon-lib/state/files_item.go b/erigon-lib/state/files_item.go
@@ -62,13 +62,30 @@ type filesItem struct {
 	canDelete atomic.Bool
 }
 
+type FilesItem interface {
+	Segment() *seg.Decompressor
+	AccessorIndex() *recsplit.Index
+	BtIndex() *BtIndex
+	ExistenceFilter() *ExistenceFilter
+}
+
+var _ FilesItem = (*filesItem)(nil)
+
 func newFilesItem(startTxNum, endTxNum, stepSize uint64) *filesItem {
 	startStep := startTxNum / stepSize
 	endStep := endTxNum / stepSize
 	frozen := endStep-startStep == config3.StepsInFrozenFile
 	return &filesItem{startTxNum: startTxNum, endTxNum: endTxNum, frozen: frozen}
 }
 
+func (i *filesItem) Segment() *seg.Decompressor { return i.decompressor }
+
+func (i *filesItem) AccessorIndex() *recsplit.Index { return i.index }
+
+func (i *filesItem) BtIndex() *BtIndex { return i.bindex }
+
+func (i *filesItem) ExistenceFilter() *ExistenceFilter { return i.existence }
+
 // isSubsetOf - when `j` covers `i` but not equal `i`
 func (i *filesItem) isSubsetOf(j *filesItem) bool {
 	return (j.startTxNum <= i.startTxNum && i.endTxNum <= j.endTxNum) && (j.startTxNum != i.startTxNum || i.endTxNum != j.endTxNum)

diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go
@@ -506,6 +506,10 @@ type MergeRange struct {
 	to        uint64
 }
 
+func NewMergeRange(name string, needMerge bool, from uint64, to uint64) *MergeRange {
+	return &MergeRange{name: name, needMerge: needMerge, from: from, to: to}
+}
+
 func (mr *MergeRange) FromTo() (uint64, uint64) {
 	return mr.from, mr.to
 }

diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go
@@ -81,6 +81,10 @@ type DomainRanges struct {
 	aggStep uint64
 }
 
+func NewDomainRanges(name kv.Domain, values MergeRange, history HistoryRanges, aggStep uint64) DomainRanges {
+	return DomainRanges{name: name, values: values, history: history, aggStep: aggStep}
+}
+
 func (r DomainRanges) String() string {
 	var b strings.Builder
 	if r.values.needMerge {
@@ -219,6 +223,10 @@ type HistoryRanges struct {
 	index   MergeRange
 }
 
+func NewHistoryRanges(history MergeRange, index MergeRange) HistoryRanges {
+	return HistoryRanges{history: history, index: index}
+}
+
 func (r HistoryRanges) String(aggStep uint64) string {
 	var str string
 	if r.history.needMerge {

diff --git a/erigon-lib/state/squeeze.go b/erigon-lib/state/squeeze.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
+
 	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/datadir"
 	"github.com/erigontech/erigon-lib/common/dir"
@@ -132,7 +133,7 @@ func (ac *AggregatorRoTx) SqueezeCommitmentFiles() error {
 			},
 		},
 	}
-	sf, err := ac.staticFilesInRange(rng)
+	sf, err := ac.StaticFilesInRange(rng)
 	if err != nil {
 		return err
 	}
@@ -331,7 +332,7 @@ func (a *Aggregator) RebuildCommitmentFiles(ctx context.Context, rwDb kv.RwDB, t
 			},
 		},
 	}
-	sf, err := acRo.staticFilesInRange(rng)
+	sf, err := acRo.StaticFilesInRange(rng)
 	if err != nil {
 		return nil, err
 	}

diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go
@@ -300,12 +300,6 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
 		return err
 	}
 
-	if cfg.silkworm != nil {
-		if err := cfg.blockReader.Snapshots().(silkworm.CanAddSnapshotsToSilkwarm).AddSnapshotsToSilkworm(cfg.silkworm); err != nil {
-			return err
-		}
-	}
-
 	indexWorkers := estimate.IndexSnapshot.Workers()
 	diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "E3 Indexing"})
 	if err := agg.BuildMissedIndices(ctx, indexWorkers); err != nil {
@@ -321,6 +315,18 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
 		cfg.notifier.Events.OnNewSnapshot()
 	}
 
+	if cfg.silkworm != nil {
+		repository := silkworm.NewSnapshotsRepository(
+			cfg.silkworm,
+			cfg.blockReader.Snapshots().(*freezeblocks.RoSnapshots),
+			agg,
+			logger,
+		)
+		if err := repository.Update(); err != nil {
+			return err
+		}
+	}
+
 	frozenBlocks := cfg.blockReader.FrozenBlocks()
 	if s.BlockNumber < frozenBlocks { // allow genesis
 		if err := s.Update(tx, frozenBlocks); err != nil {

diff --git a/go.mod b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116
 	github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f
 	github.com/erigontech/secp256k1 v1.1.0
-	github.com/erigontech/silkworm-go v0.18.0
+	github.com/erigontech/silkworm-go v0.24.0
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -275,8 +275,8 @@ github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f h1:2uZv1SGd7
 github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f/go.mod h1:lkqHAZqXtFaIPlvTaGAx3VUDuGYZcuhve1l4JVVN1Z0=
 github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY=
 github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls=
-github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA=
-github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
+github.com/erigontech/silkworm-go v0.24.0 h1:fFe74CjQM5LI7ouMYjmqfFaqIFzQTpMrt+ls+a5PxpE=
+github.com/erigontech/silkworm-go v0.24.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
 github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U=
 github.com/erigontech/speedtest v0.0.2/go.mod h1:vulsRNiM51BmSTbVtch4FWxKxx53pS2D35lZTtao0bw=
 github.com/erigontech/torrent v1.54.3-alpha-1 h1:oyT9YpMr82g566v0STVKW0ZTdX/eun03cW2mKmKuTAQ=

diff --git a/turbo/silkworm/silkworm.go b/turbo/silkworm/silkworm.go
@@ -33,16 +33,18 @@ type SilkwormLogLevel = silkworm_go.SilkwormLogLevel
 type SentrySettings = silkworm_go.SentrySettings
 type RpcDaemonSettings = silkworm_go.RpcDaemonSettings
 type RpcInterfaceLogSettings = silkworm_go.RpcInterfaceLogSettings
-type MappedHeaderSnapshot = silkworm_go.MappedHeaderSnapshot
-type MappedBodySnapshot = silkworm_go.MappedBodySnapshot
-type MappedTxnSnapshot = silkworm_go.MappedTxnSnapshot
-type MappedChainSnapshot = silkworm_go.MappedChainSnapshot
-
-var NewMemoryMappedRegion = silkworm_go.NewMemoryMappedRegion
-var NewMappedHeaderSnapshot = silkworm_go.NewMappedHeaderSnapshot
-var NewMappedBodySnapshot = silkworm_go.NewMappedBodySnapshot
-var NewMappedTxnSnapshot = silkworm_go.NewMappedTxnSnapshot
 
+type HeadersSnapshot = silkworm_go.HeadersSnapshot
+type BodiesSnapshot = silkworm_go.BodiesSnapshot
+type TransactionsSnapshot = silkworm_go.TransactionsSnapshot
+type BlocksSnapshotBundle = silkworm_go.BlocksSnapshotBundle
+type InvertedIndexSnapshot = silkworm_go.InvertedIndexSnapshot
+type HistorySnapshot = silkworm_go.HistorySnapshot
+type DomainSnapshot = silkworm_go.DomainSnapshot
+type StateSnapshotBundleLatest = silkworm_go.StateSnapshotBundleLatest
+type StateSnapshotBundleHistorical = silkworm_go.StateSnapshotBundleHistorical
+
+var NewFilePath = silkworm_go.NewFilePath
 var ErrInterrupted = silkworm_go.ErrInterrupted
 
 func New(dataDirPath string, libMdbxVersion string, numIOContexts uint32, logLevel log.Lvl) (*Silkworm, error) {
@@ -125,7 +127,3 @@ func ExecuteBlocksPerpetual(s *Silkworm, db kv.RwDB, chainID *big.Int, startBloc
 	}
 	return lastExecutedBlock, err
 }
-
-type CanAddSnapshotsToSilkwarm interface {
-	AddSnapshotsToSilkworm(*Silkworm) error
-}