feat(v2): import/export, hashing, clean up

v2/cmd/bench/bench.go

@@ -0,0 +1,160 @@
+package bench
+
+import (
+	"log/slog"
+	"net/http"
+	"os"
+	"runtime/pprof"
+	"time"
+
+	"github.com/cosmos/iavl/v2"
+	"github.com/cosmos/iavl/v2/metrics"
+	"github.com/cosmos/iavl/v2/testutil"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+)
+
+func Command() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "bench",
+		Short: "run benchmarks",
+	}
+	cmd.AddCommand(benchCommand())
+	return cmd
+}
+
+func benchCommand() *cobra.Command {
+	var (
+		dbPath        string
+		changelogPath string
+		loadSnapshot  bool
+		usePrometheus bool
+		cpuProfile    string
+	)
+	cmd := &cobra.Command{
+		Use:   "std",
+		Short: "run the std development benchmark",
+		Long: `Runs a longer benchmark for the IAVL tree. This is useful for development and testing.
+Pre-requisites this command:
+$ go run ./cmd gen tree --db /tmp/iavl-v2 --limit 1 --type osmo-like-many
+mkdir -p /tmp/osmo-like-many/v2 && go run ./cmd gen emit --start 2 --limit 1000 --type osmo-like-many --out /tmp/osmo-like-many/v2
+
+Optional for --snapshot arg:
+$ go run ./cmd snapshot --db /tmp/iavl-v2 --version 1
+`,
+
+		RunE: func(_ *cobra.Command, _ []string) error {
+			if cpuProfile != "" {
+				f, err := os.Create(cpuProfile)
+				if err != nil {
+					return err
+				}
+				if err := pprof.StartCPUProfile(f); err != nil {
+					return err
+				}
+				defer func() {
+					pprof.StopCPUProfile()
+					f.Close()
+				}()
+			}
+			treeOpts := iavl.DefaultTreeOptions()
+			treeOpts.CheckpointInterval = 80
+			treeOpts.StateStorage = true
+			treeOpts.HeightFilter = 1
+			treeOpts.EvictionDepth = 22
+			treeOpts.MetricsProxy = metrics.NewStructMetrics()
+			if usePrometheus {
+				treeOpts.MetricsProxy = newPrometheusMetricsProxy()
+			}
+
+			logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))
+			var multiTree *iavl.MultiTree
+			if loadSnapshot {
+				var err error
+				multiTree, err = iavl.ImportMultiTree(logger, 1, dbPath, treeOpts)
+				if err != nil {
+					return err
+				}
+			} else {
+				multiTree = iavl.NewMultiTree(logger, dbPath, treeOpts)
+				if err := multiTree.MountTrees(); err != nil {
+					return err
+				}
+				if err := multiTree.LoadVersion(1); err != nil {
+					return err
+				}
+				if err := multiTree.WarmLeaves(); err != nil {
+					return err
+				}
+			}
+
+			opts := testutil.CompactedChangelogs(changelogPath)
+			opts.SampleRate = 250_000
+
+			// opts.Until = 1_000
+			// opts.UntilHash = "557663181d9ab97882ecfc6538e3b4cfe31cd805222fae905c4b4f4403ca5cda"
+			opts.Until = 500
+			opts.UntilHash = "2670bd5767e70f2bf9e4f723b5f205759e39afdb5d8cfb6b54a4a3ecc27a1377"
+
+			_, err := multiTree.TestBuild(opts)
+			return err
+		},
+	}
+	cmd.Flags().StringVar(&dbPath, "db", "/tmp/iavl-v2", "the path to the database at version 1")
+	cmd.Flags().StringVar(&changelogPath, "changelog", "/tmp/osmo-like-many/v2", "the path to the changelog")
+	cmd.Flags().BoolVar(&loadSnapshot, "snapshot", false, "load the snapshot at version 1 before running the benchmarks (loads full tree into memory)")
+	cmd.Flags().BoolVar(&usePrometheus, "prometheus", false, "enable prometheus metrics")
+	cmd.Flags().StringVar(&cpuProfile, "cpu-profile", "", "write cpu profile to file")
+
+	if err := cmd.MarkFlagRequired("changelog"); err != nil {
+		panic(err)
+	}
+	if err := cmd.MarkFlagRequired("db"); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
+var _ metrics.Proxy = &prometheusMetricsProxy{}
+
+type prometheusMetricsProxy struct {
+	workingSize  prometheus.Gauge
+	workingBytes prometheus.Gauge
+}
+
+func newPrometheusMetricsProxy() *prometheusMetricsProxy {
+	p := &prometheusMetricsProxy{}
+	p.workingSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "iavl_working_size",
+		Help: "working size",
+	})
+	p.workingBytes = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "iavl_working_bytes",
+		Help: "working bytes",
+	})
+	http.Handle("/metrics", promhttp.Handler())
+	go func() {
+		err := http.ListenAndServe(":2112", nil)
+		if err != nil {
+			panic(err)
+		}
+	}()
+	return p
+}
+
+func (p *prometheusMetricsProxy) IncrCounter(_ float32, _ ...string) {
+}
+
+func (p *prometheusMetricsProxy) SetGauge(val float32, keys ...string) {
+	k := keys[1]
+	switch k {
+	case "working_size":
+		p.workingSize.Set(float64(val))
+	case "working_bytes":
+		p.workingBytes.Set(float64(val))
+	}
+}
+
+func (p *prometheusMetricsProxy) MeasureSince(_ time.Time, _ ...string) {}

v2/cmd/gen/gen.go

@@ -144,8 +144,8 @@ func treeCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "tree",
 		Short: "build and save a Tree to disk, taking generated changesets as input",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			multiTree := iavl.NewMultiTree(iavl.NewTestLogger(), dbPath, iavl.TreeOptions{StateStorage: true})
+		RunE: func(_ *cobra.Command, _ []string) error {
+			multiTree := iavl.NewMultiTree(iavl.NewDebugLogger(), dbPath, iavl.DefaultTreeOptions())
 			defer func(mt *iavl.MultiTree) {
 				err := mt.Close()
 				if err != nil {

v2/cmd/root.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/cosmos/iavl/v2/cmd/bench"
 	"github.com/cosmos/iavl/v2/cmd/gen"
 	"github.com/cosmos/iavl/v2/cmd/rollback"
 	"github.com/cosmos/iavl/v2/cmd/scan"
@@ -19,6 +20,7 @@ func RootCommand() (*cobra.Command, error) {
 		rollback.Command(),
 		scan.Command(),
 		latestCommand(),
+		bench.Command(),
 	)
 	return cmd, nil
 }

v2/export.go

@@ -13,30 +13,41 @@ const (
 )
 
 type Exporter struct {
-	tree  *Tree
+	sql   *SqliteDb
 	out   chan *Node
 	errCh chan error
 }
 
-func (tree *Tree) Export(order TraverseOrderType) *Exporter {
+func (tree *Tree) Export(version int64, order TraverseOrderType) (*Exporter, error) {
+	sql := tree.sql
+	root := tree.root
+	if version != tree.Version() {
+		cloned, err := tree.ReadonlyClone()
+		if err != nil {
+			return nil, err
+		}
+		if err = cloned.LoadVersion(version); err != nil {
+			return nil, err
+		}
+		root = cloned.root
+		sql = cloned.sql
+	}
+
 	exporter := &Exporter{
-		tree:  tree,
+		sql:   sql,
 		out:   make(chan *Node),
 		errCh: make(chan error),
 	}
-
 	go func(traverseOrder TraverseOrderType) {
 		defer close(exporter.out)
-		defer close(exporter.errCh)
-
 		if traverseOrder == PostOrder {
-			exporter.postOrderNext(tree.root)
+			exporter.postOrderNext(root)
 		} else if traverseOrder == PreOrder {
-			exporter.preOrderNext(tree.root)
+			exporter.preOrderNext(root)
 		}
 	}(order)
 
-	return exporter
+	return exporter, nil
 }
 
 func (e *Exporter) postOrderNext(node *Node) {
@@ -45,14 +56,14 @@ func (e *Exporter) postOrderNext(node *Node) {
 		return
 	}
 
-	left, err := node.getLeftNode(e.tree)
+	left, err := node.getLeftNode(e.sql)
 	if err != nil {
 		e.errCh <- err
 		return
 	}
 	e.postOrderNext(left)
 
-	right, err := node.getRightNode(e.tree)
+	right, err := node.getRightNode(e.sql)
 	if err != nil {
 		e.errCh <- err
 		return
@@ -68,36 +79,38 @@ func (e *Exporter) preOrderNext(node *Node) {
 		return
 	}
 
-	left, err := node.getLeftNode(e.tree)
+	left, err := node.getLeftNode(e.sql)
 	if err != nil {
 		e.errCh <- err
 		return
 	}
 	e.preOrderNext(left)
 
-	right, err := node.getRightNode(e.tree)
+	right, err := node.getRightNode(e.sql)
 	if err != nil {
 		e.errCh <- err
 		return
 	}
 	e.preOrderNext(right)
 }
 
-func (e *Exporter) Next() (*SnapshotNode, error) {
+func (e *Exporter) Next() (*Node, error) {
 	select {
 	case node, ok := <-e.out:
 		if !ok {
 			return nil, ErrorExportDone
 		}
-		return &SnapshotNode{
-			Key:     node.key,
-			Value:   node.value,
-			Version: node.nodeKey.Version(),
-			Height:  node.subtreeHeight,
-		}, nil
+		if node == nil {
+			panic("unexpected nil node")
+		}
+		return node, nil
 	case err := <-e.errCh:
 		return nil, err
 	}
 }
 
+func (e *Exporter) Close() error {
+	return e.sql.Close()
+}
+
 var ErrorExportDone = errors.New("export done")