fix tree test

Author: kocubinski

metrics/metrics.go

@@ -30,6 +30,7 @@ type TreeMetrics struct {
 	TreeUpdate  int64
 	TreeNewNode int64
 	TreeDelete  int64
+	TreeHash    int64
 }
 
 type DbMetrics struct {

tree.go

@@ -258,6 +258,9 @@ func (tree *Tree) deepHash(node *Node, depth int8) {
 		}
 	}
 
+	if node.hash == nil {
+		tree.metrics.TreeHash++
+	}
 	node._hash()
 
 	// when heightFilter > 0 remove the leaf nodes from memory.

tree_test.go

@@ -3,7 +3,6 @@
 package iavl
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
@@ -64,16 +63,19 @@ func testTreeBuild(t *testing.T, multiTree *MultiTree, opts *testutil.TreeBuildO
 			workingSize  int64
 			writeLeaves  int64
 			writeTime    time.Duration
+			hashCount    int64
 		)
 		for _, tr := range multiTree.Trees {
 			m := tr.sql.metrics
 			workingBytes += tr.workingBytes
 			workingSize += tr.workingSize
 			writeLeaves += m.WriteLeaves
 			writeTime += m.WriteTime
+			hashCount += tr.metrics.TreeHash
 			m.WriteDurations = nil
 			m.WriteLeaves = 0
 			m.WriteTime = 0
+			tr.metrics.TreeHash = 0
 		}
 		fmt.Printf("leaves=%s time=%s last=%s μ=%s version=%d work-size=%s work-bytes=%s %s\n",
 			humanize.Comma(cnt),
@@ -86,11 +88,12 @@ func testTreeBuild(t *testing.T, multiTree *MultiTree, opts *testutil.TreeBuildO
 			MemUsage())
 
 		if writeTime > 0 {
-			fmt.Printf("writes: cnt=%s wr/s=%s dur/wr=%s dur=%s\n",
+			fmt.Printf("writes: cnt=%s wr/s=%s dur/wr=%s dur=%s hashes=%s\n",
 				humanize.Comma(writeLeaves),
 				humanize.Comma(int64(float64(writeLeaves)/writeTime.Seconds())),
 				time.Duration(int64(writeTime)/writeLeaves),
 				writeTime.Round(time.Millisecond),
+				humanize.Comma(hashCount),
 			)
 		}
 
@@ -158,7 +161,6 @@ func TestTree_Hash(t *testing.T) {
 	var err error
 
 	tmpDir := t.TempDir()
-	//tmpDir := "/tmp/iavl-test"
 	t.Logf("levelDb tmpDir: %s\n", tmpDir)
 
 	require.NoError(t, err)
@@ -187,8 +189,7 @@ func TestTree_Hash(t *testing.T) {
 
 func TestTree_Build_Load(t *testing.T) {
 	// build the initial version of the tree with periodic checkpoints
-	//tmpDir := t.TempDir()
-	tmpDir := "/tmp/iavl-v2-test"
+	tmpDir := t.TempDir()
 	opts := testutil.NewTreeBuildOptions().With10_000()
 	multiTree := NewMultiTree(tmpDir, TreeOptions{CheckpointInterval: 4000, HeightFilter: 0, StateStorage: false})
 	itrs, ok := opts.Iterator.(*bench.ChangesetIterators)
@@ -215,86 +216,58 @@ func TestTree_Build_Load(t *testing.T) {
 	opts.UntilHash = "3a037f8dd67a5e1a9ef83a53b81c619c9ac0233abee6f34a400fb9b9dfbb4f8d"
 	testTreeBuild(t, mt, opts)
 	require.NoError(t, mt.Close())
-
-	t.Log("export the tree at version 12,000 and import it into a sql db in pre-order")
-	traverseOrder := PreOrder
-	restorePreOrderMt := NewMultiTree(t.TempDir(), TreeOptions{CheckpointInterval: 4000})
-	for sk, tree := range multiTree.Trees {
-		require.NoError(t, restorePreOrderMt.MountTree(sk))
-		exporter := tree.Export(traverseOrder)
-
-		restoreTree := restorePreOrderMt.Trees[sk]
-		_, err := restoreTree.sql.WriteSnapshot(context.Background(), tree.Version(), exporter.Next, SnapshotOptions{WriteCheckpoint: true, TraverseOrder: traverseOrder})
-		require.NoError(t, err)
-		require.NoError(t, restoreTree.LoadSnapshot(tree.Version(), traverseOrder))
-	}
-	require.NoError(t, restorePreOrderMt.Close())
-
-	t.Log("export the tree at version 12,000 and import it into a sql db in post-order")
-	traverseOrder = PostOrder
-	restorePostOrderMt := NewMultiTree(t.TempDir(), TreeOptions{CheckpointInterval: 4000})
-	for sk, tree := range multiTree.Trees {
-		require.NoError(t, restorePostOrderMt.MountTree(sk))
-		exporter := tree.Export(traverseOrder)
-
-		restoreTree := restorePostOrderMt.Trees[sk]
-		_, err := restoreTree.sql.WriteSnapshot(context.Background(), tree.Version(), exporter.Next, SnapshotOptions{WriteCheckpoint: true, TraverseOrder: traverseOrder})
-		require.NoError(t, err)
-		require.NoError(t, restoreTree.LoadSnapshot(tree.Version(), traverseOrder))
-	}
-	require.Equal(t, restorePostOrderMt.Hash(), restorePreOrderMt.Hash())
-
-	t.Log("build tree to version 20,000 and verify hash")
-	require.NoError(t, opts.Iterator.Next())
-	require.Equal(t, int64(12_001), opts.Iterator.Version())
-	opts.Until = 20_000
-	opts.UntilHash = "25907b193c697903218d92fa70a87ef6cdd6fa5b9162d955a4d70a9d5d2c4824"
-	testTreeBuild(t, restorePostOrderMt, opts)
-	require.NoError(t, restorePostOrderMt.Close())
 }
 
 // pre-requisites for the 2 tests below:
 // $ go run ./cmd gen tree --db /tmp/iavl-v2 --limit 1 --type osmo-like-many
 // $ go run ./cmd snapshot --db /tmp/iavl-v2 --version 1
 // mkdir -p /tmp/osmo-like-many/v2 && go run ./cmd gen emit --start 2 --limit 5000 --type osmo-like-many --out /tmp/osmo-like-many/v2
 func TestOsmoLike_HotStart(t *testing.T) {
-	tmpDir := "/tmp/iavl-v2"
-	// logDir := "/tmp/osmo-like-many-v2"
-	logDir := "/Users/mattk/src/scratch/osmo-like-many/v2"
+	tmpDir := "/Users/mattk/.costor/iavl-v2"
+	logDir := "/Users/mattk/src/devmos/osmo-like-many/v2"
+
 	pool := NewNodePool()
-	multiTree, err := ImportMultiTree(pool, 1, tmpDir, TreeOptions{HeightFilter: 0, StateStorage: false})
+	multiTree, err := ImportMultiTree(pool, 1, tmpDir, TreeOptions{
+		HeightFilter:       0,
+		StateStorage:       false,
+		CheckpointInterval: 1001,
+	})
 	require.NoError(t, err)
 	require.NotNil(t, multiTree)
 	opts := testutil.CompactedChangelogs(logDir)
 	opts.SampleRate = 250_000
 
-	opts.Until = 1_000
-	opts.UntilHash = "557663181d9ab97882ecfc6538e3b4cfe31cd805222fae905c4b4f4403ca5cda"
+	// opts.Until = 1_000
+	// opts.UntilHash = "557663181d9ab97882ecfc6538e3b4cfe31cd805222fae905c4b4f4403ca5cda"
+	opts.Until = 500
+	opts.UntilHash = "2670bd5767e70f2bf9e4f723b5f205759e39afdb5d8cfb6b54a4a3ecc27a1377"
 
 	testTreeBuild(t, multiTree, opts)
 }
 
 func TestOsmoLike_ColdStart(t *testing.T) {
-	tmpDir := "/tmp/iavl-v2"
+	tmpDir := "/Users/mattk/.costor/iavl-v2"
+	logDir := "/Users/mattk/src/devmos/osmo-like-many/v2"
 
 	treeOpts := DefaultTreeOptions()
-	treeOpts.CheckpointInterval = -1
-	treeOpts.CheckpointMemory = 1.5 * 1024 * 1024 * 1024
-	treeOpts.StateStorage = false
+	treeOpts.CheckpointInterval = 50
+	// treeOpts.CheckpointMemory = 1.5 * 1024 * 1024 * 1024
+	treeOpts.StateStorage = true
 	treeOpts.HeightFilter = 1
-	treeOpts.EvictionDepth = 16
+	// treeOpts.EvictionDepth = 22
 	treeOpts.MetricsProxy = newPrometheusMetricsProxy()
 	multiTree := NewMultiTree(tmpDir, treeOpts)
 	require.NoError(t, multiTree.MountTrees())
 	require.NoError(t, multiTree.LoadVersion(1))
-	// require.NoError(t, multiTree.WarmLeaves())
+	require.NoError(t, multiTree.WarmLeaves())
 
-	// logDir := "/tmp/osmo-like-many-v2"
-	opts := testutil.CompactedChangelogs("/Users/mattk/src/scratch/osmo-like-many/v2")
+	opts := testutil.CompactedChangelogs(logDir)
 	opts.SampleRate = 250_000
 
-	opts.Until = 1_000
-	opts.UntilHash = "557663181d9ab97882ecfc6538e3b4cfe31cd805222fae905c4b4f4403ca5cda"
+	// opts.Until = 1_000
+	// opts.UntilHash = "557663181d9ab97882ecfc6538e3b4cfe31cd805222fae905c4b4f4403ca5cda"
+	opts.Until = 500
+	opts.UntilHash = "2670bd5767e70f2bf9e4f723b5f205759e39afdb5d8cfb6b54a4a3ecc27a1377"
 
 	testTreeBuild(t, multiTree, opts)
 }