Skip to content

Commit 2c5614f

Browse files
kocubinskitac0turtle
kocubinski
and
tac0turtle
authored
feat: iavl v2 (#872)
Co-authored-by: Marko <marbar3778@yahoo.com>
1 parent 013ea5c commit 2c5614f

39 files changed

+9302
-0
lines changed

‎v2/.gitignore

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
vendor
2+
.glide
3+
*.swp
4+
*.swo
5+
6+
# created in test code
7+
test.db
8+
9+
# profiling data
10+
*\.test
11+
cpu*.out
12+
mem*.out
13+
cpu*.pdf
14+
mem*.pdf
15+
16+
# IDE files
17+
.idea/*
18+
.vscode/*
19+
20+
go.work
21+
go.work.sum

‎v2/.golangci.yml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
run:
2+
tests: true
3+
# timeout for analysis, e.g. 30s, 5m, default is 1m
4+
timeout: 5m
5+
6+
linters:
7+
disable-all: true
8+
enable:
9+
- bodyclose
10+
- dogsled
11+
- errcheck
12+
- exportloopref
13+
- goconst
14+
- gocritic
15+
- gofumpt
16+
- gosec
17+
- gosimple
18+
- govet
19+
- ineffassign
20+
- misspell
21+
- nakedret
22+
- nolintlint
23+
- prealloc
24+
- revive
25+
- staticcheck
26+
- stylecheck
27+
- typecheck
28+
- unconvert
29+
- unparam
30+
- unused
31+
32+
linters-settings:
33+
nolintlint:
34+
allow-leading-space: true
35+
require-explanation: false
36+
require-specific: true
37+
38+
issues:
39+
exclude-rules:
40+
- text: "Use of weak random number generator"
41+
linters:
42+
- gosec
43+
- text: "comment on exported var"
44+
linters:
45+
- golint
46+
- text: "don't use an underscore in package name"
47+
linters:
48+
- golint
49+
- text: "should be written without leading space as"
50+
linters:
51+
- nolintlint
52+
- text: "ST1003:"
53+
linters:
54+
- stylecheck
55+
# FIXME: Disabled until golangci-lint updates stylecheck with this fix:
56+
# https://github.com/dominikh/go-tools/issues/389
57+
- text: "ST1016:"
58+
linters:
59+
- stylecheck
60+
- path: "migrations"
61+
text: "SA1019:"
62+
linters:
63+
- staticcheck
64+
65+
max-issues-per-linter: 10000
66+
max-same-issues: 10000

‎v2/cmd/gen/gen.go

+234
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
package gen
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"sync"
7+
"time"
8+
9+
"github.com/cosmos/iavl-bench/bench"
10+
"github.com/cosmos/iavl/v2"
11+
"github.com/cosmos/iavl/v2/testutil"
12+
"github.com/dustin/go-humanize"
13+
"github.com/kocubinski/costor-api/compact"
14+
"github.com/kocubinski/costor-api/core"
15+
"github.com/rs/zerolog"
16+
zlog "github.com/rs/zerolog/log"
17+
"github.com/spf13/cobra"
18+
)
19+
20+
var log = zlog.Output(zerolog.ConsoleWriter{
21+
Out: os.Stderr,
22+
TimeFormat: time.Stamp,
23+
})
24+
25+
func Command() *cobra.Command {
26+
cmd := &cobra.Command{
27+
Use: "gen",
28+
Short: "generate changesets",
29+
}
30+
31+
cmd.AddCommand(emitCommand(), treeCommand())
32+
33+
return cmd
34+
}
35+
36+
func getChangesetIterator(typ string) (bench.ChangesetIterator, error) {
37+
switch typ {
38+
case "osmo-like":
39+
return testutil.OsmoLike().Iterator, nil
40+
case "osmo-like-many":
41+
return testutil.OsmoLikeManyTrees().Iterator, nil
42+
case "height-zero":
43+
return testutil.NewTreeBuildOptions().Iterator, nil
44+
default:
45+
return nil, fmt.Errorf("unknown generator type %s", typ)
46+
}
47+
}
48+
49+
func emitCommand() *cobra.Command {
50+
var (
51+
typ string
52+
out string
53+
start int
54+
limit int
55+
)
56+
cmd := &cobra.Command{
57+
Use: "emit",
58+
Short: "emit generated changesets to disk",
59+
RunE: func(cmd *cobra.Command, args []string) error {
60+
itr, err := getChangesetIterator(typ)
61+
if err != nil {
62+
return err
63+
}
64+
ctx := core.Context{Context: cmd.Context()}
65+
66+
stream := compact.StreamingContext{
67+
In: make(chan compact.Sequenced),
68+
Context: ctx,
69+
OutDir: out,
70+
MaxFileSize: 100 * 1024 * 1024,
71+
}
72+
73+
var wg sync.WaitGroup
74+
wg.Add(1)
75+
go func() {
76+
stats, err := stream.Compact()
77+
if err != nil {
78+
log.Fatal().Err(err).Msg("failed to compact")
79+
}
80+
log.Info().Msgf(stats.Report())
81+
wg.Done()
82+
}()
83+
84+
var cnt int64
85+
for ; itr.Valid(); err = itr.Next() {
86+
if err != nil {
87+
return err
88+
}
89+
if limit > 0 && itr.Version() > int64(limit) {
90+
break
91+
}
92+
nodes := itr.Nodes()
93+
for ; nodes.Valid(); err = nodes.Next() {
94+
cnt++
95+
96+
if itr.Version() < int64(start) {
97+
if cnt%5_000_000 == 0 {
98+
log.Info().Msgf("fast forward version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
99+
}
100+
continue
101+
}
102+
103+
if cnt%500_000 == 0 {
104+
log.Info().Msgf("version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
105+
}
106+
107+
select {
108+
case <-cmd.Context().Done():
109+
close(stream.In)
110+
wg.Wait()
111+
return nil
112+
default:
113+
}
114+
115+
if err != nil {
116+
return err
117+
}
118+
stream.In <- nodes.GetNode()
119+
}
120+
}
121+
close(stream.In)
122+
wg.Wait()
123+
124+
return nil
125+
},
126+
}
127+
128+
cmd.Flags().StringVar(&typ, "type", "", "the type of changeset to generate")
129+
if err := cmd.MarkFlagRequired("type"); err != nil {
130+
panic(err)
131+
}
132+
cmd.Flags().StringVar(&out, "out", "", "the directory to write changesets to")
133+
if err := cmd.MarkFlagRequired("out"); err != nil {
134+
panic(err)
135+
}
136+
cmd.Flags().IntVar(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
137+
cmd.Flags().IntVar(&start, "start", 1, "the version (inclusive) to start generation at")
138+
139+
return cmd
140+
}
141+
142+
func treeCommand() *cobra.Command {
143+
var (
144+
dbPath string
145+
genType string
146+
limit int64
147+
)
148+
cmd := &cobra.Command{
149+
Use: "tree",
150+
Short: "build and save a Tree to disk, taking generated changesets as input",
151+
RunE: func(cmd *cobra.Command, args []string) error {
152+
multiTree := iavl.NewMultiTree(dbPath, iavl.TreeOptions{StateStorage: true})
153+
defer func(mt *iavl.MultiTree) {
154+
err := mt.Close()
155+
if err != nil {
156+
log.Error().Err(err).Msg("failed to close db")
157+
}
158+
}(multiTree)
159+
160+
itr, err := getChangesetIterator(genType)
161+
if err != nil {
162+
return err
163+
}
164+
165+
var i int64
166+
var lastHash []byte
167+
var lastVersion int64
168+
start := time.Now()
169+
for ; itr.Valid(); err = itr.Next() {
170+
if err != nil {
171+
return err
172+
}
173+
if limit > -1 && itr.Version() > limit {
174+
break
175+
}
176+
177+
changeset := itr.Nodes()
178+
for ; changeset.Valid(); err = changeset.Next() {
179+
if err != nil {
180+
return err
181+
}
182+
node := changeset.GetNode()
183+
key := node.Key
184+
185+
tree, ok := multiTree.Trees[node.StoreKey]
186+
if !ok {
187+
if err = multiTree.MountTree(node.StoreKey); err != nil {
188+
return err
189+
}
190+
tree = multiTree.Trees[node.StoreKey]
191+
}
192+
if node.Delete {
193+
_, _, err = tree.Remove(key)
194+
if err != nil {
195+
return err
196+
}
197+
} else {
198+
_, err = tree.Set(key, node.Value)
199+
if err != nil {
200+
return err
201+
}
202+
}
203+
204+
i++
205+
if i%100_000 == 0 {
206+
log.Info().Msgf("leaves=%s dur=%s rate=%s version=%d",
207+
humanize.Comma(i),
208+
time.Since(start),
209+
humanize.Comma(int64(100_000/time.Since(start).Seconds())),
210+
itr.Version(),
211+
)
212+
start = time.Now()
213+
}
214+
}
215+
216+
lastHash, lastVersion, err = multiTree.SaveVersionConcurrently()
217+
if err != nil {
218+
return err
219+
}
220+
}
221+
222+
log.Info().Msgf("last version=%d hash=%x", lastVersion, lastHash)
223+
224+
return nil
225+
},
226+
}
227+
cmd.Flags().StringVar(&genType, "type", "", "the type of changeset to generate")
228+
if err := cmd.MarkFlagRequired("type"); err != nil {
229+
panic(err)
230+
}
231+
cmd.Flags().StringVar(&dbPath, "db", "/tmp", "the path to the database")
232+
cmd.Flags().Int64Var(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
233+
return cmd
234+
}

‎v2/cmd/latest.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package main
2+
3+
import (
4+
"github.com/cosmos/iavl/v2"
5+
"github.com/spf13/cobra"
6+
)
7+
8+
func latestCommand() *cobra.Command {
9+
var (
10+
dbPath string
11+
version int64
12+
)
13+
cmd := &cobra.Command{
14+
Use: "latest",
15+
Short: "fill the latest table with the latest version of leaf nodes in a tree",
16+
RunE: func(cmd *cobra.Command, args []string) error {
17+
paths, err := iavl.FindDbsInPath(dbPath)
18+
if err != nil {
19+
return err
20+
}
21+
var (
22+
pool = iavl.NewNodePool()
23+
done = make(chan struct{})
24+
errors = make(chan error)
25+
cnt = 0
26+
)
27+
for _, path := range paths {
28+
cnt++
29+
sqlOpts := iavl.SqliteDbOptions{Path: path}
30+
sql, err := iavl.NewSqliteDb(pool, sqlOpts)
31+
if err != nil {
32+
return err
33+
}
34+
tree := iavl.NewTree(sql, pool, iavl.TreeOptions{})
35+
if err = tree.LoadVersion(version); err != nil {
36+
return err
37+
}
38+
go func() {
39+
fillErr := tree.WriteLatestLeaves()
40+
if fillErr != nil {
41+
errors <- fillErr
42+
}
43+
fillErr = tree.Close()
44+
if fillErr != nil {
45+
errors <- fillErr
46+
}
47+
done <- struct{}{}
48+
}()
49+
}
50+
for i := 0; i < cnt; i++ {
51+
select {
52+
case <-done:
53+
continue
54+
case err := <-errors:
55+
return err
56+
}
57+
}
58+
return nil
59+
},
60+
}
61+
cmd.Flags().StringVar(&dbPath, "db", "", "the path to the db to fill the latest table for")
62+
if err := cmd.MarkFlagRequired("db"); err != nil {
63+
panic(err)
64+
}
65+
cmd.Flags().Int64Var(&version, "version", 0, "version to fill from")
66+
if err := cmd.MarkFlagRequired("version"); err != nil {
67+
panic(err)
68+
}
69+
return cmd
70+
}

0 commit comments

Comments
 (0)