Skip to content

Commit 5b4efe3

Browse files
kocubinskikocubinski
committed
Initial version of v2 work
1 parent 08b8626 commit 5b4efe3

40 files changed

+8531
-0
lines changed

‎v2/.gitignore

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
vendor
2+
.glide
3+
*.swp
4+
*.swo
5+
6+
# created in test code
7+
test.db
8+
9+
# profiling data
10+
*\.test
11+
cpu*.out
12+
mem*.out
13+
cpu*.pdf
14+
mem*.pdf
15+
16+
# IDE files
17+
.idea/*
18+
.vscode/*
19+
20+
go.work
21+
go.work.sum

‎v2/.golangci.yml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
run:
2+
tests: true
3+
# timeout for analysis, e.g. 30s, 5m, default is 1m
4+
timeout: 5m
5+
6+
linters:
7+
disable-all: true
8+
enable:
9+
- bodyclose
10+
- dogsled
11+
- errcheck
12+
- exportloopref
13+
- goconst
14+
- gocritic
15+
- gofumpt
16+
- gosec
17+
- gosimple
18+
- govet
19+
- ineffassign
20+
- misspell
21+
- nakedret
22+
- nolintlint
23+
- prealloc
24+
- revive
25+
- staticcheck
26+
- stylecheck
27+
- typecheck
28+
- unconvert
29+
- unparam
30+
- unused
31+
32+
linters-settings:
33+
nolintlint:
34+
allow-leading-space: true
35+
require-explanation: false
36+
require-specific: true
37+
38+
issues:
39+
exclude-rules:
40+
- text: "Use of weak random number generator"
41+
linters:
42+
- gosec
43+
- text: "comment on exported var"
44+
linters:
45+
- golint
46+
- text: "don't use an underscore in package name"
47+
linters:
48+
- golint
49+
- text: "should be written without leading space as"
50+
linters:
51+
- nolintlint
52+
- text: "ST1003:"
53+
linters:
54+
- stylecheck
55+
# FIXME: Disabled until golangci-lint updates stylecheck with this fix:
56+
# https://github.com/dominikh/go-tools/issues/389
57+
- text: "ST1016:"
58+
linters:
59+
- stylecheck
60+
- path: "migrations"
61+
text: "SA1019:"
62+
linters:
63+
- staticcheck
64+
65+
max-issues-per-linter: 10000
66+
max-same-issues: 10000

‎v2/cache.go

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package iavl
2+
3+
import (
4+
"sync"
5+
6+
metrics2 "github.com/cosmos/iavl/v2/metrics"
7+
"github.com/dustin/go-humanize"
8+
)
9+
10+
type NodeCache struct {
11+
cache map[NodeKey]*Node
12+
nextCache map[NodeKey]*Node
13+
pool sync.Pool
14+
nodes []*Node
15+
16+
metrics *metrics2.TreeMetrics
17+
}
18+
19+
func NewNodeCache(metrics *metrics2.TreeMetrics) *NodeCache {
20+
return &NodeCache{
21+
nextCache: make(map[NodeKey]*Node),
22+
cache: make(map[NodeKey]*Node),
23+
metrics: metrics,
24+
//missCount: metrics.Default.NewCounter("node_cache.miss"),
25+
//hitCount: metrics.Default.NewCounter("node_cache.hit"),
26+
}
27+
}
28+
29+
func (nc *NodeCache) Swap() {
30+
l := log.With().Str("module", "node_cache").Logger()
31+
nc.cache, nc.nextCache = nc.nextCache, nc.cache
32+
l.Info().Msgf("emptying %s cache=%s",
33+
humanize.Comma(int64(len(nc.nextCache))),
34+
humanize.Comma(int64(len(nc.cache))),
35+
)
36+
for _, n := range nc.nextCache {
37+
nc.pool.Put(n)
38+
}
39+
nc.nextCache = make(map[NodeKey]*Node)
40+
nc.metrics.CacheHit = 0
41+
nc.metrics.CacheMiss = 0
42+
}
43+
44+
func (nc *NodeCache) Get(nodeKey NodeKey) *Node {
45+
n, ok := nc.cache[nodeKey]
46+
if ok {
47+
nc.metrics.CacheHit++
48+
} else {
49+
nc.metrics.CacheMiss++
50+
}
51+
return n
52+
}
53+
54+
func (nc *NodeCache) SetThis(node *Node) {
55+
nc.cache[node.nodeKey] = node
56+
}
57+
58+
func (nc *NodeCache) Set(node *Node) {
59+
if len(nc.nextCache) > 10_000_000 {
60+
return
61+
}
62+
nc.nextCache[node.nodeKey] = node
63+
}

‎v2/cmd/gen/gen.go

+234
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
package gen
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"sync"
7+
"time"
8+
9+
"github.com/cosmos/iavl-bench/bench"
10+
"github.com/cosmos/iavl/v2"
11+
"github.com/cosmos/iavl/v2/testutil"
12+
"github.com/dustin/go-humanize"
13+
"github.com/kocubinski/costor-api/compact"
14+
"github.com/kocubinski/costor-api/core"
15+
"github.com/rs/zerolog"
16+
zlog "github.com/rs/zerolog/log"
17+
"github.com/spf13/cobra"
18+
)
19+
20+
var log = zlog.Output(zerolog.ConsoleWriter{
21+
Out: os.Stderr,
22+
TimeFormat: time.Stamp,
23+
})
24+
25+
func Command() *cobra.Command {
26+
cmd := &cobra.Command{
27+
Use: "gen",
28+
Short: "generate changesets",
29+
}
30+
31+
cmd.AddCommand(emitCommand(), treeCommand())
32+
33+
return cmd
34+
}
35+
36+
func getChangesetIterator(typ string) (bench.ChangesetIterator, error) {
37+
switch typ {
38+
case "osmo-like":
39+
return testutil.OsmoLike().Iterator, nil
40+
case "osmo-like-many":
41+
return testutil.OsmoLikeManyTrees().Iterator, nil
42+
case "height-zero":
43+
return testutil.NewTreeBuildOptions().Iterator, nil
44+
default:
45+
return nil, fmt.Errorf("unknown generator type %s", typ)
46+
}
47+
}
48+
49+
func emitCommand() *cobra.Command {
50+
var (
51+
typ string
52+
out string
53+
start int
54+
limit int
55+
)
56+
cmd := &cobra.Command{
57+
Use: "emit",
58+
Short: "emit generated changesets to disk",
59+
RunE: func(cmd *cobra.Command, args []string) error {
60+
itr, err := getChangesetIterator(typ)
61+
if err != nil {
62+
return err
63+
}
64+
ctx := core.Context{Context: cmd.Context()}
65+
66+
stream := compact.StreamingContext{
67+
In: make(chan compact.Sequenced),
68+
Context: ctx,
69+
OutDir: out,
70+
MaxFileSize: 100 * 1024 * 1024,
71+
}
72+
73+
var wg sync.WaitGroup
74+
wg.Add(1)
75+
go func() {
76+
stats, err := stream.Compact()
77+
if err != nil {
78+
log.Fatal().Err(err).Msg("failed to compact")
79+
}
80+
log.Info().Msgf(stats.Report())
81+
wg.Done()
82+
}()
83+
84+
var cnt int64
85+
for ; itr.Valid(); err = itr.Next() {
86+
if err != nil {
87+
return err
88+
}
89+
if limit > 0 && itr.Version() > int64(limit) {
90+
break
91+
}
92+
nodes := itr.Nodes()
93+
for ; nodes.Valid(); err = nodes.Next() {
94+
cnt++
95+
96+
if itr.Version() < int64(start) {
97+
if cnt%5_000_000 == 0 {
98+
log.Info().Msgf("fast forward version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
99+
}
100+
continue
101+
}
102+
103+
if cnt%500_000 == 0 {
104+
log.Info().Msgf("version=%d nodes=%s", itr.Version(), humanize.Comma(cnt))
105+
}
106+
107+
select {
108+
case <-cmd.Context().Done():
109+
close(stream.In)
110+
wg.Wait()
111+
return nil
112+
default:
113+
}
114+
115+
if err != nil {
116+
return err
117+
}
118+
stream.In <- nodes.GetNode()
119+
}
120+
}
121+
close(stream.In)
122+
wg.Wait()
123+
124+
return nil
125+
},
126+
}
127+
128+
cmd.Flags().StringVar(&typ, "type", "", "the type of changeset to generate")
129+
if err := cmd.MarkFlagRequired("type"); err != nil {
130+
panic(err)
131+
}
132+
cmd.Flags().StringVar(&out, "out", "", "the directory to write changesets to")
133+
if err := cmd.MarkFlagRequired("out"); err != nil {
134+
panic(err)
135+
}
136+
cmd.Flags().IntVar(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
137+
cmd.Flags().IntVar(&start, "start", 1, "the version (inclusive) to start generation at")
138+
139+
return cmd
140+
}
141+
142+
func treeCommand() *cobra.Command {
143+
var (
144+
dbPath string
145+
genType string
146+
limit int64
147+
)
148+
cmd := &cobra.Command{
149+
Use: "tree",
150+
Short: "build and save a Tree to disk, taking generated changesets as input",
151+
RunE: func(cmd *cobra.Command, args []string) error {
152+
multiTree := iavl.NewMultiTree(dbPath, iavl.TreeOptions{StateStorage: true})
153+
defer func(mt *iavl.MultiTree) {
154+
err := mt.Close()
155+
if err != nil {
156+
log.Error().Err(err).Msg("failed to close db")
157+
}
158+
}(multiTree)
159+
160+
itr, err := getChangesetIterator(genType)
161+
if err != nil {
162+
return err
163+
}
164+
165+
var i int64
166+
var lastHash []byte
167+
var lastVersion int64
168+
start := time.Now()
169+
for ; itr.Valid(); err = itr.Next() {
170+
if err != nil {
171+
return err
172+
}
173+
if limit > -1 && itr.Version() > limit {
174+
break
175+
}
176+
177+
changeset := itr.Nodes()
178+
for ; changeset.Valid(); err = changeset.Next() {
179+
if err != nil {
180+
return err
181+
}
182+
node := changeset.GetNode()
183+
key := node.Key
184+
185+
tree, ok := multiTree.Trees[node.StoreKey]
186+
if !ok {
187+
if err = multiTree.MountTree(node.StoreKey); err != nil {
188+
return err
189+
}
190+
tree = multiTree.Trees[node.StoreKey]
191+
}
192+
if node.Delete {
193+
_, _, err = tree.Remove(key)
194+
if err != nil {
195+
return err
196+
}
197+
} else {
198+
_, err = tree.Set(key, node.Value)
199+
if err != nil {
200+
return err
201+
}
202+
}
203+
204+
i++
205+
if i%100_000 == 0 {
206+
log.Info().Msgf("leaves=%s dur=%s rate=%s version=%d",
207+
humanize.Comma(i),
208+
time.Since(start),
209+
humanize.Comma(int64(100_000/time.Since(start).Seconds())),
210+
itr.Version(),
211+
)
212+
start = time.Now()
213+
}
214+
}
215+
216+
lastHash, lastVersion, err = multiTree.SaveVersionConcurrently()
217+
if err != nil {
218+
return err
219+
}
220+
}
221+
222+
log.Info().Msgf("last version=%d hash=%x", lastVersion, lastHash)
223+
224+
return nil
225+
},
226+
}
227+
cmd.Flags().StringVar(&genType, "type", "", "the type of changeset to generate")
228+
if err := cmd.MarkFlagRequired("type"); err != nil {
229+
panic(err)
230+
}
231+
cmd.Flags().StringVar(&dbPath, "db", "/tmp", "the path to the database")
232+
cmd.Flags().Int64Var(&limit, "limit", -1, "the version (inclusive) to halt generation at. -1 means no limit")
233+
return cmd
234+
}

0 commit comments

Comments
 (0)