Skip to content
This repository has been archived by the owner on May 24, 2022. It is now read-only.

Commit

Permalink
[fileflatdb] close files after opening
Browse files Browse the repository at this point in the history
  • Loading branch information
@miguelmota
miguelmota committed Feb 20, 2019
1 parent 09e8517 commit 479b2e2
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 53 deletions.
3 changes: 1 addition & 2 deletions client/p2p/peers/peers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,7 @@ func (p *Peers) Log(event peerstypes.EventEnum, iface interface{}) error {
return ErrNilEvent
}

// TODO: log pinfo? or peer id?
logger.Infof("[peers] peer event: %s, interface: %v", event.String(), iface)
logger.Infof("[peers] peer event: %s", event.String())

p.handleEvent(event, iface)

Expand Down
2 changes: 1 addition & 1 deletion cmd/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func setup() {
Short: "Start node",
Long: `Starts the Polkadot node`,
RunE: func(cmd *cobra.Command, args []string) error {
priv, pub, err := ic.GenerateKeyPairWithReader(ic.RSA, 256, rand.Reader)
priv, pub, err := ic.GenerateKeyPairWithReader(ic.RSA, 2048, rand.Reader)
if err != nil {
return err
}
Expand Down
26 changes: 17 additions & 9 deletions common/fileflatdb/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ func (c *Cache) GetCachedBranch(branchAt int64) []byte {
branch, found := c.lruBranch[branchAt]
if !found {
branch = make([]byte, branchSize)
fd := os.NewFile(c.fd(), "temp")
_, err := fd.ReadAt(branch, branchAt)
if err != nil {
f := c.openFile()
defer f.Close()

if _, err := f.ReadAt(branch, branchAt); err != nil {
log.Fatalf("[fileflatdb/cache] get cached branch error: %v", err)
}

Expand All @@ -56,9 +57,11 @@ func (c *Cache) GetCachedData(dataAt int64, length int64) []byte {
data, found := c.lruData[dataAt]
if !found {
data = make([]byte, length)
fd := os.NewFile(c.fd(), "temp")
_, err := fd.ReadAt(data, dataAt)
if err != nil {

f := c.openFile()
defer f.Close()

if _, err := f.ReadAt(data, dataAt); err != nil {
log.Fatalf("[fileflatdb/cache] get cached data error: %v", err)
}

Expand All @@ -68,8 +71,8 @@ func (c *Cache) GetCachedData(dataAt int64, length int64) []byte {
return data
}

// fd ...
func (c *Cache) fd() uintptr {
// openFile ...
func (c *Cache) openFile() *os.File {
filepath := c.file.path

if _, err := os.Stat(filepath); os.IsNotExist(err) {
Expand All @@ -81,5 +84,10 @@ func (c *Cache) fd() uintptr {
log.Fatalf("[fileflatdb/cache] error opening file: %v", err)
}

return file.Fd()
return file
}

// fd ...
func (c *Cache) fd() uintptr {
return c.file.fd
}
36 changes: 18 additions & 18 deletions common/fileflatdb/compact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func NewCompact(file string) *Compact {
// Maintain ...
func (c *Compact) Maintain(fn *db.ProgressCB) {
if c.fd != -1 {
log.Fatal("database cannot be open for compacting")
log.Fatalln("[fileflatdb/compact] database cannot be open for compacting")
}

start := time.Now().Unix()
Expand All @@ -44,23 +44,23 @@ func (c *Compact) Maintain(fn *db.ProgressCB) {

newStat, err := os.Stat(newFile)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to stat new file: %s\n", err)
}
oldStat, err := os.Stat(c.file)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to stat old file: %s\n", err)
}
percentage := 100 * (newStat.Size() / oldStat.Size())
sizeMB := newStat.Size() / (1024 * 1024)
elapsed := time.Now().Unix() - start

err = os.Remove(c.file)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to remove file: %s\n", err)
}
err = os.Rename(newFile, c.file)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to rename file: %s\n", err)
}

log.Printf("compacted in %d, %dk keys, %dMB (%d%%)", elapsed, keys/1e3, sizeMB, percentage)
Expand All @@ -74,13 +74,13 @@ func (c *Compact) Open(file string, startEmpty bool) uintptr {
data := make([]byte, branchSize)
err := ioutil.WriteFile(file, data, os.ModePerm)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed write to file after opening: %s\n", err)
}
}

f, err := os.OpenFile(file, os.O_RDONLY, 0755)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed write to open file: %s\n", err)
}

return f.Fd()
Expand Down Expand Up @@ -112,7 +112,7 @@ func (c *Compact) doCompact(keys *int, percent int, fn db.ProgressCB, newFd, old

c.doCompact(keys, percent, fn, newFd, oldFd, int(headerAt), int(dataAt.Uint64()), depth+1)
} else {
log.Fatalf("Unknown entry type %d", entryType)
log.Fatalf("[fileflatdb/compact] unknown entry type %d\n", entryType)
}

var isCompleted bool
Expand Down Expand Up @@ -148,7 +148,7 @@ func (c *Compact) CompactReadEntry(fd uintptr, at int, index int) []byte {
file := os.NewFile(fd, "temp")
_, err := file.ReadAt(entry, int64(entryAt))
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to read entry: %s\n", err)
}

return entry
Expand All @@ -160,7 +160,7 @@ func (c *Compact) CompactReadKey(fd uintptr, at int64) ([]byte, []byte) {
file := os.NewFile(fd, "temp")
_, err := file.ReadAt(key, at)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to read key: %s\n", err)
}

valueLength := new(big.Int)
Expand All @@ -172,7 +172,7 @@ func (c *Compact) CompactReadKey(fd uintptr, at int64) ([]byte, []byte) {
value := make([]byte, valueLength.Uint64())
_, err = file.ReadAt(value, int64(valueAt.Uint64()))
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to read value: %s\n", err)
}

return key, value
Expand All @@ -183,7 +183,7 @@ func (c *Compact) CompactWriteKey(fd uintptr, key, value []byte) int64 {
file := os.NewFile(fd, "temp")
stat, err := file.Stat()
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to stat file: %s\n", err)
}
valueAt := stat.Size()
keyAt := valueAt + int64(len(value))
Expand All @@ -192,11 +192,11 @@ func (c *Compact) CompactWriteKey(fd uintptr, key, value []byte) int64 {

_, err = file.WriteAt(value, valueAt)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to write value: %s\n", err)
}
_, err = file.WriteAt(key, keyAt)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to compact write key: %s\n", err)
}

return keyAt
Expand All @@ -213,7 +213,7 @@ func (c *Compact) CompactUpdateLink(fd uintptr, at int, index int, pointer int64
file := os.NewFile(fd, "temp")
_, err := file.WriteAt(entry, int64(entryAt))
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to compact update link write entry: %s\n", err)
}
}

Expand All @@ -222,14 +222,14 @@ func (c *Compact) CompactWriteHeader(fd uintptr, at int, index int) int64 {
file := os.NewFile(fd, "temp")
stat, err := file.Stat()
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to stat file: %s\n", err)
}
headerAt := stat.Size()

header := make([]byte, branchSize)
_, err = file.WriteAt(header, headerAt)
if err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to write header: %s\n", err)
}

c.CompactUpdateLink(fd, at, index, headerAt, SlotBranch)
Expand All @@ -240,6 +240,6 @@ func (c *Compact) CompactWriteHeader(fd uintptr, at int, index int) int64 {
func closeFd(fd uintptr) {
// close file descriptor
if err := syscall.Close(int(fd)); err != nil {
log.Fatal(err)
log.Fatalf("[fileflatdb/compact] failed to close file descriptor: %s\n", err)
}
}
52 changes: 29 additions & 23 deletions common/fileflatdb/impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (i *Impl) FindKey(key *NibbleBuffer, doCreate bool, keyIndex, branchAt int6
case SlotLeaf:
return i.RetrieveLeaf(doCreate, branch, int(branchAt), entryIndex, int(keyIndex), key)
default:
log.Fatalf("Unhandled entry type %v\n", entryType)
log.Fatalf("[fileflatdb] unhandled entry type %v\n", entryType)
}
return nil
}
Expand Down Expand Up @@ -152,10 +152,10 @@ func (i *Impl) WriteValue(keyAt int, keyValue []byte, value []byte) *Value {

writeUIntBE(keyValue, int64(len(value)), int64(keySize), int64(uintSize))
writeUIntBE(keyValue, int64(valueAt), int64(keySize)+int64(uintSize), int64(uintSize))
file := os.NewFile(i.fd(), "fileflatdb")
_, err := file.WriteAt(keyValue[keySize:keySize+2*uintSize], int64(keyAt)+int64(keySize))
if err != nil {
log.Fatal(err)
file := i.openFile()
defer file.Close()
if _, err := file.WriteAt(keyValue[keySize:keySize+2*uintSize], int64(keyAt)+int64(keySize)); err != nil {
log.Fatalf("[fileflatdb] failed to write value: %v\n", err)
}

return &Value{
Expand Down Expand Up @@ -232,10 +232,10 @@ func (i *Impl) WriteNewBranch(branch []byte, branchAt int64, entryIndex int64, k

writeUIntBE(branch, int64(newBranchAt), int64(entryIndex)+1, int64(uintSize))

file := os.NewFile(i.fd(), "fileflatdb")
_, err := file.WriteAt(branch[entryIndex:entryIndex+int64(entrySize)], int64(branchAt)+int64(entryIndex))
if err != nil {
log.Fatal(err)
file := i.openFile()
defer file.Close()
if _, err := file.WriteAt(branch[entryIndex:entryIndex+int64(entrySize)], int64(branchAt)+int64(entryIndex)); err != nil {
log.Fatalf("[fileflatdb] failed to write branch: %v\n", err)
}

return &Key{
Expand All @@ -251,10 +251,10 @@ func (i *Impl) WriteNewLeaf(branch []byte, branchAt int64, entryIndex int64, key

branch[entryIndex] = byte(SlotLeaf)
writeUIntBE(branch, int64(newKey.KeyAt), int64(entryIndex)+1, int64(uintSize))
file := os.NewFile(i.fd(), "fileflatdb")
_, err := file.WriteAt(branch[entryIndex:entryIndex+int64(entrySize)], int64(branchAt)+int64(entryIndex))
if err != nil {
log.Fatal(err)
file := i.openFile()
defer file.Close()
if _, err := file.WriteAt(branch[entryIndex:entryIndex+int64(entrySize)], int64(branchAt)+int64(entryIndex)); err != nil {
log.Fatalf("[fileflatdb] failed to write leaf: %v\n", err)
}

return &Key{
Expand All @@ -267,9 +267,9 @@ func (i *Impl) WriteNewLeaf(branch []byte, branchAt int64, entryIndex int64, key
// WriteUpdatedBuffer ...
func (i *Impl) WriteUpdatedBuffer(buffer []byte, bufferAt int64) int64 {

file := os.NewFile(i.fd(), "fileflatdb")
_, err := file.WriteAt(buffer, bufferAt)
if err != nil {
file := i.openFile()
defer file.Close()
if _, err := file.WriteAt(buffer, bufferAt); err != nil {
log.Fatal(err)
}

Expand All @@ -281,10 +281,10 @@ func (i *Impl) WriteUpdatedBuffer(buffer []byte, bufferAt int64) int64 {
func (i *Impl) WriteNewBuffer(buffer []byte, withCache bool) int64 {
startAt := i.cache.file.fileSize

file := os.NewFile(i.fd(), "fileflatdb")
_, err := file.WriteAt(buffer, startAt)
if err != nil {
log.Fatalf("error writing new buffer to file: %v \n", err)
file := i.openFile()
defer file.Close()
if _, err := file.WriteAt(buffer, startAt); err != nil {
log.Fatalf("[fileflatdb] error writing new buffer to file: %v\n", err)
}

if withCache {
Expand Down Expand Up @@ -313,8 +313,8 @@ func (i *Impl) WriteNewBuffers(buffers [][]byte) int64 {
return i.WriteNewBuffer(concenatedBuffers, false)
}

// fd ...
func (i *Impl) fd() uintptr {
// openFile ...
func (i *Impl) openFile() *os.File {
filepath := i.cache.file.path

if _, err := os.Stat(filepath); os.IsNotExist(err) {
Expand All @@ -325,5 +325,11 @@ func (i *Impl) fd() uintptr {
if err != nil {
log.Fatalf("[fileflatdb] error opening file: %v", err)
}
return file.Fd()

return file
}

// fd ...
func (i *Impl) fd() uintptr {
return i.cache.file.fd
}

0 comments on commit 479b2e2

Please sign in to comment.