From 3391e737ad95927e8cab78d82a5316b2188600a9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 12:30:57 -0700 Subject: [PATCH 1/3] Add badger performance options --- storage/badger_storage.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/storage/badger_storage.go b/storage/badger_storage.go index 8c88c8ac..d1f29a98 100644 --- a/storage/badger_storage.go +++ b/storage/badger_storage.go @@ -150,6 +150,36 @@ func DefaultBadgerOptions(dir string) badger.Options { return opts } +// PerformanceBadgerOptions are performance geared +// BadgerDB options that use much more RAM than the +// default settings. +func PerformanceBadgerOptions(dir string) badger.Options { + opts := badger.DefaultOptions(dir) + + // By default, we do not compress the table at all. Doing so can + // significantly increase memory usage. + opts.Compression = DefaultCompressionMode + + // Don't load tables into memory. + opts.TableLoadingMode = options.LoadToRAM + opts.ValueLogLoadingMode = options.LoadToRAM + + // This option will have a significant effect the memory. If the level is kept + // in-memory, read are faster but the tables will be kept in memory. By default, + // this is set to false. + opts.KeepL0InMemory = true + + // We don't compact L0 on close as this can greatly delay shutdown time. + opts.CompactL0OnClose = false + + // LoadBloomsOnOpen=false will improve the db startup speed. This is also + // a waste to enable with a limited index cache size (as many of the loaded bloom + // filters will be immediately discarded from the cache). + opts.LoadBloomsOnOpen = false + + return opts +} + // NewBadgerStorage creates a new BadgerStorage. func NewBadgerStorage( ctx context.Context, From dd7cf1e843147fc83f2bcfee4f207631ed94e342 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 12:39:44 -0700 Subject: [PATCH 2/3] Fix invalid config --- storage/badger_storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/badger_storage.go b/storage/badger_storage.go index d1f29a98..293e5180 100644 --- a/storage/badger_storage.go +++ b/storage/badger_storage.go @@ -160,9 +160,9 @@ func PerformanceBadgerOptions(dir string) badger.Options { // significantly increase memory usage. opts.Compression = DefaultCompressionMode - // Don't load tables into memory. + // Load tables into memory and memory map value logs. opts.TableLoadingMode = options.LoadToRAM - opts.ValueLogLoadingMode = options.LoadToRAM + opts.ValueLogLoadingMode = options.MemoryMap // This option will have a significant effect the memory. If the level is kept // in-memory, read are faster but the tables will be kept in memory. By default, From 99f826bf56a6acf3a4e12f882fc2d761e4c61341 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 12:54:00 -0700 Subject: [PATCH 3/3] Use all mmap --- storage/badger_storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/badger_storage.go b/storage/badger_storage.go index 293e5180..4897ec8a 100644 --- a/storage/badger_storage.go +++ b/storage/badger_storage.go @@ -161,13 +161,13 @@ func PerformanceBadgerOptions(dir string) badger.Options { opts.Compression = DefaultCompressionMode // Load tables into memory and memory map value logs. - opts.TableLoadingMode = options.LoadToRAM + opts.TableLoadingMode = options.MemoryMap opts.ValueLogLoadingMode = options.MemoryMap // This option will have a significant effect the memory. If the level is kept // in-memory, read are faster but the tables will be kept in memory. By default, // this is set to false. - opts.KeepL0InMemory = true + opts.KeepL0InMemory = false // We don't compact L0 on close as this can greatly delay shutdown time. opts.CompactL0OnClose = false