From 3f164e16d58fdea2aa178ae35168ab14edb52fc0 Mon Sep 17 00:00:00 2001 From: Ulimo Date: Tue, 19 Mar 2024 19:36:35 +0100 Subject: [PATCH] Add support to configure default page size in bplustree (#414) * Add support to configure default page size in bplustree * Default page size to 1024 --- .../StateManager/Internal/IStateClient.cs | 1 + .../Internal/Sync/StateManagerSyncClient.cs | 6 ++++++ .../StateManager/Internal/Sync/SyncStateClient.cs | 7 ++++++- .../StateManager/StateManagerOptions.cs | 2 ++ .../StateManager/StateManagerSync.cs | 4 ++-- src/FlowtideDotNet.Storage/Tree/BPlusTreeOptions.cs | 5 ++++- src/FlowtideDotNet.Storage/Tree/Internal/BPlusTree.cs | 10 +++++++--- 7 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/FlowtideDotNet.Storage/StateManager/Internal/IStateClient.cs b/src/FlowtideDotNet.Storage/StateManager/Internal/IStateClient.cs index 2dd20f4c0..2b8ede991 100644 --- a/src/FlowtideDotNet.Storage/StateManager/Internal/IStateClient.cs +++ b/src/FlowtideDotNet.Storage/StateManager/Internal/IStateClient.cs @@ -22,5 +22,6 @@ internal interface IStateClient ValueTask Commit(); void Delete(in long key); ValueTask Reset(bool clearMetadata); + int BPlusTreePageSize { get; } } } diff --git a/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/StateManagerSyncClient.cs b/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/StateManagerSyncClient.cs index 1f294235c..941cf7e74 100644 --- a/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/StateManagerSyncClient.cs +++ b/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/StateManagerSyncClient.cs @@ -37,6 +37,12 @@ public IStateManagerClient GetChildManager(string name) public async ValueTask> GetOrCreateTree(string name, BPlusTreeOptions options) { var stateClient = await CreateStateClient(name, new BPlusTreeSerializer(options.KeySerializer, options.ValueSerializer)); + + if (options.BucketSize == null) + { + options.BucketSize = stateClient.BPlusTreePageSize; + } + var tree = new BPlusTree(stateClient, options); await tree.InitializeAsync(); return tree; diff --git a/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/SyncStateClient.cs b/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/SyncStateClient.cs index def20b299..73bc1375d 100644 --- a/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/SyncStateClient.cs +++ b/src/FlowtideDotNet.Storage/StateManager/Internal/Sync/SyncStateClient.cs @@ -28,6 +28,7 @@ internal class SyncStateClient : StateClient, IStateClient options; private readonly bool useReadCache; + private readonly int m_bplusTreePageSize; private readonly ConcurrentDictionary m_modified; private readonly object m_lock = new object(); private readonly FlowtideDotNet.Storage.FileCache.FileCache m_fileCache; @@ -50,7 +51,8 @@ public SyncStateClient( StateClientOptions options, FileCacheOptions fileCacheOptions, Meter meter, - bool useReadCache) + bool useReadCache, + int bplusTreePageSize) { this.stateManager = stateManager; this.metadataId = metadataId; @@ -58,6 +60,7 @@ public SyncStateClient( this.session = session; this.options = options; this.useReadCache = useReadCache; + this.m_bplusTreePageSize = bplusTreePageSize; m_fileCache = new FlowtideDotNet.Storage.FileCache.FileCache(fileCacheOptions, name); m_modified = new ConcurrentDictionary(); m_fileCacheVersion = new ConcurrentDictionary(); @@ -79,6 +82,8 @@ public TMetadata? Metadata } } + public int BPlusTreePageSize => m_bplusTreePageSize; + public bool AddOrUpdate(in long key, V value) { lock (m_lock) diff --git a/src/FlowtideDotNet.Storage/StateManager/StateManagerOptions.cs b/src/FlowtideDotNet.Storage/StateManager/StateManagerOptions.cs index b8d3e553f..d7598452a 100644 --- a/src/FlowtideDotNet.Storage/StateManager/StateManagerOptions.cs +++ b/src/FlowtideDotNet.Storage/StateManager/StateManagerOptions.cs @@ -60,5 +60,7 @@ public class StateManagerOptions /// Useful if persistent storage is not on disk and instead in a remote location. /// public bool UseReadCache { get; set; } = false; + + public int DefaultBPlusTreePageSize { get; set; } = 1024; } } diff --git a/src/FlowtideDotNet.Storage/StateManager/StateManagerSync.cs b/src/FlowtideDotNet.Storage/StateManager/StateManagerSync.cs index 11f4a03bc..4dfd6009f 100644 --- a/src/FlowtideDotNet.Storage/StateManager/StateManagerSync.cs +++ b/src/FlowtideDotNet.Storage/StateManager/StateManagerSync.cs @@ -233,7 +233,7 @@ internal async ValueTask> CreateClientAsync(new ByteMemoryOwner(bytes), bytes.Length); var persistentSession = m_persistentStorage.CreateSession(); - var stateClient = new SyncStateClient(this, client, location, metadata, persistentSession, options, m_fileCacheOptions, meter, this.options.UseReadCache); + var stateClient = new SyncStateClient(this, client, location, metadata, persistentSession, options, m_fileCacheOptions, meter, this.options.UseReadCache, this.options.DefaultBPlusTreePageSize); lock (m_lock) { _stateClients.Add(client, stateClient); @@ -257,7 +257,7 @@ internal async ValueTask> CreateClientAsync(this, client, clientMetadataPageId, clientMetadata, session, options, m_fileCacheOptions, meter, this.options.UseReadCache); + var stateClient = new SyncStateClient(this, client, clientMetadataPageId, clientMetadata, session, options, m_fileCacheOptions, meter, this.options.UseReadCache, this.options.DefaultBPlusTreePageSize); _stateClients.Add(client, stateClient); return stateClient; } diff --git a/src/FlowtideDotNet.Storage/Tree/BPlusTreeOptions.cs b/src/FlowtideDotNet.Storage/Tree/BPlusTreeOptions.cs index 182ac4eaf..8f35eda06 100644 --- a/src/FlowtideDotNet.Storage/Tree/BPlusTreeOptions.cs +++ b/src/FlowtideDotNet.Storage/Tree/BPlusTreeOptions.cs @@ -14,7 +14,10 @@ namespace FlowtideDotNet.Storage.Tree { public class BPlusTreeOptions { - public int BucketSize { get; set; } = 64; + /// + /// Override the default page size. This should only be set if the operator works best with a specific size. + /// + public int? BucketSize { get; set; } public required IComparer Comparer { get; set; } diff --git a/src/FlowtideDotNet.Storage/Tree/Internal/BPlusTree.cs b/src/FlowtideDotNet.Storage/Tree/Internal/BPlusTree.cs index 639a6ece5..72fa57528 100644 --- a/src/FlowtideDotNet.Storage/Tree/Internal/BPlusTree.cs +++ b/src/FlowtideDotNet.Storage/Tree/Internal/BPlusTree.cs @@ -11,6 +11,7 @@ // limitations under the License. using FlowtideDotNet.Storage.StateManager.Internal; +using Microsoft.Extensions.Options; using System.Diagnostics; using System.Text; @@ -25,14 +26,16 @@ internal partial class BPlusTree : IBPlusTree public BPlusTree(IStateClient stateClient, BPlusTreeOptions options) { + Debug.Assert(options.BucketSize.HasValue); this.m_stateClient = stateClient; this.m_options = options; - minSize = options.BucketSize / 3; + minSize = options.BucketSize.Value / 3; this.m_keyComparer = options.Comparer; } public Task InitializeAsync() { + Debug.Assert(m_options.BucketSize.HasValue); if (m_stateClient.Metadata == null) { var rootId = m_stateClient.GetNewPageId(); @@ -40,7 +43,7 @@ public Task InitializeAsync() m_stateClient.Metadata = new BPlusTreeMetadata() { Root = rootId, - BucketLength = m_options.BucketSize, + BucketLength = m_options.BucketSize.Value, Left = rootId }; m_stateClient.AddOrUpdate(rootId, root); @@ -172,6 +175,7 @@ private sealed class RMWContainer public async ValueTask Clear() { + Debug.Assert(m_options.BucketSize.HasValue); // Clear the current state from the state storage await m_stateClient.Reset(true); @@ -181,7 +185,7 @@ public async ValueTask Clear() m_stateClient.Metadata = new BPlusTreeMetadata() { Root = rootId, - BucketLength = m_options.BucketSize, + BucketLength = m_options.BucketSize.Value, Left = rootId }; m_stateClient.AddOrUpdate(rootId, root);