From 234333584d882920f428b7116bae5ad637458325 Mon Sep 17 00:00:00 2001 From: nadavsteindler Date: Sun, 26 Jan 2025 18:39:01 +0200 Subject: [PATCH] MOAR storageid --- pkg/graveler/committed/iterator_test.go | 4 ++-- pkg/graveler/committed/manager_test.go | 4 ++-- pkg/graveler/committed/merge_test.go | 6 ++--- .../committed/meta_range_writer_test.go | 8 +++---- pkg/graveler/sstable/range_manager_test.go | 22 +++++++++---------- pkg/graveler/sstable/writer_test.go | 12 +++++----- pkg/graveler/testutil/fakes.go | 18 +++++++-------- pkg/pyramid/tier_fs_test.go | 12 +++++----- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/pkg/graveler/committed/iterator_test.go b/pkg/graveler/committed/iterator_test.go index 9ee186a3679..38fbab547fa 100644 --- a/pkg/graveler/committed/iterator_test.go +++ b/pkg/graveler/committed/iterator_test.go @@ -171,7 +171,7 @@ func TestIterator(t *testing.T) { key = committed.Key(p.Keys[len(p.Keys)-1]) } manager.EXPECT(). - NewRangeIterator(gomock.Any(), gomock.Eq(namespace), committed.ID(key)). + NewRangeIterator(gomock.Any(), gomock.Any(), gomock.Eq(namespace), committed.ID(key)). Return(makeRangeIterator(p.Keys), nil) lastKey = key } @@ -195,7 +195,7 @@ func TestIterator(t *testing.T) { key = committed.Key(p.Keys[len(p.Keys)-1]) } manager.EXPECT(). - NewRangeIterator(gomock.Any(), gomock.Eq(namespace), committed.ID(key)). + NewRangeIterator(gomock.Any(), gomock.Any(), gomock.Eq(namespace), committed.ID(key)). Return(makeRangeIterator(p.Keys), nil). AnyTimes() lastKey = key diff --git a/pkg/graveler/committed/manager_test.go b/pkg/graveler/committed/manager_test.go index 20a08ea5613..f36ae9e97e1 100644 --- a/pkg/graveler/committed/manager_test.go +++ b/pkg/graveler/committed/manager_test.go @@ -56,7 +56,7 @@ func TestManager_WriteRange(t *testing.T) { rangeWriter := mock.NewMockRangeWriter(ctrl) rangeWriter.EXPECT().Abort().Return(nil) - rangeManager.EXPECT().GetWriter(context.Background(), committed.Namespace(ns), nil).Return(rangeWriter, nil) + rangeManager.EXPECT().GetWriter(context.Background(), committed.StorageID(ns), committed.Namespace(ns), nil).Return(rangeWriter, nil) sut := committed.NewCommittedManager(metarangeManager, rangeManager, params) @@ -69,7 +69,7 @@ func TestManager_WriteRange(t *testing.T) { rangeWriter.EXPECT().SetMetadata(committed.MetadataTypeKey, committed.MetadataRangesType) it := testutils.NewFakeValueIterator(tt.records) - rangeInfo, err := sut.WriteRange(context.Background(), ns, it) + rangeInfo, err := sut.WriteRange(context.Background(), "", ns, it) require.NoError(t, err) require.Equal(t, &graveler.RangeInfo{ ID: graveler.RangeID(writeResult.RangeID), diff --git a/pkg/graveler/committed/merge_test.go b/pkg/graveler/committed/merge_test.go index ec58c09ae7e..bc416c19945 100644 --- a/pkg/graveler/committed/merge_test.go +++ b/pkg/graveler/committed/merge_test.go @@ -1719,9 +1719,9 @@ func runMergeTests(tests testCases, t *testing.T) { sourceMetaRangeID := tst.sourceRange.GetMetaRangeID() destMetaRangeID := tst.destRange.GetMetaRangeID() baseMetaRangeID := tst.baseRange.GetMetaRangeID() - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), baseMetaRangeID).AnyTimes().Return(createIter(tst.baseRange), nil) - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil) - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil) + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), baseMetaRangeID).AnyTimes().Return(createIter(tst.baseRange), nil) + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil) + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil) rangeManager := mock.NewMockRangeManager(ctrl) diff --git a/pkg/graveler/committed/meta_range_writer_test.go b/pkg/graveler/committed/meta_range_writer_test.go index e00ae43a07e..1987325d4c9 100644 --- a/pkg/graveler/committed/meta_range_writer_test.go +++ b/pkg/graveler/committed/meta_range_writer_test.go @@ -39,7 +39,7 @@ func TestWriter_WriteRecords(t *testing.T) { fakeWriter := NewFakeRangeWriter(&writeResult, nil) rangeManager := mock.NewMockRangeManager(ctrl) - rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil) + rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil) metaWriteResult := committed.WriteResult{ RangeID: committed.ID("meta-range-id"), @@ -51,7 +51,7 @@ func TestWriter_WriteRecords(t *testing.T) { fakeMetaWriter.ExpectAnyRecord() rangeManagerMeta := mock.NewMockRangeManager(ctrl) - rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil) + rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil) namespace := committed.Namespace("ns") w := committed.NewGeneralMetaRangeWriter(ctx, rangeManager, rangeManagerMeta, ¶ms, namespace, nil) @@ -136,8 +136,8 @@ func TestWriter_RecordRangeAndClose(t *testing.T) { rng := committed.Range{ID: "rng2-id", MinKey: committed.Key("a"), MaxKey: committed.Key("g"), Count: 4} // get writer - once for record writer, once for range writer - rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil) - rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil) + rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil) + rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil) // Never attempt to split files: fake writers return size 0. diff --git a/pkg/graveler/sstable/range_manager_test.go b/pkg/graveler/sstable/range_manager_test.go index 816ccf9222c..8418359dd8b 100644 --- a/pkg/graveler/sstable/range_manager_test.go +++ b/pkg/graveler/sstable/range_manager_test.go @@ -16,8 +16,8 @@ import ( fsMock "github.com/treeverse/lakefs/pkg/pyramid/mock" ) -func makeNewReader(r fakeReader) func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { - return func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { +func makeNewReader(r fakeReader) func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { + return func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { return r.Reader, nil } } @@ -43,7 +43,7 @@ func TestGetEntrySuccess(t *testing.T) { ns := "some-ns" sstableID := "some-id" - val, err := sut.GetValue(ctx, committed.Namespace(ns), committed.ID(sstableID), committed.Key(keys[len(keys)/3])) + val, err := sut.GetValue(ctx, "", committed.Namespace(ns), committed.ID(sstableID), committed.Key(keys[len(keys)/3])) require.NoError(t, err) require.NotNil(t, val) @@ -58,14 +58,14 @@ func TestGetEntryCacheFailure(t *testing.T) { mockFS := fsMock.NewMockFS(ctrl) - sut := sstable.NewPebbleSSTableRangeManagerWithNewReader(func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { + sut := sstable.NewPebbleSSTableRangeManagerWithNewReader(func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) { return nil, expectedErr }, &NoCache{}, mockFS, crypto.SHA256) ns := "some-ns" sstableID := committed.ID("some-id") - val, err := sut.GetValue(ctx, committed.Namespace(ns), sstableID, committed.Key("some-key")) + val, err := sut.GetValue(ctx, "", committed.Namespace(ns), sstableID, committed.Key("some-key")) require.Error(t, expectedErr, err) require.Nil(t, val) } @@ -87,7 +87,7 @@ func TestGetEntryNotFound(t *testing.T) { ns := "some-ns" sstableID := committed.ID("some-id") - val, err := sut.GetValue(ctx, committed.Namespace(ns), sstableID, committed.Key("does-not-exist")) + val, err := sut.GetValue(ctx, "", committed.Namespace(ns), sstableID, committed.Key("does-not-exist")) require.Error(t, err) require.Nil(t, val) @@ -104,9 +104,9 @@ func TestGetWriterSuccess(t *testing.T) { ns := "some-ns" mockFile := fsMock.NewMockStoredFile(ctrl) - mockFS.EXPECT().Create(ctx, ns).Return(mockFile, nil).Times(1) + mockFS.EXPECT().Create(ctx, "", ns).Return(mockFile, nil).Times(1) - writer, err := sut.GetWriter(ctx, committed.Namespace(ns), nil) + writer, err := sut.GetWriter(ctx, "", committed.Namespace(ns), nil) require.NoError(t, err) require.NotNil(t, writer) @@ -133,7 +133,7 @@ func TestNewPartIteratorSuccess(t *testing.T) { ns := "some-ns" sstableID := committed.ID("some-id") - iter, err := sut.NewRangeIterator(ctx, committed.Namespace(ns), sstableID) + iter, err := sut.NewRangeIterator(ctx, "", committed.Namespace(ns), sstableID) require.NoError(t, err) require.NotNil(t, iter) @@ -163,9 +163,9 @@ func TestGetWriterRangeID(t *testing.T) { mockFile.EXPECT().Sync().Return(nil).AnyTimes() mockFile.EXPECT().Close().Return(nil).Times(1) mockFile.EXPECT().Store(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockFS.EXPECT().Create(ctx, ns).Return(mockFile, nil).Times(1) + mockFS.EXPECT().Create(ctx, "", ns).Return(mockFile, nil).Times(1) - writer, err := sut.GetWriter(ctx, ns, nil) + writer, err := sut.GetWriter(ctx, "", ns, nil) require.NoError(t, err) require.NotNil(t, writer) err = writer.WriteRecord(committed.Record{ diff --git a/pkg/graveler/sstable/writer_test.go b/pkg/graveler/sstable/writer_test.go index d890d312f45..ef53aac8931 100644 --- a/pkg/graveler/sstable/writer_test.go +++ b/pkg/graveler/sstable/writer_test.go @@ -24,10 +24,10 @@ func TestWriter(t *testing.T) { // create the mock file with the matching file-system mockFile := mock.NewMockStoredFile(ctrl) mockFile.EXPECT().Close().Return(nil).Times(1) - mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil) + mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil) writes := 500 - dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil) + dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil) require.NoError(t, err) require.NotNil(t, dw) @@ -77,9 +77,9 @@ func TestWriterAbort(t *testing.T) { mockFile := mock.NewMockStoredFile(ctrl) mockFile.EXPECT().Abort(gomock.Any()).Return(nil).Times(1) mockFile.EXPECT().Close().Return(nil).Times(1) - mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil) + mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil) - dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil) + dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil) require.NoError(t, err) require.NotNil(t, dw) @@ -111,14 +111,14 @@ func TestWriterAbortAfterClose(t *testing.T) { // create the mock file with the matching file-system mockFile := mock.NewMockStoredFile(ctrl) mockFile.EXPECT().Close().Return(nil).Times(1) - mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil) + mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil) // expect any write file actions mockFile.EXPECT().Write(gomock.Any()).DoAndReturn(func(b []byte) (int, error) { return len(b), nil }).Times(1) mockFile.EXPECT().Sync().Return(nil).Times(1) mockFile.EXPECT().Store(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, filename string) error { return nil }).Times(1) // Create writer - dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil) + dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil) require.NoError(t, err) require.NotNil(t, dw) diff --git a/pkg/graveler/testutil/fakes.go b/pkg/graveler/testutil/fakes.go index 468ff6ed8ba..3d076dbee7e 100644 --- a/pkg/graveler/testutil/fakes.go +++ b/pkg/graveler/testutil/fakes.go @@ -30,7 +30,7 @@ type CommittedFake struct { AppliedData AppliedData } -func (c *CommittedFake) GetRangeIDByKey(_ context.Context, _ graveler.StorageNamespace, _ graveler.MetaRangeID, _ graveler.Key) (graveler.RangeID, error) { +func (c *CommittedFake) GetRangeIDByKey(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _ graveler.MetaRangeID, _ graveler.Key) (graveler.RangeID, error) { panic("implement me") } @@ -42,21 +42,21 @@ func (t *MetaRangeFake) ID() graveler.MetaRangeID { return t.id } -func (c *CommittedFake) Exists(context.Context, graveler.StorageNamespace, graveler.MetaRangeID) (bool, error) { +func (c *CommittedFake) Exists(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID) (bool, error) { if c.Err != nil { return false, c.Err } return true, nil } -func (c *CommittedFake) Get(_ context.Context, _ graveler.StorageNamespace, _ graveler.MetaRangeID, key graveler.Key) (*graveler.Value, error) { +func (c *CommittedFake) Get(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _ graveler.MetaRangeID, key graveler.Key) (*graveler.Value, error) { if c.Err != nil { return nil, c.Err } return c.ValuesByKey[string(key)], nil } -func (c *CommittedFake) List(_ context.Context, _ graveler.StorageNamespace, mr graveler.MetaRangeID) (graveler.ValueIterator, error) { +func (c *CommittedFake) List(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, mr graveler.MetaRangeID) (graveler.ValueIterator, error) { if c.Err != nil { return nil, c.Err } @@ -66,14 +66,14 @@ func (c *CommittedFake) List(_ context.Context, _ graveler.StorageNamespace, mr return c.ValueIterator, nil } -func (c *CommittedFake) Diff(context.Context, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) { +func (c *CommittedFake) Diff(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) { if c.Err != nil { return nil, c.Err } return c.DiffIterator, nil } -func (c *CommittedFake) Compare(context.Context, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) { +func (c *CommittedFake) Compare(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) { if c.Err != nil { return nil, c.Err } @@ -87,14 +87,14 @@ func (c *CommittedFake) Merge(_ context.Context, _ graveler.StorageNamespace, _, return c.MetaRangeID, nil } -func (c *CommittedFake) Import(_ context.Context, _ graveler.StorageNamespace, _, _ graveler.MetaRangeID, _ []graveler.Prefix, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, error) { +func (c *CommittedFake) Import(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _, _ graveler.MetaRangeID, _ []graveler.Prefix, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, error) { if c.Err != nil { return "", c.Err } return c.MetaRangeID, nil } -func (c *CommittedFake) Commit(_ context.Context, _ graveler.StorageNamespace, baseMetaRangeID graveler.MetaRangeID, changes graveler.ValueIterator, _ bool, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, graveler.DiffSummary, error) { +func (c *CommittedFake) Commit(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, baseMetaRangeID graveler.MetaRangeID, changes graveler.ValueIterator, _ bool, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, graveler.DiffSummary, error) { if c.Err != nil { return "", graveler.DiffSummary{}, c.Err } @@ -110,7 +110,7 @@ func (c *CommittedFake) WriteMetaRangeByIterator(context.Context, graveler.Stora return &c.MetaRangeID, nil } -func (c *CommittedFake) WriteRange(context.Context, graveler.StorageNamespace, graveler.ValueIterator) (*graveler.RangeInfo, error) { +func (c *CommittedFake) WriteRange(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.ValueIterator) (*graveler.RangeInfo, error) { return &c.RangeInfo, nil } diff --git a/pkg/pyramid/tier_fs_test.go b/pkg/pyramid/tier_fs_test.go index fe7f08f6b62..e6d047b80e5 100644 --- a/pkg/pyramid/tier_fs_test.go +++ b/pkg/pyramid/tier_fs_test.go @@ -40,7 +40,7 @@ func TestReadFailDuringWrite(t *testing.T) { ctx := context.Background() namespace := uniqueNamespace() filename := "file1" - f, err := fs.Create(ctx, namespace) + f, err := fs.Create(ctx, "", namespace) require.NoError(t, err) content := []byte("some content") @@ -48,7 +48,7 @@ func TestReadFailDuringWrite(t *testing.T) { require.NoError(t, err) require.Equal(t, len(content), n) - readF, err := fs.Open(ctx, namespace, filename) + readF, err := fs.Open(ctx, "", namespace, filename) require.Nil(t, readF) require.Error(t, err) require.NoError(t, f.Close()) @@ -127,7 +127,7 @@ func TestStartup(t *testing.T) { // package os this does not matter. assert.Error(t, err, os.ErrNotExist, "expected %s not to exist", workspacePath) - f, err := localFS.Open(ctx, "mem://"+namespaceID, filename) + f, err := localFS.Open(ctx, "", "mem://"+namespaceID, filename) defer func() { _ = f.Close() }() assert.NoError(t, err) @@ -156,7 +156,7 @@ func testEviction(t *testing.T, namespaces ...string) { for i := 0; i < numFiles; i++ { filename := "file_" + strconv.Itoa(i) - f, err := fs.Open(ctx, namespaces[i%len(namespaces)], filename) + f, err := fs.Open(ctx, "", namespaces[i%len(namespaces)], filename) require.NoError(t, err) _, err = io.ReadAll(f) @@ -206,7 +206,7 @@ func TestMultipleConcurrentReads(t *testing.T) { func writeToFile(t *testing.T, ctx context.Context, namespace, filename string, content []byte) { t.Helper() - f, err := fs.Create(ctx, namespace) + f, err := fs.Create(ctx, "", namespace) require.NoError(t, err) n, err := f.Write(content) @@ -219,7 +219,7 @@ func writeToFile(t *testing.T, ctx context.Context, namespace, filename string, func checkContent(t *testing.T, ctx context.Context, namespace string, filename string, content []byte) { t.Helper() - f, err := fs.Open(ctx, namespace, filename) + f, err := fs.Open(ctx, "", namespace, filename) if err != nil { t.Errorf("Failed to open namespace:%s filename:%s - %s", namespace, filename, err) return