Skip to content

Commit

Permalink
MOAR storageid
Browse files Browse the repository at this point in the history
  • Loading branch information
@nadavsteindler
nadavsteindler committed Jan 26, 2025
1 parent 317f0cf commit 2343335
Show file tree
Hide file tree
Showing 8 changed files with 43 additions and 43 deletions.
4 changes: 2 additions & 2 deletions pkg/graveler/committed/iterator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ func TestIterator(t *testing.T) {
key = committed.Key(p.Keys[len(p.Keys)-1])
}
manager.EXPECT().
NewRangeIterator(gomock.Any(), gomock.Eq(namespace), committed.ID(key)).
NewRangeIterator(gomock.Any(), gomock.Any(), gomock.Eq(namespace), committed.ID(key)).
Return(makeRangeIterator(p.Keys), nil)
lastKey = key
}
Expand All @@ -195,7 +195,7 @@ func TestIterator(t *testing.T) {
key = committed.Key(p.Keys[len(p.Keys)-1])
}
manager.EXPECT().
NewRangeIterator(gomock.Any(), gomock.Eq(namespace), committed.ID(key)).
NewRangeIterator(gomock.Any(), gomock.Any(), gomock.Eq(namespace), committed.ID(key)).
Return(makeRangeIterator(p.Keys), nil).
AnyTimes()
lastKey = key
Expand Down
4 changes: 2 additions & 2 deletions pkg/graveler/committed/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func TestManager_WriteRange(t *testing.T) {
rangeWriter := mock.NewMockRangeWriter(ctrl)

rangeWriter.EXPECT().Abort().Return(nil)
rangeManager.EXPECT().GetWriter(context.Background(), committed.Namespace(ns), nil).Return(rangeWriter, nil)
rangeManager.EXPECT().GetWriter(context.Background(), committed.StorageID(ns), committed.Namespace(ns), nil).Return(rangeWriter, nil)

sut := committed.NewCommittedManager(metarangeManager, rangeManager, params)

Expand All @@ -69,7 +69,7 @@ func TestManager_WriteRange(t *testing.T) {
rangeWriter.EXPECT().SetMetadata(committed.MetadataTypeKey, committed.MetadataRangesType)

it := testutils.NewFakeValueIterator(tt.records)
rangeInfo, err := sut.WriteRange(context.Background(), ns, it)
rangeInfo, err := sut.WriteRange(context.Background(), "", ns, it)
require.NoError(t, err)
require.Equal(t, &graveler.RangeInfo{
ID: graveler.RangeID(writeResult.RangeID),
Expand Down
6 changes: 3 additions & 3 deletions pkg/graveler/committed/merge_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1719,9 +1719,9 @@ func runMergeTests(tests testCases, t *testing.T) {
sourceMetaRangeID := tst.sourceRange.GetMetaRangeID()
destMetaRangeID := tst.destRange.GetMetaRangeID()
baseMetaRangeID := tst.baseRange.GetMetaRangeID()
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), baseMetaRangeID).AnyTimes().Return(createIter(tst.baseRange), nil)
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil)
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil)
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), baseMetaRangeID).AnyTimes().Return(createIter(tst.baseRange), nil)
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil)
metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil)

rangeManager := mock.NewMockRangeManager(ctrl)

Expand Down
8 changes: 4 additions & 4 deletions pkg/graveler/committed/meta_range_writer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestWriter_WriteRecords(t *testing.T) {
fakeWriter := NewFakeRangeWriter(&writeResult, nil)

rangeManager := mock.NewMockRangeManager(ctrl)
rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil)
rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil)

metaWriteResult := committed.WriteResult{
RangeID: committed.ID("meta-range-id"),
Expand All @@ -51,7 +51,7 @@ func TestWriter_WriteRecords(t *testing.T) {
fakeMetaWriter.ExpectAnyRecord()

rangeManagerMeta := mock.NewMockRangeManager(ctrl)
rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil)
rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil)
namespace := committed.Namespace("ns")
w := committed.NewGeneralMetaRangeWriter(ctx, rangeManager, rangeManagerMeta, &params, namespace, nil)

Expand Down Expand Up @@ -136,8 +136,8 @@ func TestWriter_RecordRangeAndClose(t *testing.T) {
rng := committed.Range{ID: "rng2-id", MinKey: committed.Key("a"), MaxKey: committed.Key("g"), Count: 4}

// get writer - once for record writer, once for range writer
rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil)
rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil)
rangeManager.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeWriter, nil)
rangeManagerMeta.EXPECT().GetWriter(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeMetaWriter, nil)

// Never attempt to split files: fake writers return size 0.

Expand Down
22 changes: 11 additions & 11 deletions pkg/graveler/sstable/range_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
fsMock "github.com/treeverse/lakefs/pkg/pyramid/mock"
)

func makeNewReader(r fakeReader) func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
return func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
func makeNewReader(r fakeReader) func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
return func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
return r.Reader, nil
}
}
Expand All @@ -43,7 +43,7 @@ func TestGetEntrySuccess(t *testing.T) {
ns := "some-ns"
sstableID := "some-id"

val, err := sut.GetValue(ctx, committed.Namespace(ns), committed.ID(sstableID), committed.Key(keys[len(keys)/3]))
val, err := sut.GetValue(ctx, "", committed.Namespace(ns), committed.ID(sstableID), committed.Key(keys[len(keys)/3]))
require.NoError(t, err)
require.NotNil(t, val)

Expand All @@ -58,14 +58,14 @@ func TestGetEntryCacheFailure(t *testing.T) {

mockFS := fsMock.NewMockFS(ctrl)

sut := sstable.NewPebbleSSTableRangeManagerWithNewReader(func(context.Context, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
sut := sstable.NewPebbleSSTableRangeManagerWithNewReader(func(context.Context, committed.StorageID, committed.Namespace, committed.ID) (*pebblesst.Reader, error) {
return nil, expectedErr
}, &NoCache{}, mockFS, crypto.SHA256)

ns := "some-ns"
sstableID := committed.ID("some-id")

val, err := sut.GetValue(ctx, committed.Namespace(ns), sstableID, committed.Key("some-key"))
val, err := sut.GetValue(ctx, "", committed.Namespace(ns), sstableID, committed.Key("some-key"))
require.Error(t, expectedErr, err)
require.Nil(t, val)
}
Expand All @@ -87,7 +87,7 @@ func TestGetEntryNotFound(t *testing.T) {
ns := "some-ns"
sstableID := committed.ID("some-id")

val, err := sut.GetValue(ctx, committed.Namespace(ns), sstableID, committed.Key("does-not-exist"))
val, err := sut.GetValue(ctx, "", committed.Namespace(ns), sstableID, committed.Key("does-not-exist"))
require.Error(t, err)
require.Nil(t, val)

Expand All @@ -104,9 +104,9 @@ func TestGetWriterSuccess(t *testing.T) {

ns := "some-ns"
mockFile := fsMock.NewMockStoredFile(ctrl)
mockFS.EXPECT().Create(ctx, ns).Return(mockFile, nil).Times(1)
mockFS.EXPECT().Create(ctx, "", ns).Return(mockFile, nil).Times(1)

writer, err := sut.GetWriter(ctx, committed.Namespace(ns), nil)
writer, err := sut.GetWriter(ctx, "", committed.Namespace(ns), nil)
require.NoError(t, err)
require.NotNil(t, writer)

Expand All @@ -133,7 +133,7 @@ func TestNewPartIteratorSuccess(t *testing.T) {
ns := "some-ns"
sstableID := committed.ID("some-id")

iter, err := sut.NewRangeIterator(ctx, committed.Namespace(ns), sstableID)
iter, err := sut.NewRangeIterator(ctx, "", committed.Namespace(ns), sstableID)
require.NoError(t, err)
require.NotNil(t, iter)

Expand Down Expand Up @@ -163,9 +163,9 @@ func TestGetWriterRangeID(t *testing.T) {
mockFile.EXPECT().Sync().Return(nil).AnyTimes()
mockFile.EXPECT().Close().Return(nil).Times(1)
mockFile.EXPECT().Store(gomock.Any(), gomock.Any()).Return(nil).Times(1)
mockFS.EXPECT().Create(ctx, ns).Return(mockFile, nil).Times(1)
mockFS.EXPECT().Create(ctx, "", ns).Return(mockFile, nil).Times(1)

writer, err := sut.GetWriter(ctx, ns, nil)
writer, err := sut.GetWriter(ctx, "", ns, nil)
require.NoError(t, err)
require.NotNil(t, writer)
err = writer.WriteRecord(committed.Record{
Expand Down
12 changes: 6 additions & 6 deletions pkg/graveler/sstable/writer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ func TestWriter(t *testing.T) {
// create the mock file with the matching file-system
mockFile := mock.NewMockStoredFile(ctrl)
mockFile.EXPECT().Close().Return(nil).Times(1)
mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil)
mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil)

writes := 500
dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil)
dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil)
require.NoError(t, err)
require.NotNil(t, dw)

Expand Down Expand Up @@ -77,9 +77,9 @@ func TestWriterAbort(t *testing.T) {
mockFile := mock.NewMockStoredFile(ctrl)
mockFile.EXPECT().Abort(gomock.Any()).Return(nil).Times(1)
mockFile.EXPECT().Close().Return(nil).Times(1)
mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil)
mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil)

dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil)
dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil)
require.NoError(t, err)
require.NotNil(t, dw)

Expand Down Expand Up @@ -111,14 +111,14 @@ func TestWriterAbortAfterClose(t *testing.T) {
// create the mock file with the matching file-system
mockFile := mock.NewMockStoredFile(ctrl)
mockFile.EXPECT().Close().Return(nil).Times(1)
mockFS.EXPECT().Create(gomock.Any(), string(ns)).Return(mockFile, nil)
mockFS.EXPECT().Create(gomock.Any(), "", string(ns)).Return(mockFile, nil)
// expect any write file actions
mockFile.EXPECT().Write(gomock.Any()).DoAndReturn(func(b []byte) (int, error) { return len(b), nil }).Times(1)
mockFile.EXPECT().Sync().Return(nil).Times(1)
mockFile.EXPECT().Store(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, filename string) error { return nil }).Times(1)

// Create writer
dw, err := sstable.NewDiskWriter(ctx, mockFS, ns, sha256.New(), nil)
dw, err := sstable.NewDiskWriter(ctx, mockFS, "", ns, sha256.New(), nil)
require.NoError(t, err)
require.NotNil(t, dw)

Expand Down
18 changes: 9 additions & 9 deletions pkg/graveler/testutil/fakes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type CommittedFake struct {
AppliedData AppliedData
}

func (c *CommittedFake) GetRangeIDByKey(_ context.Context, _ graveler.StorageNamespace, _ graveler.MetaRangeID, _ graveler.Key) (graveler.RangeID, error) {
func (c *CommittedFake) GetRangeIDByKey(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _ graveler.MetaRangeID, _ graveler.Key) (graveler.RangeID, error) {
panic("implement me")
}

Expand All @@ -42,21 +42,21 @@ func (t *MetaRangeFake) ID() graveler.MetaRangeID {
return t.id
}

func (c *CommittedFake) Exists(context.Context, graveler.StorageNamespace, graveler.MetaRangeID) (bool, error) {
func (c *CommittedFake) Exists(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID) (bool, error) {
if c.Err != nil {
return false, c.Err
}
return true, nil
}

func (c *CommittedFake) Get(_ context.Context, _ graveler.StorageNamespace, _ graveler.MetaRangeID, key graveler.Key) (*graveler.Value, error) {
func (c *CommittedFake) Get(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _ graveler.MetaRangeID, key graveler.Key) (*graveler.Value, error) {
if c.Err != nil {
return nil, c.Err
}
return c.ValuesByKey[string(key)], nil
}

func (c *CommittedFake) List(_ context.Context, _ graveler.StorageNamespace, mr graveler.MetaRangeID) (graveler.ValueIterator, error) {
func (c *CommittedFake) List(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, mr graveler.MetaRangeID) (graveler.ValueIterator, error) {
if c.Err != nil {
return nil, c.Err
}
Expand All @@ -66,14 +66,14 @@ func (c *CommittedFake) List(_ context.Context, _ graveler.StorageNamespace, mr
return c.ValueIterator, nil
}

func (c *CommittedFake) Diff(context.Context, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) {
func (c *CommittedFake) Diff(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) {
if c.Err != nil {
return nil, c.Err
}
return c.DiffIterator, nil
}

func (c *CommittedFake) Compare(context.Context, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) {
func (c *CommittedFake) Compare(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.MetaRangeID, graveler.MetaRangeID, graveler.MetaRangeID) (graveler.DiffIterator, error) {
if c.Err != nil {
return nil, c.Err
}
Expand All @@ -87,14 +87,14 @@ func (c *CommittedFake) Merge(_ context.Context, _ graveler.StorageNamespace, _,
return c.MetaRangeID, nil
}

func (c *CommittedFake) Import(_ context.Context, _ graveler.StorageNamespace, _, _ graveler.MetaRangeID, _ []graveler.Prefix, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, error) {
func (c *CommittedFake) Import(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, _, _ graveler.MetaRangeID, _ []graveler.Prefix, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, error) {
if c.Err != nil {
return "", c.Err
}
return c.MetaRangeID, nil
}

func (c *CommittedFake) Commit(_ context.Context, _ graveler.StorageNamespace, baseMetaRangeID graveler.MetaRangeID, changes graveler.ValueIterator, _ bool, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, graveler.DiffSummary, error) {
func (c *CommittedFake) Commit(_ context.Context, _ graveler.StorageID, _ graveler.StorageNamespace, baseMetaRangeID graveler.MetaRangeID, changes graveler.ValueIterator, _ bool, _ ...graveler.SetOptionsFunc) (graveler.MetaRangeID, graveler.DiffSummary, error) {
if c.Err != nil {
return "", graveler.DiffSummary{}, c.Err
}
Expand All @@ -110,7 +110,7 @@ func (c *CommittedFake) WriteMetaRangeByIterator(context.Context, graveler.Stora
return &c.MetaRangeID, nil
}

func (c *CommittedFake) WriteRange(context.Context, graveler.StorageNamespace, graveler.ValueIterator) (*graveler.RangeInfo, error) {
func (c *CommittedFake) WriteRange(context.Context, graveler.StorageID, graveler.StorageNamespace, graveler.ValueIterator) (*graveler.RangeInfo, error) {
return &c.RangeInfo, nil
}

Expand Down
12 changes: 6 additions & 6 deletions pkg/pyramid/tier_fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ func TestReadFailDuringWrite(t *testing.T) {
ctx := context.Background()
namespace := uniqueNamespace()
filename := "file1"
f, err := fs.Create(ctx, namespace)
f, err := fs.Create(ctx, "", namespace)
require.NoError(t, err)

content := []byte("some content")
n, err := f.Write(content)
require.NoError(t, err)
require.Equal(t, len(content), n)

readF, err := fs.Open(ctx, namespace, filename)
readF, err := fs.Open(ctx, "", namespace, filename)
require.Nil(t, readF)
require.Error(t, err)
require.NoError(t, f.Close())
Expand Down Expand Up @@ -127,7 +127,7 @@ func TestStartup(t *testing.T) {
// package os this does not matter.
assert.Error(t, err, os.ErrNotExist, "expected %s not to exist", workspacePath)

f, err := localFS.Open(ctx, "mem://"+namespaceID, filename)
f, err := localFS.Open(ctx, "", "mem://"+namespaceID, filename)
defer func() { _ = f.Close() }()
assert.NoError(t, err)

Expand Down Expand Up @@ -156,7 +156,7 @@ func testEviction(t *testing.T, namespaces ...string) {
for i := 0; i < numFiles; i++ {
filename := "file_" + strconv.Itoa(i)

f, err := fs.Open(ctx, namespaces[i%len(namespaces)], filename)
f, err := fs.Open(ctx, "", namespaces[i%len(namespaces)], filename)
require.NoError(t, err)

_, err = io.ReadAll(f)
Expand Down Expand Up @@ -206,7 +206,7 @@ func TestMultipleConcurrentReads(t *testing.T) {

func writeToFile(t *testing.T, ctx context.Context, namespace, filename string, content []byte) {
t.Helper()
f, err := fs.Create(ctx, namespace)
f, err := fs.Create(ctx, "", namespace)
require.NoError(t, err)

n, err := f.Write(content)
Expand All @@ -219,7 +219,7 @@ func writeToFile(t *testing.T, ctx context.Context, namespace, filename string,

func checkContent(t *testing.T, ctx context.Context, namespace string, filename string, content []byte) {
t.Helper()
f, err := fs.Open(ctx, namespace, filename)
f, err := fs.Open(ctx, "", namespace, filename)
if err != nil {
t.Errorf("Failed to open namespace:%s filename:%s - %s", namespace, filename, err)
return
Expand Down

0 comments on commit 2343335

Please sign in to comment.