From 317f0cf40f4d326a91067188743658530f9ed86a Mon Sep 17 00:00:00 2001 From: nadavsteindler Date: Sun, 26 Jan 2025 16:37:54 +0200 Subject: [PATCH] MOAR storageid --- pkg/graveler/committed/import_test.go | 8 ++++---- pkg/graveler/committed/manager.go | 4 ++-- pkg/graveler/committed/meta_range_writer.go | 5 +++-- pkg/graveler/committed/mock/range_manager.go | 8 ++++---- pkg/graveler/committed/range_manager.go | 2 +- pkg/graveler/graveler.go | 4 ++-- pkg/graveler/mock/graveler.go | 8 ++++---- pkg/graveler/sstable/range_manager.go | 4 ++-- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pkg/graveler/committed/import_test.go b/pkg/graveler/committed/import_test.go index 1096c42390d..8b980a0a8d6 100644 --- a/pkg/graveler/committed/import_test.go +++ b/pkg/graveler/committed/import_test.go @@ -362,9 +362,9 @@ func Test_import(t *testing.T) { metaRangeManager.EXPECT().NewWriter(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(writer) sourceMetaRangeID := tst.sourceRange.GetMetaRangeID() destMetaRangeID := tst.destRange.GetMetaRangeID() - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), graveler.MetaRangeID("")).AnyTimes().Return(committed.NewEmptyIterator(), nil) // empty base - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil) - metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil) + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), graveler.MetaRangeID("")).AnyTimes().Return(committed.NewEmptyIterator(), nil) // empty base + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil) + metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil) rangeManager := mock.NewMockRangeManager(ctrl) @@ -372,7 +372,7 @@ func Test_import(t *testing.T) { metaRangeId := graveler.MetaRangeID("import") writer.EXPECT().Close(gomock.Any()).Return(&metaRangeId, nil).AnyTimes() committedManager := committed.NewCommittedManager(metaRangeManager, rangeManager, params) - _, err := committedManager.Import(ctx, "ns", destMetaRangeID, sourceMetaRangeID, tst.prefixes) + _, err := committedManager.Import(ctx, "", "ns", destMetaRangeID, sourceMetaRangeID, tst.prefixes) if !errors.Is(err, expectedResult.expectedErr) { t.Fatalf("Import error = '%v', expected '%v'", err, expectedResult.expectedErr) } diff --git a/pkg/graveler/committed/manager.go b/pkg/graveler/committed/manager.go index a00a4bfbfcf..e19c014fc30 100644 --- a/pkg/graveler/committed/manager.go +++ b/pkg/graveler/committed/manager.go @@ -61,8 +61,8 @@ func (c *committedManager) List(ctx context.Context, storageID graveler.StorageI return NewValueIterator(it), nil } -func (c *committedManager) WriteRange(ctx context.Context, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) { - writer, err := c.RangeManager.GetWriter(ctx, Namespace(ns), nil) +func (c *committedManager) WriteRange(ctx context.Context, storageID graveler.StorageID, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) { + writer, err := c.RangeManager.GetWriter(ctx, StorageID(storageID), Namespace(ns), nil) if err != nil { return nil, fmt.Errorf("failed creating range writer: %w", err) } diff --git a/pkg/graveler/committed/meta_range_writer.go b/pkg/graveler/committed/meta_range_writer.go index 17fb04f9cbb..2d8282626f7 100644 --- a/pkg/graveler/committed/meta_range_writer.go +++ b/pkg/graveler/committed/meta_range_writer.go @@ -15,6 +15,7 @@ type GeneralMetaRangeWriter struct { ctx context.Context metadata graveler.Metadata params *Params // for breaking ranges + storageID StorageID namespace Namespace metaRangeManager RangeManager rangeManager RangeManager @@ -58,7 +59,7 @@ func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error var err error if w.rangeWriter == nil { - w.rangeWriter, err = w.rangeManager.GetWriter(w.ctx, w.namespace, w.metadata) + w.rangeWriter, err = w.rangeManager.GetWriter(w.ctx, w.storageID, w.namespace, w.metadata) if err != nil { return fmt.Errorf("get range writer: %w", err) } @@ -145,7 +146,7 @@ func (w *GeneralMetaRangeWriter) shouldBreakAtKey(key graveler.Key) bool { // writeRangesToMetaRange writes all ranges to a MetaRange and returns the MetaRangeID func (w *GeneralMetaRangeWriter) writeRangesToMetaRange(ctx context.Context) (*graveler.MetaRangeID, error) { - metaRangeWriter, err := w.metaRangeManager.GetWriter(w.ctx, w.namespace, w.metadata) + metaRangeWriter, err := w.metaRangeManager.GetWriter(w.ctx, w.storageID, w.namespace, w.metadata) if err != nil { return nil, fmt.Errorf("failed creating metarange writer: %w", err) } diff --git a/pkg/graveler/committed/mock/range_manager.go b/pkg/graveler/committed/mock/range_manager.go index edb94a8d955..11182a2cfae 100644 --- a/pkg/graveler/committed/mock/range_manager.go +++ b/pkg/graveler/committed/mock/range_manager.go @@ -186,18 +186,18 @@ func (mr *MockRangeManagerMockRecorder) GetValueGE(ctx, storageID, ns, id, key i } // GetWriter mocks base method. -func (m *MockRangeManager) GetWriter(ctx context.Context, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) { +func (m *MockRangeManager) GetWriter(ctx context.Context, storageID committed.StorageID, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetWriter", ctx, ns, metadata) + ret := m.ctrl.Call(m, "GetWriter", ctx, storageID, ns, metadata) ret0, _ := ret[0].(committed.RangeWriter) ret1, _ := ret[1].(error) return ret0, ret1 } // GetWriter indicates an expected call of GetWriter. -func (mr *MockRangeManagerMockRecorder) GetWriter(ctx, ns, metadata interface{}) *gomock.Call { +func (mr *MockRangeManagerMockRecorder) GetWriter(ctx, storageID, ns, metadata interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWriter", reflect.TypeOf((*MockRangeManager)(nil).GetWriter), ctx, ns, metadata) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWriter", reflect.TypeOf((*MockRangeManager)(nil).GetWriter), ctx, storageID, ns, metadata) } // NewRangeIterator mocks base method. diff --git a/pkg/graveler/committed/range_manager.go b/pkg/graveler/committed/range_manager.go index 901b2e6e0f3..d871fe045c8 100644 --- a/pkg/graveler/committed/range_manager.go +++ b/pkg/graveler/committed/range_manager.go @@ -60,7 +60,7 @@ type RangeManager interface { NewRangeIterator(ctx context.Context, storageID StorageID, ns Namespace, pid ID) (ValueIterator, error) // GetWriter returns a new Range writer instance - GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error) + GetWriter(ctx context.Context, storageID StorageID, ns Namespace, metadata graveler.Metadata) (RangeWriter, error) // GetURI returns a URI from which to read the contents of id. If id does not exist // it may return a URI that resolves nowhere rather than an error. diff --git a/pkg/graveler/graveler.go b/pkg/graveler/graveler.go index c4e186e77bc..67368a7249f 100644 --- a/pkg/graveler/graveler.go +++ b/pkg/graveler/graveler.go @@ -1018,7 +1018,7 @@ type CommittedManager interface { // WriteRange creates a new Range from the iterator values. // Keeps Range closing logic, so might not exhaust the iterator. - WriteRange(ctx context.Context, ns StorageNamespace, it ValueIterator) (*RangeInfo, error) + WriteRange(ctx context.Context, storageID StorageID, ns StorageNamespace, it ValueIterator) (*RangeInfo, error) // WriteMetaRange creates a new MetaRange from the given Ranges. WriteMetaRange(ctx context.Context, ns StorageNamespace, ranges []*RangeInfo) (*MetaRangeInfo, error) @@ -1243,7 +1243,7 @@ func (g *Graveler) WriteRange(ctx context.Context, repository *RepositoryRecord, if repository.ReadOnly && !options.Force { return nil, ErrReadOnlyRepository } - return g.CommittedManager.WriteRange(ctx, repository.StorageNamespace, it) + return g.CommittedManager.WriteRange(ctx, repository.StorageID, repository.StorageNamespace, it) } func (g *Graveler) WriteMetaRange(ctx context.Context, repository *RepositoryRecord, ranges []*RangeInfo, opts ...SetOptionsFunc) (*MetaRangeInfo, error) { diff --git a/pkg/graveler/mock/graveler.go b/pkg/graveler/mock/graveler.go index 56497f4ee74..f6221ff184f 100644 --- a/pkg/graveler/mock/graveler.go +++ b/pkg/graveler/mock/graveler.go @@ -2813,18 +2813,18 @@ func (mr *MockCommittedManagerMockRecorder) WriteMetaRangeByIterator(ctx, ns, it } // WriteRange mocks base method. -func (m *MockCommittedManager) WriteRange(ctx context.Context, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) { +func (m *MockCommittedManager) WriteRange(ctx context.Context, storageID graveler.StorageID, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WriteRange", ctx, ns, it) + ret := m.ctrl.Call(m, "WriteRange", ctx, storageID, ns, it) ret0, _ := ret[0].(*graveler.RangeInfo) ret1, _ := ret[1].(error) return ret0, ret1 } // WriteRange indicates an expected call of WriteRange. -func (mr *MockCommittedManagerMockRecorder) WriteRange(ctx, ns, it interface{}) *gomock.Call { +func (mr *MockCommittedManagerMockRecorder) WriteRange(ctx, storageID, ns, it interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteRange", reflect.TypeOf((*MockCommittedManager)(nil).WriteRange), ctx, ns, it) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteRange", reflect.TypeOf((*MockCommittedManager)(nil).WriteRange), ctx, storageID, ns, it) } // MockStagingManager is a mock of StagingManager interface. diff --git a/pkg/graveler/sstable/range_manager.go b/pkg/graveler/sstable/range_manager.go index fdb050d68f4..a765a4b874b 100644 --- a/pkg/graveler/sstable/range_manager.go +++ b/pkg/graveler/sstable/range_manager.go @@ -164,8 +164,8 @@ func (m *RangeManager) NewRangeIterator(ctx context.Context, storageID committed } // GetWriter returns a new SSTable writer instance -func (m *RangeManager) GetWriter(ctx context.Context, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) { - return NewDiskWriter(ctx, m.fs, ns, m.hash.New(), metadata) +func (m *RangeManager) GetWriter(ctx context.Context, storageID committed.StorageID, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) { + return NewDiskWriter(ctx, m.fs, storageID, ns, m.hash.New(), metadata) } func (m *RangeManager) GetURI(ctx context.Context, ns committed.Namespace, id committed.ID) (string, error) {