MOAR storageid

treeverse · Jan 26, 2025 · 317f0cf · 317f0cf
1 parent 46c153c
commit 317f0cf
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 21 deletions.
diff --git a/pkg/graveler/committed/import_test.go b/pkg/graveler/committed/import_test.go
@@ -362,17 +362,17 @@ func Test_import(t *testing.T) {
 				metaRangeManager.EXPECT().NewWriter(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(writer)
 				sourceMetaRangeID := tst.sourceRange.GetMetaRangeID()
 				destMetaRangeID := tst.destRange.GetMetaRangeID()
-				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), graveler.MetaRangeID("")).AnyTimes().Return(committed.NewEmptyIterator(), nil) // empty base
-				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil)
-				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil)
+				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), graveler.MetaRangeID("")).AnyTimes().Return(committed.NewEmptyIterator(), nil) // empty base
+				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), sourceMetaRangeID).AnyTimes().Return(createIter(tst.sourceRange), nil)
+				metaRangeManager.EXPECT().NewMetaRangeIterator(gomock.Any(), gomock.Any(), gomock.Any(), destMetaRangeID).AnyTimes().Return(createIter(tst.destRange), nil)
 
 				rangeManager := mock.NewMockRangeManager(ctrl)
 
 				writer.EXPECT().Abort().AnyTimes()
 				metaRangeId := graveler.MetaRangeID("import")
 				writer.EXPECT().Close(gomock.Any()).Return(&metaRangeId, nil).AnyTimes()
 				committedManager := committed.NewCommittedManager(metaRangeManager, rangeManager, params)
-				_, err := committedManager.Import(ctx, "ns", destMetaRangeID, sourceMetaRangeID, tst.prefixes)
+				_, err := committedManager.Import(ctx, "", "ns", destMetaRangeID, sourceMetaRangeID, tst.prefixes)
 				if !errors.Is(err, expectedResult.expectedErr) {
 					t.Fatalf("Import error = '%v', expected '%v'", err, expectedResult.expectedErr)
 				}

diff --git a/pkg/graveler/committed/manager.go b/pkg/graveler/committed/manager.go
@@ -61,8 +61,8 @@ func (c *committedManager) List(ctx context.Context, storageID graveler.StorageI
 	return NewValueIterator(it), nil
 }
 
-func (c *committedManager) WriteRange(ctx context.Context, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) {
-	writer, err := c.RangeManager.GetWriter(ctx, Namespace(ns), nil)
+func (c *committedManager) WriteRange(ctx context.Context, storageID graveler.StorageID, ns graveler.StorageNamespace, it graveler.ValueIterator) (*graveler.RangeInfo, error) {
+	writer, err := c.RangeManager.GetWriter(ctx, StorageID(storageID), Namespace(ns), nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed creating range writer: %w", err)
 	}

diff --git a/pkg/graveler/committed/meta_range_writer.go b/pkg/graveler/committed/meta_range_writer.go
@@ -15,6 +15,7 @@ type GeneralMetaRangeWriter struct {
 	ctx              context.Context
 	metadata         graveler.Metadata
 	params           *Params // for breaking ranges
+	storageID        StorageID
 	namespace        Namespace
 	metaRangeManager RangeManager
 	rangeManager     RangeManager
@@ -58,7 +59,7 @@ func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error
 
 	var err error
 	if w.rangeWriter == nil {
-		w.rangeWriter, err = w.rangeManager.GetWriter(w.ctx, w.namespace, w.metadata)
+		w.rangeWriter, err = w.rangeManager.GetWriter(w.ctx, w.storageID, w.namespace, w.metadata)
 		if err != nil {
 			return fmt.Errorf("get range writer: %w", err)
 		}
@@ -145,7 +146,7 @@ func (w *GeneralMetaRangeWriter) shouldBreakAtKey(key graveler.Key) bool {
 
 // writeRangesToMetaRange writes all ranges to a MetaRange and returns the MetaRangeID
 func (w *GeneralMetaRangeWriter) writeRangesToMetaRange(ctx context.Context) (*graveler.MetaRangeID, error) {
-	metaRangeWriter, err := w.metaRangeManager.GetWriter(w.ctx, w.namespace, w.metadata)
+	metaRangeWriter, err := w.metaRangeManager.GetWriter(w.ctx, w.storageID, w.namespace, w.metadata)
 	if err != nil {
 		return nil, fmt.Errorf("failed creating metarange writer: %w", err)
 	}

diff --git a/pkg/graveler/committed/mock/range_manager.go b/pkg/graveler/committed/mock/range_manager.go
diff --git a/pkg/graveler/committed/range_manager.go b/pkg/graveler/committed/range_manager.go
@@ -60,7 +60,7 @@ type RangeManager interface {
 	NewRangeIterator(ctx context.Context, storageID StorageID, ns Namespace, pid ID) (ValueIterator, error)
 
 	// GetWriter returns a new Range writer instance
-	GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error)
+	GetWriter(ctx context.Context, storageID StorageID, ns Namespace, metadata graveler.Metadata) (RangeWriter, error)
 
 	// GetURI returns a URI from which to read the contents of id.  If id does not exist
 	// it may return a URI that resolves nowhere rather than an error.

diff --git a/pkg/graveler/graveler.go b/pkg/graveler/graveler.go
@@ -1018,7 +1018,7 @@ type CommittedManager interface {
 
 	// WriteRange creates a new Range from the iterator values.
 	// Keeps Range closing logic, so might not exhaust the iterator.
-	WriteRange(ctx context.Context, ns StorageNamespace, it ValueIterator) (*RangeInfo, error)
+	WriteRange(ctx context.Context, storageID StorageID, ns StorageNamespace, it ValueIterator) (*RangeInfo, error)
 
 	// WriteMetaRange creates a new MetaRange from the given Ranges.
 	WriteMetaRange(ctx context.Context, ns StorageNamespace, ranges []*RangeInfo) (*MetaRangeInfo, error)
@@ -1243,7 +1243,7 @@ func (g *Graveler) WriteRange(ctx context.Context, repository *RepositoryRecord,
 	if repository.ReadOnly && !options.Force {
 		return nil, ErrReadOnlyRepository
 	}
-	return g.CommittedManager.WriteRange(ctx, repository.StorageNamespace, it)
+	return g.CommittedManager.WriteRange(ctx, repository.StorageID, repository.StorageNamespace, it)
 }
 
 func (g *Graveler) WriteMetaRange(ctx context.Context, repository *RepositoryRecord, ranges []*RangeInfo, opts ...SetOptionsFunc) (*MetaRangeInfo, error) {

diff --git a/pkg/graveler/mock/graveler.go b/pkg/graveler/mock/graveler.go
diff --git a/pkg/graveler/sstable/range_manager.go b/pkg/graveler/sstable/range_manager.go
@@ -164,8 +164,8 @@ func (m *RangeManager) NewRangeIterator(ctx context.Context, storageID committed
 }
 
 // GetWriter returns a new SSTable writer instance
-func (m *RangeManager) GetWriter(ctx context.Context, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) {
-	return NewDiskWriter(ctx, m.fs, ns, m.hash.New(), metadata)
+func (m *RangeManager) GetWriter(ctx context.Context, storageID committed.StorageID, ns committed.Namespace, metadata graveler.Metadata) (committed.RangeWriter, error) {
+	return NewDiskWriter(ctx, m.fs, storageID, ns, m.hash.New(), metadata)
 }
 
 func (m *RangeManager) GetURI(ctx context.Context, ns committed.Namespace, id committed.ID) (string, error) {