From bafe34ff48f65094a8daf16168a0edad3f8ce6d0 Mon Sep 17 00:00:00 2001 From: Nir Ozery Date: Tue, 28 Jan 2025 12:44:31 -0500 Subject: [PATCH] StorageConfigList response changes --- modules/block/factory/build.go | 2 +- pkg/api/controller.go | 72 ++++++--- pkg/block/adapter.go | 31 ++++ pkg/block/factory/build.go | 3 +- pkg/block/params/block.go | 9 -- pkg/catalog/catalog.go | 5 +- pkg/config/config.go | 259 +++++++++++++++++++-------------- pkg/ingest/store/factory.go | 5 +- pkg/stats/metadata.go | 2 +- 9 files changed, 243 insertions(+), 145 deletions(-) diff --git a/modules/block/factory/build.go b/modules/block/factory/build.go index 2d18d253c16..d1327c11251 100644 --- a/modules/block/factory/build.go +++ b/modules/block/factory/build.go @@ -10,7 +10,7 @@ import ( ) func BuildBlockAdapter(ctx context.Context, statsCollector stats.Collector, c config.Config) (block.Adapter, error) { - adapter, err := factory.BuildBlockAdapter(ctx, statsCollector, c.GetBaseConfig()) + adapter, err := factory.BuildBlockAdapter(ctx, statsCollector, c.StorageConfig().GetStorageByID("")) if err != nil { return nil, err } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index ccdf2b7a64c..1d17914363a 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -32,6 +32,7 @@ import ( "github.com/treeverse/lakefs/pkg/auth/setup" "github.com/treeverse/lakefs/pkg/authentication" "github.com/treeverse/lakefs/pkg/block" + blockfactory "github.com/treeverse/lakefs/pkg/block/factory" "github.com/treeverse/lakefs/pkg/catalog" "github.com/treeverse/lakefs/pkg/cloud" "github.com/treeverse/lakefs/pkg/config" @@ -765,7 +766,7 @@ func (c *Controller) LinkPhysicalAddress(w http.ResponseWriter, r *http.Request, ifAbsent = true } - blockStoreType := c.BlockAdapter.BlockstoreType() + blockStoreType := c.Config.StorageConfig().GetStorageByID(repo.StorageID).BlockstoreType() expectedType := qk.GetStorageType().BlockstoreType() if expectedType != blockStoreType { c.Logger.WithContext(ctx).WithFields(logging.Fields{ @@ -1852,8 +1853,7 @@ func (c *Controller) GetConfig(w http.ResponseWriter, r *http.Request) { } storageCfg := c.getStorageConfig() - // TODO (niro): Needs to be populated - storageListCfg := apigen.StorageConfigList{} + storageListCfg := c.getStorageConfigList() versionConfig := c.getVersionConfig() writeResponse(w, r, http.StatusOK, apigen.Config{StorageConfig: &storageCfg, VersionConfig: &versionConfig, StorageConfigList: &storageListCfg}) } @@ -1872,22 +1872,54 @@ func (c *Controller) GetStorageConfig(w http.ResponseWriter, r *http.Request) { } func (c *Controller) getStorageConfig() apigen.StorageConfig { - info := c.BlockAdapter.GetStorageNamespaceInfo() - defaultNamespacePrefix := swag.String(info.DefaultNamespacePrefix) - if c.Config.GetBaseConfig().Blockstore.DefaultNamespacePrefix != nil { - defaultNamespacePrefix = c.Config.GetBaseConfig().Blockstore.DefaultNamespacePrefix - } + storageConfig := block.GetStorageConfig(c.BlockAdapter, c.Config.GetBaseConfig().Blockstore.DefaultNamespacePrefix) return apigen.StorageConfig{ - BlockstoreType: c.Config.GetBaseConfig().Blockstore.Type, - BlockstoreNamespaceValidityRegex: info.ValidityRegex, - BlockstoreNamespaceExample: info.Example, - DefaultNamespacePrefix: defaultNamespacePrefix, - PreSignSupport: info.PreSignSupport, - PreSignSupportUi: info.PreSignSupportUI, - ImportSupport: info.ImportSupport, - ImportValidityRegex: info.ImportValidityRegex, - PreSignMultipartUpload: swag.Bool(info.PreSignSupportMultipart), + BlockstoreType: storageConfig.Type, + BlockstoreNamespaceValidityRegex: storageConfig.NamespaceValidityRegex, + BlockstoreNamespaceExample: storageConfig.NamespaceExample, + DefaultNamespacePrefix: swag.String(storageConfig.DefaultNamespacePrefix), + PreSignSupport: storageConfig.PreSignSupport, + PreSignSupportUi: storageConfig.PreSignSupportUI, + ImportSupport: storageConfig.ImportSupport, + ImportValidityRegex: storageConfig.ImportValidityRegex, + PreSignMultipartUpload: swag.Bool(storageConfig.PreSignMultipartUpload), + } +} + +func (c *Controller) getStorageConfigList() apigen.StorageConfigList { + configList := apigen.StorageConfigList{} + storageConfig := c.Config.StorageConfig() + for _, id := range c.Config.StorageConfig().GetStorageIDs() { + storage := storageConfig.GetStorageByID(id) + if storage == nil { + c.Logger.Error("no storage found for id: %s", id) + continue + } + adapter, err := blockfactory.BuildBlockAdapter(context.Background(), nil, storage) + if err != nil { + c.Logger.WithError(err).WithField("storage_id", id). + Warn("Failed to initialize adapter") + continue + } + info := block.GetStorageConfig(adapter, storage.GetDefaultNamespacePrefix()) + configList = append(configList, apigen.StorageConfig{ + BlockstoreDescription: swag.String(storage.BlockstoreDescription()), + BlockstoreExtras: &apigen.StorageConfig_BlockstoreExtras{ + AdditionalProperties: storage.GetBlockstoreExtras(), + }, + BlockstoreId: swag.String(id), + BlockstoreNamespaceValidityRegex: info.NamespaceValidityRegex, + BlockstoreNamespaceExample: info.NamespaceExample, + BlockstoreType: info.Type, + DefaultNamespacePrefix: swag.String(info.DefaultNamespacePrefix), + ImportSupport: info.ImportSupport, + ImportValidityRegex: info.ImportValidityRegex, + PreSignMultipartUpload: swag.Bool(info.PreSignMultipartUpload), + PreSignSupport: info.PreSignSupport, + PreSignSupportUi: info.PreSignSupportUI, + }) } + return configList } func (c *Controller) HealthCheck(w http.ResponseWriter, r *http.Request) { @@ -2000,7 +2032,7 @@ func (c *Controller) CreateRepository(w http.ResponseWriter, r *http.Request, bo retErr = err reason = "bad_url" case errors.Is(err, block.ErrInvalidAddress): - retErr = fmt.Errorf("%w, must match: %s", err, c.BlockAdapter.BlockstoreType()) + retErr = fmt.Errorf("%w, must match: %s", err, c.Config.StorageConfig().GetStorageByID(storageID).BlockstoreType()) reason = "invalid_namespace" case errors.Is(err, ErrStorageNamespaceInUse): retErr = err @@ -3357,7 +3389,7 @@ func (c *Controller) StageObject(w http.ResponseWriter, r *http.Request, body ap uriRegex := c.BlockAdapter.GetStorageNamespaceInfo().ValidityRegex if match, err := regexp.MatchString(uriRegex, body.PhysicalAddress); err != nil || !match { writeError(w, r, http.StatusBadRequest, fmt.Sprintf("physical address is not valid for block adapter: %s", - c.BlockAdapter.BlockstoreType(), + c.Config.StorageConfig().GetStorageByID(repo.StorageID).BlockstoreType(), )) return } @@ -5145,7 +5177,7 @@ func (c *Controller) SetupCommPrefs(w http.ResponseWriter, r *http.Request, body InstallationID: installationID, FeatureUpdates: commPrefs.FeatureUpdates, SecurityUpdates: commPrefs.SecurityUpdates, - BlockstoreType: c.Config.GetBaseConfig().BlockstoreType(), + BlockstoreType: c.BlockAdapter.BlockstoreType(), } // collect comm prefs go c.Collector.CollectCommPrefs(commPrefsED) diff --git a/pkg/block/adapter.go b/pkg/block/adapter.go index 7f6db15b180..04f7b6f4a53 100644 --- a/pkg/block/adapter.go +++ b/pkg/block/adapter.go @@ -162,6 +162,19 @@ func (r *PutResponse) GetMtime() time.Time { return time.Now() } +type StorageConfig struct { + ID string + Type string + NamespaceValidityRegex string + NamespaceExample string + DefaultNamespacePrefix string + PreSignSupport bool + PreSignSupportUI bool + ImportSupport bool + ImportValidityRegex string + PreSignMultipartUpload bool +} + // Adapter abstract Storage Adapter for persistence of version controlled data. The methods generally map to S3 API methods // - Generally some type of Object Storage // - Can also be block storage or even in-memory @@ -204,3 +217,21 @@ type Adapter interface { RuntimeStats() map[string]string } + +func GetStorageConfig(adapter Adapter, defaultNamespacePrefix *string) StorageConfig { + info := adapter.GetStorageNamespaceInfo() + if defaultNamespacePrefix != nil { + info.DefaultNamespacePrefix = *defaultNamespacePrefix + } + return StorageConfig{ + Type: adapter.BlockstoreType(), + NamespaceValidityRegex: info.ValidityRegex, + NamespaceExample: info.Example, + DefaultNamespacePrefix: info.DefaultNamespacePrefix, + PreSignSupport: info.PreSignSupport, + PreSignSupportUI: info.PreSignSupportUI, + ImportSupport: info.ImportSupport, + ImportValidityRegex: info.ImportValidityRegex, + PreSignMultipartUpload: info.PreSignSupportMultipart, + } +} diff --git a/pkg/block/factory/build.go b/pkg/block/factory/build.go index 641e718a6b1..de4d134f7f0 100644 --- a/pkg/block/factory/build.go +++ b/pkg/block/factory/build.go @@ -15,6 +15,7 @@ import ( "github.com/treeverse/lakefs/pkg/block/params" s3a "github.com/treeverse/lakefs/pkg/block/s3" "github.com/treeverse/lakefs/pkg/block/transient" + "github.com/treeverse/lakefs/pkg/config" "github.com/treeverse/lakefs/pkg/logging" "github.com/treeverse/lakefs/pkg/stats" "golang.org/x/oauth2/google" @@ -26,7 +27,7 @@ const ( googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform" ) -func BuildBlockAdapter(ctx context.Context, statsCollector stats.Collector, c params.AdapterConfig) (block.Adapter, error) { +func BuildBlockAdapter(ctx context.Context, statsCollector stats.Collector, c config.AdapterConfig) (block.Adapter, error) { blockstore := strings.ToLower(c.BlockstoreType()) logging.FromContext(ctx). WithField("type", blockstore). diff --git a/pkg/block/params/block.go b/pkg/block/params/block.go index 335a259c808..ef7aec14758 100644 --- a/pkg/block/params/block.go +++ b/pkg/block/params/block.go @@ -4,15 +4,6 @@ import ( "time" ) -// AdapterConfig configures a block adapter. -type AdapterConfig interface { - BlockstoreType() string - BlockstoreLocalParams() (Local, error) - BlockstoreS3Params() (S3, error) - BlockstoreGSParams() (GS, error) - BlockstoreAzureParams() (Azure, error) -} - type Mem struct{} type Local struct { diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 009204130e6..ed183da3c1d 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -23,9 +23,9 @@ import ( "github.com/hashicorp/go-multierror" lru "github.com/hnlq715/golang-lru" "github.com/rs/xid" + blockfactory "github.com/treeverse/lakefs/modules/block/factory" "github.com/treeverse/lakefs/pkg/batch" "github.com/treeverse/lakefs/pkg/block" - blockfactory "github.com/treeverse/lakefs/pkg/block/factory" "github.com/treeverse/lakefs/pkg/config" "github.com/treeverse/lakefs/pkg/graveler" "github.com/treeverse/lakefs/pkg/graveler/branch" @@ -312,8 +312,9 @@ func New(ctx context.Context, cfg Config) (*Catalog, error) { cancelFn() return nil, fmt.Errorf("build block adapter: %w", err) } + // TODO (niro): This part will break using multiple blockstores. We should change the catalog logic to get the walker from the adapter if cfg.WalkerFactory == nil { - cfg.WalkerFactory = store.NewFactory(cfg.Config) + cfg.WalkerFactory = store.NewFactory(&cfg.Config.Blockstore) } tierFSParams, err := pyramidparams.NewCommittedTierFSParams(cfg.Config, adapter) diff --git a/pkg/config/config.go b/pkg/config/config.go index a2fd73a95a3..24fc74a7d4d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -149,6 +149,18 @@ type ApproximatelyCorrectOwnership struct { Acquire time.Duration `mapstructure:"acquire"` } +// AdapterConfig configures a blockstore adapter. +type AdapterConfig interface { + BlockstoreType() string + BlockstoreDescription() string + BlockstoreLocalParams() (blockparams.Local, error) + BlockstoreS3Params() (blockparams.S3, error) + BlockstoreGSParams() (blockparams.GS, error) + BlockstoreAzureParams() (blockparams.Azure, error) + GetDefaultNamespacePrefix() *string + GetBlockstoreExtras() map[string]string +} + type Blockstore struct { Signing struct { SecretKey SecureString `mapstructure:"secret_key"` @@ -211,12 +223,147 @@ type Blockstore struct { } `mapstructure:"gs"` } +func (b *Blockstore) GetStorageByID(id string) AdapterConfig { + if id != "" { + return nil + } + + return b +} + +func (b *Blockstore) BlockstoreType() string { + return b.Type +} + +func (b *Blockstore) BlockstoreS3Params() (blockparams.S3, error) { + var webIdentity *blockparams.S3WebIdentity + if b.S3.WebIdentity != nil { + webIdentity = &blockparams.S3WebIdentity{ + SessionDuration: b.S3.WebIdentity.SessionDuration, + SessionExpiryWindow: b.S3.WebIdentity.SessionExpiryWindow, + } + } + + var creds blockparams.S3Credentials + if b.S3.Credentials != nil { + creds.AccessKeyID = b.S3.Credentials.AccessKeyID.SecureValue() + creds.SecretAccessKey = b.S3.Credentials.SecretAccessKey.SecureValue() + creds.SessionToken = b.S3.Credentials.SessionToken.SecureValue() + } + + return blockparams.S3{ + Region: b.S3.Region, + Profile: b.S3.Profile, + CredentialsFile: b.S3.CredentialsFile, + Credentials: creds, + MaxRetries: b.S3.MaxRetries, + Endpoint: b.S3.Endpoint, + ForcePathStyle: b.S3.ForcePathStyle, + DiscoverBucketRegion: b.S3.DiscoverBucketRegion, + SkipVerifyCertificateTestOnly: b.S3.SkipVerifyCertificateTestOnly, + ServerSideEncryption: b.S3.ServerSideEncryption, + ServerSideEncryptionKmsKeyID: b.S3.ServerSideEncryptionKmsKeyID, + PreSignedExpiry: b.S3.PreSignedExpiry, + PreSignedEndpoint: b.S3.PreSignedEndpoint, + DisablePreSigned: b.S3.DisablePreSigned, + DisablePreSignedUI: b.S3.DisablePreSignedUI, + DisablePreSignedMultipart: b.S3.DisablePreSignedMultipart, + ClientLogRetries: b.S3.ClientLogRetries, + ClientLogRequest: b.S3.ClientLogRequest, + WebIdentity: webIdentity, + }, nil +} + +func (b *Blockstore) BlockstoreLocalParams() (blockparams.Local, error) { + localPath := b.Local.Path + path, err := homedir.Expand(localPath) + if err != nil { + return blockparams.Local{}, fmt.Errorf("parse blockstore location URI %s: %w", localPath, err) + } + + params := blockparams.Local(*b.Local) + params.Path = path + return params, nil +} + +func (b *Blockstore) BlockstoreGSParams() (blockparams.GS, error) { + var customerSuppliedKey []byte = nil + if b.GS.ServerSideEncryptionCustomerSupplied != "" { + v, err := hex.DecodeString(b.GS.ServerSideEncryptionCustomerSupplied) + if err != nil { + return blockparams.GS{}, err + } + if len(v) != gcpAESKeyLength { + return blockparams.GS{}, ErrBadGCPCSEKValue + } + customerSuppliedKey = v + if b.GS.ServerSideEncryptionKmsKeyID != "" { + return blockparams.GS{}, ErrGCPEncryptKeyConflict + } + } + + credPath, err := homedir.Expand(b.GS.CredentialsFile) + if err != nil { + return blockparams.GS{}, fmt.Errorf("parse GS credentials path '%s': %w", b.GS.CredentialsFile, err) + } + return blockparams.GS{ + CredentialsFile: credPath, + CredentialsJSON: b.GS.CredentialsJSON, + PreSignedExpiry: b.GS.PreSignedExpiry, + DisablePreSigned: b.GS.DisablePreSigned, + DisablePreSignedUI: b.GS.DisablePreSignedUI, + ServerSideEncryptionCustomerSupplied: customerSuppliedKey, + ServerSideEncryptionKmsKeyID: b.GS.ServerSideEncryptionKmsKeyID, + }, nil +} + +func (b *Blockstore) BlockstoreAzureParams() (blockparams.Azure, error) { + if b.Azure.AuthMethod != "" { + logging.ContextUnavailable().Warn("blockstore.azure.auth_method is deprecated. Value is no longer used.") + } + if b.Azure.ChinaCloudDeprecated { + logging.ContextUnavailable().Warn("blockstore.azure.china_cloud is deprecated. Value is no longer used. Please pass Domain = 'blob.core.chinacloudapi.cn'") + b.Azure.Domain = "blob.core.chinacloudapi.cn" + } + return blockparams.Azure{ + StorageAccount: b.Azure.StorageAccount, + StorageAccessKey: b.Azure.StorageAccessKey, + TryTimeout: b.Azure.TryTimeout, + PreSignedExpiry: b.Azure.PreSignedExpiry, + TestEndpointURL: b.Azure.TestEndpointURL, + Domain: b.Azure.Domain, + DisablePreSigned: b.Azure.DisablePreSigned, + DisablePreSignedUI: b.Azure.DisablePreSignedUI, + }, nil +} + +func (b *Blockstore) BlockstoreDescription() string { + return "" +} + +func (b *Blockstore) GetStorageIDs() []string { + return []string{""} +} + +func (b *Blockstore) GetDefaultNamespacePrefix() *string { + return b.DefaultNamespacePrefix +} + +func (b *Blockstore) GetBlockstoreExtras() map[string]string { + return nil +} + type Config interface { GetBaseConfig() *BaseConfig - StorageConfig() interface{} + StorageConfig() StorageConfig Validate() error } +type StorageConfig interface { + GetStorageByID(storageID string) AdapterConfig + GetStorageIDs() []string +} + // BaseConfig - Output struct of configuration, used to validate. If you read a key using a viper accessor // rather than accessing a field of this struct, that key will *not* be validated. So don't // do that. @@ -480,116 +627,10 @@ func (c *BaseConfig) Validate() error { return ValidateBlockstore(&c.Blockstore) } -func (c *BaseConfig) BlockstoreType() string { - return c.Blockstore.Type -} - -func (c *BaseConfig) BlockstoreS3Params() (blockparams.S3, error) { - var webIdentity *blockparams.S3WebIdentity - if c.Blockstore.S3.WebIdentity != nil { - webIdentity = &blockparams.S3WebIdentity{ - SessionDuration: c.Blockstore.S3.WebIdentity.SessionDuration, - SessionExpiryWindow: c.Blockstore.S3.WebIdentity.SessionExpiryWindow, - } - } - - var creds blockparams.S3Credentials - if c.Blockstore.S3.Credentials != nil { - creds.AccessKeyID = c.Blockstore.S3.Credentials.AccessKeyID.SecureValue() - creds.SecretAccessKey = c.Blockstore.S3.Credentials.SecretAccessKey.SecureValue() - creds.SessionToken = c.Blockstore.S3.Credentials.SessionToken.SecureValue() - } - - return blockparams.S3{ - Region: c.Blockstore.S3.Region, - Profile: c.Blockstore.S3.Profile, - CredentialsFile: c.Blockstore.S3.CredentialsFile, - Credentials: creds, - MaxRetries: c.Blockstore.S3.MaxRetries, - Endpoint: c.Blockstore.S3.Endpoint, - ForcePathStyle: c.Blockstore.S3.ForcePathStyle, - DiscoverBucketRegion: c.Blockstore.S3.DiscoverBucketRegion, - SkipVerifyCertificateTestOnly: c.Blockstore.S3.SkipVerifyCertificateTestOnly, - ServerSideEncryption: c.Blockstore.S3.ServerSideEncryption, - ServerSideEncryptionKmsKeyID: c.Blockstore.S3.ServerSideEncryptionKmsKeyID, - PreSignedExpiry: c.Blockstore.S3.PreSignedExpiry, - PreSignedEndpoint: c.Blockstore.S3.PreSignedEndpoint, - DisablePreSigned: c.Blockstore.S3.DisablePreSigned, - DisablePreSignedUI: c.Blockstore.S3.DisablePreSignedUI, - DisablePreSignedMultipart: c.Blockstore.S3.DisablePreSignedMultipart, - ClientLogRetries: c.Blockstore.S3.ClientLogRetries, - ClientLogRequest: c.Blockstore.S3.ClientLogRequest, - WebIdentity: webIdentity, - }, nil -} - -func (c *BaseConfig) BlockstoreLocalParams() (blockparams.Local, error) { - localPath := c.Blockstore.Local.Path - path, err := homedir.Expand(localPath) - if err != nil { - return blockparams.Local{}, fmt.Errorf("parse blockstore location URI %s: %w", localPath, err) - } - - params := blockparams.Local(*c.Blockstore.Local) - params.Path = path - return params, nil -} - const ( gcpAESKeyLength = 32 ) -func (c *BaseConfig) BlockstoreGSParams() (blockparams.GS, error) { - var customerSuppliedKey []byte = nil - if c.Blockstore.GS.ServerSideEncryptionCustomerSupplied != "" { - v, err := hex.DecodeString(c.Blockstore.GS.ServerSideEncryptionCustomerSupplied) - if err != nil { - return blockparams.GS{}, err - } - if len(v) != gcpAESKeyLength { - return blockparams.GS{}, ErrBadGCPCSEKValue - } - customerSuppliedKey = v - if c.Blockstore.GS.ServerSideEncryptionKmsKeyID != "" { - return blockparams.GS{}, ErrGCPEncryptKeyConflict - } - } - - credPath, err := homedir.Expand(c.Blockstore.GS.CredentialsFile) - if err != nil { - return blockparams.GS{}, fmt.Errorf("parse GS credentials path '%s': %w", c.Blockstore.GS.CredentialsFile, err) - } - return blockparams.GS{ - CredentialsFile: credPath, - CredentialsJSON: c.Blockstore.GS.CredentialsJSON, - PreSignedExpiry: c.Blockstore.GS.PreSignedExpiry, - DisablePreSigned: c.Blockstore.GS.DisablePreSigned, - DisablePreSignedUI: c.Blockstore.GS.DisablePreSignedUI, - ServerSideEncryptionCustomerSupplied: customerSuppliedKey, - ServerSideEncryptionKmsKeyID: c.Blockstore.GS.ServerSideEncryptionKmsKeyID, - }, nil -} - -func (c *BaseConfig) BlockstoreAzureParams() (blockparams.Azure, error) { - if c.Blockstore.Azure.AuthMethod != "" { - logging.ContextUnavailable().Warn("blockstore.azure.auth_method is deprecated. Value is no longer used.") - } - if c.Blockstore.Azure.ChinaCloudDeprecated { - logging.ContextUnavailable().Warn("blockstore.azure.china_cloud is deprecated. Value is no longer used. Please pass Domain = 'blob.core.chinacloudapi.cn'") - c.Blockstore.Azure.Domain = "blob.core.chinacloudapi.cn" - } - return blockparams.Azure{ - StorageAccount: c.Blockstore.Azure.StorageAccount, - StorageAccessKey: c.Blockstore.Azure.StorageAccessKey, - TryTimeout: c.Blockstore.Azure.TryTimeout, - PreSignedExpiry: c.Blockstore.Azure.PreSignedExpiry, - TestEndpointURL: c.Blockstore.Azure.TestEndpointURL, - Domain: c.Blockstore.Azure.Domain, - DisablePreSigned: c.Blockstore.Azure.DisablePreSigned, - DisablePreSignedUI: c.Blockstore.Azure.DisablePreSignedUI, - }, nil -} - const ( AuthRBACNone = "none" AuthRBACSimplified = "simplified" @@ -644,6 +685,6 @@ func (c *BaseConfig) GetBaseConfig() *BaseConfig { return c } -func (c *BaseConfig) StorageConfig() interface{} { - return c.Blockstore +func (c *BaseConfig) StorageConfig() StorageConfig { + return &c.Blockstore } diff --git a/pkg/ingest/store/factory.go b/pkg/ingest/store/factory.go index b65c0a981be..a236714c9ea 100644 --- a/pkg/ingest/store/factory.go +++ b/pkg/ingest/store/factory.go @@ -17,6 +17,7 @@ import ( "github.com/treeverse/lakefs/pkg/block/local" "github.com/treeverse/lakefs/pkg/block/params" "github.com/treeverse/lakefs/pkg/block/s3" + "github.com/treeverse/lakefs/pkg/config" ) var ErrNotSupported = errors.New("no storage adapter found") @@ -52,10 +53,10 @@ func (ww *WalkerWrapper) GetSkippedEntries() []block.ObjectStoreEntry { } type WalkerFactory struct { - params params.AdapterConfig + params config.AdapterConfig } -func NewFactory(params params.AdapterConfig) *WalkerFactory { +func NewFactory(params config.AdapterConfig) *WalkerFactory { return &WalkerFactory{params: params} } diff --git a/pkg/stats/metadata.go b/pkg/stats/metadata.go index 81ff245aecc..0cda877cd17 100644 --- a/pkg/stats/metadata.go +++ b/pkg/stats/metadata.go @@ -60,7 +60,7 @@ func BuildMetadataProvider(logger logging.Logger, c *config.BaseConfig) cloud.Me case block.BlockstoreTypeGS: return gcp.NewMetadataProvider(logger) case block.BlockstoreTypeS3: - s3Params, err := c.BlockstoreS3Params() + s3Params, err := c.Blockstore.BlockstoreS3Params() if err != nil { logger.WithError(err).Warn("Failed to create S3 client for MetadataProvider") return &noopMetadataProvider{}