diff --git a/cmd/lakefs-loadtest/cmd/entry.go b/cmd/lakefs-loadtest/cmd/entry.go index af2f9ba96a1..f215675551b 100644 --- a/cmd/lakefs-loadtest/cmd/entry.go +++ b/cmd/lakefs-loadtest/cmd/entry.go @@ -13,8 +13,8 @@ import ( nanoid "github.com/matoous/go-nanoid/v2" "github.com/schollz/progressbar/v3" "github.com/spf13/cobra" + configfactory "github.com/treeverse/lakefs/modules/config/factory" "github.com/treeverse/lakefs/pkg/catalog" - "github.com/treeverse/lakefs/pkg/config" "github.com/treeverse/lakefs/pkg/kv" "github.com/treeverse/lakefs/pkg/kv/kvparams" "github.com/treeverse/lakefs/pkg/logging" @@ -50,10 +50,12 @@ var entryCmd = &cobra.Command{ ctx := cmd.Context() - conf, err := config.NewConfig("") + confInterface, err := configfactory.BuildConfig("") if err != nil { fmt.Printf("config: %s\n", err) } + conf := confInterface.GetBaseConfig() + err = conf.Validate() if err != nil { fmt.Printf("invalid config: %s\n", err) diff --git a/cmd/lakefs/cmd/flare.go b/cmd/lakefs/cmd/flare.go index 4276fcaba4c..66e0222d8aa 100644 --- a/cmd/lakefs/cmd/flare.go +++ b/cmd/lakefs/cmd/flare.go @@ -39,7 +39,7 @@ var flareCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { flare.SetBaselinePermissions(flare.FlareUmask) now := strings.ReplaceAll(time.Now().String(), " ", "") - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() envVarBlacklist := addAppEnvVarPrefix(config.GetSecureStringKeyPaths(cfg)) flr, err := flare.NewFlare(flare.WithEnvVarBlacklist(envVarBlacklist)) if err != nil { diff --git a/cmd/lakefs/cmd/kv.go b/cmd/lakefs/cmd/kv.go index de254714ea6..dadbe13f92a 100644 --- a/cmd/lakefs/cmd/kv.go +++ b/cmd/lakefs/cmd/kv.go @@ -35,7 +35,7 @@ var kvGetCmd = &cobra.Command{ Short: "Return the value for the given path under the given partition", Args: cobra.ExactArgs(GetCmdNumArgs), RunE: func(cmd *cobra.Command, args []string) error { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() pretty, err := cmd.Flags().GetBool("pretty") if err != nil { @@ -83,7 +83,7 @@ var kvScanCmd = &cobra.Command{ Short: "Scan through keys and values under the given partition. An optional path can be specified as a starting point (inclusive)", Args: cobra.RangeArgs(ScanCmdMinArgs, ScanCmdMaxArgs), RunE: func(cmd *cobra.Command, args []string) error { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() limit, err := cmd.Flags().GetInt("limit") if err != nil { diff --git a/cmd/lakefs/cmd/migrate.go b/cmd/lakefs/cmd/migrate.go index 4469a7537f3..7ec0b3ad875 100644 --- a/cmd/lakefs/cmd/migrate.go +++ b/cmd/lakefs/cmd/migrate.go @@ -22,7 +22,7 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print current migration version and available version", Run: func(cmd *cobra.Command, args []string) { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() kvParams, err := kvparams.NewConfig(&cfg.Database) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "KV params: %s\n", err) @@ -63,7 +63,7 @@ var upCmd = &cobra.Command{ Use: "up", Short: "Apply all up migrations", Run: func(cmd *cobra.Command, args []string) { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() kvParams, err := kvparams.NewConfig(&cfg.Database) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "KV params: %s\n", err) @@ -100,7 +100,7 @@ var upCmd = &cobra.Command{ }, } -func DoMigration(ctx context.Context, kvStore kv.Store, _ *config.Config, _ bool) error { +func DoMigration(ctx context.Context, kvStore kv.Store, _ *config.BaseConfig, _ bool) error { var ( version int err error @@ -136,7 +136,7 @@ var gotoCmd = &cobra.Command{ Use: "goto", Short: "Migrate to version V.", Run: func(cmd *cobra.Command, args []string) { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() kvParams, err := kvparams.NewConfig(&cfg.Database) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "KV params: %s\n", err) diff --git a/cmd/lakefs/cmd/migrate_test.go b/cmd/lakefs/cmd/migrate_test.go index 7a0e6663ffd..495dd69d4ec 100644 --- a/cmd/lakefs/cmd/migrate_test.go +++ b/cmd/lakefs/cmd/migrate_test.go @@ -33,7 +33,7 @@ func TestDoMigrate(t *testing.T) { }) t.Run("initial_kv_version", func(t *testing.T) { - cfg := config.Config{} + cfg := config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified cfg.Auth.Encrypt.SecretKey = "test" kvStore := kvtest.GetStore(ctx, t) @@ -46,7 +46,7 @@ func TestDoMigrate(t *testing.T) { }) t.Run("from_acl_v1_force", func(t *testing.T) { - cfg := config.Config{} + cfg := config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified kvStore := kvtest.GetStore(ctx, t) require.NoError(t, kv.SetDBSchemaVersion(ctx, kvStore, kv.ACLNoReposMigrateVersion)) @@ -58,7 +58,7 @@ func TestDoMigrate(t *testing.T) { }) t.Run("from_acl_v2", func(t *testing.T) { - cfg := config.Config{} + cfg := config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified startVer := kv.ACLNoReposMigrateVersion for !kv.IsLatestSchemaVersion(startVer) { @@ -74,7 +74,7 @@ func TestDoMigrate(t *testing.T) { }) t.Run("latest_version", func(t *testing.T) { - cfg := config.Config{} + cfg := config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified kvStore := kvtest.GetStore(ctx, t) require.NoError(t, kv.SetDBSchemaVersion(ctx, kvStore, kv.NextSchemaVersion-1)) @@ -86,7 +86,7 @@ func TestDoMigrate(t *testing.T) { }) t.Run("next_version", func(t *testing.T) { - cfg := config.Config{} + cfg := config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACSimplified kvStore := kvtest.GetStore(ctx, t) require.NoError(t, kv.SetDBSchemaVersion(ctx, kvStore, kv.NextSchemaVersion)) diff --git a/cmd/lakefs/cmd/root.go b/cmd/lakefs/cmd/root.go index 15f21a0f101..bf87214868c 100644 --- a/cmd/lakefs/cmd/root.go +++ b/cmd/lakefs/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" + configfactory "github.com/treeverse/lakefs/modules/config/factory" "github.com/treeverse/lakefs/pkg/block" "github.com/treeverse/lakefs/pkg/config" "github.com/treeverse/lakefs/pkg/kv/local" @@ -52,7 +53,9 @@ func init() { rootCmd.PersistentFlags().Bool(config.QuickstartConfiguration, false, "Use lakeFS quickstart configuration") } -func validateQuickstartEnv(cfg *config.Config) { +// TODO (niro): All this validation logic should be in the config package + +func validateQuickstartEnv(cfg *config.BaseConfig) { if (cfg.Database.Type != local.DriverName && cfg.Database.Type != mem.DriverName) || cfg.Blockstore.Type != block.BlockstoreTypeLocal { _, _ = fmt.Fprint(os.Stderr, "\nFATAL: quickstart mode can only run with local settings\n") os.Exit(1) @@ -78,27 +81,38 @@ func useConfig(flagName string) bool { return res } -func newConfig() (*config.Config, error) { +func newConfig() (config.Config, error) { name := "" configurations := []string{config.QuickstartConfiguration, config.UseLocalConfiguration} if idx := slices.IndexFunc(configurations, useConfig); idx != -1 { name = configurations[idx] } - cfg, err := config.NewConfig(name) + cfg, err := configfactory.BuildConfig(name) if err != nil { return nil, err } if name == config.QuickstartConfiguration { - validateQuickstartEnv(cfg) + validateQuickstartEnv(cfg.GetBaseConfig()) } - return cfg, nil + return cfg.GetBaseConfig(), nil } -func loadConfig() *config.Config { - initOnce.Do(initConfig) +func loadConfig() config.Config { + log := logging.ContextUnavailable().WithField("phase", "startup") + initOnce.Do(func() { + initConfig(log) + }) + // setup config used by the executed command cfg, err := newConfig() + if err != nil { + log.WithError(err).Fatal("Load config") + } else { + log.Info("Config loaded") + } + + log.WithFields(config.MapLoggingFields(cfg)).Info("Config") if err != nil { fmt.Println("Failed to load config file", err) os.Exit(1) @@ -107,10 +121,9 @@ func loadConfig() *config.Config { } // initConfig reads in config file and ENV variables if set. -func initConfig() { - logger := logging.ContextUnavailable().WithField("phase", "startup") +func initConfig(log logging.Logger) { if cfgFile != "" { - logger.WithField("file", cfgFile).Info("Configuration file") + log.WithField("file", cfgFile).Info("Configuration file") // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { @@ -128,10 +141,10 @@ func initConfig() { // read the configuration file err := viper.ReadInConfig() - logger = logger.WithField("file", viper.ConfigFileUsed()) // should be called after SetConfigFile + log = log.WithField("file", viper.ConfigFileUsed()) // should be called after SetConfigFile var errFileNotFound viper.ConfigFileNotFoundError if err != nil && !errors.As(err, &errFileNotFound) { - logger.WithError(err).Fatal("Failed to find a config file") + log.WithError(err).Fatal("Failed to find a config file") } // fallback - try to load the previous supported $HOME/.lakefs.yaml // if err is set it will be file-not-found based on the previous check @@ -139,28 +152,13 @@ func initConfig() { fallbackCfgFile := path.Join(getHomeDir(), ".lakefs.yaml") if cfgFile != fallbackCfgFile { viper.SetConfigFile(fallbackCfgFile) - logger = logger.WithField("file", viper.ConfigFileUsed()) // should be called after SetConfigFile + log = log.WithField("file", viper.ConfigFileUsed()) // should be called after SetConfigFile err = viper.ReadInConfig() if err != nil && !os.IsNotExist(err) { - logger.WithError(err).Fatal("Failed to read config file") + log.WithError(err).Fatal("Failed to read config file") } } } - - // setup config used by the executed command - cfg, err := newConfig() - if err != nil { - logger.WithError(err).Fatal("Load config") - } else { - logger.Info("Config loaded") - } - - err = cfg.Validate() - if err != nil { - logger.WithError(err).Fatal("Invalid config") - } - - logger.WithFields(config.MapLoggingFields(cfg)).Info("Config") } // getHomeDir find and return the home directory diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go index ce3cdb63460..0acd03a2221 100644 --- a/cmd/lakefs/cmd/run.go +++ b/cmd/lakefs/cmd/run.go @@ -62,7 +62,7 @@ var ( errInvalidAuth = errors.New("invalid auth configuration") ) -func checkAuthModeSupport(cfg *config.Config) error { +func checkAuthModeSupport(cfg *config.BaseConfig) error { if cfg.IsAuthBasic() { // Basic mode return nil } @@ -75,7 +75,7 @@ func checkAuthModeSupport(cfg *config.Config) error { return nil } -func NewAuthService(ctx context.Context, cfg *config.Config, logger logging.Logger, kvStore kv.Store, metadataManager *auth.KVMetadataManager) auth.Service { +func NewAuthService(ctx context.Context, cfg *config.BaseConfig, logger logging.Logger, kvStore kv.Store, metadataManager *auth.KVMetadataManager) auth.Service { if err := checkAuthModeSupport(cfg); err != nil { logger.WithError(err).Fatal("Unsupported auth mode") } @@ -137,10 +137,10 @@ var runCmd = &cobra.Command{ Short: "Run lakeFS", Run: func(cmd *cobra.Command, args []string) { logger := logging.ContextUnavailable() - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() viper.WatchConfig() viper.OnConfigChange(func(in fsnotify.Event) { - var c config.Config + var c config.BaseConfig if err := config.Unmarshal(&c); err != nil { logger.WithError(err).Error("Failed to unmarshal config while reload") return diff --git a/cmd/lakefs/cmd/run_test.go b/cmd/lakefs/cmd/run_test.go index 77dcf89faf3..2ce0fe9d5f6 100644 --- a/cmd/lakefs/cmd/run_test.go +++ b/cmd/lakefs/cmd/run_test.go @@ -13,7 +13,7 @@ import ( func TestGetAuthService(t *testing.T) { t.Run("maintain_inviter", func(t *testing.T) { - cfg := &config.Config{} + cfg := &config.BaseConfig{} cfg.Auth.UIConfig.RBAC = config.AuthRBACInternal cfg.Auth.API.Endpoint = "http://localhost:8000" cfg.Auth.API.SkipHealthCheck = true @@ -24,7 +24,7 @@ func TestGetAuthService(t *testing.T) { } }) t.Run("maintain_service", func(t *testing.T) { - cfg := &config.Config{} + cfg := &config.BaseConfig{} kvStore := kvtest.GetStore(context.Background(), t) meta := auth.NewKVMetadataManager("serve_test", cfg.Installation.FixedID, cfg.Database.Type, kvStore) cfg.Auth.UIConfig.RBAC = config.AuthRBACNone diff --git a/cmd/lakefs/cmd/setup.go b/cmd/lakefs/cmd/setup.go index 46633557484..e88e6e942af 100644 --- a/cmd/lakefs/cmd/setup.go +++ b/cmd/lakefs/cmd/setup.go @@ -23,7 +23,7 @@ var setupCmd = &cobra.Command{ Aliases: []string{"init"}, Short: "Setup a new lakeFS instance with initial credentials", Run: func(cmd *cobra.Command, args []string) { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() ctx := cmd.Context() kvParams, err := kvparams.NewConfig(&cfg.Database) @@ -100,7 +100,7 @@ var setupCmd = &cobra.Command{ }, } -func setupLakeFS(ctx context.Context, cfg *config.Config, metadataManager auth.MetadataManager, authService auth.Service, userName string, accessKeyID string, secretAccessKey string) (*model.Credential, error) { +func setupLakeFS(ctx context.Context, cfg *config.BaseConfig, metadataManager auth.MetadataManager, authService auth.Service, userName string, accessKeyID string, secretAccessKey string) (*model.Credential, error) { initialized, err := metadataManager.IsInitialized(ctx) if err != nil || initialized { // return on error or if already initialized diff --git a/cmd/lakefs/cmd/superuser.go b/cmd/lakefs/cmd/superuser.go index fbc039911b8..b67470ad5bf 100644 --- a/cmd/lakefs/cmd/superuser.go +++ b/cmd/lakefs/cmd/superuser.go @@ -31,7 +31,7 @@ To do that provide the user name as well as the access key ID to import. If the wrong user or credentials were chosen it is possible to delete the user and perform the action again. `, Run: func(cmd *cobra.Command, args []string) { - cfg := loadConfig() + cfg := loadConfig().GetBaseConfig() if cfg.Auth.UIConfig.RBAC == config.AuthRBACExternal { fmt.Printf("Can't create additional admin while using external auth API - auth.api.endpoint is configured.\n") os.Exit(1) diff --git a/go.work b/go.work index 737e8e235ff..f9c81d77d23 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,7 @@ go 1.23 use ( . - ./modules/block/factory ./webui + ./modules/block/factory + ./modules/config/factory ) diff --git a/modules/config/factory/build.go b/modules/config/factory/build.go new file mode 100644 index 00000000000..789ad480496 --- /dev/null +++ b/modules/config/factory/build.go @@ -0,0 +1,25 @@ +package factory + +import ( + "github.com/treeverse/lakefs/pkg/config" +) + +func BuildConfig(cfgType string) (config.Config, error) { + c := &config.BaseConfig{} + c, err := config.NewConfig(cfgType, c) + if err != nil { + return nil, err + } + + // Perform required validations + if err = c.Validate(); err != nil { + return nil, err + } + + err = c.ValidateDomainNames() + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/modules/config/factory/go.mod b/modules/config/factory/go.mod new file mode 100644 index 00000000000..5f434528397 --- /dev/null +++ b/modules/config/factory/go.mod @@ -0,0 +1,3 @@ +module github.com/treeverse/lakefs/modules/config/factory + +go 1.23 diff --git a/pkg/api/controller.go b/pkg/api/controller.go index b4e7a9867da..8c0ce67be80 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -88,7 +88,7 @@ type Migrator interface { } type Controller struct { - Config *config.Config + Config *config.BaseConfig Catalog *catalog.Catalog Authenticator auth.Authenticator Auth auth.Service @@ -108,7 +108,7 @@ type Controller struct { var usageCounter = stats.NewUsageCounter() -func NewController(cfg *config.Config, catalog *catalog.Catalog, authenticator auth.Authenticator, authService auth.Service, authenticationService authentication.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, sessionStore sessions.Store, pathProvider upload.PathProvider, usageReporter stats.UsageReporterOperations) *Controller { +func NewController(cfg *config.BaseConfig, catalog *catalog.Catalog, authenticator auth.Authenticator, authService auth.Service, authenticationService authentication.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, sessionStore sessions.Store, pathProvider upload.PathProvider, usageReporter stats.UsageReporterOperations) *Controller { return &Controller{ Config: cfg, Catalog: catalog, @@ -4948,7 +4948,7 @@ func (c *Controller) GetTag(w http.ResponseWriter, r *http.Request, repository, writeResponse(w, r, http.StatusOK, response) } -func newLoginConfig(c *config.Config) *apigen.LoginConfig { +func newLoginConfig(c *config.BaseConfig) *apigen.LoginConfig { return &apigen.LoginConfig{ RBAC: &c.Auth.UIConfig.RBAC, LoginUrl: c.Auth.UIConfig.LoginURL, diff --git a/pkg/api/serve.go b/pkg/api/serve.go index 51fac13a3ea..b396e73d1d4 100644 --- a/pkg/api/serve.go +++ b/pkg/api/serve.go @@ -33,7 +33,7 @@ const ( extensionValidationExcludeBody = "x-validation-exclude-body" ) -func Serve(cfg *config.Config, catalog *catalog.Catalog, middlewareAuthenticator auth.Authenticator, authService auth.Service, authenticationService authentication.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, gatewayDomains []string, snippets []params.CodeSnippet, pathProvider upload.PathProvider, usageReporter stats.UsageReporterOperations) http.Handler { +func Serve(cfg *config.BaseConfig, catalog *catalog.Catalog, middlewareAuthenticator auth.Authenticator, authService auth.Service, authenticationService authentication.Service, blockAdapter block.Adapter, metadataManager auth.MetadataManager, migrator Migrator, collector stats.Collector, cloudMetadataProvider cloud.MetadataProvider, actions actionsHandler, auditChecker AuditChecker, logger logging.Logger, gatewayDomains []string, snippets []params.CodeSnippet, pathProvider upload.PathProvider, usageReporter stats.UsageReporterOperations) http.Handler { logger.Info("initialize OpenAPI server") swagger, err := apigen.GetSwagger() if err != nil { diff --git a/pkg/api/serve_test.go b/pkg/api/serve_test.go index 3a6cf44a88d..91e01faebfb 100644 --- a/pkg/api/serve_test.go +++ b/pkg/api/serve_test.go @@ -112,8 +112,8 @@ func setupHandler(t testing.TB) (http.Handler, *dependencies) { viper.Set("auth.api.endpoint", config.DefaultListenAddress) collector := &memCollector{} - - cfg, err := config.NewConfig("") + cfg := &config.BaseConfig{} + cfg, err := config.NewConfig("", cfg) testutil.MustDo(t, "config", err) kvStore := kvtest.GetStore(ctx, t) factory := store.NewFactory(nil) diff --git a/pkg/auth/setup/setup.go b/pkg/auth/setup/setup.go index 9ace1369c17..fd86505a4d0 100644 --- a/pkg/auth/setup/setup.go +++ b/pkg/auth/setup/setup.go @@ -156,7 +156,7 @@ func CreateRBACBaseGroups(ctx context.Context, authService auth.Service, ts time } // CreateAdminUser setup base groups, policies and create admin user -func CreateAdminUser(ctx context.Context, authService auth.Service, cfg *config.Config, superuser *model.SuperuserConfiguration) (*model.Credential, error) { +func CreateAdminUser(ctx context.Context, authService auth.Service, cfg *config.BaseConfig, superuser *model.SuperuserConfiguration) (*model.Credential, error) { // Set up the basic groups and policies now := time.Now() err := CreateBaseGroups(ctx, authService, cfg, now) @@ -204,11 +204,11 @@ func AddAdminUser(ctx context.Context, authService auth.Service, user *model.Sup return creds, nil } -func CreateInitialAdminUser(ctx context.Context, authService auth.Service, cfg *config.Config, metadataManger auth.MetadataManager, username string) (*model.Credential, error) { +func CreateInitialAdminUser(ctx context.Context, authService auth.Service, cfg *config.BaseConfig, metadataManger auth.MetadataManager, username string) (*model.Credential, error) { return CreateInitialAdminUserWithKeys(ctx, authService, cfg, metadataManger, username, nil, nil) } -func CreateInitialAdminUserWithKeys(ctx context.Context, authService auth.Service, cfg *config.Config, metadataManger auth.MetadataManager, username string, accessKeyID *string, secretAccessKey *string) (*model.Credential, error) { +func CreateInitialAdminUserWithKeys(ctx context.Context, authService auth.Service, cfg *config.BaseConfig, metadataManger auth.MetadataManager, username string, accessKeyID *string, secretAccessKey *string) (*model.Credential, error) { adminUser := &model.SuperuserConfiguration{ User: model.User{ CreatedAt: time.Now(), @@ -240,7 +240,7 @@ func CreateInitialAdminUserWithKeys(ctx context.Context, authService auth.Servic return cred, err } -func CreateBaseGroups(ctx context.Context, authService auth.Service, cfg *config.Config, ts time.Time) error { +func CreateBaseGroups(ctx context.Context, authService auth.Service, cfg *config.BaseConfig, ts time.Time) error { if !cfg.IsAdvancedAuth() { return nil } diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 1ebbb1521ee..8dc65fc29a4 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -214,7 +214,7 @@ type WriteRangeRequest struct { } type Config struct { - Config *config.Config + Config *config.BaseConfig KVStore kv.Store WalkerFactory WalkerFactory SettingsManagerOption settings.ManagerOption diff --git a/pkg/config/config.go b/pkg/config/config.go index 1cfe91bd5cf..40026608555 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -142,17 +142,85 @@ type Database struct { } `mapstructure:"cosmosdb"` } -// ApproximateOwnership configures an approximate ("mostly correct") ownership. +// ApproximatelyCorrectOwnership configures an approximate ("mostly correct") ownership. type ApproximatelyCorrectOwnership struct { Enabled bool `mapstructure:"enabled"` Refresh time.Duration `mapstructure:"refresh"` Acquire time.Duration `mapstructure:"acquire"` } -// Config - Output struct of configuration, used to validate. If you read a key using a viper accessor +type Blockstore struct { + Signing struct { + SecretKey SecureString `mapstructure:"secret_key"` + } `mapstructure:"signing"` + Type string `mapstructure:"type"` + DefaultNamespacePrefix *string `mapstructure:"default_namespace_prefix"` + Local *struct { + Path string `mapstructure:"path"` + ImportEnabled bool `mapstructure:"import_enabled"` + ImportHidden bool `mapstructure:"import_hidden"` + AllowedExternalPrefixes []string `mapstructure:"allowed_external_prefixes"` + } `mapstructure:"local"` + S3 *struct { + S3AuthInfo `mapstructure:",squash"` + Region string `mapstructure:"region"` + Endpoint string `mapstructure:"endpoint"` + MaxRetries int `mapstructure:"max_retries"` + ForcePathStyle bool `mapstructure:"force_path_style"` + DiscoverBucketRegion bool `mapstructure:"discover_bucket_region"` + SkipVerifyCertificateTestOnly bool `mapstructure:"skip_verify_certificate_test_only"` + ServerSideEncryption string `mapstructure:"server_side_encryption"` + ServerSideEncryptionKmsKeyID string `mapstructure:"server_side_encryption_kms_key_id"` + PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` + // Endpoint for pre-signed URLs, if set, will override the default pre-signed URL S3 endpoint (only for pre-sign URL generation) + PreSignedEndpoint string `mapstructure:"pre_signed_endpoint"` + DisablePreSigned bool `mapstructure:"disable_pre_signed"` + DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` + DisablePreSignedMultipart bool `mapstructure:"disable_pre_signed_multipart"` + ClientLogRetries bool `mapstructure:"client_log_retries"` + ClientLogRequest bool `mapstructure:"client_log_request"` + WebIdentity *struct { + SessionDuration time.Duration `mapstructure:"session_duration"` + SessionExpiryWindow time.Duration `mapstructure:"session_expiry_window"` + } `mapstructure:"web_identity"` + } `mapstructure:"s3"` + Azure *struct { + TryTimeout time.Duration `mapstructure:"try_timeout"` + StorageAccount string `mapstructure:"storage_account"` + StorageAccessKey string `mapstructure:"storage_access_key"` + // Deprecated: Value ignored + AuthMethod string `mapstructure:"auth_method"` + PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` + DisablePreSigned bool `mapstructure:"disable_pre_signed"` + DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` + // Deprecated: Value ignored + ChinaCloudDeprecated bool `mapstructure:"china_cloud"` + TestEndpointURL string `mapstructure:"test_endpoint_url"` + // Domain by default points to Azure default domain blob.core.windows.net, can be set to other Azure domains (China/Gov) + Domain string `mapstructure:"domain"` + } `mapstructure:"azure"` + GS *struct { + S3Endpoint string `mapstructure:"s3_endpoint"` + CredentialsFile string `mapstructure:"credentials_file"` + CredentialsJSON string `mapstructure:"credentials_json"` + PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` + DisablePreSigned bool `mapstructure:"disable_pre_signed"` + DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` + ServerSideEncryptionCustomerSupplied string `mapstructure:"server_side_encryption_customer_supplied"` + ServerSideEncryptionKmsKeyID string `mapstructure:"server_side_encryption_kms_key_id"` + } `mapstructure:"gs"` +} + +type Config interface { + GetBaseConfig() *BaseConfig + StorageConfig() interface{} + Validate() error +} + +// BaseConfig - Output struct of configuration, used to validate. If you read a key using a viper accessor // rather than accessing a field of this struct, that key will *not* be validated. So don't // do that. -type Config struct { +type BaseConfig struct { ListenAddress string `mapstructure:"listen_address"` TLS struct { Enabled bool `mapstructure:"enabled"` @@ -234,68 +302,8 @@ type Config struct { LogoutURL string `mapstructure:"logout_url"` } `mapstructure:"ui_config"` } `mapstructure:"auth"` - Blockstore struct { - Signing struct { - SecretKey SecureString `mapstructure:"secret_key" validate:"required"` - } `mapstructure:"signing"` - Type string `mapstructure:"type" validate:"required"` - DefaultNamespacePrefix *string `mapstructure:"default_namespace_prefix"` - Local *struct { - Path string `mapstructure:"path"` - ImportEnabled bool `mapstructure:"import_enabled"` - ImportHidden bool `mapstructure:"import_hidden"` - AllowedExternalPrefixes []string `mapstructure:"allowed_external_prefixes"` - } `mapstructure:"local"` - S3 *struct { - S3AuthInfo `mapstructure:",squash"` - Region string `mapstructure:"region"` - Endpoint string `mapstructure:"endpoint"` - MaxRetries int `mapstructure:"max_retries"` - ForcePathStyle bool `mapstructure:"force_path_style"` - DiscoverBucketRegion bool `mapstructure:"discover_bucket_region"` - SkipVerifyCertificateTestOnly bool `mapstructure:"skip_verify_certificate_test_only"` - ServerSideEncryption string `mapstructure:"server_side_encryption"` - ServerSideEncryptionKmsKeyID string `mapstructure:"server_side_encryption_kms_key_id"` - PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` - // Endpoint for pre-signed URLs, if set, will override the default pre-signed URL S3 endpoint (only for pre-sign URL generation) - PreSignedEndpoint string `mapstructure:"pre_signed_endpoint"` - DisablePreSigned bool `mapstructure:"disable_pre_signed"` - DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` - DisablePreSignedMultipart bool `mapstructure:"disable_pre_signed_multipart"` - ClientLogRetries bool `mapstructure:"client_log_retries"` - ClientLogRequest bool `mapstructure:"client_log_request"` - WebIdentity *struct { - SessionDuration time.Duration `mapstructure:"session_duration"` - SessionExpiryWindow time.Duration `mapstructure:"session_expiry_window"` - } `mapstructure:"web_identity"` - } `mapstructure:"s3"` - Azure *struct { - TryTimeout time.Duration `mapstructure:"try_timeout"` - StorageAccount string `mapstructure:"storage_account"` - StorageAccessKey string `mapstructure:"storage_access_key"` - // Deprecated: Value ignored - AuthMethod string `mapstructure:"auth_method"` - PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` - DisablePreSigned bool `mapstructure:"disable_pre_signed"` - DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` - // Deprecated: Value ignored - ChinaCloudDeprecated bool `mapstructure:"china_cloud"` - TestEndpointURL string `mapstructure:"test_endpoint_url"` - // Domain by default points to Azure default domain blob.core.windows.net, can be set to other Azure domains (China/Gov) - Domain string `mapstructure:"domain"` - } `mapstructure:"azure"` - GS *struct { - S3Endpoint string `mapstructure:"s3_endpoint"` - CredentialsFile string `mapstructure:"credentials_file"` - CredentialsJSON string `mapstructure:"credentials_json"` - PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` - DisablePreSigned bool `mapstructure:"disable_pre_signed"` - DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` - ServerSideEncryptionCustomerSupplied string `mapstructure:"server_side_encryption_customer_supplied"` - ServerSideEncryptionKmsKeyID string `mapstructure:"server_side_encryption_kms_key_id"` - } `mapstructure:"gs"` - } `mapstructure:"blockstore"` - Committed struct { + Blockstore Blockstore `mapstructure:"blockstore"` + Committed struct { LocalCache struct { SizeBytes int64 `mapstructure:"size_bytes"` Dir string `mapstructure:"dir"` @@ -390,42 +398,46 @@ type Config struct { } `mapstructure:"usage_report"` } -func NewConfig(cfgType string) (*Config, error) { - return newConfig(cfgType) +func ValidateBlockstore(c *Blockstore) error { + if c.Signing.SecretKey == "" { + return fmt.Errorf("'blockstore.signing.secret_key: %w", ErrMissingRequiredKeys) + } + if c.Type == "" { + return fmt.Errorf("'blockstore.type: %w", ErrMissingRequiredKeys) + } + return nil } -func newConfig(cfgType string) (*Config, error) { - c := &Config{} - +// NewConfig - General (common) configuration +func NewConfig(cfgType string, c Config) (*BaseConfig, error) { // Inform viper of all expected fields. Otherwise, it fails to deserialize from the // environment. - keys := GetStructKeys(reflect.TypeOf(c), "mapstructure", "squash") - for _, key := range keys { - viper.SetDefault(key, nil) - } - setDefaults(cfgType) - + SetDefaults(cfgType, c) err := Unmarshal(c) if err != nil { return nil, err } - err = c.validateDomainNames() + cfg := c.GetBaseConfig() + // setup logging package + logging.SetOutputFormat(cfg.Logging.Format) + err = logging.SetOutputs(cfg.Logging.Output, cfg.Logging.FileMaxSizeMB, cfg.Logging.FilesKeep) if err != nil { return nil, err } + logging.SetLevel(cfg.Logging.Level) + return cfg, nil +} - // setup logging package - logging.SetOutputFormat(c.Logging.Format) - err = logging.SetOutputs(c.Logging.Output, c.Logging.FileMaxSizeMB, c.Logging.FilesKeep) - if err != nil { - return nil, err +func SetDefaults(cfgType string, c Config) { + keys := GetStructKeys(reflect.TypeOf(c), "mapstructure", "squash") + for _, key := range keys { + viper.SetDefault(key, nil) } - logging.SetLevel(c.Logging.Level) - return c, nil + setBaseDefaults(cfgType) } -func Unmarshal(c *Config) error { +func Unmarshal(c Config) error { return viper.UnmarshalExact(&c, viper.DecodeHook( mapstructure.ComposeDecodeHookFunc( @@ -441,7 +453,7 @@ func stringReverse(s string) string { return string(chars) } -func (c *Config) validateDomainNames() error { +func (c *BaseConfig) ValidateDomainNames() error { domainStrings := c.Gateways.S3.DomainNames domainNames := make([]string, len(domainStrings)) copy(domainNames, domainStrings) @@ -460,19 +472,19 @@ func (c *Config) validateDomainNames() error { return nil } -func (c *Config) Validate() error { +func (c *BaseConfig) Validate() error { missingKeys := ValidateMissingRequiredKeys(c, "mapstructure", "squash") if len(missingKeys) > 0 { return fmt.Errorf("%w: %v", ErrMissingRequiredKeys, missingKeys) } - return nil + return ValidateBlockstore(&c.Blockstore) } -func (c *Config) BlockstoreType() string { +func (c *BaseConfig) BlockstoreType() string { return c.Blockstore.Type } -func (c *Config) BlockstoreS3Params() (blockparams.S3, error) { +func (c *BaseConfig) BlockstoreS3Params() (blockparams.S3, error) { var webIdentity *blockparams.S3WebIdentity if c.Blockstore.S3.WebIdentity != nil { webIdentity = &blockparams.S3WebIdentity{ @@ -511,7 +523,7 @@ func (c *Config) BlockstoreS3Params() (blockparams.S3, error) { }, nil } -func (c *Config) BlockstoreLocalParams() (blockparams.Local, error) { +func (c *BaseConfig) BlockstoreLocalParams() (blockparams.Local, error) { localPath := c.Blockstore.Local.Path path, err := homedir.Expand(localPath) if err != nil { @@ -527,7 +539,7 @@ const ( gcpAESKeyLength = 32 ) -func (c *Config) BlockstoreGSParams() (blockparams.GS, error) { +func (c *BaseConfig) BlockstoreGSParams() (blockparams.GS, error) { var customerSuppliedKey []byte = nil if c.Blockstore.GS.ServerSideEncryptionCustomerSupplied != "" { v, err := hex.DecodeString(c.Blockstore.GS.ServerSideEncryptionCustomerSupplied) @@ -558,7 +570,7 @@ func (c *Config) BlockstoreGSParams() (blockparams.GS, error) { }, nil } -func (c *Config) BlockstoreAzureParams() (blockparams.Azure, error) { +func (c *BaseConfig) BlockstoreAzureParams() (blockparams.Azure, error) { if c.Blockstore.Azure.AuthMethod != "" { logging.ContextUnavailable().Warn("blockstore.azure.auth_method is deprecated. Value is no longer used.") } @@ -585,33 +597,33 @@ const ( AuthRBACInternal = "internal" ) -func (c *Config) IsAuthBasic() bool { +func (c *BaseConfig) IsAuthBasic() bool { return c.Auth.UIConfig.RBAC == AuthRBACNone } -func (c *Config) IsAuthUISimplified() bool { +func (c *BaseConfig) IsAuthUISimplified() bool { return c.Auth.UIConfig.RBAC == AuthRBACSimplified } -func (c *Config) IsAuthenticationTypeAPI() bool { +func (c *BaseConfig) IsAuthenticationTypeAPI() bool { return c.Auth.AuthenticationAPI.Endpoint != "" } -func (c *Config) IsAuthTypeAPI() bool { +func (c *BaseConfig) IsAuthTypeAPI() bool { return c.Auth.API.Endpoint != "" } -func (c *Config) IsExternalPrincipalsEnabled() bool { +func (c *BaseConfig) IsExternalPrincipalsEnabled() bool { // IsAuthTypeAPI must be true since the local auth service doesnt support external principals // ExternalPrincipalsEnabled indicates that the remote auth service enables external principals support since its optional extension return c.IsAuthTypeAPI() && c.Auth.AuthenticationAPI.ExternalPrincipalsEnabled } -func (c *Config) IsAdvancedAuth() bool { +func (c *BaseConfig) IsAdvancedAuth() bool { return c.IsAuthTypeAPI() && (c.Auth.UIConfig.RBAC == AuthRBACExternal || c.Auth.UIConfig.RBAC == AuthRBACInternal) } -func (c *Config) UISnippets() []apiparams.CodeSnippet { +func (c *BaseConfig) UISnippets() []apiparams.CodeSnippet { snippets := make([]apiparams.CodeSnippet, 0, len(c.UI.Snippets)) for _, item := range c.UI.Snippets { snippets = append(snippets, apiparams.CodeSnippet{ @@ -621,3 +633,11 @@ func (c *Config) UISnippets() []apiparams.CodeSnippet { } return snippets } + +func (c *BaseConfig) GetBaseConfig() *BaseConfig { + return c +} + +func (c *BaseConfig) StorageConfig() interface{} { + return c.Blockstore +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 54c327b7ef8..7d5e9ba7ae6 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -13,6 +13,7 @@ import ( "github.com/go-test/deep" "github.com/spf13/viper" blockfactory "github.com/treeverse/lakefs/modules/block/factory" + configfactory "github.com/treeverse/lakefs/modules/config/factory" "github.com/treeverse/lakefs/pkg/block" "github.com/treeverse/lakefs/pkg/block/gs" "github.com/treeverse/lakefs/pkg/block/local" @@ -22,27 +23,28 @@ import ( "github.com/treeverse/lakefs/pkg/testutil" ) -func newConfigFromFile(fn string) (*config.Config, error) { +func newConfigFromFile(fn string) (*config.BaseConfig, error) { viper.SetConfigFile(fn) err := viper.ReadInConfig() if err != nil { return nil, err } - cfg, err := config.NewConfig("") + cfg, err := configfactory.BuildConfig("") if err != nil { return nil, err } err = cfg.Validate() - return cfg, err + return cfg.GetBaseConfig(), err } func TestConfig_Setup(t *testing.T) { // test defaults - c, err := config.NewConfig("") + cfg := &config.BaseConfig{} + cfg, err := config.NewConfig("", cfg) testutil.Must(t, err) // Don't validate, some tested configs don't have all required fields. - if c.ListenAddress != config.DefaultListenAddress { - t.Fatalf("expected listen addr '%s', got '%s'", config.DefaultListenAddress, c.ListenAddress) + if cfg.ListenAddress != config.DefaultListenAddress { + t.Fatalf("expected listen addr '%s', got '%s'", config.DefaultListenAddress, cfg.ListenAddress) } } diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 6033ff72040..0def3fe2840 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -20,7 +20,7 @@ const ( ) //nolint:mnd -func setDefaults(cfgType string) { +func setBaseDefaults(cfgType string) { switch cfgType { case QuickstartConfiguration: viper.SetDefault("installation.user_name", DefaultQuickstartUsername) @@ -34,18 +34,16 @@ func setDefaults(cfgType string) { viper.SetDefault("auth.encrypt.secret_key", DefaultAuthSecret) viper.SetDefault(BlockstoreTypeKey, "local") } + viper.SetDefault("installation.allow_inter_region_storage", true) - viper.SetDefault("blockstore.signing.secret_key", DefaultSigningSecretKey) viper.SetDefault("listen_address", DefaultListenAddress) viper.SetDefault("logging.format", "text") viper.SetDefault("logging.level", DefaultLoggingLevel) viper.SetDefault("logging.output", "-") - viper.SetDefault("logging.files_keep", 100) viper.SetDefault("logging.audit_log_level", DefaultLoggingAuditLogLevel) - viper.SetDefault("logging.file_max_size_mb", (1<<10)*100) // 100MiB viper.SetDefault("actions.enabled", true) @@ -58,6 +56,7 @@ func setDefaults(cfgType string) { viper.SetDefault("auth.cache.jitter", 3*time.Second) viper.SetDefault("auth.logout_redirect_url", "/auth/login") + viper.SetDefault("auth.login_duration", 7*24*time.Hour) viper.SetDefault("auth.login_max_duration", 14*24*time.Hour) @@ -72,14 +71,6 @@ func setDefaults(cfgType string) { viper.SetDefault("auth.oidc.persist_friendly_name", false) viper.SetDefault("auth.cookie_auth_verification.persist_friendly_name", false) - viper.SetDefault("blockstore.local.path", "~/lakefs/data/block") - viper.SetDefault("blockstore.s3.region", "us-east-1") - viper.SetDefault("blockstore.s3.max_retries", 5) - viper.SetDefault("blockstore.s3.discover_bucket_region", true) - viper.SetDefault("blockstore.s3.pre_signed_expiry", 15*time.Minute) - viper.SetDefault("blockstore.s3.web_identity.session_expiry_window", 5*time.Minute) - viper.SetDefault("blockstore.s3.disable_pre_signed_ui", true) - viper.SetDefault("committed.local_cache.size_bytes", 1*1024*1024*1024) viper.SetDefault("committed.local_cache.dir", "~/lakefs/data/cache") viper.SetDefault("committed.local_cache.max_uploaders_per_writer", 10) @@ -87,19 +78,37 @@ func setDefaults(cfgType string) { viper.SetDefault("committed.local_cache.metarange_proportion", 0.1) viper.SetDefault("committed.block_storage_prefix", "_lakefs") + viper.SetDefault("committed.permanent.min_range_size_bytes", 0) viper.SetDefault("committed.permanent.max_range_size_bytes", 20*1024*1024) viper.SetDefault("committed.permanent.range_raggedness_entries", 50_000) + viper.SetDefault("committed.sstable.memory.cache_size_bytes", 400_000_000) viper.SetDefault("gateways.s3.domain_name", "s3.local.lakefs.io") viper.SetDefault("gateways.s3.region", "us-east-1") viper.SetDefault("gateways.s3.verify_unsupported", true) + // blockstore defaults + viper.SetDefault("blockstore.signing.secret_key", DefaultSigningSecretKey) + + viper.SetDefault("blockstore.local.path", "~/lakefs/data/block") + + viper.SetDefault("blockstore.s3.region", "us-east-1") + viper.SetDefault("blockstore.s3.max_retries", 5) + viper.SetDefault("blockstore.s3.discover_bucket_region", true) + viper.SetDefault("blockstore.s3.pre_signed_expiry", 15*time.Minute) + viper.SetDefault("blockstore.s3.web_identity.session_expiry_window", 5*time.Minute) + viper.SetDefault("blockstore.s3.disable_pre_signed_ui", true) + viper.SetDefault("blockstore.gs.s3_endpoint", "https://storage.googleapis.com") viper.SetDefault("blockstore.gs.pre_signed_expiry", 15*time.Minute) viper.SetDefault("blockstore.gs.disable_pre_signed_ui", true) + viper.SetDefault("blockstore.azure.try_timeout", 10*time.Minute) + viper.SetDefault("blockstore.azure.pre_signed_expiry", 15*time.Minute) + viper.SetDefault("blockstore.azure.disable_pre_signed_ui", true) + viper.SetDefault("stats.enabled", true) viper.SetDefault("stats.address", "https://stats.lakefs.io") viper.SetDefault("stats.flush_interval", 30*time.Second) @@ -107,10 +116,6 @@ func setDefaults(cfgType string) { viper.SetDefault("email_subscription.enabled", true) - viper.SetDefault("blockstore.azure.try_timeout", 10*time.Minute) - viper.SetDefault("blockstore.azure.pre_signed_expiry", 15*time.Minute) - viper.SetDefault("blockstore.azure.disable_pre_signed_ui", true) - viper.SetDefault("security.audit_check_interval", 24*time.Hour) viper.SetDefault("security.audit_check_url", "https://audit.lakefs.io/audit") viper.SetDefault("security.check_latest_version", true) diff --git a/pkg/config/testdata/domain_name_prefix.yaml b/pkg/config/testdata/domain_name_prefix.yaml index cdd4b7b2dee..3e16dc61d65 100644 --- a/pkg/config/testdata/domain_name_prefix.yaml +++ b/pkg/config/testdata/domain_name_prefix.yaml @@ -5,6 +5,10 @@ database: blockstore: type: local +auth: + encrypt: + secret_key: "required in config" + gateways: s3: domain_name: diff --git a/pkg/gateway/testutil/gateway_setup.go b/pkg/gateway/testutil/gateway_setup.go index b2eebf2a811..1fa8d6ec646 100644 --- a/pkg/gateway/testutil/gateway_setup.go +++ b/pkg/gateway/testutil/gateway_setup.go @@ -42,7 +42,8 @@ func GetBasicHandler(t *testing.T, authService *FakeAuthService, repoName string blockstoreType, _ := os.LookupEnv(testutil.EnvKeyUseBlockAdapter) blockAdapter := testutil.NewBlockAdapterByType(t, blockstoreType) - conf, err := config.NewConfig("") + conf := &config.BaseConfig{} + conf, err = config.NewConfig("", conf) testutil.MustDo(t, "config", err) c, err := catalog.New(ctx, catalog.Config{ diff --git a/pkg/loadtest/local_load_test.go b/pkg/loadtest/local_load_test.go index 6595f03ee6e..ab5c814d204 100644 --- a/pkg/loadtest/local_load_test.go +++ b/pkg/loadtest/local_load_test.go @@ -39,7 +39,8 @@ func TestLocalLoad(t *testing.T) { ctx := context.Background() viper.Set(config.BlockstoreTypeKey, block.BlockstoreTypeLocal) - conf, err := config.NewConfig("") + cfg := &config.BaseConfig{} + cfg, err := config.NewConfig("", cfg) testutil.MustDo(t, "config", err) superuser := &authmodel.SuperuserConfiguration{ @@ -51,7 +52,7 @@ func TestLocalLoad(t *testing.T) { kvStore := kvtest.GetStore(ctx, t) authService := auth.NewBasicAuthService(kvStore, crypt.NewSecretStore([]byte("some secret")), authparams.ServiceCache{}, logging.FromContext(ctx)) - meta := auth.NewKVMetadataManager("local_load_test", conf.Installation.FixedID, conf.Database.Type, kvStore) + meta := auth.NewKVMetadataManager("local_load_test", cfg.Installation.FixedID, cfg.Database.Type, kvStore) blockstoreType := os.Getenv(testutil.EnvKeyUseBlockAdapter) if blockstoreType == "" { @@ -60,7 +61,7 @@ func TestLocalLoad(t *testing.T) { blockAdapter := testutil.NewBlockAdapterByType(t, blockstoreType) c, err := catalog.New(ctx, catalog.Config{ - Config: conf, + Config: cfg, KVStore: kvStore, PathProvider: upload.DefaultPathProvider, }) @@ -77,15 +78,15 @@ func TestLocalLoad(t *testing.T) { testutil.Must(t, err) authenticator := auth.NewBuiltinAuthenticator(authService) - kvParams, err := kvparams.NewConfig(&conf.Database) + kvParams, err := kvparams.NewConfig(&cfg.Database) testutil.Must(t, err) migrator := kv.NewDatabaseMigrator(kvParams) t.Cleanup(func() { _ = c.Close() }) - auditChecker := version.NewDefaultAuditChecker(conf.Security.AuditCheckURL, "", nil) + auditChecker := version.NewDefaultAuditChecker(cfg.Security.AuditCheckURL, "", nil) authenticationService := authentication.NewDummyService() - handler := api.Serve(conf, c, authenticator, authService, authenticationService, blockAdapter, meta, migrator, &stats.NullCollector{}, nil, actionsService, auditChecker, logging.ContextUnavailable(), nil, nil, upload.DefaultPathProvider, stats.DefaultUsageReporter) + handler := api.Serve(cfg, c, authenticator, authService, authenticationService, blockAdapter, meta, migrator, &stats.NullCollector{}, nil, actionsService, auditChecker, logging.ContextUnavailable(), nil, nil, upload.DefaultPathProvider, stats.DefaultUsageReporter) ts := httptest.NewServer(handler) defer ts.Close() diff --git a/pkg/pyramid/params/params.go b/pkg/pyramid/params/params.go index d87e779751c..afbd5aba0c9 100644 --- a/pkg/pyramid/params/params.go +++ b/pkg/pyramid/params/params.go @@ -94,7 +94,7 @@ func (p ExtParams) WithLogger(logger logging.Logger) ExtParams { // NewCommittedTierFSParams returns parameters for building a tierFS. // Caller must separately build and populate Adapter. -func NewCommittedTierFSParams(c *config.Config, adapter block.Adapter) (*ExtParams, error) { +func NewCommittedTierFSParams(c *config.BaseConfig, adapter block.Adapter) (*ExtParams, error) { const floatSumTolerance = 1e-6 rangePro := c.Committed.LocalCache.RangeProportion metaRangePro := c.Committed.LocalCache.MetaRangeProportion diff --git a/pkg/stats/metadata.go b/pkg/stats/metadata.go index b22a6087e70..81ff245aecc 100644 --- a/pkg/stats/metadata.go +++ b/pkg/stats/metadata.go @@ -55,7 +55,7 @@ func (n *noopMetadataProvider) GetMetadata() map[string]string { return nil } -func BuildMetadataProvider(logger logging.Logger, c *config.Config) cloud.MetadataProvider { +func BuildMetadataProvider(logger logging.Logger, c *config.BaseConfig) cloud.MetadataProvider { switch c.Blockstore.Type { case block.BlockstoreTypeGS: return gcp.NewMetadataProvider(logger)