From 8ae59ac6ecf854f67fece800b71d703383c5724e Mon Sep 17 00:00:00 2001 From: jt-dd <112463504+jt-dd@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:15:11 +0200 Subject: [PATCH] Adding local config file (#233) * adding local config file * removing datadog agent flags * removing required flag as it can be handle in the config file * typo * inLine config path overwrite local config file * adding comment * cleaning the config files --- cmd/kubehound/ingest.go | 1 - configs/etc/kubehound-dd.yaml | 80 ------------------------- configs/etc/kubehound-ingestor.yaml | 60 ------------------- configs/etc/kubehound-reference.yaml | 13 ++-- configs/etc/kubehound.yaml | 15 +++++ pkg/cmd/dump.go | 13 ---- pkg/config/config.go | 90 +++++++++++++++++----------- 7 files changed, 78 insertions(+), 194 deletions(-) delete mode 100644 configs/etc/kubehound-dd.yaml delete mode 100644 configs/etc/kubehound-ingestor.yaml diff --git a/cmd/kubehound/ingest.go b/cmd/kubehound/ingest.go index a44b240aa..042a93b55 100644 --- a/cmd/kubehound/ingest.go +++ b/cmd/kubehound/ingest.go @@ -42,7 +42,6 @@ var ( Long: `Run an ingestion on KHaaS from a bucket to build the attack path, by default it will rehydrate the latest snapshot previously dumped on a KHaaS instance from all clusters`, PreRunE: func(cobraCmd *cobra.Command, args []string) error { viper.BindPFlag(config.IngestorAPIEndpoint, cobraCmd.Flags().Lookup("khaas-server")) //nolint: errcheck - cobraCmd.MarkFlagRequired("khaas-server") //nolint: errcheck viper.BindPFlag(config.IngestorAPIInsecure, cobraCmd.Flags().Lookup("insecure")) //nolint: errcheck if !isIngestRemoteDefault() { diff --git a/configs/etc/kubehound-dd.yaml b/configs/etc/kubehound-dd.yaml deleted file mode 100644 index d37fccfbe..000000000 --- a/configs/etc/kubehound-dd.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Default Datadog KubeHound configuration -# NOTE: this is optimized for large clusters in the Datadog environment -# - -# K8s collector configuration -collector: - # Type of collector to use - type: live-k8s-api-collector - - # Live collector configuration - live: - # Rate limit of requests/second to the Kubernetes API - rate_limit_per_second: 150 - - # Number of pages to buffer - page_buffer_size: 50 - -# General storage configuration -storage: - # Number of connection retries before declaring an error - retry: 5 - - # Delay between connection retries - retry_delay: 10s - -# Store database configuration -mongodb: - # Connection URL to the mongo DB instance - url: "mongodb://localhost:27017" - - # Timeout on requests to the mongo DB instance - connection_timeout: 30s - -# Graph database configuration -janusgraph: - # Connection URL to the JanusGraph DB instance - url: "ws://localhost:8182/gremlin" - - # Timeout on requests to the JanusGraph DB instance - connection_timeout: 30s - -# Datadog telemetry configuration -telemetry: - # Whether to enable Datadog telemetry (default false) - enabled: true - - # Default tags to add to all telemetry (free form key-value map) - tags: - team: ase - - # Statsd configuration for metics support - statsd: - # URL to send statsd data to the Datadog agent - url: "127.0.0.1:8225" - - # Tracer configuration for APM support - tracer: - # URL to send tracer data to the Datadog agent - url: "127.0.0.1:8226" - -# Graph builder configuration -builder: - # Vertex builder configuration - vertex: - # Batch size for vertex inserts - batch_size: 500 - - # Edge builder configuration - edge: - worker_pool_size: 5 - - # Batch size for edge inserts - batch_size: 1000 - - # Cluster impact batch size for edge inserts - batch_size_cluster_impact: 10 - - # Enable for large clusters to prevent number of edges growing exponentially - large_cluster_optimizations: true diff --git a/configs/etc/kubehound-ingestor.yaml b/configs/etc/kubehound-ingestor.yaml deleted file mode 100644 index 32a749c5c..000000000 --- a/configs/etc/kubehound-ingestor.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# -# Default KubeHound configuration -# NOTE: this is optimized for smaller clusters of 1-2k pods -# - -# General storage configuration -storage: - # Number of connection retries before declaring an error - retry: 5 - - # Delay between connection retries - retry_delay: 10s - -# Store database configuration -mongodb: - # Connection URL to the mongo DB instance - url: "mongodb://localhost:27017" - - # Timeout on requests to the mongo DB instance - connection_timeout: 30s - -# Graph database configuration -janusgraph: - # Connection URL to the JanusGraph DB instance - url: "ws://localhost:8182/gremlin" - - # Timeout on requests to the JanusGraph DB instance - connection_timeout: 30s - -# Graph builder configuration -builder: - # Vertex builder configuration - vertex: - # Batch size for vertex inserts - batch_size: 500 - - # Edge builder configuration - edge: - worker_pool_size: 2 - - # Batch size for edge inserts - batch_size: 500 - - # Cluster impact batch size for edge inserts - batch_size_cluster_impact: 10 - - # Enable for large clusters to prevent number of edges growing exponentially - large_cluster_optimizations: true - -# Ingestor configuration (for KHaaS) -ingestor: - blob: - bucket: "" # (i.e.: s3://) - region: "" # (i.e.: us-west-2) - temp_dir: "/tmp/kubehound" - archive_name: "archive.tar.gz" - max_archive_size: 2147483648 # 2GB - api: # GRPC endpoint for the ingestor - endpoint: "127.0.0.1:9000" - insecure: true \ No newline at end of file diff --git a/configs/etc/kubehound-reference.yaml b/configs/etc/kubehound-reference.yaml index a4a511da0..f2132d7d7 100644 --- a/configs/etc/kubehound-reference.yaml +++ b/configs/etc/kubehound-reference.yaml @@ -128,11 +128,14 @@ builder: # Ingestor configuration (for KHaaS) # ingestor: # blob: -# bucket: "" # (i.e.: s3://your-bucket) -# region: "" # (i.e.: us-east-1) +# # (i.e.: s3://) +# bucket: "" +# # (i.e.: us-east-1) +# region: "" # temp_dir: "/tmp/kubehound" # archive_name: "archive.tar.gz" -# max_archive_size: 2147483648 # 2GB -# api: # GRPC endpoint for the ingestor +# max_archive_size: 2147483648 # 2GB +# # GRPC endpoint for the ingestor +# api: # endpoint: "127.0.0.1:9000" -# insecure: true \ No newline at end of file +# insecure: true diff --git a/configs/etc/kubehound.yaml b/configs/etc/kubehound.yaml index fc7a25677..b36c66c79 100644 --- a/configs/etc/kubehound.yaml +++ b/configs/etc/kubehound.yaml @@ -56,3 +56,18 @@ builder: # Enable for large clusters to prevent number of edges growing exponentially large_cluster_optimizations: true + +# Ingestor configuration (for KHaaS) +ingestor: + blob: + # (i.e.: s3://) + bucket: "" + # (i.e.: us-east-1) + region: "" + temp_dir: "/tmp/kubehound" + archive_name: "archive.tar.gz" + max_archive_size: 2147483648 # 2GB + # GRPC endpoint for the ingestor + api: + endpoint: "127.0.0.1:9000" + insecure: true diff --git a/pkg/cmd/dump.go b/pkg/cmd/dump.go index 16fb9c903..dd9367936 100644 --- a/pkg/cmd/dump.go +++ b/pkg/cmd/dump.go @@ -12,21 +12,9 @@ func InitRootCmd(cmd *cobra.Command) { } func InitDumpCmd(cmd *cobra.Command) { - cmd.PersistentFlags().String("statsd", config.DefaultTelemetryStatsdUrl, "URL of the statsd endpoint") - viper.BindPFlag(config.TelemetryStatsdUrl, cmd.PersistentFlags().Lookup("statsd")) //nolint: errcheck - - cmd.PersistentFlags().String("profiler", config.DefaultTelemetryProfilerUrl, "URL of the profiler endpoint") - viper.BindPFlag(config.TelemetryTracerUrl, cmd.PersistentFlags().Lookup("profiler")) //nolint: errcheck - cmd.PersistentFlags().Bool("telemetry", false, "Enable telemetry with default settings") viper.BindPFlag(config.TelemetryEnabled, cmd.PersistentFlags().Lookup("telemetry")) //nolint: errcheck - cmd.PersistentFlags().Duration("period", config.DefaultProfilerPeriod, "Period specifies the interval at which to collect profiles") - viper.BindPFlag(config.TelemetryProfilerPeriod, cmd.PersistentFlags().Lookup("period")) //nolint: errcheck - - cmd.PersistentFlags().Duration("cpu-duration", config.DefaultProfilerCPUDuration, "CPU Duration specifies the length at which to collect CPU profiles") - viper.BindPFlag(config.TelemetryProfilerCPUDuration, cmd.PersistentFlags().Lookup("cpu-duration")) //nolint: errcheck - cmd.PersistentFlags().Int("rate", config.DefaultK8sAPIRateLimitPerSecond, "Rate limit of requests/second to the Kubernetes API") viper.BindPFlag(config.CollectorLiveRate, cmd.PersistentFlags().Lookup("rate")) //nolint: errcheck @@ -51,7 +39,6 @@ func InitLocalDumpCmd(cmd *cobra.Command) { func InitRemoteDumpCmd(cmd *cobra.Command) { cmd.Flags().String("bucket", "", "Bucket to use to push k8s resources (e.g.: s3://)") viper.BindPFlag(config.CollectorFileBlobBucket, cmd.Flags().Lookup("bucket")) //nolint: errcheck - cmd.MarkFlagRequired("bucket") //nolint: errcheck cmd.Flags().String("region", "", "Region to retrieve the configuration (only for s3) (e.g.: us-east-1)") viper.BindPFlag(config.CollectorFileBlobRegion, cmd.Flags().Lookup("region")) //nolint: errcheck diff --git a/pkg/config/config.go b/pkg/config/config.go index 8429b4989..2cad68b47 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,7 @@ var ( const ( DefaultConfigType = "yaml" DefaultClusterName = "unknown" + DefaultConfigName = "kubehound" GlobalDebug = "debug" ) @@ -88,53 +89,55 @@ func NewKubehoundConfig(configPath string, inLine bool) *KubehoundConfig { } // SetDefaultValues loads the default value from the different modules -func SetDefaultValues(c *viper.Viper) { +func SetDefaultValues(v *viper.Viper) { // K8s Live collector module - c.SetDefault(CollectorLivePageSize, DefaultK8sAPIPageSize) - c.SetDefault(CollectorLivePageBufferSize, DefaultK8sAPIPageBufferSize) - c.SetDefault(CollectorLiveRate, DefaultK8sAPIRateLimitPerSecond) - c.SetDefault(CollectorNonInteractive, DefaultK8sAPINonInteractive) + v.SetDefault(CollectorLivePageSize, DefaultK8sAPIPageSize) + v.SetDefault(CollectorLivePageBufferSize, DefaultK8sAPIPageBufferSize) + v.SetDefault(CollectorLiveRate, DefaultK8sAPIRateLimitPerSecond) + v.SetDefault(CollectorNonInteractive, DefaultK8sAPINonInteractive) // File collector module - c.SetDefault(CollectorFileArchiveNoCompress, DefaultArchiveNoCompress) + v.SetDefault(CollectorFileArchiveNoCompress, DefaultArchiveNoCompress) // Default values for storage provider - c.SetDefault("storage.wipe", true) - c.SetDefault("storage.retry", DefaultRetry) - c.SetDefault("storage.retry_delay", DefaultRetryDelay) + v.SetDefault("storage.wipe", true) + v.SetDefault("storage.retry", DefaultRetry) + v.SetDefault("storage.retry_delay", DefaultRetryDelay) // Disable Datadog telemetry by default - c.SetDefault(TelemetryEnabled, false) + v.SetDefault(TelemetryEnabled, false) // Default value for MongoDB - c.SetDefault("mongodb.url", DefaultMongoUrl) - c.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout) + v.SetDefault("mongodb.url", DefaultMongoUrl) + v.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout) // Defaults values for JanusGraph - c.SetDefault("janusgraph.url", DefaultJanusGraphUrl) - c.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout) + v.SetDefault("janusgraph.url", DefaultJanusGraphUrl) + v.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout) // Profiler values - c.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod) - c.SetDefault(TelemetryProfilerCPUDuration, DefaultProfilerCPUDuration) + v.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod) + v.SetDefault(TelemetryProfilerCPUDuration, DefaultProfilerCPUDuration) // Default values for graph builder - c.SetDefault("builder.vertex.batch_size", DefaultVertexBatchSize) - c.SetDefault("builder.vertex.batch_size_small", DefaultVertexBatchSizeSmall) - c.SetDefault("builder.edge.worker_pool_size", DefaultEdgeWorkerPoolSize) - c.SetDefault("builder.edge.worker_pool_capacity", DefaultEdgeWorkerPoolCapacity) - c.SetDefault("builder.edge.batch_size", DefaultEdgeBatchSize) - c.SetDefault("builder.edge.batch_size_small", DefaultEdgeBatchSizeSmall) - c.SetDefault("builder.edge.batch_size_cluster_impact", DefaultEdgeBatchSizeClusterImpact) - c.SetDefault("builder.stop_on_error", DefaultStopOnError) - c.SetDefault("builder.edge.large_cluster_optimizations", DefaultLargeClusterOptimizations) - - c.SetDefault(IngestorAPIEndpoint, DefaultIngestorAPIEndpoint) - c.SetDefault(IngestorAPIInsecure, DefaultIngestorAPIInsecure) - c.SetDefault(IngestorBlobBucketName, DefaultBucketName) - c.SetDefault(IngestorTempDir, DefaultTempDir) - c.SetDefault(IngestorMaxArchiveSize, DefaultMaxArchiveSize) - c.SetDefault(IngestorArchiveName, DefaultArchiveName) + v.SetDefault("builder.vertex.batch_size", DefaultVertexBatchSize) + v.SetDefault("builder.vertex.batch_size_small", DefaultVertexBatchSizeSmall) + v.SetDefault("builder.edge.worker_pool_size", DefaultEdgeWorkerPoolSize) + v.SetDefault("builder.edge.worker_pool_capacity", DefaultEdgeWorkerPoolCapacity) + v.SetDefault("builder.edge.batch_size", DefaultEdgeBatchSize) + v.SetDefault("builder.edge.batch_size_small", DefaultEdgeBatchSizeSmall) + v.SetDefault("builder.edge.batch_size_cluster_impact", DefaultEdgeBatchSizeClusterImpact) + v.SetDefault("builder.stop_on_error", DefaultStopOnError) + v.SetDefault("builder.edge.large_cluster_optimizations", DefaultLargeClusterOptimizations) + + v.SetDefault(IngestorAPIEndpoint, DefaultIngestorAPIEndpoint) + v.SetDefault(IngestorAPIInsecure, DefaultIngestorAPIInsecure) + v.SetDefault(IngestorBlobBucketName, DefaultBucketName) + v.SetDefault(IngestorTempDir, DefaultTempDir) + v.SetDefault(IngestorMaxArchiveSize, DefaultMaxArchiveSize) + v.SetDefault(IngestorArchiveName, DefaultArchiveName) + + SetLocalConfig(v) } // SetEnvOverrides enables environment variable overrides for the config. @@ -173,12 +176,13 @@ func unmarshalConfig(v *viper.Viper) (*KubehoundConfig, error) { // NewConfig creates a new config instance from the provided file using viper. func NewConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error) { - v.SetConfigType(DefaultConfigType) - v.SetConfigFile(configPath) - // Configure default values SetDefaultValues(v) + // Loading inLine config path + v.SetConfigType(DefaultConfigType) + v.SetConfigFile(configPath) + // Configure environment variable override SetEnvOverrides(v) if err := v.ReadInConfig(); err != nil { @@ -209,6 +213,22 @@ func NewInlineConfig(v *viper.Viper) (*KubehoundConfig, error) { return kc, nil } +// Load local config file if it exists, check for local file in current dir or in $HOME/.config/ +// Not returning any error as it is not mandatory to have a local config file +func SetLocalConfig(v *viper.Viper) { + v.SetConfigName(DefaultConfigName) // name of config file (without extension) + v.SetConfigType(DefaultConfigType) // REQUIRED if the config file does not have the extension in the name + v.AddConfigPath("$HOME/.config/") // call multiple times to add many search paths + v.AddConfigPath(".") // optionally look for config in the working directory + + err := v.ReadInConfig() + if err != nil { + log.I.Warnf("No local config file was found (%s.%s)", DefaultConfigName, DefaultConfigType) + // log.I.Debugf("Error reading config: %v", err) + } + log.I.Infof("Using %s for default config\n", viper.ConfigFileUsed()) +} + // NewEmbedConfig creates a new config instance from an embedded config file using viper. func NewEmbedConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error) { v.SetConfigType(DefaultConfigType)