From 8cca242883a4d55750675c721e31a59a67659ea3 Mon Sep 17 00:00:00 2001 From: Matt Conway Date: Wed, 22 Sep 2021 14:31:34 -0400 Subject: [PATCH] enable setting of log level from project mapping crds --- README.md | 1 + helm/helmv2/templates/projectmapping.yaml | 4 + helm/kubetruth/crds/projectmapping.yaml | 4 + lib/kubetruth/config.rb | 1 + lib/kubetruth/etl.rb | 154 ++++++++++++---------- lib/kubetruth/logging.rb | 8 ++ spec/kubetruth/etl_spec.rb | 54 +++++++- spec/kubetruth/logging_spec.rb | 19 +++ 8 files changed, 174 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 8d8f7f9..e9daaeb 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ Parameterize the helm install with `--set *` or `--values yourConfig.yaml` to co | projectMappings.root.project_selector | A regexp to limit the projects acted against (client-side). Supplies any named matches for template evaluation | string | "" | no | | projectMappings.root.key_selector | A regexp to limit the keys acted against (client-side). Supplies any named matches for template evaluation | string | "" | no | | projectMappings.root.skip | Skips the generation of resources for the selected projects | flag | false | no | +| projectMappings.root.log_level | Sets the kubetruth logging level while handling the selected projects | enum(debug, info, warn, error, fatal) | as set by cli | no | | projectMappings.root.included_projects | Include the parameters from other projects into the selected ones. This can be recursive in a depth first fashion, so if A imports B and B imports C, then A will get B's and C's parameters. For key conflicts, if A includes B and B includes C, then the precendence is A overrides B overrides C. If A includes \[B, C], then the precendence is A overrides C overrides B. | list | [] | no | | projectMappings.root.context | Additional variables made available to the resource templates. Can also be templates | map | [default](helm/kubetruth/values.yaml#L93-L129) | no | | projectMappings.root.resource_templates | The templates to use in generating kubernetes resources (ConfigMap/Secrets/other) | map | [default](helm/kubetruth/values.yaml#L93-L129) | no | diff --git a/helm/helmv2/templates/projectmapping.yaml b/helm/helmv2/templates/projectmapping.yaml index ab893b8..7e14901 100644 --- a/helm/helmv2/templates/projectmapping.yaml +++ b/helm/helmv2/templates/projectmapping.yaml @@ -30,6 +30,10 @@ spec: skip: type: boolean description: Skips the generation of resources for the selected projects. Useful for excluding projects that should only be included into others. + log_level: + type: string + description: The level of logging to use + enum: ["debug", "info", "warn", "error", "fatal"] included_projects: type: array items: diff --git a/helm/kubetruth/crds/projectmapping.yaml b/helm/kubetruth/crds/projectmapping.yaml index 9f067c3..6335f52 100644 --- a/helm/kubetruth/crds/projectmapping.yaml +++ b/helm/kubetruth/crds/projectmapping.yaml @@ -32,6 +32,10 @@ spec: skip: type: boolean description: Skips the generation of resources for the selected projects + log_level: + type: string + description: The level of logging to use + enum: ["debug", "info", "warn", "error", "fatal"] included_projects: type: array items: diff --git a/lib/kubetruth/config.rb b/lib/kubetruth/config.rb index e976cf9..94776c1 100644 --- a/lib/kubetruth/config.rb +++ b/lib/kubetruth/config.rb @@ -14,6 +14,7 @@ class DuplicateSelection < Kubetruth::Error; end :key_selector, :environment, :skip, + :log_level, :included_projects, :context, :resource_templates, diff --git a/lib/kubetruth/etl.rb b/lib/kubetruth/etl.rb index 9e0fdaf..8114bef 100644 --- a/lib/kubetruth/etl.rb +++ b/lib/kubetruth/etl.rb @@ -115,6 +115,16 @@ def load_config configs end + def with_log_level(level) + original_root_log_level = Kubetruth::Logging.root_log_level + begin + Kubetruth::Logging.root_log_level = level if level + yield + ensure + Kubetruth::Logging.root_log_level = original_root_log_level + end + end + def apply async(annotation: "ETL Event Loop") do logger.warn("Performing dry-run") if @dry_run @@ -123,89 +133,93 @@ def apply CtApi.reset load_config do |namespace, config| - project_collection = ProjectCollection.new - - # Load all projects that are used - all_specs = [config.root_spec] + config.override_specs - project_selectors = all_specs.collect(&:project_selector) - included_projects = all_specs.collect(&:included_projects).flatten.uniq - - project_collection.names.each do |project_name| - active = included_projects.any? {|p| p == project_name } - active ||= project_selectors.any? {|s| s =~ project_name } - if active - project_spec = config.spec_for_project(project_name) - project_collection.create_project(name: project_name, spec: project_spec) + with_log_level(config.root_spec.log_level) do + project_collection = ProjectCollection.new + + # Load all projects that are used + all_specs = [config.root_spec] + config.override_specs + project_selectors = all_specs.collect(&:project_selector) + included_projects = all_specs.collect(&:included_projects).flatten.uniq + + project_collection.names.each do |project_name| + active = included_projects.any? {|p| p == project_name } + active ||= project_selectors.any? {|s| s =~ project_name } + if active + project_spec = config.spec_for_project(project_name) + project_collection.create_project(name: project_name, spec: project_spec) + end end - end - project_collection.projects.values.each do |project| + project_collection.projects.values.each do |project| + with_log_level(project.spec.log_level) do + logger.info "Processing project '#{project.name}'" - match = project.name.match(project.spec.project_selector) - if match.nil? - logger.info "Skipping project '#{project.name}' as it does not match any selectors" - next - end + match = project.name.match(project.spec.project_selector) + if match.nil? + logger.info "Skipping project '#{project.name}' as it does not match any selectors" + next + end - if project.spec.skip - logger.info "Skipping project '#{project.name}'" - next - end + if project.spec.skip + logger.info "Skipping project '#{project.name}'" + next + end - async(annotation: "Project: #{project.name}") do + async(annotation: "Project: #{project.name}") do - # constructing the hash will cause any overrides to happen in the right - # order (includer wins over last included over first included) - params = project.all_parameters - parts = params.group_by(&:secret) - config_params, secret_params = (parts[false] || []), (parts[true] || []) - config_param_hash = params_to_hash(config_params) - secret_param_hash = params_to_hash(secret_params) + # constructing the hash will cause any overrides to happen in the right + # order (includer wins over last included over first included) + params = project.all_parameters + parts = params.group_by(&:secret) + config_params, secret_params = (parts[false] || []), (parts[true] || []) + config_param_hash = params_to_hash(config_params) + secret_param_hash = params_to_hash(secret_params) - parameter_origins = project.parameter_origins - param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) } - config_origins = Hash[param_origins_parts[true] || []] - secret_origins = Hash[param_origins_parts[false] || []] + parameter_origins = project.parameter_origins + param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) } + config_origins = Hash[param_origins_parts[true] || []] + secret_origins = Hash[param_origins_parts[false] || []] - config_param_hash = config_param_hash.reject do |k, v| - logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil? - v.nil? - end - secret_param_hash = secret_param_hash.reject do |k, v| - logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil? - v.nil? - end + config_param_hash = config_param_hash.reject do |k, v| + logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil? + v.nil? + end + secret_param_hash = secret_param_hash.reject do |k, v| + logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil? + v.nil? + end + + project.spec.resource_templates.each_with_index do |pair, i| + template_name, template = *pair + logger.debug { "Processing template '#{template_name}' (#{i+1}/#{project.spec.resource_templates.size})" } + resource_yml = template.render( + template: template_name, + kubetruth_namespace: kubeapi.namespace, + mapping_namespace: namespace, + project: project.name, + project_heirarchy: project.heirarchy, + debug: logger.debug?, + parameters: config_param_hash, + parameter_origins: config_origins, + secrets: secret_param_hash, + secret_origins: secret_origins, + templates: Template::TemplatesDrop.new(project: project.name, environment: project.spec.environment), + context: project.spec.context + ) + + template_id = "mapping: #{project.spec.name}, mapping_namespace: #{namespace}, project: #{project.name}, template: #{template_name}" + parsed_ymls = YAML.safe_load_stream(resource_yml, template_id) + logger.debug {"Skipping empty template"} if parsed_ymls.empty? + parsed_ymls.each do |parsed_yml| + async(annotation: "Apply Template: #{template_id}") do + kube_apply(parsed_yml) + end + end - project.spec.resource_templates.each_with_index do |pair, i| - template_name, template = *pair - logger.debug { "Processing template '#{template_name}' (#{i+1}/#{project.spec.resource_templates.size})" } - resource_yml = template.render( - template: template_name, - kubetruth_namespace: kubeapi.namespace, - mapping_namespace: namespace, - project: project.name, - project_heirarchy: project.heirarchy, - debug: logger.debug?, - parameters: config_param_hash, - parameter_origins: config_origins, - secrets: secret_param_hash, - secret_origins: secret_origins, - templates: Template::TemplatesDrop.new(project: project.name, environment: project.spec.environment), - context: project.spec.context - ) - - template_id = "mapping: #{project.spec.name}, mapping_namespace: #{namespace}, project: #{project.name}, template: #{template_name}" - parsed_ymls = YAML.safe_load_stream(resource_yml, template_id) - logger.debug {"Skipping empty template"} if parsed_ymls.empty? - parsed_ymls.each do |parsed_yml| - async(annotation: "Apply Template: #{template_id}") do - kube_apply(parsed_yml) end end - end end - end end end.wait diff --git a/lib/kubetruth/logging.rb b/lib/kubetruth/logging.rb index f8c23ea..99ea145 100644 --- a/lib/kubetruth/logging.rb +++ b/lib/kubetruth/logging.rb @@ -67,6 +67,14 @@ def self.clear sio&.clear end + def self.root_log_level=(level) + ::Logging.logger.root.level = level + end + + def self.root_log_level + ::Logging.levelify(::Logging::LNAMES[::Logging.logger.root.level]) + end + def self.setup_logging(level: :info, color: true) init_logger diff --git a/spec/kubetruth/etl_spec.rb b/spec/kubetruth/etl_spec.rb index a13b2d8..1a9452f 100644 --- a/spec/kubetruth/etl_spec.rb +++ b/spec/kubetruth/etl_spec.rb @@ -207,6 +207,26 @@ class ForceExit < Exception; end end + describe "#with_log_level" do + + it "does not set nil log level" do + expect(Kubetruth::Logging.root_log_level).to eq("debug") + etl.with_log_level(nil) do + expect(Kubetruth::Logging.root_log_level).to eq("debug") + end + expect(Kubetruth::Logging.root_log_level).to eq("debug") + end + + it "temporarily sets log level" do + expect(Kubetruth::Logging.root_log_level).to eq("debug") + etl.with_log_level("error") do + expect(Kubetruth::Logging.root_log_level).to eq("error") + end + expect(Kubetruth::Logging.root_log_level).to eq("debug") + end + + end + describe "#kube_apply" do it "calls kube to create new resource" do @@ -510,7 +530,7 @@ class ForceExit < Exception; end ]) project end - expect(collection).to receive(:names).and_return(["proj1"]) + allow(collection).to receive(:names).and_return(["proj1"]) allow(etl).to receive(:kube_apply) end @@ -546,6 +566,38 @@ class ForceExit < Exception; end etl.apply() end + it "sets log level when supplied by root pm in config" do + Kubetruth::Logging.root_log_level = "error" # undo debug logging set by test harness + config.root_spec.log_level = "error" + + allow(etl).to receive(:load_config).and_yield(@ns, config) + allow(config.root_spec.resource_templates["configmap"]).to receive(:render).and_return("") + + etl.apply() + expect(Logging.contents).to_not match(/DEBUG/) + Logging.clear + + config.root_spec.log_level = "debug" + etl.apply() + expect(Logging.contents).to match(/DEBUG/) + end + + it "sets log level when supplied by override pm in config" do + Kubetruth::Logging.root_log_level = "error" # undo debug logging set by test harness + + override_crd = {scope: "override", project_selector: "proj1", log_level: "debug"} + config = Kubetruth::Config.new([root_spec_crd, override_crd]) + + allow(etl).to receive(:load_config).and_yield(@ns, config) + allow(config.root_spec.resource_templates["configmap"]).to receive(:render).and_return("") + + etl.apply() + expect(Logging.contents).to_not match(/DEBUG.*Config ProjectSpec for root mapping/) + expect(Logging.contents).to match(/INFO.*ETL Processing project 'proj1'/) + expect(Logging.contents).to match(/DEBUG.*Template Evaluating template/) + end + + end describe "verify async behavior" do diff --git a/spec/kubetruth/logging_spec.rb b/spec/kubetruth/logging_spec.rb index 4e8f9a1..5e6eb7f 100644 --- a/spec/kubetruth/logging_spec.rb +++ b/spec/kubetruth/logging_spec.rb @@ -39,5 +39,24 @@ module Kubetruth end + describe "#root_log_level" do + + it "gets and sets root log level" do + expect(described_class.root_log_level).to eq("debug") + described_class.root_log_level = "error" + expect(described_class.root_log_level).to eq("error") + end + + it "logs at set log level" do + described_class.root_log_level = "info" + logger.info("infolog") + expect(Logging.contents).to include("infolog") + logger.debug("debuglog") + expect(Logging.contents).to_not include("debuglog") + end + + end + end + end \ No newline at end of file