From 06ddf404d9b05b923071444bcf9e43cb39e28909 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Fri, 15 Nov 2024 16:13:02 +0300 Subject: [PATCH] refactoring Signed-off-by: Pavel Okhlopkov --- cmd/shell-operator/main.go | 5 +- go.mod | 21 ++-- go.sum | 43 ++++--- pkg/app/debug.go | 13 +- pkg/app/webhook.go | 3 +- pkg/executor/executor.go | 117 ++++++++++++------ pkg/executor/executor_test.go | 79 +++++++----- pkg/filter/filter.go | 6 + pkg/filter/jq/apply_jq_exec.go | 29 +++++ .../jq/apply_libjq_go.go} | 21 +++- pkg/{ => filter}/jq/jq_exec.go | 0 .../binding_context.go | 6 +- .../binding_context_test.go | 4 +- pkg/hook/config/config.go | 3 +- pkg/hook/config/config_v0.go | 11 +- pkg/hook/config/config_v1.go | 24 ++-- .../admission_bindings_controller.go | 5 +- .../conversion_bindings_controller.go | 5 +- pkg/hook/controller/hook_controller.go | 15 ++- pkg/hook/controller/hook_controller_test.go | 12 +- .../kubernetes_bindings_controller.go | 14 +-- .../schedule_bindings_controller.go | 10 +- pkg/hook/hook.go | 41 +++--- pkg/hook/hook_manager.go | 42 ++++--- pkg/hook/hook_manager_test.go | 6 +- pkg/hook/hook_test.go | 6 +- pkg/hook/task_metadata/task_metadata.go | 14 +-- pkg/hook/task_metadata/task_metadata_test.go | 4 +- pkg/hook/types/bindings.go | 6 +- pkg/jq/apply_jq_exec.go | 17 --- .../error_handler.go | 10 +- .../factory.go | 5 +- .../filter.go | 11 +- .../filter_test.go | 8 +- .../kube_events_manager.go | 16 +-- .../kube_events_manager_test.go | 8 +- .../monitor.go | 14 +-- .../monitor_config.go | 6 +- .../monitor_test.go | 10 +- .../namespace_informer.go | 6 +- .../resource_informer.go | 22 ++-- .../types/types.go | 3 +- .../types/types_test.go | 0 .../util.go | 4 +- .../util_test.go | 2 +- .../metric_storage.go | 8 +- .../operation/operation.go | 0 .../operation/operation_test.go | 0 .../vault/vault.go | 2 +- .../vault/vault_test.go | 3 +- pkg/metric/storage.go | 3 +- pkg/metric/storage_mock.go | 2 +- pkg/metric/storage_test.go | 6 +- .../object_patch => object-patch}/helpers.go | 11 +- .../operation.go | 10 +- .../object_patch => object-patch}/options.go | 2 +- .../object_patch => object-patch}/patch.go | 5 +- .../patch_collector.go | 11 +- .../patch_test.go | 8 +- .../serialized_operations/invalid_create.yaml | 0 .../serialized_operations/invalid_delete.yaml | 0 .../serialized_operations/invalid_patch.yaml | 0 .../serialized_operations/valid_create.yaml | 0 .../serialized_operations/valid_delete.yaml | 0 .../serialized_operations/valid_patch.yaml | 0 .../validation.go | 2 +- .../validation_test.go | 2 +- .../schedule_manager.go | 6 +- .../schedule_manager_test.go | 5 +- .../types/types.go | 0 pkg/shell-operator/bootstrap.go | 14 +-- pkg/shell-operator/combine_binding_context.go | 3 +- .../combine_binding_context_test.go | 64 +++++----- pkg/shell-operator/kube_client.go | 14 +-- pkg/shell-operator/manager_events_handler.go | 14 +-- pkg/shell-operator/metrics_hooks.go | 6 +- pkg/shell-operator/metrics_operator.go | 10 +- pkg/shell-operator/operator.go | 42 +++---- pkg/shell-operator/operator_test.go | 2 +- pkg/task/queue/queue_set.go | 6 +- pkg/task/queue/task_queue.go | 6 +- pkg/webhook/admission/manager.go | 2 +- pkg/webhook/conversion/manager.go | 2 +- test/hook/context/context_combiner.go | 13 +- test/hook/context/generator.go | 13 +- test/hook/context/generator_test.go | 3 +- test/hook/context/state.go | 5 +- .../kube_event_manager_test.go | 12 +- .../kubeclient/object_patch_test.go | 31 +++-- test/integration/suite/run.go | 11 +- 90 files changed, 596 insertions(+), 470 deletions(-) create mode 100644 pkg/filter/filter.go create mode 100644 pkg/filter/jq/apply_jq_exec.go rename pkg/{jq/apply_libjq-go.go => filter/jq/apply_libjq_go.go} (61%) rename pkg/{ => filter}/jq/jq_exec.go (100%) rename pkg/hook/{binding_context => binding-context}/binding_context.go (98%) rename pkg/hook/{binding_context => binding-context}/binding_context_test.go (99%) delete mode 100644 pkg/jq/apply_jq_exec.go rename pkg/{kube_events_manager => kube-events-manager}/error_handler.go (90%) rename pkg/{kube_events_manager => kube-events-manager}/factory.go (99%) rename pkg/{kube_events_manager => kube-events-manager}/filter.go (79%) rename pkg/{kube_events_manager => kube-events-manager}/filter_test.go (72%) rename pkg/{kube_events_manager => kube-events-manager}/kube_events_manager.go (91%) rename pkg/{kube_events_manager => kube-events-manager}/kube_events_manager_test.go (99%) rename pkg/{kube_events_manager => kube-events-manager}/monitor.go (96%) rename pkg/{kube_events_manager => kube-events-manager}/monitor_config.go (97%) rename pkg/{kube_events_manager => kube-events-manager}/monitor_test.go (98%) rename pkg/{kube_events_manager => kube-events-manager}/namespace_informer.go (99%) rename pkg/{kube_events_manager => kube-events-manager}/resource_informer.go (95%) rename pkg/{kube_events_manager => kube-events-manager}/types/types.go (99%) rename pkg/{kube_events_manager => kube-events-manager}/types/types_test.go (100%) rename pkg/{kube_events_manager => kube-events-manager}/util.go (96%) rename pkg/{kube_events_manager => kube-events-manager}/util_test.go (87%) rename pkg/{metric_storage => metric-storage}/metric_storage.go (98%) rename pkg/{metric_storage => metric-storage}/operation/operation.go (100%) rename pkg/{metric_storage => metric-storage}/operation/operation_test.go (100%) rename pkg/{metric_storage => metric-storage}/vault/vault.go (100%) rename pkg/{metric_storage => metric-storage}/vault/vault_test.go (99%) rename pkg/{kube/object_patch => object-patch}/helpers.go (91%) rename pkg/{kube/object_patch => object-patch}/operation.go (97%) rename pkg/{kube/object_patch => object-patch}/options.go (99%) rename pkg/{kube/object_patch => object-patch}/patch.go (99%) rename pkg/{kube/object_patch => object-patch}/patch_collector.go (84%) rename pkg/{kube/object_patch => object-patch}/patch_test.go (99%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/invalid_create.yaml (100%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/invalid_delete.yaml (100%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/invalid_patch.yaml (100%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/valid_create.yaml (100%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/valid_delete.yaml (100%) rename pkg/{kube/object_patch => object-patch}/testdata/serialized_operations/valid_patch.yaml (100%) rename pkg/{kube/object_patch => object-patch}/validation.go (99%) rename pkg/{kube/object_patch => object-patch}/validation_test.go (96%) rename pkg/{schedule_manager => schedule-manager}/schedule_manager.go (96%) rename pkg/{schedule_manager => schedule-manager}/schedule_manager_test.go (97%) rename pkg/{schedule_manager => schedule-manager}/types/types.go (100%) diff --git a/cmd/shell-operator/main.go b/cmd/shell-operator/main.go index 16f8a727..2c0e7ae5 100644 --- a/cmd/shell-operator/main.go +++ b/cmd/shell-operator/main.go @@ -10,7 +10,7 @@ import ( "github.com/flant/kube-client/klogtolog" "github.com/flant/shell-operator/pkg/app" "github.com/flant/shell-operator/pkg/debug" - "github.com/flant/shell-operator/pkg/jq" + "github.com/flant/shell-operator/pkg/filter/jq" shell_operator "github.com/flant/shell-operator/pkg/shell-operator" utils_signal "github.com/flant/shell-operator/pkg/utils/signal" ) @@ -33,7 +33,8 @@ func main() { // print version kpApp.Command("version", "Show version.").Action(func(_ *kingpin.ParseContext) error { fmt.Printf("%s %s\n", app.AppName, app.Version) - fmt.Println(jq.FilterInfo()) + fl := jq.NewFilter(app.JqLibraryPath) + fmt.Println(fl.FilterInfo()) return nil }) diff --git a/go.mod b/go.mod index d5739ada..5c3b541c 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.29.8 - k8s.io/apiextensions-apiserver v0.28.4 + k8s.io/apiextensions-apiserver v0.29.0 k8s.io/apimachinery v0.29.8 k8s.io/client-go v0.29.8 sigs.k8s.io/yaml v1.4.0 @@ -42,10 +42,10 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/analysis v0.19.10 // indirect github.com/go-openapi/errors v0.19.7 // indirect @@ -61,7 +61,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/imdario/mergo v0.3.15 // indirect @@ -70,18 +70,17 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/pretty v1.2.0 // indirect go.mongodb.org/mongo-driver v1.5.4 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect @@ -93,9 +92,9 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index d25d28f3..6df4ea81 100644 --- a/go.sum +++ b/go.sum @@ -22,15 +22,16 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241106140903-258b93b3334e h1:QUQy+5Bv7/UzhfrytiG3c5gfLGhPppepVbRpbMisVIw= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241106140903-258b93b3334e/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/flant/go-openapi-validate v0.19.12-flant.0 h1:xk6kvc9fHKMgUdB6J7kbpbLR5vJOUzKAK8p3nrD7mDk= github.com/flant/go-openapi-validate v0.19.12-flant.0/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= github.com/flant/kube-client v1.2.2 h1:27LBs+PKJEFnkQXjPU9eIps7a7iyI13AKcSYj897DCU= @@ -39,12 +40,11 @@ github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee h1:evii83J+/6QGNv github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee/go.mod h1:f+REaGl/+pZR97rbTcwHEka/MAipoQQ2Mc0iQUj4ak0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -154,8 +154,8 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -207,8 +207,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -240,8 +240,9 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -279,9 +280,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= @@ -343,7 +343,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= @@ -410,18 +409,18 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.29.8 h1:ZBKg9clWnIGtQ5yGhNwMw2zyyrsIAQaXhZACcYNflQE= k8s.io/api v0.29.8/go.mod h1:XlGIpmpzKGrtVca7GlgNryZJ19SvQdI808NN7fy1SgQ= -k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= -k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.29.8 h1:uBHc9WuKiTHClIspJqtR84WNpG0aOGn45HWqxgXkk8Y= k8s.io/apimachinery v0.29.8/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= k8s.io/client-go v0.29.8 h1:QMRKcIzqE/qawknXcsi51GdIAYN8UP39S/M5KnFu/J0= k8s.io/client-go v0.29.8/go.mod h1:ZzrAAVrqO2jVXMb8My/jTke8n0a/mIynnA3y/1y1UB0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/pkg/app/debug.go b/pkg/app/debug.go index 5d9d93fc..19860a08 100644 --- a/pkg/app/debug.go +++ b/pkg/app/debug.go @@ -10,7 +10,10 @@ var DebugUnixSocket = "/var/run/shell-operator/debug.socket" var DebugHttpServerAddr = "" -var DebugKeepTmpFiles = "no" +var ( + DebugKeepTmpFilesVar = "no" + DebugKeepTmpFiles = false +) var DebugKubernetesAPI = false @@ -27,8 +30,12 @@ func DefineDebugFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause) { cmd.Flag("debug-keep-tmp-files", "set to yes to disable cleanup of temporary files"). Envar("DEBUG_KEEP_TMP_FILES"). Hidden(). - Default(DebugKeepTmpFiles). - StringVar(&DebugKeepTmpFiles) + Default(DebugKeepTmpFilesVar).Action(func(_ *kingpin.ParseContext) error { + DebugKeepTmpFiles = DebugKeepTmpFilesVar == "yes" + + return nil + }). + StringVar(&DebugKeepTmpFilesVar) cmd.Flag("debug-kubernetes-api", "enable client-go debug messages"). Envar("DEBUG_KUBERNETES_API"). diff --git a/pkg/app/webhook.go b/pkg/app/webhook.go index f6b36ecc..951abf22 100644 --- a/pkg/app/webhook.go +++ b/pkg/app/webhook.go @@ -1,11 +1,10 @@ package app import ( - "gopkg.in/alecthomas/kingpin.v2" - "github.com/flant/shell-operator/pkg/webhook/admission" "github.com/flant/shell-operator/pkg/webhook/conversion" "github.com/flant/shell-operator/pkg/webhook/server" + "gopkg.in/alecthomas/kingpin.v2" ) var ValidatingWebhookSettings = &admission.WebhookSettings{ diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 68cda40e..baf17a60 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -14,17 +14,9 @@ import ( "time" "github.com/deckhouse/deckhouse/pkg/log" - - "github.com/flant/shell-operator/pkg/app" utils "github.com/flant/shell-operator/pkg/utils/labels" ) -type CmdUsage struct { - Sys time.Duration - User time.Duration - MaxRss int64 -} - func Run(cmd *exec.Cmd) error { // TODO context: hook name, hook phase, hook binding // TODO observability @@ -33,21 +25,86 @@ func Run(cmd *exec.Cmd) error { return cmd.Run() } -func RunAndLogLines(cmd *exec.Cmd, logLabels map[string]string, logger *log.Logger) (*CmdUsage, error) { - // TODO observability +type Executor struct { + cmd *exec.Cmd + logProxyHookJSON bool + proxyJsonKey string + logger *log.Logger +} + +func (e *Executor) WithLogProxyHookJSON(logProxyHookJSON bool) *Executor { + e.logProxyHookJSON = logProxyHookJSON + + return e +} + +func (e *Executor) WithLogProxyHookJSONKey(logProxyHookJSONKey string) *Executor { + if logProxyHookJSONKey == "" { + return e + } + + e.proxyJsonKey = logProxyHookJSONKey + + return e +} + +func (e *Executor) WithLogger(logger *log.Logger) *Executor { + e.logger = logger + + return e +} + +func (e *Executor) WithCMDStdout(w io.Writer) *Executor { + e.cmd.Stdout = w + + return e +} + +func (e *Executor) WithCMDStderr(w io.Writer) *Executor { + e.cmd.Stderr = w + + return e +} + +func NewExecutor(dir string, entrypoint string, args []string, envs []string) *Executor { + cmd := exec.Command(entrypoint, args...) + cmd.Env = append(cmd.Env, envs...) + cmd.Dir = dir + + ex := &Executor{ + cmd: cmd, + proxyJsonKey: "proxyJsonLog", + logger: log.NewLogger(log.Options{}).Named("auto-executor"), + } + + return ex +} + +func (e *Executor) Output() ([]byte, error) { + e.logger.Debugf("Executing command '%s' in '%s' dir", strings.Join(e.cmd.Args, " "), e.cmd.Dir) + return e.cmd.Output() +} + +type CmdUsage struct { + Sys time.Duration + User time.Duration + MaxRss int64 +} + +func (e *Executor) RunAndLogLines(logLabels map[string]string) (*CmdUsage, error) { stdErr := bytes.NewBuffer(nil) - logEntry := utils.EnrichLoggerWithLabels(logger, logLabels) + logEntry := utils.EnrichLoggerWithLabels(e.logger, logLabels) stdoutLogEntry := logEntry.With("output", "stdout") stderrLogEntry := logEntry.With("output", "stderr") - logEntry.Debugf("Executing command '%s' in '%s' dir", strings.Join(cmd.Args, " "), cmd.Dir) + logEntry.Debugf("Executing command '%s' in '%s' dir", strings.Join(e.cmd.Args, " "), e.cmd.Dir) - plo := &proxyLogger{app.LogProxyHookJSON, stdoutLogEntry, make([]byte, 0)} - ple := &proxyLogger{app.LogProxyHookJSON, stderrLogEntry, make([]byte, 0)} - cmd.Stdout = plo - cmd.Stderr = io.MultiWriter(ple, stdErr) + plo := &proxyLogger{e.logProxyHookJSON, e.proxyJsonKey, stdoutLogEntry, make([]byte, 0)} + ple := &proxyLogger{e.logProxyHookJSON, e.proxyJsonKey, stderrLogEntry, make([]byte, 0)} + e.cmd.Stdout = plo + e.cmd.Stderr = io.MultiWriter(ple, stdErr) - err := cmd.Run() + err := e.cmd.Run() if err != nil { if len(stdErr.Bytes()) > 0 { return nil, fmt.Errorf("%s", stdErr.String()) @@ -57,14 +114,14 @@ func RunAndLogLines(cmd *exec.Cmd, logLabels map[string]string, logger *log.Logg } var usage *CmdUsage - if cmd.ProcessState != nil { + if e.cmd.ProcessState != nil { usage = &CmdUsage{ - Sys: cmd.ProcessState.SystemTime(), - User: cmd.ProcessState.UserTime(), + Sys: e.cmd.ProcessState.SystemTime(), + User: e.cmd.ProcessState.UserTime(), } // FIXME Maxrss is Unix specific. - sysUsage := cmd.ProcessState.SysUsage() + sysUsage := e.cmd.ProcessState.SysUsage() if v, ok := sysUsage.(*syscall.Rusage); ok { // v.Maxrss is int32 on arm/v7 usage.MaxRss = int64(v.Maxrss) //nolint:unconvert @@ -76,6 +133,7 @@ func RunAndLogLines(cmd *exec.Cmd, logLabels map[string]string, logger *log.Logg type proxyLogger struct { logProxyHookJSON bool + proxyJsonLogKey string logger *log.Logger @@ -116,7 +174,7 @@ func (pl *proxyLogger) Write(p []byte) (int, error) { return len(p), err } - logger := pl.logger.With(app.ProxyJsonLogKey, true) + logger := pl.logger.With(pl.proxyJsonLogKey, true) logLineRaw, _ := json.Marshal(logMap) logLine := string(logLineRaw) @@ -176,18 +234,3 @@ func (pl *proxyLogger) writerScanner(p []byte) { pl.logger.Error("reading from scanner", slog.String("error", err.Error())) } } - -func Output(cmd *exec.Cmd) (output []byte, err error) { - // TODO context: hook name, hook phase, hook binding - // TODO observability - log.Debugf("Executing command '%s' in '%s' dir", strings.Join(cmd.Args, " "), cmd.Dir) - output, err = cmd.Output() - return -} - -func MakeCommand(dir string, entrypoint string, args []string, envs []string) *exec.Cmd { - cmd := exec.Command(entrypoint, args...) - cmd.Env = append(cmd.Env, envs...) - cmd.Dir = dir - return cmd -} diff --git a/pkg/executor/executor_test.go b/pkg/executor/executor_test.go index 0a9d72d8..0377d26a 100644 --- a/pkg/executor/executor_test.go +++ b/pkg/executor/executor_test.go @@ -5,16 +5,14 @@ import ( "io" "math/rand/v2" "os" - "os/exec" "regexp" "testing" "time" - "github.com/deckhouse/deckhouse/pkg/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/flant/shell-operator/pkg/app" + "github.com/deckhouse/deckhouse/pkg/log" ) func TestRunAndLogLines(t *testing.T) { @@ -35,11 +33,11 @@ func TestRunAndLogLines(t *testing.T) { logger.SetOutput(&buf) t.Run("simple log", func(t *testing.T) { - app.LogProxyHookJSON = true - - cmd := exec.Command("echo", `{"foo": "baz"}`) + ex := NewExecutor("", "echo", []string{`{"foo": "baz"}`}, []string{}). + WithLogProxyHookJSON(true). + WithLogger(logger) - _, err := RunAndLogLines(cmd, map[string]string{"a": "b"}, logger) + _, err := ex.RunAndLogLines(map[string]string{"a": "b"}) assert.NoError(t, err) assert.Equal(t, buf.String(), `{"level":"fatal","msg":"hook result","a":"b","hook":{"foo":"baz"},"output":"stdout","proxyJsonLog":true,"time":"2006-01-02T15:04:05Z"}`+"\n") @@ -48,10 +46,10 @@ func TestRunAndLogLines(t *testing.T) { }) t.Run("not json log", func(t *testing.T) { - app.LogProxyHookJSON = false - cmd := exec.Command("echo", `foobar`) + ex := NewExecutor("", "echo", []string{"foobar"}, []string{}). + WithLogger(logger) - _, err := RunAndLogLines(cmd, map[string]string{"a": "b"}, logger) + _, err := ex.RunAndLogLines(map[string]string{"a": "b"}) assert.NoError(t, err) assert.Equal(t, buf.String(), `{"level":"info","msg":"foobar","a":"b","output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n") @@ -67,10 +65,11 @@ func TestRunAndLogLines(t *testing.T) { _, _ = io.WriteString(f, `{"foo": "`+randStringRunes(1024*1024)+`"}`) - app.LogProxyHookJSON = true - cmd := exec.Command("cat", f.Name()) + ex := NewExecutor("", "cat", []string{f.Name()}, []string{}). + WithLogProxyHookJSON(true). + WithLogger(logger) - _, err = RunAndLogLines(cmd, map[string]string{"a": "b"}, logger) + _, err = ex.RunAndLogLines(map[string]string{"a": "b"}) assert.NoError(t, err) reg := regexp.MustCompile(`{"level":"fatal","msg":"hook result","a":"b","hook":{"truncated":".*:truncated"},"output":"stdout","proxyJsonLog":true,"time":"2006-01-02T15:04:05Z"`) @@ -87,10 +86,10 @@ func TestRunAndLogLines(t *testing.T) { _, _ = io.WriteString(f, `result `+randStringRunes(1024*1024)) - app.LogProxyHookJSON = false - cmd := exec.Command("cat", f.Name()) + ex := NewExecutor("", "cat", []string{f.Name()}, []string{}). + WithLogger(logger) - _, err = RunAndLogLines(cmd, map[string]string{"a": "b"}, logger) + _, err = ex.RunAndLogLines(map[string]string{"a": "b"}) assert.NoError(t, err) reg := regexp.MustCompile(`{"level":"info","msg":"result .*:truncated","a":"b","output":"stdout","time":"2006-01-02T15:04:05Z"`) @@ -101,25 +100,32 @@ func TestRunAndLogLines(t *testing.T) { t.Run("invalid json structure", func(t *testing.T) { logger.SetLevel(log.LevelDebug) - app.LogProxyHookJSON = true - cmd := exec.Command("echo", `["a","b","c"]`) - _, err := RunAndLogLines(cmd, map[string]string{"a": "b"}, logger) + + ex := NewExecutor("", "echo", []string{`["a","b","c"]`}, []string{}). + WithLogProxyHookJSON(true). + WithLogger(logger) + + _, err := ex.RunAndLogLines(map[string]string{"a": "b"}) assert.NoError(t, err) - assert.Equal(t, buf.String(), `{"level":"debug","msg":"Executing command 'echo [\"a\",\"b\",\"c\"]' in '' dir","source":"executor/executor.go:43","a":"b","time":"2006-01-02T15:04:05Z"}`+"\n"+ - `{"level":"debug","msg":"json log line not map[string]interface{}","source":"executor/executor.go:111","a":"b","line":["a","b","c"],"output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n"+ - `{"level":"info","msg":"[\"a\",\"b\",\"c\"]\n","source":"executor/executor.go:114","a":"b","output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n") + + assert.Equal(t, buf.String(), `{"level":"debug","msg":"Executing command 'echo [\"a\",\"b\",\"c\"]' in '' dir","source":"executor/executor.go:100","a":"b","time":"2006-01-02T15:04:05Z"}`+"\n"+ + `{"level":"debug","msg":"json log line not map[string]interface{}","source":"executor/executor.go:169","a":"b","line":["a","b","c"],"output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n"+ + `{"level":"info","msg":"[\"a\",\"b\",\"c\"]\n","source":"executor/executor.go:172","a":"b","output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n") buf.Reset() }) t.Run("multiline", func(t *testing.T) { logger.SetLevel(log.LevelInfo) - app.LogProxyHookJSON = true - cmd := exec.Command("echo", ` + arg := ` {"a":"b", "c":"d"} -`) - _, err := RunAndLogLines(cmd, map[string]string{"foor": "baar"}, logger) +` + ex := NewExecutor("", "echo", []string{arg}, []string{}). + WithLogProxyHookJSON(true). + WithLogger(logger) + + _, err := ex.RunAndLogLines(map[string]string{"foor": "baar"}) assert.NoError(t, err) assert.Equal(t, buf.String(), `{"level":"fatal","msg":"hook result","foor":"baar","hook":{"a":"b","c":"d"},"output":"stdout","proxyJsonLog":true,"time":"2006-01-02T15:04:05Z"}`+"\n") @@ -127,12 +133,14 @@ func TestRunAndLogLines(t *testing.T) { }) t.Run("multiline non json", func(t *testing.T) { - app.LogProxyHookJSON = false - cmd := exec.Command("echo", ` + arg := ` a b c d -`) - _, err := RunAndLogLines(cmd, map[string]string{"foor": "baar"}, logger) +` + ex := NewExecutor("", "echo", []string{arg}, []string{}). + WithLogger(logger) + + _, err := ex.RunAndLogLines(map[string]string{"foor": "baar"}) assert.NoError(t, err) assert.Equal(t, buf.String(), `{"level":"info","msg":"a b","foor":"baar","output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n"+ `{"level":"info","msg":"c d","foor":"baar","output":"stdout","time":"2006-01-02T15:04:05Z"}`+"\n") @@ -141,12 +149,15 @@ c d }) t.Run("multiline json", func(t *testing.T) { - app.LogProxyHookJSON = true - cmd := exec.Command("echo", `{ + arg := `{ "a":"b", "c":"d" -}`) - _, err := RunAndLogLines(cmd, map[string]string{"foor": "baar"}, logger) +}` + ex := NewExecutor("", "echo", []string{arg}, []string{}). + WithLogProxyHookJSON(true). + WithLogger(logger) + + _, err := ex.RunAndLogLines(map[string]string{"foor": "baar"}) assert.NoError(t, err) assert.Equal(t, buf.String(), `{"level":"fatal","msg":"hook result","foor":"baar","hook":{"a":"b","c":"d"},"output":"stdout","proxyJsonLog":true,"time":"2006-01-02T15:04:05Z"}`+"\n") diff --git a/pkg/filter/filter.go b/pkg/filter/filter.go new file mode 100644 index 00000000..c24d29d6 --- /dev/null +++ b/pkg/filter/filter.go @@ -0,0 +1,6 @@ +package filter + +type Filter interface { + ApplyFilter(filterStr string, data []byte) (string, error) + FilterInfo() string +} diff --git a/pkg/filter/jq/apply_jq_exec.go b/pkg/filter/jq/apply_jq_exec.go new file mode 100644 index 00000000..6543cf31 --- /dev/null +++ b/pkg/filter/jq/apply_jq_exec.go @@ -0,0 +1,29 @@ +//go:build !cgo || (cgo && !use_libjq) +// +build !cgo cgo,!use_libjq + +package jq + +import "github.com/flant/shell-operator/pkg/filter" + +var _ filter.Filter = (*Filter)(nil) + +func NewFilter(libpath string) *Filter { + return &Filter{ + Libpath: libpath, + } +} + +type Filter struct { + Libpath string +} + +// ApplyJqFilter runs jq expression provided in jqFilter with jsonData as input. +// +// It uses jq as a subprocess. +func (f *Filter) ApplyFilter(jqFilter string, jsonData []byte) (string, error) { + return jqExec(jqFilter, jsonData, f.Libpath) +} + +func (f *Filter) FilterInfo() string { + return "jqFilter implementation: use jq binary from $PATH" +} diff --git a/pkg/jq/apply_libjq-go.go b/pkg/filter/jq/apply_libjq_go.go similarity index 61% rename from pkg/jq/apply_libjq-go.go rename to pkg/filter/jq/apply_libjq_go.go index 633d103d..aae9535a 100644 --- a/pkg/jq/apply_libjq-go.go +++ b/pkg/filter/jq/apply_libjq_go.go @@ -8,27 +8,40 @@ import ( "os" . "github.com/flant/libjq-go" + "github.com/flant/shell-operator/pkg/filter" ) +var _ filter.Filter = (*Filter)(nil) + +func NewFilter(libpath string) *Filter { + return &Filter{ + Libpath: libpath, + } +} + +type Filter struct { + Libpath string +} + // Note: add build tag 'use_libjg' to build with libjq-go. // ApplyJqFilter runs jq expression provided in jqFilter with jsonData as input. // // It uses libjq-go or executes jq as a binary if $JQ_EXEC is set to "yes". -func ApplyJqFilter(jqFilter string, jsonData []byte, libPath string) (string, error) { +func (f *Filter) ApplyFilter(jqFilter string, jsonData []byte) (string, error) { // Use jq exec filtering if environment variable is present. if os.Getenv("JQ_EXEC") == "yes" { - return jqExec(jqFilter, jsonData, libPath) + return jqExec(jqFilter, jsonData, f.Libpath) } - result, err := Jq().WithLibPath(libPath).Program(jqFilter).Cached().Run(string(jsonData)) + result, err := Jq().WithLibPath(f.Libpath).Program(jqFilter).Cached().Run(string(jsonData)) if err != nil { return "", fmt.Errorf("libjq filter '%s': '%s'", jqFilter, err) } return result, nil } -func FilterInfo() string { +func (f *Filter) FilterInfo() string { if os.Getenv("JQ_EXEC") == "yes" { return "jqFilter implementation: use jq binary from $PATH (JQ_EXEC=yes is set)" } diff --git a/pkg/jq/jq_exec.go b/pkg/filter/jq/jq_exec.go similarity index 100% rename from pkg/jq/jq_exec.go rename to pkg/filter/jq/jq_exec.go diff --git a/pkg/hook/binding_context/binding_context.go b/pkg/hook/binding-context/binding_context.go similarity index 98% rename from pkg/hook/binding_context/binding_context.go rename to pkg/hook/binding-context/binding_context.go index 126b5b99..ab7cd941 100644 --- a/pkg/hook/binding_context/binding_context.go +++ b/pkg/hook/binding-context/binding_context.go @@ -1,14 +1,14 @@ -package binding_context +package bindingcontext import ( "encoding/json" - "github.com/deckhouse/deckhouse/pkg/log" v1 "k8s.io/api/admission/v1" apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "github.com/deckhouse/deckhouse/pkg/log" . "github.com/flant/shell-operator/pkg/hook/types" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" ) // BindingContext contains information about event for hook diff --git a/pkg/hook/binding_context/binding_context_test.go b/pkg/hook/binding-context/binding_context_test.go similarity index 99% rename from pkg/hook/binding_context/binding_context_test.go rename to pkg/hook/binding-context/binding_context_test.go index efba5588..57b7d47f 100644 --- a/pkg/hook/binding_context/binding_context_test.go +++ b/pkg/hook/binding-context/binding_context_test.go @@ -1,4 +1,4 @@ -package binding_context +package bindingcontext // TODO: need refactoring // change JQ tests for another testing tool @@ -12,7 +12,7 @@ package binding_context // . "github.com/flant/libjq-go" // . "github.com/flant/shell-operator/pkg/hook/types" -// . "github.com/flant/shell-operator/pkg/kube_events_manager/types" +// . "github.com/flant/shell-operator/pkg/kube-events-manager/types" // ) // func JqEqual(t *testing.T, input []byte, program string, expected string) { diff --git a/pkg/hook/config/config.go b/pkg/hook/config/config.go index f42bac2d..d2673444 100644 --- a/pkg/hook/config/config.go +++ b/pkg/hook/config/config.go @@ -3,9 +3,8 @@ package config import ( "fmt" - "sigs.k8s.io/yaml" - . "github.com/flant/shell-operator/pkg/hook/types" + "sigs.k8s.io/yaml" ) var validBindingTypes = []BindingType{OnStartup, Schedule, OnKubernetesEvent, KubernetesValidating, KubernetesMutating, KubernetesConversion} diff --git a/pkg/hook/config/config_v0.go b/pkg/hook/config/config_v0.go index e0c463fc..c141c47c 100644 --- a/pkg/hook/config/config_v0.go +++ b/pkg/hook/config/config_v0.go @@ -3,13 +3,12 @@ package config import ( "fmt" + . "github.com/flant/shell-operator/pkg/hook/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + . "github.com/flant/shell-operator/pkg/schedule-manager/types" "gopkg.in/robfig/cron.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - . "github.com/flant/shell-operator/pkg/schedule_manager/types" ) type HookConfigV0 struct { @@ -69,7 +68,7 @@ func (cv0 *HookConfigV0) ConvertAndCheck(c *HookConfig) (err error) { return fmt.Errorf("invalid onKubernetesEvent config [%d]: %v", i, err) } - monitor := &kube_events_manager.MonitorConfig{} + monitor := &kubeeventsmanager.MonitorConfig{} monitor.Metadata.DebugName = MonitorDebugName(kubeCfg.Name, i) monitor.Metadata.MonitorId = MonitorConfigID() monitor.Metadata.LogLabels = map[string]string{} diff --git a/pkg/hook/config/config_v1.go b/pkg/hook/config/config_v1.go index 16307950..afc6e775 100644 --- a/pkg/hook/config/config_v1.go +++ b/pkg/hook/config/config_v1.go @@ -5,6 +5,13 @@ import ( "strconv" "time" + . "github.com/flant/shell-operator/pkg/hook/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + . "github.com/flant/shell-operator/pkg/schedule-manager/types" + "github.com/flant/shell-operator/pkg/webhook/admission" + "github.com/flant/shell-operator/pkg/webhook/conversion" + "github.com/flant/shell-operator/pkg/webhook/validating/validation" "github.com/hashicorp/go-multierror" "gopkg.in/robfig/cron.v2" v1 "k8s.io/api/admissionregistration/v1" @@ -12,13 +19,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "github.com/flant/shell-operator/pkg/app" - . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - . "github.com/flant/shell-operator/pkg/schedule_manager/types" - "github.com/flant/shell-operator/pkg/webhook/admission" - "github.com/flant/shell-operator/pkg/webhook/conversion" - "github.com/flant/shell-operator/pkg/webhook/validating/validation" ) type HookConfigV1 struct { @@ -118,7 +118,7 @@ func (cv1 *HookConfigV1) ConvertAndCheck(c *HookConfig) (err error) { return fmt.Errorf("invalid kubernetes config [%d]: %v", i, err) } - monitor := &kube_events_manager.MonitorConfig{} + monitor := &kubeeventsmanager.MonitorConfig{} monitor.Metadata.DebugName = MonitorDebugName(kubeCfg.Name, i) monitor.Metadata.MonitorId = MonitorConfigID() monitor.Metadata.LogLabels = map[string]string{} @@ -370,14 +370,14 @@ func (cv1 *HookConfigV1) CheckOnKubernetesEvent(kubeCfg OnKubernetesEventConfigV } if kubeCfg.LabelSelector != nil { - _, err := kube_events_manager.FormatLabelSelector(kubeCfg.LabelSelector) + _, err := kubeeventsmanager.FormatLabelSelector(kubeCfg.LabelSelector) if err != nil { allErr = multierror.Append(allErr, fmt.Errorf("labelSelector is invalid: %v", err)) } } if kubeCfg.FieldSelector != nil { - _, err := kube_events_manager.FormatFieldSelector((*FieldSelector)(kubeCfg.FieldSelector)) + _, err := kubeeventsmanager.FormatFieldSelector((*FieldSelector)(kubeCfg.FieldSelector)) if err != nil { allErr = multierror.Append(allErr, fmt.Errorf("fieldSelector is invalid: %v", err)) } @@ -407,14 +407,14 @@ func (cv1 *HookConfigV1) CheckAdmission(kubeConfigs []OnKubernetesEventConfig, c } if cfgV1.LabelSelector != nil { - _, err := kube_events_manager.FormatLabelSelector(cfgV1.LabelSelector) + _, err := kubeeventsmanager.FormatLabelSelector(cfgV1.LabelSelector) if err != nil { allErr = multierror.Append(allErr, fmt.Errorf("labelSelector is invalid: %v", err)) } } if cfgV1.Namespace != nil && cfgV1.Namespace.LabelSelector != nil { - _, err := kube_events_manager.FormatLabelSelector(cfgV1.Namespace.LabelSelector) + _, err := kubeeventsmanager.FormatLabelSelector(cfgV1.Namespace.LabelSelector) if err != nil { allErr = multierror.Append(allErr, fmt.Errorf("namespace.labelSelector is invalid: %v", err)) } diff --git a/pkg/hook/controller/admission_bindings_controller.go b/pkg/hook/controller/admission_bindings_controller.go index cbeeec32..90e564a4 100644 --- a/pkg/hook/controller/admission_bindings_controller.go +++ b/pkg/hook/controller/admission_bindings_controller.go @@ -2,11 +2,10 @@ package controller import ( "github.com/deckhouse/deckhouse/pkg/log" - v1 "k8s.io/api/admission/v1" - - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/flant/shell-operator/pkg/hook/types" "github.com/flant/shell-operator/pkg/webhook/admission" + v1 "k8s.io/api/admission/v1" ) // AdmissionBindingToWebhookLink is a link between a hook and a webhook configuration. diff --git a/pkg/hook/controller/conversion_bindings_controller.go b/pkg/hook/controller/conversion_bindings_controller.go index 4548e01e..fed0a0dc 100644 --- a/pkg/hook/controller/conversion_bindings_controller.go +++ b/pkg/hook/controller/conversion_bindings_controller.go @@ -2,11 +2,10 @@ package controller import ( "github.com/deckhouse/deckhouse/pkg/log" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/flant/shell-operator/pkg/hook/types" "github.com/flant/shell-operator/pkg/webhook/conversion" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) // A link between a hook and a kube monitor diff --git a/pkg/hook/controller/hook_controller.go b/pkg/hook/controller/hook_controller.go index 54cd166d..f473f2b2 100644 --- a/pkg/hook/controller/hook_controller.go +++ b/pkg/hook/controller/hook_controller.go @@ -2,15 +2,14 @@ package controller import ( "github.com/deckhouse/deckhouse/pkg/log" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/schedule_manager" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" "github.com/flant/shell-operator/pkg/webhook/admission" "github.com/flant/shell-operator/pkg/webhook/conversion" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) type BindingExecutionInfo struct { @@ -53,7 +52,7 @@ type HookController struct { logger *log.Logger } -func (hc *HookController) InitKubernetesBindings(bindings []OnKubernetesEventConfig, kubeEventMgr kube_events_manager.KubeEventsManager, logger *log.Logger) { +func (hc *HookController) InitKubernetesBindings(bindings []OnKubernetesEventConfig, kubeEventMgr kubeeventsmanager.KubeEventsManager, logger *log.Logger) { if len(bindings) == 0 { return } @@ -66,7 +65,7 @@ func (hc *HookController) InitKubernetesBindings(bindings []OnKubernetesEventCon hc.logger = logger } -func (hc *HookController) InitScheduleBindings(bindings []ScheduleConfig, scheduleMgr schedule_manager.ScheduleManager) { +func (hc *HookController) InitScheduleBindings(bindings []ScheduleConfig, scheduleMgr schedulemanager.ScheduleManager) { if len(bindings) == 0 { return } diff --git a/pkg/hook/controller/hook_controller_test.go b/pkg/hook/controller/hook_controller_test.go index 92683f9f..8b05114f 100644 --- a/pkg/hook/controller/hook_controller_test.go +++ b/pkg/hook/controller/hook_controller_test.go @@ -5,14 +5,14 @@ import ( "testing" "github.com/deckhouse/deckhouse/pkg/log" + bindingcontext "github.com/flant/shell-operator/pkg/hook/binding-context" + "github.com/flant/shell-operator/pkg/hook/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + types2 "github.com/flant/shell-operator/pkg/kube-events-manager/types" . "github.com/onsi/gomega" "github.com/flant/kube-client/fake" - "github.com/flant/shell-operator/pkg/hook/binding_context" "github.com/flant/shell-operator/pkg/hook/config" - "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - types2 "github.com/flant/shell-operator/pkg/kube_events_manager/types" ) // Test updating snapshots for combined contexts. @@ -20,7 +20,7 @@ func Test_UpdateSnapshots(t *testing.T) { g := NewWithT(t) fc := fake.NewFakeCluster(fake.ClusterVersionV121) - mgr := kube_events_manager.NewKubeEventsManager(context.Background(), fc.Client, log.NewNop()) + mgr := kubeeventsmanager.NewKubeEventsManager(context.Background(), fc.Client, log.NewNop()) testHookConfig := ` configVersion: v1 @@ -51,7 +51,7 @@ kubernetes: hc.EnableScheduleBindings() // Test case: combined binding context for binding_2 and binding_3. - bcs := []binding_context.BindingContext{ + bcs := []bindingcontext.BindingContext{ { Binding: "binding_2", Type: types2.TypeEvent, diff --git a/pkg/hook/controller/kubernetes_bindings_controller.go b/pkg/hook/controller/kubernetes_bindings_controller.go index dd395d31..f2696271 100644 --- a/pkg/hook/controller/kubernetes_bindings_controller.go +++ b/pkg/hook/controller/kubernetes_bindings_controller.go @@ -4,11 +4,11 @@ import ( "fmt" "github.com/deckhouse/deckhouse/pkg/log" - - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + utils "github.com/flant/shell-operator/pkg/utils/labels" ) @@ -21,7 +21,7 @@ type KubernetesBindingToMonitorLink struct { // KubernetesBindingsController handles kubernetes bindings for one hook. type KubernetesBindingsController interface { WithKubernetesBindings([]OnKubernetesEventConfig) - WithKubeEventsManager(kube_events_manager.KubeEventsManager) + WithKubeEventsManager(kubeeventsmanager.KubeEventsManager) EnableKubernetesBindings() ([]BindingExecutionInfo, error) UpdateMonitor(monitorId string, kind, apiVersion string) error UnlockEvents() @@ -47,7 +47,7 @@ type kubernetesBindingsController struct { KubernetesBindings []OnKubernetesEventConfig // dependencies - kubeEventsManager kube_events_manager.KubeEventsManager + kubeEventsManager kubeeventsmanager.KubeEventsManager logger *log.Logger } @@ -67,7 +67,7 @@ func (c *kubernetesBindingsController) WithKubernetesBindings(bindings []OnKuber c.KubernetesBindings = bindings } -func (c *kubernetesBindingsController) WithKubeEventsManager(kubeEventsManager kube_events_manager.KubeEventsManager) { +func (c *kubernetesBindingsController) WithKubeEventsManager(kubeEventsManager kubeeventsmanager.KubeEventsManager) { c.kubeEventsManager = kubeEventsManager } diff --git a/pkg/hook/controller/schedule_bindings_controller.go b/pkg/hook/controller/schedule_bindings_controller.go index 3f83438a..ca29d7dd 100644 --- a/pkg/hook/controller/schedule_bindings_controller.go +++ b/pkg/hook/controller/schedule_bindings_controller.go @@ -1,9 +1,9 @@ package controller import ( - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/schedule_manager" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" ) // A link between a hook and a kube monitor @@ -20,7 +20,7 @@ type ScheduleBindingToCrontabLink struct { // ScheduleBindingsController handles schedule bindings for one hook. type ScheduleBindingsController interface { WithScheduleBindings([]ScheduleConfig) - WithScheduleManager(schedule_manager.ScheduleManager) + WithScheduleManager(schedulemanager.ScheduleManager) EnableScheduleBindings() DisableScheduleBindings() CanHandleEvent(crontab string) bool @@ -36,7 +36,7 @@ type scheduleBindingsController struct { ScheduleBindings []ScheduleConfig // dependencies - scheduleManager schedule_manager.ScheduleManager + scheduleManager schedulemanager.ScheduleManager } // kubernetesHooksController should implement the KubernetesHooksController @@ -53,7 +53,7 @@ func (c *scheduleBindingsController) WithScheduleBindings(bindings []ScheduleCon c.ScheduleBindings = bindings } -func (c *scheduleBindingsController) WithScheduleManager(scheduleManager schedule_manager.ScheduleManager) { +func (c *scheduleBindingsController) WithScheduleManager(scheduleManager schedulemanager.ScheduleManager) { c.scheduleManager = scheduleManager } diff --git a/pkg/hook/hook.go b/pkg/hook/hook.go index 803c55b1..93ee51fb 100644 --- a/pkg/hook/hook.go +++ b/pkg/hook/hook.go @@ -8,18 +8,17 @@ import ( "path/filepath" "strings" - "github.com/deckhouse/deckhouse/pkg/log" uuid "github.com/gofrs/uuid/v5" "github.com/kennygrant/sanitize" "golang.org/x/time/rate" - "github.com/flant/shell-operator/pkg/app" + "github.com/deckhouse/deckhouse/pkg/log" "github.com/flant/shell-operator/pkg/executor" - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" "github.com/flant/shell-operator/pkg/hook/config" "github.com/flant/shell-operator/pkg/hook/controller" . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/metric_storage/operation" + "github.com/flant/shell-operator/pkg/metric-storage/operation" "github.com/flant/shell-operator/pkg/webhook/admission" "github.com/flant/shell-operator/pkg/webhook/conversion" ) @@ -44,17 +43,24 @@ type Hook struct { HookController *controller.HookController RateLimiter *rate.Limiter - TmpDir string + TmpDir string + KeepTemporaryHookFiles bool + + LogProxyHookJSON bool + LogProxyHookJSONKey string Logger *log.Logger } -func NewHook(name, path string, logger *log.Logger) *Hook { +func NewHook(name, path string, keepTemporaryHookFiles bool, logProxyHookJSON bool, logProxyHookJSONKey string, logger *log.Logger) *Hook { return &Hook{ - Name: name, - Path: path, - Config: &config.HookConfig{}, - Logger: logger, + Name: name, + Path: path, + Config: &config.HookConfig{}, + KeepTemporaryHookFiles: keepTemporaryHookFiles, + LogProxyHookJSON: logProxyHookJSON, + LogProxyHookJSONKey: logProxyHookJSONKey, + Logger: logger, } } @@ -118,7 +124,7 @@ func (h *Hook) Run(_ BindingType, context []BindingContext, logLabels map[string // remove tmp file on hook exit defer func() { - if app.DebugKeepTmpFiles != "yes" { + if h.KeepTemporaryHookFiles { _ = os.Remove(contextPath) _ = os.Remove(metricsPath) _ = os.Remove(conversionPath) @@ -130,7 +136,7 @@ func (h *Hook) Run(_ BindingType, context []BindingContext, logLabels map[string envs := make([]string, 0) envs = append(envs, os.Environ()...) if contextPath != "" { - envs = append(envs, fmt.Sprintf("BINDING_CONTEXT_PATH=%s", contextPath)) + envs = append(envs, fmt.Sprintf("bindingcontext_PATH=%s", contextPath)) envs = append(envs, fmt.Sprintf("METRICS_PATH=%s", metricsPath)) envs = append(envs, fmt.Sprintf("CONVERSION_RESPONSE_PATH=%s", conversionPath)) envs = append(envs, fmt.Sprintf("VALIDATING_RESPONSE_PATH=%s", admissionPath)) @@ -138,11 +144,18 @@ func (h *Hook) Run(_ BindingType, context []BindingContext, logLabels map[string envs = append(envs, fmt.Sprintf("KUBERNETES_PATCH_PATH=%s", kubernetesPatchPath)) } - hookCmd := executor.MakeCommand(path.Dir(h.Path), h.Path, []string{}, envs) + hookCmd := executor.NewExecutor( + path.Dir(h.Path), + h.Path, + []string{}, + envs). + WithLogProxyHookJSON(h.LogProxyHookJSON). + WithLogProxyHookJSONKey(h.LogProxyHookJSONKey). + WithLogger(h.Logger.Named("executor")) result := &Result{} - result.Usage, err = executor.RunAndLogLines(hookCmd, logLabels, h.Logger) + result.Usage, err = hookCmd.RunAndLogLines(logLabels) if err != nil { return result, fmt.Errorf("%s FAILED: %s", h.Name, err) } diff --git a/pkg/hook/hook_manager.go b/pkg/hook/hook_manager.go index 5d31c725..6da58a49 100644 --- a/pkg/hook/hook_manager.go +++ b/pkg/hook/hook_manager.go @@ -9,25 +9,26 @@ import ( "strings" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/app" + . "github.com/flant/shell-operator/pkg/hook/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" + "github.com/flant/shell-operator/pkg/webhook/admission" + "github.com/flant/shell-operator/pkg/webhook/conversion" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "github.com/flant/shell-operator/pkg/executor" "github.com/flant/shell-operator/pkg/hook/controller" - . "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/schedule_manager" utils_file "github.com/flant/shell-operator/pkg/utils/file" - "github.com/flant/shell-operator/pkg/webhook/admission" - "github.com/flant/shell-operator/pkg/webhook/conversion" ) type Manager struct { // dependencies workingDir string tempDir string - kubeEventsManager kube_events_manager.KubeEventsManager - scheduleManager schedule_manager.ScheduleManager + kubeEventsManager kubeeventsmanager.KubeEventsManager + scheduleManager schedulemanager.ScheduleManager conversionWebhookManager *conversion.WebhookManager admissionWebhookManager *admission.WebhookManager @@ -49,8 +50,8 @@ type Manager struct { type ManagerConfig struct { WorkingDir string TempDir string - Kmgr kube_events_manager.KubeEventsManager - Smgr schedule_manager.ScheduleManager + Kmgr kubeeventsmanager.KubeEventsManager + Smgr schedulemanager.ScheduleManager Wmgr *admission.WebhookManager Cmgr *conversion.WebhookManager @@ -132,7 +133,7 @@ func (hm *Manager) loadHook(hookPath string) (hook *Hook, err error) { if err != nil { return nil, err } - hook = NewHook(hookName, hookPath, hm.logger.Named("hook")) + hook = NewHook(hookName, hookPath, app.DebugKeepTmpFiles, app.LogProxyHookJSON, app.ProxyJsonLogKey, hm.logger.Named("hook")) hookEntry := hm.logger.With("hook", hook.Name). With("phase", "config") @@ -210,16 +211,23 @@ func (hm *Manager) loadHook(hookPath string) (hook *Hook, err error) { func (hm *Manager) execCommandOutput(hookName string, dir string, entrypoint string, envs []string, args []string) ([]byte, error) { envs = append(os.Environ(), envs...) - cmd := executor.MakeCommand(dir, entrypoint, args, envs) - cmd.Stdout = nil - cmd.Stderr = nil + hookCmd := executor.NewExecutor( + dir, + entrypoint, + args, + envs). + WithLogProxyHookJSON(app.LogProxyHookJSON). + WithLogProxyHookJSONKey(app.ProxyJsonLogKey). + WithCMDStdout(nil). + WithCMDStderr(nil). + WithLogger(hm.logger.Named("executor")) debugEntry := hm.logger.With("hook", hookName). - With("cmd", strings.Join(cmd.Args, " ")) + With("cmd", strings.Join(args, " ")) - debugEntry.Debugf("Executing hook in %s", cmd.Dir) + debugEntry.Debugf("Executing hook in %s", dir) - output, err := executor.Output(cmd) + output, err := hookCmd.Output() if err != nil { return output, err } diff --git a/pkg/hook/hook_manager_test.go b/pkg/hook/hook_manager_test.go index f98d7a73..85607050 100644 --- a/pkg/hook/hook_manager_test.go +++ b/pkg/hook/hook_manager_test.go @@ -6,13 +6,13 @@ import ( "testing" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/hook/types" + "github.com/flant/shell-operator/pkg/webhook/admission" + "github.com/flant/shell-operator/pkg/webhook/conversion" . "github.com/onsi/gomega" "github.com/flant/shell-operator/pkg/app" "github.com/flant/shell-operator/pkg/hook/controller" - "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/webhook/admission" - "github.com/flant/shell-operator/pkg/webhook/conversion" ) func newHookManager(t *testing.T, testdataDir string) *Manager { diff --git a/pkg/hook/hook_test.go b/pkg/hook/hook_test.go index b7c28cf8..5aec0982 100644 --- a/pkg/hook/hook_test.go +++ b/pkg/hook/hook_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" - "github.com/deckhouse/deckhouse/pkg/log" . "github.com/onsi/gomega" "golang.org/x/time/rate" + "github.com/deckhouse/deckhouse/pkg/log" "github.com/flant/shell-operator/pkg/hook/config" . "github.com/flant/shell-operator/pkg/hook/types" ) @@ -24,7 +24,7 @@ func Test_Hook_SafeName(t *testing.T) { t.Error(err) } - h := NewHook(hookName, hookPath, log.NewNop()) + h := NewHook(hookName, hookPath, false, false, "", log.NewNop()) g.Expect(h.SafeName()).To(Equal("002-cool-hooks-monitor-namespaces-py")) } @@ -136,7 +136,7 @@ func Test_Hook_WithConfig(t *testing.T) { for _, test := range tests { t.Run(test.name, func(_ *testing.T) { - hook = NewHook("hook-sh", "/hooks/hook.sh", log.NewNop()) + hook = NewHook("hook-sh", "/hooks/hook.sh", false, false, "", log.NewNop()) _, err = hook.LoadConfig([]byte(test.jsonData)) test.fn() }) diff --git a/pkg/hook/task_metadata/task_metadata.go b/pkg/hook/task_metadata/task_metadata.go index 0b0a3b38..78bbd1a3 100644 --- a/pkg/hook/task_metadata/task_metadata.go +++ b/pkg/hook/task_metadata/task_metadata.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/deckhouse/deckhouse/pkg/log" - - "github.com/flant/shell-operator/pkg/hook/binding_context" + bindingcontext "github.com/flant/shell-operator/pkg/hook/binding-context" "github.com/flant/shell-operator/pkg/hook/types" + "github.com/flant/shell-operator/pkg/task" ) @@ -22,7 +22,7 @@ type HookNameAccessor interface { } type BindingContextAccessor interface { - GetBindingContext() []binding_context.BindingContext + GetBindingContext() []bindingcontext.BindingContext } type MonitorIDAccessor interface { @@ -34,7 +34,7 @@ type HookMetadata struct { Binding string // binding name Group string BindingType types.BindingType - BindingContext []binding_context.BindingContext + BindingContext []bindingcontext.BindingContext AllowFailure bool // Task considered as 'ok' if hook failed. False by default. Can be true for some schedule hooks. MonitorIDs []string // monitor ids for Synchronization tasks @@ -66,7 +66,7 @@ func (m HookMetadata) GetHookName() string { return m.HookName } -func (m HookMetadata) GetBindingContext() []binding_context.BindingContext { +func (m HookMetadata) GetBindingContext() []bindingcontext.BindingContext { return m.BindingContext } @@ -88,12 +88,12 @@ func (m *HookMetadata) WithBinding(binding types.BindingType) *HookMetadata { return m } -func (m *HookMetadata) WithBindingContext(context []binding_context.BindingContext) *HookMetadata { +func (m *HookMetadata) WithBindingContext(context []bindingcontext.BindingContext) *HookMetadata { m.BindingContext = context return m } -func (m *HookMetadata) AppendBindingContext(context binding_context.BindingContext) *HookMetadata { +func (m *HookMetadata) AppendBindingContext(context bindingcontext.BindingContext) *HookMetadata { m.BindingContext = append(m.BindingContext, context) return m } diff --git a/pkg/hook/task_metadata/task_metadata_test.go b/pkg/hook/task_metadata/task_metadata_test.go index ce3dd0e2..f0c3beda 100644 --- a/pkg/hook/task_metadata/task_metadata_test.go +++ b/pkg/hook/task_metadata/task_metadata_test.go @@ -5,10 +5,10 @@ import ( "strings" "testing" + . "github.com/flant/shell-operator/pkg/hook/binding-context" + . "github.com/flant/shell-operator/pkg/hook/types" . "github.com/onsi/gomega" - . "github.com/flant/shell-operator/pkg/hook/binding_context" - . "github.com/flant/shell-operator/pkg/hook/types" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" ) diff --git a/pkg/hook/types/bindings.go b/pkg/hook/types/bindings.go index c32b43ab..25e34fae 100644 --- a/pkg/hook/types/bindings.go +++ b/pkg/hook/types/bindings.go @@ -3,8 +3,8 @@ package types import ( "time" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/schedule_manager/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/schedule-manager/types" "github.com/flant/shell-operator/pkg/webhook/admission" "github.com/flant/shell-operator/pkg/webhook/conversion" ) @@ -41,7 +41,7 @@ type ScheduleConfig struct { type OnKubernetesEventConfig struct { CommonBindingConfig - Monitor *kube_events_manager.MonitorConfig + Monitor *kubeeventsmanager.MonitorConfig IncludeSnapshotsFrom []string Queue string Group string diff --git a/pkg/jq/apply_jq_exec.go b/pkg/jq/apply_jq_exec.go deleted file mode 100644 index afeb0b2a..00000000 --- a/pkg/jq/apply_jq_exec.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build !cgo || (cgo && !use_libjq) -// +build !cgo cgo,!use_libjq - -package jq - -// Note: this implementation is enabled by default. - -// ApplyJqFilter runs jq expression provided in jqFilter with jsonData as input. -// -// It uses jq as a subprocess. -func ApplyJqFilter(jqFilter string, jsonData []byte, libPath string) (string, error) { - return jqExec(jqFilter, jsonData, libPath) -} - -func FilterInfo() string { - return "jqFilter implementation: use jq binary from $PATH" -} diff --git a/pkg/kube_events_manager/error_handler.go b/pkg/kube-events-manager/error_handler.go similarity index 90% rename from pkg/kube_events_manager/error_handler.go rename to pkg/kube-events-manager/error_handler.go index 852b789e..72952566 100644 --- a/pkg/kube_events_manager/error_handler.go +++ b/pkg/kube-events-manager/error_handler.go @@ -1,25 +1,25 @@ -package kube_events_manager +package kubeeventsmanager import ( "io" - "github.com/deckhouse/deckhouse/pkg/log" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" - "github.com/flant/shell-operator/pkg/metric_storage" + "github.com/deckhouse/deckhouse/pkg/log" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" utils "github.com/flant/shell-operator/pkg/utils/labels" ) type WatchErrorHandler struct { description string kind string - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage logger *log.Logger } -func newWatchErrorHandler(description string, kind string, logLabels map[string]string, metricStorage *metric_storage.MetricStorage, logger *log.Logger) *WatchErrorHandler { +func newWatchErrorHandler(description string, kind string, logLabels map[string]string, metricStorage *metricstorage.MetricStorage, logger *log.Logger) *WatchErrorHandler { return &WatchErrorHandler{ description: description, kind: kind, diff --git a/pkg/kube_events_manager/factory.go b/pkg/kube-events-manager/factory.go similarity index 99% rename from pkg/kube_events_manager/factory.go rename to pkg/kube-events-manager/factory.go index fc439025..5e22ad2f 100644 --- a/pkg/kube_events_manager/factory.go +++ b/pkg/kube-events-manager/factory.go @@ -1,17 +1,18 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" "sync" "time" - "github.com/deckhouse/deckhouse/pkg/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/tools/cache" + + "github.com/deckhouse/deckhouse/pkg/log" ) var ( diff --git a/pkg/kube_events_manager/filter.go b/pkg/kube-events-manager/filter.go similarity index 79% rename from pkg/kube_events_manager/filter.go rename to pkg/kube-events-manager/filter.go index 2165f2c9..6bbab69c 100644 --- a/pkg/kube_events_manager/filter.go +++ b/pkg/kube-events-manager/filter.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" @@ -10,16 +10,15 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/jq" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + "github.com/flant/shell-operator/pkg/filter" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" ) // applyFilter filters object json representation with jq expression, calculate checksum // over result and return ObjectAndFilterResult. If jqFilter is empty, no filter // is required and checksum is calculated over full json representation of the object. -func applyFilter(jqFilter string, filterFn func(obj *unstructured.Unstructured) (result interface{}, err error), obj *unstructured.Unstructured) (*ObjectAndFilterResult, error) { +func applyFilter(jqFilter string, fl filter.Filter, filterFn func(obj *unstructured.Unstructured) (result interface{}, err error), obj *unstructured.Unstructured) (*ObjectAndFilterResult, error) { defer trace.StartRegion(context.Background(), "ApplyJqFilter").End() res := &ObjectAndFilterResult{ @@ -57,7 +56,7 @@ func applyFilter(jqFilter string, filterFn func(obj *unstructured.Unstructured) } else { var err error var filtered string - filtered, err = jq.ApplyJqFilter(jqFilter, data, app.JqLibraryPath) + filtered, err = fl.ApplyFilter(jqFilter, data) if err != nil { return nil, fmt.Errorf("jqFilter: %v", err) } diff --git a/pkg/kube_events_manager/filter_test.go b/pkg/kube-events-manager/filter_test.go similarity index 72% rename from pkg/kube_events_manager/filter_test.go rename to pkg/kube-events-manager/filter_test.go index a641e4fe..a6c64dc0 100644 --- a/pkg/kube_events_manager/filter_test.go +++ b/pkg/kube-events-manager/filter_test.go @@ -1,9 +1,10 @@ -package kube_events_manager +package kubeeventsmanager import ( "encoding/json" "testing" + "github.com/flant/shell-operator/pkg/filter/jq" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -11,8 +12,9 @@ import ( func TestApplyFilter(t *testing.T) { t.Run("filter func with error", func(t *testing.T) { uns := &unstructured.Unstructured{Object: map[string]interface{}{"foo": "bar"}} - _, err := applyFilter("", filterFuncWithError, uns) - assert.EqualError(t, err, "filterFn (github.com/flant/shell-operator/pkg/kube_events_manager.filterFuncWithError) contains an error: invalid character 'a' looking for beginning of value") + filter := &jq.Filter{} + _, err := applyFilter("", filter, filterFuncWithError, uns) + assert.EqualError(t, err, "filterFn (github.com/flant/shell-operator/pkg/kube-events-manager.filterFuncWithError) contains an error: invalid character 'a' looking for beginning of value") }) } diff --git a/pkg/kube_events_manager/kube_events_manager.go b/pkg/kube-events-manager/kube_events_manager.go similarity index 91% rename from pkg/kube_events_manager/kube_events_manager.go rename to pkg/kube-events-manager/kube_events_manager.go index cc8b583c..7b30344f 100644 --- a/pkg/kube_events_manager/kube_events_manager.go +++ b/pkg/kube-events-manager/kube_events_manager.go @@ -1,19 +1,19 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" "runtime/trace" "sync" - "github.com/deckhouse/deckhouse/pkg/log" - klient "github.com/flant/kube-client/client" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/metric_storage" + + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" ) type KubeEventsManager interface { - WithMetricStorage(mstor *metric_storage.MetricStorage) + WithMetricStorage(mstor *metricstorage.MetricStorage) AddMonitor(monitorConfig *MonitorConfig) error HasMonitor(monitorID string) bool GetMonitor(monitorID string) Monitor @@ -33,7 +33,7 @@ type kubeEventsManager struct { ctx context.Context cancel context.CancelFunc - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage m sync.RWMutex Monitors map[string]Monitor @@ -59,7 +59,7 @@ func NewKubeEventsManager(ctx context.Context, client *klient.Client, logger *lo return em } -func (mgr *kubeEventsManager) WithMetricStorage(mstor *metric_storage.MetricStorage) { +func (mgr *kubeEventsManager) WithMetricStorage(mstor *metricstorage.MetricStorage) { mgr.metricStorage = mstor } diff --git a/pkg/kube_events_manager/kube_events_manager_test.go b/pkg/kube-events-manager/kube_events_manager_test.go similarity index 99% rename from pkg/kube_events_manager/kube_events_manager_test.go rename to pkg/kube-events-manager/kube_events_manager_test.go index ed55b4d5..5e1a83dd 100644 --- a/pkg/kube_events_manager/kube_events_manager_test.go +++ b/pkg/kube-events-manager/kube_events_manager_test.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/deckhouse/deckhouse/pkg/log" + klient "github.com/flant/kube-client/client" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -14,8 +14,8 @@ import ( "k8s.io/apimachinery/pkg/version" fakediscovery "k8s.io/client-go/discovery/fake" - klient "github.com/flant/kube-client/client" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" ) func Test_MainKubeEventsManager_Run(t *testing.T) { diff --git a/pkg/kube_events_manager/monitor.go b/pkg/kube-events-manager/monitor.go similarity index 96% rename from pkg/kube_events_manager/monitor.go rename to pkg/kube-events-manager/monitor.go index 1cbc4b65..8ab8acc2 100644 --- a/pkg/kube_events_manager/monitor.go +++ b/pkg/kube-events-manager/monitor.go @@ -1,15 +1,15 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" "fmt" "sort" - "github.com/deckhouse/deckhouse/pkg/log" - klient "github.com/flant/kube-client/client" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/metric_storage" + + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" utils "github.com/flant/shell-operator/pkg/utils/labels" ) @@ -45,12 +45,12 @@ type monitor struct { ctx context.Context cancel context.CancelFunc - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage logger *log.Logger } -func NewMonitor(ctx context.Context, client *klient.Client, mstor *metric_storage.MetricStorage, config *MonitorConfig, eventCb func(KubeEvent), logger *log.Logger) *monitor { +func NewMonitor(ctx context.Context, client *klient.Client, mstor *metricstorage.MetricStorage, config *MonitorConfig, eventCb func(KubeEvent), logger *log.Logger) *monitor { cctx, cancel := context.WithCancel(ctx) return &monitor{ diff --git a/pkg/kube_events_manager/monitor_config.go b/pkg/kube-events-manager/monitor_config.go similarity index 97% rename from pkg/kube_events_manager/monitor_config.go rename to pkg/kube-events-manager/monitor_config.go index 08e87566..06bfecf7 100644 --- a/pkg/kube_events_manager/monitor_config.go +++ b/pkg/kube-events-manager/monitor_config.go @@ -1,11 +1,11 @@ -package kube_events_manager +package kubeeventsmanager import ( - "github.com/deckhouse/deckhouse/pkg/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" ) // MonitorConfig is a config that suits the latest diff --git a/pkg/kube_events_manager/monitor_test.go b/pkg/kube-events-manager/monitor_test.go similarity index 98% rename from pkg/kube_events_manager/monitor_test.go rename to pkg/kube-events-manager/monitor_test.go index b71d9a8b..89148aa1 100644 --- a/pkg/kube_events_manager/monitor_test.go +++ b/pkg/kube-events-manager/monitor_test.go @@ -1,18 +1,18 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" "fmt" "testing" - "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/kube-client/fake" + "github.com/flant/kube-client/manifest" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/flant/kube-client/fake" - "github.com/flant/kube-client/manifest" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" ) func Test_Monitor_should_handle_dynamic_ns_events(t *testing.T) { diff --git a/pkg/kube_events_manager/namespace_informer.go b/pkg/kube-events-manager/namespace_informer.go similarity index 99% rename from pkg/kube_events_manager/namespace_informer.go rename to pkg/kube-events-manager/namespace_informer.go index 726cee94..828bfb98 100644 --- a/pkg/kube_events_manager/namespace_informer.go +++ b/pkg/kube-events-manager/namespace_informer.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager // Namespace manager monitor namespaces for onKubernetesEvent config. @@ -6,14 +6,14 @@ import ( "context" "fmt" - "github.com/deckhouse/deckhouse/pkg/log" + klient "github.com/flant/kube-client/client" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" corev1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/cache" - klient "github.com/flant/kube-client/client" + "github.com/deckhouse/deckhouse/pkg/log" ) type namespaceInformer struct { diff --git a/pkg/kube_events_manager/resource_informer.go b/pkg/kube-events-manager/resource_informer.go similarity index 95% rename from pkg/kube_events_manager/resource_informer.go rename to pkg/kube-events-manager/resource_informer.go index 3f7d25c0..b19e03b9 100644 --- a/pkg/kube_events_manager/resource_informer.go +++ b/pkg/kube-events-manager/resource_informer.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager import ( "context" @@ -7,16 +7,18 @@ import ( "sync" "time" - "github.com/deckhouse/deckhouse/pkg/log" + klient "github.com/flant/kube-client/client" "github.com/gofrs/uuid/v5" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" - klient "github.com/flant/kube-client/client" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/metric_storage" + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/app" + "github.com/flant/shell-operator/pkg/filter/jq" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" "github.com/flant/shell-operator/pkg/utils/measure" ) @@ -56,7 +58,7 @@ type resourceInformer struct { ctx context.Context cancel context.CancelFunc - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage // a flag to stop handle events after Stop() stopped bool @@ -67,7 +69,7 @@ type resourceInformer struct { // resourceInformer should implement ResourceInformer type resourceInformerConfig struct { client *klient.Client - mstor *metric_storage.MetricStorage + mstor *metricstorage.MetricStorage eventCb func(KubeEvent) monitor *MonitorConfig @@ -213,7 +215,8 @@ func (ei *resourceInformer) loadExistedObjects() error { defer measure.Duration(func(d time.Duration) { ei.metricStorage.HistogramObserve("{PREFIX}kube_jq_filter_duration_seconds", d.Seconds(), ei.Monitor.Metadata.MetricLabels, nil) })() - objFilterRes, err = applyFilter(ei.Monitor.JqFilter, ei.Monitor.FilterFunc, &obj) + filter := &jq.Filter{Libpath: app.JqLibraryPath} + objFilterRes, err = applyFilter(ei.Monitor.JqFilter, filter, ei.Monitor.FilterFunc, &obj) }() if err != nil { @@ -292,7 +295,8 @@ func (ei *resourceInformer) handleWatchEvent(object interface{}, eventType Watch defer measure.Duration(func(d time.Duration) { ei.metricStorage.HistogramObserve("{PREFIX}kube_jq_filter_duration_seconds", d.Seconds(), ei.Monitor.Metadata.MetricLabels, nil) })() - objFilterRes, err = applyFilter(ei.Monitor.JqFilter, ei.Monitor.FilterFunc, obj) + filter := &jq.Filter{Libpath: app.JqLibraryPath} + objFilterRes, err = applyFilter(ei.Monitor.JqFilter, filter, ei.Monitor.FilterFunc, obj) }() if err != nil { log.Errorf("%s: WATCH %s: %s", diff --git a/pkg/kube_events_manager/types/types.go b/pkg/kube-events-manager/types/types.go similarity index 99% rename from pkg/kube_events_manager/types/types.go rename to pkg/kube-events-manager/types/types.go index dabb7c16..1c12a5e9 100644 --- a/pkg/kube_events_manager/types/types.go +++ b/pkg/kube-events-manager/types/types.go @@ -5,9 +5,10 @@ import ( "fmt" "strings" - "github.com/deckhouse/deckhouse/pkg/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/deckhouse/deckhouse/pkg/log" ) type WatchEventType string diff --git a/pkg/kube_events_manager/types/types_test.go b/pkg/kube-events-manager/types/types_test.go similarity index 100% rename from pkg/kube_events_manager/types/types_test.go rename to pkg/kube-events-manager/types/types_test.go diff --git a/pkg/kube_events_manager/util.go b/pkg/kube-events-manager/util.go similarity index 96% rename from pkg/kube_events_manager/util.go rename to pkg/kube-events-manager/util.go index b4459d2e..7a2a4958 100644 --- a/pkg/kube_events_manager/util.go +++ b/pkg/kube-events-manager/util.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager import ( "fmt" @@ -9,7 +9,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" ) // ResourceId describes object with namespace, kind and name diff --git a/pkg/kube_events_manager/util_test.go b/pkg/kube-events-manager/util_test.go similarity index 87% rename from pkg/kube_events_manager/util_test.go rename to pkg/kube-events-manager/util_test.go index 0356126e..ecb46e6a 100644 --- a/pkg/kube_events_manager/util_test.go +++ b/pkg/kube-events-manager/util_test.go @@ -1,4 +1,4 @@ -package kube_events_manager +package kubeeventsmanager import ( "fmt" diff --git a/pkg/metric_storage/metric_storage.go b/pkg/metric-storage/metric_storage.go similarity index 98% rename from pkg/metric_storage/metric_storage.go rename to pkg/metric-storage/metric_storage.go index 3c74fae2..d5a9ea87 100644 --- a/pkg/metric_storage/metric_storage.go +++ b/pkg/metric-storage/metric_storage.go @@ -1,4 +1,4 @@ -package metric_storage +package metricstorage import ( "context" @@ -7,13 +7,13 @@ import ( "strings" "sync" - "github.com/deckhouse/deckhouse/pkg/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/deckhouse/deckhouse/pkg/log" "github.com/flant/shell-operator/pkg/metric" - "github.com/flant/shell-operator/pkg/metric_storage/operation" - "github.com/flant/shell-operator/pkg/metric_storage/vault" + "github.com/flant/shell-operator/pkg/metric-storage/operation" + "github.com/flant/shell-operator/pkg/metric-storage/vault" . "github.com/flant/shell-operator/pkg/utils/labels" ) diff --git a/pkg/metric_storage/operation/operation.go b/pkg/metric-storage/operation/operation.go similarity index 100% rename from pkg/metric_storage/operation/operation.go rename to pkg/metric-storage/operation/operation.go diff --git a/pkg/metric_storage/operation/operation_test.go b/pkg/metric-storage/operation/operation_test.go similarity index 100% rename from pkg/metric_storage/operation/operation_test.go rename to pkg/metric-storage/operation/operation_test.go diff --git a/pkg/metric_storage/vault/vault.go b/pkg/metric-storage/vault/vault.go similarity index 100% rename from pkg/metric_storage/vault/vault.go rename to pkg/metric-storage/vault/vault.go index 118107fe..16c44d9f 100644 --- a/pkg/metric_storage/vault/vault.go +++ b/pkg/metric-storage/vault/vault.go @@ -4,9 +4,9 @@ import ( "fmt" "sync" - "github.com/deckhouse/deckhouse/pkg/log" "github.com/prometheus/client_golang/prometheus" + "github.com/deckhouse/deckhouse/pkg/log" "github.com/flant/shell-operator/pkg/metric" . "github.com/flant/shell-operator/pkg/utils/labels" ) diff --git a/pkg/metric_storage/vault/vault_test.go b/pkg/metric-storage/vault/vault_test.go similarity index 99% rename from pkg/metric_storage/vault/vault_test.go rename to pkg/metric-storage/vault/vault_test.go index bc4fd2f1..cd1720e1 100644 --- a/pkg/metric_storage/vault/vault_test.go +++ b/pkg/metric-storage/vault/vault_test.go @@ -5,10 +5,11 @@ import ( "strings" "testing" - "github.com/deckhouse/deckhouse/pkg/log" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/deckhouse/deckhouse/pkg/log" ) func Test_CounterAdd(t *testing.T) { diff --git a/pkg/metric/storage.go b/pkg/metric/storage.go index 3bcf207b..55f7bee2 100644 --- a/pkg/metric/storage.go +++ b/pkg/metric/storage.go @@ -1,9 +1,8 @@ package metric import ( + "github.com/flant/shell-operator/pkg/metric-storage/operation" "github.com/prometheus/client_golang/prometheus" - - "github.com/flant/shell-operator/pkg/metric_storage/operation" ) type Storage interface { diff --git a/pkg/metric/storage_mock.go b/pkg/metric/storage_mock.go index 787e815b..11828525 100644 --- a/pkg/metric/storage_mock.go +++ b/pkg/metric/storage_mock.go @@ -9,7 +9,7 @@ import ( mm_atomic "sync/atomic" mm_time "time" - "github.com/flant/shell-operator/pkg/metric_storage/operation" + "github.com/flant/shell-operator/pkg/metric-storage/operation" "github.com/gojuno/minimock/v3" "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/metric/storage_test.go b/pkg/metric/storage_test.go index b7f907d8..265abd3b 100644 --- a/pkg/metric/storage_test.go +++ b/pkg/metric/storage_test.go @@ -2,12 +2,12 @@ package metric_test import ( "github.com/flant/shell-operator/pkg/metric" - "github.com/flant/shell-operator/pkg/metric_storage" - "github.com/flant/shell-operator/pkg/metric_storage/vault" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" + "github.com/flant/shell-operator/pkg/metric-storage/vault" ) var ( - _ metric.Storage = (*metric_storage.MetricStorage)(nil) + _ metric.Storage = (*metricstorage.MetricStorage)(nil) _ metric.Storage = (*metric.StorageMock)(nil) _ metric.GroupedStorage = (*vault.GroupedVault)(nil) diff --git a/pkg/kube/object_patch/helpers.go b/pkg/object-patch/helpers.go similarity index 91% rename from pkg/kube/object_patch/helpers.go rename to pkg/object-patch/helpers.go index 507c8374..c4821601 100644 --- a/pkg/kube/object_patch/helpers.go +++ b/pkg/object-patch/helpers.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( "bytes" @@ -6,14 +6,13 @@ import ( "fmt" "io" + "github.com/flant/kube-client/manifest" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" k8yaml "sigs.k8s.io/yaml" - "github.com/flant/kube-client/manifest" - "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/jq" + "github.com/flant/shell-operator/pkg/filter" ) func unmarshalFromJSONOrYAML(specs []byte) ([]OperationSpec, error) { @@ -65,13 +64,13 @@ func unmarshalFromYaml(yamlSpecs []byte) ([]OperationSpec, error) { return specSlice, nil } -func applyJQPatch(jqFilter string, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { +func applyJQPatch(jqFilter string, fl filter.Filter, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { objBytes, err := obj.MarshalJSON() if err != nil { return nil, err } - filterResult, err := jq.ApplyJqFilter(jqFilter, objBytes, app.JqLibraryPath) + filterResult, err := fl.ApplyFilter(jqFilter, objBytes) if err != nil { return nil, fmt.Errorf("failed to apply jqFilter:\n%sto Object:\n%s\n"+ "error: %s", jqFilter, obj, err) diff --git a/pkg/kube/object_patch/operation.go b/pkg/object-patch/operation.go similarity index 97% rename from pkg/kube/object_patch/operation.go rename to pkg/object-patch/operation.go index e58ea968..54514a95 100644 --- a/pkg/kube/object_patch/operation.go +++ b/pkg/object-patch/operation.go @@ -1,13 +1,16 @@ -package object_patch +package objectpatch import ( "fmt" - "github.com/deckhouse/deckhouse/pkg/log" "github.com/hashicorp/go-multierror" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/app" + "github.com/flant/shell-operator/pkg/filter/jq" ) // A JSON and YAML representation of the operation for shell hooks @@ -192,7 +195,8 @@ func NewFromOperationSpec(spec OperationSpec) Operation { case JQPatch: return NewFilterPatchOperation( func(u *unstructured.Unstructured) (*unstructured.Unstructured, error) { - return applyJQPatch(spec.JQFilter, u) + filter := &jq.Filter{Libpath: app.JqLibraryPath} + return applyJQPatch(spec.JQFilter, filter, u) }, spec.ApiVersion, spec.Kind, spec.Namespace, spec.Name, WithSubresource(spec.Subresource), diff --git a/pkg/kube/object_patch/options.go b/pkg/object-patch/options.go similarity index 99% rename from pkg/kube/object_patch/options.go rename to pkg/object-patch/options.go index c6900ee9..1980a4fe 100644 --- a/pkg/kube/object_patch/options.go +++ b/pkg/object-patch/options.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/kube/object_patch/patch.go b/pkg/object-patch/patch.go similarity index 99% rename from pkg/kube/object_patch/patch.go rename to pkg/object-patch/patch.go index f46e22b7..4f3ece7a 100644 --- a/pkg/kube/object_patch/patch.go +++ b/pkg/object-patch/patch.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( "bytes" @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/deckhouse/deckhouse/pkg/log" "github.com/hashicorp/go-multierror" gerror "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/equality" @@ -19,6 +18,8 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" + + "github.com/deckhouse/deckhouse/pkg/log" ) type ObjectPatcher struct { diff --git a/pkg/kube/object_patch/patch_collector.go b/pkg/object-patch/patch_collector.go similarity index 84% rename from pkg/kube/object_patch/patch_collector.go rename to pkg/object-patch/patch_collector.go index 0d2c365e..faccb287 100644 --- a/pkg/kube/object_patch/patch_collector.go +++ b/pkg/object-patch/patch_collector.go @@ -1,9 +1,18 @@ -package object_patch +package objectpatch import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +type IPatchCollector interface { + Create(object interface{}, options ...CreateOption) + Delete(apiVersion string, kind string, namespace string, name string, options ...DeleteOption) + Filter(filterFunc func(*unstructured.Unstructured) (*unstructured.Unstructured, error), apiVersion string, kind string, namespace string, name string, options ...FilterOption) + JSONPatch(jsonPatch interface{}, apiVersion string, kind string, namespace string, name string, options ...PatchOption) + MergePatch(mergePatch interface{}, apiVersion string, kind string, namespace string, name string, options ...PatchOption) + Operations() []Operation +} + type PatchCollector struct { patchOperations []Operation } diff --git a/pkg/kube/object_patch/patch_test.go b/pkg/object-patch/patch_test.go similarity index 99% rename from pkg/kube/object_patch/patch_test.go rename to pkg/object-patch/patch_test.go index bc0a29ce..65268b32 100644 --- a/pkg/kube/object_patch/patch_test.go +++ b/pkg/object-patch/patch_test.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( "context" @@ -6,7 +6,8 @@ import ( "os" "testing" - "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/kube-client/fake" + "github.com/flant/kube-client/manifest" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -14,8 +15,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "github.com/flant/kube-client/fake" - "github.com/flant/kube-client/manifest" + "github.com/deckhouse/deckhouse/pkg/log" ) func mustReadFile(t *testing.T, filePath string) []byte { diff --git a/pkg/kube/object_patch/testdata/serialized_operations/invalid_create.yaml b/pkg/object-patch/testdata/serialized_operations/invalid_create.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/invalid_create.yaml rename to pkg/object-patch/testdata/serialized_operations/invalid_create.yaml diff --git a/pkg/kube/object_patch/testdata/serialized_operations/invalid_delete.yaml b/pkg/object-patch/testdata/serialized_operations/invalid_delete.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/invalid_delete.yaml rename to pkg/object-patch/testdata/serialized_operations/invalid_delete.yaml diff --git a/pkg/kube/object_patch/testdata/serialized_operations/invalid_patch.yaml b/pkg/object-patch/testdata/serialized_operations/invalid_patch.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/invalid_patch.yaml rename to pkg/object-patch/testdata/serialized_operations/invalid_patch.yaml diff --git a/pkg/kube/object_patch/testdata/serialized_operations/valid_create.yaml b/pkg/object-patch/testdata/serialized_operations/valid_create.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/valid_create.yaml rename to pkg/object-patch/testdata/serialized_operations/valid_create.yaml diff --git a/pkg/kube/object_patch/testdata/serialized_operations/valid_delete.yaml b/pkg/object-patch/testdata/serialized_operations/valid_delete.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/valid_delete.yaml rename to pkg/object-patch/testdata/serialized_operations/valid_delete.yaml diff --git a/pkg/kube/object_patch/testdata/serialized_operations/valid_patch.yaml b/pkg/object-patch/testdata/serialized_operations/valid_patch.yaml similarity index 100% rename from pkg/kube/object_patch/testdata/serialized_operations/valid_patch.yaml rename to pkg/object-patch/testdata/serialized_operations/valid_patch.yaml diff --git a/pkg/kube/object_patch/validation.go b/pkg/object-patch/validation.go similarity index 99% rename from pkg/kube/object_patch/validation.go rename to pkg/object-patch/validation.go index e8055a67..a651b89f 100644 --- a/pkg/kube/object_patch/validation.go +++ b/pkg/object-patch/validation.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( "encoding/json" diff --git a/pkg/kube/object_patch/validation_test.go b/pkg/object-patch/validation_test.go similarity index 96% rename from pkg/kube/object_patch/validation_test.go rename to pkg/object-patch/validation_test.go index 73d28580..5240744d 100644 --- a/pkg/kube/object_patch/validation_test.go +++ b/pkg/object-patch/validation_test.go @@ -1,4 +1,4 @@ -package object_patch +package objectpatch import ( "testing" diff --git a/pkg/schedule_manager/schedule_manager.go b/pkg/schedule-manager/schedule_manager.go similarity index 96% rename from pkg/schedule_manager/schedule_manager.go rename to pkg/schedule-manager/schedule_manager.go index 6681b5bd..50ee4b2d 100644 --- a/pkg/schedule_manager/schedule_manager.go +++ b/pkg/schedule-manager/schedule_manager.go @@ -1,12 +1,12 @@ -package schedule_manager +package schedulemanager import ( "context" - "github.com/deckhouse/deckhouse/pkg/log" "gopkg.in/robfig/cron.v2" - . "github.com/flant/shell-operator/pkg/schedule_manager/types" + "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/schedule-manager/types" ) type ScheduleManager interface { diff --git a/pkg/schedule_manager/schedule_manager_test.go b/pkg/schedule-manager/schedule_manager_test.go similarity index 97% rename from pkg/schedule_manager/schedule_manager_test.go rename to pkg/schedule-manager/schedule_manager_test.go index 21efbc2b..521bf18c 100644 --- a/pkg/schedule_manager/schedule_manager_test.go +++ b/pkg/schedule-manager/schedule_manager_test.go @@ -1,12 +1,11 @@ -package schedule_manager +package schedulemanager import ( "context" "testing" "github.com/deckhouse/deckhouse/pkg/log" - - "github.com/flant/shell-operator/pkg/schedule_manager/types" + "github.com/flant/shell-operator/pkg/schedule-manager/types" ) func Test_ScheduleManager_Add(t *testing.T) { diff --git a/pkg/schedule_manager/types/types.go b/pkg/schedule-manager/types/types.go similarity index 100% rename from pkg/schedule_manager/types/types.go rename to pkg/schedule-manager/types/types.go diff --git a/pkg/shell-operator/bootstrap.go b/pkg/shell-operator/bootstrap.go index 760726a4..439675fe 100644 --- a/pkg/shell-operator/bootstrap.go +++ b/pkg/shell-operator/bootstrap.go @@ -6,14 +6,13 @@ import ( "log/slog" "github.com/deckhouse/deckhouse/pkg/log" - "github.com/flant/shell-operator/pkg/app" "github.com/flant/shell-operator/pkg/config" "github.com/flant/shell-operator/pkg/debug" + "github.com/flant/shell-operator/pkg/filter/jq" "github.com/flant/shell-operator/pkg/hook" - "github.com/flant/shell-operator/pkg/jq" - "github.com/flant/shell-operator/pkg/kube_events_manager" - "github.com/flant/shell-operator/pkg/schedule_manager" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" "github.com/flant/shell-operator/pkg/task/queue" utils "github.com/flant/shell-operator/pkg/utils/file" "github.com/flant/shell-operator/pkg/webhook/admission" @@ -29,7 +28,8 @@ func Init(logger *log.Logger) (*ShellOperator, error) { // Log version and jq filtering implementation. logger.Info(app.AppStartMessage) - logger.Debug(jq.FilterInfo()) + fl := jq.NewFilter(app.JqLibraryPath) + logger.Debug(fl.FilterInfo()) hooksDir, err := utils.RequireExistingDirectory(app.HooksDir) if err != nil { @@ -154,10 +154,10 @@ func (op *ShellOperator) SetupEventManagers() { op.TaskQueues.WithMetricStorage(op.MetricStorage) // Initialize schedule manager. - op.ScheduleManager = schedule_manager.NewScheduleManager(op.ctx, op.logger.Named("schedule-manager")) + op.ScheduleManager = schedulemanager.NewScheduleManager(op.ctx, op.logger.Named("schedule-manager")) // Initialize kubernetes events manager. - op.KubeEventsManager = kube_events_manager.NewKubeEventsManager(op.ctx, op.KubeClient, op.logger.Named("kube-events-manager")) + op.KubeEventsManager = kubeeventsmanager.NewKubeEventsManager(op.ctx, op.KubeClient, op.logger.Named("kube-events-manager")) op.KubeEventsManager.WithMetricStorage(op.MetricStorage) // Initialize events handler that emit tasks to run hooks diff --git a/pkg/shell-operator/combine_binding_context.go b/pkg/shell-operator/combine_binding_context.go index 7638f4db..61c8a571 100644 --- a/pkg/shell-operator/combine_binding_context.go +++ b/pkg/shell-operator/combine_binding_context.go @@ -3,7 +3,8 @@ package shell_operator import ( "fmt" - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" + . "github.com/flant/shell-operator/pkg/hook/task_metadata" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" diff --git a/pkg/shell-operator/combine_binding_context_test.go b/pkg/shell-operator/combine_binding_context_test.go index 6caef60b..e62b2c93 100644 --- a/pkg/shell-operator/combine_binding_context_test.go +++ b/pkg/shell-operator/combine_binding_context_test.go @@ -5,12 +5,12 @@ import ( "testing" "github.com/deckhouse/deckhouse/pkg/log" + bindingcontext "github.com/flant/shell-operator/pkg/hook/binding-context" + "github.com/flant/shell-operator/pkg/hook/types" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" . "github.com/onsi/gomega" - "github.com/flant/shell-operator/pkg/hook/binding_context" . "github.com/flant/shell-operator/pkg/hook/task_metadata" - "github.com/flant/shell-operator/pkg/hook/types" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" ) @@ -31,7 +31,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -42,7 +42,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -53,7 +53,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "schedule", }, @@ -63,7 +63,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -74,7 +74,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook2.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -85,7 +85,7 @@ func Test_CombineBindingContext_MultipleHooks(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -125,7 +125,7 @@ func Test_CombineBindingContext_Nil_On_NoCombine(t *testing.T) { WithQueueName("test_no_combine"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -136,7 +136,7 @@ func Test_CombineBindingContext_Nil_On_NoCombine(t *testing.T) { WithQueueName("test_no_combine"). WithMetadata(HookMetadata{ HookName: "hook2.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -147,7 +147,7 @@ func Test_CombineBindingContext_Nil_On_NoCombine(t *testing.T) { WithQueueName("test_no_combine"). WithMetadata(HookMetadata{ HookName: "hook3.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "schedule", }, @@ -179,7 +179,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { } }) - bcMeta := binding_context.BindingContext{}.Metadata + bcMeta := bindingcontext.BindingContext{}.Metadata bcMeta.Group = "pods" tasks := []task.Task{ @@ -188,7 +188,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "kubernetes", @@ -200,7 +200,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "kubernetes", @@ -212,7 +212,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "schedule", @@ -223,7 +223,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -235,7 +235,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook2.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -246,7 +246,7 @@ func Test_CombineBindingContext_Group_Compaction(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -281,11 +281,11 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { } }) - bcMeta := binding_context.BindingContext{}.Metadata + bcMeta := bindingcontext.BindingContext{}.Metadata bcMeta.Group = "pods" bcMeta.BindingType = types.OnKubernetesEvent - schMeta := binding_context.BindingContext{}.Metadata + schMeta := bindingcontext.BindingContext{}.Metadata schMeta.Group = "pods" schMeta.BindingType = types.Schedule @@ -294,7 +294,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "kubernetes", @@ -306,7 +306,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "kubernetes2", @@ -318,7 +318,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "schedule", @@ -332,7 +332,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -344,7 +344,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: schMeta, Binding: "schedule", @@ -356,10 +356,10 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { - Metadata: func() binding_context.BindingContext { - bc := binding_context.BindingContext{} + Metadata: func() bindingcontext.BindingContext { + bc := bindingcontext.BindingContext{} bc.Metadata.BindingType = types.Schedule return bc }().Metadata, @@ -372,7 +372,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Metadata: bcMeta, Binding: "kubernetes", @@ -385,7 +385,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, @@ -396,7 +396,7 @@ func Test_CombineBindingContext_Group_Type(t *testing.T) { WithQueueName("test_multiple_hooks"). WithMetadata(HookMetadata{ HookName: "hook1.sh", - BindingContext: []binding_context.BindingContext{ + BindingContext: []bindingcontext.BindingContext{ { Binding: "kubernetes", Type: TypeEvent, diff --git a/pkg/shell-operator/kube_client.go b/pkg/shell-operator/kube_client.go index 733341ac..3f96ae35 100644 --- a/pkg/shell-operator/kube_client.go +++ b/pkg/shell-operator/kube_client.go @@ -4,11 +4,11 @@ import ( "fmt" "github.com/deckhouse/deckhouse/pkg/log" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" + objectpatch "github.com/flant/shell-operator/pkg/object-patch" klient "github.com/flant/kube-client/client" "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/kube/object_patch" - "github.com/flant/shell-operator/pkg/metric_storage" utils "github.com/flant/shell-operator/pkg/utils/labels" ) @@ -19,7 +19,7 @@ var ( // defaultMainKubeClient creates a Kubernetes client for hooks. No timeout specified, because // timeout will reset connections for Watchers. -func defaultMainKubeClient(metricStorage *metric_storage.MetricStorage, metricLabels map[string]string, logger *log.Logger) *klient.Client { +func defaultMainKubeClient(metricStorage *metricstorage.MetricStorage, metricLabels map[string]string, logger *log.Logger) *klient.Client { client := klient.New(klient.WithLogger(logger)) client.WithContextName(app.KubeContext) client.WithConfigPath(app.KubeConfig) @@ -29,7 +29,7 @@ func defaultMainKubeClient(metricStorage *metric_storage.MetricStorage, metricLa return client } -func initDefaultMainKubeClient(metricStorage *metric_storage.MetricStorage, logger *log.Logger) (*klient.Client, error) { +func initDefaultMainKubeClient(metricStorage *metricstorage.MetricStorage, logger *log.Logger) (*klient.Client, error) { //nolint:staticcheck klient.RegisterKubernetesClientMetrics(metricStorage, defaultMainKubeClientMetricLabels) kubeClient := defaultMainKubeClient(metricStorage, defaultMainKubeClientMetricLabels, logger.Named("main-kube-client")) @@ -41,7 +41,7 @@ func initDefaultMainKubeClient(metricStorage *metric_storage.MetricStorage, logg } // defaultObjectPatcherKubeClient initializes a Kubernetes client for ObjectPatcher. Timeout is specified here. -func defaultObjectPatcherKubeClient(metricStorage *metric_storage.MetricStorage, metricLabels map[string]string, logger *log.Logger) *klient.Client { +func defaultObjectPatcherKubeClient(metricStorage *metricstorage.MetricStorage, metricLabels map[string]string, logger *log.Logger) *klient.Client { client := klient.New(klient.WithLogger(logger)) client.WithContextName(app.KubeContext) client.WithConfigPath(app.KubeConfig) @@ -52,11 +52,11 @@ func defaultObjectPatcherKubeClient(metricStorage *metric_storage.MetricStorage, return client } -func initDefaultObjectPatcher(metricStorage *metric_storage.MetricStorage, logger *log.Logger) (*object_patch.ObjectPatcher, error) { +func initDefaultObjectPatcher(metricStorage *metricstorage.MetricStorage, logger *log.Logger) (*objectpatch.ObjectPatcher, error) { patcherKubeClient := defaultObjectPatcherKubeClient(metricStorage, defaultObjectPatcherKubeClientMetricLabels, logger.Named("object-patcher-kube-client")) err := patcherKubeClient.Init() if err != nil { return nil, fmt.Errorf("initialize Kubernetes client for Object patcher: %s\n", err) } - return object_patch.NewObjectPatcher(patcherKubeClient, logger), nil + return objectpatch.NewObjectPatcher(patcherKubeClient, logger), nil } diff --git a/pkg/shell-operator/manager_events_handler.go b/pkg/shell-operator/manager_events_handler.go index 643c8de0..1d05f890 100644 --- a/pkg/shell-operator/manager_events_handler.go +++ b/pkg/shell-operator/manager_events_handler.go @@ -4,18 +4,18 @@ import ( "context" "github.com/deckhouse/deckhouse/pkg/log" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/schedule_manager" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" ) type managerEventsHandlerConfig struct { tqs *queue.TaskQueueSet - mgr kube_events_manager.KubeEventsManager - smgr schedule_manager.ScheduleManager + mgr kubeeventsmanager.KubeEventsManager + smgr schedulemanager.ScheduleManager logger *log.Logger } @@ -24,8 +24,8 @@ type ManagerEventsHandler struct { ctx context.Context cancel context.CancelFunc - kubeEventsManager kube_events_manager.KubeEventsManager - scheduleManager schedule_manager.ScheduleManager + kubeEventsManager kubeeventsmanager.KubeEventsManager + scheduleManager schedulemanager.ScheduleManager kubeEventCb func(kubeEvent KubeEvent) []task.Task scheduleCb func(crontab string) []task.Task diff --git a/pkg/shell-operator/metrics_hooks.go b/pkg/shell-operator/metrics_hooks.go index 24e2bfa5..e134cbcf 100644 --- a/pkg/shell-operator/metrics_hooks.go +++ b/pkg/shell-operator/metrics_hooks.go @@ -4,11 +4,11 @@ import ( "net/http" "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/metric_storage" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" ) func (op *ShellOperator) setupHookMetricStorage() { - metricStorage := metric_storage.NewMetricStorage(op.ctx, app.PrometheusMetricsPrefix, true, op.logger.Named("metric-storage")) + metricStorage := metricstorage.NewMetricStorage(op.ctx, app.PrometheusMetricsPrefix, true, op.logger.Named("metric-storage")) op.APIServer.RegisterRoute(http.MethodGet, "/metrics/hooks", metricStorage.Handler().ServeHTTP) // create new metric storage for hooks @@ -17,7 +17,7 @@ func (op *ShellOperator) setupHookMetricStorage() { } // specific metrics for shell-operator HookManager -func registerHookMetrics(metricStorage *metric_storage.MetricStorage) { +func registerHookMetrics(metricStorage *metricstorage.MetricStorage) { // Metrics for enable kubernetes bindings. metricStorage.RegisterGauge("{PREFIX}hook_enable_kubernetes_bindings_seconds", map[string]string{"hook": ""}) metricStorage.RegisterCounter("{PREFIX}hook_enable_kubernetes_bindings_errors_total", map[string]string{"hook": ""}) diff --git a/pkg/shell-operator/metrics_operator.go b/pkg/shell-operator/metrics_operator.go index 021335a9..65d91ee0 100644 --- a/pkg/shell-operator/metrics_operator.go +++ b/pkg/shell-operator/metrics_operator.go @@ -4,12 +4,12 @@ import ( "net/http" "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/metric_storage" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" ) // setupMetricStorage creates and initializes metrics storage for built-in operator metrics func (op *ShellOperator) setupMetricStorage(kubeEventsManagerLabels map[string]string) { - metricStorage := metric_storage.NewMetricStorage(op.ctx, app.PrometheusMetricsPrefix, false, op.logger.Named("metric-storage")) + metricStorage := metricstorage.NewMetricStorage(op.ctx, app.PrometheusMetricsPrefix, false, op.logger.Named("metric-storage")) registerCommonMetrics(metricStorage) registerTaskQueueMetrics(metricStorage) @@ -23,13 +23,13 @@ func (op *ShellOperator) setupMetricStorage(kubeEventsManagerLabels map[string]s // registerCommonMetrics register base metric // This function is used in the addon-operator -func registerCommonMetrics(metricStorage *metric_storage.MetricStorage) { +func registerCommonMetrics(metricStorage *metricstorage.MetricStorage) { metricStorage.RegisterCounter("{PREFIX}live_ticks", map[string]string{}) } // registerTaskQueueMetrics // This function is used in the addon-operator -func registerTaskQueueMetrics(metricStorage *metric_storage.MetricStorage) { +func registerTaskQueueMetrics(metricStorage *metricstorage.MetricStorage) { metricStorage.RegisterHistogram( "{PREFIX}tasks_queue_action_duration_seconds", map[string]string{ @@ -50,7 +50,7 @@ func registerTaskQueueMetrics(metricStorage *metric_storage.MetricStorage) { // registerKubeEventsManagerMetrics registers metrics for kube_event_manager // This function is used in the addon-operator -func registerKubeEventsManagerMetrics(metricStorage *metric_storage.MetricStorage, labels map[string]string) { +func registerKubeEventsManagerMetrics(metricStorage *metricstorage.MetricStorage, labels map[string]string) { // Count of objects in snapshot for one kubernets bindings. metricStorage.RegisterGauge("{PREFIX}kube_snapshot_objects", labels) // Duration of jqFilter applying. diff --git a/pkg/shell-operator/operator.go b/pkg/shell-operator/operator.go index 73306142..388dfe14 100644 --- a/pkg/shell-operator/operator.go +++ b/pkg/shell-operator/operator.go @@ -6,26 +6,26 @@ import ( "time" "github.com/deckhouse/deckhouse/pkg/log" + bindingcontext "github.com/flant/shell-operator/pkg/hook/binding-context" + "github.com/flant/shell-operator/pkg/hook/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + kemTypes "github.com/flant/shell-operator/pkg/kube-events-manager/types" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" + objectpatch "github.com/flant/shell-operator/pkg/object-patch" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" + "github.com/flant/shell-operator/pkg/webhook/admission" + "github.com/flant/shell-operator/pkg/webhook/conversion" "github.com/gofrs/uuid/v5" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" klient "github.com/flant/kube-client/client" "github.com/flant/shell-operator/pkg/hook" - "github.com/flant/shell-operator/pkg/hook/binding_context" "github.com/flant/shell-operator/pkg/hook/controller" "github.com/flant/shell-operator/pkg/hook/task_metadata" - "github.com/flant/shell-operator/pkg/hook/types" - "github.com/flant/shell-operator/pkg/kube/object_patch" - "github.com/flant/shell-operator/pkg/kube_events_manager" - kemTypes "github.com/flant/shell-operator/pkg/kube_events_manager/types" - "github.com/flant/shell-operator/pkg/metric_storage" - "github.com/flant/shell-operator/pkg/schedule_manager" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" utils "github.com/flant/shell-operator/pkg/utils/labels" "github.com/flant/shell-operator/pkg/utils/measure" - "github.com/flant/shell-operator/pkg/webhook/admission" - "github.com/flant/shell-operator/pkg/webhook/conversion" ) var WaitQueuesTimeout = time.Second * 10 @@ -40,14 +40,14 @@ type ShellOperator struct { APIServer *baseHTTPServer // MetricStorage collects and store metrics for built-in operator primitives, hook execution - MetricStorage *metric_storage.MetricStorage + MetricStorage *metricstorage.MetricStorage // HookMetricStorage separate metric storage for metrics, which are returned by user hooks - HookMetricStorage *metric_storage.MetricStorage + HookMetricStorage *metricstorage.MetricStorage KubeClient *klient.Client - ObjectPatcher *object_patch.ObjectPatcher + ObjectPatcher *objectpatch.ObjectPatcher - ScheduleManager schedule_manager.ScheduleManager - KubeEventsManager kube_events_manager.KubeEventsManager + ScheduleManager schedulemanager.ScheduleManager + KubeEventsManager kubeeventsmanager.KubeEventsManager TaskQueues *queue.TaskQueueSet @@ -622,12 +622,12 @@ func (op *ShellOperator) handleRunHook(t task.Task, taskHook *hook.Hook, hookMet result, err := taskHook.Run(hookMeta.BindingType, hookMeta.BindingContext, hookLogLabels) if err != nil { if result != nil && len(result.KubernetesPatchBytes) > 0 { - operations, patchStatusErr := object_patch.ParseOperations(result.KubernetesPatchBytes) + operations, patchStatusErr := objectpatch.ParseOperations(result.KubernetesPatchBytes) if patchStatusErr != nil { return fmt.Errorf("%s: couldn't patch status: %s", err, patchStatusErr) } - patchStatusErr = op.ObjectPatcher.ExecuteOperations(object_patch.GetPatchStatusOperationsOnHookError(operations)) + patchStatusErr = op.ObjectPatcher.ExecuteOperations(objectpatch.GetPatchStatusOperationsOnHookError(operations)) if patchStatusErr != nil { return fmt.Errorf("%s: couldn't patch status: %s", err, patchStatusErr) } @@ -644,7 +644,7 @@ func (op *ShellOperator) handleRunHook(t task.Task, taskHook *hook.Hook, hookMet // Try to apply Kubernetes actions. if len(result.KubernetesPatchBytes) > 0 { - operations, err := object_patch.ParseOperations(result.KubernetesPatchBytes) + operations, err := objectpatch.ParseOperations(result.KubernetesPatchBytes) if err != nil { return err } @@ -735,7 +735,7 @@ func (op *ShellOperator) CombineBindingContextForHook(q *queue.TaskQueue, t task } // Combine binding context and make a map to delete excess tasks - combinedContext := make([]binding_context.BindingContext, 0) + combinedContext := make([]bindingcontext.BindingContext, 0) monitorIDs := taskMeta.(task_metadata.MonitorIDAccessor).GetMonitorIDs() tasksFilter := make(map[string]bool) // current task always remain in queue @@ -759,7 +759,7 @@ func (op *ShellOperator) CombineBindingContextForHook(q *queue.TaskQueue, t task }) // group is used to compact binding contexts when only snapshots are needed - compactedContext := make([]binding_context.BindingContext, 0) + compactedContext := make([]bindingcontext.BindingContext, 0) for i := 0; i < len(combinedContext); i++ { keep := true @@ -807,7 +807,7 @@ func (op *ShellOperator) bootstrapMainQueue(tqs *queue.TaskQueueSet) { } for _, hookName := range onStartupHooks { - bc := binding_context.BindingContext{ + bc := bindingcontext.BindingContext{ Binding: string(types.OnStartup), } bc.Metadata.BindingType = types.OnStartup @@ -816,7 +816,7 @@ func (op *ShellOperator) bootstrapMainQueue(tqs *queue.TaskQueueSet) { WithMetadata(task_metadata.HookMetadata{ HookName: hookName, BindingType: types.OnStartup, - BindingContext: []binding_context.BindingContext{bc}, + BindingContext: []bindingcontext.BindingContext{bc}, }). WithQueuedAt(time.Now()) mainQueue.AddLast(newTask) diff --git a/pkg/shell-operator/operator_test.go b/pkg/shell-operator/operator_test.go index 5e8ee610..cf2be54c 100644 --- a/pkg/shell-operator/operator_test.go +++ b/pkg/shell-operator/operator_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/hook/types" . "github.com/onsi/gomega" . "github.com/flant/shell-operator/pkg/hook/task_metadata" - . "github.com/flant/shell-operator/pkg/hook/types" "github.com/flant/shell-operator/pkg/task" utils "github.com/flant/shell-operator/pkg/utils/file" ) diff --git a/pkg/task/queue/queue_set.go b/pkg/task/queue/queue_set.go index 5c876f25..3e87cf45 100644 --- a/pkg/task/queue/queue_set.go +++ b/pkg/task/queue/queue_set.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/flant/shell-operator/pkg/metric_storage" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" "github.com/flant/shell-operator/pkg/task" ) @@ -15,7 +15,7 @@ const MainQueueName = "main" type TaskQueueSet struct { MainName string - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage ctx context.Context cancel context.CancelFunc @@ -40,7 +40,7 @@ func (tqs *TaskQueueSet) WithContext(ctx context.Context) { tqs.ctx, tqs.cancel = context.WithCancel(ctx) } -func (tqs *TaskQueueSet) WithMetricStorage(mstor *metric_storage.MetricStorage) { +func (tqs *TaskQueueSet) WithMetricStorage(mstor *metricstorage.MetricStorage) { tqs.metricStorage = mstor } diff --git a/pkg/task/queue/task_queue.go b/pkg/task/queue/task_queue.go index 32019550..c586a014 100644 --- a/pkg/task/queue/task_queue.go +++ b/pkg/task/queue/task_queue.go @@ -9,8 +9,8 @@ import ( "time" "github.com/deckhouse/deckhouse/pkg/log" + metricstorage "github.com/flant/shell-operator/pkg/metric-storage" - "github.com/flant/shell-operator/pkg/metric_storage" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/utils/exponential_backoff" "github.com/flant/shell-operator/pkg/utils/measure" @@ -56,7 +56,7 @@ type TaskResult struct { type TaskQueue struct { m sync.RWMutex - metricStorage *metric_storage.MetricStorage + metricStorage *metricstorage.MetricStorage ctx context.Context cancel context.CancelFunc @@ -101,7 +101,7 @@ func (q *TaskQueue) WithContext(ctx context.Context) { q.ctx, q.cancel = context.WithCancel(ctx) } -func (q *TaskQueue) WithMetricStorage(mstor *metric_storage.MetricStorage) { +func (q *TaskQueue) WithMetricStorage(mstor *metricstorage.MetricStorage) { q.metricStorage = mstor } diff --git a/pkg/webhook/admission/manager.go b/pkg/webhook/admission/manager.go index b34459c0..4224b2b6 100644 --- a/pkg/webhook/admission/manager.go +++ b/pkg/webhook/admission/manager.go @@ -4,9 +4,9 @@ import ( "os" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/webhook/server" klient "github.com/flant/kube-client/client" - "github.com/flant/shell-operator/pkg/webhook/server" ) // DefaultConfigurationId is a ConfigurationId for ValidatingWebhookConfiguration/MutatingWebhookConfiguration diff --git a/pkg/webhook/conversion/manager.go b/pkg/webhook/conversion/manager.go index 44f5287f..7acec13d 100644 --- a/pkg/webhook/conversion/manager.go +++ b/pkg/webhook/conversion/manager.go @@ -5,10 +5,10 @@ import ( "os" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/flant/shell-operator/pkg/webhook/server" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" klient "github.com/flant/kube-client/client" - "github.com/flant/shell-operator/pkg/webhook/server" ) type EventHandlerFn func(cdrName string, request *v1.ConversionRequest) (*Response, error) diff --git a/test/hook/context/context_combiner.go b/test/hook/context/context_combiner.go index 62b30f25..776c82b7 100644 --- a/test/hook/context/context_combiner.go +++ b/test/hook/context/context_combiner.go @@ -4,10 +4,11 @@ import ( "context" "fmt" - "github.com/flant/shell-operator/pkg/hook/binding_context" + bindingcontext "github.com/flant/shell-operator/pkg/hook/binding-context" "github.com/flant/shell-operator/pkg/hook/controller" - "github.com/flant/shell-operator/pkg/hook/task_metadata" "github.com/flant/shell-operator/pkg/hook/types" + + "github.com/flant/shell-operator/pkg/hook/task_metadata" shell_operator "github.com/flant/shell-operator/pkg/shell-operator" "github.com/flant/shell-operator/pkg/task" "github.com/flant/shell-operator/pkg/task/queue" @@ -54,10 +55,10 @@ func (c *ContextCombiner) AddBindingContext(bindingType types.BindingType, info // CombinedContext returns a combined context or a binding context // from the first task. -func (c *ContextCombiner) Combined() []binding_context.BindingContext { +func (c *ContextCombiner) Combined() []bindingcontext.BindingContext { firstTask := c.q.GetFirst() if firstTask == nil { - return []binding_context.BindingContext{} + return []bindingcontext.BindingContext{} } taskMeta := firstTask.GetMetadata() @@ -81,11 +82,11 @@ func (c *ContextCombiner) QueueLen() int { return c.q.Length() } -func ConvertToGeneratedBindingContexts(bindingContexts []binding_context.BindingContext) (GeneratedBindingContexts, error) { +func ConvertToGeneratedBindingContexts(bindingContexts []bindingcontext.BindingContext) (GeneratedBindingContexts, error) { res := GeneratedBindingContexts{} // Support only v1 binding contexts. - bcList := binding_context.ConvertBindingContextList("v1", bindingContexts) + bcList := bindingcontext.ConvertBindingContextList("v1", bindingContexts) data, err := bcList.Json() if err != nil { return res, fmt.Errorf("marshaling binding context error: %v", err) diff --git a/test/hook/context/generator.go b/test/hook/context/generator.go index a786237e..0f95a410 100644 --- a/test/hook/context/generator.go +++ b/test/hook/context/generator.go @@ -7,14 +7,15 @@ import ( "time" "github.com/deckhouse/deckhouse/pkg/log" - - "github.com/flant/kube-client/fake" "github.com/flant/shell-operator/pkg/hook" - . "github.com/flant/shell-operator/pkg/hook/binding_context" + . "github.com/flant/shell-operator/pkg/hook/binding-context" "github.com/flant/shell-operator/pkg/hook/controller" "github.com/flant/shell-operator/pkg/hook/types" - kubeeventsmanager "github.com/flant/shell-operator/pkg/kube_events_manager" - schedulemanager "github.com/flant/shell-operator/pkg/schedule_manager" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + + "github.com/flant/kube-client/fake" + "github.com/flant/shell-operator/pkg/app" + schedulemanager "github.com/flant/shell-operator/pkg/schedule-manager" ) func init() { @@ -102,7 +103,7 @@ func (b *BindingContextController) Run(initialState string) (GeneratedBindingCon } if b.Hook == nil { - testHook := hook.NewHook("test", "test", b.logger.Named("hook")) + testHook := hook.NewHook("test", "test", app.DebugKeepTmpFiles, app.LogProxyHookJSON, app.ProxyJsonLogKey, b.logger.Named("hook")) testHook, err = testHook.LoadConfig([]byte(b.HookConfig)) if err != nil { return GeneratedBindingContexts{}, fmt.Errorf("couldn't load or validate hook configuration: %v", err) diff --git a/test/hook/context/generator_test.go b/test/hook/context/generator_test.go index 09028938..1f90f0b5 100644 --- a/test/hook/context/generator_test.go +++ b/test/hook/context/generator_test.go @@ -5,9 +5,8 @@ import ( "testing" "github.com/deckhouse/deckhouse/pkg/log" + . "github.com/flant/shell-operator/pkg/hook/binding-context" . "github.com/onsi/gomega" - - . "github.com/flant/shell-operator/pkg/hook/binding_context" ) func parseContexts(contexts string) []BindingContext { diff --git a/test/hook/context/state.go b/test/hook/context/state.go index d99a965d..6f6ce2aa 100644 --- a/test/hook/context/state.go +++ b/test/hook/context/state.go @@ -5,10 +5,11 @@ import ( "reflect" "time" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + "github.com/flant/shell-operator/pkg/kube-events-manager/types" + "github.com/flant/kube-client/fake" "github.com/flant/kube-client/manifest" - kubeeventsmanager "github.com/flant/shell-operator/pkg/kube_events_manager" - "github.com/flant/shell-operator/pkg/kube_events_manager/types" ) // if we use default, then we are not able to emulate global resources due to fake cluster limitations diff --git a/test/integration/kube_event_manager/kube_event_manager_test.go b/test/integration/kube_event_manager/kube_event_manager_test.go index 51d1ecdd..db25fd2d 100644 --- a/test/integration/kube_event_manager/kube_event_manager_test.go +++ b/test/integration/kube_event_manager/kube_event_manager_test.go @@ -13,8 +13,8 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/flant/shell-operator/pkg/app" - "github.com/flant/shell-operator/pkg/kube_events_manager" - . "github.com/flant/shell-operator/pkg/kube_events_manager/types" + kubeeventsmanager "github.com/flant/shell-operator/pkg/kube-events-manager" + . "github.com/flant/shell-operator/pkg/kube-events-manager/types" . "github.com/flant/shell-operator/test/integration/suite" testutils "github.com/flant/shell-operator/test/utils" ) @@ -24,17 +24,17 @@ func Test(t *testing.T) { } var _ = Describe("Binding 'kubernetes' with kind 'Pod' should emit KubeEvent objects", func() { - var KubeEventsManager kube_events_manager.KubeEventsManager + var KubeEventsManager kubeeventsmanager.KubeEventsManager BeforeEach(func() { - KubeEventsManager = kube_events_manager.NewKubeEventsManager(context.Background(), KubeClient, log.NewNop()) + KubeEventsManager = kubeeventsmanager.NewKubeEventsManager(context.Background(), KubeClient, log.NewNop()) }) Context("with configVersion: v1", func() { - var monitorConfig *kube_events_manager.MonitorConfig + var monitorConfig *kubeeventsmanager.MonitorConfig BeforeEach(func() { - monitorConfig = &kube_events_manager.MonitorConfig{ + monitorConfig = &kubeeventsmanager.MonitorConfig{ Kind: "Pod", ApiVersion: "v1", KeepFullObjectsInMemory: true, diff --git a/test/integration/kubeclient/object_patch_test.go b/test/integration/kubeclient/object_patch_test.go index 22182317..2db6a064 100644 --- a/test/integration/kubeclient/object_patch_test.go +++ b/test/integration/kubeclient/object_patch_test.go @@ -7,6 +7,8 @@ import ( "context" "encoding/json" + objectpatch "github.com/flant/shell-operator/pkg/object-patch" + . "github.com/flant/shell-operator/test/integration/suite" uuid "github.com/gofrs/uuid/v5" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -16,9 +18,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" - - "github.com/flant/shell-operator/pkg/kube/object_patch" - . "github.com/flant/shell-operator/test/integration/suite" ) const ( @@ -65,7 +64,7 @@ var _ = Describe("Kubernetes API object patching", func() { }) It("should fail to Create() an object if it already exists", func() { - err := ObjectPatcher.ExecuteOperation(object_patch.NewCreateOperation(unstructuredCM)) + err := ObjectPatcher.ExecuteOperation(objectpatch.NewCreateOperation(unstructuredCM)) Expect(err).To(Not(Succeed())) }) @@ -79,7 +78,7 @@ var _ = Describe("Kubernetes API object patching", func() { unstructuredNewTestCM, err := generateUnstructured(newTestCM) Expect(err).To(Succeed()) - err = ObjectPatcher.ExecuteOperation(object_patch.NewCreateOperation(unstructuredNewTestCM, object_patch.UpdateIfExists())) + err = ObjectPatcher.ExecuteOperation(objectpatch.NewCreateOperation(unstructuredNewTestCM, objectpatch.UpdateIfExists())) Expect(err).To(Succeed()) cm, err := KubeClient.CoreV1().ConfigMaps(testCM.Namespace).Get(context.TODO(), newTestCM.Name, metav1.GetOptions{}) @@ -94,7 +93,7 @@ var _ = Describe("Kubernetes API object patching", func() { unstructuredSeparateTestCM, err := generateUnstructured(separateTestCM) Expect(err).To(Succeed()) - err = ObjectPatcher.ExecuteOperation(object_patch.NewCreateOperation(unstructuredSeparateTestCM, object_patch.UpdateIfExists())) + err = ObjectPatcher.ExecuteOperation(objectpatch.NewCreateOperation(unstructuredSeparateTestCM, objectpatch.UpdateIfExists())) Expect(err).To(Succeed()) _, err = KubeClient.CoreV1().ConfigMaps(testCM.Namespace).Get(context.TODO(), separateTestCM.Name, metav1.GetOptions{}) @@ -118,9 +117,9 @@ var _ = Describe("Kubernetes API object patching", func() { }) It("should successfully delete an object", func() { - err := ObjectPatcher.ExecuteOperation(object_patch.NewDeleteOperation( + err := ObjectPatcher.ExecuteOperation(objectpatch.NewDeleteOperation( testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name, - object_patch.InBackground())) + objectpatch.InBackground())) Expect(err).Should(Succeed()) _, err = KubeClient.CoreV1().ConfigMaps(testCM.Namespace).Get(context.TODO(), testCM.Name, metav1.GetOptions{}) @@ -128,7 +127,7 @@ var _ = Describe("Kubernetes API object patching", func() { }) It("should successfully delete an object if it doesn't exist", func() { - err := ObjectPatcher.ExecuteOperation(object_patch.NewDeleteOperation(testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name)) + err := ObjectPatcher.ExecuteOperation(objectpatch.NewDeleteOperation(testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name)) Expect(err).Should(Succeed()) }) }) @@ -149,8 +148,8 @@ var _ = Describe("Kubernetes API object patching", func() { }) It("should successfully JQPatch an object", func() { - err := ObjectPatcher.ExecuteOperation(object_patch.NewFromOperationSpec(object_patch.OperationSpec{ - Operation: object_patch.JQPatch, + err := ObjectPatcher.ExecuteOperation(objectpatch.NewFromOperationSpec(objectpatch.OperationSpec{ + Operation: objectpatch.JQPatch, ApiVersion: testCM.APIVersion, Kind: testCM.Kind, Namespace: testCM.Namespace, @@ -176,7 +175,7 @@ data: mergePatchJson, err := json.Marshal(mergePatch) Expect(err).To(Succeed()) - err = ObjectPatcher.ExecuteOperation(object_patch.NewMergePatchOperation(mergePatchJson, testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name)) + err = ObjectPatcher.ExecuteOperation(objectpatch.NewMergePatchOperation(mergePatchJson, testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name)) Expect(err).To(Succeed()) existingCM, err := KubeClient.CoreV1().ConfigMaps(testCM.Namespace).Get(context.TODO(), testCM.Name, metav1.GetOptions{}) @@ -185,7 +184,7 @@ data: }) It("should successfully JSONPatch an object", func() { - err := ObjectPatcher.ExecuteOperation(object_patch.NewJSONPatchOperation( + err := ObjectPatcher.ExecuteOperation(objectpatch.NewJSONPatchOperation( []byte(`[{ "op": "replace", "path": "/data/firstField", "value": "jsonPatched"}]`), testCM.APIVersion, testCM.Kind, testCM.Namespace, testCM.Name)) Expect(err).To(Succeed()) @@ -205,7 +204,7 @@ func ensureNamespace(name string) error { panic(err) } - return ObjectPatcher.ExecuteOperation(object_patch.NewCreateOperation(unstructuredNS, object_patch.UpdateIfExists())) + return ObjectPatcher.ExecuteOperation(objectpatch.NewCreateOperation(unstructuredNS, objectpatch.UpdateIfExists())) } func ensureTestObject(_ string, obj interface{}) error { @@ -214,11 +213,11 @@ func ensureTestObject(_ string, obj interface{}) error { panic(err) } - return ObjectPatcher.ExecuteOperation(object_patch.NewCreateOperation(unstructuredObj, object_patch.UpdateIfExists())) + return ObjectPatcher.ExecuteOperation(objectpatch.NewCreateOperation(unstructuredObj, objectpatch.UpdateIfExists())) } func removeNamespace(name string) error { - return ObjectPatcher.ExecuteOperation(object_patch.NewDeleteOperation("", "Namespace", "", name)) + return ObjectPatcher.ExecuteOperation(objectpatch.NewDeleteOperation("", "Namespace", "", name)) } func generateUnstructured(obj interface{}) (*unstructured.Unstructured, error) { diff --git a/test/integration/suite/run.go b/test/integration/suite/run.go index f6fb6dc6..47326ba6 100644 --- a/test/integration/suite/run.go +++ b/test/integration/suite/run.go @@ -7,19 +7,18 @@ import ( "os" "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/deckhouse/deckhouse/pkg/log" klient "github.com/flant/kube-client/client" - "github.com/flant/shell-operator/pkg/kube/object_patch" + objectpatch "github.com/flant/shell-operator/pkg/object-patch" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var ( ClusterName string ContextName string KubeClient *klient.Client - ObjectPatcher *object_patch.ObjectPatcher + ObjectPatcher *objectpatch.ObjectPatcher ) func RunIntegrationSuite(t *testing.T, description string, clusterPrefix string) { @@ -37,5 +36,5 @@ var _ = BeforeSuite(func() { err := KubeClient.Init() Expect(err).ShouldNot(HaveOccurred()) - ObjectPatcher = object_patch.NewObjectPatcher(KubeClient, log.NewNop()) + ObjectPatcher = objectpatch.NewObjectPatcher(KubeClient, log.NewNop()) })