Support skipping a namespace if was recently used based on annotation…

… timestamp (#2)

* Support skipping a namespace if was recently used based on annotation timestamp

CHANGELOG.md

@@ -1,3 +1,7 @@
+## Unreleased
+
+* Support skipping a namespace if was recently used based on annotation timestamp
+
 ## v0.1.0 / 2021-06-07
 
 * Initial release

README.md

@@ -37,8 +37,9 @@ For Open OnDemand the following adjustments can be made to get a working install
 ```
 helm install k8-namespace-reaper k8-namespace-reaper/k8-namespace-reaper \
 -n k8-namespace-reaper --create-namespace \
---prometheus-address=http://prometheus:9090
---set config.namespaceLabels='app.kubernetes.io/name=open-ondemand'
+--prometheus-address=http://prometheus:9090 \
+--set config.namespaceLabels='app.kubernetes.io/name=open-ondemand' \
+--set config.namespaceLastUsedAnnotation='openondemand.org/last-hook-execution'
 ```
 
 ### Install with YAML
@@ -69,6 +70,9 @@ If you wish to scope the namespaces searched for reaping change either `--namesp
 
 The minimum age of a namespace to reap is set with `--reap-after`. This flag also sets how far back to look for active namespaces by looking at pod metrics. If `--reap-after` is default of `168h` then a namespace older than 7 days with no pods active in last 7 days will be deleted.
 
+Use `--namespace-last-used-annotation` to define a namespace annotation that marks when the namespace was last used.
+A namespace will not be reaped if that last usage is more recent than the duration defined with `--last-used-threshold`.
+
 ## Configuration Details
 
 The k8-namespace-reaper is intended to be deployed inside a Kubernetes cluster. It can also be run outside the cluster via cron.
@@ -79,9 +83,11 @@ The following flags and environment variables can modify the behavior of the k8-
 |---------|----------------------|-------------|
 | --namespace-labels | NAMESPACE_LABELS | Sets namespaces labels for which namespaces to consider for reaping, required if `--namespace-regexp` is not set. |
 | --namespace-regexp | NAMESPACE_REGEXP | Sets namespace regular expression for which namespaces to consider for reaping, required if `--namespace-labels` is not set. |
+| --namespace-last-used-annotation | NAMESPACE\_LAST\_USED_ANNOTATION | Annotation of when namespace was last used, must be Unix timestamp |
 | --prometheus-address | PROMETHEUS_ADDRESS | Prometheus address, eg: http://prometheus:9090, this is required |
 | --prometheus-timeout=30s | PROMETHEUS_TIMEOUT=30s | Prometheus query timeout [Duration](https://golang.org/pkg/time/#ParseDuration) |
 | --reap-after=168h | REAP_AFTER=168h |  [Duration](https://golang.org/pkg/time/#ParseDuration) minimum age of namespaces to reap as well as how far back to look for active pods |
+| --last-used-threshold=4h | LAST\_USED_THRESHOLD=4h | How long after last used can a namespace be reaped (must be a [Duration](https://golang.org/pkg/time/#ParseDuration)) |
 | --interval=6h | INTERVAL=6h | [Duration](https://golang.org/pkg/time/#ParseDuration) between each reaping execution when run in loop |
 | --listen-address=:8080 | LISTEN_ADDRESS=:8080| Address to listen for HTTP requests |
 | --no-process-metrics | PROCESS_METRICS=false | Disable metrics about the running processes such as CPU, memory and Go stats |

charts/k8-namespace-reaper/ci/test-values.yaml

@@ -1,5 +1,6 @@
 config:
   prometheusAddress: http://prometheus.prometheus.svc.cluster.local:9090
   namespaceLabels: app.kubernetes.io/name=open-ondemand
+  namespaceLastUsedAnnotation: openondemand.org/last-hook-execution
 extraArgs:
   - --log-level=debug

charts/k8-namespace-reaper/templates/deployment.yaml

@@ -40,6 +40,9 @@ spec:
           {{- if .Values.config.namespaceRegexp }}
             - --namespace-regexp={{ .Values.config.namespaceRegexp }}
           {{- end }}
+          {{- if .Values.config.namespaceLastUsedAnnotation }}
+            - --namespace-last-used-annotation={{ .Values.config.namespaceLastUsedAnnotation }}
+          {{- end }}
           {{- if .Values.config.prometheusAddress }}
             - --prometheus-address={{ .Values.config.prometheusAddress }}
           {{- end }}
@@ -49,6 +52,9 @@ spec:
           {{- if .Values.config.reapAfter }}
             - --reap-after={{ .Values.config.reapAfter }}
           {{- end }}
+          {{- if .Values.config.lastUsedThreshold }}
+            - --last-used-threshold={{ .Values.config.lastUsedThreshold }}
+          {{- end }}
           {{- if .Values.config.interval }}
             - --interval={{ .Values.config.interval }}
           {{- end }}

charts/k8-namespace-reaper/values.yaml

@@ -7,10 +7,12 @@ config:
   namespaceLabels: ""
   # For OnDemand
   # namespaceLabels: app.kubernetes.io/name=open-ondemand
+  # namespaceLastUsedAnnotation: openondemand.org/last-hook-execution
   namespaceRegexp: ""
   prometheusAddress: ""
   prometheusTimeout: 30s
   reapAfter: 168h
+  lastUsedThreshold: 4h
   interval: 6h
 extraArgs: []
 

install/ondemand-deployment.yaml

@@ -25,6 +25,7 @@ spec:
         args:
         - --prometheus-address=http://prometheus:9090
         - --namespace-labels=app.kubernetes.io/name=open-ondemand
+        - --namespace-last-used-annotation=openondemand.org/last-hook-execution
         - --listen-address=:8080
         - --log-level=info
         - --log-format=logfmt

main.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -47,19 +48,21 @@ const (
 )
 
 var (
-	namespaceLabels   = kingpin.Flag("namespace-labels", "Labels to use when filtering namespaces").Default("").Envar("NAMESPACE_LABELS").String()
-	namespaceRegexp   = kingpin.Flag("namespace-regexp", "Regular expression of namespaces to reap").Default("").Envar("NAMESPACE_REGEXP").String()
-	prometheusAddress = kingpin.Flag("prometheus-address", "URL for Prometheus, eg http://prometheus:9090").Envar("PROMETHEUS_ADDRESS").Required().String()
-	prometheusTimeout = kingpin.Flag("prometheus-timeout", "Duration to timeout Prometheus query").Default("30s").Envar("PROMETHEUS_TIMEOUT").Duration()
-	reapAfter         = kingpin.Flag("reap-after", "How long to wait before reaping unused namespaces").Default("168h").Envar("REAP_AFTER").Duration()
-	interval          = kingpin.Flag("interval", "Duration between reap runs").Default("6h").Envar("INTERLVAL").Duration()
-	listenAddress     = kingpin.Flag("listen-address", "Address to listen for HTTP requests").Default(":8080").Envar("LISTEN_ADDRESS").String()
-	processMetrics    = kingpin.Flag("process-metrics", "Collect metrics about running process such as CPU and memory and Go stats").Default("true").Envar("PROCESS_METRICS").Bool()
-	runOnce           = kingpin.Flag("run-once", "Set application to run once then exit, ie executed with cron").Default("false").Envar("RUN_ONCE").Bool()
-	kubeconfig        = kingpin.Flag("kubeconfig", "Path to kubeconfig when running outside Kubernetes cluster").Default("").Envar("KUBECONFIG").String()
-	logLevel          = kingpin.Flag("log-level", "Log level, One of: [debug, info, warn, error]").Default("info").Envar("LOG_LEVEL").String()
-	logFormat         = kingpin.Flag("log-format", "Log format, One of: [logfmt, json]").Default("logfmt").Envar("LOG_FORMAT").String()
-	timestampFormat   = log.TimestampFormat(
+	namespaceLabels             = kingpin.Flag("namespace-labels", "Labels to use when filtering namespaces").Default("").Envar("NAMESPACE_LABELS").String()
+	namespaceRegexp             = kingpin.Flag("namespace-regexp", "Regular expression of namespaces to reap").Default("").Envar("NAMESPACE_REGEXP").String()
+	namespaceLastUsedAnnotation = kingpin.Flag("namespace-last-used-annotation", "Annotation of when namespace was last used, must be Unix timestamp").Default("").Envar("NAMESPACE_LAST_USED_ANNOTATION").String()
+	prometheusAddress           = kingpin.Flag("prometheus-address", "URL for Prometheus, eg http://prometheus:9090").Envar("PROMETHEUS_ADDRESS").Required().String()
+	prometheusTimeout           = kingpin.Flag("prometheus-timeout", "Duration to timeout Prometheus query").Default("30s").Envar("PROMETHEUS_TIMEOUT").Duration()
+	reapAfter                   = kingpin.Flag("reap-after", "How long to wait before reaping unused namespaces").Default("168h").Envar("REAP_AFTER").Duration()
+	lastUsedThreshold           = kingpin.Flag("last-used-threshold", "How long after last used can a namespace be reaped").Default("4h").Envar("LAST_USED_THRESHOLD").Duration()
+	interval                    = kingpin.Flag("interval", "Duration between reap runs").Default("6h").Envar("INTERLVAL").Duration()
+	listenAddress               = kingpin.Flag("listen-address", "Address to listen for HTTP requests").Default(":8080").Envar("LISTEN_ADDRESS").String()
+	processMetrics              = kingpin.Flag("process-metrics", "Collect metrics about running process such as CPU and memory and Go stats").Default("true").Envar("PROCESS_METRICS").Bool()
+	runOnce                     = kingpin.Flag("run-once", "Set application to run once then exit, ie executed with cron").Default("false").Envar("RUN_ONCE").Bool()
+	kubeconfig                  = kingpin.Flag("kubeconfig", "Path to kubeconfig when running outside Kubernetes cluster").Default("").Envar("KUBECONFIG").String()
+	logLevel                    = kingpin.Flag("log-level", "Log level, One of: [debug, info, warn, error]").Default("info").Envar("LOG_LEVEL").String()
+	logFormat                   = kingpin.Flag("log-format", "Log format, One of: [logfmt, json]").Default("logfmt").Envar("LOG_FORMAT").String()
+	timestampFormat             = log.TimestampFormat(
 		func() time.Time { return time.Now().UTC() },
 		"2006-01-02T15:04:05.000Z07:00",
 	)
@@ -260,6 +263,22 @@ func getNamespaces(clientset kubernetes.Interface, logger log.Logger) ([]string,
 				level.Debug(logger).Log("msg", "Skipping namespace due to age", "namespace", namespace.Name, "age", currentAge.String())
 				continue
 			}
+			if *namespaceLastUsedAnnotation != "" {
+				if val, ok := namespace.Annotations[*namespaceLastUsedAnnotation]; ok {
+					sec, err := strconv.ParseInt(val, 10, 64)
+					if err != nil {
+						level.Error(logger).Log("msg", "Unable to parse namespace last used annotation", "namespace", namespace.Name, "err", err)
+						continue
+					}
+					timeSinceLastUsed := timeNow().Sub(time.Unix(sec, 0))
+					if timeSinceLastUsed < *lastUsedThreshold {
+						level.Debug(logger).Log("msg", "Skipping namespace due to recently used", "namespace", namespace.Name, "last-used", timeSinceLastUsed.String())
+						continue
+					}
+				} else {
+					level.Debug(logger).Log("msg", "Namespace lacks last used annotation", "namespace", namespace.Name)
+				}
+			}
 			namespaces = append(namespaces, namespace.Name)
 		}
 	}

main_test.go

@@ -51,6 +51,10 @@ func clientset() kubernetes.Interface {
 			Labels: map[string]string{
 				"app.kubernetes.io/name": "open-ondemand",
 			},
+			Annotations: map[string]string{
+				// date --date="01/08/2020 14:00:00" +%s
+				"openondemand.org/last-hook-execution": "1578510000",
+			},
 			CreationTimestamp: metav1.NewTime(creationTime),
 		},
 	}, &v1.Namespace{
@@ -67,6 +71,9 @@ func clientset() kubernetes.Interface {
 			Labels: map[string]string{
 				"app.kubernetes.io/name": "foo",
 			},
+			Annotations: map[string]string{
+				"openondemand.org/last-hook-execution": "foo",
+			},
 			CreationTimestamp: metav1.NewTime(creationTime.Add(time.Hour * 24)),
 		},
 	})
@@ -148,6 +155,46 @@ func TestGetNamespacesByRegexp(t *testing.T) {
 	}
 }
 
+func TestGetNamespacesLastUsedAnnotation(t *testing.T) {
+	args := []string{
+		"--namespace-regexp=user-.+",
+		"--namespace-last-used-annotation=openondemand.org/last-hook-execution",
+		"--prometheus-address=foobar",
+	}
+	if _, err := kingpin.CommandLine.Parse(args); err != nil {
+		t.Fatal(err)
+	}
+	timeNow = func() time.Time {
+		return creationTime.Add((time.Hour * 24 * 7) + time.Hour)
+	}
+	w := log.NewSyncWriter(os.Stderr)
+	logger := log.NewLogfmtLogger(w)
+	clientset := clientset()
+	namespaces, err := getNamespaces(clientset, logger)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if len(namespaces) != 0 {
+		t.Errorf("Unexpected number of namespaces: %d", len(namespaces))
+	}
+	timeNow = func() time.Time {
+		return creationTime.Add((time.Hour * 24 * 8) + time.Hour)
+	}
+	namespaces, err = getNamespaces(clientset, logger)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if len(namespaces) != 2 {
+		t.Errorf("Unexpected number of namespaces: %d", len(namespaces))
+	}
+	expected := []string{"user-user1", "user-user2"}
+	sort.Strings(expected)
+	sort.Strings(namespaces)
+	if !reflect.DeepEqual(namespaces, expected) {
+		t.Errorf("Unexpected value for namespaces\nExpected: %v\nGot: %v", expected, namespaces)
+	}
+}
+
 func TestGetNamespacesByRegexpAndLabel(t *testing.T) {
 	args := []string{
 		"--prometheus-address=foobar",