Merge pull request #152 from mallardduck/new-charts/pt3

.github/workflows/prom-fed-e2e-ci.yaml

@@ -98,7 +98,7 @@ jobs:
       -
         name: Perform pre-e2e image build
         run: |
-          EMBEDED_CHART_VERSION=0.3.4 REPO=${REPO} TAG=${TAG} make build;
+          REPO=${REPO} TAG=${TAG} make build;
           REPO=${REPO} TAG=${TAG} make package;
       -
         name : Install k3d

.github/workflows/publish.yaml

@@ -45,15 +45,10 @@ jobs:
         uses: azure/setup-helm@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Prepare helm charts for debug artifacts
+      - name: Prepare helm charts (needed for build)
         run: |
-          BUILD_TARGET=helm-locker make package-helm && echo "hl: chart success"
-          BUILD_TARGET=helm-project-operator make package-helm && echo "hpo: embeded chart success"
-          BUILD_TARGET=prometheus-federator make package-helm && echo "pf: embeded chart success"
-      - name: Prepare embedded helm chart (needed for build)
-        run: |
-          BUILD_TARGET=helm-project-operator make build-chart
-          BUILD_TARGET=prometheus-federator make build-chart
+          BUILD_TARGET=prometheus-federator make package-helm && echo "pf: release chart prepared"
+          BUILD_TARGET=prometheus-federator make build-chart && echo "pf: embedded project-monitoring chart prepared"
       - uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser

charts/prometheus-federator/Chart.yaml

@@ -0,0 +1,25 @@
+annotations:
+  catalog.cattle.io/certified: rancher
+  catalog.cattle.io/display-name: Prometheus Federator
+  catalog.cattle.io/namespace: cattle-monitoring-system
+  catalog.cattle.io/os: linux,windows
+  catalog.cattle.io/permits-os: linux,windows
+  catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1
+  catalog.cattle.io/release-name: prometheus-federator
+apiVersion: v2
+appVersion: 9999
+description: Prometheus Federator - installs rancher-project-monitoring in project namespaces.
+icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg
+keywords:
+  - prometheus
+  - monitoring
+  - project-monitoring
+maintainers:
+  - email: alexandre.lamarre@suse.com
+    name: Alexandre
+  - email: dan.pock@suse.com
+    name: Dan
+  - email: julia.suriano@suse.com
+    name: Julia
+name: prometheus-federator
+version: 9999

charts/prometheus-federator/app-README.md

@@ -0,0 +1,27 @@
+# Prometheus Federator
+
+This chart deploys an operator that manages Project Monitoring Stacks composed of the following set of resources that are scoped to project namespaces:
+- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator))
+- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator))
+- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart)
+- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/)
+- Default ServiceMonitors that watch the deployed Prometheus, Grafana, and Alertmanager
+
+Since this Project Monitoring Stack deploys Prometheus Operator CRs, an existing Prometheus Operator instance must already be deployed in the cluster for Prometheus Federator to successfully be able to deploy Project Monitoring Stacks. It is recommended to use [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) for this. For more information on how the chart works or advanced configurations, please read the `README.md`.
+
+## Upgrading to Kubernetes v1.25+
+
+Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. 
+
+As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`.
+​
+> **Note:**
+> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`.
+    ​
+> **Note:**
+> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).**
+>
+> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets.
+Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart.
+​
+As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards.

charts/prometheus-federator/questions.yaml

@@ -0,0 +1,43 @@
+questions:
+- variable: global.cattle.psp.enabled
+  default: "false"
+  description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false."
+  label: "Enable PodSecurityPolicies"
+  type: boolean
+  group: "Security Settings"
+- variable: helmProjectOperator.helmController.enabled
+  label: Enable Embedded Helm Controller
+  description: 'Note: If you are running Prometheus Federator in an RKE2 / K3s cluster before v1.23.14 / v1.24.8 / v1.25.4, this should be disabled.'
+  type: boolean
+  group: Helm Controller
+- variable: helmProjectOperator.helmLocker.enabled
+  label: Enable Embedded Helm Locker
+  type: boolean
+  group: Helm Locker
+- variable: helmProjectOperator.projectReleaseNamespaces.labelValue
+  label: Project Release Namespace Project ID
+  description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx)
+  type: string
+  required: false
+  group: Namespaces
+- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin
+  label: Admin ClusterRole
+  description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx)
+  type: string
+  default: admin
+  required: false
+  group: RBAC
+- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit
+  label: Edit ClusterRole
+  description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx)
+  type: string
+  default: edit
+  required: false
+  group: RBAC
+- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view
+  label: View ClusterRole
+  description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx)
+  type: string
+  default: view
+  required: false
+  group: RBAC

charts/prometheus-federator/templates/NOTES.txt

@@ -0,0 +1,3 @@
+{{ $.Chart.Name }} has been installed. Check its status by running:
+  kubectl --namespace {{ template "prometheus-federator.namespace" . }} get pods -l "release={{ $.Release.Name }}"
+

charts/prometheus-federator/templates/_helpers.tpl

@@ -0,0 +1,89 @@
+# Rancher
+{{- define "system_default_registry" -}}
+{{- if .Values.global.cattle.systemDefaultRegistry -}}
+{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Define the image registry to use; either values, or systemdefault if set, or nothing */}}
+{{- define "prometheus-federator.imageRegistry" -}}
+{{- if and .Values.image .Values.image.registry }}{{- printf "%s/" .Values.image.registry -}}
+{{- else if .Values.helmProjectOperator.image.registry }}{{- printf "%s/" .Values.helmProjectOperator.image.registry -}}
+{{- else }}{{ template "system_default_registry" .  }}
+{{- end }}
+{{- end }}
+
+{{- define "prometheus-federator.imageRepository" -}}
+{{- if and .Values.image .Values.image.repository }}{{ .Values.image.repository }}
+{{- else if .Values.helmProjectOperator.image.repository }}{{ .Values.helmProjectOperator.image.repository }}
+{{- end }}
+{{- end }}
+
+{{- define "prometheus-federator.imageTag" -}}
+{{- if and .Values.image .Values.image.tag }}{{ .Values.image.tag }}
+{{- else if and .Values.helmProjectOperator.image.tag }}{{ .Values.helmProjectOperator.image.tag }}
+{{ else }}{{ .Chart.AppVersion }}
+{{- end }}
+{{- end }}
+
+# Windows Support
+
+{{/*
+Windows cluster will add default taint for linux nodes,
+add below linux tolerations to workloads could be scheduled to those linux nodes
+*/}}
+
+{{- define "linux-node-tolerations" -}}
+- key: "cattle.io/os"
+  value: "linux"
+  effect: "NoSchedule"
+  operator: "Equal"
+{{- end -}}
+
+{{- define "linux-node-selector" -}}
+{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+beta.kubernetes.io/os: linux
+{{- else -}}
+kubernetes.io/os: linux
+{{- end -}}
+{{- end -}}
+
+# Helm Project Operator
+
+{{/* vim: set filetype=mustache: */}}
+{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}}
+{{- define "prometheus-federator.name" -}}
+{{- default .Chart.Name (default .Values.helmProjectOperator.nameOverride .Values.nameOverride) | trunc 50 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Allow the release namespace to be overridden for multi-namespace deployments in combined charts
+*/}}
+{{- define "prometheus-federator.namespace" -}}
+  {{- if .Values.namespaceOverride -}}
+    {{- .Values.namespaceOverride -}}
+  {{- else if .Values.helmProjectOperator.namespaceOverride -}}
+    {{- .Values.helmProjectOperator.namespaceOverride -}}
+  {{- else -}}
+    {{- .Release.Namespace -}}
+  {{- end -}}
+{{- end -}}
+
+{{/* Create chart name and version as used by the chart label. */}}
+{{- define "prometheus-federator.chartref" -}}
+{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}}
+{{- end }}
+
+{{/* Generate basic labels */}}
+{{- define "prometheus-federator.labels" }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}"
+app.kubernetes.io/part-of: {{ template "prometheus-federator.name" . }}
+chart: {{ template "prometheus-federator.chartref" . }}
+release: {{ $.Release.Name | quote }}
+heritage: {{ $.Release.Service | quote }}
+{{- if .Values.commonLabels}}
+{{ toYaml .Values.commonLabels }}
+{{- end }}
+{{- end }}

charts/prometheus-federator/templates/cleanup.yaml

@@ -0,0 +1,82 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "prometheus-federator.name" . }}-cleanup
+  namespace: {{ template "prometheus-federator.namespace" . }}
+  labels: {{ include "prometheus-federator.labels" . | indent 4 }}
+    app: {{ template "prometheus-federator.name" . }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+spec:
+  template:
+    metadata:
+      name: {{ template "prometheus-federator.name" . }}-cleanup
+      labels: {{ include "prometheus-federator.labels" . | indent 8 }}
+        app: {{ template "prometheus-federator.name" . }}
+    spec:
+      serviceAccountName: {{ template "prometheus-federator.name" . }}
+{{- if .Values.helmProjectOperator.cleanup.securityContext }}
+      securityContext: {{ toYaml .Values.helmProjectOperator.cleanup.securityContext | nindent 8 }}
+{{- end }}
+      initContainers:
+        - name: add-cleanup-annotations
+          image: {{ template "system_default_registry" . }}{{ .Values.helmProjectOperator.cleanup.image.repository }}:{{ .Values.helmProjectOperator.cleanup.image.tag }}
+          imagePullPolicy: "{{ .Values.helmProjectOperator.image.pullPolicy }}"
+          command:
+          - /bin/sh
+          - -c
+          - >
+            echo "Labeling all ProjectHelmCharts with helm.cattle.io/helm-project-operator-cleanup=true";
+            EXPECTED_HELM_API_VERSION={{ .Values.helmProjectOperator.helmApiVersion }};
+            IFS=$'\n';
+            for namespace in $(kubectl get namespaces -l helm.cattle.io/helm-project-operated=true --no-headers -o=custom-columns=NAME:.metadata.name); do
+              for projectHelmChartAndHelmApiVersion in $(kubectl get projecthelmcharts -n ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name,HELMAPIVERSION:.spec.helmApiVersion); do
+                projectHelmChartAndHelmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | xargs);
+                projectHelmChart=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f1);
+                helmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f2);
+                if [[ ${helmApiVersion} != ${EXPECTED_HELM_API_VERSION} ]]; then
+                  echo "Skipping marking ${namespace}/${projectHelmChart} with cleanup annotation since spec.helmApiVersion: ${helmApiVersion} is not ${EXPECTED_HELM_API_VERSION}";
+                  continue;
+                fi;
+                kubectl label projecthelmcharts -n ${namespace} ${projectHelmChart} helm.cattle.io/helm-project-operator-cleanup=true --overwrite;
+              done;
+            done;
+{{- if .Values.helmProjectOperator.cleanup.resources }}
+          resources: {{ toYaml .Values.helmProjectOperator.cleanup.resources | nindent 12 }}
+{{- end }}
+{{- if .Values.helmProjectOperator.cleanup.containerSecurityContext }}
+          securityContext: {{ toYaml .Values.helmProjectOperator.cleanup.containerSecurityContext | nindent 12 }}
+{{- end }}
+      containers:
+        - name: ensure-subresources-deleted
+          image: {{ template "system_default_registry" . }}{{ .Values.helmProjectOperator.cleanup.image.repository }}:{{ .Values.helmProjectOperator.cleanup.image.tag }}
+          imagePullPolicy: IfNotPresent
+          command:
+          - /bin/sh
+          - -c
+          - >
+            SYSTEM_NAMESPACE={{ .Release.Namespace }}
+            EXPECTED_HELM_API_VERSION={{ .Values.helmProjectOperator.helmApiVersion }};
+            HELM_API_VERSION_TRUNCATED=$(echo ${EXPECTED_HELM_API_VERSION} | cut -d'/' -f0);
+            echo "Ensuring HelmCharts and HelmReleases are deleted from ${SYSTEM_NAMESPACE}...";
+            while [[ "$(kubectl get helmcharts,helmreleases -l helm.cattle.io/helm-api-version=${HELM_API_VERSION_TRUNCATED} -n ${SYSTEM_NAMESPACE} 2>&1)" != "No resources found in ${SYSTEM_NAMESPACE} namespace." ]]; do
+              echo "waiting for HelmCharts and HelmReleases to be deleted from ${SYSTEM_NAMESPACE}... sleeping 3 seconds";
+              sleep 3;
+            done;
+            echo "Successfully deleted all HelmCharts and HelmReleases in ${SYSTEM_NAMESPACE}!";
+{{- if .Values.helmProjectOperator.cleanup.resources }}
+          resources: {{ toYaml .Values.helmProjectOperator.cleanup.resources | nindent 12 }}
+{{- end }}
+{{- if .Values.helmProjectOperator.cleanup.containerSecurityContext }}
+          securityContext: {{ toYaml .Values.helmProjectOperator.cleanup.containerSecurityContext | nindent 12 }}
+{{- end }}
+      restartPolicy: OnFailure
+      nodeSelector: {{ include "linux-node-selector" . | nindent 8 }}
+      {{- if .Values.helmProjectOperator.cleanup.nodeSelector }}
+      {{- toYaml .Values.helmProjectOperator.cleanup.nodeSelector | nindent 8 }}
+      {{- end }}
+      tolerations: {{ include "linux-node-tolerations" . | nindent 8 }}
+      {{- if .Values.helmProjectOperator.cleanup.tolerations }}
+      {{- toYaml .Values.helmProjectOperator.cleanup.tolerations | nindent 8 }}
+      {{- end }}

charts/prometheus-federator/templates/clusterrole.yaml

@@ -0,0 +1,57 @@
+{{- if and .Values.helmProjectOperator.global.rbac.create .Values.helmProjectOperator.global.rbac.userRoles.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "prometheus-federator.name" . }}-admin
+  labels: {{ include "prometheus-federator.labels" . | indent 4 }}
+  {{- if .Values.helmProjectOperator.global.rbac.userRoles.aggregateToDefaultRoles }}
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+  {{- end }}
+rules:
+- apiGroups:
+  - helm.cattle.io
+  resources:
+  - projecthelmcharts
+  - projecthelmcharts/finalizers
+  - projecthelmcharts/status
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "prometheus-federator.name" . }}-edit
+  labels: {{ include "prometheus-federator.labels" . | indent 4 }}
+  {{- if .Values.helmProjectOperator.global.rbac.userRoles.aggregateToDefaultRoles }}
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+  {{- end }}
+rules:
+- apiGroups:
+  - helm.cattle.io
+  resources:
+  - projecthelmcharts
+  - projecthelmcharts/status
+  verbs:
+  - 'get'
+  - 'list'
+  - 'watch'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "prometheus-federator.name" . }}-view
+  labels: {{ include "prometheus-federator.labels" . | indent 4 }}
+  {{- if .Values.helmProjectOperator.global.rbac.userRoles.aggregateToDefaultRoles }}
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  {{- end }}
+rules:
+- apiGroups:
+  - helm.cattle.io
+  resources:
+  - projecthelmcharts
+  - projecthelmcharts/status
+  verbs:
+  - 'get'
+  - 'list'
+  - 'watch'
+{{- end }}

charts/prometheus-federator/templates/configmap.yaml

@@ -0,0 +1,14 @@
+## Note: If you add another entry to this ConfigMap, make sure a corresponding env var is set
+## in the deployment of the operator to ensure that a Helm upgrade will force the operator
+## to reload the values in the ConfigMap and redeploy
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "prometheus-federator.name" . }}-config
+  namespace: {{ template "prometheus-federator.namespace" . }}
+  labels: {{ include "prometheus-federator.labels" . | indent 4 }}
+data:
+  hardened.yaml: |-
+{{ .Values.helmProjectOperator.hardenedNamespaces.configuration | toYaml | indent 4 }}
+  values.yaml: |-
+{{ .Values.helmProjectOperator.valuesOverride | toYaml | indent 4 }}