From 91733e73e009179ee5c2f63ba8c64f75c8db3d0c Mon Sep 17 00:00:00 2001 From: Douglass Kirkley Date: Wed, 12 Feb 2025 07:38:12 -0500 Subject: [PATCH] feat: Add TLS config to Perses client and mount certs to Deployment and StatefulSet Signed-off-by: Douglass Kirkley --- api/v1alpha1/perses_types.go | 40 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 62 +++++++++ config/crd/bases/perses.dev_perses.yaml | 67 ++++++++++ .../perses.dev_v1alpha1_perses_tls.yaml | 39 ++++++ controllers/perses/deployment_controller.go | 52 +------- controllers/perses/statefulset_controller.go | 46 +------ internal/perses/common/perses_args.go | 21 +++ .../perses/common/perses_client_factory.go | 33 ++++- internal/perses/common/volumes.go | 120 ++++++++++++++++++ 9 files changed, 384 insertions(+), 96 deletions(-) create mode 100644 config/samples/perses.dev_v1alpha1_perses_tls.yaml create mode 100644 internal/perses/common/perses_args.go create mode 100644 internal/perses/common/volumes.go diff --git a/api/v1alpha1/perses_types.go b/api/v1alpha1/perses_types.go index a46c2fd..3772253 100644 --- a/api/v1alpha1/perses_types.go +++ b/api/v1alpha1/perses_types.go @@ -26,8 +26,14 @@ type PersesSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec Metadata *Metadata `json:"metadata,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec + // Client perses client configuration + Client Client `json:"client,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec Config PersesConfig `json:"config,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec + // Args extra arguments to pass to perses + Args []string `json:"args,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec ContainerPort int32 `json:"containerPort,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec Replicas *int32 `json:"replicas,omitempty"` @@ -45,6 +51,40 @@ type Metadata struct { Annotations map[string]string `json:"annotations,omitempty"` } +type Client struct { + // +optional + // TLS the equivalent to the tls_config for perses client + TLS *TLS `json:"tls,omitempty"` +} + +type TLS struct { + Enable bool `json:"enable"` + InsecureSkipVerify bool `json:"insecureSkipVerify"` + CaCert Certificate `json:"caCert"` + // +optional + UserCert *Certificate `json:"userCert,omitempty"` +} + +// CertificateType types of certificate sources in k8s +type CertificateType string + +const ( + CertificateTypeSecret CertificateType = "secret" + CertificateTypeConfigMap CertificateType = "configmap" +) + +type Certificate struct { + // +kubebuilder:validation:Enum:={"secret", "configmap"} + // Type source type of certificate + Type CertificateType `json:"type"` + // Name of certificate k8s resource + Name string `json:"name"` + // CertFile path to certificate + CertFile string `json:"certFile"` + // CertKeyFile path to certificate key file + CertKeyFile string `json:"certKeyFile"` +} + // PersesStatus defines the observed state of Perses type PersesStatus struct { // +operator-sdk:csv:customresourcedefinitions:type=status diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8ad8e49..a56eccf 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,41 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Certificate) DeepCopyInto(out *Certificate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Certificate. +func (in *Certificate) DeepCopy() *Certificate { + if in == nil { + return nil + } + out := new(Certificate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Client) DeepCopyInto(out *Client) { + *out = *in + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Client. +func (in *Client) DeepCopy() *Client { + if in == nil { + return nil + } + out := new(Client) + in.DeepCopyInto(out) + return out +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Dashboard. func (in *Dashboard) DeepCopy() *Dashboard { if in == nil { @@ -314,7 +349,13 @@ func (in *PersesSpec) DeepCopyInto(out *PersesSpec) { *out = new(Metadata) (*in).DeepCopyInto(*out) } + in.Client.DeepCopyInto(&out.Client) in.Config.DeepCopyInto(&out.Config) + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) @@ -372,3 +413,24 @@ func (in *PersesStatus) DeepCopy() *PersesStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLS) DeepCopyInto(out *TLS) { + *out = *in + out.CaCert = in.CaCert + if in.UserCert != nil { + in, out := &in.UserCert, &out.UserCert + *out = new(Certificate) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. +func (in *TLS) DeepCopy() *TLS { + if in == nil { + return nil + } + out := new(TLS) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/perses.dev_perses.yaml b/config/crd/bases/perses.dev_perses.yaml index 5177283..443f5c1 100644 --- a/config/crd/bases/perses.dev_perses.yaml +++ b/config/crd/bases/perses.dev_perses.yaml @@ -959,6 +959,73 @@ spec: x-kubernetes-list-type: atomic type: object type: object + args: + description: Args extra arguments to pass to perses + items: + type: string + type: array + client: + description: Client perses client configuration + properties: + tls: + description: TLS the equivalent to the tls_config for perses client + properties: + caCert: + properties: + certFile: + description: CertFile path to certificate + type: string + certKeyFile: + description: CertKeyFile path to certificate key file + type: string + name: + description: Name of certificate k8s resource + type: string + type: + description: Type source type of certificate + enum: + - secret + - configmap + type: string + required: + - certFile + - certKeyFile + - name + - type + type: object + enable: + type: boolean + insecureSkipVerify: + type: boolean + userCert: + properties: + certFile: + description: CertFile path to certificate + type: string + certKeyFile: + description: CertKeyFile path to certificate key file + type: string + name: + description: Name of certificate k8s resource + type: string + type: + description: Type source type of certificate + enum: + - secret + - configmap + type: string + required: + - certFile + - certKeyFile + - name + - type + type: object + required: + - caCert + - enable + - insecureSkipVerify + type: object + type: object config: properties: api_prefix: diff --git a/config/samples/perses.dev_v1alpha1_perses_tls.yaml b/config/samples/perses.dev_v1alpha1_perses_tls.yaml new file mode 100644 index 0000000..f4574f4 --- /dev/null +++ b/config/samples/perses.dev_v1alpha1_perses_tls.yaml @@ -0,0 +1,39 @@ +apiVersion: perses.dev/v1alpha1 +kind: Perses +metadata: + labels: + app.kubernetes.io/name: perses + app.kubernetes.io/instance: perses-tls-sample + app.kubernetes.io/part-of: perses-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: perses-operator + name: perses-tls-sample + namespace: perses-dev +spec: + client: + tls: + enable: true + caCert: + type: secret + name: perses-certs + certFile: ca.crt + userCert: + type: secret + name: perses-certs + certFile: tls.crt + certKeyFile: tls.key + + config: + database: + file: + folder: "/etc/perses/storage" + extension: "yaml" + schemas: + panels_path: "/etc/perses/cue/schemas/panels" + queries_path: "/etc/perses/cue/schemas/queries" + datasources_path: "/etc/perses/cue/schemas/datasources" + variables_path: "/etc/perses/cue/schemas/variables" + ephemeral_dashboard: + enable: false + cleanup_interval: "1s" + containerPort: 8080 diff --git a/controllers/perses/deployment_controller.go b/controllers/perses/deployment_controller.go index 1c177a4..002057d 100644 --- a/controllers/perses/deployment_controller.go +++ b/controllers/perses/deployment_controller.go @@ -29,7 +29,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -119,7 +118,6 @@ func (r *PersesReconciler) reconcileDeployment(ctx context.Context, req ctrl.Req func (r *PersesReconciler) createPersesDeployment( perses *v1alpha1.Perses) (*appsv1.Deployment, error) { - configName := common.GetConfigName(perses.Name) ls, err := common.LabelsForPerses(r.Config.PersesImage, perses.Name, perses.Name, perses.Spec.Metadata) if err != nil { @@ -179,54 +177,10 @@ func (r *PersesReconciler) createPersesDeployment( ContainerPort: perses.Spec.ContainerPort, Name: "perses", }}, - VolumeMounts: []corev1.VolumeMount{ - // TODO: check if perses supports passing certificates for TLS - // { - // Name: "serving-cert", - // ReadOnly: true, - // MountPath: "/var/serving-cert", - // }, - { - Name: "config", - ReadOnly: true, - MountPath: "/perses/config", - }, - { - Name: "storage", - ReadOnly: false, - MountPath: "/etc/perses/storage", - }, - }, - Args: []string{"--config=/perses/config/config.yaml"}, + VolumeMounts: common.GetVolumeMounts(perses.Spec.Client.TLS), + Args: common.GetPersesArgs(perses.Spec.Client.TLS, perses.Spec.Args), }}, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: configName, - }, - DefaultMode: ptr.To[int32](420), - }, - }, - }, - { - Name: "storage", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - // { - // Name: "serving-cert", - // VolumeSource: corev1.VolumeSource{ - // Secret: &corev1.SecretVolumeSource{ - // SecretName: "perses-serving-cert", - // DefaultMode: &[]int32{420}[0], - // }, - // }, - // }, - }, + Volumes: common.GetVolumes(perses.Name, perses.Spec.Client.TLS), RestartPolicy: "Always", DNSPolicy: "ClusterFirst", }, diff --git a/controllers/perses/statefulset_controller.go b/controllers/perses/statefulset_controller.go index ab50c1d..69d02ce 100644 --- a/controllers/perses/statefulset_controller.go +++ b/controllers/perses/statefulset_controller.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -121,7 +120,6 @@ func (r *PersesReconciler) reconcileStatefulSet(ctx context.Context, req ctrl.Re func (r *PersesReconciler) createPersesStatefulSet( perses *v1alpha1.Perses) (*appsv1.StatefulSet, error) { - configName := common.GetConfigName(perses.Name) ls, err := common.LabelsForPerses(r.Config.PersesImage, perses.Name, perses.Name, perses.Spec.Metadata) if err != nil { @@ -183,48 +181,10 @@ func (r *PersesReconciler) createPersesStatefulSet( ContainerPort: perses.Spec.ContainerPort, Name: "perses", }}, - VolumeMounts: []corev1.VolumeMount{ - // TODO: check if perses supports passing certificates for TLS - // { - // Name: "serving-cert", - // ReadOnly: true, - // MountPath: "/var/serving-cert", - // }, - { - Name: "config", - ReadOnly: true, - MountPath: "/perses/config", - }, - { - Name: storageName, - ReadOnly: false, - MountPath: "/etc/perses/storage", - }, - }, - Args: []string{"--config=/perses/config/config.yaml"}, + VolumeMounts: common.GetVolumeMounts(perses.Spec.Client.TLS), + Args: common.GetPersesArgs(perses.Spec.Client.TLS, perses.Spec.Args), }}, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: configName, - }, - DefaultMode: ptr.To[int32](420), - }, - }, - }, - // { - // Name: "serving-cert", - // VolumeSource: corev1.VolumeSource{ - // Secret: &corev1.SecretVolumeSource{ - // SecretName: "perses-serving-cert", - // DefaultMode: &[]int32{420}[0], - // }, - // }, - // }, - }, + Volumes: common.GetVolumes(perses.Name, perses.Spec.Client.TLS), RestartPolicy: "Always", DNSPolicy: "ClusterFirst", }, diff --git a/internal/perses/common/perses_args.go b/internal/perses/common/perses_args.go new file mode 100644 index 0000000..a6ed707 --- /dev/null +++ b/internal/perses/common/perses_args.go @@ -0,0 +1,21 @@ +package common + +import ( + "fmt" + "github.com/perses/perses-operator/api/v1alpha1" +) + +func GetPersesArgs(tls *v1alpha1.TLS, args []string) []string { + defaultArgs := []string{"--config=/perses/config/config.yaml"} + + // append tls cert args if user cert and key is provided + if tls != nil && tls.Enable && tls.UserCert != nil { + defaultArgs = append(defaultArgs, fmt.Sprintf("--web.tls-cert-file=/tls/%s", tls.UserCert.CertFile)) + defaultArgs = append(defaultArgs, fmt.Sprintf("--web.tls-key-file=/tls/%s", tls.UserCert.CertKeyFile)) + } + + // append user provided args + defaultArgs = append(defaultArgs, args...) + + return defaultArgs +} diff --git a/internal/perses/common/perses_client_factory.go b/internal/perses/common/perses_client_factory.go index 26c3c69..6a9aa66 100644 --- a/internal/perses/common/perses_client_factory.go +++ b/internal/perses/common/perses_client_factory.go @@ -3,11 +3,15 @@ package common import ( "flag" "fmt" + "path/filepath" + + "github.com/perses/perses/pkg/model/api/v1/secret" - persesv1alpha1 "github.com/perses/perses-operator/api/v1alpha1" v1 "github.com/perses/perses/pkg/client/api/v1" clientConfig "github.com/perses/perses/pkg/client/config" "github.com/perses/perses/pkg/model/api/v1/common" + + persesv1alpha1 "github.com/perses/perses-operator/api/v1alpha1" ) type PersesClientFactory interface { @@ -23,19 +27,40 @@ func NewWithConfig() PersesClientFactory { func (f *PersesClientFactoryWithConfig) CreateClient(perses persesv1alpha1.Perses) (v1.ClientInterface, error) { var urlStr string + var httpProtocol = "http" + if perses.Spec.Client.TLS != nil && perses.Spec.Client.TLS.Enable { + httpProtocol = "https" + } + if flag.Lookup("perses-server-url").Value.String() != "" { urlStr = flag.Lookup("perses-server-url").Value.String() } else { - urlStr = fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", perses.Name, perses.Namespace, perses.Spec.ContainerPort) + urlStr = fmt.Sprintf("%s://%s.%s.svc.cluster.local:%d", httpProtocol, perses.Name, perses.Namespace, perses.Spec.ContainerPort) } parsedURL, err := common.ParseURL(urlStr) if err != nil { return nil, err } - restClient, err := clientConfig.NewRESTClient(clientConfig.RestConfigClient{ + config := clientConfig.RestConfigClient{ URL: parsedURL, - }) + } + + if perses.Spec.Client.TLS != nil && perses.Spec.Client.TLS.Enable { + tlsConfig := &secret.TLSConfig{ + InsecureSkipVerify: perses.Spec.Client.TLS.InsecureSkipVerify, + CAFile: filepath.Join("/ca", perses.Spec.Client.TLS.CaCert.CertFile), + } + + if perses.Spec.Client.TLS.UserCert != nil { + tlsConfig.CertFile = filepath.Join("/tls", perses.Spec.Client.TLS.UserCert.CertFile) + tlsConfig.KeyFile = filepath.Join("/tls", perses.Spec.Client.TLS.UserCert.CertKeyFile) + } + + config.TLSConfig = tlsConfig + } + + restClient, err := clientConfig.NewRESTClient(config) if err != nil { return nil, err } diff --git a/internal/perses/common/volumes.go b/internal/perses/common/volumes.go new file mode 100644 index 0000000..f1632a1 --- /dev/null +++ b/internal/perses/common/volumes.go @@ -0,0 +1,120 @@ +package common + +import ( + "github.com/perses/perses-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +func GetVolumes(name string, tls *v1alpha1.TLS) []corev1.Volume { + configName := GetConfigName(name) + + volumes := []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configName, + }, + DefaultMode: ptr.To[int32](420), + }, + }, + }, + { + Name: "storage", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if tls != nil && tls.Enable { + switch tls.CaCert.Type { + case v1alpha1.CertificateTypeSecret: + volumes = append(volumes, corev1.Volume{ + Name: "ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tls.CaCert.Name, + DefaultMode: &[]int32{420}[0], + }, + }, + }) + case v1alpha1.CertificateTypeConfigMap: + volumes = append(volumes, corev1.Volume{ + Name: "ca", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tls.CaCert.Name, + }, + }, + }, + }) + } + + if tls.UserCert != nil { + switch tls.UserCert.Type { + case v1alpha1.CertificateTypeSecret: + volumes = append(volumes, corev1.Volume{ + Name: "tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tls.UserCert.Name, + DefaultMode: &[]int32{420}[0], + }, + }, + }) + case v1alpha1.CertificateTypeConfigMap: + volumes = append(volumes, corev1.Volume{ + Name: "tls", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tls.UserCert.Name, + }, + DefaultMode: &[]int32{420}[0], + }, + }, + }) + } + } + } + + return volumes +} + +func GetVolumeMounts(tls *v1alpha1.TLS) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config", + ReadOnly: true, + MountPath: "/perses/config", + }, + { + Name: "storage", + ReadOnly: false, + MountPath: "/etc/perses/storage", + }, + } + + if tls != nil && tls.Enable { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "ca", + ReadOnly: true, + MountPath: "/ca", + SubPath: tls.CaCert.CertFile, + }) + + if tls.UserCert != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "tls", + ReadOnly: true, + MountPath: "/tls", + }) + } + } + + return volumeMounts +}