From 330b93e217126a984dfbd8101169aba3ab1bf644 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Tue, 24 Sep 2024 16:28:37 -0400 Subject: [PATCH 1/4] Added Agent Server and Fluent-bit client certificates to implement mtls on agent endpoint (#106) * added server and client certificates to implement mtls on agent endpoint --- .../templates/certmanager.yaml | 67 +++++++++++++++++++ .../linux/cloudwatch-agent-daemonset.yaml | 47 +++++++++++++ .../templates/linux/fluent-bit-daemonset.yaml | 20 ++++++ .../values.yaml | 5 ++ 4 files changed, 139 insertions(+) diff --git a/charts/amazon-cloudwatch-observability/templates/certmanager.yaml b/charts/amazon-cloudwatch-observability/templates/certmanager.yaml index 50b8ecb..a8a1a04 100644 --- a/charts/amazon-cloudwatch-observability/templates/certmanager.yaml +++ b/charts/amazon-cloudwatch-observability/templates/certmanager.yaml @@ -63,6 +63,57 @@ spec: kind: Issuer name: "agent-ca" secretName: "amazon-cloudwatch-observability-agent-cert" +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }} + name: "amazon-cloudwatch-observability-agent-server-cert" + namespace: {{ .Release.Namespace }} +spec: + commonName: "agent-server" + dnsNames: + - "cloudwatch-agent" + - "cloudwatch-agent.amazon-cloudwatch.svc" + issuerRef: + kind: Issuer + name: "agent-ca" + secretName: "amazon-cloudwatch-observability-agent-server-cert" + usages: + - digital signature + - key encipherment + - cert sign + keyUsages: + critical: true + usages: + - digitalSignature + - keyEncipherment + - certSign +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }} + name: "amazon-cloudwatch-observability-agent-client-cert" + namespace: {{ .Release.Namespace }} +spec: + commonName: "agent-client" + issuerRef: + kind: Issuer + name: "agent-ca" + secretName: "amazon-cloudwatch-observability-agent-client-cert" + usages: + - digital signature + - key encipherment + - cert sign + keyUsages: + critical: true + usages: + - digitalSignature + - keyEncipherment + - certSign {{- if not .Values.agent.certManager.issuerRef }} --- apiVersion: cert-manager.io/v1 @@ -87,5 +138,21 @@ metadata: {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }} name: "amazon-cloudwatch-observability-agent-cert" namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }} + name: "amazon-cloudwatch-observability-agent-server-cert" + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4 }} + name: "amazon-cloudwatch-observability-agent-client-cert" + namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml b/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml index 693cecb..4a68ccc 100644 --- a/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml +++ b/charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-daemonset.yaml @@ -1,8 +1,11 @@ {{- if .Values.agent.enabled }} {{- if and (.Values.agent.autoGenerateCert.enabled) (not .Values.agent.certManager.enabled) -}} {{- $altNames := list ( printf "%s-service" (include "dcgm-exporter.name" .) ) ( printf "%s-service" (include "neuron-monitor.name" .) ) ( printf "%s-service.%s.svc" (include "dcgm-exporter.name" .) .Release.Namespace ) ( printf "%s-service.%s.svc" (include "neuron-monitor.name" .) .Release.Namespace ) -}} +{{- $agentAltNames := list ( printf "%s" (include "cloudwatch-agent.name" .) ) ( printf "%s.%s.svc" (include "cloudwatch-agent.name" .) .Release.Namespace ) -}} {{- $ca := genCA ("agent-ca") ( .Values.agent.autoGenerateCert.expiryDays | int ) -}} {{- $cert := genSignedCert ("agent") nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}} +{{- $serverCert := genSignedCert ("agent-server") nil $agentAltNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}} +{{- $clientCert := genSignedCert ("agent-client") nil nil ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}} apiVersion: v1 kind: Secret metadata: @@ -15,6 +18,30 @@ data: tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} --- +apiVersion: v1 +kind: Secret +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}} + name: "amazon-cloudwatch-observability-agent-server-cert" + namespace: {{ .Release.Namespace }} +data: + ca.crt: {{ $ca.Cert | b64enc }} + tls.crt: {{ $serverCert.Cert | b64enc }} + tls.key: {{ $serverCert.Key | b64enc }} +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}} + name: "amazon-cloudwatch-observability-agent-client-cert" + namespace: {{ .Release.Namespace }} +data: + ca.crt: {{ $ca.Cert | b64enc }} + tls.crt: {{ $clientCert.Cert | b64enc }} + tls.key: {{ $clientCert.Key | b64enc }} +--- {{- end -}} {{- $clusterName := .Values.clusterName | required ".Values.clusterName is required." -}} @@ -69,6 +96,12 @@ spec: - mountPath: /etc/amazon-cloudwatch-observability-agent-cert name: agenttls readOnly: true + - mountPath: /etc/amazon-cloudwatch-observability-agent-client-cert + name: agentclienttls + readOnly: true + - mountPath: /etc/amazon-cloudwatch-observability-agent-server-cert + name: agentservertls + readOnly: true - mountPath: /var/lib/kubelet/pod-resources name: kubelet-podresources volumes: @@ -100,6 +133,20 @@ spec: items: - key: ca.crt path: tls-ca.crt + - name: agentclienttls + secret: + secretName: amazon-cloudwatch-observability-agent-client-cert + items: + - key: ca.crt + path: tls-ca.crt + - name: agentservertls + secret: + secretName: amazon-cloudwatch-observability-agent-server-cert + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key env: - name: K8S_NODE_NAME valueFrom: diff --git a/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml b/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml index 5d1eb54..7aff77f 100644 --- a/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml +++ b/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-daemonset.yaml @@ -68,6 +68,12 @@ spec: - name: dmesg mountPath: /var/log/dmesg readOnly: true + - mountPath: /etc/amazon-cloudwatch-observability-agent-client-cert + name: agentclienttls + readOnly: true + - mountPath: /etc/amazon-cloudwatch-observability-agent-server-cert + name: agentservertls + readOnly: true terminationGracePeriodSeconds: 10 hostNetwork: true dnsPolicy: ClusterFirstWithHostNet @@ -90,6 +96,20 @@ spec: - name: dmesg hostPath: path: /var/log/dmesg + - name: agentclienttls + secret: + secretName: amazon-cloudwatch-observability-agent-client-cert + items: + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + - name: agentservertls + secret: + secretName: amazon-cloudwatch-observability-agent-server-cert + items: + - key: ca.crt + path: tls-ca.crt serviceAccountName: {{ template "cloudwatch-agent.serviceAccountName" . }} affinity: nodeAffinity: diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml index d3954c6..13b9a96 100644 --- a/charts/amazon-cloudwatch-observability/values.yaml +++ b/charts/amazon-cloudwatch-observability/values.yaml @@ -118,6 +118,10 @@ containerLogs: Refresh_Interval 10 Read_from_Head ${READ_FROM_HEAD} + [FILTER] + Name aws + Match application.* + [FILTER] Name kubernetes Match application.* @@ -132,6 +136,7 @@ containerLogs: Use_Kubelet On Kubelet_Port 10250 Buffer_Size 0 + Use_Pod_Association On [OUTPUT] Name cloudwatch_logs From d40aca103930f0e21d9d7de6c90dbc3910573ba6 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Mon, 28 Oct 2024 14:52:44 -0400 Subject: [PATCH 2/4] added latest fluent-bit config for application logs files to support sending entity (#118) --- charts/amazon-cloudwatch-observability/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml index 13b9a96..78afe0f 100644 --- a/charts/amazon-cloudwatch-observability/values.yaml +++ b/charts/amazon-cloudwatch-observability/values.yaml @@ -121,6 +121,8 @@ containerLogs: [FILTER] Name aws Match application.* + az false + Enable_Entity true [FILTER] Name kubernetes @@ -146,6 +148,7 @@ containerLogs: log_stream_prefix ${HOST_NAME}- auto_create_group true extra_user_agent container-insights + add_entity true dataplane-log.conf: | [INPUT] Name systemd From f981807c550c1a1d3624e950846c99793c0033b3 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Thu, 7 Nov 2024 11:13:30 -0500 Subject: [PATCH 3/4] added flag to retrieve instance id behind entity flag in aws filter plugin for application logs (#122) --- charts/amazon-cloudwatch-observability/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml index 78afe0f..ec7028a 100644 --- a/charts/amazon-cloudwatch-observability/values.yaml +++ b/charts/amazon-cloudwatch-observability/values.yaml @@ -122,6 +122,7 @@ containerLogs: Name aws Match application.* az false + ec2_instance_id false Enable_Entity true [FILTER] From b3b00292e55c4d8db276a7c08c35917bda6cee65 Mon Sep 17 00:00:00 2001 From: lisguo Date: Thu, 7 Nov 2024 12:25:28 -0500 Subject: [PATCH 4/4] Increment fluentbit version for linux --- charts/amazon-cloudwatch-observability/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml index ec7028a..f99b35b 100644 --- a/charts/amazon-cloudwatch-observability/values.yaml +++ b/charts/amazon-cloudwatch-observability/values.yaml @@ -32,7 +32,7 @@ containerLogs: fluentBit: image: repository: aws-for-fluent-bit - tag: 2.32.2.20240627 + tag: 2.32.4 tagWindows: 2.31.12-windowsservercore repositoryDomainMap: public: public.ecr.aws/aws-observability