parameterize maestro server k8s, DB and MI settings (#1092)

Signed-off-by: Gerd Oberlechner <goberlec@redhat.com>
Azure · Jan 14, 2025 · 5a63476 · 5a63476
1 parent ae89b0c
commit 5a63476
Show file tree

Hide file tree

Showing 14 changed files with 140 additions and 34 deletions.
diff --git a/config/config.msft.yaml b/config/config.msft.yaml
@@ -56,6 +56,12 @@ defaults:
 
   # Maestro
   maestro:
+    server:
+      mqttClientName: maestro-server
+      managedIdentityName: maestro-server
+      k8s:
+        namespace: maestro
+        serviceAccountName: maestro
     eventGrid:
       name: arohcp-maestro-{{ .ctx.regionShort }}
       maxClientSessionsPerAuthName: 4
@@ -68,9 +74,9 @@ defaults:
       deploy: false
       private: false
       minTLSVersion: 'TLSV1.2'
+      databaseName: maestro
     restrictIstioIngress: true
     consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
-    serverMqttClientName: maestro-server
     imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro
 
   # Cluster Service

diff --git a/config/config.schema.json b/config/config.schema.json
@@ -381,8 +381,38 @@
           "consumerName": {
             "type": "string"
           },
-          "serverMqttClientName": {
-            "type": "string"
+          "server": {
+            "type": "object",
+            "properties": {
+              "mqttClientName": {
+                "type": "string"
+              },
+              "managedIdentityName": {
+                "type": "string"
+              },
+              "k8s": {
+                "type": "object",
+                "properties": {
+                  "namespace": {
+                    "type": "string"
+                  },
+                  "serviceAccountName": {
+                    "type": "string"
+                  }
+                },
+                "additionalProperties": false,
+                "required": [
+                  "namespace",
+                  "serviceAccountName"
+                ]
+              }
+            },
+            "additionalProperties": false,
+            "required": [
+              "mqttClientName",
+              "managedIdentityName",
+              "k8s"
+            ]
           },
           "eventGrid": {
             "type": "object",
@@ -431,6 +461,9 @@
               "minTLSVersion": {
                 "type": "string",
                 "enum": ["TLSV1.2", "TLSV1.3"]
+              },
+              "databaseName": {
+                "type": "string"
               }
             },
             "additionalProperties": false,
@@ -440,7 +473,8 @@
               "private",
               "serverStorageSizeGB",
               "serverVersion",
-              "minTLSVersion"
+              "minTLSVersion",
+              "databaseName"
             ]
           },
           "restrictIstioIngress": {
@@ -451,7 +485,7 @@
         "required": [
           "certDomain",
           "consumerName",
-          "serverMqttClientName",
+          "server",
           "eventGrid",
           "imageBase",
           "imageTag",

diff --git a/config/config.yaml b/config/config.yaml
@@ -56,6 +56,12 @@ defaults:
 
   # Maestro
   maestro:
+    server:
+      mqttClientName: maestro-server
+      managedIdentityName: maestro-server
+      k8s:
+        namespace: maestro
+        serviceAccountName: maestro
     eventGrid:
       name: arohcp-maestro-{{ .ctx.regionShort }}
       maxClientSessionsPerAuthName: 4
@@ -68,8 +74,8 @@ defaults:
       deploy: true
       private: false
       minTLSVersion: 'TLSV1.2'
+      databaseName: maestro
     restrictIstioIngress: true
-    serverMqttClientName: maestro-server
     consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
     imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro
 
@@ -249,7 +255,8 @@ clouds:
           regionalDNSSubdomain: '{{ .ctx.region }}'
           # Maestro
           maestro:
-            serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-dev'
+            server:
+              mqttClientName: 'maestro-server-{{ .ctx.regionShort }}-dev'
           # Frontend
           frontend:
             cosmosDB:
@@ -273,7 +280,8 @@ clouds:
           # Maestro
           maestro:
             restrictIstioIngress: false
-            serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-cs'
+            server:
+              mqttClientName: 'maestro-server-{{ .ctx.regionShort }}-cs'
           # Frontend
           frontend:
             cosmosDB:
@@ -291,7 +299,8 @@ clouds:
           maestro:
             postgres:
               deploy: false
-            serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}'
+            server:
+              mqttClientName: 'maestro-server-{{ .ctx.regionShort }}'
           # Frontend
           frontend:
             cosmosDB:

diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json
@@ -109,6 +109,7 @@
     "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
     "imageTag": "c9a36e110a32c0c25aa5025cfe6d51af797e6d4b",
     "postgres": {
+      "databaseName": "maestro",
       "deploy": true,
       "minTLSVersion": "TLSV1.2",
       "name": "arohcp-maestro-cspr",
@@ -117,7 +118,14 @@
       "serverVersion": "15"
     },
     "restrictIstioIngress": false,
-    "serverMqttClientName": "maestro-server-cspr-cs"
+    "server": {
+      "k8s": {
+        "namespace": "maestro",
+        "serviceAccountName": "maestro"
+      },
+      "managedIdentityName": "maestro-server",
+      "mqttClientName": "maestro-server-cspr-cs"
+    }
   },
   "mce": {
     "clcStateMetrics": {

diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json
@@ -109,6 +109,7 @@
     "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
     "imageTag": "c9a36e110a32c0c25aa5025cfe6d51af797e6d4b",
     "postgres": {
+      "databaseName": "maestro",
       "deploy": true,
       "minTLSVersion": "TLSV1.2",
       "name": "arohcp-maestro-dev",
@@ -117,7 +118,14 @@
       "serverVersion": "15"
     },
     "restrictIstioIngress": true,
-    "serverMqttClientName": "maestro-server-dev-dev"
+    "server": {
+      "k8s": {
+        "namespace": "maestro",
+        "serviceAccountName": "maestro"
+      },
+      "managedIdentityName": "maestro-server",
+      "mqttClientName": "maestro-server-dev-dev"
+    }
   },
   "mce": {
     "clcStateMetrics": {

diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json
@@ -109,6 +109,7 @@
     "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
     "imageTag": "c9a36e110a32c0c25aa5025cfe6d51af797e6d4b",
     "postgres": {
+      "databaseName": "maestro",
       "deploy": false,
       "minTLSVersion": "TLSV1.2",
       "name": "arohcp-maestro-int",
@@ -117,7 +118,14 @@
       "serverVersion": "15"
     },
     "restrictIstioIngress": true,
-    "serverMqttClientName": "maestro-server"
+    "server": {
+      "k8s": {
+        "namespace": "maestro",
+        "serviceAccountName": "maestro"
+      },
+      "managedIdentityName": "maestro-server",
+      "mqttClientName": "maestro-server"
+    }
   },
   "mgmt": {
     "clusterServiceResourceId": "todo",

diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json
@@ -109,6 +109,7 @@
     "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
     "imageTag": "c9a36e110a32c0c25aa5025cfe6d51af797e6d4b",
     "postgres": {
+      "databaseName": "maestro",
       "deploy": false,
       "minTLSVersion": "TLSV1.2",
       "name": "arohcp-maestro-usw3tst",
@@ -117,7 +118,14 @@
       "serverVersion": "15"
     },
     "restrictIstioIngress": true,
-    "serverMqttClientName": "maestro-server-usw3tst"
+    "server": {
+      "k8s": {
+        "namespace": "maestro",
+        "serviceAccountName": "maestro"
+      },
+      "managedIdentityName": "maestro-server",
+      "mqttClientName": "maestro-server-usw3tst"
+    }
   },
   "mce": {
     "clcStateMetrics": {

diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam
@@ -20,13 +20,17 @@ param deployFrontendCosmos = {{ .frontend.cosmosDB.deploy }}
 param rpCosmosDbName = '{{ .frontend.cosmosDB.name }}'
 param rpCosmosDbPrivate = {{ .frontend.cosmosDB.private }}
 
+param maestroMIName = '{{ .maestro.server.managedIdentityName }}'
+param maestroNamespace = '{{ .maestro.server.k8s.namespace }}'
+param maestroServiceAccountName = '{{ .maestro.server.k8s.serviceAccountName }}'
 param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
-param maestroServerMqttClientName = '{{ .maestro.serverMqttClientName }}'
+param maestroServerMqttClientName = '{{ .maestro.server.mqttClientName }}'
 param maestroCertDomain = '{{ .maestro.certDomain}}'
 param maestroPostgresServerName = '{{ .maestro.postgres.name }}'
 param maestroPostgresServerMinTLSVersion = '{{ .maestro.postgres.minTLSVersion }}'
 param maestroPostgresServerVersion = '{{ .maestro.postgres.serverVersion }}'
 param maestroPostgresServerStorageSizeGB = {{ .maestro.postgres.serverStorageSizeGB }}
+param maestroPostgresDatabaseName = '{{ .maestro.postgres.databaseName }}'
 param deployMaestroPostgres = {{ .maestro.postgres.deploy }}
 param maestroPostgresPrivate = {{ .maestro.postgres.private }}
 

diff --git a/dev-infrastructure/modules/maestro/maestro-server.bicep b/dev-infrastructure/modules/maestro/maestro-server.bicep
@@ -52,7 +52,7 @@ param privateEndpointSubnetId string = ''
 param privateEndpointVnetId string = ''
 
 @description('The name of the database to create for Maestro')
-param maestroDatabaseName string = 'maestro'
+param maestroDatabaseName string
 
 @description('The name of the Managed Identity for the Maestro cluster service')
 param maestroServerManagedIdentityName string

diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep
@@ -107,9 +107,21 @@ param maestroPostgresServerMinTLSVersion string
 @description('The size of the Postgres server for Maestro')
 param maestroPostgresServerStorageSizeGB int
 
+@description('The name of the Maestro Postgres database')
+param maestroPostgresDatabaseName string
+
 @description('The name of Maestro Server MQTT client')
 param maestroServerMqttClientName string
 
+@description('The name of the maestro managed identity')
+param maestroMIName string
+
+@description('The namespace of the maestro managed identity')
+param maestroNamespace string
+
+@description('The service account name of the maestro managed identity')
+param maestroServiceAccountName string
+
 @description('The name of the service keyvault')
 param serviceKeyVaultName string
 
@@ -184,9 +196,9 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
         serviceAccountName: 'backend'
       }
       maestro_wi: {
-        uamiName: 'maestro-server'
-        namespace: 'maestro'
-        serviceAccountName: 'maestro'
+        uamiName: maestroMIName
+        namespace: maestroNamespace
+        serviceAccountName: maestroServiceAccountName
       }
       cs_wi: {
         uamiName: clusterServiceMIName
@@ -263,16 +275,14 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = {
     postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB
     privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId
     privateEndpointVnetId: svcCluster.outputs.aksVnetId
+    maestroDatabaseName: maestroPostgresDatabaseName
     postgresServerPrivate: maestroPostgresPrivate
     postgresAdministrationManagedIdentityId: aroDevopsMsiId
     maestroServerManagedIdentityPrincipalId: filter(
       svcCluster.outputs.userAssignedIdentities,
-      id => id.uamiName == 'maestro-server'
+      id => id.uamiName == maestroMIName
     )[0].uamiPrincipalID
-    maestroServerManagedIdentityName: filter(
-      svcCluster.outputs.userAssignedIdentities,
-      id => id.uamiName == 'maestro-server'
-    )[0].uamiName
+    maestroServerManagedIdentityName: maestroMIName
   }
   dependsOn: [
     serviceKeyVault

diff --git a/maestro/registration/Makefile b/maestro/registration/Makefile
@@ -6,11 +6,11 @@ include config.mk
 HELM_CMD ?= helm upgrade --install
 
 deploy:
-	@if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \
-		echo "Error: Service 'maestro' not found in namespace 'maestro'"; \
+	@if ! kubectl get service maestro -n ${NAMESPACE} > /dev/null 2>&1; then \
+		echo "Error: Service 'maestro' not found in namespace '${NAMESPACE}'"; \
 		exit 1; \
 	fi
 	${HELM_CMD} ${CONSUMER_NAME} ./helm \
-		--namespace maestro \
+		--namespace ${NAMESPACE} \
 		--set consumerName=${CONSUMER_NAME}
 .PHONY: deploy
diff --git a/maestro/registration/config.tmpl.mk b/maestro/registration/config.tmpl.mk
@@ -1 +1,2 @@
 CONSUMER_NAME ?= {{ .maestro.consumerName }}
+NAMESPACE ?= {{ .maestro.server.k8s.namespace }}
diff --git a/maestro/server/Makefile b/maestro/server/Makefile
@@ -3,16 +3,17 @@
 HELM_CMD ?= helm upgrade --install
 
 deploy:
-	@kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f -
-	@kubectl label ${KUBECTL_DRY_RUN} namespace maestro "istio.io/rev=${ISTO_TAG}" --overwrite=true
+	@kubectl create namespace ${NAMESPACE} --dry-run=client -o json | kubectl apply -f -
+	@kubectl label ${KUBECTL_DRY_RUN} namespace ${NAMESPACE} "istio.io/rev=${ISTO_TAG}" --overwrite=true
 	@EVENTGRID_HOSTNAME=$(shell az resource show -n ${EVENTGRID_NAME} -g ${REGION_RG} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) && \
 	TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
-	MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n maestro-server --query clientId -o tsv) && \
-	DATABASE_HOST=$$(if [ "${USE_AZURE_DB}" = "true" ]; then az postgres flexible-server show -g ${SVC_RG} -n ${DATABASE_NAME} --query fullyQualifiedDomainName -o tsv; else echo "maestro-db"; fi) && \
+	MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n ${MANAGED_IDENTITY_NAME} --query clientId -o tsv) && \
+	DATABASE_HOST=$$(if [ "${USE_AZURE_DB}" = "true" ]; then az postgres flexible-server show -g ${SVC_RG} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv; else echo "maestro-db"; fi) && \
 	OVERRIDES=$$(if [ "${USE_AZURE_DB}" = "true" ]; then echo "azuredb.values.yaml"; else echo "containerdb.values.yaml"; fi) && \
 	${HELM_CMD} maestro-server ./helm \
-		--namespace maestro \
+		--namespace ${NAMESPACE} \
 		-f helm/$${OVERRIDES} \
+		--set maestro.serviceAccount=${SERVICE_ACCOUNT_NAME} \
 		--set broker.host=$${EVENTGRID_HOSTNAME} \
 		--set credsKeyVault.name=${KEYVAULT_NAME} \
 		--set credsKeyVault.secret=${MQTT_CLIENT_NAME} \
@@ -21,5 +22,6 @@ deploy:
 		--set istio.restrictIngress=${ISTIO_RESTRICT_INGRESS} \
 		--set image.base=${IMAGE_BASE} \
 		--set image.tag=${IMAGE_TAG} \
-		--set database.host=$${DATABASE_HOST}
+		--set database.host=$${DATABASE_HOST} \
+		--set database.name=${DATABASE_NAME}
 .PHONY: deploy
diff --git a/maestro/server/pipeline.yaml b/maestro/server/pipeline.yaml
@@ -28,13 +28,21 @@ resourceGroups:
       configRef: maestro.imageTag
     - name: USE_AZURE_DB
       configRef: maestro.postgres.deploy
-    - name: DATABASE_NAME
+    - name: DATABASE_SERVER_NAME
       configRef: maestro.postgres.name
+    - name: DATABASE_NAME
+      configRef: maestro.postgres.databaseName
     - name: ISTIO_RESTRICT_INGRESS
       configRef: maestro.restrictIstioIngress
     - name: KEYVAULT_NAME
       configRef: serviceKeyVault.name
     - name: MQTT_CLIENT_NAME
-      configRef: maestro.serverMqttClientName
+      configRef: maestro.server.mqttClientName
     - name: ISTO_TAG
       configRef: svc.istio.tag
+    - name: NAMESPACE
+      configRef: maestro.server.k8s.namespace
+    - name: SERVICE_ACCOUNT_NAME
+      configRef: maestro.server.k8s.serviceAccountName
+    - name: MANAGED_IDENTITY_NAME
+      configRef: maestro.server.managedIdentityName
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		CONSUMER_NAME ?= {{ .maestro.consumerName }}
		NAMESPACE ?= {{ .maestro.server.k8s.namespace }}