grafana monitoring workspace cross subscription registration

* grafana and monitoring workspaces can now existin different subscriptions * grafana setup extracted to dedicated template (still part of the global pipeline though) * cleanup of unused resources (monitoring msi)
Azure · Feb 25, 2025 · ecf2d10 · ecf2d10
1 parent ae6684d
commit ecf2d10
Show file tree

Hide file tree

Showing 19 changed files with 158 additions and 143 deletions.
diff --git a/config/config.msft.yaml b/config/config.msft.yaml
@@ -189,7 +189,6 @@ defaults:
   # Metrics
   monitoring:
     workspaceName: 'arohcp-{{ .ctx.regionShort }}'
-    msiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}'
 
   # Logs
   logs:

diff --git a/config/config.schema.json b/config/config.schema.json
@@ -799,9 +799,6 @@
           "grafanaName": {
             "type": "string"
           },
-          "msiName": {
-            "type": "string"
-          },
           "workspaceName": {
             "type": "string"
           }
@@ -810,7 +807,6 @@
         "required": [
           "grafanaAdminGroupPrincipalId",
           "grafanaName",
-          "msiName",
           "workspaceName"
         ]
       },

diff --git a/config/config.yaml b/config/config.yaml
@@ -294,7 +294,6 @@ clouds:
       monitoring:
         workspaceName: 'arohcp-{{ .ctx.regionShort }}'
         grafanaName: 'arohcp-dev'
-        msiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}'
         grafanaAdminGroupPrincipalId: 6b6d3adf-8476-4727-9812-20ffdef2b85c
       # Logs
       logs:

diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json
@@ -210,7 +210,6 @@
   "monitoring": {
     "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
     "grafanaName": "arohcp-dev",
-    "msiName": "aro-hcp-metrics-msi-cspr",
     "workspaceName": "arohcp-cspr"
   },
   "msiKeyVault": {

diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json
@@ -210,7 +210,6 @@
   "monitoring": {
     "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
     "grafanaName": "arohcp-dev",
-    "msiName": "aro-hcp-metrics-msi-dev",
     "workspaceName": "arohcp-dev"
   },
   "msiKeyVault": {

diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json
@@ -210,7 +210,6 @@
   "monitoring": {
     "grafanaAdminGroupPrincipalId": "2fdb57d4-3fd3-415d-b604-1d0e37a188fe",
     "grafanaName": "arohcp-int",
-    "msiName": "aro-hcp-metrics-msi-int",
     "workspaceName": "arohcp-int"
   },
   "msiKeyVault": {

diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json
@@ -210,7 +210,6 @@
   "monitoring": {
     "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
     "grafanaName": "arohcp-dev",
-    "msiName": "aro-hcp-metrics-msi-usw3tst",
     "workspaceName": "arohcp-usw3tst"
   },
   "msiKeyVault": {

diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore
@@ -15,6 +15,7 @@ configurations/output-svc.bicepparam
 configurations/mock-identities.bicepparam
 configurations/global-acr.bicepparam
 configurations/global-infra.bicepparam
+configurations/global-grafana.bicepparam
 configurations/output-global.bicepparam
 configurations/output-svc.bicepparam
 config.mk

diff --git a/dev-infrastructure/configurations/global-grafana.tmpl.bicepparam b/dev-infrastructure/configurations/global-grafana.tmpl.bicepparam
@@ -0,0 +1,5 @@
+using '../templates/global-grafana.bicep'
+
+param globalMSIName = '{{ .global.globalMSIName }}'
+param grafanaName = '{{ .monitoring.grafanaName }}'
+param grafanaAdminGroupPrincipalId = '{{ .monitoring.grafanaAdminGroupPrincipalId }}'
diff --git a/dev-infrastructure/configurations/global-infra.tmpl.bicepparam b/dev-infrastructure/configurations/global-infra.tmpl.bicepparam
@@ -3,7 +3,5 @@ using '../templates/global-infra.bicep'
 param globalMSIName = '{{ .global.globalMSIName }}'
 param cxParentZoneName = '{{ .dns.cxParentZoneName }}'
 param svcParentZoneName = '{{ .dns.svcParentZoneName }}'
-param grafanaName = '{{ .monitoring.grafanaName }}'
-param grafanaAdminGroupPrincipalId = '{{ .monitoring.grafanaAdminGroupPrincipalId }}'
 //  SafeDnsIntApplication object ID use to delegate child DNS
 param safeDnsIntAppObjectId = '{{ .global.safeDnsIntAppObjectId }}'
diff --git a/dev-infrastructure/configurations/metrics.tmpl.bicepparam b/dev-infrastructure/configurations/metrics.tmpl.bicepparam
@@ -1,6 +1,4 @@
 using '../modules/metrics/metrics.bicep'
 
 param monitorName = '{{ .monitoring.workspaceName }}'
-param grafanaName = '{{ .monitoring.grafanaName }}'
-param msiName = '{{ .monitoring.msiName }}'
-param globalResourceGroup = '{{ .global.rg }}'
+param grafanaResourceId = '__grafanaResourceId__'
diff --git a/dev-infrastructure/global-pipeline.yaml b/dev-infrastructure/global-pipeline.yaml
@@ -11,6 +11,12 @@ resourceGroups:
     template: templates/global-infra.bicep
     parameters: configurations/global-infra.tmpl.bicepparam
     deploymentLevel: ResourceGroup
+  # creates the global Grafana instance for the ARO HCP services
+  - name: global-grafana
+    action: ARM
+    template: templates/global-grafana.bicep
+    parameters: configurations/global-grafana.tmpl.bicepparam
+    deploymentLevel: ResourceGroup
   # creates DNS delegation for the ARO HCP global SVC zone
   - name: svcChildZone
     action: DelegateChildZone

diff --git a/dev-infrastructure/modules/metrics/metrics.bicep b/dev-infrastructure/modules/metrics/metrics.bicep
@@ -1,25 +1,18 @@
 // this module is only used in dev
-@description('Metrics global resource group name')
-param globalResourceGroup string
 
-@description('Metrics global MSI name')
-param msiName string
+@description('The grafana instance to integrate with')
+param grafanaResourceId string
 
 @description('Metrics regional monitor name')
 param monitorName string
 
-@description('Metrics global Grafana name')
-param grafanaName string
-
 module monitor 'monitor.bicep' = {
   name: 'monitor'
   params: {
-    globalResourceGroup: globalResourceGroup
-    msiName: msiName
+    grafanaResourceId: grafanaResourceId
     monitorName: monitorName
-    grafanaName: grafanaName
   }
 }
 
-output msiId string = monitor.outputs.msiId
 output monitorId string = monitor.outputs.monitorId
+output monitorPrometheusQueryEndpoint string = monitor.outputs.monitorPrometheusQueryEndpoint
diff --git a/dev-infrastructure/modules/metrics/monitor.bicep b/dev-infrastructure/modules/metrics/monitor.bicep
@@ -1,14 +1,12 @@
-@description('Metrics global resource group name')
-param globalResourceGroup string
+@description('The grafana instance to integrate with')
+param grafanaResourceId string
 
-@description('Metrics global MSI name')
-param msiName string
+@description('Metrics region monitor name')
+param monitorName string
 
-@description('Metrics global Grafana name')
-param grafanaName string
+import * as res from '../resource.bicep'
 
-@description('Metrics region monitor name')
-param monitorName string = 'aro-hcp-monitor'
+var grafanaRef = res.grafanaRefFromId(grafanaResourceId)
 
 resource monitor 'microsoft.monitor/accounts@2021-06-03-preview' = {
   name: monitorName
@@ -23,17 +21,13 @@ module defaultRuleGroups 'rules/defaultRecordingRuleGroups.bicep' = {
     regionalResourceGroup: resourceGroup().name
   }
 }
+
 // Assign the Monitoring Data Reader role to the Azure Managed Grafana system-assigned managed identity at the workspace scope
 var dataReader = 'b0d8363b-8ddd-447d-831f-62ca05bff136'
 
-resource msi 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
-  name: msiName
-  scope: resourceGroup(globalResourceGroup)
-}
-
 resource grafana 'Microsoft.Dashboard/grafana@2023-09-01' existing = {
-  name: grafanaName
-  scope: resourceGroup(globalResourceGroup)
+  name: grafanaRef.name
+  scope: resourceGroup(grafanaRef.resourceGroup.subscriptionId, grafanaRef.resourceGroup.name)
 }
 
 resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
@@ -59,5 +53,5 @@ module generatedAlerts 'rules/generatedPrometheusAlertingRules.bicep' = {
   }
 }
 
-output msiId string = msi.id
 output monitorId string = monitor.id
+output monitorPrometheusQueryEndpoint string = monitor.properties.metrics.prometheusQueryEndpoint
diff --git a/dev-infrastructure/modules/resource.bicep b/dev-infrastructure/modules/resource.bicep
@@ -71,3 +71,15 @@ func eventgridNamespaceRefFromId(eventgridNamespaceResourceId string) eventgridN
   resourceGroup: resourceGroupFromResourceId(eventgridNamespaceResourceId)
   name: last(split(eventgridNamespaceResourceId, '/'))
 }
+
+@export()
+type grafanaRef = {
+  resourceGroup: resourceGroupReference
+  name: string
+}
+
+@export()
+func grafanaRefFromId(grafanaResourceId string) grafanaRef => {
+  resourceGroup: resourceGroupFromResourceId(grafanaResourceId)
+  name: last(split(grafanaResourceId, '/'))
+}
diff --git a/dev-infrastructure/region-pipeline.yaml b/dev-infrastructure/region-pipeline.yaml
@@ -52,23 +52,31 @@ resourceGroups:
     template: modules/metrics/metrics.bicep
     parameters: configurations/metrics.tmpl.bicepparam
     deploymentLevel: ResourceGroup
+    variables:
+    - name: grafanaResourceId
+      input:
+        step: global-output
+        name: grafanaResourceId
     dependsOn:
     - region
-# this will not work in case grafana is in another subscription as the workspace
-# - name: {{ .global.rg }}
-#   subscription: {{ .global.subscription }}
-#   steps:
-#   - name: add-grafana-datasource
-#     action: Shell
-#     command: scripts/add-grafana-datasource.sh
-#     variables:
-#       - name: GRAFANA_NAME
-#         configRef: monitoring.grafanaName
-#       - name: GRAFANA_RG
-#         configRef: global.rg
-#       - name: MONITOR_NAME
-#         configRef: monitoring.workspaceName
-#       - name: MONITOR_RG
-#         configRef: regionRG
-#     dependsOn:
-#     - metrics-infra
+- name: {{ .global.rg }}
+  subscription: {{ .global.subscription }}
+  steps:
+  - name: add-grafana-datasource
+    action: Shell
+    command: scripts/add-grafana-datasource.sh
+    variables:
+      - name: GRAFANA_RESOURCE_ID
+        input:
+          step: global-output
+          name: grafanaResourceId
+      - name: PROM_QUERY_URL
+        input:
+          step: metrics-infra
+          name: monitorPrometheusQueryEndpoint
+      - name: MONITOR_ID
+        input:
+          step: metrics-infra
+          name: monitorId
+    dependsOn:
+    - metrics-infra
diff --git a/dev-infrastructure/scripts/add-grafana-datasource.sh b/dev-infrastructure/scripts/add-grafana-datasource.sh
@@ -1,23 +1,37 @@
 #!/bin/bash
 # This script integrates an existing Azure Monitoring Workspace with the global Azure Managed Grafana Instance.
-set -e
-
-MONITOR_DATA_SOURCE=Managed_Prometheus_${MONITOR_NAME}
-
-DATA_SOURCE_URL=$(az grafana data-source list --name ${GRAFANA_NAME} \
-    --resource-group ${GRAFANA_RG} \
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# parse resource IDs
+IFS='/'
+read -ra ADDR <<< "$GRAFANA_RESOURCE_ID"
+GRAFANA_SUBSCRIPTION_ID=${ADDR[2]}
+GRAFANA_RG=${ADDR[4]}
+GRAFANA_NAME=${ADDR[8]}
+read -ra ADDR <<< "$MONITOR_ID"
+MONITOR_DATA_SOURCE_SUBSCRIPTION_ID=${ADDR[2]}
+MONITOR_RG=${ADDR[4]}
+MONITOR_NAME=${ADDR[8]}
+IFS=' '
+
+echo "Grafana subscription id: ${GRAFANA_SUBSCRIPTION_ID}"
+echo "Grafana resource group: ${GRAFANA_RG}"
+echo "Grafana name: ${GRAFANA_NAME}"
+echo "Monitor subscription id: ${MONITOR_DATA_SOURCE_SUBSCRIPTION_ID}"
+echo "Monitor resource group: ${MONITOR_RG}"
+echo "Monitor name: ${MONITOR_NAME}"
+
+# lookup existing azure monitoring workspace registration
+MONITOR_DATA_SOURCE="Managed_Prometheus_${MONITOR_NAME}"
+EXISTING_DATA_SOURCE_URL=$(az grafana data-source list --name ${GRAFANA_NAME} \
+    --resource-group ${GRAFANA_RG} --subscription ${GRAFANA_SUBSCRIPTION_ID} \
     --query "[?contains(name, '${MONITOR_DATA_SOURCE}')].url | [0]" -o tsv)
 
-MONITOR_JSON=$(az monitor account list \
-    --query "[?contains(name, '${MONITOR_NAME}')].{id:id, name: name, promUrl: metrics.prometheusQueryEndpoint}"[0])
-
-PROM_QUERY_URL=$(echo $MONITOR_JSON | jq '.promUrl' -r )
-
-MONITOR_ID=$(echo $MONITOR_JSON | jq '.id' )
-
 # In dev resource groups are purged which causes data sources to become out of sync in the Azure Grafana Instance.
 # If prometheus urls don't match then delete the integration to cleanup the data source.
-if [[ -n "${DATA_SOURCE_URL}" && ${DATA_SOURCE_URL} != ${PROM_QUERY_URL} ]];
+if [[ -n "${EXISTING_DATA_SOURCE_URL}" && ${EXISTING_DATA_SOURCE_URL} != ${PROM_QUERY_URL} ]];
 then
     echo "Removing ${MONITOR_NAME} integration from ${GRAFANA_NAME}"
     az grafana integrations monitor delete \
@@ -26,25 +40,15 @@ then
         --monitor-name ${MONITOR_NAME} \
         --monitor-resource-group-name ${MONITOR_RG} \
         --skip-role-assignment true
-
-    az resource wait --updated --ids $(az grafana show --name ${GRAFANA_NAME} --resource-group ${GRAFANA_RG} --query 'id' -o tsv)
+    az resource wait --updated --ids $GRAFANA_RESOURCE_ID
 fi
 
-MONITORS=$(az grafana integrations monitor list \
-    --name ${GRAFANA_NAME} \
-    --resource-group ${GRAFANA_RG})
-
-IS_INTEGRATED=$(echo ${MONITORS} | jq "contains([${MONITOR_ID}])")
-
+# add the azure monitor workspace to grafana if it is not already integrated
+MONITORS=$(az grafana show --ids ${GRAFANA_RESOURCE_ID} -o json | jq .properties.grafanaIntegrations.azureMonitorWorkspaceIntegrations)
+IS_INTEGRATED=$(echo "$MONITORS" | jq --arg id "${MONITOR_ID}" 'map(.azureMonitorWorkspaceResourceId) | contains([$id])')
 if [[ ${IS_INTEGRATED} == "false" ]];
 then
-    echo "Adding monitor account ${MONITOR_NAME} as a data source to ${GRAFANA_NAME}"
-    az grafana integrations monitor add \
-        --name ${GRAFANA_NAME} \
-        --resource-group ${GRAFANA_RG} \
-        --monitor-name ${MONITOR_NAME} \
-        --monitor-resource-group-name ${MONITOR_RG} \
-        --skip-role-assignments true
-
-    az resource wait --updated --ids $(az grafana show --name ${GRAFANA_NAME} --resource-group ${GRAFANA_RG} --query 'id' -o tsv)
+    MONITOR_UPDATES=$(echo "${MONITORS}" | jq --arg id "${MONITOR_ID}" '. + [{"azureMonitorWorkspaceResourceId": $id}]')
+    az resource update --ids ${GRAFANA_RESOURCE_ID} --set properties.grafanaIntegrations.azureMonitorWorkspaceIntegrations="${MONITOR_UPDATES}"
+    az resource wait --updated --ids $GRAFANA_RESOURCE_ID
 fi
diff --git a/dev-infrastructure/templates/global-grafana.bicep b/dev-infrastructure/templates/global-grafana.bicep
@@ -0,0 +1,58 @@
+@description('The global msi name')
+param globalMSIName string
+
+@description('Metrics global Grafana name')
+param grafanaName string
+
+@description('The admin group principal ID to manage Grafana')
+param grafanaAdminGroupPrincipalId string
+
+resource ev2MSI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
+  name: globalMSIName
+}
+
+// Azure Managed Grafana Workspace Contributor: Can manage Azure Managed Grafana resources, without providing access to the workspaces themselves.
+// https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/monitor#azure-managed-grafana-workspace-contributor
+var grafanaContributor = '5c2d7e57-b7c2-4d8a-be4f-82afa42c6e95'
+
+// Grafana Admin: Perform all Grafana operations, including the ability to manage data sources, create dashboards, and manage role assignments within Grafana.
+// https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/monitor#grafana-admin
+var grafanaAdminRole = '22926164-76b3-42b3-bc55-97df8dab3e41'
+
+var grafanaAdminGroup = {
+  principalId: grafanaAdminGroupPrincipalId
+  principalType: 'group'
+}
+
+resource grafana 'Microsoft.Dashboard/grafana@2023-09-01' = {
+  name: grafanaName
+  location: resourceGroup().location
+  sku: {
+    name: 'Standard'
+  }
+  identity: {
+    type: 'SystemAssigned'
+  }
+}
+
+resource contributorRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(grafana.id, ev2MSI.id, grafanaContributor)
+  scope: grafana
+  properties: {
+    principalId: ev2MSI.properties.principalId
+    principalType: 'ServicePrincipal'
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', grafanaContributor)
+  }
+}
+
+resource adminRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(grafana.id, grafanaAdminGroup.principalId, grafanaAdminRole)
+  scope: grafana
+  properties: {
+    principalId: grafanaAdminGroup.principalId
+    principalType: grafanaAdminGroup.principalType
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', grafanaAdminRole)
+  }
+}
+
+output grafanaId string = grafana.id