Support & enable audit logging for Key Vault and AKS for dev & cspr environments

config/config.msft.yaml

@@ -203,6 +203,7 @@ defaults:
 
   # Logs
   logs:
+    enableLogAnalytics: false
     namespace: logs
     msiName: logs-mdsd
     serviceAccountName: genevabit-aggregator

config/config.schema.json

@@ -820,6 +820,9 @@
     "logs": {
       "type": "object",
       "properties": {
+        "enableLogAnalytics": {
+          "type": "boolean"
+        },
         "namespace": {
           "type": "string"
         },
@@ -832,6 +835,7 @@
       },
       "additionalProperties": false,
       "required": [
+        "enableLogAnalytics",
         "namespace",
         "msiName",
         "serviceAccountName"
@@ -995,4 +999,4 @@
     "svcAcrName",
     "svcAcrZoneRedundantMode"
   ]
-}
+}

config/config.yaml

@@ -22,6 +22,9 @@ defaults:
     namespace: hypershift
     additionalInstallArg: '--tech-preview-no-upgrade'
 
+  logs:
+    enableLogAnalytics: false
+
   # SVC cluster specifics
   svc:
     subscription: ARO Hosted Control Planes (EA Subscription 1)
@@ -324,6 +327,8 @@ clouds:
       dev:
         # this is the integrated DEV environment
         defaults:
+          logs:
+            enableLogAnalytics: true
           mgmt:
             aks:
               systemAgentPool:
@@ -353,6 +358,8 @@ clouds:
       cs-pr:
         # this is the cluster service PR check and full cycle test environment
         defaults:
+          logs:
+            enableLogAnalytics: true
           svc:
             aks:
               # MC AKS nodepools

config/public-cloud-cs-pr.json

@@ -142,6 +142,7 @@
   },
   "kvCertOfficerPrincipalId": "c9b1819d-bb29-4ac2-9abe-39e4fe9b59eb",
   "logs": {
+    "enableLogAnalytics": true,
     "msiName": "logs-mdsd",
     "namespace": "logs",
     "serviceAccountName": "genevabit-aggregator"

config/public-cloud-dev.json

@@ -142,6 +142,7 @@
   },
   "kvCertOfficerPrincipalId": "c9b1819d-bb29-4ac2-9abe-39e4fe9b59eb",
   "logs": {
+    "enableLogAnalytics": true,
     "msiName": "logs-mdsd",
     "namespace": "logs",
     "serviceAccountName": "genevabit-aggregator"

config/public-cloud-msft-int.json

@@ -142,6 +142,7 @@
   },
   "kvCertOfficerPrincipalId": "32af88de-a61c-4f71-b709-50538598c4f2",
   "logs": {
+    "enableLogAnalytics": false,
     "msiName": "logs-mdsd",
     "namespace": "logs",
     "serviceAccountName": "genevabit-aggregator"

config/public-cloud-personal-dev.json

@@ -142,6 +142,7 @@
   },
   "kvCertOfficerPrincipalId": "c9b1819d-bb29-4ac2-9abe-39e4fe9b59eb",
   "logs": {
+    "enableLogAnalytics": false,
     "msiName": "logs-mdsd",
     "namespace": "logs",
     "serviceAccountName": "genevabit-aggregator"

dev-infrastructure/configurations/global-infra.tmpl.bicepparam

@@ -9,3 +9,5 @@ param safeDnsIntAppObjectId = '{{ .global.safeDnsIntAppObjectId }}'
 param grafanaName = '{{ .monitoring.grafanaName }}'
 param grafanaAdminGroupPrincipalId = '{{ .monitoring.grafanaAdminGroupPrincipalId }}'
 param grafanaZoneRedundantMode = '{{ .monitoring.grafanaZoneRedundantMode }}'
+
+param enableLogAnalytics = {{ .logs.enableLogAnalytics }}

dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam

@@ -51,3 +51,6 @@ param azureMonitoringWorkspaceId = '__azureMonitoringWorkspaceId__'
 param logsNamespace = '{{ .logs.namespace }}'
 param logsMSI = '{{ .logs.msiName }}'
 param logsServiceAccount = '{{ .logs.serviceAccountName }}'
+
+// Log Analytics Workspace ID will be passed from global pipeline if enabled in config
+param logAnalyticsWorkspaceId = '__logAnalyticsWorkspaceId__'

dev-infrastructure/configurations/mgmt-infra.tmpl.bicepparam

@@ -24,3 +24,6 @@ param aroDevopsMsiId = '{{ .aroDevopsMsiId }}'
 // Cluster Service identity
 // used for Key Vault access
 param clusterServiceMIResourceId = '__clusterServiceMIResourceId__'
+
+// Log Analytics Workspace ID will be passed from global pipeline if enabled in config
+param logAnalyticsWorkspaceId = '__logAnalyticsWorkspaceId__'

dev-infrastructure/configurations/output-global.tmpl.bicepparam

@@ -5,3 +5,4 @@ param ocpAcrName = '{{ .ocpAcrName }}'
 param cxParentZoneName = '{{ .dns.cxParentZoneName }}'
 param svcParentZoneName = '{{ .dns.svcParentZoneName }}'
 param grafanaName = '{{ .monitoring.grafanaName }}'
+param enableLogAnalytics = {{ .logs.enableLogAnalytics }}

dev-infrastructure/configurations/region.tmpl.bicepparam

@@ -21,3 +21,6 @@ param maestroCertificateIssuer = '{{ .maestro.certIssuer }}'
 
 // MI for resource access during pipeline runs
 param aroDevopsMsiId = '{{ .aroDevopsMsiId }}'
+
+// Log Analytics Workspace ID will be passed from region pipeline if enabled in config
+param logAnalyticsWorkspaceId = '__logAnalyticsWorkspaceId__'

dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam

@@ -81,3 +81,6 @@ param azureMonitoringWorkspaceId = '__azureMonitoringWorkspaceId__'
 param logsNamespace = '{{ .logs.namespace }}'
 param logsMSI = '{{ .logs.msiName }}'
 param logsServiceAccount = '{{ .logs.serviceAccountName }}'
+
+// Log Analytics Workspace ID will be passed from global pipeline if enabled in config
+param logAnalyticsWorkspaceId = '__logAnalyticsWorkspaceId__'

dev-infrastructure/configurations/svc-infra.tmpl.bicepparam

@@ -11,3 +11,6 @@ param aroDevopsMsiId = '{{ .aroDevopsMsiId }}'
 
 // SP for KV certificate issuer registration
 param kvCertOfficerPrincipalId = '{{ .kvCertOfficerPrincipalId }}'
+
+// Log Analytics Workspace ID will be passed from global pipeline if enabled in config
+param logAnalyticsWorkspaceId = '__logAnalyticsWorkspaceId__'

dev-infrastructure/mgmt-pipeline.yaml

@@ -42,7 +42,12 @@ resourceGroups:
       input:
         step: svc-output
         name: cs
+    - name: logAnalyticsWorkspaceId
+      input:
+        step: global-output
+        name: logAnalyticsWorkspaceId
     dependsOn:
+    - global-output
     - svc-output
   # Configure certificate issuers for the MC KVs
   - name: cx-oncert-public-kv-issuer
@@ -98,6 +103,10 @@ resourceGroups:
       input:
         step: region-output
         name: maestroEventGridNamespaceId
+    - name: logAnalyticsWorkspaceId
+      input:
+        step: global-output
+        name: logAnalyticsWorkspaceId
     dependsOn:
     - cx-oncert-public-kv-issuer
     - mgmt-oncert-private-kv-issuer

dev-infrastructure/modules/aks-cluster-base.bicep

@@ -62,6 +62,8 @@ var istioIngressGatewayIPAddressIPTagsArray = [
 @maxLength(24)
 param aksKeyVaultName string
 
+param logAnalyticsWorkspaceId string = ''
+
 // Local Params
 @description('Optional DNS prefix to use with hosted Kubernetes API server FQDN.')
 param dnsPrefix string = aksClusterName
@@ -460,6 +462,82 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
   ]
 }
 
+resource aksDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = if (logAnalyticsWorkspaceId != '') {
+  scope: aksCluster
+  name: aksClusterName
+  properties: {
+    logs: [
+      {
+        category: 'kube-audit'
+        enabled: true
+      }
+      {
+        category: 'kube-audit-admin'
+        enabled: true
+      }
+    ]
+    workspaceId: logAnalyticsWorkspaceId
+  }
+}
+
+resource aksClusterDcr 'Microsoft.Insights/dataCollectionRules@2023-03-11' = if (logAnalyticsWorkspaceId != '') {
+  name: '${aksClusterName}-dcr'
+  location: location
+  kind: 'Linux'
+  properties: {
+    dataSources: {
+      extensions: [
+        {
+          name: 'ContainerInsightsExtension'
+          streams: [
+            'Microsoft-ContainerLog'
+            'Microsoft-ContainerLogV2'
+            'Microsoft-KubeEvents'
+            'Microsoft-KubePodInventory'
+          ]
+          extensionSettings: {
+            dataCollectionSettings: {
+              interval: '1m'
+              namespaceFilteringMode: 'Off'
+              enableContainerLogV2: true
+            }
+          }
+          extensionName: 'ContainerInsights'
+        }
+      ]
+    }
+    destinations: {
+      logAnalytics: [
+        {
+          name: 'ContainerInsightsWorkspace'
+          workspaceResourceId: logAnalyticsWorkspaceId
+        }
+      ]
+    }
+    dataFlows: [
+      {
+        destinations: [
+          'ContainerInsightsWorkspace'
+        ]
+        streams: [
+          'Microsoft-ContainerLog'
+          'Microsoft-ContainerLogV2'
+          'Microsoft-KubeEvents'
+        ]
+      }
+    ]
+  }
+}
+
+resource aksClusterDcra 'Microsoft.Insights/dataCollectionRuleAssociations@2023-03-11' = if (logAnalyticsWorkspaceId != '') {
+  name: '${aksClusterName}-dcra'
+  scope: aksCluster
+  properties: {
+    description: 'Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster.'
+    dataCollectionRuleId: aksClusterDcr.id
+  }
+}
+
 resource userAgentPools 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = [
   for i in range(0, userAgentPoolAZCount): {
     parent: aksCluster

dev-infrastructure/modules/keyvault/keyvault.bicep

@@ -13,6 +13,9 @@ param private bool
 @description('Purpose of the keyvault.')
 param purpose string
 
+@description('Log Analytics Workspace ID if logging to Log Analytics')
+param logAnalyticsWorkspaceId string = ''
+
 resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = {
   location: location
   name: keyVaultName
@@ -35,6 +38,24 @@ resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = {
   }
 }
 
+resource keyVaultDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = if (logAnalyticsWorkspaceId != '') {
+  scope: keyVault
+  name: keyVaultName
+  properties: {
+    logs: [
+      {
+        category: 'AuditEvent'
+        enabled: true
+      }
+      {
+        category: 'AzurePolicyEvaluationDetails'
+        enabled: true
+      }
+    ]
+    workspaceId: logAnalyticsWorkspaceId
+  }
+}
+
 output kvId string = keyVault.id
 
 output kvName string = keyVault.name

dev-infrastructure/modules/maestro/maestro-infra.bicep

@@ -24,6 +24,9 @@ param maxClientSessionsPerAuthName int
 ])
 param publicNetworkAccess string
 
+@description('Log Analytics Workspace ID if logging to Log Analytics')
+param logAnalyticsWorkspaceId string = ''
+
 param certificateIssuer string
 
 //
@@ -53,6 +56,44 @@ resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2024-12-15-preview'
   }
 }
 
+resource eventGridNamespaceDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (logAnalyticsWorkspaceId != '') {
+  scope: eventGridNamespace
+  name: eventGridNamespaceName
+  properties: {
+    logs: [
+      {
+        category: 'SuccessfulMqttConnections'
+        enabled: true
+      }
+      {
+        category: 'FailedMqttConnections'
+        enabled: true
+      }
+      {
+        category: 'MqttDisconnections'
+        enabled: true
+      }
+      {
+        category: 'FailedMqttPublishedMessages'
+        enabled: true
+      }
+      {
+        category: 'FailedMqttSubscriptionOperations'
+        enabled: true
+      }
+      {
+        category: 'SuccessfulHttpDataPlaneOperations'
+        enabled: true
+      }
+      {
+        category: 'FailedHttpDataPlaneOperations'
+        enabled: true
+      }
+    ]
+    workspaceId: logAnalyticsWorkspaceId
+  }
+}
+
 // find a better way to register the OneCert
 resource certificateSignerCA 'Microsoft.EventGrid/namespaces/caCertificates@2024-12-15-preview' = if (startsWith(
   certificateIssuer,

dev-infrastructure/region-pipeline.yaml

@@ -45,6 +45,10 @@ resourceGroups:
         input:
           step: global-output
           name: svcParentZoneResourceId
+      - name: logAnalyticsWorkspaceId
+        input:
+          step: global-output
+          name: logAnalyticsWorkspaceId
     dependsOn:
     - global-output
   - name: metrics-infra

dev-infrastructure/svc-pipeline.yaml

@@ -41,6 +41,13 @@ resourceGroups:
     template: templates/svc-infra.bicep
     parameters: configurations/svc-infra.tmpl.bicepparam
     deploymentLevel: ResourceGroup
+    variables:
+    - name: logAnalyticsWorkspaceId
+      input:
+        step: global-output
+        name: logAnalyticsWorkspaceId
+    dependsOn:
+    - global-output
   # Configure certificate issuers for the SVC KV
   - name: svc-oncert-private-kv-issuer
     action: SetCertificateIssuer
@@ -81,6 +88,10 @@ resourceGroups:
       input:
         step: region-output
         name: azureMonitoringWorkspaceId
+    - name: logAnalyticsWorkspaceId
+      input:
+        step: global-output
+        name: logAnalyticsWorkspaceId
     dependsOn:
     - svc-oncert-private-kv-issuer
     - svc-oncert-public-kv-issuer