From 026ce1cfdb9eb0b437f75b149b3a9e69f117e988 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 14 Jan 2025 09:03:34 +0100 Subject: [PATCH] refactor DNS settings (#1077) * collect dns settings under `dns` in config * proper name for CX DNS zone `cxParentZoneName` (previously `baseDnsZoneName`) * regional svc zone --- cluster-service/Makefile | 2 +- cluster-service/pipeline.yaml | 6 +-- config/config.msft.yaml | 12 +++-- config/config.schema.json | 42 +++++++++++------ config/config.yaml | 17 ++++--- config/public-cloud-cs-pr.json | 10 ++-- config/public-cloud-dev.json | 10 ++-- config/public-cloud-msft-int.json | 10 ++-- config/public-cloud-personal-dev.json | 10 ++-- .../global-infra.tmpl.bicepparam | 4 +- .../configurations/region.tmpl.bicepparam | 7 +-- .../svc-cluster.tmpl.bicepparam | 2 +- .../configurations/svc-infra.tmpl.bicepparam | 2 +- dev-infrastructure/docs/development-setup.md | 6 ++- .../modules/cluster-service.bicep | 13 ++++-- dev-infrastructure/templates/region.bicep | 46 +++++++++++++++---- .../templates/svc-cluster.bicep | 6 +-- dev-infrastructure/templates/svc-infra.bicep | 9 ++-- 18 files changed, 139 insertions(+), 75 deletions(-) diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 0bc47bd41..c13be0efa 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -2,7 +2,7 @@ -include ../helm-cmd.mk HELM_CMD ?= helm upgrade --install -ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${BASE_DNS_ZONE_NAME}" +ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${CX_PARENT_DNS_ZONE_NAME}" deploy: diff --git a/cluster-service/pipeline.yaml b/cluster-service/pipeline.yaml index 7afb12841..d0ea48a0c 100644 --- a/cluster-service/pipeline.yaml +++ b/cluster-service/pipeline.yaml @@ -48,10 +48,10 @@ resourceGroups: value: msiMockCert - name: ARM_HELPER_CERT_NAME value: armHelperCert - - name: BASE_DNS_ZONE_NAME - configRef: baseDnsZoneName + - name: CX_PARENT_DNS_ZONE_NAME + configRef: dns.cxParentZoneName - name: REGIONAL_DNS_SUBDOMAIN - configRef: regionalDNSSubdomain + configRef: dns.regionalSubdomain - name: USE_AZURE_DB configRef: clusterService.postgres.deploy - name: DATABASE_SERVER_NAME diff --git a/config/config.msft.yaml b/config/config.msft.yaml index 66c6872ef..afdbb217a 100644 --- a/config/config.msft.yaml +++ b/config/config.msft.yaml @@ -130,8 +130,9 @@ defaults: private: false # DNS - baseDnsZoneRG: global-shared-resources - regionalDNSSubdomain: '{{ .ctx.region }}' + dns: + baseDnsZoneRG: global-shared-resources + regionalSubdomain: '{{ .ctx.region }}' # Metrics monitoring: @@ -202,10 +203,11 @@ clouds: vmSize: 'Standard_D16s_v3' osDiskSizeGB: 128 azCount: 3 + # DNS - baseDnsZoneName: aroapp-hcp.azure-test.net - regionalDNSSubdomain: '{{ .ctx.region }}' - svcParentZoneName: "aro-hcp.azure-test.net" + dns: + cxParentZoneName: aroapp-hcp.azure-test.net + svcParentZoneName: aro-hcp.azure-test.net # ACR svcAcrName: arohcpsvcint diff --git a/config/config.schema.json b/config/config.schema.json index 2b90f691c..1b6303006 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -23,15 +23,33 @@ "aroDevopsMsiId": { "type": "string" }, - "baseDnsZoneName": { - "type": "string" - }, - "svcParentZoneName": { - "type": "string", - "description": "The service cluster component domain name" - }, - "baseDnsZoneRG": { - "type": "string" + "dns": { + "type": "object", + "properties": { + "baseDnsZoneRG": { + "type": "string", + "description": "The Azure RG that holds the parent DNS zones" + }, + "cxParentZoneName": { + "type": "string", + "description": "The parent DNS zone name for regional HCP cluster DNS zones" + }, + "svcParentZoneName": { + "type": "string", + "description": "The parent DNS zone name for regional ARO-HCP infrastructure, e.g. the RP" + }, + "regionalSubdomain": { + "type": "string", + "description": "The regional subdomain to be used to construct the regional hcp and svc zones under their respective parents, e.g. regionalSubdomain.svcParentZoneName" + } + }, + "additionalProperties": false, + "required": [ + "baseDnsZoneRG", + "cxParentZoneName", + "svcParentZoneName", + "regionalSubdomain" + ] }, "clusterService": { "type": "object", @@ -676,9 +694,6 @@ "regionRG": { "type": "string" }, - "regionalDNSSubdomain": { - "type": "string" - }, "serviceKeyVault": { "type": "object", "properties": { @@ -806,8 +821,6 @@ "required": [ "aksName", "aroDevopsMsiId", - "baseDnsZoneName", - "baseDnsZoneRG", "clusterService", "cxKeyVault", "firstPartyAppClientId", @@ -828,7 +841,6 @@ "podSubnetPrefix", "region", "regionRG", - "regionalDNSSubdomain", "serviceKeyVault", "subnetPrefix", "svc", diff --git a/config/config.yaml b/config/config.yaml index c3b1f789d..e7ed7e4e5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -136,16 +136,16 @@ defaults: softDelete: true private: true - # DNS - baseDnsZoneRG: 'global' clouds: public: # this configuration serves as a template for for all RH DEV subscription deployments # the following vars need approprivate overrides: defaults: # DNS - baseDnsZoneName: 'hcp.osadev.cloud' - svcParentZoneName: "hcpsvc.osadev.cloud" + dns: + baseDnsZoneRG: global + cxParentZoneName: hcp.osadev.cloud + svcParentZoneName: hcpsvc.osadev.cloud # 1P app firstPartyAppClientId: 57e54810-3138-4f38-bd3b-29cb33f4c358 # Mock Managed Identities Service Princiapl @@ -252,7 +252,8 @@ clouds: minCount: 2 maxCount: 12 # DNS - regionalDNSSubdomain: '{{ .ctx.region }}' + dns: + regionalSubdomain: '{{ .ctx.region }}' # Maestro maestro: server: @@ -276,7 +277,8 @@ clouds: minCount: 2 maxCount: 12 # DNS - regionalDNSSubdomain: '{{ .ctx.region }}-cs' + dns: + regionalSubdomain: '{{ .ctx.region }}-cs' # Maestro maestro: restrictIstioIngress: false @@ -294,7 +296,8 @@ clouds: postgres: deploy: false # DNS - regionalDNSSubdomain: '{{ .ctx.regionShort }}' + dns: + regionalSubdomain: '{{ .ctx.regionShort }}' # Maestro maestro: postgres: diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 461f60314..bce554aed 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -7,8 +7,6 @@ "backend": { "imageTag": "" }, - "baseDnsZoneName": "hcp.osadev.cloud", - "baseDnsZoneRG": "global", "clusterService": { "acrRG": "global", "azureOperatorsManagedIdentities": { @@ -45,6 +43,12 @@ "private": false, "softDelete": false }, + "dns": { + "baseDnsZoneRG": "global", + "cxParentZoneName": "hcp.osadev.cloud", + "regionalSubdomain": "westus3-cs", + "svcParentZoneName": "hcpsvc.osadev.cloud" + }, "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontend": { @@ -177,7 +181,6 @@ "podSubnetPrefix": "10.128.64.0/18", "region": "westus3", "regionRG": "hcp-underlay-cspr", - "regionalDNSSubdomain": "westus3-cs", "serviceKeyVault": { "name": "aro-hcp-dev-svc-kv", "private": false, @@ -208,6 +211,5 @@ } }, "svcAcrName": "arohcpsvcdev", - "svcParentZoneName": "hcpsvc.osadev.cloud", "vnetAddressPrefix": "10.128.0.0/14" } diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index f2f83e99b..ceacde95b 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -7,8 +7,6 @@ "backend": { "imageTag": "" }, - "baseDnsZoneName": "hcp.osadev.cloud", - "baseDnsZoneRG": "global", "clusterService": { "acrRG": "global", "azureOperatorsManagedIdentities": { @@ -45,6 +43,12 @@ "private": false, "softDelete": false }, + "dns": { + "baseDnsZoneRG": "global", + "cxParentZoneName": "hcp.osadev.cloud", + "regionalSubdomain": "westus3", + "svcParentZoneName": "hcpsvc.osadev.cloud" + }, "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontend": { @@ -177,7 +181,6 @@ "podSubnetPrefix": "10.128.64.0/18", "region": "westus3", "regionRG": "hcp-underlay-dev", - "regionalDNSSubdomain": "westus3", "serviceKeyVault": { "name": "aro-hcp-dev-svc-kv", "private": false, @@ -208,6 +211,5 @@ } }, "svcAcrName": "arohcpsvcdev", - "svcParentZoneName": "hcpsvc.osadev.cloud", "vnetAddressPrefix": "10.128.0.0/14" } diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json index 4fecb83af..a2626b032 100644 --- a/config/public-cloud-msft-int.json +++ b/config/public-cloud-msft-int.json @@ -7,8 +7,6 @@ "backend": { "imageTag": "0b3c08f" }, - "baseDnsZoneName": "aroapp-hcp.azure-test.net", - "baseDnsZoneRG": "global-shared-resources", "clusterService": { "acrRG": "global-shared-resources", "azureOperatorsManagedIdentities": { @@ -45,6 +43,12 @@ "private": false, "softDelete": false }, + "dns": { + "baseDnsZoneRG": "global-shared-resources", + "cxParentZoneName": "aroapp-hcp.azure-test.net", + "regionalSubdomain": "westus3", + "svcParentZoneName": "aro-hcp.azure-test.net" + }, "extraVars": {}, "firstPartyAppClientId": "??? the one used by CS to do first party stuff ???", "frontend": { @@ -172,7 +176,6 @@ "podSubnetPrefix": "10.128.64.0/18", "region": "westus3", "regionRG": "westus3-shared-resources", - "regionalDNSSubdomain": "westus3", "serviceKeyVault": { "name": "arohcp-svc-int", "private": false, @@ -203,6 +206,5 @@ } }, "svcAcrName": "arohcpsvcint", - "svcParentZoneName": "aro-hcp.azure-test.net", "vnetAddressPrefix": "10.128.0.0/14" } diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index 53c66a47a..201efb3b4 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -7,8 +7,6 @@ "backend": { "imageTag": "" }, - "baseDnsZoneName": "hcp.osadev.cloud", - "baseDnsZoneRG": "global", "clusterService": { "acrRG": "global", "azureOperatorsManagedIdentities": { @@ -45,6 +43,12 @@ "private": false, "softDelete": false }, + "dns": { + "baseDnsZoneRG": "global", + "cxParentZoneName": "hcp.osadev.cloud", + "regionalSubdomain": "usw3tst", + "svcParentZoneName": "hcpsvc.osadev.cloud" + }, "extraVars": {}, "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358", "frontend": { @@ -177,7 +181,6 @@ "podSubnetPrefix": "10.128.64.0/18", "region": "westus3", "regionRG": "hcp-underlay-usw3tst", - "regionalDNSSubdomain": "usw3tst", "serviceKeyVault": { "name": "aro-hcp-dev-svc-kv", "private": false, @@ -208,6 +211,5 @@ } }, "svcAcrName": "arohcpsvcdev", - "svcParentZoneName": "hcpsvc.osadev.cloud", "vnetAddressPrefix": "10.128.0.0/14" } diff --git a/dev-infrastructure/configurations/global-infra.tmpl.bicepparam b/dev-infrastructure/configurations/global-infra.tmpl.bicepparam index 70a4089ff..b6a55bfd2 100644 --- a/dev-infrastructure/configurations/global-infra.tmpl.bicepparam +++ b/dev-infrastructure/configurations/global-infra.tmpl.bicepparam @@ -1,5 +1,5 @@ using '../templates/global-infra.bicep' param globalMSIName = '{{ .global.globalMSIName }}' -param cxParentZoneName = '{{ .baseDnsZoneName }}' -param svcParentZoneName = '{{ .svcParentZoneName }}' +param cxParentZoneName = '{{ .dns.cxParentZoneName }}' +param svcParentZoneName = '{{ .dns.svcParentZoneName }}' diff --git a/dev-infrastructure/configurations/region.tmpl.bicepparam b/dev-infrastructure/configurations/region.tmpl.bicepparam index db9c33144..766afc9ba 100644 --- a/dev-infrastructure/configurations/region.tmpl.bicepparam +++ b/dev-infrastructure/configurations/region.tmpl.bicepparam @@ -10,9 +10,10 @@ param ocpAcrName = '{{ .ocpAcrName }}' param svcAcrName = '{{ .svcAcrName }}' // dns -param baseDNSZoneName = '{{ .baseDnsZoneName }}' -param baseDNSZoneResourceGroup = '{{ .baseDnsZoneRG }}' -param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}' +param cxBaseDNSZoneName = '{{ .dns.cxParentZoneName }}' +param svcBaseDNSZoneName = '{{ .dns.svcParentZoneName }}' +param baseDNSZoneResourceGroup = '{{ .dns.baseDnsZoneRG }}' +param regionalDNSSubdomain = '{{ .dns.regionalSubdomain }}' // maestro param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}' diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam index bc84a591b..7d308e322 100644 --- a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -49,7 +49,7 @@ param useCustomACRTokenManagementRole = {{ .global.manageTokenCustomRole }} param oidcStorageAccountName = '{{ .oidcStorageAccountName }}' param aroDevopsMsiId = '{{ .aroDevopsMsiId }}' -param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}' +param regionalCXDNSZoneName = '{{ .dns.regionalSubdomain }}.{{ .dns.cxParentZoneName }}' param regionalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/configurations/svc-infra.tmpl.bicepparam b/dev-infrastructure/configurations/svc-infra.tmpl.bicepparam index 2238bcabe..68b9a83a8 100644 --- a/dev-infrastructure/configurations/svc-infra.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-infra.tmpl.bicepparam @@ -7,7 +7,7 @@ param serviceKeyVaultLocation = '{{ .serviceKeyVault.region }}' param serviceKeyVaultSoftDelete = {{ .serviceKeyVault.softDelete }} param serviceKeyVaultPrivate = {{ .serviceKeyVault.private }} -param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .svcParentZoneName }}' +param regionalSvcDNSZoneName = '{{ .dns.regionalSubdomain }}.{{ .dns.svcParentZoneName }}' // MI for deployment scripts diff --git a/dev-infrastructure/docs/development-setup.md b/dev-infrastructure/docs/development-setup.md index f3ecd801e..1c0e7b14a 100644 --- a/dev-infrastructure/docs/development-setup.md +++ b/dev-infrastructure/docs/development-setup.md @@ -120,11 +120,13 @@ defaults: (1) clouds: public: (2) defaults: (3) - baseDnsZoneName: "arohcp.azure.com" + dns: + cxParentZoneName: "arohcp.azure.com" environments: personal-dev: (4) defaults: - baseDnsZoneName: "hcp.osadev.cloud" (5) + dns: + cxParentZoneName: "hcp.osadev.cloud" (5) production: defaults: regions: diff --git a/dev-infrastructure/modules/cluster-service.bicep b/dev-infrastructure/modules/cluster-service.bicep index 4aebfe508..3a88e2c32 100644 --- a/dev-infrastructure/modules/cluster-service.bicep +++ b/dev-infrastructure/modules/cluster-service.bicep @@ -31,8 +31,13 @@ param serviceKeyVaultName string @description('The resource group of the service keyvault') param serviceKeyVaultResourceGroup string -@description('The name of the regional DNS zone') -param regionalDNSZoneName string +@description( + ''' + The regional DNS zone to hold ARO HCP customer cluster DNS records. + CS requires write access to this zone to provision the DNS records for HCPs. + ''' +) +param regionalCXDNSZoneName string @description('The regional resourece group') param regionalResourceGroup string @@ -138,10 +143,10 @@ module csServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep // module csDnsZoneContributor '../modules/dns/zone-contributor.bicep' = { - name: guid(regionalDNSZoneName, clusterServiceManagedIdentityPrincipalId) + name: guid(regionalCXDNSZoneName, clusterServiceManagedIdentityPrincipalId) scope: resourceGroup(regionalResourceGroup) params: { - zoneName: regionalDNSZoneName + zoneName: regionalCXDNSZoneName zoneContributerManagedIdentityPrincipalId: clusterServiceManagedIdentityPrincipalId } } diff --git a/dev-infrastructure/templates/region.bicep b/dev-infrastructure/templates/region.bicep index 9b542d522..ac9f464aa 100644 --- a/dev-infrastructure/templates/region.bicep +++ b/dev-infrastructure/templates/region.bicep @@ -13,8 +13,17 @@ param maestroEventGridPrivate bool @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false -@description('This is a global DNS zone name that will be the parent of regional DNS zones to host ARO HCP customer cluster DNS records') -param baseDNSZoneName string +@description(''' + This is the global parent DNS zone for ARO HCP customer cluster DNS. + It is prefixed with regionalDNSSubdomain to form the actual regional DNS zone name + ''') +param cxBaseDNSZoneName string + +@description(''' + This is the global parent DNS zone for ARO HCP service DNS records. + It is prefixed with regionalDNSSubdomain to form the actual regional DNS zone name + ''') +param svcBaseDNSZoneName string @description('The resource group to deploy the base DNS zone to') param baseDNSZoneResourceGroup string = 'global' @@ -39,21 +48,40 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { } // -// R E G I O N A L D N S Z O N E +// R E G I O N A L C X D N S Z O N E // -resource regionalZone 'Microsoft.Network/dnsZones@2018-05-01' = { - name: '${regionalDNSSubdomain}.${baseDNSZoneName}' +resource regionalCxZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: '${regionalDNSSubdomain}.${cxBaseDNSZoneName}' location: 'global' } -module regionalZoneDelegation '../modules/dns/zone-delegation.bicep' = { - name: '${regionalDNSSubdomain}-zone-deleg' +module regionalCxZoneDelegation '../modules/dns/zone-delegation.bicep' = { + name: '${regionalDNSSubdomain}-cx-zone-deleg' scope: resourceGroup(baseDNSZoneResourceGroup) params: { childZoneName: regionalDNSSubdomain - childZoneNameservers: regionalZone.properties.nameServers - parentZoneName: baseDNSZoneName + childZoneNameservers: regionalCxZone.properties.nameServers + parentZoneName: cxBaseDNSZoneName + } +} + +// +// R E G I O N A L S V C D N S Z O N E +// + +resource regionalSvcZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: '${regionalDNSSubdomain}.${svcBaseDNSZoneName}' + location: 'global' +} + +module regionalSvcZoneDelegation '../modules/dns/zone-delegation.bicep' = { + name: '${regionalDNSSubdomain}-svc-zone-deleg' + scope: resourceGroup(baseDNSZoneResourceGroup) + params: { + childZoneName: regionalDNSSubdomain + childZoneNameservers: regionalSvcZone.properties.nameServers + parentZoneName: cxBaseDNSZoneName } } diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 1767f3f8b..b4f2fd41d 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -146,8 +146,8 @@ param useCustomACRTokenManagementRole bool @description('MSI that will be used to run the deploymentScript') param aroDevopsMsiId string -@description('This is a regional DNS zone') -param regionalDNSZoneName string +@description('The regional DNS zone to hold ARO HCP customer cluster DNS records') +param regionalCXDNSZoneName string @description('Frontend Ingress Certificate Name') param frontendIngressCertName string @@ -327,7 +327,7 @@ module cs '../modules/cluster-service.bicep' = { clusterServiceManagedIdentityName: clusterServiceMIName serviceKeyVaultName: serviceKeyVault.name serviceKeyVaultResourceGroup: serviceKeyVaultResourceGroup - regionalDNSZoneName: regionalDNSZoneName + regionalCXDNSZoneName: regionalCXDNSZoneName regionalResourceGroup: regionalResourceGroup acrResourceGroupNames: clustersServiceAcrResourceGroupNames postgresAdministrationManagedIdentityId: aroDevopsMsiId diff --git a/dev-infrastructure/templates/svc-infra.bicep b/dev-infrastructure/templates/svc-infra.bicep index 9eb64ac76..4e5b83584 100644 --- a/dev-infrastructure/templates/svc-infra.bicep +++ b/dev-infrastructure/templates/svc-infra.bicep @@ -19,8 +19,11 @@ param aroDevopsMsiId string @description('Frontend Certificate Name') param certName string -@description('This is a regional DNS zone') -param regionalDNSZoneName string +@description(''' + This is a regional DNS zone name to hold records for ARO HCP service components, + e.g. the RP + ''') +param regionalSvcDNSZoneName string @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false @@ -57,7 +60,7 @@ output svcKeyVaultName string = serviceKeyVault.outputs.kvName // C E R T I F I C A T E C R E A T I O N // -var clientAuthenticationName = 'frontend.${regionalDNSZoneName}' +var clientAuthenticationName = 'frontend.${regionalSvcDNSZoneName}' module clientCertificate '../modules/keyvault/key-vault-cert.bicep' = { name: 'frontend-cert-${uniqueString(resourceGroup().name)}'