From 66f57baccd58304d9119b85841b509caca7e2862 Mon Sep 17 00:00:00 2001 From: Jean-Francois Chevrette Date: Fri, 7 Feb 2025 09:51:21 -0500 Subject: [PATCH] refactor AZ script and helper func for proper AZ selection ... to account for regions with varied number of AZs and the eventuality that AZ names could be something other than 1,2,3 --- .../modules/aks-cluster-base.bicep | 21 +- dev-infrastructure/scripts/list-az-regions.sh | 18 +- dev-infrastructure/templates/common.bicep | 507 ++++++++++++++---- .../templates/mgmt-cluster.bicep | 6 + .../templates/svc-cluster.bicep | 9 +- 5 files changed, 440 insertions(+), 121 deletions(-) diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index 29f6bd1a4..8b065bf2e 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -25,6 +25,10 @@ param dnsServiceIP string = '10.130.0.10' // Passed Params and Overrides param location string +@description('List of Availability Zones to use for the AKS cluster') +param locationAvailabilityZones array +var locationHasAvailabilityZones = length(locationAvailabilityZones) > 0 + @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false @@ -58,13 +62,6 @@ var istioIngressGatewayIPAddressIPTagsArray = [ } ] -@description('List of Availability Zones for zone-redundant resources') -param zoneRedundancyZones array = [ - '1' - '2' - '3' -] - @maxLength(24) param aksKeyVaultName string @@ -255,7 +252,7 @@ module istioIngressGatewayIPAddress '../modules/network/publicipaddress.bicep' = name: istioIngressGatewayIPAddressName ipTags: istioIngressGatewayIPAddressIPTagsArray location: location - zones: zoneRedundancyZones + zones: locationHasAvailabilityZones ? locationAvailabilityZones : null // Role Assignment needed for the public IP address to be used on the Load Balancer roleAssignmentProperties: { principalId: aksClusterUserDefinedManagedIdentity.properties.principalId @@ -272,7 +269,7 @@ module aksClusterOutboundIPAddress '../modules/network/publicipaddress.bicep' = name: aksClusterOutboundIPAddressName ipTags: aksClusterOutboundIPAddressIPTagsArray location: location - zones: zoneRedundancyZones + zones: locationHasAvailabilityZones ? locationAvailabilityZones : null // Role Assignment needed for the public IP address to be used on the Load Balancer roleAssignmentProperties: { principalId: aksClusterUserDefinedManagedIdentity.properties.principalId @@ -338,7 +335,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ vnetSubnetID: aksNodeSubnet.id podSubnetID: aksPodSubnet.id maxPods: 100 - availabilityZones: zoneRedundancyZones + availabilityZones: locationHasAvailabilityZones ? locationAvailabilityZones : null securityProfile: { enableSecureBoot: false enableVTPM: false @@ -492,9 +489,7 @@ resource userAgentPools 'Microsoft.ContainerService/managedClusters/agentPools@2 vnetSubnetID: aksNodeSubnet.id podSubnetID: aksPodSubnet.id maxPods: 225 - availabilityZones: [ - '${(i + 1)}' - ] + availabilityZones: locationHasAvailabilityZones ? [locationAvailabilityZones[i]] : null securityProfile: { enableSecureBoot: false enableVTPM: false diff --git a/dev-infrastructure/scripts/list-az-regions.sh b/dev-infrastructure/scripts/list-az-regions.sh index d665d1263..eb65346a3 100755 --- a/dev-infrastructure/scripts/list-az-regions.sh +++ b/dev-infrastructure/scripts/list-az-regions.sh @@ -7,12 +7,12 @@ az rest \ --method get \ --uri "/subscriptions/$(az account show --query id --output tsv)/locations?api-version=2024-11-01" \ | jq -r ' - # First, print items with availabilityZoneMappings set - "Regions with Availability Zones:", - (.value[] | select(.availabilityZoneMappings) | .name), - "", - # Then, print items without availabilityZoneMappings - "Regions without Availability Zones:", - (.value[] | select(.availabilityZoneMappings | not) | .name) - ' - + reduce .value[] as $item ({}; .[$item.name] = { + availabilityZones: ( + if $item.availabilityZoneMappings then + $item.availabilityZoneMappings | map(.logicalZone) + else + [] + end + ) + }) | to_entries | sort_by(.key) | from_entries' diff --git a/dev-infrastructure/templates/common.bicep b/dev-infrastructure/templates/common.bicep index a42c59140..ed45d9ac0 100644 --- a/dev-infrastructure/templates/common.bicep +++ b/dev-infrastructure/templates/common.bicep @@ -1,101 +1,414 @@ // https://learn.microsoft.com/en-us/azure/reliability/availability-zones-region-support // See helper script in dev-infrastructure/scripts/list-az-locations.sh -var _zoneRedundantLocations = [ - 'australiaeast' - 'brazilsouth' - 'canadacentral' - 'centralindia' - 'centralus' - 'centraluseuap' - 'eastasia' - 'eastus' - 'eastus2' - 'eastus2euap' - 'francecentral' - 'germanywestcentral' - 'israelcentral' - 'italynorth' - 'japaneast' - 'koreacentral' - 'mexicocentral' - 'newzealandnorth' - 'northeurope' - 'norwayeast' - 'polandcentral' - 'qatarcentral' - 'southafricanorth' - 'southcentralus' - 'southeastasia' - 'spaincentral' - 'swedencentral' - 'switzerlandnorth' - 'uaenorth' - 'uksouth' - 'westeurope' - 'westus2' - 'westus3' - - // The following regions do not support availability zones - // asia - // asiapacific - // australia - // australiacentral - // australiacentral2 - // australiasoutheast - // brazil - // brazilsoutheast - // brazilus - // canada - // canadaeast - // centralusstage - // eastasiastage - // eastus2stage - // eastusstage - // eastusstg - // europe - // france - // francesouth - // germany - // germanynorth - // global - // india - // israel - // italy - // japan - // japanwest - // jioindiacentral - // jioindiawest - // korea - // koreasouth - // newzealand - // northcentralus - // northcentralusstage - // norway - // norwaywest - // poland - // qatar - // singapore - // southafrica - // southafricawest - // southcentralusstage - // southcentralusstg - // southeastasiastage - // southindia - // sweden - // switzerland - // switzerlandwest - // uae - // uaecentral - // uk - // ukwest - // unitedstates - // unitedstateseuap - // westcentralus - // westindia - // westus - // westus2stage - // westusstage -] +var _locationAvailabilityZones = { + asia: { + availabilityZones: [] + } + asiapacific: { + availabilityZones: [] + } + australia: { + availabilityZones: [] + } + australiacentral: { + availabilityZones: [] + } + australiacentral2: { + availabilityZones: [] + } + australiaeast: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + australiasoutheast: { + availabilityZones: [] + } + brazil: { + availabilityZones: [] + } + brazilsouth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + brazilsoutheast: { + availabilityZones: [] + } + brazilus: { + availabilityZones: [] + } + canada: { + availabilityZones: [] + } + canadacentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + canadaeast: { + availabilityZones: [] + } + centralindia: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + centralus: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + centraluseuap: { + availabilityZones: [ + '1' + '2' + ] + } + centralusstage: { + availabilityZones: [] + } + eastasia: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + eastasiastage: { + availabilityZones: [] + } + eastus: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + eastus2: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + eastus2euap: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + eastus2stage: { + availabilityZones: [] + } + eastusstage: { + availabilityZones: [] + } + eastusstg: { + availabilityZones: [] + } + europe: { + availabilityZones: [] + } + france: { + availabilityZones: [] + } + francecentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + francesouth: { + availabilityZones: [] + } + germany: { + availabilityZones: [] + } + germanynorth: { + availabilityZones: [] + } + germanywestcentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + global: { + availabilityZones: [] + } + india: { + availabilityZones: [] + } + israel: { + availabilityZones: [] + } + israelcentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + italy: { + availabilityZones: [] + } + italynorth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + japan: { + availabilityZones: [] + } + japaneast: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + japanwest: { + availabilityZones: [] + } + jioindiacentral: { + availabilityZones: [] + } + jioindiawest: { + availabilityZones: [] + } + korea: { + availabilityZones: [] + } + koreacentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + koreasouth: { + availabilityZones: [] + } + mexicocentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + newzealand: { + availabilityZones: [] + } + newzealandnorth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + northcentralus: { + availabilityZones: [] + } + northcentralusstage: { + availabilityZones: [] + } + northeurope: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + norway: { + availabilityZones: [] + } + norwayeast: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + norwaywest: { + availabilityZones: [] + } + poland: { + availabilityZones: [] + } + polandcentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + qatar: { + availabilityZones: [] + } + qatarcentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + singapore: { + availabilityZones: [] + } + southafrica: { + availabilityZones: [] + } + southafricanorth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + southafricawest: { + availabilityZones: [] + } + southcentralus: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + southcentralusstage: { + availabilityZones: [] + } + southcentralusstg: { + availabilityZones: [] + } + southeastasia: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + southeastasiastage: { + availabilityZones: [] + } + southindia: { + availabilityZones: [] + } + spaincentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + sweden: { + availabilityZones: [] + } + swedencentral: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + switzerland: { + availabilityZones: [] + } + switzerlandnorth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + switzerlandwest: { + availabilityZones: [] + } + uae: { + availabilityZones: [] + } + uaecentral: { + availabilityZones: [] + } + uaenorth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + uk: { + availabilityZones: [] + } + uksouth: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + ukwest: { + availabilityZones: [] + } + unitedstates: { + availabilityZones: [] + } + unitedstateseuap: { + availabilityZones: [] + } + westcentralus: { + availabilityZones: [] + } + westeurope: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + westindia: { + availabilityZones: [] + } + westus: { + availabilityZones: [] + } + westus2: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + westus2stage: { + availabilityZones: [] + } + westus3: { + availabilityZones: [ + '1' + '2' + '3' + ] + } + westusstage: { + availabilityZones: [] + } +} @export() -func locationIsZoneRedundant(region string) bool => contains(_zoneRedundantLocations, region) +func getLocationAvailabilityZones(region string) array => _locationAvailabilityZones[region].availabilityZones diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep index 2944db701..25faa7960 100644 --- a/dev-infrastructure/templates/mgmt-cluster.bicep +++ b/dev-infrastructure/templates/mgmt-cluster.bicep @@ -1,6 +1,11 @@ +import { getLocationAvailabilityZones } from 'common.bicep' + @description('Azure Region Location') param location string = resourceGroup().location +@description('List of Availability Zones to use for the AKS cluster') +param locationAvailabilityZones array = getLocationAvailabilityZones(location) + @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false @@ -94,6 +99,7 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = { scope: resourceGroup() params: { location: location + locationAvailabilityZones: locationAvailabilityZones persist: persist aksClusterName: aksClusterName aksNodeResourceGroupName: aksNodeResourceGroupName diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 4e1ffaf32..1ba0d153d 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -1,8 +1,12 @@ -import { locationIsZoneRedundant } from '../templates/common.bicep' +import { getLocationAvailabilityZones } from 'common.bicep' @description('Azure Region Location') param location string = resourceGroup().location +@description('List of Availability Zones to use for the AKS cluster') +param locationAvailabilityZones array = getLocationAvailabilityZones(location) +var locationHasAvailabilityZones = length(locationAvailabilityZones) > 0 + @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false @@ -191,6 +195,7 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { scope: resourceGroup() params: { location: location + locationAvailabilityZones: locationAvailabilityZones persist: persist aksClusterName: aksClusterName aksNodeResourceGroupName: aksNodeResourceGroupName @@ -376,7 +381,7 @@ module oidc '../modules/oidc/main.bicep' = { location: location storageAccountName: oidcStorageAccountName rpMsiName: csMIName - skuName: locationIsZoneRedundant(location) ? 'Standard_ZRS' : 'Standard_LRS' + skuName: locationHasAvailabilityZones ? 'Standard_ZRS' : 'Standard_LRS' msiId: aroDevopsMsiId deploymentScriptLocation: location }