add CAPZ windows prow jobs

.github/workflows/deploy-lts-prow.yaml

@@ -37,11 +37,14 @@ jobs:
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
-      - name: create resourceGroup
+      - name: create resourceGroups
         run: |
           if [ $(az group exists --name ${{ secrets.AZURE_RG }}) = false ]; then
               az group create --name ${{ secrets.AZURE_RG }} --location ${{ secrets.AZURE_LOCATION }}
           fi
+          if [ $(az group exists --name ${{ secrets.CAPZ_RG }}) = false ]; then
+              az group create --name ${{ secrets.CAPZ_RG }} --location ${{ secrets.AZURE_LOCATION }}
+          fi
 
       - name: Deploy Prow cluster Bicep
         id: bicep
@@ -53,12 +56,23 @@ jobs:
           parameters: aks_cluster_region=${{ secrets.AZURE_LOCATION }} aks_cluster_admin_groups="${{ secrets.PROW_ADMIN_GROUPS }}" aks_cluster_admin_users="${{ secrets.PROW_ADMIN_USERS }}"
           failOnStdErr: false
 
+      - name: Deploy CAPZ Bicep
+        id: capzbicep
+        uses: azure/arm-deploy@v2
+        with:
+          subscriptionId: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: ${{ secrets.CAPZ_RG }}
+          template: ./config/capz/capz.bicep
+          parameters: location=${{ secrets.AZURE_LOCATION }}
+          failOnStdErr: false
+
       - name: Fetch config
         run: |
           echo "PROW_HOST=${{ steps.bicep.outputs.prowHostName }}" >> "$GITHUB_ENV"
           echo "AZURE_STORAGE_ACCOUNT_USER=${{ steps.bicep.outputs.storageAccountName }}" >> "$GITHUB_ENV"
           echo "PUBLIC_IP_NAME=${{ steps.bicep.outputs.publicIpName }}" >> "$GITHUB_ENV"
           echo "CLUSTER_RG=${{ steps.bicep.outputs.resourceGroupName }}" >> "$GITHUB_ENV"
+          echo "CAPZ_SA=${{ steps.capzbicep.outputs.capzsastorage_name }}" >> "$GITHUB_ENV"
 
       - name: Install Kubectl
         uses: azure/setup-kubectl@v4
@@ -102,6 +116,12 @@ jobs:
           PUBLIC_IP_ADDRESS=$(az network public-ip show -g ${{ secrets.AZURE_RG }} -n ${{ steps.bicep.outputs.publicIpName }} | jq -r '.ipAddress')
           echo "::add-mask::$PUBLIC_IP_ADDRESS"
           echo "PUBLIC_IP_ADDRESS=$PUBLIC_IP_ADDRESS" >> "$GITHUB_ENV"
+          CAPZ_CI_REGISTRY=$(az acr show -g ${{ secrets.CAPZ_RG }} -n ${{ steps.capzbicep.outputs.capzci_registry_name }}  | jq .loginServer)
+          echo "::add-mask::$CAPZ_CI_REGISTRY"
+          echo "CAPZ_CI_REGISTRY=$CAPZ_CI_REGISTRY" >> "$GITHUB_ENV"
+          echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> "$GITHUB_ENV"
+          echo "::add-mask::${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}"
+          echo "CAPZ_GMSA_KV=${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" >> "$GITHUB_ENV"
 
       - name: 'Create job configs'
         run: |
@@ -110,6 +130,10 @@ jobs:
           envsubst < config/prow/release-branch-jobs/1.28.yaml >> cm.yaml
           kubectl create configmap config -n prow --from-file=config.yaml=cm.yaml -o yaml --dry-run=client | kubectl apply -f -
           rm cm.yaml
+        env:
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          CAPZ_RG: ${{ secrets.CAPZ_RG }}
+          CAPZ_GMSA_KV: ${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}
 
       - name: 'Apply Prowjob CRD'
         run: for f in config/prow/k8s/prowjob/*.yaml; do kubectl apply --server-side=true -f $f; done

config/capz/capz.bicep

@@ -0,0 +1,155 @@
+param resource_prefix string = 'capz'
+@secure()
+param location string = resourceGroup().location
+
+param random_suffix string = substring(uniqueString(resourceGroup().id, location), 0, 8)
+
+// https://github.com/kubernetes/k8s.io/tree/main/infra/azure/terraform/capz 
+// https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/sig-windows/release-master-windows.yaml
+// https://github.com/kubernetes-sigs/windows-testing/tree/master/capz
+
+resource cloudproviderId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  name: 'cloud-provider-user-identity'
+  location: location
+}
+
+resource domainVMId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  name: 'domain-vm-identity'
+  location: location
+}
+
+resource gmsaId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  name: 'gmsa-user-identity'
+  location: location
+}
+
+resource gmsa_kv 'Microsoft.KeyVault/vaults@2023-07-01' = {
+  name: '${resource_prefix}gmsakv${random_suffix}'
+  location: location
+  properties: {
+    sku: {
+      family: 'A'
+      name: 'standard'
+    }
+    tenantId: subscription().tenantId
+    accessPolicies: [
+      {
+        tenantId: subscription().tenantId
+        objectId: domainVMId.properties.principalId
+        permissions: {
+          secrets: ['set']
+        }
+      }
+      {
+        tenantId: subscription().tenantId
+        objectId: gmsaId.properties.principalId
+        permissions: {
+          secrets: ['get']
+        }
+      }
+    ]
+  }
+}
+
+resource capzci_registry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = {
+  name: '${resource_prefix}ci${random_suffix}'
+  location: location
+  sku: {
+    name: 'Premium'
+  }
+  properties:{
+    anonymousPullEnabled: true
+    policies: {
+      retentionPolicy: {
+        days: 7
+        status: 'enabled'
+      }
+    }
+  }
+}
+
+resource registrytask 'Microsoft.ContainerRegistry/registries/tasks@2019-06-01-preview' = {
+  name: 'midnight_capz_purge'
+  parent: capzci_registry
+  location: location
+  properties: {
+    platform: {
+      os: 'Linux'
+      architecture: 'amd64'
+    }
+    trigger:{
+      timerTriggers: [
+        {
+          name: 't1'
+          schedule: '0 0 * * *'
+          status: 'enabled'
+        } 
+      ]
+      baseImageTrigger: {
+        name: 'defaultBaseimageTriggerName'
+        baseImageTriggerType: 'Runtime'
+        updateTriggerPayloadType: 'Default'
+      }
+    }
+    agentConfiguration: {
+      cpu: 2
+    }
+
+    step: {
+      type: 'EncodedTask'
+      encodedTaskContent: base64('''
+version: v1.1.0
+steps:
+  - cmd: acr purge --filter azdisk:* --filter azure-cloud-controller-manager:* --filter azure-cloud-node-manager-arm64:* --filter azure-cloud-node-manager:* --filter cluster-api-azure:* --ago 1d --untagged
+    disableWorkingDirectoryOverride: true
+    timeout: 3600
+''')
+    }
+  }
+}
+
+resource e2eprivatecommunity 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = {
+  name: '${resource_prefix}e2e${random_suffix}'
+  location: location
+  sku: {
+    name: 'Premium'
+  }
+  properties:{
+    anonymousPullEnabled: true
+    policies: {
+      retentionPolicy: {
+        days: 7
+        status: 'enabled'
+      }
+    }
+  }
+}
+
+resource capzsa 'Microsoft.Storage/storageAccounts@2022-05-01' = {
+  name: '${resource_prefix}sa${random_suffix}'
+  location: location
+  sku: {
+    name: 'Standard_ZRS'
+  }
+  kind: 'StorageV2'
+  properties: {
+    accessTier: 'Hot'
+    minimumTlsVersion: 'TLS1_2'
+    supportsHttpsTrafficOnly: true
+  } 
+}
+
+resource cloudproviderblobcontributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid('storage-rbac', cloudproviderId.id, capzsa.id, 'Storage Blob Data Contributor')
+  scope: capzsa
+  properties: {
+    principalId: cloudproviderId.properties.principalId
+    roleDefinitionId: tenantResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Storage Blob Data Contributor
+    principalType: 'ServicePrincipal'
+    description: 'Allow controlPlane VM to download private build from storage account'
+  }
+}
+
+output capzci_registry_name string = capzci_registry.name
+output capz_gmsa_kv_name string = gmsa_kv.name
+output capzsastorage_name string = capzsa.name

config/prow/cluster/prow-cluster.bicep

@@ -99,11 +99,11 @@ resource aks 'Microsoft.ContainerService/managedClusters@2024-06-01' = {
       }
     }
     oidcIssuerProfile: {
-      enabled: true
+      enabled: true // windows capz prowjob need this
     }
     securityProfile: {
       workloadIdentity: {
-        enabled: true
+        enabled: true // windows capz prowjob need this
       }
     }
   }

config/prow/k8s/test-pods/capz.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: capz-clusteradmin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: azure
+  namespace: test-pods

config/prow/release-branch-jobs/1.27.yaml

@@ -309,7 +309,7 @@
             memory: 9Gi
         securityContext:
           privileged: true
-  - always_run: true
+  - always_run: true # pull-kubernetes-verify
     branches:
     - release-1.27-lts
     context: pull-kubernetes-verify
@@ -345,5 +345,62 @@
           requests:
             cpu: "7"
             memory: 16Gi
+        securityContext:
+          privileged: true
+  - always_run: false # pull-kubernetes-e2e-capz-windows
+    branches:
+    - release-1.27-lts
+    name: pull-kubernetes-e2e-capz-windows
+    decorate: true
+    decoration_config:
+      timeout: 4h
+    labels:
+      preset-dind-enabled: "true"
+      preset-kind-volume-mounts: "true"
+      preset-capz-windows-common-pull: "true"
+      preset-capz-windows-2019: "true"
+      preset-capz-containerd-1-7-latest: "true"
+      preset-azure-community: "true"
+    extra_refs:
+    - base_ref: release-1.16
+      org: kubernetes-sigs
+      path_alias: sigs.k8s.io/cluster-api-provider-azure
+      repo: cluster-api-provider-azure
+      workdir: false
+    - base_ref: release-1.27
+      org: kubernetes-sigs
+      path_alias: sigs.k8s.io/cloud-provider-azure
+      repo: cloud-provider-azure
+    - org: haitch
+      repo: windows-testing
+      base_ref: master
+      path_alias: k8s.io/windows-testing
+      workdir: true
+    optional: true
+    path_alias: k8s.io/kubernetes
+    run_if_changed: azure.*\.go$|.*windows\.go$|test/e2e/windows/.*
+    spec:
+      serviceAccountName: azure
+      containers:
+      - command:
+        - "runner.sh"
+        - "env"
+        - "./capz/run-capz-e2e.sh"
+        image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240705-131cd74733-1.27
+        name: ""
+        env:
+        - name: CAPI_VERSION
+          value: "v1.7.3" # newer version (v1.7.4 and beyond) use kubeadm.k8s.io/v1beta4, which is not support by 1.27 kubeadm.
+        - name: AZURE_LOCATION
+          value: "australiaeast"
+        - name: "KUBERNETES_VERSION"
+          value: "1.27"
+        resources:
+          requests:
+            cpu: "2"
+            memory: 9Gi
+          limits:
+            cpu: "2"
+            memory: 9Gi
         securityContext:
           privileged: true