diff --git a/.github/workflows/deploy-lts-prow.yaml b/.github/workflows/deploy-lts-prow.yaml index 1190c80..6f53dd5 100644 --- a/.github/workflows/deploy-lts-prow.yaml +++ b/.github/workflows/deploy-lts-prow.yaml @@ -37,11 +37,14 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - name: create resourceGroup + - name: create resourceGroups run: | if [ $(az group exists --name ${{ secrets.AZURE_RG }}) = false ]; then az group create --name ${{ secrets.AZURE_RG }} --location ${{ secrets.AZURE_LOCATION }} fi + if [ $(az group exists --name ${{ secrets.CAPZ_RG }}) = false ]; then + az group create --name ${{ secrets.CAPZ_RG }} --location ${{ secrets.AZURE_LOCATION }} + fi - name: Deploy Prow cluster Bicep id: bicep @@ -53,12 +56,23 @@ jobs: parameters: aks_cluster_region=${{ secrets.AZURE_LOCATION }} aks_cluster_admin_groups="${{ secrets.PROW_ADMIN_GROUPS }}" aks_cluster_admin_users="${{ secrets.PROW_ADMIN_USERS }}" failOnStdErr: false + - name: Deploy CAPZ Bicep + id: capzbicep + uses: azure/arm-deploy@v2 + with: + subscriptionId: ${{ vars.AZURE_SUBSCRIPTION_ID }} + resourceGroupName: ${{ secrets.CAPZ_RG }} + template: ./config/capz/capz.bicep + parameters: location=${{ secrets.AZURE_LOCATION }} + failOnStdErr: false + - name: Fetch config run: | echo "PROW_HOST=${{ steps.bicep.outputs.prowHostName }}" >> "$GITHUB_ENV" echo "AZURE_STORAGE_ACCOUNT_USER=${{ steps.bicep.outputs.storageAccountName }}" >> "$GITHUB_ENV" echo "PUBLIC_IP_NAME=${{ steps.bicep.outputs.publicIpName }}" >> "$GITHUB_ENV" echo "CLUSTER_RG=${{ steps.bicep.outputs.resourceGroupName }}" >> "$GITHUB_ENV" + echo "CAPZ_SA=${{ steps.capzbicep.outputs.capzsastorage_name }}" >> "$GITHUB_ENV" - name: Install Kubectl uses: azure/setup-kubectl@v4 @@ -102,6 +116,12 @@ jobs: PUBLIC_IP_ADDRESS=$(az network public-ip show -g ${{ secrets.AZURE_RG }} -n ${{ steps.bicep.outputs.publicIpName }} | jq -r '.ipAddress') echo "::add-mask::$PUBLIC_IP_ADDRESS" echo "PUBLIC_IP_ADDRESS=$PUBLIC_IP_ADDRESS" >> "$GITHUB_ENV" + CAPZ_CI_REGISTRY=$(az acr show -g ${{ secrets.CAPZ_RG }} -n ${{ steps.capzbicep.outputs.capzci_registry_name }} | jq .loginServer) + echo "::add-mask::$CAPZ_CI_REGISTRY" + echo "CAPZ_CI_REGISTRY=$CAPZ_CI_REGISTRY" >> "$GITHUB_ENV" + echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> "$GITHUB_ENV" + echo "::add-mask::${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" + echo "CAPZ_GMSA_KV=${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" >> "$GITHUB_ENV" - name: 'Create job configs' run: | @@ -110,6 +130,10 @@ jobs: envsubst < config/prow/release-branch-jobs/1.28.yaml >> cm.yaml kubectl create configmap config -n prow --from-file=config.yaml=cm.yaml -o yaml --dry-run=client | kubectl apply -f - rm cm.yaml + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + CAPZ_RG: ${{ secrets.CAPZ_RG }} + CAPZ_GMSA_KV: ${{ steps.capzbicep.outputs.capz_gmsa_kv_name }} - name: 'Apply Prowjob CRD' run: for f in config/prow/k8s/prowjob/*.yaml; do kubectl apply --server-side=true -f $f; done diff --git a/config/capz/capz.bicep b/config/capz/capz.bicep new file mode 100644 index 0000000..3071964 --- /dev/null +++ b/config/capz/capz.bicep @@ -0,0 +1,155 @@ +param resource_prefix string = 'capz' +@secure() +param location string = resourceGroup().location + +param random_suffix string = substring(uniqueString(resourceGroup().id, location), 0, 8) + +// https://github.com/kubernetes/k8s.io/tree/main/infra/azure/terraform/capz +// https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/sig-windows/release-master-windows.yaml +// https://github.com/kubernetes-sigs/windows-testing/tree/master/capz + +resource cloudproviderId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: 'cloud-provider-user-identity' + location: location +} + +resource domainVMId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: 'domain-vm-identity' + location: location +} + +resource gmsaId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: 'gmsa-user-identity' + location: location +} + +resource gmsa_kv 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: '${resource_prefix}gmsakv${random_suffix}' + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + accessPolicies: [ + { + tenantId: subscription().tenantId + objectId: domainVMId.properties.principalId + permissions: { + secrets: ['set'] + } + } + { + tenantId: subscription().tenantId + objectId: gmsaId.properties.principalId + permissions: { + secrets: ['get'] + } + } + ] + } +} + +resource capzci_registry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = { + name: '${resource_prefix}ci${random_suffix}' + location: location + sku: { + name: 'Premium' + } + properties:{ + anonymousPullEnabled: true + policies: { + retentionPolicy: { + days: 7 + status: 'enabled' + } + } + } +} + +resource registrytask 'Microsoft.ContainerRegistry/registries/tasks@2019-06-01-preview' = { + name: 'midnight_capz_purge' + parent: capzci_registry + location: location + properties: { + platform: { + os: 'Linux' + architecture: 'amd64' + } + trigger:{ + timerTriggers: [ + { + name: 't1' + schedule: '0 0 * * *' + status: 'enabled' + } + ] + baseImageTrigger: { + name: 'defaultBaseimageTriggerName' + baseImageTriggerType: 'Runtime' + updateTriggerPayloadType: 'Default' + } + } + agentConfiguration: { + cpu: 2 + } + + step: { + type: 'EncodedTask' + encodedTaskContent: base64(''' +version: v1.1.0 +steps: + - cmd: acr purge --filter azdisk:* --filter azure-cloud-controller-manager:* --filter azure-cloud-node-manager-arm64:* --filter azure-cloud-node-manager:* --filter cluster-api-azure:* --ago 1d --untagged + disableWorkingDirectoryOverride: true + timeout: 3600 +''') + } + } +} + +resource e2eprivatecommunity 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = { + name: '${resource_prefix}e2e${random_suffix}' + location: location + sku: { + name: 'Premium' + } + properties:{ + anonymousPullEnabled: true + policies: { + retentionPolicy: { + days: 7 + status: 'enabled' + } + } + } +} + +resource capzsa 'Microsoft.Storage/storageAccounts@2022-05-01' = { + name: '${resource_prefix}sa${random_suffix}' + location: location + sku: { + name: 'Standard_ZRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + } +} + +resource cloudproviderblobcontributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('storage-rbac', cloudproviderId.id, capzsa.id, 'Storage Blob Data Contributor') + scope: capzsa + properties: { + principalId: cloudproviderId.properties.principalId + roleDefinitionId: tenantResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Storage Blob Data Contributor + principalType: 'ServicePrincipal' + description: 'Allow controlPlane VM to download private build from storage account' + } +} + +output capzci_registry_name string = capzci_registry.name +output capz_gmsa_kv_name string = gmsa_kv.name +output capzsastorage_name string = capzsa.name diff --git a/config/prow/cluster/prow-cluster.bicep b/config/prow/cluster/prow-cluster.bicep index 273eeec..8f8b2e5 100644 --- a/config/prow/cluster/prow-cluster.bicep +++ b/config/prow/cluster/prow-cluster.bicep @@ -99,11 +99,11 @@ resource aks 'Microsoft.ContainerService/managedClusters@2024-06-01' = { } } oidcIssuerProfile: { - enabled: true + enabled: true // windows capz prowjob need this } securityProfile: { workloadIdentity: { - enabled: true + enabled: true // windows capz prowjob need this } } } diff --git a/config/prow/k8s/test-pods/capz.yaml b/config/prow/k8s/test-pods/capz.yaml new file mode 100644 index 0000000..6101993 --- /dev/null +++ b/config/prow/k8s/test-pods/capz.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capz-clusteradmin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: azure + namespace: test-pods \ No newline at end of file diff --git a/config/prow/release-branch-jobs/1.27.yaml b/config/prow/release-branch-jobs/1.27.yaml index 2545dd1..48111d4 100644 --- a/config/prow/release-branch-jobs/1.27.yaml +++ b/config/prow/release-branch-jobs/1.27.yaml @@ -309,7 +309,7 @@ memory: 9Gi securityContext: privileged: true - - always_run: true + - always_run: true # pull-kubernetes-verify branches: - release-1.27-lts context: pull-kubernetes-verify @@ -345,5 +345,62 @@ requests: cpu: "7" memory: 16Gi + securityContext: + privileged: true + - always_run: false # pull-kubernetes-e2e-capz-windows + branches: + - release-1.27-lts + name: pull-kubernetes-e2e-capz-windows + decorate: true + decoration_config: + timeout: 4h + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + preset-capz-windows-common-pull: "true" + preset-capz-windows-2019: "true" + preset-capz-containerd-1-7-latest: "true" + preset-azure-community: "true" + extra_refs: + - base_ref: release-1.16 + org: kubernetes-sigs + path_alias: sigs.k8s.io/cluster-api-provider-azure + repo: cluster-api-provider-azure + workdir: false + - base_ref: release-1.27 + org: kubernetes-sigs + path_alias: sigs.k8s.io/cloud-provider-azure + repo: cloud-provider-azure + - org: haitch + repo: windows-testing + base_ref: master + path_alias: k8s.io/windows-testing + workdir: true + optional: true + path_alias: k8s.io/kubernetes + run_if_changed: azure.*\.go$|.*windows\.go$|test/e2e/windows/.* + spec: + serviceAccountName: azure + containers: + - command: + - "runner.sh" + - "env" + - "./capz/run-capz-e2e.sh" + image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240705-131cd74733-1.27 + name: "" + env: + - name: CAPI_VERSION + value: "v1.7.3" # newer version (v1.7.4 and beyond) use kubeadm.k8s.io/v1beta4, which is not support by 1.27 kubeadm. + - name: AZURE_LOCATION + value: "australiaeast" + - name: "KUBERNETES_VERSION" + value: "1.27" + resources: + requests: + cpu: "2" + memory: 9Gi + limits: + cpu: "2" + memory: 9Gi securityContext: privileged: true \ No newline at end of file diff --git a/config/prow/release-branch-jobs/base.yaml b/config/prow/release-branch-jobs/base.yaml index ed85db1..55a3366 100644 --- a/config/prow/release-branch-jobs/base.yaml +++ b/config/prow/release-branch-jobs/base.yaml @@ -44,16 +44,18 @@ plank: github_api_endpoints: - http://ghproxy - https://api.github.com - github_app_id: "$GITHUB_APP_ID" - github_app_private_key_secret: - name: github-token - key: cert + # disable for now + # github_app_id: "$GITHUB_APP_ID" + # github_app_private_key_secret: + # name: github-token + # key: cert s3_credentials_secret: s3-credentials utility_images: clonerefs: us-docker.pkg.dev/k8s-infra-prow/images/clonerefs:$K8S_PROW_IMAGE_TAG entrypoint: us-docker.pkg.dev/k8s-infra-prow/images/entrypoint:$K8S_PROW_IMAGE_TAG initupload: us-docker.pkg.dev/k8s-infra-prow/images/initupload:$K8S_PROW_IMAGE_TAG sidecar: us-docker.pkg.dev/k8s-infra-prow/images/sidecar:$K8S_PROW_IMAGE_TAG + blobless_fetch: true presets: # docker-in-docker (with images/bootstrap) preset @@ -108,6 +110,67 @@ presets: valueFrom: resourceFieldRef: resource: limits.cpu +# capz-windows-common-pull +- labels: + preset-capz-windows-common-pull: "true" + env: + - name: WINDOWS + value: "true" + - name: CONFORMANCE_NODES + value: "4" + - name: AZURE_NODE_MACHINE_TYPE + value: "Standard_D4s_v3" +# capz-windows-2019 +- labels: + preset-capz-windows-2019: "true" + env: + - name: WINDOWS_FLAVOR + value: "containerd" +# capz-windows-2019-latest +- labels: + preset-capz-containerd-1-7-latest: "true" + env: + - name: WINDOWS_CONTAINERD_URL + value: "https://github.com/containerd/containerd/releases/download/v1.7.18/containerd-1.7.18-windows-amd64.tar.gz" +# preset-azure-community +- labels: + preset-azure-community: "true" + env: # below env values are not secrets + - name: AZURE_CLIENT_ID + value: "07a59928-edb3-4a5c-a09b-56473261713c" + - name: AZURE_SUBSCRIPTION_ID + value: "$AZURE_SUBSCRIPTION_ID" + - name: AZURE_TENANT_ID + value: "72f988bf-86f1-41af-91ab-2d7cd011db47" + - name: AZURE_FEDERATED_TOKEN_FILE + value: "/var/run/secrets/azure-token/serviceaccount/token" + - name: AZURE_STORAGE_ACCOUNT + value: "$CAPZ_SA" + - name: REGISTRY + value: $CAPZ_CI_REGISTRY + - name: GMSA_KEYVAULT + value: $CAPZ_GMSA_KV + - name: USE_LOCAL_KIND_REGISTRY + value: "false" + - name: CI_RG + value: $CAPZ_RG + - name: TEST_K8S + value: "true" + - name: SKIP_CLEANUP + value: "true" + volumes: + - name: azure-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + expirationSeconds: 86400 + path: token + audience: api://AzureADTokenExchange + volumeMounts: + - mountPath: /var/run/secrets/azure-token/serviceaccount + name: azure-token + readOnly: true tide: queries: