Test/multiarch clone

Makefile

@@ -66,14 +66,24 @@ ifneq ($(IS_ARM_MACHINE), )
 	ifneq ($(GOARCH), amd64)
 		GOARCH := arm64
 	endif
-	PLATFORM := --platform=linux/$(GOARCH)
 else
 	# currently we only support arm64 and amd64 as a GOARCH option.
 	ifneq ($(GOARCH), arm64)
 		GOARCH := amd64
 	endif
 endif
 
+PLATFORM := --platform=linux/$(GOARCH)
+PLATFORM_MULTIARCH := $(PLATFORM)
+LOAD_OR_PUSH := --load
+ifeq ($(MULTIARCH), true)
+	PLATFORM_MULTIARCH := --platform=linux/amd64,linux/arm64
+	LOAD_OR_PUSH :=
+
+	ifeq ($(MULTIARCH_PUSH), true)
+		LOAD_OR_PUSH := --push
+	endif
+endif
 
 GOOS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
 
@@ -105,7 +115,7 @@ UTILS_DONOR_IMAGE ?= busybox:uclibc
 # https://github.com/solo-io/envoy-gloo-ee/blob/main/ci/Dockerfile#L7 - check /etc/debian_version in the ubuntu version used
 # This is the true base image for GLOO_DISTROLESS_BASE_IMAGE and GLOO_DISTROLESS_BASE_WITH_UTILS_IMAGE
 # Since we only publish amd64 images, we use the amd64 variant. If we decide to change this, we need to update the distroless dockerfiles as well
-DISTROLESS_BASE_IMAGE ?= gcr.io/distroless/base-debian11:latest-amd64
+DISTROLESS_BASE_IMAGE ?= gcr.io/distroless/base-debian11:latest
 # DISTROLESS_BASE_IMAGE + ca-certificates
 GLOO_DISTROLESS_BASE_IMAGE ?= $(IMAGE_REGISTRY)/distroless-base:$(VERSION)
 # GLOO_DISTROLESS_BASE_IMAGE + utility binaries (sh, wget, sleep, nc, echo, ls, cat, vi)
@@ -498,10 +508,9 @@ $(DISTROLESS_OUTPUT_DIR)/Dockerfile: $(DISTROLESS_DIR)/Dockerfile
 
 .PHONY: distroless-docker
 distroless-docker: $(DISTROLESS_OUTPUT_DIR)/Dockerfile
-	docker buildx build --load $(PLATFORM) $(DISTROLESS_OUTPUT_DIR) -f $(DISTROLESS_OUTPUT_DIR)/Dockerfile \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(DISTROLESS_OUTPUT_DIR) -f $(DISTROLESS_OUTPUT_DIR)/Dockerfile \
 		--build-arg PACKAGE_DONOR_IMAGE=$(PACKAGE_DONOR_IMAGE) \
 		--build-arg BASE_IMAGE=$(DISTROLESS_BASE_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t $(GLOO_DISTROLESS_BASE_IMAGE) $(QUAY_EXPIRATION_LABEL)
 
 $(DISTROLESS_OUTPUT_DIR)/Dockerfile.utils: $(DISTROLESS_DIR)/Dockerfile.utils
@@ -510,10 +519,9 @@ $(DISTROLESS_OUTPUT_DIR)/Dockerfile.utils: $(DISTROLESS_DIR)/Dockerfile.utils
 
 .PHONY: distroless-with-utils-docker
 distroless-with-utils-docker: distroless-docker $(DISTROLESS_OUTPUT_DIR)/Dockerfile.utils
-	docker buildx build --load $(PLATFORM) $(DISTROLESS_OUTPUT_DIR) -f $(DISTROLESS_OUTPUT_DIR)/Dockerfile.utils \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(DISTROLESS_OUTPUT_DIR) -f $(DISTROLESS_OUTPUT_DIR)/Dockerfile.utils \
 		--build-arg UTILS_DONOR_IMAGE=$(UTILS_DONOR_IMAGE) \
 		--build-arg BASE_IMAGE=$(GLOO_DISTROLESS_BASE_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t  $(GLOO_DISTROLESS_BASE_WITH_UTILS_IMAGE) $(QUAY_EXPIRATION_LABEL)
 
 #----------------------------------------------------------------------------------
@@ -802,19 +810,17 @@ $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen: $(CERTGEN_DIR)/Dockerfile
 
 .PHONY: certgen-docker
 certgen-docker: $(CERTGEN_OUTPUT_DIR)/certgen-linux-$(GOARCH) $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen
-	docker buildx build --load $(PLATFORM) $(CERTGEN_OUTPUT_DIR) -f $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(CERTGEN_OUTPUT_DIR) -f $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen \
 		--build-arg BASE_IMAGE=$(ALPINE_BASE_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t $(IMAGE_REGISTRY)/certgen:$(VERSION) $(QUAY_EXPIRATION_LABEL)
 
 $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen.distroless: $(CERTGEN_DIR)/Dockerfile.distroless
 	cp $< $@
 
 .PHONY: certgen-distroless-docker
 certgen-distroless-docker: $(CERTGEN_OUTPUT_DIR)/certgen-linux-$(GOARCH) $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen.distroless distroless-docker
-	docker buildx build --load $(PLATFORM) $(CERTGEN_OUTPUT_DIR) -f $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen.distroless \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(CERTGEN_OUTPUT_DIR) -f $(CERTGEN_OUTPUT_DIR)/Dockerfile.certgen.distroless \
 		--build-arg BASE_IMAGE=$(GLOO_DISTROLESS_BASE_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t $(IMAGE_REGISTRY)/certgen:$(VERSION)-distroless $(QUAY_EXPIRATION_LABEL)
 
 #----------------------------------------------------------------------------------
@@ -830,9 +836,8 @@ $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl: $(KUBECTL_DIR)/Dockerfile
 
 .PHONY: kubectl-docker
 kubectl-docker: $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl
-	docker buildx build --load $(PLATFORM) $(KUBECTL_OUTPUT_DIR) -f $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(KUBECTL_OUTPUT_DIR) -f $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl \
 		--build-arg BASE_IMAGE=$(ALPINE_BASE_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t $(IMAGE_REGISTRY)/kubectl:$(VERSION) $(QUAY_EXPIRATION_LABEL)
 
 $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl.distroless: $(KUBECTL_DIR)/Dockerfile.distroless
@@ -841,9 +846,8 @@ $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl.distroless: $(KUBECTL_DIR)/Dockerfile.d
 
 .PHONY: kubectl-distroless-docker
 kubectl-distroless-docker: $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl.distroless distroless-with-utils-docker
-	docker buildx build --load $(PLATFORM) $(KUBECTL_OUTPUT_DIR) -f $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl.distroless \
+	docker buildx build $(LOAD_OR_PUSH) $(PLATFORM_MULTIARCH) $(KUBECTL_OUTPUT_DIR) -f $(KUBECTL_OUTPUT_DIR)/Dockerfile.kubectl.distroless \
 		--build-arg BASE_IMAGE=$(GLOO_DISTROLESS_BASE_WITH_UTILS_IMAGE) \
-		--build-arg GOARCH=$(GOARCH) \
 		-t $(IMAGE_REGISTRY)/kubectl:$(VERSION)-distroless $(QUAY_EXPIRATION_LABEL)
 
 #----------------------------------------------------------------------------------
@@ -1011,20 +1015,28 @@ docker-standard-push: docker-push-gloo
 docker-standard-push: docker-push-discovery
 docker-standard-push: docker-push-gloo-envoy-wrapper
 docker-standard-push: docker-push-sds
+ifeq ($(MULTIARCH), )
 docker-standard-push: docker-push-certgen
+endif
 docker-standard-push: docker-push-ingress
 docker-standard-push: docker-push-access-logger
+ifeq ($(MULTIARCH), )
 docker-standard-push: docker-push-kubectl
+endif
 
 .PHONY: docker-distroless-push
 docker-distroless-push: docker-push-gloo-distroless
 docker-distroless-push: docker-push-discovery-distroless
 docker-distroless-push: docker-push-gloo-envoy-wrapper-distroless
 docker-distroless-push: docker-push-sds-distroless
+ifeq ($(MULTIARCH), )
 docker-distroless-push: docker-push-certgen-distroless
+endif
 docker-distroless-push: docker-push-ingress-distroless
 docker-distroless-push: docker-push-access-logger-distroless
+ifeq ($(MULTIARCH), )
 docker-distroless-push: docker-push-kubectl-distroless
+endif
 
 # Push docker images to the defined IMAGE_REGISTRY
 .PHONY: docker-push

changelog/v1.19.0-beta1/ci-multiarch-updates.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NON_USER_FACING
+    description: >-
+      Multiarch updates for base and test images.
+
+      skipCI-docs-build:true

ci/cloudbuild/publish-artifacts.yaml

@@ -43,16 +43,37 @@ steps:
   - 'compute/zone'
   - 'us-central1-a'
 
+# Initialize Docker Buildx for multiarch builds
+- name: 'gcr.io/cloud-builders/docker'
+  id: 'init-buildx'
+  args:
+  - buildx
+  - create
+  - --use
+
+- name: 'gcr.io/$PROJECT_ID/go-mod-make:0.10.2'
+  id: 'build-certgen-arm64-binary'
+  args:
+  - 'certgen-docker'
+  dir: *dir
+  env:
+  - 'GOARCH=arm64'
+
 # Run make targets to push docker images to quay.io
 - name: 'gcr.io/$PROJECT_ID/go-mod-make:0.10.2'
   id: 'publish-docker'
   args:
   - 'publish-docker'
   dir: *dir
+  env:
+  - 'MULTIARCH=true'
+  - 'MULTIARCH_PUSH=true'
   secretEnv:
   - 'GITHUB_TOKEN'
   waitFor:
   - 'docker-login'
+  - 'init-buildx'
+  - 'build-certgen-arm64-binary'
 
 # Publish helm chart, compile manifests, produce release artifacts, deploy docs
 # isolating this portion of the release in order to force the manifest to be regenerated with the tagged version

jobs/certgen/cmd/Dockerfile

@@ -1,16 +1,14 @@
 ARG BASE_IMAGE
 
 FROM $BASE_IMAGE
-
-
-ARG GOARCH=amd64
+ARG TARGETARCH
 
 # Needed for access to AWS
 RUN apk upgrade --update-cache \
     && apk add ca-certificates \
     && rm -rf /var/cache/apk/*
 
-COPY certgen-linux-$GOARCH /usr/local/bin/certgen
+COPY certgen-linux-$TARGETARCH /usr/local/bin/certgen
 
 USER 10101
 

jobs/certgen/cmd/Dockerfile.distroless

@@ -1,10 +1,9 @@
 ARG BASE_IMAGE
 
-
 FROM $BASE_IMAGE
-ARG GOARCH=amd64
+ARG TARGETARCH
 
-COPY certgen-linux-$GOARCH /usr/local/bin/certgen
+COPY certgen-linux-$TARGETARCH /usr/local/bin/certgen
 
 USER 10101
 

projects/distroless/Dockerfile

@@ -1,15 +1,18 @@
-# Since we only publish amd64 images, we use the amd64 variant.
-
 ARG PACKAGE_DONOR_IMAGE
 ARG BASE_IMAGE
+ARG TARGETARCH
+
+FROM --platform=linux/amd64 $PACKAGE_DONOR_IMAGE AS donor-amd64
+FROM --platform=linux/arm64 $PACKAGE_DONOR_IMAGE AS donor-arm64
+
+FROM $PACKAGE_DONOR_IMAGE AS donor
 
-FROM --platform=linux/amd64 $PACKAGE_DONOR_IMAGE as donor
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update \
     && apt-get upgrade -y \
     && apt-get install --no-install-recommends -y ca-certificates
 
-FROM --platform=linux/amd64 $BASE_IMAGE
+FROM $BASE_IMAGE
 
 # ca-certificates depends on openssl
 COPY --from=donor /etc/ssl /etc/ssl
@@ -18,9 +21,13 @@ COPY --from=donor /usr/bin/c_rehash usr/bin/c_rehash
 COPY --from=donor /usr/bin/openssl /usr/bin/openssl
 
 # openssl depends on libssl1.1
-COPY --from=donor /usr/lib/x86_64-linux-gnu/engines-1.1 /usr/lib/x86_64-linux-gnu/engines-1.1
-COPY --from=donor /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
-COPY --from=donor /usr/lib/x86_64-linux-gnu/libssl.so.1.1 /usr/lib/x86_64-linux-gnu/libssl.so.1.1
+COPY --from=donor-amd64 /usr/lib/x86_64-linux-gnu/engines-1.1 /usr/lib/x86_64-linux-gnu/engines-1.1
+COPY --from=donor-amd64 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
+COPY --from=donor-amd64 /usr/lib/x86_64-linux-gnu/libssl.so.1.1 /usr/lib/x86_64-linux-gnu/libssl.so.1.1
+
+COPY --from=donor-arm64 /usr/lib/aarch64-linux-gnu/engines-1.1 /usr/lib/aarch64-linux-gnu/engines-1.1
+COPY --from=donor-arm64 /usr/lib/aarch64-linux-gnu/libcrypto.so.1.1 /usr/lib/aarch64-linux-gnu/libcrypto.so.1.1
+COPY --from=donor-arm64 /usr/lib/aarch64-linux-gnu/libssl.so.1.1 /usr/lib/aarch64-linux-gnu/libssl.so.1.1
 
 # Copy over the certs
 COPY --from=donor /usr/share/ca-certificates /usr/share/ca-certificates

test/kube2e/containers/testrunner/Dockerfile

@@ -1,10 +1,12 @@
+# docker buildx build --push --platform=linux/amd64,linux/arm64 -t gcr.io/solo-test-236622/testrunner:v1.7.0-beta18 .
 FROM ubuntu:22.04
 
 RUN apt update && apt install -y curl
 COPY --from=lachlanevenson/k8s-kubectl:v1.10.3 /usr/local/bin/kubectl /usr/local/bin/kubectl
 
 # Python
-RUN apt-get install -y python; apt clean
+RUN apt-get install -y python2; apt clean
+RUN ln -s /bin/python2 /bin/python
 
 COPY root.crt /
 

test/kube2e/helper/http_echo.go

@@ -1,7 +1,8 @@
 package helper
 
 const (
-	defaultHttpEchoImage = "kennship/http-echo@sha256:144322e8e96be2be6675dcf6e3ee15697c5d052d14d240e8914871a2a83990af"
+	// kennship/http-echo@sha256:144322e8e96be2be6675dcf6e3ee15697c5d052d14d240e8914871a2a83990af
+	defaultHttpEchoImage = "gcr.io/solo-test-236622/kennship-http-echo:latest"
 	HttpEchoName         = "http-echo"
 	HttpEchoPort         = 3000
 )

test/kube2e/helper/testserver.go

@@ -10,74 +10,15 @@ import (
 )
 
 const (
-	defaultTestServerImage = "quay.io/solo-io/testrunner:v1.7.0-beta17"
+	defaultTestServerImage = "gcr.io/solo-test-236622/testrunner:v1.7.0-beta18"
 	TestServerName         = "testserver"
 	TestServerPort         = 1234
 
 	// This response is given by the testserver when the SimpleServer is started
 	SimpleHttpResponse = `<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"><html>
-<title>Directory listing for /</title>
-<body>
-<h2>Directory listing for /</h2>
-<hr>
-<ul>
-<li><a href="bin/">bin/</a>
-<li><a href="boot/">boot/</a>
-<li><a href="dev/">dev/</a>
-<li><a href="etc/">etc/</a>
-<li><a href="home/">home/</a>
-<li><a href="lib/">lib/</a>
-<li><a href="lib64/">lib64/</a>
-<li><a href="media/">media/</a>
-<li><a href="mnt/">mnt/</a>
-<li><a href="opt/">opt/</a>
-<li><a href="proc/">proc/</a>
-<li><a href="product_name">product_name</a>
-<li><a href="product_uuid">product_uuid</a>
-<li><a href="root/">root/</a>
-<li><a href="root.crt">root.crt</a>
-<li><a href="run/">run/</a>
-<li><a href="sbin/">sbin/</a>
-<li><a href="srv/">srv/</a>
-<li><a href="sys/">sys/</a>
-<li><a href="tmp/">tmp/</a>
-<li><a href="usr/">usr/</a>
-<li><a href="var/">var/</a>
-</ul>
-<hr>
-</body>
-</html>`
+<title>Directory listing for /</title>`
 	SimpleHttpResponseArm = `<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"><html>
-<title>Directory listing for /</title>
-<body>
-<h2>Directory listing for /</h2>
-<hr>
-<ul>
-<li><a href="bin/">bin/</a>
-<li><a href="boot/">boot/</a>
-<li><a href="dev/">dev/</a>
-<li><a href="etc/">etc/</a>
-<li><a href="home/">home/</a>
-<li><a href="lib/">lib/</a>
-<li><a href="lib64/">lib64/</a>
-<li><a href="media/">media/</a>
-<li><a href="mnt/">mnt/</a>
-<li><a href="opt/">opt/</a>
-<li><a href="proc/">proc/</a>
-<li><a href="product_uuid">product_uuid</a>
-<li><a href="root/">root/</a>
-<li><a href="root.crt">root.crt</a>
-<li><a href="run/">run/</a>
-<li><a href="sbin/">sbin/</a>
-<li><a href="srv/">srv/</a>
-<li><a href="sys/">sys/</a>
-<li><a href="tmp/">tmp/</a>
-<li><a href="usr/">usr/</a>
-<li><a href="var/">var/</a>
-</ul>
-<hr>
-</body>
-</html>`
+<title>Directory listing for /</title>`
 )
 
 // tests relying on the test server should be ported using the default nginx deployment located at

test/kube2e/util.go

@@ -38,7 +38,7 @@ const (
 func GetHttpEchoImage() string {
 	httpEchoImage := "hashicorp/http-echo"
 	if runtime.GOARCH == "arm64" {
-		httpEchoImage = "gcr.io/solo-test-236622/http-echo"
+		httpEchoImage = "gcr.io/solo-test-236622/http-echo:0.2.4"
 	}
 	return httpEchoImage
 }

test/kubernetes/testutils/helper/util.go

@@ -24,7 +24,7 @@ import (
 func GetHttpEchoImage() string {
 	httpEchoImage := "hashicorp/http-echo"
 	if runtime.GOARCH == "arm64" {
-		httpEchoImage = "gcr.io/solo-test-236622/http-echo"
+		httpEchoImage = "gcr.io/solo-test-236622/http-echo:0.2.4"
 	}
 	return httpEchoImage
 }