diff --git a/Makefile b/Makefile
index e455cad33dc..5978ae79a7e 100644
--- a/Makefile
+++ b/Makefile
@@ -27,14 +27,6 @@ GOLANGCI_LINT_VERSION := v1.54.0
 GOLANGCI_LINT_BINDIR  := $(CURDIR)/.golangci-bin
 GOLANGCI_LINT_BIN     := $(GOLANGCI_LINT_BINDIR)/$(GOLANGCI_LINT_VERSION)/golangci-lint
 
-BUILD_TAG :=
-ifndef CUSTOM_BUILD_TAG
-	BUILD_TAG = $(shell build/images/build-tag.sh)
-else
-	BUILD_TAG = $(CUSTOM_BUILD_TAG)
-	DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG)
-endif
-
 DOCKER_BUILD_ARGS :=
 ifeq ($(NO_PULL),)
 	DOCKER_BUILD_ARGS += --pull
@@ -44,7 +36,6 @@ ifneq ($(NO_CACHE),)
 endif
 DOCKER_BUILD_ARGS += --build-arg OVS_VERSION=$(OVS_VERSION)
 DOCKER_BUILD_ARGS += --build-arg GO_VERSION=$(GO_VERSION)
-DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG)
 
 export CGO_ENABLED
 
@@ -53,6 +44,15 @@ all: build
 
 include versioning.mk
 
+BUILD_TAG :=
+ifndef CUSTOM_BUILD_TAG
+	BUILD_TAG = $(shell build/images/build-tag.sh)
+else
+	BUILD_TAG = $(CUSTOM_BUILD_TAG)
+	DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG)
+endif
+DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG)
+
 LDFLAGS += $(VERSION_LDFLAGS)
 
 UNAME_S := $(shell uname -s)
diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
index 988941848ec..ac06d842ed3 100755
--- a/ci/jenkins/test.sh
+++ b/ci/jenkins/test.sh
@@ -191,7 +191,7 @@ function clean_antrea {
     for antrea_yml in ${WORKDIR}/*.yml; do
         kubectl delete -f $antrea_yml --ignore-not-found=true || true
     done
-    docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi || true
+    docker images --format "{{.Repository}}:{{.Tag}}" | grep '${BUILD_TAG}' | xargs -r docker rmi || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
     check_and_cleanup_docker_build_cache
 }