Skip to content

Commit

Permalink
Setup local registry for faster iteration on self-signer dev
Browse files Browse the repository at this point in the history
  • Loading branch information
udnay authored and NishanthNalluri committed Feb 11, 2025
1 parent 116b559 commit 65a993d
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 5 deletions.
33 changes: 28 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
UNAME_S := $(shell uname -s)
NC := $(shell tput sgr0) # No Color
ifeq ($(UNAME_S),Linux)
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
Expand All @@ -21,7 +22,10 @@ ifeq ($(UNAME_S),Darwin)
endif

K3D_CLUSTER ?= chart-testing
REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
REGISTRY ?= gcr.io
REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
DOCKER_NETWORK_NAME ?= ${K3D_CLUSTER}
LOCAL_REGISTRY ?= "localhost:5000"

export BUNDLE_IMAGE ?= cockroach-operator-bundle
export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
Expand Down Expand Up @@ -56,7 +60,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
build/self-signer: bin/yq ## build the self-signer image
@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .

##@ Release

Expand All @@ -66,12 +70,31 @@ release: ## publish the build artifacts to S3
build-and-push/self-signer: bin/yq ## push the self-signer image
@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .

##@ Dev
dev/clean: ## remove built artifacts
@rm -r build/artifacts/

## Setup/teardown registries for easier local dev
dev/registries/up: bin/k3d
@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
cd ../../bin/k3d; ./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
fi

dev/registries/down: bin/k3d
@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
fi

dev/push/local: dev/registries/up
@echo "$(CYAN)Pushing image to local registry...$(NC)"
@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .

##@ Test
test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing

Expand All @@ -94,9 +117,9 @@ test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach
docker pull $$i; \
bin/k3d image import $$i -c $(K3D_CLUSTER); \
done
docker pull ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
docker pull ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
bin/k3d image import \
${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
-c $(K3D_CLUSTER)

test/template: bin/cockroach bin/helm ## Run template tests
Expand Down
52 changes: 52 additions & 0 deletions tests/k3d/dev-cluster.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/usr/bin/env bash

CLUSTER_NAME=local

NETWORK_NAME=k3d-local

if [ $# -eq 0 ]
then
echo "No arguments supplied: "
echo " up: Start cluster."
echo " --nodes x: The cluster should have x nodes (default 1)"
echo " --version x: The version of Kubernetes (default 1.24.14)"
echo " down: Delete cluster."

exit 1
fi

COMMAND="${1-}"
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

nodes=${environment:-1}
version=${version:-1.24.14}

while [ $# -gt 0 ]; do

if [[ $1 == *"--"* ]]; then
param="${1/--/}"
declare $param="$2"
# echo $1 $2 // Optional to see the parameter:value result
fi

shift
done

case $COMMAND in
up)
k3d cluster create ${CLUSTER_NAME} \
--network ${NETWORK_NAME} \
--registry-config "$SCRIPT_DIR/registries.yaml" \
--image rancher/k3s:v${version}-k3s1 \
--agents ${nodes} \
--k3s-node-label "topology.kubernetes.io/region=us-east-1@agent:0" \
--k3s-node-label "topology.kubernetes.io/region=us-east-1@server:0"
;;
down)
k3d cluster delete ${CLUSTER_NAME}
;;
*)
echo "Unknown command: $COMMAND"
exit 1;
;;
esac
65 changes: 65 additions & 0 deletions tests/k3d/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
version: "3.3"

#
# Each proxy needs a 'dns' section added. When it wasn't there, the registries were unable to resolve
# remote DNS names, only DNS names set up within the network of the docker-compose. Here the dns is pointed
# at the Google DNS servers.
#

services:
registry-localhost:
image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
restart: "always"
volumes:
- "registry:/var/lib/registry"
ports:
- "5000:5000"

registry-quayio:
image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
restart: "always"
dns:
- 8.8.8.8
volumes:
- "registry:/var/lib/registry"
environment:
REGISTRY_PROXY_REMOTEURL: "https://quay.io/repository"
REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED: "true"

registry-dockerio:
image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
restart: "always"
dns:
- 8.8.8.8
volumes:
- "registry:/var/lib/registry"
environment:
REGISTRY_PROXY_REMOTEURL: "http://registry-1.docker.io"

registry-us-gcr-io:
image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
restart: "always"
dns:
- 8.8.8.8
volumes:
- "registry:/var/lib/registry"
environment:
REGISTRY_PROXY_REMOTEURL: "https://us.gcr.io"

us-docker-pkg-dev:
image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
restart: "always"
dns:
- 8.8.8.8
volumes:
- "registry:/var/lib/registry"
environment:
REGISTRY_PROXY_REMOTEURL: "https://us-docker.pkg.dev"

volumes:
registry: { }

networks:
default:
external: true
name: ${DOCKER_NETWORK_NAME}
32 changes: 32 additions & 0 deletions tests/k3d/registries.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

set -euxo pipefail

# Figure out, regardless of any symlinks, aliases, etc, where this script
# is located.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"


COMMAND="${1-}"

DOCKER_REGISTRY_PROJECT_NAME=${2:-k3d-local}

# The name of the docker network. This must change if it changes in the docker-compose.yaml file.
DOCKER_REGISTRY_NETWORK_NAME=${2:-k3d-local}


case $COMMAND in
up)
docker network create --driver bridge ${DOCKER_REGISTRY_NETWORK_NAME} || true
DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml up -d
;;
down)
DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml down
;;
*)
echo "Unknown command: $COMMAND"
exit 1;
;;
esac
16 changes: 16 additions & 0 deletions tests/k3d/registries.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
mirrors:
"localhost:5000":
endpoint:
- "http://registry-localhost:5000"
quay.io:
endpoint:
- "http://registry-quayio:5000"
docker.io:
endpoint:
- "http://registry-dockerio:5000"
us.gcr.io:
endpoint:
- "http://registry-us-gcr-io:5000"
us-docker.pkg.dev:
endpoint:
- "http://us-docker-pkg-dev:5000"

0 comments on commit 65a993d

Please sign in to comment.