Skip to content

Commit

Permalink
[CRDB-47257] Setup local registry for testing self-signer on local k3…
Browse files Browse the repository at this point in the history
…d cluster
  • Loading branch information
NishanthNalluri committed Feb 11, 2025
1 parent 65a993d commit dcfebb2
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 40 deletions.
3 changes: 1 addition & 2 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ name: Helm Chart Package CI
on:
pull_request:
branches:
- 'master'
- 'cert-manager-feature-branch'
- '*'

jobs:

Expand Down
20 changes: 13 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ endif
K3D_CLUSTER ?= chart-testing
REGISTRY ?= gcr.io
REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
DOCKER_NETWORK_NAME ?= ${K3D_CLUSTER}
DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
LOCAL_REGISTRY ?= "localhost:5000"
CLUSTER_SIZE ?= 1

export BUNDLE_IMAGE ?= cockroach-operator-bundle
export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
Expand Down Expand Up @@ -89,20 +90,25 @@ dev/registries/down: bin/k3d
cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
fi

dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up

dev/push/local: dev/registries/up
@echo "$(CYAN)Pushing image to local registry...$(NC)"
@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
@docker push "${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)"

##@ Test
test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing
test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing

test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing

test/cluster_up: bin/k3d
@bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER)
test/cluster/up: bin/k3d
@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)

test/cluster_down: bin/k3d
bin/k3d cluster delete $(K3D_CLUSTER)
test/cluster/down: bin/k3d
./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"

test/e2e/%: PKG=$*
test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
Expand Down
87 changes: 56 additions & 31 deletions tests/k3d/dev-cluster.sh
Original file line number Diff line number Diff line change
@@ -1,52 +1,77 @@
#!/usr/bin/env bash
region="us-east-1"
zones=3

CLUSTER_NAME=local
K3D_PATH="./bin/k3d"

NETWORK_NAME=k3d-local
if [ $# -eq 0 ]; then
echo "No arguments supplied: "
echo " up: Start cluster."
echo " --nodes x: The cluster should have x nodes (default 1)"
echo " --version x: The version of Kubernetes (default 1.24.14)"
echo " --name x: The name of the cluster (default local)"
echo " --network_name x: The name of the cluster's network (default k3d-\${name})"
echo " --region x: The name of the cluster's region for node labels topology.kubernetes.io/region (default us-east-1)"
echo " --zones x: The number of zones in the region for node labels topology.kubernetes.io/zone (default 3)"

if [ $# -eq 0 ]
then
echo "No arguments supplied: "
echo " up: Start cluster."
echo " --nodes x: The cluster should have x nodes (default 1)"
echo " --version x: The version of Kubernetes (default 1.24.14)"
echo " down: Delete cluster."
echo " down: Delete cluster."

exit 1
exit 1
fi

COMMAND="${1-}"
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)

nodes=${environment:-1}
version=${version:-1.24.14}
version=${version:-1.31.2}

while [ $# -gt 0 ]; do

if [[ $1 == *"--"* ]]; then
param="${1/--/}"
declare $param="$2"
# echo $1 $2 // Optional to see the parameter:value result
fi
if [[ $1 == *"--"* ]]; then
param="${1/--/}"
declare $param="$2"
# echo $1 $2 // Optional to see the parameter:value result
fi

shift
done

name=${name:-local}
network_name=${network_name:-"k3d-${name}"}

# Function to set topology.kubernetes.io/zone labels in a round-robin fashion
set_node_labels() {
local nodes=$1
local region=$2
local zones=$3
local labels=""
local az=(a b c d e f g h i j k l m n o p q r s t u v w x y z)

for ((i = 0; i < nodes; i++)); do
zone="${region}${az[$((i % zones))]}"
labels+="--k3s-node-label topology.kubernetes.io/zone=${zone}@agent:${i} "
labels+="--k3s-node-label topology.kubernetes.io/region=${region}@agent:${i} "
done

echo "${labels}"
}

case $COMMAND in
up)
k3d cluster create ${CLUSTER_NAME} \
--network ${NETWORK_NAME} \
--registry-config "$SCRIPT_DIR/registries.yaml" \
--image rancher/k3s:v${version}-k3s1 \
--agents ${nodes} \
--k3s-node-label "topology.kubernetes.io/region=us-east-1@agent:0" \
--k3s-node-label "topology.kubernetes.io/region=us-east-1@server:0"
up)
node_labels=$(set_node_labels ${nodes} ${region} ${zones})
${K3D_PATH} cluster create ${name} \
--network ${network_name} \
--registry-config "$SCRIPT_DIR/registries.yaml" \
--image rancher/k3s:v${version}-k3s1 \
--agents ${nodes} \
--k3s-node-label "topology.kubernetes.io/region=${region}@server:0" \
${node_labels}
;;
down)
k3d cluster delete ${CLUSTER_NAME}
down)
${K3D_PATH} cluster delete ${name}
;;
*)
echo "Unknown command: $COMMAND"
exit 1;
*)
echo "Unknown command: $COMMAND"
exit 1
;;
esac
esac

0 comments on commit dcfebb2

Please sign in to comment.