From dfa5bc6e08e5e4b2578cb2cea6fa1f2ecdee8a4f Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Wed, 7 May 2025 21:26:15 +0000 Subject: [PATCH 1/4] [tmpnet] Define reusable flags for configuring kubernetes Previously `tmpnetctl start-kind-cluster` and the bootstrap-monitor determined their kubeconfig and context separately. Simpler to do it one way and this will be reusable by the kube node runtime. --- Taskfile.yml | 4 ++ flake.lock | 10 +-- flake.nix | 1 + scripts/start_kind_cluster.sh | 12 ++++ scripts/tests.e2e.bootstrap_monitor.sh | 5 +- tests/fixture/bootstrapmonitor/common.go | 3 +- .../fixture/bootstrapmonitor/e2e/e2e_test.go | 6 +- tests/fixture/tmpnet/flags/kube_config.go | 63 +++++++++++++++++++ tests/fixture/tmpnet/tmpnetctl/main.go | 35 ++++------- 9 files changed, 105 insertions(+), 34 deletions(-) create mode 100755 scripts/start_kind_cluster.sh create mode 100644 tests/fixture/tmpnet/flags/kube_config.go diff --git a/Taskfile.yml b/Taskfile.yml index fa6d64c18064..4082af03b6f6 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -89,6 +89,10 @@ tasks: desc: Generates protobuf cmd: ./scripts/protobuf_codegen.sh + ginkgo-build: + desc: Runs ginkgo against the current working directory + cmd: ./bin/ginkgo build {{.USER_WORKING_DIR}} + install-nix: desc: Installs nix with the determinate systems installer cmd: curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install diff --git a/flake.lock b/flake.lock index 188f5937e544..88eb13d83daa 100644 --- a/flake.lock +++ b/flake.lock @@ -2,12 +2,12 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1742512142, - "narHash": "sha256-8XfURTDxOm6+33swQJu/hx6xw1Tznl8vJJN5HwVqckg=", - "rev": "7105ae3957700a9646cc4b766f5815b23ed0c682", - "revCount": 715908, + "lastModified": 1742937945, + "narHash": "sha256-lWc+79eZRyvHp/SqMhHTMzZVhpxkRvthsP1Qx6UCq0E=", + "rev": "d02d88f8de5b882ccdde0465d8fa2db3aa1169f7", + "revCount": 716288, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2411.715908%2Brev-7105ae3957700a9646cc4b766f5815b23ed0c682/0195b8ff-82a6-7d19-b362-1f70dcb1c7f7/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2411.716288%2Brev-d02d88f8de5b882ccdde0465d8fa2db3aa1169f7/0195d574-d7fe-7866-9aa3-2e5ea0618cf6/source.tar.gz" }, "original": { "type": "tarball", diff --git a/flake.nix b/flake.nix index a3d1e647db3e..d9c1f97c6adf 100644 --- a/flake.nix +++ b/flake.nix @@ -45,6 +45,7 @@ # Kube tools kubectl # Kubernetes CLI + k9s # Kubernetes TUI kind # Kubernetes-in-Docker kubernetes-helm # Helm CLI (Kubernetes package manager) self.packages.${system}.kind-with-registry # Script installing kind configured with a local registry diff --git a/scripts/start_kind_cluster.sh b/scripts/start_kind_cluster.sh new file mode 100755 index 000000000000..f0f376fdeb5b --- /dev/null +++ b/scripts/start_kind_cluster.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Enable reuse of the arguments to ginkgo relevant to starting a cluster +START_CLUSTER_ARGS=() +for arg in "$@"; do + if [[ "${arg}" =~ "--kubeconfig" || "${arg}" =~ "--kubeconfig-context" ]]; then + START_CLUSTER_ARGS+=("${arg}") + fi +done +./bin/tmpnetctl start-kind-cluster "${START_CLUSTER_ARGS[@]}" diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh index 84763ff177af..8e0543f067c5 100755 --- a/scripts/tests.e2e.bootstrap_monitor.sh +++ b/scripts/tests.e2e.bootstrap_monitor.sh @@ -9,6 +9,5 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then exit 255 fi -./bin/tmpnetctl start-kind-cluster - -KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e +./scripts/start_kind_cluster.sh "$@" +./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e "$@" diff --git a/tests/fixture/bootstrapmonitor/common.go b/tests/fixture/bootstrapmonitor/common.go index 73d5da7aeb7d..e9032ff3cb10 100644 --- a/tests/fixture/bootstrapmonitor/common.go +++ b/tests/fixture/bootstrapmonitor/common.go @@ -19,6 +19,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet/flags" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/version" @@ -221,6 +222,6 @@ func getLatestImageDetails( func getClientset(log logging.Logger) (*kubernetes.Clientset, error) { log.Info("Initializing clientset") - kubeConfigPath := os.Getenv("KUBECONFIG") + kubeConfigPath := os.Getenv(flags.KubeConfigPathEnvVar) return tmpnet.GetClientset(log, kubeConfigPath, "") } diff --git a/tests/fixture/bootstrapmonitor/e2e/e2e_test.go b/tests/fixture/bootstrapmonitor/e2e/e2e_test.go index 7ccc67a9f7a2..9121898479e6 100644 --- a/tests/fixture/bootstrapmonitor/e2e/e2e_test.go +++ b/tests/fixture/bootstrapmonitor/e2e/e2e_test.go @@ -24,6 +24,7 @@ import ( "github.com/ava-labs/avalanchego/tests/fixture/bootstrapmonitor" "github.com/ava-labs/avalanchego/tests/fixture/e2e" "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet/flags" "github.com/ava-labs/avalanchego/utils/constants" appsv1 "k8s.io/api/apps/v1" @@ -60,6 +61,7 @@ const ( ) var ( + kubeconfigVars *flags.KubeconfigVars skipAvalanchegoImageBuild bool skipMonitorImageBuild bool @@ -67,6 +69,7 @@ var ( ) func init() { + kubeconfigVars = flags.NewKubeconfigFlagVars() flag.BoolVar( &skipAvalanchegoImageBuild, "skip-avalanchego-image-build", @@ -103,8 +106,7 @@ var _ = ginkgo.Describe("[Bootstrap Tester]", func() { } ginkgo.By("Configuring a kubernetes client") - kubeconfigPath := os.Getenv("KUBECONFIG") - kubeconfig, err := tmpnet.GetClientConfig(tc.Log(), kubeconfigPath, "") + kubeconfig, err := tmpnet.GetClientConfig(tc.Log(), kubeconfigVars.Path, kubeconfigVars.Context) require.NoError(err) clientset, err := kubernetes.NewForConfig(kubeconfig) require.NoError(err) diff --git a/tests/fixture/tmpnet/flags/kube_config.go b/tests/fixture/tmpnet/flags/kube_config.go new file mode 100644 index 000000000000..4c1600b8ef2e --- /dev/null +++ b/tests/fixture/tmpnet/flags/kube_config.go @@ -0,0 +1,63 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package flags + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/pflag" + + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" +) + +const KubeConfigPathEnvVar = "KUBECONFIG" + +type KubeconfigVars struct { + Path string + Context string +} + +// NewKubeconfigFlagVars registers kubeconfig flag variables for stdlib flag +func NewKubeconfigFlagVars() *KubeconfigVars { + return newKubeconfigFlagVars("") +} + +// internal method enabling configuration of the doc prefix +func newKubeconfigFlagVars(docPrefix string) *KubeconfigVars { + v := &KubeconfigVars{} + v.register(flag.StringVar, docPrefix) + return v +} + +// NewKubeconfigFlagSetVars registers kubeconfig flag variables for pflag +func NewKubeconfigFlagSetVars(flagSet *pflag.FlagSet) *KubeconfigVars { + return newKubeconfigFlagSetVars(flagSet, "") +} + +// internal method enabling configuration of the doc prefix +func newKubeconfigFlagSetVars(flagSet *pflag.FlagSet, docPrefix string) *KubeconfigVars { + v := &KubeconfigVars{} + v.register(flagSet.StringVar, docPrefix) + return v +} + +func (v *KubeconfigVars) register(stringVar varFunc[string], docPrefix string) { + stringVar( + &v.Path, + "kubeconfig", + tmpnet.GetEnvWithDefault(KubeConfigPathEnvVar, os.ExpandEnv("$HOME/.kube/config")), + docPrefix+fmt.Sprintf( + "The path to a kubernetes configuration file for the target cluster. Also possible to configure via the %s env variable.", + KubeConfigPathEnvVar, + ), + ) + stringVar( + &v.Context, + "kubeconfig-context", + "", + docPrefix+"The optional kubeconfig context to use", + ) +} diff --git a/tests/fixture/tmpnet/tmpnetctl/main.go b/tests/fixture/tmpnet/tmpnetctl/main.go index bd736119c0b8..99422cab6c51 100644 --- a/tests/fixture/tmpnet/tmpnetctl/main.go +++ b/tests/fixture/tmpnet/tmpnetctl/main.go @@ -12,7 +12,6 @@ import ( "path/filepath" "github.com/spf13/cobra" - "github.com/spf13/pflag" "go.uber.org/zap" "github.com/ava-labs/avalanchego/tests" @@ -24,7 +23,10 @@ import ( const cliVersion = "0.0.1" -var errNetworkDirRequired = fmt.Errorf("--network-dir or %s are required", tmpnet.NetworkDirEnvName) +var ( + errNetworkDirRequired = fmt.Errorf("--network-dir or %s is required", tmpnet.NetworkDirEnvName) + errKubeconfigRequired = errors.New("--kubeconfig is required") +) func main() { var ( @@ -226,10 +228,7 @@ func main() { ) rootCmd.AddCommand(checkLogsCmd) - var ( - kubeConfigPath string - kubeConfigContext string - ) + var kubeconfigVars *flags.KubeconfigVars startKindClusterCmd := &cobra.Command{ Use: "start-kind-cluster", Short: "Starts a local kind cluster with an integrated registry", @@ -240,10 +239,15 @@ func main() { if err != nil { return err } - return tmpnet.StartKindCluster(ctx, log, kubeConfigPath, kubeConfigContext) + // A valid kubeconfig is required for local kind usage but this is not validated by KubeconfigVars + // since unlike kind, tmpnet usage may involve an implicit in-cluster config. + if len(kubeconfigVars.Path) == 0 { + return errKubeconfigRequired + } + return tmpnet.StartKindCluster(ctx, log, kubeconfigVars.Path, kubeconfigVars.Context) }, } - SetKubeConfigFlags(startKindClusterCmd.PersistentFlags(), &kubeConfigPath, &kubeConfigContext) + kubeconfigVars = flags.NewKubeconfigFlagSetVars(startKindClusterCmd.PersistentFlags()) rootCmd.AddCommand(startKindClusterCmd) if err := rootCmd.Execute(); err != nil { @@ -252,18 +256,3 @@ func main() { } os.Exit(0) } - -func SetKubeConfigFlags(flagSet *pflag.FlagSet, kubeConfigPath *string, kubeConfigContext *string) { - flagSet.StringVar( - kubeConfigPath, - "kubeconfig", - os.Getenv("KUBECONFIG"), - "The path to a kubernetes configuration file for the target cluster", - ) - flagSet.StringVar( - kubeConfigContext, - "kubeconfig-context", - "", - "The path to a kubernetes configuration file for the target cluster", - ) -} From f3a6bc55c21428c3e38cb852af8fa07f9794b252 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Thu, 8 May 2025 00:36:45 +0000 Subject: [PATCH 2/4] fixup: Document requirement to use --arg=value for kubeconfig args --- scripts/start_kind_cluster.sh | 3 +++ scripts/tests.e2e.bootstrap_monitor.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/scripts/start_kind_cluster.sh b/scripts/start_kind_cluster.sh index f0f376fdeb5b..92a7b61a2cd0 100755 --- a/scripts/start_kind_cluster.sh +++ b/scripts/start_kind_cluster.sh @@ -2,6 +2,9 @@ set -euo pipefail +# --kubeconfig and --kubeconfig-context should be provided in the form --arg=value +# to work with the simplistic mechanism enabling flag reuse. + # Enable reuse of the arguments to ginkgo relevant to starting a cluster START_CLUSTER_ARGS=() for arg in "$@"; do diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh index 8e0543f067c5..3e761d5a1b5c 100755 --- a/scripts/tests.e2e.bootstrap_monitor.sh +++ b/scripts/tests.e2e.bootstrap_monitor.sh @@ -3,6 +3,9 @@ set -euo pipefail # Run e2e tests for bootstrap monitor. +# +# --kubeconfig and --kubeconfig-context should be provided in the form --arg=value +# to work with the simplistic mechanism enabling flag reuse. if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then echo "must be run from repository root" From dea4e91c65aecb35b705af64484b25bebf1a3dc6 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Thu, 8 May 2025 01:04:02 +0000 Subject: [PATCH 3/4] fixup: Respond to review feedback --- scripts/start_kind_cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start_kind_cluster.sh b/scripts/start_kind_cluster.sh index 92a7b61a2cd0..7dba75d16983 100755 --- a/scripts/start_kind_cluster.sh +++ b/scripts/start_kind_cluster.sh @@ -8,7 +8,7 @@ set -euo pipefail # Enable reuse of the arguments to ginkgo relevant to starting a cluster START_CLUSTER_ARGS=() for arg in "$@"; do - if [[ "${arg}" =~ "--kubeconfig" || "${arg}" =~ "--kubeconfig-context" ]]; then + if [[ "${arg}" =~ "--kubeconfig=" || "${arg}" =~ "--kubeconfig-context=" ]]; then START_CLUSTER_ARGS+=("${arg}") fi done From 2977b0ecc3c4577bb791a84a642d31aadf6382a9 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Mon, 12 May 2025 20:07:16 +0100 Subject: [PATCH 4/4] fixup: Consistently use kubeconfig as a word --- tests/fixture/bootstrapmonitor/common.go | 4 ++-- .../fixture/tmpnet/flags/{kube_config.go => kubeconfig.go} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename tests/fixture/tmpnet/flags/{kube_config.go => kubeconfig.go} (92%) diff --git a/tests/fixture/bootstrapmonitor/common.go b/tests/fixture/bootstrapmonitor/common.go index e9032ff3cb10..da675379e22d 100644 --- a/tests/fixture/bootstrapmonitor/common.go +++ b/tests/fixture/bootstrapmonitor/common.go @@ -222,6 +222,6 @@ func getLatestImageDetails( func getClientset(log logging.Logger) (*kubernetes.Clientset, error) { log.Info("Initializing clientset") - kubeConfigPath := os.Getenv(flags.KubeConfigPathEnvVar) - return tmpnet.GetClientset(log, kubeConfigPath, "") + kubeconfigPath := os.Getenv(flags.KubeconfigPathEnvVar) + return tmpnet.GetClientset(log, kubeconfigPath, "") } diff --git a/tests/fixture/tmpnet/flags/kube_config.go b/tests/fixture/tmpnet/flags/kubeconfig.go similarity index 92% rename from tests/fixture/tmpnet/flags/kube_config.go rename to tests/fixture/tmpnet/flags/kubeconfig.go index 4c1600b8ef2e..b8bfe9353094 100644 --- a/tests/fixture/tmpnet/flags/kube_config.go +++ b/tests/fixture/tmpnet/flags/kubeconfig.go @@ -13,7 +13,7 @@ import ( "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" ) -const KubeConfigPathEnvVar = "KUBECONFIG" +const KubeconfigPathEnvVar = "KUBECONFIG" type KubeconfigVars struct { Path string @@ -48,10 +48,10 @@ func (v *KubeconfigVars) register(stringVar varFunc[string], docPrefix string) { stringVar( &v.Path, "kubeconfig", - tmpnet.GetEnvWithDefault(KubeConfigPathEnvVar, os.ExpandEnv("$HOME/.kube/config")), + tmpnet.GetEnvWithDefault(KubeconfigPathEnvVar, os.ExpandEnv("$HOME/.kube/config")), docPrefix+fmt.Sprintf( "The path to a kubernetes configuration file for the target cluster. Also possible to configure via the %s env variable.", - KubeConfigPathEnvVar, + KubeconfigPathEnvVar, ), ) stringVar(