[tmpnet] Refactor in preparation for kube

Author: maru-ava

flake.lock

@@ -39,6 +39,7 @@
 
             # Kube tools
             kubectl                                    # Kubernetes CLI
+            k9s                                        # Kubernetes TUI
             kind                                       # Kubernetes-in-Docker
             kubernetes-helm                            # Helm CLI (Kubernetes package manager)
             self.packages.${system}.kind-with-registry # Script installing kind configured with a local registry

flake.nix

@@ -87,5 +87,5 @@ else
                                 "${AVALANCHE_PATH}/build/antithesis/xsvm" \
                                 "AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVAGO_PLUGIN_DIR=${AVALANCHE_PATH}/build/plugins"
 
-  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
+  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/Dockerfile.node"
 fi

scripts/build_antithesis_images.sh

@@ -9,6 +9,8 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
   exit 255
 fi
 
+export KUBECONFIG="$HOME/.kube/config"
+
 ./bin/tmpnetctl start-kind-cluster
 
-KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e
+./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e

scripts/tests.e2e.bootstrap_monitor.sh

@@ -20,14 +20,17 @@ fi
 # the instructions to build non-portable BLST.
 source ./scripts/constants.sh
 
-# Enable subnet testing by building xsvm
-./scripts/build_xsvm.sh
-echo ""
-
 # Ensure an absolute path to avoid dependency on the working directory
 # of script execution.
-AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
-E2E_ARGS="--avalanchego-path=${AVALANCHEGO_PATH}"
+E2E_ARGS="${*:-}"
+if ! [[ "${E2E_ARGS}" =~ "--runtime=kube" ]]; then
+  # If not running in kubernetes, use the local avalanchego binary
+  AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
+  E2E_ARGS+=" --avalanchego-path=${AVALANCHEGO_PATH}"
+
+  # Enable subnet testing by building the xsvm binary
+  ./scripts/build_xsvm.sh
+fi
 
 #################################
 # Determine ginkgo args
@@ -58,5 +61,5 @@ else
 fi
 
 #################################
-# shellcheck disable=SC2086
-./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}"
+# shellcheck disable=SC2086,SC2068
+./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- ${E2E_ARGS[@]}

scripts/tests.e2e.sh

@@ -174,7 +174,7 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
 		}
 
 		// Apply configuration appropriate to a test network
-		for k, v := range tmpnet.DefaultTestFlags() {
+		for k, v := range tmpnet.DefaultTmpnetFlags() {
 			switch value := v.(type) {
 			case string:
 				env[k] = value

tests/antithesis/compose.go

@@ -61,23 +61,27 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	require.NoError(tc, err)
 
 	upgradeBase64 := base64.StdEncoding.EncodeToString(upgradeJSON)
+
+	defaultFlags := tmpnet.FlagsMap{
+		config.UpgradeFileContentKey: upgradeBase64,
+		// Ensure a min stake duration compatible with testing staking logic
+		config.MinStakeDurationKey: "1s",
+	}
+	defaultFlags.SetDefaults(tmpnet.DefaultE2EFlags())
+
 	return e2e.NewTestEnvironment(
 		tc,
 		flagVars,
 		&tmpnet.Network{
-			Owner: flagVars.NetworkOwner(),
-			DefaultFlags: tmpnet.FlagsMap{
-				config.UpgradeFileContentKey: upgradeBase64,
-				// Ensure a min stake duration compatible with testing staking logic
-				config.MinStakeDurationKey: tmpnet.DefaultMinStakeDuration,
-			},
-			Nodes:   nodes,
-			Subnets: subnets,
+			Owner:        flagVars.NetworkOwner(),
+			DefaultFlags: defaultFlags,
+			Nodes:        nodes,
+			Subnets:      subnets,
 		},
 	).Marshal()
 }, func(envBytes []byte) {
 	// Run in every ginkgo process
 
 	// Initialize the local test environment from the global state
-	e2e.InitSharedTestEnvironment(ginkgo.GinkgoT(), envBytes)
+	e2e.InitSharedTestEnvironment(e2e.NewTestContext(), envBytes)
 })

tests/e2e/e2e_test.go

@@ -44,7 +44,7 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
 		_ = e2e.AddEphemeralNode(tc, network, node2)
 
 		tc.By("checking that the second new node fails to become healthy before timeout")
-		err := tmpnet.WaitForHealthy(tc.DefaultContext(), node2)
+		err := tmpnet.WaitForHealthyNode(tc.DefaultContext(), tc.Log(), node2)
 		require.ErrorIs(err, context.DeadlineExceeded)
 
 		tc.By("stopping the first new node")

tests/e2e/faultinjection/duplicate_node_id.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ava-labs/avalanchego/tests/fixture/e2e"
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 	"github.com/ava-labs/avalanchego/utils/constants"
+	"github.com/ava-labs/avalanchego/utils/logging"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -282,7 +283,15 @@ func newNodeStatefulSet(name string, flags map[string]string) *appsv1.StatefulSe
 }
 
 func defaultPodFlags() map[string]string {
-	return tmpnet.DefaultPodFlags(constants.LocalName, nodeDataDir)
+	return map[string]string{
+		config.DataDirKey:                nodeDataDir,
+		config.NetworkNameKey:            constants.LocalName,
+		config.SybilProtectionEnabledKey: "false",
+		config.HealthCheckFreqKey:        "500ms", // Ensure rapid detection of a healthy state
+		config.LogDisplayLevelKey:        logging.Info.String(),
+		config.LogLevelKey:               logging.Off.String(),
+		config.HTTPHostKey:               "0.0.0.0", // Need to bind to pod IP to ensure kubelet can access the http port for the readiness check
+	}
 }
 
 // waitForPodCondition waits until the specified pod reports the specified condition

tests/fixture/bootstrapmonitor/e2e/e2e_test.go

@@ -26,11 +26,12 @@ import (
 // access to the shared env to GetEnv which adds a test context.
 var env *TestEnvironment
 
-func InitSharedTestEnvironment(t require.TestingT, envBytes []byte) {
-	require := require.New(t)
+func InitSharedTestEnvironment(tc tests.TestContext, envBytes []byte) {
+	require := require.New(tc)
 	require.Nil(env, "env already initialized")
 	env = &TestEnvironment{}
 	require.NoError(json.Unmarshal(envBytes, env))
+	env.testContext = tc
 
 	// Ginkgo parallelization is at the process level, so a given key
 	// can safely be used by all tests in a given process without fear
@@ -81,6 +82,7 @@ func (te *TestEnvironment) Marshal() []byte {
 // Initialize a new test environment with a shared network (either pre-existing or newly created).
 func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork *tmpnet.Network) *TestEnvironment {
 	require := require.New(tc)
+	log := tc.Log()
 
 	var network *tmpnet.Network
 
@@ -126,34 +128,34 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 
 		if len(networkDir) > 0 {
 			var err error
-			network, err = tmpnet.ReadNetwork(networkDir)
+			network, err = tmpnet.ReadNetwork(tc.DefaultContext(), log, networkDir)
 			require.NoError(err)
-			tc.Log().Info("loaded a network",
+			log.Info("loaded a network",
 				zap.String("networkDir", networkDir),
 			)
 		}
 
 		if networkCmd == StopNetworkCmd {
 			if len(networkSymlink) > 0 {
 				// Remove the symlink to avoid attempts to reuse the stopped network
-				tc.Log().Info("removing symlink",
+				log.Info("removing symlink",
 					zap.String("path", networkSymlink),
 				)
 				if err := os.Remove(networkSymlink); !errors.Is(err, os.ErrNotExist) {
 					require.NoError(err)
 				}
 			}
 			if network != nil {
-				tc.Log().Info("stopping network")
+				log.Info("stopping network")
 				require.NoError(network.Stop(tc.DefaultContext()))
 			} else {
-				tc.Log().Warn("no network to stop")
+				log.Warn("no network to stop")
 			}
 			os.Exit(0)
 		}
 
 		if network != nil && networkCmd == RestartNetworkCmd {
-			require.NoError(network.Restart(tc.DefaultContext(), tc.Log()))
+			require.NoError(network.Restart(tc.DefaultContext()))
 		}
 	}
 
@@ -194,7 +196,7 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 
 	uris := network.GetNodeURIs()
 	require.NotEmpty(uris, "network contains no nodes")
-	tc.Log().Info("network nodes are available",
+	log.Info("network nodes are available",
 		zap.Any("uris", uris),
 	)
 
@@ -221,8 +223,9 @@ func (te *TestEnvironment) GetRandomNodeURI() tmpnet.NodeURI {
 
 // Retrieve the network to target for testing.
 func (te *TestEnvironment) GetNetwork() *tmpnet.Network {
-	network, err := tmpnet.ReadNetwork(te.NetworkDir)
-	require.NoError(te.testContext, err)
+	tc := te.testContext
+	network, err := tmpnet.ReadNetwork(tc.DefaultContext(), tc.Log(), te.NetworkDir)
+	require.NoError(tc, err)
 	return network
 }
 
@@ -233,14 +236,15 @@ func (te *TestEnvironment) NewKeychain() *secp256k1fx.Keychain {
 
 // Create a new private network that is not shared with other tests.
 func (te *TestEnvironment) StartPrivateNetwork(network *tmpnet.Network) {
-	require := require.New(te.testContext)
+	tc := te.testContext
+	require := require.New(tc)
 	// Use the same configuration as the shared network
-	sharedNetwork, err := tmpnet.ReadNetwork(te.NetworkDir)
+	sharedNetwork, err := tmpnet.ReadNetwork(tc.DefaultContext(), tc.Log(), te.NetworkDir)
 	require.NoError(err)
 	network.DefaultRuntimeConfig = sharedNetwork.DefaultRuntimeConfig
 
 	StartNetwork(
-		te.testContext,
+		tc,
 		network,
 		te.RootNetworkDir,
 		te.PrivateNetworkShutdownDelay,

tests/fixture/e2e/env.go

@@ -136,7 +136,7 @@ func NewEthClient(tc tests.TestContext, nodeURI tmpnet.NodeURI) ethclient.Client
 func AddEphemeralNode(tc tests.TestContext, network *tmpnet.Network, node *tmpnet.Node) *tmpnet.Node {
 	require := require.New(tc)
 
-	require.NoError(network.StartNode(tc.DefaultContext(), tc.Log(), node))
+	require.NoError(network.StartNode(tc.DefaultContext(), node))
 
 	tc.DeferCleanup(func() {
 		tc.Log().Info("shutting down ephemeral node",
@@ -150,11 +150,11 @@ func AddEphemeralNode(tc tests.TestContext, network *tmpnet.Network, node *tmpne
 }
 
 // Wait for the given node to report healthy.
-func WaitForHealthy(t require.TestingT, node *tmpnet.Node) {
+func WaitForHealthy(tc tests.TestContext, node *tmpnet.Node) {
 	// Need to use explicit context (vs DefaultContext()) to support use with DeferCleanup
 	ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout)
 	defer cancel()
-	require.NoError(t, tmpnet.WaitForHealthy(ctx, node))
+	require.NoError(tc, tmpnet.WaitForHealthyNode(ctx, tc.Log(), node))
 }
 
 // Sends an eth transaction and waits for the transaction receipt from the
@@ -236,7 +236,7 @@ func CheckBootstrapIsPossible(tc tests.TestContext, network *tmpnet.Network) *tm
 	}
 
 	node := tmpnet.NewEphemeralNode(flags)
-	require.NoError(network.StartNode(tc.DefaultContext(), tc.Log(), node))
+	require.NoError(network.StartNode(tc.DefaultContext(), node))
 	// StartNode will initiate node stop if an error is encountered during start,
 	// so no further cleanup effort is required if an error is seen here.
 
@@ -248,7 +248,7 @@ func CheckBootstrapIsPossible(tc tests.TestContext, network *tmpnet.Network) *tm
 	})
 
 	// Check that the node becomes healthy within timeout
-	require.NoError(tmpnet.WaitForHealthy(tc.DefaultContext(), node))
+	require.NoError(tmpnet.WaitForHealthyNode(tc.DefaultContext(), tc.Log(), node))
 
 	// Ensure that the primary validators are still healthy
 	for _, node := range network.Nodes {
@@ -272,10 +272,11 @@ func StartNetwork(
 	networkCmd NetworkCmd,
 ) {
 	require := require.New(tc)
+	log := tc.Log()
 
 	err := tmpnet.BootstrapNewNetwork(
 		tc.DefaultContext(),
-		tc.Log(),
+		log,
 		network,
 		rootNetworkDir,
 	)
@@ -289,7 +290,7 @@ func StartNetwork(
 		require.NoError(err, "failed to bootstrap network")
 	}
 
-	tc.Log().Info("network started successfully")
+	log.Info("network started successfully")
 
 	symlinkPath, err := tmpnet.GetReusableNetworkPathForOwner(network.Owner)
 	require.NoError(err)
@@ -298,36 +299,36 @@ func StartNetwork(
 		// Symlink the path of the created network to the default owner path (e.g. latest_avalanchego-e2e)
 		// to enable easy discovery for reuse.
 		require.NoError(os.Symlink(network.Dir, symlinkPath))
-		tc.Log().Info("symlinked network dir for reuse",
+		log.Info("symlinked network dir for reuse",
 			zap.String("networkDir", network.Dir),
 			zap.String("symlinkPath", symlinkPath),
 		)
 	}
 
 	tc.DeferCleanup(func() {
 		if networkCmd == ReuseNetworkCmd || networkCmd == RestartNetworkCmd {
-			tc.Log().Info("skipping shutdown for network intended for reuse",
+			log.Info("skipping shutdown for network intended for reuse",
 				zap.String("networkDir", network.Dir),
 				zap.String("symlinkPath", symlinkPath),
 			)
 			return
 		}
 
 		if networkCmd == StartNetworkCmd {
-			tc.Log().Info("skipping shutdown for --start-network",
+			log.Info("skipping shutdown for --start-network",
 				zap.String("networkDir", network.Dir),
 			)
 			return
 		}
 
 		if shutdownDelay > 0 {
-			tc.Log().Info("delaying network shutdown to ensure final metrics scrape",
+			log.Info("delaying network shutdown to ensure final metrics scrape",
 				zap.Duration("delay", shutdownDelay),
 			)
 			time.Sleep(shutdownDelay)
 		}
 
-		tc.Log().Info("shutting down network")
+		log.Info("shutting down network")
 		ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout)
 		defer cancel()
 		require.NoError(network.Stop(ctx))