From aa894faf0eefa46d02e23cb774164341e7d03a8d Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 24 Dec 2024 14:28:24 +0530
Subject: [PATCH 01/15] current progress

Signed-off-by: Archisman <archisman@obmondo.com>
---
 Makefile                                                  | 8 ++++----
 constants/constants.go                                    | 1 +
 pkg/core/setup_cluster.go                                 | 6 ++++++
 pkg/core/setup_kubeaid_config.go                          | 2 +-
 .../capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl      | 2 ++
 utils/git.go                                              | 3 +++
 6 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 7bc48e4..baff16a 100644
--- a/Makefile
+++ b/Makefile
@@ -57,9 +57,9 @@ bootstrap-cluster-dev-aws:
 bootstrap-cluster-dev-hetzner:
 	@go run ./cmd cluster bootstrap hetzner \
 		--debug \
-		--config /app/outputs/kubeaid-bootstrap-script.config.yaml \
-		--skip-clusterctl-move
+		--config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml
 # --skip-kubeaid-config-setup
+# --skip-clusterctl-move
 
 .PHONY: use-management-cluster
 use-management-cluster:
@@ -69,10 +69,10 @@ use-management-cluster:
 use-provisioned-cluster:
 	export KUBECONFIG=./outputs/provisioned-cluster.kubeconfig.yaml
 
-.PHONY: delete-provisioned-cluster
+.PHONY: delete-provisioned-cluster-dev
 delete-provisioned-cluster-dev:
 	@go run ./cmd cluster delete \
-		--config /app/outputs/kubeaid-bootstrap-script.config.yaml
+		--config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml
 
 .PHONY: delete-management-cluster
 delete-management-cluster:
diff --git a/constants/constants.go b/constants/constants.go
index 646a6f9..aa47c3b 100644
--- a/constants/constants.go
+++ b/constants/constants.go
@@ -62,6 +62,7 @@ const (
 const (
 	ArgoCDAppRoot              = "root"
 	ArgoCDAppCapiCluster       = "capi-cluster"
+	ArgoCDAppHetznerRobot      = "hetzner-robot"
 	ArgoCDAppClusterAutoscaler = "cluster-autoscaler"
 	ArgoCDAppVelero            = "velero"
 )
diff --git a/pkg/core/setup_cluster.go b/pkg/core/setup_cluster.go
index b82fcdf..a3ffb76 100644
--- a/pkg/core/setup_cluster.go
+++ b/pkg/core/setup_cluster.go
@@ -38,6 +38,12 @@ func SetupCluster(ctx context.Context, kubeClient client.Client) {
 		utils.SyncArgoCDApp(ctx, argoCDApp, []*argoCDV1Alpha1.SyncOperationResource{})
 	}
 
+	// If provisioning cluster in Hetzner bare metal, then sync the Hetzner Robot ArgoCD App.
+	// TODO : Explain.
+	if config.ParsedConfig.Cloud.Hetzner != nil {
+		utils.SyncArgoCDApp(ctx, constants.ArgoCDAppHetznerRobot, []*argoCDV1Alpha1.SyncOperationResource{})
+	}
+
 	// Sync the Infrastructure Provider component of the capi-cluster ArgoCD App.
 	// TODO : Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure
 	// Provider component first.
diff --git a/pkg/core/setup_kubeaid_config.go b/pkg/core/setup_kubeaid_config.go
index 5721a15..e5c48b3 100644
--- a/pkg/core/setup_kubeaid_config.go
+++ b/pkg/core/setup_kubeaid_config.go
@@ -91,7 +91,7 @@ func createOrUpdateKubeAidConfigFiles(ctx context.Context, clusterDir string, gi
 	}
 
 	// Build KubePrometheus.
-	buildKubePrometheus(ctx, clusterDir, gitAuthMethod, templateValues)
+	// buildKubePrometheus(ctx, clusterDir, gitAuthMethod, templateValues)
 }
 
 // Creates / updates all necessary Sealed Secrets files for the given cluster, in the user's KubeAid
diff --git a/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl b/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl
index 8913aec..14a525a 100644
--- a/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl
+++ b/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl
@@ -9,3 +9,5 @@ stringData:
 
   ssh-publickey: |
 {{ .HetznerConfig.RobotSSHKeyPair.PublicKey | indent 4  }}
+
+  sshkey-name: cluster
diff --git a/utils/git.go b/utils/git.go
index eb78193..0df801e 100644
--- a/utils/git.go
+++ b/utils/git.go
@@ -171,6 +171,9 @@ func getCreatePRURL(fromBranch string) string {
 	return createPRURL
 }
 
+// TODO : Sometimes we get this error while trying to detect whether the branch has been merged
+// or not : `unexpected EOF`.
+// In that case, just retry instead of erroring out.
 func WaitUntilPRMerged(ctx context.Context, repo *git.Repository, defaultBranchName string, commitHash plumbing.Hash, auth transport.AuthMethod, branchToBeMerged string) {
 	for {
 		slog.Info("Waiting for PR to be merged. Sleeping for 10 seconds....", slog.String("from-branch", branchToBeMerged), slog.String("to-branch", defaultBranchName))

From 9022279c48044d61eb3f52e059485f4008a122f4 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Wed, 25 Dec 2024 00:28:19 +0530
Subject: [PATCH 02/15] moved the hetzner robot failover script here and
 cleaned it up

Signed-off-by: Archisman <archisman@obmondo.com>
---
 README.md                                     |  2 +
 build/Dockerfile                              |  2 +-
 build/Dockerfile.dev                          |  2 +-
 cmd/hetzner-failover-script/main.go           | 84 +++++++++++++++++++
 .../cluster/bootstrap/aws.go                  |  0
 .../cluster/bootstrap/bootstrap.go            |  0
 .../cluster/bootstrap/hetzner.go              |  0
 .../cluster/cluster.go                        |  4 +-
 .../cluster/delete/delete.go                  |  0
 .../cluster/recover/aws.go                    |  0
 .../cluster/recover/hetzner.go                |  0
 .../cluster/recover/recover.go                |  0
 .../config/config.go                          |  6 +-
 .../config/generate/aws.go                    |  0
 .../config/generate/generate.go               |  0
 .../config/generate/hetzner.go                |  0
 cmd/{ => kubeaid-bootstrap-script}/main.go    |  4 +-
 go.mod                                        |  1 +
 go.sum                                        |  4 +
 pkg/core/setup_cluster.go                     |  6 --
 utils/others.go                               |  2 +-
 21 files changed, 100 insertions(+), 17 deletions(-)
 create mode 100644 cmd/hetzner-failover-script/main.go
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/bootstrap/aws.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/bootstrap/bootstrap.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/bootstrap/hetzner.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/cluster.go (76%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/delete/delete.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/recover/aws.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/recover/hetzner.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/cluster/recover/recover.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/config/config.go (81%)
 rename cmd/{ => kubeaid-bootstrap-script}/config/generate/aws.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/config/generate/generate.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/config/generate/hetzner.go (100%)
 rename cmd/{ => kubeaid-bootstrap-script}/main.go (82%)

diff --git a/README.md b/README.md
index b6e68fd..d980ca3 100644
--- a/README.md
+++ b/README.md
@@ -46,3 +46,5 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [Secret Rotation](https://github.com/bitnami-labs/sealed-secrets?tab=readme-ov-file#secret-rotation)
 
 - [Kubernetes Backups, Upgrades, Migrations - with Velero](https://youtu.be/zybLTQER0yY?si=qOZcizBqPOeouJ7y)
+
+- [Failover](https://docs.hetzner.com/robot/dedicated-server/ip/failover/)
diff --git a/build/Dockerfile b/build/Dockerfile
index b3d2a01..940dba1 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -21,7 +21,7 @@ WORKDIR /
 COPY ./scripts/install-prerequisites.sh /install-prerequisites.sh
 RUN chmod +x /install-prerequisites.sh
 RUN CPU_ARCHITECTURE=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64") \
-  /install-prerequisites.sh
+	/install-prerequisites.sh
 
 COPY --from=builder /app/kubeaid-bootstrap-script /usr/local/bin/kubeaid-bootstrap-script
 
diff --git a/build/Dockerfile.dev b/build/Dockerfile.dev
index 07b5897..bfbee7e 100644
--- a/build/Dockerfile.dev
+++ b/build/Dockerfile.dev
@@ -7,7 +7,7 @@ WORKDIR /
 COPY ./scripts/install-prerequisites.sh /install-prerequisites.sh
 RUN chmod +x /install-prerequisites.sh
 RUN CPU_ARCHITECTURE=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64") \
-  /install-prerequisites.sh
+	/install-prerequisites.sh
 
 WORKDIR /app
 
diff --git a/cmd/hetzner-failover-script/main.go b/cmd/hetzner-failover-script/main.go
new file mode 100644
index 0000000..9b54cd2
--- /dev/null
+++ b/cmd/hetzner-failover-script/main.go
@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"log"
+	"log/slog"
+	"os"
+	"time"
+
+	"github.com/Obmondo/kubeaid-bootstrap-script/utils"
+	"github.com/Obmondo/kubeaid-bootstrap-script/utils/assert"
+	"github.com/floshodan/hrobot-go/hrobot"
+)
+
+func main() {
+	ctx := context.Background()
+
+	// Read required environment variables.
+	var (
+		failoverIP = utils.GetEnv("FAILOVER_IP")
+
+		nodeIP = utils.GetEnv("NODE_IP")
+
+		username = os.Getenv("API_USERNAME") // (optional).
+		password = os.Getenv("API_PASSWORD") // (optional).
+
+		apiToken = os.Getenv("API_TOKEN") // (optional).
+	)
+
+	// Construct Hetzner Robot API client.
+	var hetznerRobotClient *hrobot.Client
+	switch {
+	case len(username) > 0 && len(password) > 0:
+		hetznerRobotClient = hrobot.NewClient(hrobot.WithBasicAuth(username, password))
+
+	case len(apiToken) > 0:
+		hetznerRobotClient = hrobot.NewClient(hrobot.WithToken(apiToken))
+
+	default:
+		log.Fatalf("Either provide username and password / api token as credentials, to communicate with the Hetzner Robot API")
+	}
+
+	/*
+		A Failover IP is an additional IP that you can switch from one server to another. You can order
+		it for any Hetzner dedicated root server, and you can switch it to any other Hetzner dedicated
+		root server, regardless of location.
+
+		Switching a failover IP takes between 90 and 110 seconds.
+
+		REFERENCE : https://docs.hetzner.com/robot/dedicated-server/ip/failover/.
+	*/
+	// Hetzner Robot Failover IP API spec : API REFERENCE : https://robot.hetzner.com/doc/webservice/en.html#failover.
+
+	// Get the Failover IP's current active server IP.
+	failoverIPDetails, _, err := hetznerRobotClient.Failover.GetFailoverIP(ctx, failoverIP)
+	assert.AssertErrNil(ctx, err, "Failed getting Failover IP details")
+
+	activeServerIP := failoverIPDetails.ActiveServerIP
+	slog.InfoContext(ctx, "Detected active server", slog.String("ip", activeServerIP))
+
+	if activeServerIP == nodeIP {
+		slog.InfoContext(ctx, "Active server IP is already same as the current server IP")
+		return
+	}
+
+	// Update Failover IP to the current node's IP (the current node, on which this script is
+	// running)
+	_, _, err = hetznerRobotClient.Failover.SwitchFailover(ctx, failoverIP)
+	assert.AssertErrNil(ctx, err, "Failed switching Failover IP to the current node IP")
+
+	// Wait for the update to complete.
+	for {
+		failoverIPDetails, _, err := hetznerRobotClient.Failover.GetFailoverIP(ctx, failoverIP)
+		assert.AssertErrNil(ctx, err, "Failed getting Failover IP details")
+
+		if failoverIPDetails.ActiveServerIP == nodeIP {
+			slog.InfoContext(ctx, "Successfully updated Failover IP", slog.String("active-server-ip", nodeIP))
+			break
+		}
+
+		slog.InfoContext(ctx, "Waiting for the Failover IP update to complete. Sleeping for a minute....")
+		time.Sleep(time.Minute)
+	}
+}
diff --git a/cmd/cluster/bootstrap/aws.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go
similarity index 100%
rename from cmd/cluster/bootstrap/aws.go
rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go
diff --git a/cmd/cluster/bootstrap/bootstrap.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/bootstrap.go
similarity index 100%
rename from cmd/cluster/bootstrap/bootstrap.go
rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/bootstrap.go
diff --git a/cmd/cluster/bootstrap/hetzner.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go
similarity index 100%
rename from cmd/cluster/bootstrap/hetzner.go
rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go
diff --git a/cmd/cluster/cluster.go b/cmd/kubeaid-bootstrap-script/cluster/cluster.go
similarity index 76%
rename from cmd/cluster/cluster.go
rename to cmd/kubeaid-bootstrap-script/cluster/cluster.go
index 85b3bb9..30042f6 100644
--- a/cmd/cluster/cluster.go
+++ b/cmd/kubeaid-bootstrap-script/cluster/cluster.go
@@ -1,8 +1,8 @@
 package cluster
 
 import (
-	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster/bootstrap"
-	delete_ "github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster/delete"
+	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster/bootstrap"
+	delete_ "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster/delete"
 	"github.com/Obmondo/kubeaid-bootstrap-script/config"
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils"
 	"github.com/spf13/cobra"
diff --git a/cmd/cluster/delete/delete.go b/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go
similarity index 100%
rename from cmd/cluster/delete/delete.go
rename to cmd/kubeaid-bootstrap-script/cluster/delete/delete.go
diff --git a/cmd/cluster/recover/aws.go b/cmd/kubeaid-bootstrap-script/cluster/recover/aws.go
similarity index 100%
rename from cmd/cluster/recover/aws.go
rename to cmd/kubeaid-bootstrap-script/cluster/recover/aws.go
diff --git a/cmd/cluster/recover/hetzner.go b/cmd/kubeaid-bootstrap-script/cluster/recover/hetzner.go
similarity index 100%
rename from cmd/cluster/recover/hetzner.go
rename to cmd/kubeaid-bootstrap-script/cluster/recover/hetzner.go
diff --git a/cmd/cluster/recover/recover.go b/cmd/kubeaid-bootstrap-script/cluster/recover/recover.go
similarity index 100%
rename from cmd/cluster/recover/recover.go
rename to cmd/kubeaid-bootstrap-script/cluster/recover/recover.go
diff --git a/cmd/config/config.go b/cmd/kubeaid-bootstrap-script/config/config.go
similarity index 81%
rename from cmd/config/config.go
rename to cmd/kubeaid-bootstrap-script/config/config.go
index cc59ce2..29281af 100644
--- a/cmd/config/config.go
+++ b/cmd/kubeaid-bootstrap-script/config/config.go
@@ -1,7 +1,7 @@
 package config
 
 import (
-	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/config/generate"
+	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/config/generate"
 	"github.com/Obmondo/kubeaid-bootstrap-script/constants"
 	"github.com/spf13/cobra"
 )
@@ -13,9 +13,7 @@ var ConfigCmd = &cobra.Command{
 	},
 }
 
-var (
-	ConfigFilePath string
-)
+var ConfigFilePath string
 
 func init() {
 	// Subcommands.
diff --git a/cmd/config/generate/aws.go b/cmd/kubeaid-bootstrap-script/config/generate/aws.go
similarity index 100%
rename from cmd/config/generate/aws.go
rename to cmd/kubeaid-bootstrap-script/config/generate/aws.go
diff --git a/cmd/config/generate/generate.go b/cmd/kubeaid-bootstrap-script/config/generate/generate.go
similarity index 100%
rename from cmd/config/generate/generate.go
rename to cmd/kubeaid-bootstrap-script/config/generate/generate.go
diff --git a/cmd/config/generate/hetzner.go b/cmd/kubeaid-bootstrap-script/config/generate/hetzner.go
similarity index 100%
rename from cmd/config/generate/hetzner.go
rename to cmd/kubeaid-bootstrap-script/config/generate/hetzner.go
diff --git a/cmd/main.go b/cmd/kubeaid-bootstrap-script/main.go
similarity index 82%
rename from cmd/main.go
rename to cmd/kubeaid-bootstrap-script/main.go
index 8ed9dc7..b6a7284 100644
--- a/cmd/main.go
+++ b/cmd/kubeaid-bootstrap-script/main.go
@@ -4,8 +4,8 @@ import (
 	"log/slog"
 	"os"
 
-	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster"
-	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/config"
+	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster"
+	"github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/config"
 	"github.com/Obmondo/kubeaid-bootstrap-script/constants"
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils/logger"
 	"github.com/spf13/cobra"
diff --git a/go.mod b/go.mod
index 3104d58..5a6eb9b 100644
--- a/go.mod
+++ b/go.mod
@@ -101,6 +101,7 @@ require (
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
diff --git a/go.sum b/go.sum
index 7b18373..1a68bba 100644
--- a/go.sum
+++ b/go.sum
@@ -254,6 +254,8 @@ github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLg
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef h1:iniS/SxdrJ0IKzq5M83/4EvSo8dR0gTKMFx3J2JRA5M=
+github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef/go.mod h1:TjlKZ0GYQbBtq6G7eUcVcRvTqgeCxI3cO0LOfBZv1/A=
 github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
 github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -920,6 +922,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -1109,6 +1112,7 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP
 gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/h2non/gentleman.v2 v2.0.5/go.mod h1:A1c7zwrTgAyyf6AbpvVksYtBayTB4STBUGmdkEtlHeA=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
diff --git a/pkg/core/setup_cluster.go b/pkg/core/setup_cluster.go
index a3ffb76..b82fcdf 100644
--- a/pkg/core/setup_cluster.go
+++ b/pkg/core/setup_cluster.go
@@ -38,12 +38,6 @@ func SetupCluster(ctx context.Context, kubeClient client.Client) {
 		utils.SyncArgoCDApp(ctx, argoCDApp, []*argoCDV1Alpha1.SyncOperationResource{})
 	}
 
-	// If provisioning cluster in Hetzner bare metal, then sync the Hetzner Robot ArgoCD App.
-	// TODO : Explain.
-	if config.ParsedConfig.Cloud.Hetzner != nil {
-		utils.SyncArgoCDApp(ctx, constants.ArgoCDAppHetznerRobot, []*argoCDV1Alpha1.SyncOperationResource{})
-	}
-
 	// Sync the Infrastructure Provider component of the capi-cluster ArgoCD App.
 	// TODO : Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure
 	// Provider component first.
diff --git a/utils/others.go b/utils/others.go
index 3bff74e..769b931 100644
--- a/utils/others.go
+++ b/utils/others.go
@@ -56,7 +56,7 @@ func InitTempDir() {
 // Panics if the environment variable isn't found.
 func GetEnv(name string) string {
 	value, found := os.LookupEnv(name)
-	if !found {
+	if !found || len(value) == 0 {
 		slog.Error("Env not found", slog.String("name", name))
 		os.Exit(1)
 	}

From 02b5dd74de19632d89d7efb16748aed4568f7cf2 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Thu, 2 Jan 2025 17:39:02 +0530
Subject: [PATCH 03/15] Updating Hetzner Failover Script | Having a Dockerfile
 for it | Some minor fixes

Signed-off-by: Archisman <archisman@obmondo.com>
---
 .github/workflows/release.yaml                | 20 ++++++++++--
 Makefile                                      | 16 +++++-----
 build/hetzner-failover-script/Dockerfile      | 28 ++++++++++++++++
 .../{ => kubeaid-bootstrap-script}/Dockerfile |  5 ++-
 .../Dockerfile.dev                            |  0
 cmd/hetzner-failover-script/main.go           |  4 ++-
 go.mod                                        | 16 +++++-----
 go.sum                                        | 32 +++++++++----------
 pkg/cloud/aws/setup_disaster_recovery.go      |  5 ++-
 pkg/core/bootstrap_cluster.go                 |  4 ++-
 pkg/core/recover_cluster.go                   |  2 +-
 pkg/core/setup_kubeaid_config.go              |  2 +-
 12 files changed, 92 insertions(+), 42 deletions(-)
 create mode 100644 build/hetzner-failover-script/Dockerfile
 rename build/{ => kubeaid-bootstrap-script}/Dockerfile (75%)
 rename build/{ => kubeaid-bootstrap-script}/Dockerfile.dev (100%)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 8432114..636f52c 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -28,11 +28,11 @@ jobs:
           username: obmondo
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push AMD64 and ARM64 container images
+      - name: Build and push KubeAid Bootstrap Script AMD64 and ARM64 container images
         uses: docker/build-push-action@v4
         with:
           context: .
-          file: build/Dockerfile
+          file: build/kubeaid-bootstrap-script/Dockerfile
           # NOTE : It takes pretty long to build container images for the ARM64 platform (even when
           # using QEMU).
           platforms: linux/amd64,linux/arm64
@@ -43,3 +43,19 @@ jobs:
           # builds.
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Build and push Hetzner Failover Script AMD64 and ARM64 container images
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: build/hetzner-failover-script/Dockerfile
+          # NOTE : It takes pretty long to build container images for the ARM64 platform (even when
+          # using QEMU).
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/obmondo/hetzner-failover-script:${{ github.event.release.tag_name }}
+          push: true
+          # Experimental cache exporter for GitHub Actions provided by buildx and BuildKit.
+          # It uses the GitHub Cache API to fetch and load the Docker layer cache blobs across
+          # builds.
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/Makefile b/Makefile
index baff16a..e6adbca 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ IMAGE_NAME=kubeaid-bootstrap-script-dev:latest
 
 .PHONY: build-image-dev
 build-image-dev:
-	@docker build -f ./build/Dockerfile.dev --build-arg CPU_ARCHITECTURE=arm64 -t $(IMAGE_NAME) .
+	@docker build -f ./build/kubeaid-bootstrap-script/Dockerfile.dev --build-arg CPU_ARCHITECTURE=arm64 -t $(IMAGE_NAME) .
 
 .PHONY: remove-image-dev
 remove-image-dev:
@@ -43,11 +43,11 @@ remove-container-dev: stop-container-dev
 
 .PHONY: generate-sample-config-aws-dev
 generate-sample-config-aws-dev:
-	@go run ./cmd config generate aws
+	@go run ./cmd/kubeaid-bootstrap-script/ config generate aws
 
 .PHONY: bootstrap-cluster-dev-aws
 bootstrap-cluster-dev-aws:
-	@go run ./cmd cluster bootstrap aws \
+	@go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap aws \
 		--debug \
 		--config /app/outputs/kubeaid-bootstrap-script.config.yaml \
 		--skip-clusterctl-move
@@ -55,11 +55,11 @@ bootstrap-cluster-dev-aws:
 
 .PHONY: bootstrap-cluster-dev-hetzner
 bootstrap-cluster-dev-hetzner:
-	@go run ./cmd cluster bootstrap hetzner \
+	@go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap hetzner \
 		--debug \
-		--config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml
+    --config /app/outputs/kubeaid-bootstrap-script.config.yaml \
+    --skip-clusterctl-move
 # --skip-kubeaid-config-setup
-# --skip-clusterctl-move
 
 .PHONY: use-management-cluster
 use-management-cluster:
@@ -71,8 +71,8 @@ use-provisioned-cluster:
 
 .PHONY: delete-provisioned-cluster-dev
 delete-provisioned-cluster-dev:
-	@go run ./cmd cluster delete \
-		--config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml
+	@go run ./cmd/kubeaid-bootstrap-script/ cluster delete \
+		--config /app/outputs/kubeaid-bootstrap-script.config.yaml
 
 .PHONY: delete-management-cluster
 delete-management-cluster:
diff --git a/build/hetzner-failover-script/Dockerfile b/build/hetzner-failover-script/Dockerfile
new file mode 100644
index 0000000..e0e5588
--- /dev/null
+++ b/build/hetzner-failover-script/Dockerfile
@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+
+#--- Builder stage ---
+
+FROM golang:1.23.0 AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN go build -o hetzner-failover-script ./cmd/hetzner-failover-script
+
+#--- Packager stage ---
+
+FROM golang:1.23.0 AS packages
+
+# Set the maintainer label
+LABEL org.opencontainers.image.authors="ashish@obmondo.com, archisman@obmondo.com"
+
+RUN apk add --no-cache procps
+
+WORKDIR /root/
+
+COPY --from=builder /app/hetzner-failover-script .
+
+CMD ["./hetzner-failover-script"]
diff --git a/build/Dockerfile b/build/kubeaid-bootstrap-script/Dockerfile
similarity index 75%
rename from build/Dockerfile
rename to build/kubeaid-bootstrap-script/Dockerfile
index 940dba1..fa7e988 100644
--- a/build/Dockerfile
+++ b/build/kubeaid-bootstrap-script/Dockerfile
@@ -10,12 +10,15 @@ COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
-RUN go build -o kubeaid-bootstrap-script ./cmd
+RUN go build -o kubeaid-bootstrap-script ./cmd/kubeaid-bootstrap-script
 
 #--- Packager stage ---
 
 FROM golang:1.23.0 AS packages
 
+# Set the maintainer label
+LABEL org.opencontainers.image.authors="ashish@obmondo.com, archisman@obmondo.com"
+
 WORKDIR /
 
 COPY ./scripts/install-prerequisites.sh /install-prerequisites.sh
diff --git a/build/Dockerfile.dev b/build/kubeaid-bootstrap-script/Dockerfile.dev
similarity index 100%
rename from build/Dockerfile.dev
rename to build/kubeaid-bootstrap-script/Dockerfile.dev
diff --git a/cmd/hetzner-failover-script/main.go b/cmd/hetzner-failover-script/main.go
index 9b54cd2..a610cb8 100644
--- a/cmd/hetzner-failover-script/main.go
+++ b/cmd/hetzner-failover-script/main.go
@@ -65,7 +65,9 @@ func main() {
 
 	// Update Failover IP to the current node's IP (the current node, on which this script is
 	// running)
-	_, _, err = hetznerRobotClient.Failover.SwitchFailover(ctx, failoverIP)
+	// NOTE : Contributed :
+	//				https://github.com/floshodan/hrobot-go/commit/700f8ef9fdac565129608b3a50583b4b6564ff34.
+	_, _, err = hetznerRobotClient.Failover.SwitchFailover(ctx, failoverIP, activeServerIP)
 	assert.AssertErrNil(ctx, err, "Failed switching Failover IP to the current node IP")
 
 	// Wait for the update to complete.
diff --git a/go.mod b/go.mod
index 5a6eb9b..ee6b610 100644
--- a/go.mod
+++ b/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.69.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7
 	github.com/creasty/defaults v1.8.0
+	github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/go-playground/validator/v10 v10.23.0
 	github.com/go-sprout/sprout v0.6.0
@@ -21,7 +22,7 @@ require (
 	github.com/siderolabs/talos/pkg/machinery v1.8.3
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/crypto v0.29.0
+	golang.org/x/crypto v0.31.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.16.3
 	k8s.io/api v0.31.2
@@ -101,7 +102,6 @@ require (
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -195,7 +195,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.20.3 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
@@ -244,12 +244,12 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
-	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sync v0.9.0 // indirect
-	golang.org/x/sys v0.27.0 // indirect
-	golang.org/x/term v0.26.0 // indirect
-	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
diff --git a/go.sum b/go.sum
index 1a68bba..b207484 100644
--- a/go.sum
+++ b/go.sum
@@ -254,8 +254,8 @@ github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLg
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef h1:iniS/SxdrJ0IKzq5M83/4EvSo8dR0gTKMFx3J2JRA5M=
-github.com/floshodan/hrobot-go v0.0.0-20240813191752-ed07e44014ef/go.mod h1:TjlKZ0GYQbBtq6G7eUcVcRvTqgeCxI3cO0LOfBZv1/A=
+github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75 h1:XOnFL5swXi62I1MjZqZ2oLIFNthzYrtp9MWXA24leZA=
+github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75/go.mod h1:TjlKZ0GYQbBtq6G7eUcVcRvTqgeCxI3cO0LOfBZv1/A=
 github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
 github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -683,8 +683,8 @@ github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
-github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
-github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -885,8 +885,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
-golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
-golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
@@ -935,8 +935,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
-golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
-golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
+golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
+golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -953,8 +953,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -999,8 +999,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
-golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1012,8 +1012,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
-golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
-golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1027,8 +1027,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
-golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
diff --git a/pkg/cloud/aws/setup_disaster_recovery.go b/pkg/cloud/aws/setup_disaster_recovery.go
index 6ce8e74..af70646 100644
--- a/pkg/cloud/aws/setup_disaster_recovery.go
+++ b/pkg/cloud/aws/setup_disaster_recovery.go
@@ -7,6 +7,7 @@ import (
 	"github.com/Obmondo/kubeaid-bootstrap-script/config"
 	"github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/aws/services"
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils"
+	"github.com/Obmondo/kubeaid-bootstrap-script/utils/assert"
 	argoCDV1Alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/sagikazarmark/slog-shim"
 )
@@ -14,9 +15,7 @@ import (
 // Sets up the provisioned cluster for Disaster Recovery.
 // NOTE : Picks up AWS credentials from the environment.
 func (a *AWS) SetupDisasterRecovery(ctx context.Context) {
-	if config.ParsedConfig.Cloud.AWS.DisasterRecovery == nil {
-		return
-	}
+	assert.AssertNotNil(ctx, config.ParsedConfig.Cloud.AWS.DisasterRecovery, "No AWS disaster-recovery config provided")
 
 	slog.InfoContext(ctx, "Setting up Disaster Recovery")
 
diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go
index 7b86adf..987f716 100644
--- a/pkg/core/bootstrap_cluster.go
+++ b/pkg/core/bootstrap_cluster.go
@@ -36,7 +36,9 @@ func BootstrapCluster(ctx context.Context, skipKubeAidConfigSetup, skipClusterct
 
 	// If the diasterRecovery section is specified in the cloud-provider specific config, then
 	// setup Disaster Recovery.
-	cloudProvider.SetupDisasterRecovery(ctx)
+	if config.ParsedConfig.Cloud.AWS.DisasterRecovery != nil {
+		cloudProvider.SetupDisasterRecovery(ctx)
+	}
 }
 
 func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMethod, skipKubeAidConfigSetup bool) {
diff --git a/pkg/core/recover_cluster.go b/pkg/core/recover_cluster.go
index e08bfdd..11c9a22 100644
--- a/pkg/core/recover_cluster.go
+++ b/pkg/core/recover_cluster.go
@@ -30,7 +30,7 @@ func RecoverCluster(ctx context.Context, cloudProvider cloud.CloudProvider) {
 		Pull and gzip decode backed up (by Sealed Secrets backuper CRONJob) Kubernetes Secrets from S3
 		bucket. Each Kubernetes Secret contains a Sealed Secrets encryption key.
 
-		The script uresponsible for this backup process can be found here :
+		The script responsible for this backup process can be found here :
 		https://github.com/Obmondo/kubeaid/blob/master/argocd-helm-charts/sealed-secrets/templates/configmap.yaml
 
 		And you can read about Sealed Secrets key rotation from these references :
diff --git a/pkg/core/setup_kubeaid_config.go b/pkg/core/setup_kubeaid_config.go
index e5c48b3..5721a15 100644
--- a/pkg/core/setup_kubeaid_config.go
+++ b/pkg/core/setup_kubeaid_config.go
@@ -91,7 +91,7 @@ func createOrUpdateKubeAidConfigFiles(ctx context.Context, clusterDir string, gi
 	}
 
 	// Build KubePrometheus.
-	// buildKubePrometheus(ctx, clusterDir, gitAuthMethod, templateValues)
+	buildKubePrometheus(ctx, clusterDir, gitAuthMethod, templateValues)
 }
 
 // Creates / updates all necessary Sealed Secrets files for the given cluster, in the user's KubeAid

From f035cccbb226b7d1184309a19e41bb91d5389925 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Fri, 3 Jan 2025 01:06:53 +0530
Subject: [PATCH 04/15] (fix) : Supporting retry feature when `clusterctl move`
 has already been executed | Installing Sealed Secrets in the sealed-secrets
 namespace

Signed-off-by: Archisman <archisman@obmondo.com>
---
 pkg/core/bootstrap_cluster.go                 | 34 ++++++++++--------
 pkg/core/delete_cluster.go                    | 35 ++++++++-----------
 .../sealed-secrets.values.yaml.tmpl           |  5 ++-
 utils/kubernetes.go                           | 13 +++++++
 4 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go
index 987f716..31b93e9 100644
--- a/pkg/core/bootstrap_cluster.go
+++ b/pkg/core/bootstrap_cluster.go
@@ -28,11 +28,27 @@ func BootstrapCluster(ctx context.Context, skipKubeAidConfigSetup, skipClusterct
 
 	os.Setenv(constants.EnvNameKubeconfig, constants.OutputPathManagementClusterKubeconfig)
 
-	// Provision the main cluster
-	provisionMainCluster(ctx, gitAuthMethod, skipKubeAidConfigSetup)
+	// Create the management cluster (using K3d), if it doesn't already exist.
+	utils.CreateK3DCluster(ctx, "management-cluster")
+
+	// Install Sealed Secrets.
+	utils.InstallSealedSecrets(ctx)
+
+	// Setup cluster directory in the user's KubeAid config repo.
+	if !skipKubeAidConfigSetup {
+		SetupKubeAidConfig(ctx, gitAuthMethod, false)
+	}
 
-	// Let the provisioned cluster manage itself.
-	dogfoodProvisionedCluster(ctx, gitAuthMethod, skipClusterctlMove, cloudProvider, isPartOfDisasterRecovery)
+	// While retrying, if `clusterctl move` has already been executed, then we skip these steps and
+	// move to the disaster recovery setup step.
+	provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig)
+	if !utils.IsClusterctlMoveExecuted(ctx, provisionedClusterClient) {
+		// Provision the main cluster
+		provisionMainCluster(ctx, gitAuthMethod, skipKubeAidConfigSetup)
+
+		// Let the provisioned cluster manage itself.
+		dogfoodProvisionedCluster(ctx, gitAuthMethod, skipClusterctlMove, cloudProvider, isPartOfDisasterRecovery)
+	}
 
 	// If the diasterRecovery section is specified in the cloud-provider specific config, then
 	// setup Disaster Recovery.
@@ -42,18 +58,8 @@ func BootstrapCluster(ctx context.Context, skipKubeAidConfigSetup, skipClusterct
 }
 
 func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMethod, skipKubeAidConfigSetup bool) {
-	// Create the management cluster (using K3d), if it doesn't already exist.
-	utils.CreateK3DCluster(ctx, "management-cluster")
 	managementClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathManagementClusterKubeconfig)
 
-	// Install Sealed Secrets.
-	utils.InstallSealedSecrets(ctx)
-
-	// Setup cluster directory in the user's KubeAid config repo.
-	if !skipKubeAidConfigSetup {
-		SetupKubeAidConfig(ctx, gitAuthMethod, false)
-	}
-
 	// Setup the management cluster.
 	SetupCluster(ctx, managementClusterClient)
 
diff --git a/pkg/core/delete_cluster.go b/pkg/core/delete_cluster.go
index 4113a26..45244cd 100644
--- a/pkg/core/delete_cluster.go
+++ b/pkg/core/delete_cluster.go
@@ -27,28 +27,23 @@ func DeleteCluster(ctx context.Context) {
 		},
 	}
 
-	{
-		provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig)
+	provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig)
 
-		// Try to find the Cluster resource in the provisioned cluster.
-		err := utils.GetClusterResource(ctx, provisionedClusterClient, cluster)
+	// The Cluster resource exists in the provisioned cluster.
+	// The means, the 'clusterctl move' command has been executed.
+	if utils.IsClusterctlMoveExecuted(ctx, provisionedClusterClient) {
+		slog.InfoContext(ctx, "Detected that the 'clusterctl move' command has been executed")
 
-		// The Cluster resource exists in the provisioned cluster.
-		// The means, the 'clusterctl move' command has been executed.
-		if err == nil {
-			slog.InfoContext(ctx, "Detected that the 'clusterctl move' command has been executed")
-
-			// Move back the ClusterAPI manifests back from the provisioned cluster to the management
-			// cluster.
-			// NOTE : We need to retry, since we can get 'failed to call webhook' error sometimes.
-			retry.Do(func() error {
-				_, err := utils.ExecuteCommand(fmt.Sprintf(
-					"clusterctl move --kubeconfig %s --to-kubeconfig %s -n %s",
-					constants.OutputPathProvisionedClusterKubeconfig, constants.OutputPathManagementClusterKubeconfig, utils.GetCapiClusterNamespace(),
-				))
-				return err
-			})
-		}
+		// Move back the ClusterAPI manifests back from the provisioned cluster to the management
+		// cluster.
+		// NOTE : We need to retry, since we can get 'failed to call webhook' error sometimes.
+		retry.Do(func() error {
+			_, err := utils.ExecuteCommand(fmt.Sprintf(
+				"clusterctl move --kubeconfig %s --to-kubeconfig %s -n %s",
+				constants.OutputPathProvisionedClusterKubeconfig, constants.OutputPathManagementClusterKubeconfig, utils.GetCapiClusterNamespace(),
+			))
+			return err
+		})
 	}
 
 	managementClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathManagementClusterKubeconfig)
diff --git a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
index 99e0f29..5e7d2b0 100644
--- a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
+++ b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
@@ -1,6 +1,9 @@
----
+sealed-secrets:
+  namespace: sealed-secrets
+
 {{- if and (.AWSConfig) (.AWSConfig.DisasterRecovery) }}
 backup:
+  namespace: sealed-secrets
   kube2iamRole: arn:aws:iam::{{ .AWSAccountID }}:role/sealed-secrets-backuper-{{ .ClusterConfig.Name }}
   backupBucket: {{ .AWSConfig.DisasterRecovery.SealedSecretsBackupS3BucketName }}
 {{- end }}
diff --git a/utils/kubernetes.go b/utils/kubernetes.go
index 8c5d2c5..b5bd973 100644
--- a/utils/kubernetes.go
+++ b/utils/kubernetes.go
@@ -202,3 +202,16 @@ func SaveKubeconfig(ctx context.Context, kubeClient client.Client) {
 
 	slog.InfoContext(ctx, "kubeconfig has been saved locally")
 }
+
+// Returns whether the `clusterctl move` command has already been executed or not.
+func IsClusterctlMoveExecuted(ctx context.Context, provisionedClusterClient client.Client) bool {
+	// If the Cluster resource is found in the provisioned cluster, that means `clusterctl move` has
+	// been executed.
+	err := GetClusterResource(ctx, provisionedClusterClient, &clusterAPIV1Beta1.Cluster{
+		ObjectMeta: metaV1.ObjectMeta{
+			Name:      config.ParsedConfig.Cluster.Name,
+			Namespace: GetCapiClusterNamespace(),
+		},
+	})
+	return err == nil
+}

From cda30dfc02a67a1e4db8e507390854615aee1647 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Mon, 6 Jan 2025 13:12:01 +0530
Subject: [PATCH 05/15] (fix) : Sealed Secrets backuper IAM policy name |
 Sealed Secrets controller name in generated KubeAid Config

Signed-off-by: Archisman <archisman@obmondo.com>
---
 pkg/cloud/aws/policies.go                                      | 2 +-
 pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/cloud/aws/policies.go b/pkg/cloud/aws/policies.go
index d7f166c..edd0aaf 100644
--- a/pkg/cloud/aws/policies.go
+++ b/pkg/cloud/aws/policies.go
@@ -20,7 +20,7 @@ func getSealedSecretsBackuperIAMPolicy() services.PolicyDocument {
 					"s3:ListMultipartUploadParts",
 				},
 				Effect:   "Allow",
-				Resource: fmt.Sprintf("arn:aws:s3:::-%s", sealedSecretBackupsS3BucketName),
+				Resource: fmt.Sprintf("arn:aws:s3:::%s/*", sealedSecretBackupsS3BucketName),
 			},
 		},
 	}
diff --git a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
index 5e7d2b0..243dcd9 100644
--- a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
+++ b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl
@@ -1,5 +1,6 @@
 sealed-secrets:
   namespace: sealed-secrets
+  fullnameOverride: sealed-secrets-controller
 
 {{- if and (.AWSConfig) (.AWSConfig.DisasterRecovery) }}
 backup:

From 824476e2976a05e3e3022bdac7902d948a45a89e Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 7 Jan 2025 17:47:51 +0530
Subject: [PATCH 06/15] (feat/audit-logging) : Allowing the user to pass Kube
 API server related options | Having default options related to audit logging

Signed-off-by: Archisman <archisman@obmondo.com>
---
 README.md                                     |  6 ++
 config/audit_logging.go                       | 96 +++++++++++++++++++
 config/config.go                              |  8 ++
 config/files/defaults/audit-policy.yaml       | 15 +++
 .../templates/aws.sample.config.yaml.tmpl     |  0
 .../templates/hetzner.sample.config.yaml.tmpl |  0
 config/generate_sample_config.go              |  2 +-
 config/{utils.go => parse.go}                 | 29 +++---
 constants/constants.go                        | 10 +-
 go.mod                                        | 27 +++---
 go.sum                                        | 74 +++++++-------
 .../argocd-apps/capi-cluster.values.yaml.tmpl |  6 ++
 12 files changed, 209 insertions(+), 64 deletions(-)
 create mode 100644 config/audit_logging.go
 create mode 100644 config/files/defaults/audit-policy.yaml
 rename config/{ => files}/templates/aws.sample.config.yaml.tmpl (100%)
 rename config/{ => files}/templates/hetzner.sample.config.yaml.tmpl (100%)
 rename config/{utils.go => parse.go} (79%)

diff --git a/README.md b/README.md
index d980ca3..3961ad4 100644
--- a/README.md
+++ b/README.md
@@ -30,6 +30,8 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [ ] Support adding admin SSH keys via config file.
 - [ ] Support using HTTPS for ArgoCD apps.
 - [ ] Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure Provider component first.
+- [ ] Support enabling `Audit Logging`.
+- [ ] Switch to IAM Role from (temporary) credentials after cluster bootstrap.
 
 ## REFERENCES
 
@@ -48,3 +50,7 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [Kubernetes Backups, Upgrades, Migrations - with Velero](https://youtu.be/zybLTQER0yY?si=qOZcizBqPOeouJ7y)
 
 - [Failover](https://docs.hetzner.com/robot/dedicated-server/ip/failover/)
+
+- [Auditing](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/)
+
+- [Kube API server args](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/)
diff --git a/config/audit_logging.go b/config/audit_logging.go
new file mode 100644
index 0000000..a1a732f
--- /dev/null
+++ b/config/audit_logging.go
@@ -0,0 +1,96 @@
+package config
+
+import (
+	_ "embed"
+
+	"github.com/Obmondo/kubeaid-bootstrap-script/constants"
+	kubeadmAPIV1Beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+)
+
+var (
+	// REFER : https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/.
+	kubeAPIServerDefaultExtraArgsForAuditLogging = map[string]string{
+		"audit-log-maxage":    "10",
+		"audit-log-maxbackup": "1",
+		"audit-log-maxsize":   "100",
+
+		constants.KubeAPIServerFlagAuditPolicyFile: "/srv/kubernetes/audit.yaml",
+
+		constants.KubeAPIServerFlagAuditLogPath: "/var/log/kube-apiserver-audit.log",
+	}
+
+	//go:embed files/defaults/audit-policy.yaml
+	defaultAuditPolicy string
+)
+
+// Hydrates the KubeAid Bootstrap Script config with the default Kube API audit logging related
+// options (if not provided by the user).
+func hydrateWithAuditLoggingOptions() {
+	if !ParsedConfig.Cluster.EnableAuditLogging {
+		return
+	}
+
+	// If the user has not specified required extra args for the Kube API server, then use the
+	// default values.
+	for key, defaultValue := range kubeAPIServerDefaultExtraArgsForAuditLogging {
+		if _, ok := ParsedConfig.Cluster.APIServer.ExtraArgs[key]; !ok {
+			ParsedConfig.Cluster.APIServer.ExtraArgs[key] = defaultValue
+		}
+	}
+
+	auditPolicyFileHostPath := ParsedConfig.Cluster.APIServer.ExtraArgs[constants.KubeAPIServerFlagAuditPolicyFile]
+
+	// If the user has not specified an Audit Policy file, then use the default one.
+	{
+		isAuditPolicyFileProvidedByUser := false
+		for _, file := range ParsedConfig.Cluster.Files {
+			if file.Path == auditPolicyFileHostPath {
+				isAuditPolicyFileProvidedByUser = true
+				break
+			}
+		}
+
+		if !isAuditPolicyFileProvidedByUser {
+			ParsedConfig.Cluster.Files = append(ParsedConfig.Cluster.Files, kubeadmAPIV1Beta1.File{
+				Path:    auditPolicyFileHostPath,
+				Content: defaultAuditPolicy,
+				Append:  false,
+			})
+		}
+	}
+
+	// Make sure that the audit policy file is mounted to the Kube API server pod.
+	ensureHostPathGetsMounted(auditPolicyFileHostPath, kubeadmAPIV1Beta1.HostPathMount{
+		Name:      constants.KubeAPIServerFlagAuditPolicyFile,
+		HostPath:  auditPolicyFileHostPath,
+		MountPath: auditPolicyFileHostPath,
+		ReadOnly:  true,
+	})
+
+	// If using the log backend, make sure that the log backend file is mounted to the Kube API
+	// server pod.
+	if logBackendHostPath, ok := kubeAPIServerDefaultExtraArgsForAuditLogging[constants.KubeAPIServerFlagAuditLogPath]; ok {
+		ensureHostPathGetsMounted(logBackendHostPath, kubeadmAPIV1Beta1.HostPathMount{
+			Name:      "log-backend",
+			HostPath:  logBackendHostPath,
+			MountPath: logBackendHostPath,
+			ReadOnly:  false,
+		})
+	}
+}
+
+// Ensures that the given host path gets mounted to the Kube API server pod. If not, then uses the
+// given default volume to do the mount.
+func ensureHostPathGetsMounted(hostPath string, volume kubeadmAPIV1Beta1.HostPathMount) {
+	hostPathAlreadyMounted := false
+	for _, userSpecifiedVolume := range ParsedConfig.Cluster.APIServer.ExtraVolumes {
+		if userSpecifiedVolume.HostPath == volume.HostPath {
+			hostPathAlreadyMounted = true
+			break
+		}
+	}
+
+	if !hostPathAlreadyMounted {
+		ParsedConfig.Cluster.APIServer.ExtraVolumes = append(ParsedConfig.Cluster.APIServer.ExtraVolumes, volume)
+	}
+}
diff --git a/config/config.go b/config/config.go
index 72ea169..7dbc1dc 100644
--- a/config/config.go
+++ b/config/config.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	coreV1 "k8s.io/api/core/v1"
+	kubeadmBootstrapProviderV1Beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 )
 
 type (
@@ -31,8 +32,15 @@ type (
 	ClusterConfig struct {
 		Name       string `yaml:"name" validate:"required,notblank"`
 		K8sVersion string `yaml:"k8sVersion" validate:"required,notblank"`
+
+		EnableAuditLogging bool `yaml:"enableAuditLogging"`
+
+		APIServer APIServerConfig                        `yaml:"apiServer"`
+		Files     []kubeadmBootstrapProviderV1Beta1.File `yaml:"files"`
 	}
 
+	APIServerConfig kubeadmBootstrapProviderV1Beta1.ControlPlaneComponent
+
 	CloudConfig struct {
 		AWS     *AWSConfig     `yaml:"aws"`
 		Hetzner *HetznerConfig `yaml:"hetzner"`
diff --git a/config/files/defaults/audit-policy.yaml b/config/files/defaults/audit-policy.yaml
new file mode 100644
index 0000000..d2dcb72
--- /dev/null
+++ b/config/files/defaults/audit-policy.yaml
@@ -0,0 +1,15 @@
+apiVersion: audit.k8s.io/v1
+kind: Policy
+
+# Don't generate audit events for all requests in the RequestReceived stage.
+omitStages:
+  - "RequestReceived"
+
+rules:
+  # Log events with metadata (requesting user, timestamp, resource, verb, etc.) but not request or
+  # response body.
+  - level: Metadata
+    # Long-running requests like watches that fall under this rule will not generate an audit event
+    # in RequestReceived stage. We will omit those logs.
+    omitStages:
+      - "RequestReceived"
diff --git a/config/templates/aws.sample.config.yaml.tmpl b/config/files/templates/aws.sample.config.yaml.tmpl
similarity index 100%
rename from config/templates/aws.sample.config.yaml.tmpl
rename to config/files/templates/aws.sample.config.yaml.tmpl
diff --git a/config/templates/hetzner.sample.config.yaml.tmpl b/config/files/templates/hetzner.sample.config.yaml.tmpl
similarity index 100%
rename from config/templates/hetzner.sample.config.yaml.tmpl
rename to config/files/templates/hetzner.sample.config.yaml.tmpl
diff --git a/config/generate_sample_config.go b/config/generate_sample_config.go
index e3f9528..41f99f6 100644
--- a/config/generate_sample_config.go
+++ b/config/generate_sample_config.go
@@ -11,7 +11,7 @@ import (
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils/templates"
 )
 
-//go:embed templates/*
+//go:embed files/templates/*
 var SampleConfigs embed.FS
 
 func GenerateSampleConfig(ctx context.Context, cloudProvider string) {
diff --git a/config/utils.go b/config/parse.go
similarity index 79%
rename from config/utils.go
rename to config/parse.go
index 98f3142..3f99dcf 100644
--- a/config/utils.go
+++ b/config/parse.go
@@ -30,14 +30,19 @@ func ParseConfig(ctx context.Context, configAsString string) {
 	slog.InfoContext(ctx, "Parsed config")
 
 	// Set defaults.
-	err = defaults.Set(ParsedConfig)
-	assert.AssertErrNil(ctx, err, "Failed setting defaults for parsed config")
-	//
-	// Read cloud credentials from CLI flags and store them in config.
-	readCloudCredentialsFromFlagsToConfig()
-	//
-	// Read SSH key-pairs from provided file paths and store them in config.
-	readSSHKeys(ParsedConfig)
+	{
+		err = defaults.Set(ParsedConfig)
+		assert.AssertErrNil(ctx, err, "Failed setting defaults for parsed config")
+
+		// Read cloud credentials from CLI flags and store them in config.
+		readCloudCredentialsFromFlagsToConfig()
+
+		// Read SSH key-pairs from provided file paths and store them in config.
+		hydrateSSHKeyConfigs()
+
+		// Hydrate with Audit Logging options (if required).
+		hydrateWithAuditLoggingOptions()
+	}
 
 	// Validate.
 	validateConfig(ParsedConfig)
@@ -62,16 +67,16 @@ func readCloudCredentialsFromFlagsToConfig() {
 	}
 }
 
-func readSSHKeys(config *Config) {
+func hydrateSSHKeyConfigs() {
 	switch {
-	case config.Cloud.Hetzner != nil:
-		readSSHKey(&config.Cloud.Hetzner.RobotSSHKeyPair)
+	case ParsedConfig.Cloud.Hetzner != nil:
+		hydrateSSHKeyConfig(&ParsedConfig.Cloud.Hetzner.RobotSSHKeyPair)
 	}
 }
 
 // Reads and validates an SSH key-pair from the provided file paths.
 // The key-pair is then stored in the SSH key config struct itself.
-func readSSHKey(sshKeyConfig *SSHKeyPairConfig) {
+func hydrateSSHKeyConfig(sshKeyConfig *SSHKeyPairConfig) {
 	ctx := context.Background()
 
 	// Read and validate the SSH public key.
diff --git a/constants/constants.go b/constants/constants.go
index aa47c3b..73d5285 100644
--- a/constants/constants.go
+++ b/constants/constants.go
@@ -44,6 +44,12 @@ const (
 	FlagNameAWSRegion       = "aws-region"
 )
 
+// Kube API server CLI flags.
+const (
+	KubeAPIServerFlagAuditPolicyFile = "audit-policy-file"
+	KubeAPIServerFlagAuditLogPath    = "audit-log-path"
+)
+
 // Cloud providers.
 const (
 	CloudProviderAWS     = "aws"
@@ -76,8 +82,8 @@ const (
 
 // Template names.
 var (
-	TemplateNameAWSSampleConfig     = "templates/aws.sample.config.yaml.tmpl"
-	TemplateNameHetznerSampleConfig = "templates/hetzner.sample.config.yaml.tmpl"
+	TemplateNameAWSSampleConfig     = "files/templates/aws.sample.config.yaml.tmpl"
+	TemplateNameHetznerSampleConfig = "files/templates/hetzner.sample.config.yaml.tmpl"
 
 	CommonNonSecretTemplateNames = []string{
 		// For ArgoCD.
diff --git a/go.mod b/go.mod
index ee6b610..da647ba 100644
--- a/go.mod
+++ b/go.mod
@@ -25,12 +25,12 @@ require (
 	golang.org/x/crypto v0.31.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.16.3
-	k8s.io/api v0.31.2
-	k8s.io/apimachinery v0.31.2
-	k8s.io/client-go v0.31.2
+	k8s.io/api v0.31.3
+	k8s.io/apimachinery v0.31.3
+	k8s.io/client-go v0.31.3
 	k8s.io/kubernetes v1.31.2
-	sigs.k8s.io/cluster-api v1.8.5
-	sigs.k8s.io/controller-runtime v0.19.1
+	sigs.k8s.io/cluster-api v1.9.3
+	sigs.k8s.io/controller-runtime v0.19.3
 )
 
 require (
@@ -100,7 +100,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
-	github.com/fatih/color v1.17.0 // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
@@ -243,9 +243,9 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
-	golang.org/x/net v0.32.0 // indirect
-	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect
@@ -255,16 +255,17 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	google.golang.org/grpc v1.66.3 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.31.2 // indirect
-	k8s.io/apiserver v0.31.2 // indirect
+	k8s.io/apiextensions-apiserver v0.31.3 // indirect
+	k8s.io/apiserver v0.31.3 // indirect
 	k8s.io/cli-runtime v0.31.2 // indirect
-	k8s.io/component-base v0.31.2 // indirect
+	k8s.io/cluster-bootstrap v0.31.3 // indirect
+	k8s.io/component-base v0.31.3 // indirect
 	k8s.io/component-helpers v0.31.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-aggregator v0.31.2 // indirect
diff --git a/go.sum b/go.sum
index b207484..05eae9a 100644
--- a/go.sum
+++ b/go.sum
@@ -249,8 +249,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
-github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
-github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -414,8 +414,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM=
-github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -635,8 +635,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay
 github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
 github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
 github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
-github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
-github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -650,8 +650,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ
 github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
 github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
-github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os=
-github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo=
+github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y=
+github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -888,8 +888,8 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -902,8 +902,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
+golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -935,14 +935,14 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
-golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
-golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
-golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
+golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1051,8 +1051,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1095,8 +1095,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
@@ -1145,22 +1145,24 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.17.8/go.mod h1:N++Llhs8kCixMUoCaXXAyMMPbo8dDVnh+IQ36xZV2/0=
-k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
-k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
-k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0=
-k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM=
+k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8=
+k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE=
+k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE=
+k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4=
 k8s.io/apimachinery v0.17.8/go.mod h1:Lg8zZ5iC/O8UjCqW6DNhcQG2m4TdjF9kwG3891OWbbA=
-k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
-k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
-k8s.io/apiserver v0.31.2 h1:VUzOEUGRCDi6kX1OyQ801m4A7AUPglpsmGvdsekmcI4=
-k8s.io/apiserver v0.31.2/go.mod h1:o3nKZR7lPlJqkU5I3Ove+Zx3JuoFjQobGX1Gctw6XuE=
+k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4=
+k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apiserver v0.31.3 h1:+1oHTtCB+OheqFEz375D0IlzHZ5VeQKX1KGXnx+TTuY=
+k8s.io/apiserver v0.31.3/go.mod h1:PrxVbebxrxQPFhJk4powDISIROkNMKHibTg9lTRQ0Qg=
 k8s.io/cli-runtime v0.31.2 h1:7FQt4C4Xnqx8V1GJqymInK0FFsoC+fAZtbLqgXYVOLQ=
 k8s.io/cli-runtime v0.31.2/go.mod h1:XROyicf+G7rQ6FQJMbeDV9jqxzkWXTYD6Uxd15noe0Q=
 k8s.io/client-go v0.17.8/go.mod h1:SJsDS64AAtt9VZyeaQMb4Ck5etCitZ/FwajWdzua5eY=
-k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
-k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
-k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA=
-k8s.io/component-base v0.31.2/go.mod h1:9PeyyFN/drHjtJZMCTkSpQJS3U9OXORnHQqMLDz0sUQ=
+k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4=
+k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs=
+k8s.io/cluster-bootstrap v0.31.3 h1:O1Yxk1bLaxZvmQCXLaJjj5iJD+lVMfJdRUuKgbUHPlA=
+k8s.io/cluster-bootstrap v0.31.3/go.mod h1:TI6TCsQQB4FfcryWgNO3SLXSKWBqHjx4DfyqSFwixj8=
+k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ=
+k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU=
 k8s.io/component-helpers v0.31.2 h1:V2yjoNeyg8WfvwrJwzfYz+RUwjlbcAIaDaHEStBbaZM=
 k8s.io/component-helpers v0.31.2/go.mod h1:cNz+1ck38R0qWrjcw/rhQgGP6+Gwgw8ngr2ziDNmSXM=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -1186,10 +1188,10 @@ oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=
 oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo=
 oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c=
 oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg=
-sigs.k8s.io/cluster-api v1.8.5 h1:lNA2fPN4fkXEs+oOQlnwxT/4VwRFBpv5kkSoJG8nqBA=
-sigs.k8s.io/cluster-api v1.8.5/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY=
-sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk=
-sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/cluster-api v1.9.3 h1:lKWbrXzyNmJh++IcX54ZbAmnO7tZ2wKgds7WvskpiXY=
+sigs.k8s.io/cluster-api v1.9.3/go.mod h1:5iojv38PSvOd4cxqu08Un5TQmy2yBkd3+0U7R/e+msk=
+sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw=
+sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g=
diff --git a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
index ec5b8ef..0f6f844 100644
--- a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
+++ b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
@@ -32,6 +32,12 @@ aws:
     instanceType: {{ .AWSConfig.ControlPlane.InstanceType }}
     ami:
       id: {{ .AWSConfig.ControlPlane.AMI.ID }}
+    {{- if .ClusterConfig.APIServer }}
+    apiServer: {{ .ClusterConfig.APIServer | toYaml | nindent 6 }}
+    {{- end }}
+    {{- if .ClusterConfig.Files }}
+    files: {{ .ClusterConfig.Files | toYaml | nindent 6 }}
+    {{- end }}
   nodeGroups:
 {{ .AWSConfig.NodeGroups | toYaml | indent 2 }}
 {{- end }}

From c0e02bdf29540f068b46e840108bbc57a1245518 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 7 Jan 2025 18:47:31 +0530
Subject: [PATCH 07/15] Switching from temporary credentials to IAM roles after
 cluster bootstrap

Signed-off-by: Archisman <archisman@obmondo.com>
---
 README.md                     |  2 ++
 pkg/core/bootstrap_cluster.go | 14 ++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/README.md b/README.md
index 3961ad4..8ea5721 100644
--- a/README.md
+++ b/README.md
@@ -54,3 +54,5 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [Auditing](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/)
 
 - [Kube API server args](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/)
+
+- [Using IAM roles in management cluster instead of AWS credentials](https://cluster-api-aws.sigs.k8s.io/topics/using-iam-roles-in-mgmt-cluster)
diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go
index 31b93e9..2156720 100644
--- a/pkg/core/bootstrap_cluster.go
+++ b/pkg/core/bootstrap_cluster.go
@@ -112,6 +112,20 @@ func dogfoodProvisionedCluster(ctx context.Context, gitAuthMethod transport.Auth
 	SetupCluster(ctx, provisionedClusterClient)
 
 	if !skipClusterctlMove {
+		// Make ClusterAPI use IAM roles instead of (temporary) credentials.
+		//
+		// NOTE : The ClusterAPI AWS InfrastructureProvider component (CAPA controller) needs to run in
+		//        a master node.
+		//        And, the master node count should be more than 1.
+		{
+			// Zero the credentials CAPA controller started with.
+			// This will force the CAPA controller to fall back to use the attached instance profiles.
+			utils.ExecuteCommandOrDie("clusterawsadm controller zero-credentials --namespace capi-cluster")
+
+			// Rollout and restart on capa-controller-manager deployment.
+			utils.ExecuteCommandOrDie("clusterawsadm controller rollout-controller --namespace capi-cluster")
+		}
+
 		// Move ClusterAPI manifests to the provisioned cluster.
 		utils.ExecuteCommandOrDie(fmt.Sprintf(
 			"clusterctl move --kubeconfig %s --namespace %s --to-kubeconfig %s",

From 3b15de2e665b3576ef68c2d9aa9bde9388efec01 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 7 Jan 2025 19:13:03 +0530
Subject: [PATCH 08/15] (fix) : initializing extraArgs, extraVolumes and files
 first with the default package

Signed-off-by: Archisman <archisman@obmondo.com>
---
 config/config.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/config/config.go b/config/config.go
index 7dbc1dc..a0fa83b 100644
--- a/config/config.go
+++ b/config/config.go
@@ -36,10 +36,13 @@ type (
 		EnableAuditLogging bool `yaml:"enableAuditLogging"`
 
 		APIServer APIServerConfig                        `yaml:"apiServer"`
-		Files     []kubeadmBootstrapProviderV1Beta1.File `yaml:"files"`
+		Files     []kubeadmBootstrapProviderV1Beta1.File `yaml:"files" default:"[]"`
 	}
 
-	APIServerConfig kubeadmBootstrapProviderV1Beta1.ControlPlaneComponent
+	APIServerConfig struct {
+		ExtraArgs    map[string]string                               `yaml:"extraArgs" default:"{}"`
+		ExtraVolumes []kubeadmBootstrapProviderV1Beta1.HostPathMount `yaml:"extraVolumes" default:"[]"`
+	}
 
 	CloudConfig struct {
 		AWS     *AWSConfig     `yaml:"aws"`

From a2a1314e0a85524faaab22be100f5d5fd319cbcb Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Thu, 9 Jan 2025 19:00:12 +0530
Subject: [PATCH 09/15] Added support for enabling Audit Logging and switching
 to IAM roles from (temporary) credentials

Signed-off-by: Archisman <archisman@obmondo.com>
---
 Makefile                                     | 17 ++++++++-----
 README.md                                    | 15 +++++++++---
 config/audit_logging.go                      | 14 ++++++-----
 config/config.go                             | 25 ++++++++++++++++----
 config/files/defaults/audit-policy.yaml      |  2 --
 go.mod                                       |  1 -
 go.sum                                       |  2 --
 pkg/cloud/hetzner/execute_failover_script.go | 18 ++++++++++++++
 pkg/core/bootstrap_cluster.go                |  7 ++++++
 utils/k3d.go                                 | 14 ++++++++++-
 10 files changed, 89 insertions(+), 26 deletions(-)
 create mode 100644 pkg/cloud/hetzner/execute_failover_script.go

diff --git a/Makefile b/Makefile
index e6adbca..70ca205 100644
--- a/Makefile
+++ b/Makefile
@@ -49,15 +49,15 @@ generate-sample-config-aws-dev:
 bootstrap-cluster-dev-aws:
 	@go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap aws \
 		--debug \
-		--config /app/outputs/kubeaid-bootstrap-script.config.yaml \
-		--skip-clusterctl-move
+		--config /app/outputs/kubeaid-bootstrap-script.aws.config.yaml
 # --skip-kubeaid-config-setup
+# --skip-clusterctl-move
 
 .PHONY: bootstrap-cluster-dev-hetzner
 bootstrap-cluster-dev-hetzner:
 	@go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap hetzner \
 		--debug \
-    --config /app/outputs/kubeaid-bootstrap-script.config.yaml \
+    --config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml \
     --skip-clusterctl-move
 # --skip-kubeaid-config-setup
 
@@ -69,10 +69,15 @@ use-management-cluster:
 use-provisioned-cluster:
 	export KUBECONFIG=./outputs/provisioned-cluster.kubeconfig.yaml
 
-.PHONY: delete-provisioned-cluster-dev
-delete-provisioned-cluster-dev:
+.PHONY: delete-provisioned-cluster-dev-aws
+delete-provisioned-cluster-dev-aws:
 	@go run ./cmd/kubeaid-bootstrap-script/ cluster delete \
-		--config /app/outputs/kubeaid-bootstrap-script.config.yaml
+		--config /app/outputs/kubeaid-bootstrap-script.aws.config.yaml
+
+.PHONY: delete-provisioned-cluster-dev-hetzner
+delete-provisioned-cluster-dev-hetzner:
+	@go run ./cmd/kubeaid-bootstrap-script/ cluster delete \
+		--config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml
 
 .PHONY: delete-management-cluster
 delete-management-cluster:
diff --git a/README.md b/README.md
index 8ea5721..621d158 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ The `KubeAid Bootstrap Script` is used to bootstrap Kubernetes clusters using Cl
 
 - [Bootstrapping a self-managed cluster in AWS](https://github.com/Obmondo/KubeAid/blob/master/docs/aws/capi/cluster.md)
 
-## Developer Guide
+## Developer Guide (AWS)
 
 > Make sure, you've Docker installed in your system.
 
@@ -22,6 +22,12 @@ Once you're inside the container, use `make generate-sample-config-aws-dev` to g
 
 Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 
+### Debugging
+
+- Check ClusterAPI related pod logs.
+
+- SSH into the control-plane node. In case of AWS, you can view cloud-init logs stored at `/var/log/cloud-init-output.log`.
+
 ## TODOs
 
 - [ ] Check Git URL if SSH agent is used.
@@ -30,8 +36,11 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [ ] Support adding admin SSH keys via config file.
 - [ ] Support using HTTPS for ArgoCD apps.
 - [ ] Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure Provider component first.
-- [ ] Support enabling `Audit Logging`.
-- [ ] Switch to IAM Role from (temporary) credentials after cluster bootstrap.
+- [x] Support enabling `Audit Logging`.
+- [x] Switch to IAM Role from (temporary) credentials after cluster bootstrap.
+- [ ] Support scale to / from zero for the node-groups.
+  > First, this needs to be solved : [Allow adding extra rules to the Role / ClusterRole template of the Cluster AutoScaler Helm chart](https://github.com/kubernetes/autoscaler/issues/7680).
+- [ ] `recover cluster` command
 
 ## REFERENCES
 
diff --git a/config/audit_logging.go b/config/audit_logging.go
index a1a732f..1082185 100644
--- a/config/audit_logging.go
+++ b/config/audit_logging.go
@@ -4,7 +4,7 @@ import (
 	_ "embed"
 
 	"github.com/Obmondo/kubeaid-bootstrap-script/constants"
-	kubeadmAPIV1Beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	v1 "k8s.io/api/core/v1"
 )
 
 var (
@@ -16,7 +16,7 @@ var (
 
 		constants.KubeAPIServerFlagAuditPolicyFile: "/srv/kubernetes/audit.yaml",
 
-		constants.KubeAPIServerFlagAuditLogPath: "/var/log/kube-apiserver-audit.log",
+		constants.KubeAPIServerFlagAuditLogPath: "/var/log/kube-apiserver-audit.logs",
 	}
 
 	//go:embed files/defaults/audit-policy.yaml
@@ -51,7 +51,7 @@ func hydrateWithAuditLoggingOptions() {
 		}
 
 		if !isAuditPolicyFileProvidedByUser {
-			ParsedConfig.Cluster.Files = append(ParsedConfig.Cluster.Files, kubeadmAPIV1Beta1.File{
+			ParsedConfig.Cluster.Files = append(ParsedConfig.Cluster.Files, FileConfig{
 				Path:    auditPolicyFileHostPath,
 				Content: defaultAuditPolicy,
 				Append:  false,
@@ -60,28 +60,30 @@ func hydrateWithAuditLoggingOptions() {
 	}
 
 	// Make sure that the audit policy file is mounted to the Kube API server pod.
-	ensureHostPathGetsMounted(auditPolicyFileHostPath, kubeadmAPIV1Beta1.HostPathMount{
+	ensureHostPathGetsMounted(auditPolicyFileHostPath, HostPathMountConfig{
 		Name:      constants.KubeAPIServerFlagAuditPolicyFile,
 		HostPath:  auditPolicyFileHostPath,
 		MountPath: auditPolicyFileHostPath,
 		ReadOnly:  true,
+		PathType:  v1.HostPathFileOrCreate,
 	})
 
 	// If using the log backend, make sure that the log backend file is mounted to the Kube API
 	// server pod.
 	if logBackendHostPath, ok := kubeAPIServerDefaultExtraArgsForAuditLogging[constants.KubeAPIServerFlagAuditLogPath]; ok {
-		ensureHostPathGetsMounted(logBackendHostPath, kubeadmAPIV1Beta1.HostPathMount{
+		ensureHostPathGetsMounted(logBackendHostPath, HostPathMountConfig{
 			Name:      "log-backend",
 			HostPath:  logBackendHostPath,
 			MountPath: logBackendHostPath,
 			ReadOnly:  false,
+			PathType:  v1.HostPathFileOrCreate,
 		})
 	}
 }
 
 // Ensures that the given host path gets mounted to the Kube API server pod. If not, then uses the
 // given default volume to do the mount.
-func ensureHostPathGetsMounted(hostPath string, volume kubeadmAPIV1Beta1.HostPathMount) {
+func ensureHostPathGetsMounted(hostPath string, volume HostPathMountConfig) {
 	hostPathAlreadyMounted := false
 	for _, userSpecifiedVolume := range ParsedConfig.Cluster.APIServer.ExtraVolumes {
 		if userSpecifiedVolume.HostPath == volume.HostPath {
diff --git a/config/config.go b/config/config.go
index a0fa83b..1ecceb4 100644
--- a/config/config.go
+++ b/config/config.go
@@ -4,7 +4,6 @@ import (
 	"context"
 
 	coreV1 "k8s.io/api/core/v1"
-	kubeadmBootstrapProviderV1Beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 )
 
 type (
@@ -35,13 +34,29 @@ type (
 
 		EnableAuditLogging bool `yaml:"enableAuditLogging"`
 
-		APIServer APIServerConfig                        `yaml:"apiServer"`
-		Files     []kubeadmBootstrapProviderV1Beta1.File `yaml:"files" default:"[]"`
+		APIServer APIServerConfig `yaml:"apiServer"`
+		Files     []FileConfig    `yaml:"files" default:"[]"`
 	}
 
 	APIServerConfig struct {
-		ExtraArgs    map[string]string                               `yaml:"extraArgs" default:"{}"`
-		ExtraVolumes []kubeadmBootstrapProviderV1Beta1.HostPathMount `yaml:"extraVolumes" default:"[]"`
+		ExtraArgs    map[string]string     `yaml:"extraArgs" default:"{}"`
+		ExtraVolumes []HostPathMountConfig `yaml:"extraVolumes" default:"[]"`
+	}
+
+	// REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".HostPathMount
+	HostPathMountConfig struct {
+		Name      string              `yaml:"name" validate:"required,notblank"`
+		HostPath  string              `yaml:"hostPath" validate:"required,notblank"`
+		MountPath string              `yaml:"mountPath" validate:"required,notblank"`
+		ReadOnly  bool                `yaml:"readOnly" default:"true"`
+		PathType  coreV1.HostPathType `yaml:"pathType" validate:"required"`
+	}
+
+	// REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".File
+	FileConfig struct {
+		Path    string `yaml:"path" validate:"required,notblank"`
+		Append  bool   `yaml:"append" default:"false"`
+		Content string `yaml:"content" validate:"required,notblank"`
 	}
 
 	CloudConfig struct {
diff --git a/config/files/defaults/audit-policy.yaml b/config/files/defaults/audit-policy.yaml
index d2dcb72..7e58b5c 100644
--- a/config/files/defaults/audit-policy.yaml
+++ b/config/files/defaults/audit-policy.yaml
@@ -1,10 +1,8 @@
 apiVersion: audit.k8s.io/v1
 kind: Policy
-
 # Don't generate audit events for all requests in the RequestReceived stage.
 omitStages:
   - "RequestReceived"
-
 rules:
   # Log events with metadata (requesting user, timestamp, resource, verb, etc.) but not request or
   # response body.
diff --git a/go.mod b/go.mod
index da647ba..3609407 100644
--- a/go.mod
+++ b/go.mod
@@ -264,7 +264,6 @@ require (
 	k8s.io/apiextensions-apiserver v0.31.3 // indirect
 	k8s.io/apiserver v0.31.3 // indirect
 	k8s.io/cli-runtime v0.31.2 // indirect
-	k8s.io/cluster-bootstrap v0.31.3 // indirect
 	k8s.io/component-base v0.31.3 // indirect
 	k8s.io/component-helpers v0.31.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
diff --git a/go.sum b/go.sum
index 05eae9a..b642ed2 100644
--- a/go.sum
+++ b/go.sum
@@ -1159,8 +1159,6 @@ k8s.io/cli-runtime v0.31.2/go.mod h1:XROyicf+G7rQ6FQJMbeDV9jqxzkWXTYD6Uxd15noe0Q
 k8s.io/client-go v0.17.8/go.mod h1:SJsDS64AAtt9VZyeaQMb4Ck5etCitZ/FwajWdzua5eY=
 k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4=
 k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs=
-k8s.io/cluster-bootstrap v0.31.3 h1:O1Yxk1bLaxZvmQCXLaJjj5iJD+lVMfJdRUuKgbUHPlA=
-k8s.io/cluster-bootstrap v0.31.3/go.mod h1:TI6TCsQQB4FfcryWgNO3SLXSKWBqHjx4DfyqSFwixj8=
 k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ=
 k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU=
 k8s.io/component-helpers v0.31.2 h1:V2yjoNeyg8WfvwrJwzfYz+RUwjlbcAIaDaHEStBbaZM=
diff --git a/pkg/cloud/hetzner/execute_failover_script.go b/pkg/cloud/hetzner/execute_failover_script.go
new file mode 100644
index 0000000..a11f45a
--- /dev/null
+++ b/pkg/cloud/hetzner/execute_failover_script.go
@@ -0,0 +1,18 @@
+package hetzner
+
+import "context"
+
+/*
+Let's say there are x (> 1) master nodes behind the Failover IP.
+
+CAPH (Cluster API Provider Hetzner) will SSH into any one of those master nodes (let's denote
+it by β) and execute `kubeadm init`. The Machine resource corresponding to β will then be
+marked as ready. We'll wait for this event.
+
+Once this event occurs, we'll SSH into β and run the Hetzner Failover Script (using the
+hetzner-robot App in KubeAid). This will make the Failover IP point to β.
+
+The Kubernetes API server of the provisioned cluster will then be reachable via that Failover
+IP.
+*/
+func ExecuteFailoverScript(ctx context.Context) {}
diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go
index 2156720..370c265 100644
--- a/pkg/core/bootstrap_cluster.go
+++ b/pkg/core/bootstrap_cluster.go
@@ -10,6 +10,7 @@ import (
 	"github.com/Obmondo/kubeaid-bootstrap-script/constants"
 	"github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud"
 	"github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/aws"
+	"github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/hetzner"
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils"
 	argoCDV1Alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/go-git/go-git/v5/plumbing/transport"
@@ -70,6 +71,12 @@ func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMetho
 	// Close ArgoCD application client.
 	constants.ArgoCDApplicationClientCloser.Close()
 
+	// CASE : Hetzner
+	// Make the Failover IP point to the master node where `kubeadm init` has been executed.
+	if config.ParsedConfig.Cloud.Hetzner != nil {
+		hetzner.ExecuteFailoverScript(ctx)
+	}
+
 	// Wait for the main cluster to be provisioned and ready.
 	utils.WaitForMainClusterToBeProvisioned(ctx, managementClusterClient)
 
diff --git a/utils/k3d.go b/utils/k3d.go
index 77b7ca8..f370c03 100644
--- a/utils/k3d.go
+++ b/utils/k3d.go
@@ -39,10 +39,22 @@ func CreateK3DCluster(ctx context.Context, name string) {
 	// Since we are mounting the Docker socket to the dev container, it can resolve DNS names of
 	// Docker networks. So use the DNS name instead of 0.0.0.0.
 	// NOTE : Consider this situation :
-	//				an existing K3D cluster may have wrong Kubernetes API server URL server.
+	// 			an existing K3D cluster may have wrong Kubernetes API server URL server.
 	ExecuteCommandOrDie(fmt.Sprintf(`
 		kubectl config set-cluster k3d-%s --server=https://k3d-%s-serverlb:6443
 	`, name, name))
+
+	// Initially the master nodes have label node-role.kubernetes.io/control-plane set to "true".
+	// We'll change the label value to "" (just like it is in Vanilla Kubernetes).
+	// Some apps (like capi-cluster) relies on this label to get scheduled to the master node.
+	ExecuteCommandOrDie(fmt.Sprintf(`
+		master_nodes=$(kubectl get nodes -l node-role.kubernetes.io/control-plane=true -o name)
+
+		for node in $master_nodes; do
+			kubectl label $node node-role.kubernetes.io/control-plane-
+			kubectl label $node node-role.kubernetes.io/control-plane=""
+		done
+	`))
 }
 
 // Returns whether the given K3d cluster exists or not.

From 4ae5a0475c818d47b57306dea5ee3835c30d9fb7 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Mon, 13 Jan 2025 05:01:53 +0530
Subject: [PATCH 10/15] (fix) : KubeadmControlPlane resource forever OutOfSync
 issue fixed | (chore) : minor changes

Signed-off-by: Archisman <archisman@obmondo.com>
---
 README.md                                           | 12 +++++++++---
 config/audit_logging.go                             |  6 ++----
 config/config.go                                    | 13 ++++++++++---
 pkg/cloud/hetzner/execute_failover_script.go        |  4 +++-
 pkg/core/bootstrap_cluster.go                       |  2 +-
 .../argocd-apps/capi-cluster.values.yaml.tmpl       |  3 ---
 scripts/install-prerequisites.sh                    |  4 ++--
 utils/k3d.go                                        |  3 ++-
 8 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 621d158..19b2a19 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ In a separate terminal window, use `make exec-container-dev` to execute into the
 
 Once you're inside the container, use `make generate-sample-config-aws-dev` to generate a sample config file at [./outputs/kubeaid-bootstrap-script.config.yaml](./outputs/kubeaid-bootstrap-script.config.yaml), targetting the AWS cloud provider. Adjust the config file according to your needs.
 
-Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
+Export your AWS credentials as environment variables and then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 
 ### Debugging
 
@@ -38,8 +38,10 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [ ] Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure Provider component first.
 - [x] Support enabling `Audit Logging`.
 - [x] Switch to IAM Role from (temporary) credentials after cluster bootstrap.
-- [ ] Support scale to / from zero for the node-groups.
-  > First, this needs to be solved : [Allow adding extra rules to the Role / ClusterRole template of the Cluster AutoScaler Helm chart](https://github.com/kubernetes/autoscaler/issues/7680).
+- [x] ETCD metrics enabled.
+- [x] Support scale to / from zero for the node-groups.
+  > Currently, I have added extra ClusterRole and ClusterRoleBinding in the KubeAid [cluster-autoscaler Helm chart](https://github.com/Obmondo/kubeaid/tree/master/argocd-helm-charts/cluster-autoscaler) to support this feature.
+  > But I have also opened an issue in the kubernetes-sigs/autoscaler repository regarding this : [Allow adding extra rules to the Role / ClusterRole template of the Cluster AutoScaler Helm chart](https://github.com/kubernetes/autoscaler/issues/7680).
 - [ ] `recover cluster` command
 
 ## REFERENCES
@@ -65,3 +67,7 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster!
 - [Kube API server args](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/)
 
 - [Using IAM roles in management cluster instead of AWS credentials](https://cluster-api-aws.sigs.k8s.io/topics/using-iam-roles-in-mgmt-cluster)
+
+- [KubeadmControlPlane CRD](https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml)
+
+- [How can you call a helm 'helper' template from a subchart with the correct context?](https://stackoverflow.com/questions/47791971/how-can-you-call-a-helm-helper-template-from-a-subchart-with-the-correct-conte)
diff --git a/config/audit_logging.go b/config/audit_logging.go
index 1082185..c6716f3 100644
--- a/config/audit_logging.go
+++ b/config/audit_logging.go
@@ -43,7 +43,7 @@ func hydrateWithAuditLoggingOptions() {
 	// If the user has not specified an Audit Policy file, then use the default one.
 	{
 		isAuditPolicyFileProvidedByUser := false
-		for _, file := range ParsedConfig.Cluster.Files {
+		for _, file := range ParsedConfig.Cluster.APIServer.Files {
 			if file.Path == auditPolicyFileHostPath {
 				isAuditPolicyFileProvidedByUser = true
 				break
@@ -51,10 +51,9 @@ func hydrateWithAuditLoggingOptions() {
 		}
 
 		if !isAuditPolicyFileProvidedByUser {
-			ParsedConfig.Cluster.Files = append(ParsedConfig.Cluster.Files, FileConfig{
+			ParsedConfig.Cluster.APIServer.Files = append(ParsedConfig.Cluster.APIServer.Files, FileConfig{
 				Path:    auditPolicyFileHostPath,
 				Content: defaultAuditPolicy,
-				Append:  false,
 			})
 		}
 	}
@@ -75,7 +74,6 @@ func hydrateWithAuditLoggingOptions() {
 			Name:      "log-backend",
 			HostPath:  logBackendHostPath,
 			MountPath: logBackendHostPath,
-			ReadOnly:  false,
 			PathType:  v1.HostPathFileOrCreate,
 		})
 	}
diff --git a/config/config.go b/config/config.go
index 1ecceb4..8297ff3 100644
--- a/config/config.go
+++ b/config/config.go
@@ -35,12 +35,12 @@ type (
 		EnableAuditLogging bool `yaml:"enableAuditLogging"`
 
 		APIServer APIServerConfig `yaml:"apiServer"`
-		Files     []FileConfig    `yaml:"files" default:"[]"`
 	}
 
 	APIServerConfig struct {
 		ExtraArgs    map[string]string     `yaml:"extraArgs" default:"{}"`
 		ExtraVolumes []HostPathMountConfig `yaml:"extraVolumes" default:"[]"`
+		Files        []FileConfig          `yaml:"files" default:"[]"`
 	}
 
 	// REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".HostPathMount
@@ -48,14 +48,21 @@ type (
 		Name      string              `yaml:"name" validate:"required,notblank"`
 		HostPath  string              `yaml:"hostPath" validate:"required,notblank"`
 		MountPath string              `yaml:"mountPath" validate:"required,notblank"`
-		ReadOnly  bool                `yaml:"readOnly" default:"true"`
 		PathType  coreV1.HostPathType `yaml:"pathType" validate:"required"`
+
+		// Whether the mount should be read-only or not.
+		// Defaults to true.
+		//
+		// NOTE : If you want the mount to be read-only, then set this true.
+		//        Otherwise, omit setting this field. It gets removed by the kubeadm control-plane
+		//        provider component, which results to the capi-cluster ArgoCD App always being in
+		//        OutOfSync state.
+		ReadOnly bool `yaml:"readOnly,omitempty"`
 	}
 
 	// REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".File
 	FileConfig struct {
 		Path    string `yaml:"path" validate:"required,notblank"`
-		Append  bool   `yaml:"append" default:"false"`
 		Content string `yaml:"content" validate:"required,notblank"`
 	}
 
diff --git a/pkg/cloud/hetzner/execute_failover_script.go b/pkg/cloud/hetzner/execute_failover_script.go
index a11f45a..edaf67a 100644
--- a/pkg/cloud/hetzner/execute_failover_script.go
+++ b/pkg/cloud/hetzner/execute_failover_script.go
@@ -15,4 +15,6 @@ hetzner-robot App in KubeAid). This will make the Failover IP point to β.
 The Kubernetes API server of the provisioned cluster will then be reachable via that Failover
 IP.
 */
-func ExecuteFailoverScript(ctx context.Context) {}
+func ExecuteFailoverScript(ctx context.Context) {
+	panic("unimplemented")
+}
diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go
index 370c265..ccaf14c 100644
--- a/pkg/core/bootstrap_cluster.go
+++ b/pkg/core/bootstrap_cluster.go
@@ -97,7 +97,7 @@ func dogfoodProvisionedCluster(ctx context.Context, gitAuthMethod transport.Auth
 
 	if isPartOfDisasterRecovery {
 		// If this is a part of the disaster recovery process, then
-		// restore Kubernetes Secrets containing a Sealed Secrets keys.
+		// restore Kubernetes Secrets containing a Sealed Secrets key.
 
 		sealedSecretsBackupBucketName := cloudProvider.GetSealedSecretsBackupBucketName()
 		manifestsDirPath := utils.GetDirPathForDownloadedStorageBucketContents(sealedSecretsBackupBucketName)
diff --git a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
index 0f6f844..4e55a27 100644
--- a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
+++ b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl
@@ -35,9 +35,6 @@ aws:
     {{- if .ClusterConfig.APIServer }}
     apiServer: {{ .ClusterConfig.APIServer | toYaml | nindent 6 }}
     {{- end }}
-    {{- if .ClusterConfig.Files }}
-    files: {{ .ClusterConfig.Files | toYaml | nindent 6 }}
-    {{- end }}
   nodeGroups:
 {{ .AWSConfig.NodeGroups | toYaml | indent 2 }}
 {{- end }}
diff --git a/scripts/install-prerequisites.sh b/scripts/install-prerequisites.sh
index 541d961..c4a5f64 100644
--- a/scripts/install-prerequisites.sh
+++ b/scripts/install-prerequisites.sh
@@ -30,8 +30,8 @@ apt install -y curl
 curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
 
 # Clusterawsadm
-wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.5.2/clusterawsadm_v2.5.2_linux_"${CPU_ARCHITECTURE}"
-mv clusterawsadm_v2.5.2_linux_"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm
+wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.7.1/clusterawsadm_v2.7.1_linux_"${CPU_ARCHITECTURE}"
+mv clusterawsadm_v2.7.1_linux_"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm
 chmod +x /usr/local/bin/clusterawsadm
 
 # Clusterctl
diff --git a/utils/k3d.go b/utils/k3d.go
index f370c03..451c86d 100644
--- a/utils/k3d.go
+++ b/utils/k3d.go
@@ -38,8 +38,9 @@ func CreateK3DCluster(ctx context.Context, name string) {
 	// resolvable from within the dev container.
 	// Since we are mounting the Docker socket to the dev container, it can resolve DNS names of
 	// Docker networks. So use the DNS name instead of 0.0.0.0.
+	//
 	// NOTE : Consider this situation :
-	// 			an existing K3D cluster may have wrong Kubernetes API server URL server.
+	//        an existing K3D cluster may have wrong Kubernetes API server URL server.
 	ExecuteCommandOrDie(fmt.Sprintf(`
 		kubectl config set-cluster k3d-%s --server=https://k3d-%s-serverlb:6443
 	`, name, name))

From 4fa5fa5b0f25d7a61d37bebd4578a71cc685b2e1 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Mon, 13 Jan 2025 05:25:32 +0530
Subject: [PATCH 11/15] (fix) : clusterawsadm latest version download URL

Signed-off-by: Archisman <archisman@obmondo.com>
---
 scripts/install-prerequisites.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-prerequisites.sh b/scripts/install-prerequisites.sh
index c4a5f64..b604979 100644
--- a/scripts/install-prerequisites.sh
+++ b/scripts/install-prerequisites.sh
@@ -30,8 +30,8 @@ apt install -y curl
 curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
 
 # Clusterawsadm
-wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.7.1/clusterawsadm_v2.7.1_linux_"${CPU_ARCHITECTURE}"
-mv clusterawsadm_v2.7.1_linux_"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm
+wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.7.1/clusterawsadm-linux-"${CPU_ARCHITECTURE}"
+mv clusterawsadm-linux-"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm
 chmod +x /usr/local/bin/clusterawsadm
 
 # Clusterctl

From 8f5250bd085b112eb32ac7f21fd15203f0fb5ff7 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Mon, 13 Jan 2025 16:21:50 +0530
Subject: [PATCH 12/15] (feat/autoscaling) : adding minSize and maxSize in
 NodeGroupConfig + validating them

Signed-off-by: Archisman <archisman@obmondo.com>
---
 config/config.go       | 23 ++++++++++++++---------
 config/validate.go     | 10 ++++++++++
 utils/assert/assert.go | 18 ++++++++++++++----
 3 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/config/config.go b/config/config.go
index 8297ff3..bfa7c9c 100644
--- a/config/config.go
+++ b/config/config.go
@@ -109,20 +109,25 @@ type (
 	}
 
 	ControlPlaneConfig struct {
-		Replicas     int       `yaml:"replicas" validate:"required"`
+		Replicas     uint      `yaml:"replicas" validate:"required"`
 		InstanceType string    `yaml:"instanceType" validate:"required,notblank"`
 		AMI          AMIConfig `yaml:"ami" validate:"required"`
 	}
 
 	NodeGroups struct {
-		Name           string            `yaml:"name" validate:"required,notblank"`
-		Replicas       int               `yaml:"replicas" validate:"required"`
-		InstanceType   string            `yaml:"instanceType" validate:"required,notblank"`
-		SSHKeyName     string            `yaml:"sshKeyName" validate:"required,notblank"`
-		AMI            AMIConfig         `yaml:"ami" validate:"required"`
-		RootVolumeSize int               `yaml:"rootVolumeSize" validate:"required"`
-		Labels         map[string]string `yaml:"labels" default:"[]"`
-		Taints         []*coreV1.Taint   `yaml:"taints" default:"[]"`
+		Name string `yaml:"name" validate:"required,notblank"`
+
+		Replicas uint `yaml:"replicas" validate:"required"`
+		MinSize  uint `yaml:"minSize" validate:"required"`
+		Maxsize  uint `yaml:"maxSize" validate:"required"`
+
+		InstanceType   string    `yaml:"instanceType" validate:"required,notblank"`
+		SSHKeyName     string    `yaml:"sshKeyName" validate:"required,notblank"`
+		AMI            AMIConfig `yaml:"ami" validate:"required"`
+		RootVolumeSize uint      `yaml:"rootVolumeSize" validate:"required"`
+
+		Labels map[string]string `yaml:"labels" default:"[]"`
+		Taints []*coreV1.Taint   `yaml:"taints" default:"[]"`
 	}
 
 	AMIConfig struct {
diff --git a/config/validate.go b/config/validate.go
index 4367fd8..df5006f 100644
--- a/config/validate.go
+++ b/config/validate.go
@@ -37,6 +37,16 @@ func validateConfig(config *Config) {
 	switch {
 	case config.Cloud.AWS != nil:
 		for _, nodeGroup := range config.Cloud.AWS.NodeGroups {
+			// Validate auto-scaling options.
+			assert.Assert(ctx,
+				nodeGroup.Replicas >= nodeGroup.MinSize,
+				"node-group replica count should be >= its min-size", slog.String("node-group", nodeGroup.Name),
+			)
+			assert.Assert(ctx,
+				nodeGroup.Replicas <= nodeGroup.Maxsize,
+				"node-group replica count should be <= its max-size", slog.String("node-group", nodeGroup.Name),
+			)
+
 			// Validate labels and taints.
 			validateLabelsAndTaints(ctx, nodeGroup.Name, nodeGroup.Labels, nodeGroup.Taints)
 		}
diff --git a/utils/assert/assert.go b/utils/assert/assert.go
index 5e87f42..88c9741 100644
--- a/utils/assert/assert.go
+++ b/utils/assert/assert.go
@@ -20,21 +20,31 @@ func AssertErrNil(ctx context.Context, err error, customErrorMessage string, att
 }
 
 // Panics if the given value isn't nil.
-func AssertNil(ctx context.Context, value interface{}, errorMessage string) {
+func AssertNil(ctx context.Context, value interface{}, errorMessage string, attributes ...any) {
 	if value == nil {
 		return
 	}
 
-	slog.ErrorContext(ctx, errorMessage)
+	slog.ErrorContext(ctx, errorMessage, attributes...)
 	os.Exit(1)
 }
 
 // Panics if the given value is nil.
-func AssertNotNil(ctx context.Context, value interface{}, errorMessage string) {
+func AssertNotNil(ctx context.Context, value interface{}, errorMessage string, attributes ...any) {
 	if value != nil {
 		return
 	}
 
-	slog.ErrorContext(ctx, errorMessage)
+	slog.ErrorContext(ctx, errorMessage, attributes...)
+	os.Exit(1)
+}
+
+// Panics if the given value is false.
+func Assert(ctx context.Context, value bool, errorMessage string, attributes ...any) {
+	if !value {
+		return
+	}
+
+	slog.ErrorContext(ctx, errorMessage, attributes...)
 	os.Exit(1)
 }

From cfb7d4cbdb6199f75d2f5737f35d467c9799070b Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 14 Jan 2025 00:27:52 +0530
Subject: [PATCH 13/15] (fix) : panicking when error occurs bootstrapping IAM
 CloudFormation stack

Signed-off-by: Archisman <archisman@obmondo.com>
---
 config/validate.go               |  4 ++--
 pkg/cloud/aws/utils.go           | 11 ++++++-----
 scripts/install-prerequisites.sh |  4 ++--
 utils/assert/assert.go           |  2 +-
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/config/validate.go b/config/validate.go
index df5006f..480ac5c 100644
--- a/config/validate.go
+++ b/config/validate.go
@@ -40,11 +40,11 @@ func validateConfig(config *Config) {
 			// Validate auto-scaling options.
 			assert.Assert(ctx,
 				nodeGroup.Replicas >= nodeGroup.MinSize,
-				"node-group replica count should be >= its min-size", slog.String("node-group", nodeGroup.Name),
+				"replica count should be >= its min-size", slog.String("node-group", nodeGroup.Name),
 			)
 			assert.Assert(ctx,
 				nodeGroup.Replicas <= nodeGroup.Maxsize,
-				"node-group replica count should be <= its max-size", slog.String("node-group", nodeGroup.Name),
+				"replica count should be <= its max-size", slog.String("node-group", nodeGroup.Name),
 			)
 
 			// Validate labels and taints.
diff --git a/pkg/cloud/aws/utils.go b/pkg/cloud/aws/utils.go
index 47c21dd..118f493 100644
--- a/pkg/cloud/aws/utils.go
+++ b/pkg/cloud/aws/utils.go
@@ -3,7 +3,6 @@ package aws
 import (
 	"context"
 	"log/slog"
-	"strings"
 
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils"
 	"github.com/Obmondo/kubeaid-bootstrap-script/utils/assert"
@@ -33,8 +32,10 @@ func CreateIAMCloudFormationStack() {
 	// NOTE : This requires admin privileges.
 	output, err := utils.ExecuteCommand("clusterawsadm bootstrap iam create-cloudformation-stack")
 
-	// Panic if an error occurs (except regarding the AWS Cloudformation stack already existing).
-	if !strings.Contains(output, "already exists, updating") {
-		assert.AssertErrNil(context.Background(), err, "Failed bootstrapping IAM CloudFormation Stack", slog.String("output", output))
-	}
+	// NOTE : If the CloudFormation template is in ROLLBACK_COMPLETE state, maybe there are
+	// pre-existing IAM resources with overlapping name. If so, then first delete them manually from
+	// the AWS Console and then retry running the script.
+	assert.AssertErrNil(context.Background(), err,
+		"Failed bootstrapping IAM CloudFormation Stack", slog.String("output", output),
+	)
 }
diff --git a/scripts/install-prerequisites.sh b/scripts/install-prerequisites.sh
index b604979..06ebd5d 100644
--- a/scripts/install-prerequisites.sh
+++ b/scripts/install-prerequisites.sh
@@ -38,13 +38,13 @@ chmod +x /usr/local/bin/clusterawsadm
 curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-"${CPU_ARCHITECTURE}" -o clusterctl
 install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl
 
-# ------------------------------------------- Utilities -------------------------------------------
-
 # Kubectl
 curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${CPU_ARCHITECTURE}/kubectl"
 chmod +x ./kubectl
 mv ./kubectl /usr/local/bin
 
+# ------------------------------------------- Utilities -------------------------------------------
+
 # K9s
 wget https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_linux_"${CPU_ARCHITECTURE}".deb
 dpkg -i k9s_linux_"${CPU_ARCHITECTURE}".deb
diff --git a/utils/assert/assert.go b/utils/assert/assert.go
index 88c9741..eb65169 100644
--- a/utils/assert/assert.go
+++ b/utils/assert/assert.go
@@ -41,7 +41,7 @@ func AssertNotNil(ctx context.Context, value interface{}, errorMessage string, a
 
 // Panics if the given value is false.
 func Assert(ctx context.Context, value bool, errorMessage string, attributes ...any) {
-	if !value {
+	if value {
 		return
 	}
 

From 84b76230b66f4ad2020b15c204268580eb954b29 Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 14 Jan 2025 00:34:06 +0530
Subject: [PATCH 14/15] (fix) : turning on
 enableClusterAPIScaleFromZeroSupport.aws for cluste-autoscaler Helm
 installation

Signed-off-by: Archisman <archisman@obmondo.com>
---
 .../templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl b/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl
index 77899f7..7d50a57 100644
--- a/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl
+++ b/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl
@@ -17,3 +17,6 @@ cluster-autoscaler:
   autoDiscovery:
     clusterName: {{ .ClusterConfig.Name }}
     namespace: {{ .CAPIClusterNamespace }}
+
+enableClusterAPIScaleFromZeroSupport:
+  aws: true

From e02f748bb82c4dbc678fee2b9771c1520e41ad1b Mon Sep 17 00:00:00 2001
From: Archisman <archisman@obmondo.com>
Date: Tue, 14 Jan 2025 08:17:36 +0530
Subject: [PATCH 15/15] (chore) : improving documentation

Signed-off-by: Archisman <archisman@obmondo.com>
---
 README.md                                     |  26 ++++-
 .../cluster/bootstrap/aws.go                  |   3 +-
 .../cluster/bootstrap/hetzner.go              |   3 +-
 .../cluster/delete/delete.go                  |   3 +-
 config/config.go                              |  10 +-
 .../templates/aws.sample.config.yaml.tmpl     | 105 ++++++++++++++----
 6 files changed, 120 insertions(+), 30 deletions(-)

diff --git a/README.md b/README.md
index 19b2a19..fd76e59 100644
--- a/README.md
+++ b/README.md
@@ -20,13 +20,31 @@ In a separate terminal window, use `make exec-container-dev` to execute into the
 
 Once you're inside the container, use `make generate-sample-config-aws-dev` to generate a sample config file at [./outputs/kubeaid-bootstrap-script.config.yaml](./outputs/kubeaid-bootstrap-script.config.yaml), targetting the AWS cloud provider. Adjust the config file according to your needs.
 
-Export your AWS credentials as environment variables and then run `make bootstrap-cluster-dev` to bootstrap the cluster!
+Export your AWS credentials as environment variables like such :
 
-### Debugging
+```sh
+export AWS_REGION=""
+export AWS_ACCESS_KEY_ID=""
+export AWS_SECRET_ACCESS_KEY=""
+export AWS_SESSION_TOKEN=""
+```
 
-- Check ClusterAPI related pod logs.
+Then run `make bootstrap-cluster-dev-aws` to bootstrap the cluster!
 
-- SSH into the control-plane node. In case of AWS, you can view cloud-init logs stored at `/var/log/cloud-init-output.log`.
+> [!NOTE]
+> If the `clusterawsadm bootstrap iam create-cloudformation-stack` command errors out with this message :
+>
+>     	the IAM CloudFormation Stack create / update failed and it's currently in a `ROLLBACK_COMPLETE` state
+>
+> then that means maybe there are pre-existing IAM resources with overlapping name. Then first delete them manually from the AWS Console and then retry running the script. Filter the IAM roles and policies in the corresponding region with the keyword : `cluster` / `clusterapi`.
+
+If cluster provisioning gets stuck, then debug by :
+
+- checking logs of ClusterAPI related pod.
+
+- SSHing into the control-plane node. You can view cloud-init output logs stored at `/var/log/cloud-init-output.log`.
+
+If you want to delete the provisioned cluster, then execute : `make delete-provisioned-cluster-dev-aws`.
 
 ## TODOs
 
diff --git a/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go
index 0e807fc..b9c4e0b 100644
--- a/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go
+++ b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go
@@ -8,7 +8,8 @@ import (
 )
 
 var AWSCmd = &cobra.Command{
-	Use: "aws",
+	Use:   "aws",
+	Short: "Bootstrap a self-managed Kubernetes cluster in AWS",
 	Run: func(cmd *cobra.Command, args []string) {
 		core.BootstrapCluster(cmd.Context(), skipKubeAidConfigSetup, skipClusterctlMove, aws.NewAWSCloudProvider(), false)
 	},
diff --git a/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go
index 561816c..5816b8e 100644
--- a/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go
+++ b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go
@@ -8,7 +8,8 @@ import (
 )
 
 var HetznerCmd = &cobra.Command{
-	Use: "hetzner",
+	Use:   "hetzner",
+	Short: "Bootstrap a self-managed Kubernetes cluster in Hetzner (bare-metal)",
 	Run: func(cmd *cobra.Command, args []string) {
 		core.BootstrapCluster(cmd.Context(), skipKubeAidConfigSetup, skipClusterctlMove, hetzner.NewHetznerCloudProvider(), false)
 	},
diff --git a/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go b/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go
index 4a2a5d4..c3a3038 100644
--- a/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go
+++ b/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go
@@ -6,7 +6,8 @@ import (
 )
 
 var DeleteCmd = &cobra.Command{
-	Use: "delete",
+	Use:   "delete",
+	Short: "Delete a provisioned cluster",
 	Run: func(cmd *cobra.Command, args []string) {
 		core.DeleteCluster(cmd.Context())
 	},
diff --git a/config/config.go b/config/config.go
index bfa7c9c..1577f76 100644
--- a/config/config.go
+++ b/config/config.go
@@ -37,6 +37,14 @@ type (
 		APIServer APIServerConfig `yaml:"apiServer"`
 	}
 
+	// REFER : https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml.
+	//
+	// NOTE : Generally, refer to the KubeadmControlPlane CRD instead of the corresponding GoLang
+	//        source types linked below.
+	//        There are some configuration options which appear in the corresponding GoLang source type,
+	//        but not in the CRD. If you set those fields, then they get removed by the Kubeadm
+	//        control-plane provider. This causes the capi-cluster ArgoCD App to always be in an
+	//        OutOfSync state, resulting to the KubeAid Bootstrap Script not making any progress!
 	APIServerConfig struct {
 		ExtraArgs    map[string]string     `yaml:"extraArgs" default:"{}"`
 		ExtraVolumes []HostPathMountConfig `yaml:"extraVolumes" default:"[]"`
@@ -54,7 +62,7 @@ type (
 		// Defaults to true.
 		//
 		// NOTE : If you want the mount to be read-only, then set this true.
-		//        Otherwise, omit setting this field. It gets removed by the kubeadm control-plane
+		//        Otherwise, omit setting this field. It gets removed by the Kubeadm control-plane
 		//        provider component, which results to the capi-cluster ArgoCD App always being in
 		//        OutOfSync state.
 		ReadOnly bool `yaml:"readOnly,omitempty"`
diff --git a/config/files/templates/aws.sample.config.yaml.tmpl b/config/files/templates/aws.sample.config.yaml.tmpl
index 60d86a7..b598928 100644
--- a/config/files/templates/aws.sample.config.yaml.tmpl
+++ b/config/files/templates/aws.sample.config.yaml.tmpl
@@ -1,50 +1,111 @@
-forks:
-  kubeaidConfig: https://github.com/xxxxxxxxxx/kubeaid-config
-
+# Git credentials used to authenticate against the Git platform you're using (Github / Gitlab etc.).
+# KubeAid Bootstrap Script will use these credentials to :
+#
+#   (1) Clone the KubeAid and KubeAid config repositories.
+#   (2) Create and push commits to a branch in the KubeAid config repository.
+#
+# So, make sure the Git password (token) you're using has permissions associated to do the above.
+#
+# Currently, we only support HTTPS authentication.
 git:
   username: xxxxxxxxxx
   password: xxxxxxxxxx
 
+forks:
+  # KubeAid repository URL (in HTTPs syntax).
+  # Defaults to Obmondo's KubeAid repository.
+  # kubeaid: https://github.com/Obmondo/KubeAid
+
+  # Your KubeAid config repository URL (in HTTPs syntax).
+  kubeaidConfig: https://github.com/xxxxxxxxxx/kubeaid-config
+
+# Kubernetes cluster and control-plane specific configurations.
 cluster:
+
+  # Kubernetes cluster name.
   name: kubeaid-demo-aws
+
+  # Kubernetes version to use.
+  # NOTE : Make sure that the AMI you're using, is targetted towards this Kubernetes version.
   k8sVersion: v1.31.0
 
+  # Kubernetes API server specific configurations.
+  # REFER : https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml.
+  #  
+  # NOTE : Generally, refer to the KubeadmControlPlane CRD instead of the corresponding GoLang
+  #        source types linked below.
+  #        There are some configuration options which appear in the corresponding GoLang source type,
+  #        but not in the CRD. If you set those fields, then they get removed by the Kubeadm
+  #        control-plane provider. This causes the capi-cluster ArgoCD App to always be in an
+  #        OutOfSync state, resulting to the KubeAid Bootstrap Script not making any progress!
+  # apiServer:
+  #
+  #   extraArgs: {}
+  #
+  #   # REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".HostPathMount
+  #   #
+  #   # NOTE : If you want a mount to be read-only, then set extraVolume.readOnly to true.
+  #   #        Otherwise, omit setting that field. It gets removed by the Kubeadm control-plane
+  #   #        provider component, which results to the capi-cluster ArgoCD App always being in
+  #   #        OutOfSync state.
+  #   extraVolumes: []
+  #
+  #   # REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".File
+  #   files: []
+
+  # Uncomment, if you just want audit-logging to be setup for you! KubeAid Bootstrap Script will set
+  # necessary configuration options in cluster.apiServer.
+  # enableAuditLogging: True
+
 cloud:
+  # AWS specific configurations.
   aws:
-		region: us-east-1
+    region: us-east-1
 
+    # AWS SSH Keypair name, which ClusterAPI components (Kubeadm bootstrap and control-plane
+    # providers specifically) will use to SSH into the main cluster's master nodes.
     sshKeyName: kubeaid-demo
 
-		bastionEnabled: False
+    # bastionEnabled: True
 
     controlPlane:
-			instanceType: t4g.medium
-    	ami:
-				id: ami-xxxxxxxxxxxxxxxxx
-    	replicas: 1
+      ami:
+        id: ami-xxxxxxxxxxxxxxxxx
+      instanceType: t4g.medium
+      replicas: 1
 
     nodeGroups:
       - name: primary
         ami:
-					id: ami-xxxxxxxxxxxxxxxxx
+          id: ami-xxxxxxxxxxxxxxxxx
         instanceType: t4g.medium
+
+        minSize: 0
         replicas: 1
+        maxSize: 3
+
         rootVolumeSize: 35
+
+        # AWS SSH Keypair name, which ClusterAPI components (Kubeadm bootstrap provider
+        # specifically) will use to SSH into each node belonging to this node-group.
         sshKeyName: kubeaid-demo
 
-				# Label should meet one of the following criterias to propagate to Node :
-				#
-				# (1) Has node-role.kubernetes.io as prefix.
-				# (2) Belongs to node-restriction.kubernetes.io domain.
-				# (3) Belongs to node.cluster.x-k8s.io domain.
-				#
-				# REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine
-				labels:
-					node.cluster.x-k8s.io/nodegroup: primary
+        # A label should meet one of the following criterias to propagate to each of the nodes :
+        #
+        # (1) Has node-role.kubernetes.io as prefix.
+        # (2) Belongs to node-restriction.kubernetes.io domain.
+        # (3) Belongs to node.cluster.x-k8s.io domain.
+        #
+        # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine
+        labels:
+          node.cluster.x-k8s.io/nodegroup: primary
           node-role.kubernetes.io/bootstrapper: ""
 
-        taints: []
+        # taints: []
 
-		disasterRecovery:
-			veleroBackupsS3BucketName: kubeaid-demo-kubernetes-objects
+    disasterRecovery:
+      veleroBackupsS3BucketName: kubeaid-demo-kubernetes-objects
       sealedSecretsBackupS3BucketName: kubeaid-demo-sealed-secrets
+
+# monitoring:
+#   kubePrometheusVersion: v0.14.0