diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8432114..636f52c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,11 +28,11 @@ jobs: username: obmondo password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push AMD64 and ARM64 container images + - name: Build and push KubeAid Bootstrap Script AMD64 and ARM64 container images uses: docker/build-push-action@v4 with: context: . - file: build/Dockerfile + file: build/kubeaid-bootstrap-script/Dockerfile # NOTE : It takes pretty long to build container images for the ARM64 platform (even when # using QEMU). platforms: linux/amd64,linux/arm64 @@ -43,3 +43,19 @@ jobs: # builds. cache-from: type=gha cache-to: type=gha,mode=max + + - name: Build and push Hetzner Failover Script AMD64 and ARM64 container images + uses: docker/build-push-action@v4 + with: + context: . + file: build/hetzner-failover-script/Dockerfile + # NOTE : It takes pretty long to build container images for the ARM64 platform (even when + # using QEMU). + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/obmondo/hetzner-failover-script:${{ github.event.release.tag_name }} + push: true + # Experimental cache exporter for GitHub Actions provided by buildx and BuildKit. + # It uses the GitHub Cache API to fetch and load the Docker layer cache blobs across + # builds. + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile index 7bc48e4..70ca205 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ IMAGE_NAME=kubeaid-bootstrap-script-dev:latest .PHONY: build-image-dev build-image-dev: - @docker build -f ./build/Dockerfile.dev --build-arg CPU_ARCHITECTURE=arm64 -t $(IMAGE_NAME) . + @docker build -f ./build/kubeaid-bootstrap-script/Dockerfile.dev --build-arg CPU_ARCHITECTURE=arm64 -t $(IMAGE_NAME) . .PHONY: remove-image-dev remove-image-dev: @@ -43,22 +43,22 @@ remove-container-dev: stop-container-dev .PHONY: generate-sample-config-aws-dev generate-sample-config-aws-dev: - @go run ./cmd config generate aws + @go run ./cmd/kubeaid-bootstrap-script/ config generate aws .PHONY: bootstrap-cluster-dev-aws bootstrap-cluster-dev-aws: - @go run ./cmd cluster bootstrap aws \ + @go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap aws \ --debug \ - --config /app/outputs/kubeaid-bootstrap-script.config.yaml \ - --skip-clusterctl-move + --config /app/outputs/kubeaid-bootstrap-script.aws.config.yaml # --skip-kubeaid-config-setup +# --skip-clusterctl-move .PHONY: bootstrap-cluster-dev-hetzner bootstrap-cluster-dev-hetzner: - @go run ./cmd cluster bootstrap hetzner \ + @go run ./cmd/kubeaid-bootstrap-script/ cluster bootstrap hetzner \ --debug \ - --config /app/outputs/kubeaid-bootstrap-script.config.yaml \ - --skip-clusterctl-move + --config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml \ + --skip-clusterctl-move # --skip-kubeaid-config-setup .PHONY: use-management-cluster @@ -69,10 +69,15 @@ use-management-cluster: use-provisioned-cluster: export KUBECONFIG=./outputs/provisioned-cluster.kubeconfig.yaml -.PHONY: delete-provisioned-cluster -delete-provisioned-cluster-dev: - @go run ./cmd cluster delete \ - --config /app/outputs/kubeaid-bootstrap-script.config.yaml +.PHONY: delete-provisioned-cluster-dev-aws +delete-provisioned-cluster-dev-aws: + @go run ./cmd/kubeaid-bootstrap-script/ cluster delete \ + --config /app/outputs/kubeaid-bootstrap-script.aws.config.yaml + +.PHONY: delete-provisioned-cluster-dev-hetzner +delete-provisioned-cluster-dev-hetzner: + @go run ./cmd/kubeaid-bootstrap-script/ cluster delete \ + --config /app/outputs/kubeaid-bootstrap-script.hetzner.config.yaml .PHONY: delete-management-cluster delete-management-cluster: diff --git a/README.md b/README.md index b6e68fd..fd76e59 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The `KubeAid Bootstrap Script` is used to bootstrap Kubernetes clusters using Cl - [Bootstrapping a self-managed cluster in AWS](https://github.com/Obmondo/KubeAid/blob/master/docs/aws/capi/cluster.md) -## Developer Guide +## Developer Guide (AWS) > Make sure, you've Docker installed in your system. @@ -20,7 +20,31 @@ In a separate terminal window, use `make exec-container-dev` to execute into the Once you're inside the container, use `make generate-sample-config-aws-dev` to generate a sample config file at [./outputs/kubeaid-bootstrap-script.config.yaml](./outputs/kubeaid-bootstrap-script.config.yaml), targetting the AWS cloud provider. Adjust the config file according to your needs. -Then run `make bootstrap-cluster-dev` to bootstrap the cluster! +Export your AWS credentials as environment variables like such : + +```sh +export AWS_REGION="" +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +export AWS_SESSION_TOKEN="" +``` + +Then run `make bootstrap-cluster-dev-aws` to bootstrap the cluster! + +> [!NOTE] +> If the `clusterawsadm bootstrap iam create-cloudformation-stack` command errors out with this message : +> +> the IAM CloudFormation Stack create / update failed and it's currently in a `ROLLBACK_COMPLETE` state +> +> then that means maybe there are pre-existing IAM resources with overlapping name. Then first delete them manually from the AWS Console and then retry running the script. Filter the IAM roles and policies in the corresponding region with the keyword : `cluster` / `clusterapi`. + +If cluster provisioning gets stuck, then debug by : + +- checking logs of ClusterAPI related pod. + +- SSHing into the control-plane node. You can view cloud-init output logs stored at `/var/log/cloud-init-output.log`. + +If you want to delete the provisioned cluster, then execute : `make delete-provisioned-cluster-dev-aws`. ## TODOs @@ -30,6 +54,13 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster! - [ ] Support adding admin SSH keys via config file. - [ ] Support using HTTPS for ArgoCD apps. - [ ] Use ArgoCD sync waves so that we don't need to explicitly sync the Infrastructure Provider component first. +- [x] Support enabling `Audit Logging`. +- [x] Switch to IAM Role from (temporary) credentials after cluster bootstrap. +- [x] ETCD metrics enabled. +- [x] Support scale to / from zero for the node-groups. + > Currently, I have added extra ClusterRole and ClusterRoleBinding in the KubeAid [cluster-autoscaler Helm chart](https://github.com/Obmondo/kubeaid/tree/master/argocd-helm-charts/cluster-autoscaler) to support this feature. + > But I have also opened an issue in the kubernetes-sigs/autoscaler repository regarding this : [Allow adding extra rules to the Role / ClusterRole template of the Cluster AutoScaler Helm chart](https://github.com/kubernetes/autoscaler/issues/7680). +- [ ] `recover cluster` command ## REFERENCES @@ -46,3 +77,15 @@ Then run `make bootstrap-cluster-dev` to bootstrap the cluster! - [Secret Rotation](https://github.com/bitnami-labs/sealed-secrets?tab=readme-ov-file#secret-rotation) - [Kubernetes Backups, Upgrades, Migrations - with Velero](https://youtu.be/zybLTQER0yY?si=qOZcizBqPOeouJ7y) + +- [Failover](https://docs.hetzner.com/robot/dedicated-server/ip/failover/) + +- [Auditing](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/) + +- [Kube API server args](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) + +- [Using IAM roles in management cluster instead of AWS credentials](https://cluster-api-aws.sigs.k8s.io/topics/using-iam-roles-in-mgmt-cluster) + +- [KubeadmControlPlane CRD](https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml) + +- [How can you call a helm 'helper' template from a subchart with the correct context?](https://stackoverflow.com/questions/47791971/how-can-you-call-a-helm-helper-template-from-a-subchart-with-the-correct-conte) diff --git a/build/hetzner-failover-script/Dockerfile b/build/hetzner-failover-script/Dockerfile new file mode 100644 index 0000000..e0e5588 --- /dev/null +++ b/build/hetzner-failover-script/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 + +#--- Builder stage --- + +FROM golang:1.23.0 AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN go build -o hetzner-failover-script ./cmd/hetzner-failover-script + +#--- Packager stage --- + +FROM golang:1.23.0 AS packages + +# Set the maintainer label +LABEL org.opencontainers.image.authors="ashish@obmondo.com, archisman@obmondo.com" + +RUN apk add --no-cache procps + +WORKDIR /root/ + +COPY --from=builder /app/hetzner-failover-script . + +CMD ["./hetzner-failover-script"] diff --git a/build/Dockerfile b/build/kubeaid-bootstrap-script/Dockerfile similarity index 71% rename from build/Dockerfile rename to build/kubeaid-bootstrap-script/Dockerfile index b3d2a01..fa7e988 100644 --- a/build/Dockerfile +++ b/build/kubeaid-bootstrap-script/Dockerfile @@ -10,18 +10,21 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go build -o kubeaid-bootstrap-script ./cmd +RUN go build -o kubeaid-bootstrap-script ./cmd/kubeaid-bootstrap-script #--- Packager stage --- FROM golang:1.23.0 AS packages +# Set the maintainer label +LABEL org.opencontainers.image.authors="ashish@obmondo.com, archisman@obmondo.com" + WORKDIR / COPY ./scripts/install-prerequisites.sh /install-prerequisites.sh RUN chmod +x /install-prerequisites.sh RUN CPU_ARCHITECTURE=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64") \ - /install-prerequisites.sh + /install-prerequisites.sh COPY --from=builder /app/kubeaid-bootstrap-script /usr/local/bin/kubeaid-bootstrap-script diff --git a/build/Dockerfile.dev b/build/kubeaid-bootstrap-script/Dockerfile.dev similarity index 91% rename from build/Dockerfile.dev rename to build/kubeaid-bootstrap-script/Dockerfile.dev index 07b5897..bfbee7e 100644 --- a/build/Dockerfile.dev +++ b/build/kubeaid-bootstrap-script/Dockerfile.dev @@ -7,7 +7,7 @@ WORKDIR / COPY ./scripts/install-prerequisites.sh /install-prerequisites.sh RUN chmod +x /install-prerequisites.sh RUN CPU_ARCHITECTURE=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64") \ - /install-prerequisites.sh + /install-prerequisites.sh WORKDIR /app diff --git a/cmd/hetzner-failover-script/main.go b/cmd/hetzner-failover-script/main.go new file mode 100644 index 0000000..a610cb8 --- /dev/null +++ b/cmd/hetzner-failover-script/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "log" + "log/slog" + "os" + "time" + + "github.com/Obmondo/kubeaid-bootstrap-script/utils" + "github.com/Obmondo/kubeaid-bootstrap-script/utils/assert" + "github.com/floshodan/hrobot-go/hrobot" +) + +func main() { + ctx := context.Background() + + // Read required environment variables. + var ( + failoverIP = utils.GetEnv("FAILOVER_IP") + + nodeIP = utils.GetEnv("NODE_IP") + + username = os.Getenv("API_USERNAME") // (optional). + password = os.Getenv("API_PASSWORD") // (optional). + + apiToken = os.Getenv("API_TOKEN") // (optional). + ) + + // Construct Hetzner Robot API client. + var hetznerRobotClient *hrobot.Client + switch { + case len(username) > 0 && len(password) > 0: + hetznerRobotClient = hrobot.NewClient(hrobot.WithBasicAuth(username, password)) + + case len(apiToken) > 0: + hetznerRobotClient = hrobot.NewClient(hrobot.WithToken(apiToken)) + + default: + log.Fatalf("Either provide username and password / api token as credentials, to communicate with the Hetzner Robot API") + } + + /* + A Failover IP is an additional IP that you can switch from one server to another. You can order + it for any Hetzner dedicated root server, and you can switch it to any other Hetzner dedicated + root server, regardless of location. + + Switching a failover IP takes between 90 and 110 seconds. + + REFERENCE : https://docs.hetzner.com/robot/dedicated-server/ip/failover/. + */ + // Hetzner Robot Failover IP API spec : API REFERENCE : https://robot.hetzner.com/doc/webservice/en.html#failover. + + // Get the Failover IP's current active server IP. + failoverIPDetails, _, err := hetznerRobotClient.Failover.GetFailoverIP(ctx, failoverIP) + assert.AssertErrNil(ctx, err, "Failed getting Failover IP details") + + activeServerIP := failoverIPDetails.ActiveServerIP + slog.InfoContext(ctx, "Detected active server", slog.String("ip", activeServerIP)) + + if activeServerIP == nodeIP { + slog.InfoContext(ctx, "Active server IP is already same as the current server IP") + return + } + + // Update Failover IP to the current node's IP (the current node, on which this script is + // running) + // NOTE : Contributed : + // https://github.com/floshodan/hrobot-go/commit/700f8ef9fdac565129608b3a50583b4b6564ff34. + _, _, err = hetznerRobotClient.Failover.SwitchFailover(ctx, failoverIP, activeServerIP) + assert.AssertErrNil(ctx, err, "Failed switching Failover IP to the current node IP") + + // Wait for the update to complete. + for { + failoverIPDetails, _, err := hetznerRobotClient.Failover.GetFailoverIP(ctx, failoverIP) + assert.AssertErrNil(ctx, err, "Failed getting Failover IP details") + + if failoverIPDetails.ActiveServerIP == nodeIP { + slog.InfoContext(ctx, "Successfully updated Failover IP", slog.String("active-server-ip", nodeIP)) + break + } + + slog.InfoContext(ctx, "Waiting for the Failover IP update to complete. Sleeping for a minute....") + time.Sleep(time.Minute) + } +} diff --git a/cmd/cluster/bootstrap/aws.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go similarity index 86% rename from cmd/cluster/bootstrap/aws.go rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go index 0e807fc..b9c4e0b 100644 --- a/cmd/cluster/bootstrap/aws.go +++ b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/aws.go @@ -8,7 +8,8 @@ import ( ) var AWSCmd = &cobra.Command{ - Use: "aws", + Use: "aws", + Short: "Bootstrap a self-managed Kubernetes cluster in AWS", Run: func(cmd *cobra.Command, args []string) { core.BootstrapCluster(cmd.Context(), skipKubeAidConfigSetup, skipClusterctlMove, aws.NewAWSCloudProvider(), false) }, diff --git a/cmd/cluster/bootstrap/bootstrap.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/bootstrap.go similarity index 100% rename from cmd/cluster/bootstrap/bootstrap.go rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/bootstrap.go diff --git a/cmd/cluster/bootstrap/hetzner.go b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go similarity index 84% rename from cmd/cluster/bootstrap/hetzner.go rename to cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go index 561816c..5816b8e 100644 --- a/cmd/cluster/bootstrap/hetzner.go +++ b/cmd/kubeaid-bootstrap-script/cluster/bootstrap/hetzner.go @@ -8,7 +8,8 @@ import ( ) var HetznerCmd = &cobra.Command{ - Use: "hetzner", + Use: "hetzner", + Short: "Bootstrap a self-managed Kubernetes cluster in Hetzner (bare-metal)", Run: func(cmd *cobra.Command, args []string) { core.BootstrapCluster(cmd.Context(), skipKubeAidConfigSetup, skipClusterctlMove, hetzner.NewHetznerCloudProvider(), false) }, diff --git a/cmd/cluster/cluster.go b/cmd/kubeaid-bootstrap-script/cluster/cluster.go similarity index 76% rename from cmd/cluster/cluster.go rename to cmd/kubeaid-bootstrap-script/cluster/cluster.go index 85b3bb9..30042f6 100644 --- a/cmd/cluster/cluster.go +++ b/cmd/kubeaid-bootstrap-script/cluster/cluster.go @@ -1,8 +1,8 @@ package cluster import ( - "github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster/bootstrap" - delete_ "github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster/delete" + "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster/bootstrap" + delete_ "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster/delete" "github.com/Obmondo/kubeaid-bootstrap-script/config" "github.com/Obmondo/kubeaid-bootstrap-script/utils" "github.com/spf13/cobra" diff --git a/cmd/cluster/delete/delete.go b/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go similarity index 80% rename from cmd/cluster/delete/delete.go rename to cmd/kubeaid-bootstrap-script/cluster/delete/delete.go index 4a2a5d4..c3a3038 100644 --- a/cmd/cluster/delete/delete.go +++ b/cmd/kubeaid-bootstrap-script/cluster/delete/delete.go @@ -6,7 +6,8 @@ import ( ) var DeleteCmd = &cobra.Command{ - Use: "delete", + Use: "delete", + Short: "Delete a provisioned cluster", Run: func(cmd *cobra.Command, args []string) { core.DeleteCluster(cmd.Context()) }, diff --git a/cmd/cluster/recover/aws.go b/cmd/kubeaid-bootstrap-script/cluster/recover/aws.go similarity index 100% rename from cmd/cluster/recover/aws.go rename to cmd/kubeaid-bootstrap-script/cluster/recover/aws.go diff --git a/cmd/cluster/recover/hetzner.go b/cmd/kubeaid-bootstrap-script/cluster/recover/hetzner.go similarity index 100% rename from cmd/cluster/recover/hetzner.go rename to cmd/kubeaid-bootstrap-script/cluster/recover/hetzner.go diff --git a/cmd/cluster/recover/recover.go b/cmd/kubeaid-bootstrap-script/cluster/recover/recover.go similarity index 100% rename from cmd/cluster/recover/recover.go rename to cmd/kubeaid-bootstrap-script/cluster/recover/recover.go diff --git a/cmd/config/config.go b/cmd/kubeaid-bootstrap-script/config/config.go similarity index 81% rename from cmd/config/config.go rename to cmd/kubeaid-bootstrap-script/config/config.go index cc59ce2..29281af 100644 --- a/cmd/config/config.go +++ b/cmd/kubeaid-bootstrap-script/config/config.go @@ -1,7 +1,7 @@ package config import ( - "github.com/Obmondo/kubeaid-bootstrap-script/cmd/config/generate" + "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/config/generate" "github.com/Obmondo/kubeaid-bootstrap-script/constants" "github.com/spf13/cobra" ) @@ -13,9 +13,7 @@ var ConfigCmd = &cobra.Command{ }, } -var ( - ConfigFilePath string -) +var ConfigFilePath string func init() { // Subcommands. diff --git a/cmd/config/generate/aws.go b/cmd/kubeaid-bootstrap-script/config/generate/aws.go similarity index 100% rename from cmd/config/generate/aws.go rename to cmd/kubeaid-bootstrap-script/config/generate/aws.go diff --git a/cmd/config/generate/generate.go b/cmd/kubeaid-bootstrap-script/config/generate/generate.go similarity index 100% rename from cmd/config/generate/generate.go rename to cmd/kubeaid-bootstrap-script/config/generate/generate.go diff --git a/cmd/config/generate/hetzner.go b/cmd/kubeaid-bootstrap-script/config/generate/hetzner.go similarity index 100% rename from cmd/config/generate/hetzner.go rename to cmd/kubeaid-bootstrap-script/config/generate/hetzner.go diff --git a/cmd/main.go b/cmd/kubeaid-bootstrap-script/main.go similarity index 82% rename from cmd/main.go rename to cmd/kubeaid-bootstrap-script/main.go index 8ed9dc7..b6a7284 100644 --- a/cmd/main.go +++ b/cmd/kubeaid-bootstrap-script/main.go @@ -4,8 +4,8 @@ import ( "log/slog" "os" - "github.com/Obmondo/kubeaid-bootstrap-script/cmd/cluster" - "github.com/Obmondo/kubeaid-bootstrap-script/cmd/config" + "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/cluster" + "github.com/Obmondo/kubeaid-bootstrap-script/cmd/kubeaid-bootstrap-script/config" "github.com/Obmondo/kubeaid-bootstrap-script/constants" "github.com/Obmondo/kubeaid-bootstrap-script/utils/logger" "github.com/spf13/cobra" diff --git a/config/audit_logging.go b/config/audit_logging.go new file mode 100644 index 0000000..c6716f3 --- /dev/null +++ b/config/audit_logging.go @@ -0,0 +1,96 @@ +package config + +import ( + _ "embed" + + "github.com/Obmondo/kubeaid-bootstrap-script/constants" + v1 "k8s.io/api/core/v1" +) + +var ( + // REFER : https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/. + kubeAPIServerDefaultExtraArgsForAuditLogging = map[string]string{ + "audit-log-maxage": "10", + "audit-log-maxbackup": "1", + "audit-log-maxsize": "100", + + constants.KubeAPIServerFlagAuditPolicyFile: "/srv/kubernetes/audit.yaml", + + constants.KubeAPIServerFlagAuditLogPath: "/var/log/kube-apiserver-audit.logs", + } + + //go:embed files/defaults/audit-policy.yaml + defaultAuditPolicy string +) + +// Hydrates the KubeAid Bootstrap Script config with the default Kube API audit logging related +// options (if not provided by the user). +func hydrateWithAuditLoggingOptions() { + if !ParsedConfig.Cluster.EnableAuditLogging { + return + } + + // If the user has not specified required extra args for the Kube API server, then use the + // default values. + for key, defaultValue := range kubeAPIServerDefaultExtraArgsForAuditLogging { + if _, ok := ParsedConfig.Cluster.APIServer.ExtraArgs[key]; !ok { + ParsedConfig.Cluster.APIServer.ExtraArgs[key] = defaultValue + } + } + + auditPolicyFileHostPath := ParsedConfig.Cluster.APIServer.ExtraArgs[constants.KubeAPIServerFlagAuditPolicyFile] + + // If the user has not specified an Audit Policy file, then use the default one. + { + isAuditPolicyFileProvidedByUser := false + for _, file := range ParsedConfig.Cluster.APIServer.Files { + if file.Path == auditPolicyFileHostPath { + isAuditPolicyFileProvidedByUser = true + break + } + } + + if !isAuditPolicyFileProvidedByUser { + ParsedConfig.Cluster.APIServer.Files = append(ParsedConfig.Cluster.APIServer.Files, FileConfig{ + Path: auditPolicyFileHostPath, + Content: defaultAuditPolicy, + }) + } + } + + // Make sure that the audit policy file is mounted to the Kube API server pod. + ensureHostPathGetsMounted(auditPolicyFileHostPath, HostPathMountConfig{ + Name: constants.KubeAPIServerFlagAuditPolicyFile, + HostPath: auditPolicyFileHostPath, + MountPath: auditPolicyFileHostPath, + ReadOnly: true, + PathType: v1.HostPathFileOrCreate, + }) + + // If using the log backend, make sure that the log backend file is mounted to the Kube API + // server pod. + if logBackendHostPath, ok := kubeAPIServerDefaultExtraArgsForAuditLogging[constants.KubeAPIServerFlagAuditLogPath]; ok { + ensureHostPathGetsMounted(logBackendHostPath, HostPathMountConfig{ + Name: "log-backend", + HostPath: logBackendHostPath, + MountPath: logBackendHostPath, + PathType: v1.HostPathFileOrCreate, + }) + } +} + +// Ensures that the given host path gets mounted to the Kube API server pod. If not, then uses the +// given default volume to do the mount. +func ensureHostPathGetsMounted(hostPath string, volume HostPathMountConfig) { + hostPathAlreadyMounted := false + for _, userSpecifiedVolume := range ParsedConfig.Cluster.APIServer.ExtraVolumes { + if userSpecifiedVolume.HostPath == volume.HostPath { + hostPathAlreadyMounted = true + break + } + } + + if !hostPathAlreadyMounted { + ParsedConfig.Cluster.APIServer.ExtraVolumes = append(ParsedConfig.Cluster.APIServer.ExtraVolumes, volume) + } +} diff --git a/config/config.go b/config/config.go index 72ea169..1577f76 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,47 @@ type ( ClusterConfig struct { Name string `yaml:"name" validate:"required,notblank"` K8sVersion string `yaml:"k8sVersion" validate:"required,notblank"` + + EnableAuditLogging bool `yaml:"enableAuditLogging"` + + APIServer APIServerConfig `yaml:"apiServer"` + } + + // REFER : https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml. + // + // NOTE : Generally, refer to the KubeadmControlPlane CRD instead of the corresponding GoLang + // source types linked below. + // There are some configuration options which appear in the corresponding GoLang source type, + // but not in the CRD. If you set those fields, then they get removed by the Kubeadm + // control-plane provider. This causes the capi-cluster ArgoCD App to always be in an + // OutOfSync state, resulting to the KubeAid Bootstrap Script not making any progress! + APIServerConfig struct { + ExtraArgs map[string]string `yaml:"extraArgs" default:"{}"` + ExtraVolumes []HostPathMountConfig `yaml:"extraVolumes" default:"[]"` + Files []FileConfig `yaml:"files" default:"[]"` + } + + // REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".HostPathMount + HostPathMountConfig struct { + Name string `yaml:"name" validate:"required,notblank"` + HostPath string `yaml:"hostPath" validate:"required,notblank"` + MountPath string `yaml:"mountPath" validate:"required,notblank"` + PathType coreV1.HostPathType `yaml:"pathType" validate:"required"` + + // Whether the mount should be read-only or not. + // Defaults to true. + // + // NOTE : If you want the mount to be read-only, then set this true. + // Otherwise, omit setting this field. It gets removed by the Kubeadm control-plane + // provider component, which results to the capi-cluster ArgoCD App always being in + // OutOfSync state. + ReadOnly bool `yaml:"readOnly,omitempty"` + } + + // REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".File + FileConfig struct { + Path string `yaml:"path" validate:"required,notblank"` + Content string `yaml:"content" validate:"required,notblank"` } CloudConfig struct { @@ -76,20 +117,25 @@ type ( } ControlPlaneConfig struct { - Replicas int `yaml:"replicas" validate:"required"` + Replicas uint `yaml:"replicas" validate:"required"` InstanceType string `yaml:"instanceType" validate:"required,notblank"` AMI AMIConfig `yaml:"ami" validate:"required"` } NodeGroups struct { - Name string `yaml:"name" validate:"required,notblank"` - Replicas int `yaml:"replicas" validate:"required"` - InstanceType string `yaml:"instanceType" validate:"required,notblank"` - SSHKeyName string `yaml:"sshKeyName" validate:"required,notblank"` - AMI AMIConfig `yaml:"ami" validate:"required"` - RootVolumeSize int `yaml:"rootVolumeSize" validate:"required"` - Labels map[string]string `yaml:"labels" default:"[]"` - Taints []*coreV1.Taint `yaml:"taints" default:"[]"` + Name string `yaml:"name" validate:"required,notblank"` + + Replicas uint `yaml:"replicas" validate:"required"` + MinSize uint `yaml:"minSize" validate:"required"` + Maxsize uint `yaml:"maxSize" validate:"required"` + + InstanceType string `yaml:"instanceType" validate:"required,notblank"` + SSHKeyName string `yaml:"sshKeyName" validate:"required,notblank"` + AMI AMIConfig `yaml:"ami" validate:"required"` + RootVolumeSize uint `yaml:"rootVolumeSize" validate:"required"` + + Labels map[string]string `yaml:"labels" default:"[]"` + Taints []*coreV1.Taint `yaml:"taints" default:"[]"` } AMIConfig struct { diff --git a/config/files/defaults/audit-policy.yaml b/config/files/defaults/audit-policy.yaml new file mode 100644 index 0000000..7e58b5c --- /dev/null +++ b/config/files/defaults/audit-policy.yaml @@ -0,0 +1,13 @@ +apiVersion: audit.k8s.io/v1 +kind: Policy +# Don't generate audit events for all requests in the RequestReceived stage. +omitStages: + - "RequestReceived" +rules: + # Log events with metadata (requesting user, timestamp, resource, verb, etc.) but not request or + # response body. + - level: Metadata + # Long-running requests like watches that fall under this rule will not generate an audit event + # in RequestReceived stage. We will omit those logs. + omitStages: + - "RequestReceived" diff --git a/config/files/templates/aws.sample.config.yaml.tmpl b/config/files/templates/aws.sample.config.yaml.tmpl new file mode 100644 index 0000000..b598928 --- /dev/null +++ b/config/files/templates/aws.sample.config.yaml.tmpl @@ -0,0 +1,111 @@ +# Git credentials used to authenticate against the Git platform you're using (Github / Gitlab etc.). +# KubeAid Bootstrap Script will use these credentials to : +# +# (1) Clone the KubeAid and KubeAid config repositories. +# (2) Create and push commits to a branch in the KubeAid config repository. +# +# So, make sure the Git password (token) you're using has permissions associated to do the above. +# +# Currently, we only support HTTPS authentication. +git: + username: xxxxxxxxxx + password: xxxxxxxxxx + +forks: + # KubeAid repository URL (in HTTPs syntax). + # Defaults to Obmondo's KubeAid repository. + # kubeaid: https://github.com/Obmondo/KubeAid + + # Your KubeAid config repository URL (in HTTPs syntax). + kubeaidConfig: https://github.com/xxxxxxxxxx/kubeaid-config + +# Kubernetes cluster and control-plane specific configurations. +cluster: + + # Kubernetes cluster name. + name: kubeaid-demo-aws + + # Kubernetes version to use. + # NOTE : Make sure that the AMI you're using, is targetted towards this Kubernetes version. + k8sVersion: v1.31.0 + + # Kubernetes API server specific configurations. + # REFER : https://github.com/kubernetes-sigs/cluster-api/blob/main/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml. + # + # NOTE : Generally, refer to the KubeadmControlPlane CRD instead of the corresponding GoLang + # source types linked below. + # There are some configuration options which appear in the corresponding GoLang source type, + # but not in the CRD. If you set those fields, then they get removed by the Kubeadm + # control-plane provider. This causes the capi-cluster ArgoCD App to always be in an + # OutOfSync state, resulting to the KubeAid Bootstrap Script not making any progress! + # apiServer: + # + # extraArgs: {} + # + # # REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".HostPathMount + # # + # # NOTE : If you want a mount to be read-only, then set extraVolume.readOnly to true. + # # Otherwise, omit setting that field. It gets removed by the Kubeadm control-plane + # # provider component, which results to the capi-cluster ArgoCD App always being in + # # OutOfSync state. + # extraVolumes: [] + # + # # REFER : "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1".File + # files: [] + + # Uncomment, if you just want audit-logging to be setup for you! KubeAid Bootstrap Script will set + # necessary configuration options in cluster.apiServer. + # enableAuditLogging: True + +cloud: + # AWS specific configurations. + aws: + region: us-east-1 + + # AWS SSH Keypair name, which ClusterAPI components (Kubeadm bootstrap and control-plane + # providers specifically) will use to SSH into the main cluster's master nodes. + sshKeyName: kubeaid-demo + + # bastionEnabled: True + + controlPlane: + ami: + id: ami-xxxxxxxxxxxxxxxxx + instanceType: t4g.medium + replicas: 1 + + nodeGroups: + - name: primary + ami: + id: ami-xxxxxxxxxxxxxxxxx + instanceType: t4g.medium + + minSize: 0 + replicas: 1 + maxSize: 3 + + rootVolumeSize: 35 + + # AWS SSH Keypair name, which ClusterAPI components (Kubeadm bootstrap provider + # specifically) will use to SSH into each node belonging to this node-group. + sshKeyName: kubeaid-demo + + # A label should meet one of the following criterias to propagate to each of the nodes : + # + # (1) Has node-role.kubernetes.io as prefix. + # (2) Belongs to node-restriction.kubernetes.io domain. + # (3) Belongs to node.cluster.x-k8s.io domain. + # + # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine + labels: + node.cluster.x-k8s.io/nodegroup: primary + node-role.kubernetes.io/bootstrapper: "" + + # taints: [] + + disasterRecovery: + veleroBackupsS3BucketName: kubeaid-demo-kubernetes-objects + sealedSecretsBackupS3BucketName: kubeaid-demo-sealed-secrets + +# monitoring: +# kubePrometheusVersion: v0.14.0 diff --git a/config/templates/hetzner.sample.config.yaml.tmpl b/config/files/templates/hetzner.sample.config.yaml.tmpl similarity index 100% rename from config/templates/hetzner.sample.config.yaml.tmpl rename to config/files/templates/hetzner.sample.config.yaml.tmpl diff --git a/config/generate_sample_config.go b/config/generate_sample_config.go index e3f9528..41f99f6 100644 --- a/config/generate_sample_config.go +++ b/config/generate_sample_config.go @@ -11,7 +11,7 @@ import ( "github.com/Obmondo/kubeaid-bootstrap-script/utils/templates" ) -//go:embed templates/* +//go:embed files/templates/* var SampleConfigs embed.FS func GenerateSampleConfig(ctx context.Context, cloudProvider string) { diff --git a/config/utils.go b/config/parse.go similarity index 79% rename from config/utils.go rename to config/parse.go index 98f3142..3f99dcf 100644 --- a/config/utils.go +++ b/config/parse.go @@ -30,14 +30,19 @@ func ParseConfig(ctx context.Context, configAsString string) { slog.InfoContext(ctx, "Parsed config") // Set defaults. - err = defaults.Set(ParsedConfig) - assert.AssertErrNil(ctx, err, "Failed setting defaults for parsed config") - // - // Read cloud credentials from CLI flags and store them in config. - readCloudCredentialsFromFlagsToConfig() - // - // Read SSH key-pairs from provided file paths and store them in config. - readSSHKeys(ParsedConfig) + { + err = defaults.Set(ParsedConfig) + assert.AssertErrNil(ctx, err, "Failed setting defaults for parsed config") + + // Read cloud credentials from CLI flags and store them in config. + readCloudCredentialsFromFlagsToConfig() + + // Read SSH key-pairs from provided file paths and store them in config. + hydrateSSHKeyConfigs() + + // Hydrate with Audit Logging options (if required). + hydrateWithAuditLoggingOptions() + } // Validate. validateConfig(ParsedConfig) @@ -62,16 +67,16 @@ func readCloudCredentialsFromFlagsToConfig() { } } -func readSSHKeys(config *Config) { +func hydrateSSHKeyConfigs() { switch { - case config.Cloud.Hetzner != nil: - readSSHKey(&config.Cloud.Hetzner.RobotSSHKeyPair) + case ParsedConfig.Cloud.Hetzner != nil: + hydrateSSHKeyConfig(&ParsedConfig.Cloud.Hetzner.RobotSSHKeyPair) } } // Reads and validates an SSH key-pair from the provided file paths. // The key-pair is then stored in the SSH key config struct itself. -func readSSHKey(sshKeyConfig *SSHKeyPairConfig) { +func hydrateSSHKeyConfig(sshKeyConfig *SSHKeyPairConfig) { ctx := context.Background() // Read and validate the SSH public key. diff --git a/config/templates/aws.sample.config.yaml.tmpl b/config/templates/aws.sample.config.yaml.tmpl deleted file mode 100644 index 60d86a7..0000000 --- a/config/templates/aws.sample.config.yaml.tmpl +++ /dev/null @@ -1,50 +0,0 @@ -forks: - kubeaidConfig: https://github.com/xxxxxxxxxx/kubeaid-config - -git: - username: xxxxxxxxxx - password: xxxxxxxxxx - -cluster: - name: kubeaid-demo-aws - k8sVersion: v1.31.0 - -cloud: - aws: - region: us-east-1 - - sshKeyName: kubeaid-demo - - bastionEnabled: False - - controlPlane: - instanceType: t4g.medium - ami: - id: ami-xxxxxxxxxxxxxxxxx - replicas: 1 - - nodeGroups: - - name: primary - ami: - id: ami-xxxxxxxxxxxxxxxxx - instanceType: t4g.medium - replicas: 1 - rootVolumeSize: 35 - sshKeyName: kubeaid-demo - - # Label should meet one of the following criterias to propagate to Node : - # - # (1) Has node-role.kubernetes.io as prefix. - # (2) Belongs to node-restriction.kubernetes.io domain. - # (3) Belongs to node.cluster.x-k8s.io domain. - # - # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine - labels: - node.cluster.x-k8s.io/nodegroup: primary - node-role.kubernetes.io/bootstrapper: "" - - taints: [] - - disasterRecovery: - veleroBackupsS3BucketName: kubeaid-demo-kubernetes-objects - sealedSecretsBackupS3BucketName: kubeaid-demo-sealed-secrets diff --git a/config/validate.go b/config/validate.go index 4367fd8..480ac5c 100644 --- a/config/validate.go +++ b/config/validate.go @@ -37,6 +37,16 @@ func validateConfig(config *Config) { switch { case config.Cloud.AWS != nil: for _, nodeGroup := range config.Cloud.AWS.NodeGroups { + // Validate auto-scaling options. + assert.Assert(ctx, + nodeGroup.Replicas >= nodeGroup.MinSize, + "replica count should be >= its min-size", slog.String("node-group", nodeGroup.Name), + ) + assert.Assert(ctx, + nodeGroup.Replicas <= nodeGroup.Maxsize, + "replica count should be <= its max-size", slog.String("node-group", nodeGroup.Name), + ) + // Validate labels and taints. validateLabelsAndTaints(ctx, nodeGroup.Name, nodeGroup.Labels, nodeGroup.Taints) } diff --git a/constants/constants.go b/constants/constants.go index 646a6f9..73d5285 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -44,6 +44,12 @@ const ( FlagNameAWSRegion = "aws-region" ) +// Kube API server CLI flags. +const ( + KubeAPIServerFlagAuditPolicyFile = "audit-policy-file" + KubeAPIServerFlagAuditLogPath = "audit-log-path" +) + // Cloud providers. const ( CloudProviderAWS = "aws" @@ -62,6 +68,7 @@ const ( const ( ArgoCDAppRoot = "root" ArgoCDAppCapiCluster = "capi-cluster" + ArgoCDAppHetznerRobot = "hetzner-robot" ArgoCDAppClusterAutoscaler = "cluster-autoscaler" ArgoCDAppVelero = "velero" ) @@ -75,8 +82,8 @@ const ( // Template names. var ( - TemplateNameAWSSampleConfig = "templates/aws.sample.config.yaml.tmpl" - TemplateNameHetznerSampleConfig = "templates/hetzner.sample.config.yaml.tmpl" + TemplateNameAWSSampleConfig = "files/templates/aws.sample.config.yaml.tmpl" + TemplateNameHetznerSampleConfig = "files/templates/hetzner.sample.config.yaml.tmpl" CommonNonSecretTemplateNames = []string{ // For ArgoCD. diff --git a/go.mod b/go.mod index 3104d58..3609407 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.69.0 github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 github.com/creasty/defaults v1.8.0 + github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75 github.com/go-git/go-git/v5 v5.12.0 github.com/go-playground/validator/v10 v10.23.0 github.com/go-sprout/sprout v0.6.0 @@ -21,15 +22,15 @@ require ( github.com/siderolabs/talos/pkg/machinery v1.8.3 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.29.0 + golang.org/x/crypto v0.31.0 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.16.3 - k8s.io/api v0.31.2 - k8s.io/apimachinery v0.31.2 - k8s.io/client-go v0.31.2 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/client-go v0.31.3 k8s.io/kubernetes v1.31.2 - sigs.k8s.io/cluster-api v1.8.5 - sigs.k8s.io/controller-runtime v0.19.1 + sigs.k8s.io/cluster-api v1.9.3 + sigs.k8s.io/controller-runtime v0.19.3 ) require ( @@ -99,7 +100,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect - github.com/fatih/color v1.17.0 // indirect + github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fvbommel/sortorder v1.1.0 // indirect @@ -194,7 +195,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.3 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -242,28 +243,28 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.9.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/term v0.26.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.3 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apiextensions-apiserver v0.31.2 // indirect - k8s.io/apiserver v0.31.2 // indirect + k8s.io/apiextensions-apiserver v0.31.3 // indirect + k8s.io/apiserver v0.31.3 // indirect k8s.io/cli-runtime v0.31.2 // indirect - k8s.io/component-base v0.31.2 // indirect + k8s.io/component-base v0.31.3 // indirect k8s.io/component-helpers v0.31.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-aggregator v0.31.2 // indirect diff --git a/go.sum b/go.sum index 7b18373..b642ed2 100644 --- a/go.sum +++ b/go.sum @@ -249,11 +249,13 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75 h1:XOnFL5swXi62I1MjZqZ2oLIFNthzYrtp9MWXA24leZA= +github.com/floshodan/hrobot-go v0.0.0-20241226081906-b7fee2ebca75/go.mod h1:TjlKZ0GYQbBtq6G7eUcVcRvTqgeCxI3cO0LOfBZv1/A= github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -412,8 +414,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -633,8 +635,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= -github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -648,8 +650,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os= -github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo= +github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y= +github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -681,8 +683,8 @@ github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= -github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -883,11 +885,11 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -900,8 +902,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -920,6 +922,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -932,14 +935,14 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -950,8 +953,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -996,8 +999,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1009,8 +1012,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1024,8 +1027,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= @@ -1048,8 +1051,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1092,8 +1095,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= @@ -1109,6 +1112,7 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/h2non/gentleman.v2 v2.0.5/go.mod h1:A1c7zwrTgAyyf6AbpvVksYtBayTB4STBUGmdkEtlHeA= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -1141,22 +1145,22 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.17.8/go.mod h1:N++Llhs8kCixMUoCaXXAyMMPbo8dDVnh+IQ36xZV2/0= -k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= -k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= -k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0= -k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE= +k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4= k8s.io/apimachinery v0.17.8/go.mod h1:Lg8zZ5iC/O8UjCqW6DNhcQG2m4TdjF9kwG3891OWbbA= -k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= -k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.2 h1:VUzOEUGRCDi6kX1OyQ801m4A7AUPglpsmGvdsekmcI4= -k8s.io/apiserver v0.31.2/go.mod h1:o3nKZR7lPlJqkU5I3Ove+Zx3JuoFjQobGX1Gctw6XuE= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.3 h1:+1oHTtCB+OheqFEz375D0IlzHZ5VeQKX1KGXnx+TTuY= +k8s.io/apiserver v0.31.3/go.mod h1:PrxVbebxrxQPFhJk4powDISIROkNMKHibTg9lTRQ0Qg= k8s.io/cli-runtime v0.31.2 h1:7FQt4C4Xnqx8V1GJqymInK0FFsoC+fAZtbLqgXYVOLQ= k8s.io/cli-runtime v0.31.2/go.mod h1:XROyicf+G7rQ6FQJMbeDV9jqxzkWXTYD6Uxd15noe0Q= k8s.io/client-go v0.17.8/go.mod h1:SJsDS64AAtt9VZyeaQMb4Ck5etCitZ/FwajWdzua5eY= -k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= -k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= -k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA= -k8s.io/component-base v0.31.2/go.mod h1:9PeyyFN/drHjtJZMCTkSpQJS3U9OXORnHQqMLDz0sUQ= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= +k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ= +k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU= k8s.io/component-helpers v0.31.2 h1:V2yjoNeyg8WfvwrJwzfYz+RUwjlbcAIaDaHEStBbaZM= k8s.io/component-helpers v0.31.2/go.mod h1:cNz+1ck38R0qWrjcw/rhQgGP6+Gwgw8ngr2ziDNmSXM= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -1182,10 +1186,10 @@ oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= -sigs.k8s.io/cluster-api v1.8.5 h1:lNA2fPN4fkXEs+oOQlnwxT/4VwRFBpv5kkSoJG8nqBA= -sigs.k8s.io/cluster-api v1.8.5/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= -sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= -sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/cluster-api v1.9.3 h1:lKWbrXzyNmJh++IcX54ZbAmnO7tZ2wKgds7WvskpiXY= +sigs.k8s.io/cluster-api v1.9.3/go.mod h1:5iojv38PSvOd4cxqu08Un5TQmy2yBkd3+0U7R/e+msk= +sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= +sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g= diff --git a/pkg/cloud/aws/policies.go b/pkg/cloud/aws/policies.go index d7f166c..edd0aaf 100644 --- a/pkg/cloud/aws/policies.go +++ b/pkg/cloud/aws/policies.go @@ -20,7 +20,7 @@ func getSealedSecretsBackuperIAMPolicy() services.PolicyDocument { "s3:ListMultipartUploadParts", }, Effect: "Allow", - Resource: fmt.Sprintf("arn:aws:s3:::-%s", sealedSecretBackupsS3BucketName), + Resource: fmt.Sprintf("arn:aws:s3:::%s/*", sealedSecretBackupsS3BucketName), }, }, } diff --git a/pkg/cloud/aws/setup_disaster_recovery.go b/pkg/cloud/aws/setup_disaster_recovery.go index 6ce8e74..af70646 100644 --- a/pkg/cloud/aws/setup_disaster_recovery.go +++ b/pkg/cloud/aws/setup_disaster_recovery.go @@ -7,6 +7,7 @@ import ( "github.com/Obmondo/kubeaid-bootstrap-script/config" "github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/aws/services" "github.com/Obmondo/kubeaid-bootstrap-script/utils" + "github.com/Obmondo/kubeaid-bootstrap-script/utils/assert" argoCDV1Alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/sagikazarmark/slog-shim" ) @@ -14,9 +15,7 @@ import ( // Sets up the provisioned cluster for Disaster Recovery. // NOTE : Picks up AWS credentials from the environment. func (a *AWS) SetupDisasterRecovery(ctx context.Context) { - if config.ParsedConfig.Cloud.AWS.DisasterRecovery == nil { - return - } + assert.AssertNotNil(ctx, config.ParsedConfig.Cloud.AWS.DisasterRecovery, "No AWS disaster-recovery config provided") slog.InfoContext(ctx, "Setting up Disaster Recovery") diff --git a/pkg/cloud/aws/utils.go b/pkg/cloud/aws/utils.go index 47c21dd..118f493 100644 --- a/pkg/cloud/aws/utils.go +++ b/pkg/cloud/aws/utils.go @@ -3,7 +3,6 @@ package aws import ( "context" "log/slog" - "strings" "github.com/Obmondo/kubeaid-bootstrap-script/utils" "github.com/Obmondo/kubeaid-bootstrap-script/utils/assert" @@ -33,8 +32,10 @@ func CreateIAMCloudFormationStack() { // NOTE : This requires admin privileges. output, err := utils.ExecuteCommand("clusterawsadm bootstrap iam create-cloudformation-stack") - // Panic if an error occurs (except regarding the AWS Cloudformation stack already existing). - if !strings.Contains(output, "already exists, updating") { - assert.AssertErrNil(context.Background(), err, "Failed bootstrapping IAM CloudFormation Stack", slog.String("output", output)) - } + // NOTE : If the CloudFormation template is in ROLLBACK_COMPLETE state, maybe there are + // pre-existing IAM resources with overlapping name. If so, then first delete them manually from + // the AWS Console and then retry running the script. + assert.AssertErrNil(context.Background(), err, + "Failed bootstrapping IAM CloudFormation Stack", slog.String("output", output), + ) } diff --git a/pkg/cloud/hetzner/execute_failover_script.go b/pkg/cloud/hetzner/execute_failover_script.go new file mode 100644 index 0000000..edaf67a --- /dev/null +++ b/pkg/cloud/hetzner/execute_failover_script.go @@ -0,0 +1,20 @@ +package hetzner + +import "context" + +/* +Let's say there are x (> 1) master nodes behind the Failover IP. + +CAPH (Cluster API Provider Hetzner) will SSH into any one of those master nodes (let's denote +it by β) and execute `kubeadm init`. The Machine resource corresponding to β will then be +marked as ready. We'll wait for this event. + +Once this event occurs, we'll SSH into β and run the Hetzner Failover Script (using the +hetzner-robot App in KubeAid). This will make the Failover IP point to β. + +The Kubernetes API server of the provisioned cluster will then be reachable via that Failover +IP. +*/ +func ExecuteFailoverScript(ctx context.Context) { + panic("unimplemented") +} diff --git a/pkg/core/bootstrap_cluster.go b/pkg/core/bootstrap_cluster.go index 7b86adf..ccaf14c 100644 --- a/pkg/core/bootstrap_cluster.go +++ b/pkg/core/bootstrap_cluster.go @@ -10,6 +10,7 @@ import ( "github.com/Obmondo/kubeaid-bootstrap-script/constants" "github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud" "github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/aws" + "github.com/Obmondo/kubeaid-bootstrap-script/pkg/cloud/hetzner" "github.com/Obmondo/kubeaid-bootstrap-script/utils" argoCDV1Alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/go-git/go-git/v5/plumbing/transport" @@ -28,21 +29,8 @@ func BootstrapCluster(ctx context.Context, skipKubeAidConfigSetup, skipClusterct os.Setenv(constants.EnvNameKubeconfig, constants.OutputPathManagementClusterKubeconfig) - // Provision the main cluster - provisionMainCluster(ctx, gitAuthMethod, skipKubeAidConfigSetup) - - // Let the provisioned cluster manage itself. - dogfoodProvisionedCluster(ctx, gitAuthMethod, skipClusterctlMove, cloudProvider, isPartOfDisasterRecovery) - - // If the diasterRecovery section is specified in the cloud-provider specific config, then - // setup Disaster Recovery. - cloudProvider.SetupDisasterRecovery(ctx) -} - -func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMethod, skipKubeAidConfigSetup bool) { // Create the management cluster (using K3d), if it doesn't already exist. utils.CreateK3DCluster(ctx, "management-cluster") - managementClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathManagementClusterKubeconfig) // Install Sealed Secrets. utils.InstallSealedSecrets(ctx) @@ -52,6 +40,27 @@ func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMetho SetupKubeAidConfig(ctx, gitAuthMethod, false) } + // While retrying, if `clusterctl move` has already been executed, then we skip these steps and + // move to the disaster recovery setup step. + provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig) + if !utils.IsClusterctlMoveExecuted(ctx, provisionedClusterClient) { + // Provision the main cluster + provisionMainCluster(ctx, gitAuthMethod, skipKubeAidConfigSetup) + + // Let the provisioned cluster manage itself. + dogfoodProvisionedCluster(ctx, gitAuthMethod, skipClusterctlMove, cloudProvider, isPartOfDisasterRecovery) + } + + // If the diasterRecovery section is specified in the cloud-provider specific config, then + // setup Disaster Recovery. + if config.ParsedConfig.Cloud.AWS.DisasterRecovery != nil { + cloudProvider.SetupDisasterRecovery(ctx) + } +} + +func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMethod, skipKubeAidConfigSetup bool) { + managementClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathManagementClusterKubeconfig) + // Setup the management cluster. SetupCluster(ctx, managementClusterClient) @@ -62,6 +71,12 @@ func provisionMainCluster(ctx context.Context, gitAuthMethod transport.AuthMetho // Close ArgoCD application client. constants.ArgoCDApplicationClientCloser.Close() + // CASE : Hetzner + // Make the Failover IP point to the master node where `kubeadm init` has been executed. + if config.ParsedConfig.Cloud.Hetzner != nil { + hetzner.ExecuteFailoverScript(ctx) + } + // Wait for the main cluster to be provisioned and ready. utils.WaitForMainClusterToBeProvisioned(ctx, managementClusterClient) @@ -82,7 +97,7 @@ func dogfoodProvisionedCluster(ctx context.Context, gitAuthMethod transport.Auth if isPartOfDisasterRecovery { // If this is a part of the disaster recovery process, then - // restore Kubernetes Secrets containing a Sealed Secrets keys. + // restore Kubernetes Secrets containing a Sealed Secrets key. sealedSecretsBackupBucketName := cloudProvider.GetSealedSecretsBackupBucketName() manifestsDirPath := utils.GetDirPathForDownloadedStorageBucketContents(sealedSecretsBackupBucketName) @@ -104,6 +119,20 @@ func dogfoodProvisionedCluster(ctx context.Context, gitAuthMethod transport.Auth SetupCluster(ctx, provisionedClusterClient) if !skipClusterctlMove { + // Make ClusterAPI use IAM roles instead of (temporary) credentials. + // + // NOTE : The ClusterAPI AWS InfrastructureProvider component (CAPA controller) needs to run in + // a master node. + // And, the master node count should be more than 1. + { + // Zero the credentials CAPA controller started with. + // This will force the CAPA controller to fall back to use the attached instance profiles. + utils.ExecuteCommandOrDie("clusterawsadm controller zero-credentials --namespace capi-cluster") + + // Rollout and restart on capa-controller-manager deployment. + utils.ExecuteCommandOrDie("clusterawsadm controller rollout-controller --namespace capi-cluster") + } + // Move ClusterAPI manifests to the provisioned cluster. utils.ExecuteCommandOrDie(fmt.Sprintf( "clusterctl move --kubeconfig %s --namespace %s --to-kubeconfig %s", diff --git a/pkg/core/delete_cluster.go b/pkg/core/delete_cluster.go index 4113a26..45244cd 100644 --- a/pkg/core/delete_cluster.go +++ b/pkg/core/delete_cluster.go @@ -27,28 +27,23 @@ func DeleteCluster(ctx context.Context) { }, } - { - provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig) + provisionedClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathProvisionedClusterKubeconfig) - // Try to find the Cluster resource in the provisioned cluster. - err := utils.GetClusterResource(ctx, provisionedClusterClient, cluster) + // The Cluster resource exists in the provisioned cluster. + // The means, the 'clusterctl move' command has been executed. + if utils.IsClusterctlMoveExecuted(ctx, provisionedClusterClient) { + slog.InfoContext(ctx, "Detected that the 'clusterctl move' command has been executed") - // The Cluster resource exists in the provisioned cluster. - // The means, the 'clusterctl move' command has been executed. - if err == nil { - slog.InfoContext(ctx, "Detected that the 'clusterctl move' command has been executed") - - // Move back the ClusterAPI manifests back from the provisioned cluster to the management - // cluster. - // NOTE : We need to retry, since we can get 'failed to call webhook' error sometimes. - retry.Do(func() error { - _, err := utils.ExecuteCommand(fmt.Sprintf( - "clusterctl move --kubeconfig %s --to-kubeconfig %s -n %s", - constants.OutputPathProvisionedClusterKubeconfig, constants.OutputPathManagementClusterKubeconfig, utils.GetCapiClusterNamespace(), - )) - return err - }) - } + // Move back the ClusterAPI manifests back from the provisioned cluster to the management + // cluster. + // NOTE : We need to retry, since we can get 'failed to call webhook' error sometimes. + retry.Do(func() error { + _, err := utils.ExecuteCommand(fmt.Sprintf( + "clusterctl move --kubeconfig %s --to-kubeconfig %s -n %s", + constants.OutputPathProvisionedClusterKubeconfig, constants.OutputPathManagementClusterKubeconfig, utils.GetCapiClusterNamespace(), + )) + return err + }) } managementClusterClient := utils.CreateKubernetesClient(ctx, constants.OutputPathManagementClusterKubeconfig) diff --git a/pkg/core/recover_cluster.go b/pkg/core/recover_cluster.go index e08bfdd..11c9a22 100644 --- a/pkg/core/recover_cluster.go +++ b/pkg/core/recover_cluster.go @@ -30,7 +30,7 @@ func RecoverCluster(ctx context.Context, cloudProvider cloud.CloudProvider) { Pull and gzip decode backed up (by Sealed Secrets backuper CRONJob) Kubernetes Secrets from S3 bucket. Each Kubernetes Secret contains a Sealed Secrets encryption key. - The script uresponsible for this backup process can be found here : + The script responsible for this backup process can be found here : https://github.com/Obmondo/kubeaid/blob/master/argocd-helm-charts/sealed-secrets/templates/configmap.yaml And you can read about Sealed Secrets key rotation from these references : diff --git a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl index ec5b8ef..4e55a27 100644 --- a/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl +++ b/pkg/core/templates/argocd-apps/capi-cluster.values.yaml.tmpl @@ -32,6 +32,9 @@ aws: instanceType: {{ .AWSConfig.ControlPlane.InstanceType }} ami: id: {{ .AWSConfig.ControlPlane.AMI.ID }} + {{- if .ClusterConfig.APIServer }} + apiServer: {{ .ClusterConfig.APIServer | toYaml | nindent 6 }} + {{- end }} nodeGroups: {{ .AWSConfig.NodeGroups | toYaml | indent 2 }} {{- end }} diff --git a/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl b/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl index 77899f7..7d50a57 100644 --- a/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl +++ b/pkg/core/templates/argocd-apps/cluster-autoscaler.values.yaml.tmpl @@ -17,3 +17,6 @@ cluster-autoscaler: autoDiscovery: clusterName: {{ .ClusterConfig.Name }} namespace: {{ .CAPIClusterNamespace }} + +enableClusterAPIScaleFromZeroSupport: + aws: true diff --git a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl index 99e0f29..243dcd9 100644 --- a/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl +++ b/pkg/core/templates/argocd-apps/sealed-secrets.values.yaml.tmpl @@ -1,6 +1,10 @@ ---- +sealed-secrets: + namespace: sealed-secrets + fullnameOverride: sealed-secrets-controller + {{- if and (.AWSConfig) (.AWSConfig.DisasterRecovery) }} backup: + namespace: sealed-secrets kube2iamRole: arn:aws:iam::{{ .AWSAccountID }}:role/sealed-secrets-backuper-{{ .ClusterConfig.Name }} backupBucket: {{ .AWSConfig.DisasterRecovery.SealedSecretsBackupS3BucketName }} {{- end }} diff --git a/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl b/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl index 8913aec..14a525a 100644 --- a/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl +++ b/pkg/core/templates/sealed-secrets/capi-cluster/hetzner-robot-ssh-keypair.yaml.tmpl @@ -9,3 +9,5 @@ stringData: ssh-publickey: | {{ .HetznerConfig.RobotSSHKeyPair.PublicKey | indent 4 }} + + sshkey-name: cluster diff --git a/scripts/install-prerequisites.sh b/scripts/install-prerequisites.sh index 541d961..06ebd5d 100644 --- a/scripts/install-prerequisites.sh +++ b/scripts/install-prerequisites.sh @@ -30,21 +30,21 @@ apt install -y curl curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash # Clusterawsadm -wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.5.2/clusterawsadm_v2.5.2_linux_"${CPU_ARCHITECTURE}" -mv clusterawsadm_v2.5.2_linux_"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm +wget https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/v2.7.1/clusterawsadm-linux-"${CPU_ARCHITECTURE}" +mv clusterawsadm-linux-"${CPU_ARCHITECTURE}" /usr/local/bin/clusterawsadm chmod +x /usr/local/bin/clusterawsadm # Clusterctl curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.7.3/clusterctl-linux-"${CPU_ARCHITECTURE}" -o clusterctl install -o root -g root -m 0755 clusterctl /usr/local/bin/clusterctl -# ------------------------------------------- Utilities ------------------------------------------- - # Kubectl curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${CPU_ARCHITECTURE}/kubectl" chmod +x ./kubectl mv ./kubectl /usr/local/bin +# ------------------------------------------- Utilities ------------------------------------------- + # K9s wget https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_linux_"${CPU_ARCHITECTURE}".deb dpkg -i k9s_linux_"${CPU_ARCHITECTURE}".deb diff --git a/utils/assert/assert.go b/utils/assert/assert.go index 5e87f42..eb65169 100644 --- a/utils/assert/assert.go +++ b/utils/assert/assert.go @@ -20,21 +20,31 @@ func AssertErrNil(ctx context.Context, err error, customErrorMessage string, att } // Panics if the given value isn't nil. -func AssertNil(ctx context.Context, value interface{}, errorMessage string) { +func AssertNil(ctx context.Context, value interface{}, errorMessage string, attributes ...any) { if value == nil { return } - slog.ErrorContext(ctx, errorMessage) + slog.ErrorContext(ctx, errorMessage, attributes...) os.Exit(1) } // Panics if the given value is nil. -func AssertNotNil(ctx context.Context, value interface{}, errorMessage string) { +func AssertNotNil(ctx context.Context, value interface{}, errorMessage string, attributes ...any) { if value != nil { return } - slog.ErrorContext(ctx, errorMessage) + slog.ErrorContext(ctx, errorMessage, attributes...) + os.Exit(1) +} + +// Panics if the given value is false. +func Assert(ctx context.Context, value bool, errorMessage string, attributes ...any) { + if value { + return + } + + slog.ErrorContext(ctx, errorMessage, attributes...) os.Exit(1) } diff --git a/utils/git.go b/utils/git.go index eb78193..0df801e 100644 --- a/utils/git.go +++ b/utils/git.go @@ -171,6 +171,9 @@ func getCreatePRURL(fromBranch string) string { return createPRURL } +// TODO : Sometimes we get this error while trying to detect whether the branch has been merged +// or not : `unexpected EOF`. +// In that case, just retry instead of erroring out. func WaitUntilPRMerged(ctx context.Context, repo *git.Repository, defaultBranchName string, commitHash plumbing.Hash, auth transport.AuthMethod, branchToBeMerged string) { for { slog.Info("Waiting for PR to be merged. Sleeping for 10 seconds....", slog.String("from-branch", branchToBeMerged), slog.String("to-branch", defaultBranchName)) diff --git a/utils/k3d.go b/utils/k3d.go index 77b7ca8..451c86d 100644 --- a/utils/k3d.go +++ b/utils/k3d.go @@ -38,11 +38,24 @@ func CreateK3DCluster(ctx context.Context, name string) { // resolvable from within the dev container. // Since we are mounting the Docker socket to the dev container, it can resolve DNS names of // Docker networks. So use the DNS name instead of 0.0.0.0. + // // NOTE : Consider this situation : - // an existing K3D cluster may have wrong Kubernetes API server URL server. + // an existing K3D cluster may have wrong Kubernetes API server URL server. ExecuteCommandOrDie(fmt.Sprintf(` kubectl config set-cluster k3d-%s --server=https://k3d-%s-serverlb:6443 `, name, name)) + + // Initially the master nodes have label node-role.kubernetes.io/control-plane set to "true". + // We'll change the label value to "" (just like it is in Vanilla Kubernetes). + // Some apps (like capi-cluster) relies on this label to get scheduled to the master node. + ExecuteCommandOrDie(fmt.Sprintf(` + master_nodes=$(kubectl get nodes -l node-role.kubernetes.io/control-plane=true -o name) + + for node in $master_nodes; do + kubectl label $node node-role.kubernetes.io/control-plane- + kubectl label $node node-role.kubernetes.io/control-plane="" + done + `)) } // Returns whether the given K3d cluster exists or not. diff --git a/utils/kubernetes.go b/utils/kubernetes.go index 8c5d2c5..b5bd973 100644 --- a/utils/kubernetes.go +++ b/utils/kubernetes.go @@ -202,3 +202,16 @@ func SaveKubeconfig(ctx context.Context, kubeClient client.Client) { slog.InfoContext(ctx, "kubeconfig has been saved locally") } + +// Returns whether the `clusterctl move` command has already been executed or not. +func IsClusterctlMoveExecuted(ctx context.Context, provisionedClusterClient client.Client) bool { + // If the Cluster resource is found in the provisioned cluster, that means `clusterctl move` has + // been executed. + err := GetClusterResource(ctx, provisionedClusterClient, &clusterAPIV1Beta1.Cluster{ + ObjectMeta: metaV1.ObjectMeta{ + Name: config.ParsedConfig.Cluster.Name, + Namespace: GetCapiClusterNamespace(), + }, + }) + return err == nil +} diff --git a/utils/others.go b/utils/others.go index 3bff74e..769b931 100644 --- a/utils/others.go +++ b/utils/others.go @@ -56,7 +56,7 @@ func InitTempDir() { // Panics if the environment variable isn't found. func GetEnv(name string) string { value, found := os.LookupEnv(name) - if !found { + if !found || len(value) == 0 { slog.Error("Env not found", slog.String("name", name)) os.Exit(1) }