From 9f7d622733f24347bdf7c5560c86cdc11746df7c Mon Sep 17 00:00:00 2001 From: ankush-sqops Date: Mon, 21 Oct 2024 11:02:03 +0530 Subject: [PATCH] Added Bottlerocket AMI support and bug fixes (#55) New releases and fixes:- - Added bottlerocket AMI support for both arm and amd based architectures. - Added an option to pass launch template name while creating custom managed AWS node group. - Added tags on AWS resources to track cost. --- README.md | 6 ++ examples/complete-ipv6/main.tf | 2 + examples/complete/README.md | 6 +- examples/complete/main.tf | 27 +++++-- modules/managed-nodegroup/README.md | 7 +- modules/managed-nodegroup/main.tf | 76 +++++++++++++------ .../templates/bootstrap-bottlerocket.toml.tpl | 26 +++++++ modules/managed-nodegroup/variables.tf | 26 +++++++ 8 files changed, 140 insertions(+), 36 deletions(-) create mode 100644 modules/managed-nodegroup/templates/bootstrap-bottlerocket.toml.tpl diff --git a/README.md b/README.md index 20877e5..5246536 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ module "managed_node_group_addons" { managed_ng_pod_capacity = 90 managed_ng_monitoring_enabled = true eks_nodes_keypair_name = "key-pair-name" + launch_template_name = local.launch_template_name + enable_bottlerocket_ami = local.enable_bottlerocket_ami + bottlerocket_node_config = { + bottlerocket_eks_node_admin_container_enabled = false + bottlerocket_eks_enable_control_container = true + } k8s_labels = { "Addons-Services" = "true" } diff --git a/examples/complete-ipv6/main.tf b/examples/complete-ipv6/main.tf index d4ae54f..a371187 100644 --- a/examples/complete-ipv6/main.tf +++ b/examples/complete-ipv6/main.tf @@ -6,6 +6,8 @@ locals { Owner = "Organization_name" Expires = "Never" Department = "Engineering" + Product = "" + Environment = local.environment } kms_user = null vpc_cidr = "10.10.0.0/16" diff --git a/examples/complete/README.md b/examples/complete/README.md index 70218cc..99315e7 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -25,9 +25,9 @@ This directory contains a complete example that demonstrates the usage of the Te | [kms](#module\_kms) | terraform-aws-modules/kms/aws | 3.1.0 | | [key\_pair\_vpn](#module\_key\_pair\_vpn) | squareops/keypair/aws | 1.0.2 | | [key\_pair\_eks](#module\_key\_pair\_eks) | squareops/keypair/aws | 1.0.2 | -| [vpc](#module\_vpc) | squareops/vpc/aws | 3.3.5 | -| [eks](#module\_eks) | squareops/eks/aws | 4.0.9 | -| [managed\_node\_group\_addons](#module\_managed\_node\_group\_addons) | squareops/eks/aws//modules/managed-nodegroup | 4.0.9 | +| [vpc](#module\_vpc) | squareops/vpc/aws | 3.4.1 | +| [eks](#module\_eks) | squareops/eks/aws | 5.1.1 | +| [managed\_node\_group\_addons](#module\_managed\_node\_group\_addons) | squareops/eks/aws//modules/managed-nodegroup | 5.1.1 | | [fargate\_profle](#module\_fargate\_profle) | squareops/eks/aws//modules/fargate-profile | n/a | ## Resources diff --git a/examples/complete/main.tf b/examples/complete/main.tf index af7cbfb..c7074d9 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -21,7 +21,7 @@ locals { cluster_version = "1.30" cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"] cluster_log_retention_in_days = 30 - managed_ng_capacity_type = "SPOT" # Can use "On_DEMAND" also + managed_ng_capacity_type = "SPOT" # Choose the capacity type ("SPOT" or "ON_DEMAND") cluster_endpoint_private_access = false cluster_endpoint_public_access = true cluster_endpoint_public_access_cidrs = ["0.0.0.0/0"] @@ -33,13 +33,17 @@ locals { vpc_private_subnets_counts = 2 vpc_database_subnets_counts = 2 vpc_intra_subnets_counts = 2 + launch_template_name = "launch-template-name" additional_aws_tags = { - Owner = "Organization_name" - Expires = "Never" - Department = "Engineering" + Owner = "Organization_name" + Expires = "Never" + Department = "Engineering" + Product = "" + Environment = local.environment } - aws_managed_node_group_arch = "" #Enter your linux arch (Example:- arm64 or amd64) + aws_managed_node_group_arch = "amd64" #Enter your linux arch (Example:- arm64 or amd64) current_identity = data.aws_caller_identity.current.arn + enable_bottlerocket_ami = false } data "aws_caller_identity" "current" {} @@ -109,7 +113,7 @@ module "key_pair_eks" { module "vpc" { source = "squareops/vpc/aws" - version = "3.3.5" + version = "3.4.1" name = local.name region = local.region vpc_cidr = local.vpc_cidr @@ -135,7 +139,7 @@ module "vpc" { module "eks" { source = "squareops/eks/aws" - version = "4.0.9" + version = "5.1.1" access_entry_enabled = true access_entries = { "example" = { @@ -177,11 +181,12 @@ module "eks" { cidr_blocks = ["10.10.0.0/16"] } } + tags = local.additional_aws_tags } module "managed_node_group_addons" { source = "squareops/eks/aws//modules/managed-nodegroup" - version = "4.0.9" + version = "5.1.1" depends_on = [module.vpc, module.eks] managed_ng_name = "Infra" managed_ng_min_size = 2 @@ -203,6 +208,12 @@ module "managed_node_group_addons" { eks_nodes_keypair_name = module.key_pair_eks.key_pair_name managed_ng_pod_capacity = 90 managed_ng_monitoring_enabled = true + launch_template_name = local.launch_template_name + enable_bottlerocket_ami = local.enable_bottlerocket_ami + bottlerocket_node_config = { + bottlerocket_eks_node_admin_container_enabled = false + bottlerocket_eks_enable_control_container = true + } k8s_labels = { "Addons-Services" = "true" } diff --git a/modules/managed-nodegroup/README.md b/modules/managed-nodegroup/README.md index c8c38b4..7a42723 100644 --- a/modules/managed-nodegroup/README.md +++ b/modules/managed-nodegroup/README.md @@ -33,10 +33,10 @@ No modules. |------|------| | [aws_eks_node_group.managed_ng](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_launch_template.eks_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | -| [aws_ami.launch_template_ami_amd64](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_ami.launch_template_ami_arm64](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_ami.launch_template_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_eks_cluster.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.launch_template_userdata_bottlerocket](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | ## Inputs @@ -71,6 +71,9 @@ No modules. | [managed\_ng\_volume\_delete\_on\_termination](#input\_managed\_ng\_volume\_delete\_on\_termination) | Set to true if delete the volumes when eks cluster is terminated. | `bool` | `true` | no | | [managed\_ng\_pod\_capacity](#input\_managed\_ng\_pod\_capacity) | Maximum number of pods you want to schedule on one node. This value should not exceed 110. | `number` | `70` | no | | [aws\_managed\_node\_group\_arch](#input\_aws\_managed\_node\_group\_arch) | Enter your linux architecture. | `string` | `"amd64"` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | The name of the launch template. | `string` | `""` | no | +| [enable\_bottlerocket\_ami](#input\_enable\_bottlerocket\_ami) | Set to true to enable the use of Bottlerocket AMIs for instances. | `bool` | `false` | no | +| [bottlerocket\_node\_config](#input\_bottlerocket\_node\_config) | Bottlerocket Node configurations for EKS. | `map(any)` | 
{
  "bottlerocket_eks_enable_control_container": true,
  "bottlerocket_eks_node_admin_container_enabled": false
}
| no | ## Outputs diff --git a/modules/managed-nodegroup/main.tf b/modules/managed-nodegroup/main.tf index a9957d7..c67e784 100644 --- a/modules/managed-nodegroup/main.tf +++ b/modules/managed-nodegroup/main.tf @@ -1,26 +1,25 @@ -data "aws_eks_cluster" "eks" { - name = var.eks_cluster_name +locals { + launch_template_name = format("%s-%s-%s", var.eks_cluster_name, var.managed_ng_name, "lt") + ami_owner = var.enable_bottlerocket_ami ? "amazon" : "602401143452" + ami_base_name = var.enable_bottlerocket_ami ? "bottlerocket-aws-k8s" : (var.aws_managed_node_group_arch == "arm64" ? "amazon-eks-arm64-node" : "amazon-eks-node") + ami_arch = var.enable_bottlerocket_ami ? (var.aws_managed_node_group_arch == "arm64" ? "aarch64*" : "x86_64*") : "v*" } -data "aws_ami" "launch_template_ami_amd64" { - owners = ["602401143452"] - most_recent = true - filter { - name = "name" - values = [format("%s-%s-%s", "amazon-eks-node", data.aws_eks_cluster.eks.version, "v*")] - } +data "aws_eks_cluster" "eks" { + name = var.eks_cluster_name } -data "aws_ami" "launch_template_ami_arm64" { - owners = ["602401143452"] +data "aws_ami" "launch_template_ami" { + owners = [local.ami_owner] most_recent = true filter { name = "name" - values = [format("%s-%s-%s", "amazon-eks-arm64-node", data.aws_eks_cluster.eks.version, "v*")] + values = [format("%s-%s-%s", local.ami_base_name, data.aws_eks_cluster.eks.version, local.ami_arch)] } } data "template_file" "launch_template_userdata" { + count = var.enable_bottlerocket_ami ? 0 : 1 template = file("${path.module}/templates/${data.aws_eks_cluster.eks.kubernetes_network_config[0].ip_family == "ipv4" ? "custom-bootstrap-script.sh.tpl" : "custom-bootstrap-scriptipv6.sh.tpl"}") vars = { @@ -34,11 +33,29 @@ data "template_file" "launch_template_userdata" { } } +data "template_file" "launch_template_userdata_bottlerocket" { + count = var.enable_bottlerocket_ami ? 1 : 0 + + template = file("${path.module}/templates/bootstrap-bottlerocket.toml.tpl") + + vars = { + cluster_name = var.eks_cluster_name + cluster_endpoint = data.aws_eks_cluster.eks.endpoint + cluster_ca_data = data.aws_eks_cluster.eks.certificate_authority[0].data + eventRecordQPS = var.eventRecordQPS + image_low_threshold_percent = var.image_low_threshold_percent + image_high_threshold_percent = var.image_high_threshold_percent + managed_ng_pod_capacity = var.managed_ng_pod_capacity + admin_container_enabled = var.bottlerocket_node_config.bottlerocket_eks_node_admin_container_enabled + enable_control_container = var.bottlerocket_node_config.bottlerocket_eks_enable_control_container + } +} + resource "aws_launch_template" "eks_template" { - name = format("%s-%s-%s", var.environment, var.managed_ng_name, "launch-template") + name = length(var.launch_template_name) > 0 ? var.launch_template_name : local.launch_template_name key_name = var.eks_nodes_keypair_name - image_id = var.aws_managed_node_group_arch == "arm64" ? data.aws_ami.launch_template_ami_arm64.image_id : data.aws_ami.launch_template_ami_amd64.image_id - user_data = base64encode(data.template_file.launch_template_userdata.rendered) + image_id = data.aws_ami.launch_template_ami.image_id + user_data = var.enable_bottlerocket_ami ? base64encode(data.template_file.launch_template_userdata_bottlerocket[0].rendered) : base64encode(data.template_file.launch_template_userdata[0].rendered) update_default_version = true block_device_mappings { device_name = "/dev/xvda" @@ -62,10 +79,21 @@ resource "aws_launch_template" "eks_template" { tag_specifications { resource_type = "instance" - tags = { - Name = format("%s-%s-%s", var.environment, var.managed_ng_name, "eks-node") - Environment = var.environment - } + tags = merge( + { + Name = format("%s-%s-%s", var.environment, var.managed_ng_name, "eks-node") + }, + var.tags + ) + } + tag_specifications { + resource_type = "volume" + tags = merge( + { + Name = format("%s-%s-%s", var.environment, var.managed_ng_name, "eks-volume") + }, + var.tags + ) } lifecycle { @@ -94,8 +122,10 @@ resource "aws_eks_node_group" "managed_ng" { update_config { max_unavailable_percentage = 50 } - tags = { - Name = format("%s-%s-%s", var.environment, var.managed_ng_name, "ng") - Environment = var.environment - } + tags = merge( + { + Name = format("%s-%s-%s", var.environment, var.managed_ng_name, "ng") + }, + var.tags + ) } diff --git a/modules/managed-nodegroup/templates/bootstrap-bottlerocket.toml.tpl b/modules/managed-nodegroup/templates/bootstrap-bottlerocket.toml.tpl new file mode 100644 index 0000000..838b3c4 --- /dev/null +++ b/modules/managed-nodegroup/templates/bootstrap-bottlerocket.toml.tpl @@ -0,0 +1,26 @@ +[settings.kubernetes] +cluster-name = "${cluster_name}" +api-server = "${cluster_endpoint}" +cluster-certificate = "${cluster_ca_data}" +max-pods = ${managed_ng_pod_capacity} + +image-gc-high-threshold-percent = ${image_high_threshold_percent} +image-gc-low-threshold-percent = ${image_low_threshold_percent} +event-qps = ${eventRecordQPS} + + +# Enable kernel lockdown in "integrity" mode. +# This prevents modifications to the running kernel, even by privileged users. +[settings.kernel] +lockdown = "integrity" + + +[settings.host-containers.admin] +enabled = ${admin_container_enabled} + + +# The control host container provides out-of-band access via SSM. +# It is enabled by default, and can be disabled if you do not expect to use SSM. +# This could leave you with no way to access the API and change settings on an existing node! +[settings.host-containers.control] +enabled = ${enable_control_container} diff --git a/modules/managed-nodegroup/variables.tf b/modules/managed-nodegroup/variables.tf index 88c56fe..528c777 100644 --- a/modules/managed-nodegroup/variables.tf +++ b/modules/managed-nodegroup/variables.tf @@ -172,3 +172,29 @@ variable "aws_managed_node_group_arch" { type = string default = "amd64" } + +variable "launch_template_name" { + description = "The name of the launch template." + type = string + default = "" + + validation { + condition = length(var.launch_template_name) <= 60 + error_message = "The launch_template_name must be 60 characters or fewer. Please provide a shorter name." + } +} + +variable "enable_bottlerocket_ami" { + description = "Set to true to enable the use of Bottlerocket AMIs for instances." + default = false + type = bool +} + +variable "bottlerocket_node_config" { + type = map(any) # Specify the type as a map for clarity + description = "Bottlerocket Node configurations for EKS." + default = { + bottlerocket_eks_node_admin_container_enabled = false ## For SSH Access + bottlerocket_eks_enable_control_container = true ## For SSM Accesws + } +}