Merge pull request #23 from Ajay-sops/main

increased pod capacity in eks worker node

README.md

@@ -81,6 +81,7 @@ module "managed_node_group_production" {
   worker_iam_role_name   = module.eks.worker_iam_role_name
   worker_iam_role_arn    = module.eks.worker_iam_role_arn
   default_addon_enabled  = true
+  managed_ng_pod_capacity= 90
   eks_nodes_keypair_name = "key-pair-name"
   k8s_labels = {
     "Addons-Services" = "true"
@@ -144,6 +145,7 @@ In this module, we have implemented the following CIS Compliance checks for EKS:
 
 | Name | Version |
 |------|---------|
+| <a name="provider_null"></a> [null](#provider\_null) | n/a |
 | <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.47 |
 | <a name="provider_template"></a> [template](#provider\_template) | n/a |
 
@@ -163,6 +165,7 @@ In this module, we have implemented the following CIS Compliance checks for EKS:
 | [aws_iam_policy.kubernetes_pvc_kms_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.node_autoscaler_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.node_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.S3Access_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.SSMManagedInstanceCore_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cni_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_kms_cluster_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
@@ -171,8 +174,11 @@ In this module, we have implemented the following CIS Compliance checks for EKS:
 | [aws_iam_role_policy_attachment.eks_worker_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.node_autoscaler_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_launch_template.eks_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource |
+| [null_resource.update_cni_prifix](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [aws_ami.launch_template_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_iam_policy.S3Access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source |
 | [aws_iam_policy.SSMManagedInstanceCore](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 | [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source |
 
 ## Inputs
@@ -216,6 +222,7 @@ In this module, we have implemented the following CIS Compliance checks for EKS:
 | <a name="input_k8s_labels"></a> [k8s\_labels](#input\_k8s\_labels) | Labels to be applied to the Kubernetes node groups. | `map(any)` | `{}` | no |
 | <a name="input_worker_iam_role_arn"></a> [worker\_iam\_role\_arn](#input\_worker\_iam\_role\_arn) | The ARN of the worker role for EKS. | `string` | `""` | no |
 | <a name="input_worker_iam_role_name"></a> [worker\_iam\_role\_name](#input\_worker\_iam\_role\_name) | The name of the EKS Worker IAM role. | `string` | `""` | no |
+| <a name="input_managed_ng_pod_capacity"></a> [managed\_ng\_pod\_capacity](#input\_managed\_ng\_pod\_capacity) | Maximum number of pods you want to schedule on one node. This value should not exceed 110. | `number` | `70` | no |
 
 ## Outputs
 

examples/complete-ipv6/main.tf

@@ -154,24 +154,25 @@ module "eks" {
 }
 
 module "managed_node_group_production" {
-  source                 = "squareops/eks/aws//modules/managed-nodegroup"
-  depends_on             = [module.vpc, module.eks]
-  name                   = "Infra"
-  min_size               = 1
-  max_size               = 3
-  desired_size           = 1
-  subnet_ids             = [module.vpc.private_subnets[0]]
-  environment            = local.environment
-  kms_key_arn            = module.kms.key_arn
-  capacity_type          = "ON_DEMAND"
-  ebs_volume_size        = 50
-  instance_types         = ["t3a.large", "t3.large", "m5.large"]
-  kms_policy_arn         = module.eks.kms_policy_arn
-  eks_cluster_name       = module.eks.cluster_name
-  default_addon_enabled  = local.default_addon_enabled
-  worker_iam_role_name   = module.eks.worker_iam_role_name
-  worker_iam_role_arn    = module.eks.worker_iam_role_arn
-  eks_nodes_keypair_name = module.key_pair_eks.key_pair_name
+  source                  = "squareops/eks/aws//modules/managed-nodegroup"
+  depends_on              = [module.vpc, module.eks]
+  name                    = "Infra"
+  min_size                = 1
+  max_size                = 3
+  desired_size            = 1
+  subnet_ids              = [module.vpc.private_subnets[0]]
+  environment             = local.environment
+  kms_key_arn             = module.kms.key_arn
+  capacity_type           = "ON_DEMAND"
+  ebs_volume_size         = 50
+  instance_types          = ["t3a.large", "t3.large", "m5.large"]
+  kms_policy_arn          = module.eks.kms_policy_arn
+  eks_cluster_name        = module.eks.cluster_name
+  default_addon_enabled   = local.default_addon_enabled
+  managed_ng_pod_capacity = 90
+  worker_iam_role_name    = module.eks.worker_iam_role_name
+  worker_iam_role_arn     = module.eks.worker_iam_role_arn
+  eks_nodes_keypair_name  = module.key_pair_eks.key_pair_name
   k8s_labels = {
     "Addon-Services" = "true"
   }

examples/complete/main.tf

@@ -118,6 +118,7 @@ module "eks" {
   cluster_endpoint_public_access       = true
   cluster_endpoint_public_access_cidrs = ["0.0.0.0/0"]
   create_aws_auth_configmap            = true
+  managed_ng_pod_capacity              = 90
   default_addon_enabled                = local.default_addon_enabled
   eks_nodes_keypair_name               = module.key_pair_eks.key_pair_name
   aws_auth_roles = [
@@ -147,24 +148,25 @@ module "eks" {
 }
 
 module "managed_node_group_production" {
-  source                 = "squareops/eks/aws//modules/managed-nodegroup"
-  depends_on             = [module.vpc, module.eks]
-  name                   = "Infra"
-  min_size               = 2
-  max_size               = 5
-  desired_size           = 2
-  subnet_ids             = [module.vpc.private_subnets[0]]
-  environment            = local.environment
-  kms_key_arn            = module.kms.key_arn
-  capacity_type          = "ON_DEMAND"
-  ebs_volume_size        = 50
-  instance_types         = ["t3a.large", "t2.large", "t2.xlarge", "t3.large", "m5.large"]
-  kms_policy_arn         = module.eks.kms_policy_arn
-  eks_cluster_name       = module.eks.cluster_name
-  default_addon_enabled  = local.default_addon_enabled
-  worker_iam_role_name   = module.eks.worker_iam_role_name
-  worker_iam_role_arn    = module.eks.worker_iam_role_arn
-  eks_nodes_keypair_name = module.key_pair_eks.key_pair_name
+  source                  = "squareops/eks/aws//modules/managed-nodegroup"
+  depends_on              = [module.vpc, module.eks]
+  name                    = "Infra"
+  min_size                = 2
+  max_size                = 5
+  desired_size            = 2
+  subnet_ids              = [module.vpc.private_subnets[0]]
+  environment             = local.environment
+  kms_key_arn             = module.kms.key_arn
+  capacity_type           = "ON_DEMAND"
+  ebs_volume_size         = 50
+  instance_types          = ["t3a.large", "t2.large", "t2.xlarge", "t3.large", "m5.large"]
+  kms_policy_arn          = module.eks.kms_policy_arn
+  eks_cluster_name        = module.eks.cluster_name
+  default_addon_enabled   = local.default_addon_enabled
+  worker_iam_role_name    = module.eks.worker_iam_role_name
+  worker_iam_role_arn     = module.eks.worker_iam_role_arn
+  managed_ng_pod_capacity = 90
+  eks_nodes_keypair_name  = module.key_pair_eks.key_pair_name
   k8s_labels = {
     "Addons-Services" = "true"
   }

main.tf

@@ -1,3 +1,5 @@
+data "aws_region" "current" {}
+
 module "eks_addon" {
   count                     = var.default_addon_enabled ? 1 : 0
   source                    = "terraform-aws-modules/eks/aws"
@@ -50,6 +52,19 @@ module "eks_addon" {
   }
 }
 
+resource "null_resource" "update_cni_prifix" {
+  count      = var.default_addon_enabled ? 1 : 0
+  depends_on = [module.eks_addon]
+  provisioner "local-exec" {
+    command = <<-EOF
+      aws eks update-kubeconfig --name ${module.eks_addons[0].cluster_name} --region ${data.aws_region.current.name} &&
+      kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true &&
+      kubectl set env daemonset aws-node -n kube-system WARM_PREFIX_TARGET=1 &&
+      kubectl set env daemonset aws-node -n kube-system WARM_ENI_TARGET=1
+    EOF
+  }
+}
+
 module "eks" {
   count                     = var.default_addon_enabled ? 0 : 1
   source                    = "terraform-aws-modules/eks/aws"
@@ -256,6 +271,7 @@ data "template_file" "launch_template_userdata" {
     cluster_auth_base64          = module.eks_addon[0].cluster_certificate_authority_data
     image_low_threshold_percent  = var.image_low_threshold_percent
     image_high_threshold_percent = var.image_high_threshold_percent
+    managed_ng_pod_capacity      = var.managed_ng_pod_capacity
 
   }
 }

modules/managed-nodegroup/README.md

@@ -22,6 +22,7 @@ No requirements.
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
 | <a name="provider_template"></a> [template](#provider\_template) | n/a |
+| <a name="provider_null"></a> [null](#provider\_null) | n/a |
 
 ## Modules
 
@@ -31,11 +32,13 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [aws_eks_addon.managed_ng_addons](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource |
 | [aws_eks_node_group.managed_ng](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource |
 | [aws_launch_template.eks_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource |
+| [null_resource.update_vpc_cni_env_var](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [aws_ami.launch_template_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_eks_cluster.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
-| [aws_iam_role.worker_iam_role_name](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 | [template_file.launch_template_userdata](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source |
 
 ## Inputs
@@ -68,6 +71,8 @@ No modules.
 | <a name="input_worker_iam_role_name"></a> [worker\_iam\_role\_name](#input\_worker\_iam\_role\_name) | The name of the EKS Worker IAM role. | `string` | `""` | no |
 | <a name="input_ipv6_enabled"></a> [ipv6\_enabled](#input\_ipv6\_enabled) | Whether IPv6 enabled or not | `bool` | `false` | no |
 | <a name="input_default_addon_enabled"></a> [default\_addon\_enabled](#input\_default\_addon\_enabled) | Enable default addon(VPC-CNI, AWS-EBS-CSI-DRIVER) with Cluster creation | `bool` | `false` | no |
+| <a name="input_managed_ng_pod_capacity"></a> [managed\_ng\_pod\_capacity](#input\_managed\_ng\_pod\_capacity) | Maximum number of pods you want to schedule on one node. This value should not exceed 110. | `number` | `70` | no |
+| <a name="input_addons"></a> [addons](#input\_addons) | A map variable representing various Kubernetes add-ons with their respective name and version. | <pre>map(object({<br>    name    = string<br>    version = string<br>  }))</pre> | <pre>{<br>  "coredns": {<br>    "name": "coredns",<br>    "version": "v1.10.1-eksbuild.4"<br>  },<br>  "ebs_csi": {<br>    "name": "aws-ebs-csi-driver",<br>    "version": "v1.28.0-eksbuild.1"<br>  },<br>  "kube_proxy": {<br>    "name": "kube-proxy",<br>    "version": "v1.27.6-eksbuild.2"<br>  },<br>  "vpc_cni": {<br>    "name": "vpc-cni",<br>    "version": "v1.16.4-eksbuild.2"<br>  }<br>}</pre> | no |
 
 ## Outputs
 

modules/managed-nodegroup/main.tf

@@ -2,6 +2,8 @@ data "aws_eks_cluster" "eks" {
   name = var.eks_cluster_name
 }
 
+data "aws_region" "current" {}
+
 data "aws_ami" "launch_template_ami" {
   owners      = ["602401143452"]
   most_recent = true
@@ -21,6 +23,7 @@ data "template_file" "launch_template_userdata" {
     cluster_auth_base64          = data.aws_eks_cluster.eks.certificate_authority[0].data
     image_low_threshold_percent  = var.image_low_threshold_percent
     image_high_threshold_percent = var.image_high_threshold_percent
+    managed_ng_pod_capacity      = var.managed_ng_pod_capacity
 
   }
 }
@@ -89,4 +92,26 @@ resource "aws_eks_node_group" "managed_ng" {
     Name        = format("%s-%s-%s", var.environment, var.name, "ng")
     Environment = var.environment
   }
-}
+}
+
+resource "aws_eks_addon" "managed_ng_addons" {
+  depends_on                  = [aws_eks_node_group.managed_ng]
+  for_each                    = var.addons
+  cluster_name                = var.eks_cluster_name
+  addon_name                  = each.value.name
+  addon_version               = each.value.version
+  resolve_conflicts_on_create = "OVERWRITE"
+}
+
+resource "null_resource" "update_vpc_cni_env_var" {
+  depends_on = [aws_eks_addon.managed_ng_addons["vpc_cni"]]
+
+  provisioner "local-exec" {
+    command = <<-EOF
+      aws eks update-kubeconfig --name ${var.eks_cluster_name} --region ${data.aws_region.current.name} &&
+      kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true &&
+      kubectl set env daemonset aws-node -n kube-system WARM_PREFIX_TARGET=1 &&
+      kubectl set env daemonset aws-node -n kube-system WARM_ENI_TARGET=1
+    EOF
+  }
+}

modules/managed-nodegroup/templates/custom-bootstrap-script.sh.tpl

@@ -25,7 +25,7 @@ fi
 yum update -y && yum install vim wget curl -y
 
 
-/etc/eks/bootstrap.sh '${cluster_name}'  --apiserver-endpoint '${endpoint}' --b64-cluster-ca '${cluster_auth_base64}'
+/etc/eks/bootstrap.sh '${cluster_name}'  --apiserver-endpoint '${endpoint}' --b64-cluster-ca '${cluster_auth_base64}' --use-max-pods false --kubelet-extra-args '--max-pods=${managed_ng_pod_capacity}'
 
 
 --==MYBOUNDARY==--

modules/managed-nodegroup/templates/custom-bootstrap-scriptipv6.sh.tpl

@@ -19,4 +19,4 @@ fi
 yum update -y
 yum install -y vim wget curl
 
-/etc/eks/bootstrap.sh '${cluster_name}' --apiserver-endpoint '${endpoint}' --b64-cluster-ca '${cluster_auth_base64}' --ip-family ipv6 --service-ipv6-cidr $(aws eks describe-cluster --name=${cluster_name} --output=text --query 'cluster.{serviceIpv6Cidr: kubernetesNetworkConfig.serviceIpv6Cidr}')
+/etc/eks/bootstrap.sh '${cluster_name}'  --apiserver-endpoint '${endpoint}' --b64-cluster-ca '${cluster_auth_base64}' --use-max-pods false --kubelet-extra-args '--max-pods=${managed_ng_pod_capacity}'

modules/managed-nodegroup/variables.tf

@@ -154,3 +154,35 @@ variable "default_addon_enabled" {
   default     = false
   type        = bool
 }
+
+variable "managed_ng_pod_capacity" {
+  description = "Maximum number of pods you want to schedule on one node. This value should not exceed 110."
+  default     = 70
+  type        = number
+}
+
+variable "addons" {
+  description = "A map variable representing various Kubernetes add-ons with their respective name and version."
+  type = map(object({
+    name    = string
+    version = string
+  }))
+  default = {
+    coredns = {
+      name    = "coredns"
+      version = "v1.10.1-eksbuild.4"
+    }
+    vpc_cni = {
+      name    = "vpc-cni"
+      version = "v1.16.4-eksbuild.2"
+    }
+    kube_proxy = {
+      name    = "kube-proxy"
+      version = "v1.27.6-eksbuild.2"
+    }
+    ebs_csi = {
+      name    = "aws-ebs-csi-driver"
+      version = "v1.28.0-eksbuild.1"
+    }
+  }
+}

variables.tf

@@ -220,3 +220,9 @@ variable "worker_iam_role_name" {
   type        = string
   default     = ""
 }
+
+variable "managed_ng_pod_capacity" {
+  description = "Maximum number of pods you want to schedule on one node. This value should not exceed 110."
+  default     = 70
+  type        = number
+}