Seperate karpenter config to a submodule (#346)

* Seperate karpenter config to a submodule It seemed a bit weird that we did a lot of work to remove all of the non-core functions from the module, just to add the support for karpenter as a core part of the module. Moving support for Karpenter to a submodule doesn't make configuring a cluster much more dificult, but does reduce the complexity of the core module, and hopefully should make ongoing maintince a little simpler. Especially if we decided to use some other node provisioning tool in the future. * fmt
cookpad · Oct 9, 2023 · 8f572e3 · 8f572e3
1 parent 6ab89fc
commit 8f572e3
Show file tree

Hide file tree

Showing 13 changed files with 130 additions and 27 deletions.
diff --git a/examples/cluster/main.tf b/examples/cluster/main.tf
@@ -34,14 +34,29 @@ module "cluster" {
       username = aws_iam_role.test_role.name
       rolearn  = aws_iam_role.test_role.arn
       groups   = ["system:masters"]
-    }
+    },
+    {
+      username = "system:node:{{EC2PrivateDNSName}}"
+      rolearn  = module.karpenter.node_role_arn
+      groups = [
+        "system:bootstrappers",
+        "system:nodes",
+      ]
+    },
   ]
 
   tags = {
     Project = "terraform-aws-eks"
   }
 }
 
+module "karpenter" {
+  source = "../../modules/karpenter"
+
+  cluster_config = module.cluster.config
+  oidc_config    = module.cluster.oidc_config
+}
+
 data "aws_security_group" "nodes" {
   id = module.cluster.config.node_security_group
 }
diff --git a/main.tf b/main.tf
@@ -61,14 +61,6 @@ locals {
             "system:node-proxier",
           ]
         },
-        {
-          rolearn  = aws_iam_role.karpenter_node.arn
-          username = "system:node:{{EC2PrivateDNSName}}"
-          groups = [
-            "system:bootstrappers",
-            "system:nodes",
-          ]
-        },
       ],
       var.aws_auth_role_map,
     ))

diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md
@@ -0,0 +1,49 @@
+# Karpenter
+
+This module configures the resources required to run the
+karpenter node-provisioning tool in an eks cluster.
+
+* Fargate Profile - to run karpenter
+* IAM roles for the fargate controller and nodes to be provisioned by karpenter
+* SQS queue to provide events (spot interruption etc) to karpenter
+
+It does not install karpenter itself to the cluster - and we recomend
+that you use helm as per the [karpenter documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#4-install-karpenter)
+
+It is provided as a submodule so the core module is less opinionated.
+
+However we test the core module and the karpenter module
+in our test suite to ensure that the different components we use in our
+clusters at cookpad intergrate correctly.
+
+
+## Example
+
+You should pass cluster and oidc config from the cluster to the karpenter module.
+
+You will also need to add the IAM role of nodes created by karpenter to the aws_auth_role_map
+so they can connect to the cluster.
+
+```hcl
+module "cluster" {
+  source  = "cookpad/eks/aws"
+
+  aws_auth_role_map = [
+    {
+      username = "system:node:{{EC2PrivateDNSName}}"
+      rolearn  = module.karpenter.node_role_arn
+      groups = [
+        "system:bootstrappers",
+        "system:nodes",
+      ]
+    },
+  ]
+}
+
+module "karpenter" {
+  source  = "cookpad/eks/aws//modules/karpenter"
+
+  cluster_config = module.cluster.config
+  oidc_config    = module.cluster.oidc_config
+}
+```
diff --git a/karpenter_controller_iam.tf → modules/karpenter/controller_iam.tf b/karpenter_controller_iam.tf → modules/karpenter/controller_iam.tf
@@ -1,7 +1,7 @@
 resource "aws_iam_role" "karpenter_controller" {
-  name               = "${var.iam_role_name_prefix}Karpenter-${var.name}"
+  name               = "${var.cluster_config.iam_role_name_prefix}Karpenter-${var.cluster_config.name}"
   assume_role_policy = data.aws_iam_policy_document.karpenter_controller_assume_role_policy.json
-  description        = "Karpenter controller role for ${var.name} cluster"
+  description        = "Karpenter controller role for ${var.cluster_config.name} cluster"
 }
 
 data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" {
@@ -11,18 +11,18 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" {
 
     condition {
       test     = "StringEquals"
-      variable = "${replace(aws_iam_openid_connect_provider.cluster_oidc.url, "https://", "")}:sub"
+      variable = "${replace(var.oidc_config.url, "https://", "")}:sub"
       values   = ["system:serviceaccount:karpenter:karpenter"]
     }
 
     condition {
       test     = "StringEquals"
-      variable = "${replace(aws_iam_openid_connect_provider.cluster_oidc.url, "https://", "")}:aud"
+      variable = "${replace(var.oidc_config.url, "https://", "")}:aud"
       values   = ["sts.amazonaws.com"]
     }
 
     principals {
-      identifiers = [aws_iam_openid_connect_provider.cluster_oidc.arn]
+      identifiers = [var.oidc_config.arn]
       type        = "Federated"
     }
   }
@@ -69,7 +69,7 @@ data "aws_iam_policy_document" "karpenter_controller" {
 
   statement {
     actions   = ["eks:DescribeCluster"]
-    resources = [aws_eks_cluster.control_plane.arn]
+    resources = [var.cluster_config.arn]
   }
 
   statement {

diff --git a/modules/karpenter/data.tf b/modules/karpenter/data.tf
@@ -0,0 +1,2 @@
+data "aws_partition" "current" {}
+
diff --git a/modules/karpenter/fargate.tf b/modules/karpenter/fargate.tf
@@ -0,0 +1,11 @@
+resource "aws_eks_fargate_profile" "critical_pods" {
+  cluster_name           = var.cluster_config.name
+  fargate_profile_name   = "${var.cluster_config.name}-karpenter"
+  pod_execution_role_arn = var.cluster_config.fargate_execution_role_arn
+  subnet_ids             = values(var.cluster_config.private_subnet_ids)
+
+  selector {
+    namespace = "karpenter"
+    labels    = {}
+  }
+}
diff --git a/karpenter_interruption_queue.tf → modules/karpenter/interruption_queue.tf b/karpenter_interruption_queue.tf → modules/karpenter/interruption_queue.tf
@@ -1,5 +1,5 @@
 resource "aws_sqs_queue" "karpenter_interruption" {
-  name                      = "Karpenter-${var.name}"
+  name                      = "Karpenter-${var.cluster_config.name}"
   message_retention_seconds = 300
   sqs_managed_sse_enabled   = true
 }
@@ -61,7 +61,7 @@ locals {
 resource "aws_cloudwatch_event_rule" "karpenter" {
   for_each = local.karpenter_events
 
-  name          = "Karpenter${each.value.name}-${var.name}"
+  name          = "Karpenter${each.value.name}-${var.cluster_config.name}"
   description   = each.value.description
   event_pattern = jsonencode(each.value.event_pattern)
 }

diff --git a/karpenter_node_iam.tf → modules/karpenter/node_iam.tf b/karpenter_node_iam.tf → modules/karpenter/node_iam.tf
@@ -1,7 +1,7 @@
 resource "aws_iam_role" "karpenter_node" {
-  name               = "${var.iam_role_name_prefix}KarpenterNode-${var.name}"
+  name               = "${var.cluster_config.iam_role_name_prefix}KarpenterNode-${var.cluster_config.name}"
   assume_role_policy = data.aws_iam_policy_document.karpenter_node_assume_role_policy.json
-  description        = "Karpenter node role for ${var.name} cluster"
+  description        = "Karpenter node role for ${var.cluster_config.name} cluster"
 }
 
 data "aws_iam_policy_document" "karpenter_node_assume_role_policy" {

diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf
@@ -0,0 +1,3 @@
+output "node_role_arn" {
+  value = aws_iam_role.karpenter_node.arn
+}
diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf
@@ -0,0 +1,18 @@
+variable "cluster_config" {
+  description = "EKS cluster config object"
+  type = object({
+    name                       = string
+    arn                        = string
+    private_subnet_ids         = map(string)
+    iam_role_name_prefix       = string
+    fargate_execution_role_arn = string
+  })
+}
+
+variable "oidc_config" {
+  description = "OIDC config object"
+  type = object({
+    url = string
+    arn = string
+  })
+}
diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.47.0"
+    }
+  }
+}
diff --git a/outputs.tf b/outputs.tf
@@ -1,12 +1,15 @@
 locals {
   config = {
-    name                = aws_eks_cluster.control_plane.name
-    endpoint            = aws_eks_cluster.control_plane.endpoint
-    ca_data             = aws_eks_cluster.control_plane.certificate_authority[0].data
-    vpc_id              = var.vpc_config.vpc_id
-    private_subnet_ids  = var.vpc_config.private_subnet_ids
-    node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id
-    tags                = var.tags
+    name                       = aws_eks_cluster.control_plane.name
+    endpoint                   = aws_eks_cluster.control_plane.endpoint
+    arn                        = aws_eks_cluster.control_plane.arn
+    ca_data                    = aws_eks_cluster.control_plane.certificate_authority[0].data
+    vpc_id                     = var.vpc_config.vpc_id
+    private_subnet_ids         = var.vpc_config.private_subnet_ids
+    node_security_group        = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id
+    tags                       = var.tags
+    iam_role_name_prefix       = var.iam_role_name_prefix
+    fargate_execution_role_arn = aws_iam_role.fargate.arn
   }
 }
 

diff --git a/variables.tf b/variables.tf
@@ -93,7 +93,7 @@ variable "security_group_ids" {
 
 variable "fargate_namespaces" {
   type        = set(string)
-  default     = ["kube-system", "karpenter", "flux-system"]
+  default     = ["kube-system", "flux-system"]
   description = "A list of namespaces to create fargate profiles for, should be set to a list of namespaces critical for flux / cluster bootstrapping"
 }