diff --git a/examples/cluster/main.tf b/examples/cluster/main.tf index 8d84da5..852645e 100644 --- a/examples/cluster/main.tf +++ b/examples/cluster/main.tf @@ -34,7 +34,15 @@ module "cluster" { username = aws_iam_role.test_role.name rolearn = aws_iam_role.test_role.arn groups = ["system:masters"] - } + }, + { + username = "system:node:{{EC2PrivateDNSName}}" + rolearn = module.karpenter.node_role_arn + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, ] tags = { @@ -42,6 +50,13 @@ module "cluster" { } } +module "karpenter" { + source = "../../modules/karpenter" + + cluster_config = module.cluster.config + oidc_config = module.cluster.oidc_config +} + data "aws_security_group" "nodes" { id = module.cluster.config.node_security_group } diff --git a/main.tf b/main.tf index 9f8f8d6..d5601b3 100644 --- a/main.tf +++ b/main.tf @@ -61,14 +61,6 @@ locals { "system:node-proxier", ] }, - { - rolearn = aws_iam_role.karpenter_node.arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - }, ], var.aws_auth_role_map, )) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md new file mode 100644 index 0000000..396bb10 --- /dev/null +++ b/modules/karpenter/README.md @@ -0,0 +1,49 @@ +# Karpenter + +This module configures the resources required to run the +karpenter node-provisioning tool in an eks cluster. + +* Fargate Profile - to run karpenter +* IAM roles for the fargate controller and nodes to be provisioned by karpenter +* SQS queue to provide events (spot interruption etc) to karpenter + +It does not install karpenter itself to the cluster - and we recomend +that you use helm as per the [karpenter documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#4-install-karpenter) + +It is provided as a submodule so the core module is less opinionated. + +However we test the core module and the karpenter module +in our test suite to ensure that the different components we use in our +clusters at cookpad intergrate correctly. + + +## Example + +You should pass cluster and oidc config from the cluster to the karpenter module. + +You will also need to add the IAM role of nodes created by karpenter to the aws_auth_role_map +so they can connect to the cluster. + +```hcl +module "cluster" { + source = "cookpad/eks/aws" + + aws_auth_role_map = [ + { + username = "system:node:{{EC2PrivateDNSName}}" + rolearn = module.karpenter.node_role_arn + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] +} + +module "karpenter" { + source = "cookpad/eks/aws//modules/karpenter" + + cluster_config = module.cluster.config + oidc_config = module.cluster.oidc_config +} +``` diff --git a/karpenter_controller_iam.tf b/modules/karpenter/controller_iam.tf similarity index 80% rename from karpenter_controller_iam.tf rename to modules/karpenter/controller_iam.tf index d107671..8f15408 100644 --- a/karpenter_controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -1,7 +1,7 @@ resource "aws_iam_role" "karpenter_controller" { - name = "${var.iam_role_name_prefix}Karpenter-${var.name}" + name = "${var.cluster_config.iam_role_name_prefix}Karpenter-${var.cluster_config.name}" assume_role_policy = data.aws_iam_policy_document.karpenter_controller_assume_role_policy.json - description = "Karpenter controller role for ${var.name} cluster" + description = "Karpenter controller role for ${var.cluster_config.name} cluster" } data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { @@ -11,18 +11,18 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { condition { test = "StringEquals" - variable = "${replace(aws_iam_openid_connect_provider.cluster_oidc.url, "https://", "")}:sub" + variable = "${replace(var.oidc_config.url, "https://", "")}:sub" values = ["system:serviceaccount:karpenter:karpenter"] } condition { test = "StringEquals" - variable = "${replace(aws_iam_openid_connect_provider.cluster_oidc.url, "https://", "")}:aud" + variable = "${replace(var.oidc_config.url, "https://", "")}:aud" values = ["sts.amazonaws.com"] } principals { - identifiers = [aws_iam_openid_connect_provider.cluster_oidc.arn] + identifiers = [var.oidc_config.arn] type = "Federated" } } @@ -69,7 +69,7 @@ data "aws_iam_policy_document" "karpenter_controller" { statement { actions = ["eks:DescribeCluster"] - resources = [aws_eks_cluster.control_plane.arn] + resources = [var.cluster_config.arn] } statement { diff --git a/modules/karpenter/data.tf b/modules/karpenter/data.tf new file mode 100644 index 0000000..c3872eb --- /dev/null +++ b/modules/karpenter/data.tf @@ -0,0 +1,2 @@ +data "aws_partition" "current" {} + diff --git a/modules/karpenter/fargate.tf b/modules/karpenter/fargate.tf new file mode 100644 index 0000000..9263120 --- /dev/null +++ b/modules/karpenter/fargate.tf @@ -0,0 +1,11 @@ +resource "aws_eks_fargate_profile" "critical_pods" { + cluster_name = var.cluster_config.name + fargate_profile_name = "${var.cluster_config.name}-karpenter" + pod_execution_role_arn = var.cluster_config.fargate_execution_role_arn + subnet_ids = values(var.cluster_config.private_subnet_ids) + + selector { + namespace = "karpenter" + labels = {} + } +} diff --git a/karpenter_interruption_queue.tf b/modules/karpenter/interruption_queue.tf similarity index 93% rename from karpenter_interruption_queue.tf rename to modules/karpenter/interruption_queue.tf index 5470fdc..e355f3c 100644 --- a/karpenter_interruption_queue.tf +++ b/modules/karpenter/interruption_queue.tf @@ -1,5 +1,5 @@ resource "aws_sqs_queue" "karpenter_interruption" { - name = "Karpenter-${var.name}" + name = "Karpenter-${var.cluster_config.name}" message_retention_seconds = 300 sqs_managed_sse_enabled = true } @@ -61,7 +61,7 @@ locals { resource "aws_cloudwatch_event_rule" "karpenter" { for_each = local.karpenter_events - name = "Karpenter${each.value.name}-${var.name}" + name = "Karpenter${each.value.name}-${var.cluster_config.name}" description = each.value.description event_pattern = jsonencode(each.value.event_pattern) } diff --git a/karpenter_node_iam.tf b/modules/karpenter/node_iam.tf similarity index 83% rename from karpenter_node_iam.tf rename to modules/karpenter/node_iam.tf index e36f70e..a1e1de4 100644 --- a/karpenter_node_iam.tf +++ b/modules/karpenter/node_iam.tf @@ -1,7 +1,7 @@ resource "aws_iam_role" "karpenter_node" { - name = "${var.iam_role_name_prefix}KarpenterNode-${var.name}" + name = "${var.cluster_config.iam_role_name_prefix}KarpenterNode-${var.cluster_config.name}" assume_role_policy = data.aws_iam_policy_document.karpenter_node_assume_role_policy.json - description = "Karpenter node role for ${var.name} cluster" + description = "Karpenter node role for ${var.cluster_config.name} cluster" } data "aws_iam_policy_document" "karpenter_node_assume_role_policy" { diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf new file mode 100644 index 0000000..4f296d4 --- /dev/null +++ b/modules/karpenter/outputs.tf @@ -0,0 +1,3 @@ +output "node_role_arn" { + value = aws_iam_role.karpenter_node.arn +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf new file mode 100644 index 0000000..4f4a9eb --- /dev/null +++ b/modules/karpenter/variables.tf @@ -0,0 +1,18 @@ +variable "cluster_config" { + description = "EKS cluster config object" + type = object({ + name = string + arn = string + private_subnet_ids = map(string) + iam_role_name_prefix = string + fargate_execution_role_arn = string + }) +} + +variable "oidc_config" { + description = "OIDC config object" + type = object({ + url = string + arn = string + }) +} diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf new file mode 100644 index 0000000..bbbe93e --- /dev/null +++ b/modules/karpenter/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47.0" + } + } +} diff --git a/outputs.tf b/outputs.tf index 95b3ad3..5664d4b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,12 +1,15 @@ locals { config = { - name = aws_eks_cluster.control_plane.name - endpoint = aws_eks_cluster.control_plane.endpoint - ca_data = aws_eks_cluster.control_plane.certificate_authority[0].data - vpc_id = var.vpc_config.vpc_id - private_subnet_ids = var.vpc_config.private_subnet_ids - node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id - tags = var.tags + name = aws_eks_cluster.control_plane.name + endpoint = aws_eks_cluster.control_plane.endpoint + arn = aws_eks_cluster.control_plane.arn + ca_data = aws_eks_cluster.control_plane.certificate_authority[0].data + vpc_id = var.vpc_config.vpc_id + private_subnet_ids = var.vpc_config.private_subnet_ids + node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id + tags = var.tags + iam_role_name_prefix = var.iam_role_name_prefix + fargate_execution_role_arn = aws_iam_role.fargate.arn } } diff --git a/variables.tf b/variables.tf index 09e02e7..0a8f739 100644 --- a/variables.tf +++ b/variables.tf @@ -93,7 +93,7 @@ variable "security_group_ids" { variable "fargate_namespaces" { type = set(string) - default = ["kube-system", "karpenter", "flux-system"] + default = ["kube-system", "flux-system"] description = "A list of namespaces to create fargate profiles for, should be set to a list of namespaces critical for flux / cluster bootstrapping" }