Update coredns manifests (#143)

* docs.aws.amazon.com/eks/latest/userguide/coredns.html
* amazon-eks.s3.us-west-2.amazonaws.com/cloudformation/2020-08-12/dns.yaml

Reverse lookups were not working with the previous configuration.

modules/asg_node_group/cloud_config.tpl

@@ -4,4 +4,4 @@ fqdn: eks-node-${cluster_name}-{{ v1.instance_id }}
 runcmd:
 - [aws, --region={{ v1.region }}, ec2, create-tags, --resources={{ v1.instance_id }}, "--tags=Key=Name,Value=eks-node-${cluster_name}-{{ v1.instance_id }}"]
 - [systemctl, restart, docker]
-- [/etc/eks/bootstrap.sh, ${cluster_name}, --kubelet-extra-args, '--node-labels=${labels} --register-with-taints="${taints}"']
+- [/etc/eks/bootstrap.sh, ${cluster_name}, --kubelet-extra-args, '--node-labels=${labels} --register-with-taints="${taints}"', --dns-cluster-ip, ${dns_cluster_ip}]

modules/asg_node_group/main.tf

@@ -34,9 +34,10 @@ data "aws_ami" "image" {
 data "template_file" "cloud_config" {
   template = file("${path.module}/cloud_config.tpl")
   vars = {
-    cluster_name = var.cluster_config.name
-    labels       = join(",", [for label, value in local.labels : "${label}=${value}"])
-    taints       = join(",", [for taint, value_effect in var.taints : "${taint}=${value_effect}"])
+    cluster_name   = var.cluster_config.name
+    labels         = join(",", [for label, value in local.labels : "${label}=${value}"])
+    taints         = join(",", [for taint, value_effect in var.taints : "${taint}=${value_effect}"])
+    dns_cluster_ip = var.cluster_config.dns_cluster_ip
   }
 }
 

modules/asg_node_group/variables.tf

@@ -5,6 +5,7 @@ variable "cluster_config" {
     private_subnet_ids    = map(string)
     node_security_group   = string
     node_instance_profile = string
+    dns_cluster_ip        = string
   })
 }
 

modules/cluster/addons.tf

@@ -19,12 +19,23 @@ module "aws_k8s_cni" {
   manifest = file("${path.module}/addons/aws-k8s-cni.yaml")
 }
 
+data "aws_vpc" "network" {
+  id = var.vpc_config.vpc_id
+}
+
+locals {
+  dns_cluster_ip = length(var.dns_cluster_ip) > 0 ? var.dns_cluster_ip : (split(".", data.aws_vpc.network.cidr_block)[0] == "10" ? "172.20.0.10" : "10.100.0.10")
+}
+
 module "coredns" {
   source = "./kubectl"
   config = local.config
   manifest = templatefile(
     "${path.module}/addons/coredns.yaml",
-    { aws_region = data.aws_region.current.name },
+    {
+      aws_region     = data.aws_region.current.name,
+      dns_cluster_ip = local.dns_cluster_ip,
+    },
   )
 }
 

modules/cluster/addons/coredns.yaml

@@ -1,42 +1,126 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    eks.amazonaws.com/component: coredns
+    k8s-app: kube-dns
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    eks.amazonaws.com/component: coredns
+    k8s-app: kube-dns
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    eks.amazonaws.com/component: coredns
+    k8s-app: kube-dns
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
+  name: coredns
+  namespace: kube-system
   labels:
     eks.amazonaws.com/component: coredns
     k8s-app: kube-dns
-    kubernetes.io/name: CoreDNS
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+          pods insecure
+          upstream
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
   name: coredns
   namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: "CoreDNS"
+    eks.amazonaws.com/component: coredns
 spec:
   replicas: 2
-  selector:
-    matchLabels:
-      eks.amazonaws.com/component: coredns
-      k8s-app: kube-dns
   strategy:
+    type: RollingUpdate
     rollingUpdate:
-      maxSurge: 1
       maxUnavailable: 1
-    type: RollingUpdate
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+      eks.amazonaws.com/component: coredns
   template:
     metadata:
       labels:
-        eks.amazonaws.com/component: coredns
         k8s-app: kube-dns
+        eks.amazonaws.com/component: coredns
+      annotations:
+        eks.amazonaws.com/compute-type: ec2
     spec:
+      serviceAccountName: coredns
+      priorityClassName: system-cluster-critical
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
-              - key: beta.kubernetes.io/os
+              - key: "beta.kubernetes.io/os"
                 operator: In
                 values:
                 - linux
-              - key: beta.kubernetes.io/arch
+              - key: "beta.kubernetes.io/arch"
                 operator: In
                 values:
                 - amd64
+                - arm64
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - podAffinityTerm:
@@ -48,23 +132,28 @@ spec:
                   - kube-dns
               topologyKey: kubernetes.io/hostname
             weight: 100
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
       containers:
-      - args:
-        - -conf
-        - /etc/coredns/Corefile
-        image: 602401143452.dkr.ecr.${aws_region}.amazonaws.com/eks/coredns:v1.6.6
+      - name: coredns
+        image: 602401143452.dkr.ecr.${aws_region}.amazonaws.com/eks/coredns:v1.6.6-eksbuild.1
         imagePullPolicy: IfNotPresent
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /health
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 60
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: coredns
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        - name: tmp
+          mountPath: /tmp
         ports:
         - containerPort: 53
           name: dns
@@ -75,71 +164,60 @@ spec:
         - containerPort: 9153
           name: metrics
           protocol: TCP
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             add:
             - NET_BIND_SERVICE
             drop:
             - all
-          procMount: Default
           readOnlyRootFilesystem: true
-        terminationMessagePath: /dev/termination-log
-        terminationMessagePolicy: File
-        volumeMounts:
-        - mountPath: /etc/coredns
-          name: config-volume
-          readOnly: true
       dnsPolicy: Default
-      priorityClassName: system-node-critical
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      serviceAccount: coredns
-      serviceAccountName: coredns
-      terminationGracePeriodSeconds: 30
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-      - key: CriticalAddonsOnly
-        operator: Exists
       volumes:
-      - configMap:
-          defaultMode: 420
-          items:
-          - key: Corefile
-            path: Corefile
-          name: coredns
-        name: config-volume
+        - name: tmp
+          emptyDir: {}
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
 ---
 apiVersion: v1
-kind: ConfigMap
+kind: Service
 metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
   labels:
-    eks.amazonaws.com/component: coredns
     k8s-app: kube-dns
-  name: coredns
-  namespace: kube-system
-data:
-  Corefile: |
-    .:53 {
-        errors
-        health
-        kubernetes cluster.local in-addr.arpa ip6.arpa {
-          pods insecure
-          upstream
-          fallthrough in-addr.arpa ip6.arpa
-        }
-        prometheus :9153
-        forward . /etc/resolv.conf
-        cache 30
-        loop
-        reload
-        loadbalance
-    }
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+    eks.amazonaws.com/component: kube-dns
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: ${dns_cluster_ip}
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP

modules/cluster/outputs.tf

@@ -7,6 +7,7 @@ locals {
     private_subnet_ids    = var.vpc_config.private_subnet_ids
     node_security_group   = aws_security_group.node.id
     node_instance_profile = var.iam_config.node_role
+    dns_cluster_ip        = local.dns_cluster_ip
   }
 }
 

modules/cluster/variables.tf

@@ -109,3 +109,9 @@ variable "kms_cmk_arn" {
   default     = ""
   description = "The ARN of the KMS (CMK) customer master key, to be used for Envelope Encryption of Kubernetes secrets, if not set a key will be generated"
 }
+
+variable "dns_cluster_ip" {
+  type        = string
+  default     = ""
+  description = "Overrides the IP address to use for DNS queries within the cluster. Defaults to 10.100.0.10 or 172.20.0.10 based on the VPC cidr"
+}