Update to Kubernetes 1.20.0, update add-ons, move to Containerd runtime

scholzj · Dec 18, 2020 · a52b124 · a52b124
1 parent 526df34
commit a52b124
Show file tree

Hide file tree

Showing 3 changed files with 167 additions and 44 deletions.
diff --git a/README.md b/README.md
@@ -16,6 +16,7 @@ AWS Kubernetes is a Kubernetes cluster deployed using [Kubeadm](https://kubernet
 
 ## Updates
 
+* *18.12.2020* Update to Kubernetes 1.20.0, update add-ons, move to Containerd runtime
 * *24.11.2020* Update to Kubernetes 1.19.4
 * *18.10.2020* Update to Kubernetes 1.19.3
 * *19.9.2020* Update to Kubernetes 1.19.2

diff --git a/scripts/init-aws-kubernetes-master.sh b/scripts/init-aws-kubernetes-master.sh
@@ -16,12 +16,13 @@ export ASG_MAX_NODES="${asg_max_nodes}"
 export AWS_REGION=${aws_region}
 export AWS_SUBNETS="${aws_subnets}"
 export ADDONS="${addons}"
-export KUBERNETES_VERSION="1.19.4"
+export KUBERNETES_VERSION="1.20.0"
 
 # Set this only after setting the defaults
 set -o nounset
 
 # We needed to match the hostname expected by kubeadm an the hostname used by kubelet
+LOCAL_IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 FULL_HOSTNAME="$(curl -s http://169.254.169.254/latest/meta-data/hostname)"
 
 # Make DNS lowercase
@@ -32,16 +33,81 @@ yum install -y epel-release
 yum install -y python2-pip
 pip install awscli --upgrade
 
+########################################
+########################################
 # Tag subnets
+########################################
+########################################
 for SUBNET in $AWS_SUBNETS
 do
   aws ec2 create-tags --resources $SUBNET --tags Key=kubernetes.io/cluster/$CLUSTER_NAME,Value=shared --region $AWS_REGION
 done
 
+########################################
+########################################
+# Disable SELinux
+########################################
+########################################
+
+# setenforce returns non zero if already SE Linux is already disabled
+is_enforced=$(getenforce)
+if [[ $is_enforced != "Disabled" ]]; then
+  setenforce 0
+  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
+fi
+
+########################################
+########################################
+# Install containerd
+########################################
+########################################
+cat <<EOF | tee /etc/modules-load.d/containerd.conf
+overlay
+br_netfilter
+EOF
+
+modprobe overlay
+modprobe br_netfilter
+
+# Setup required sysctl params, these persist across reboots.
+cat <<EOF | tee /etc/sysctl.d/99-kubernetes-cri.conf
+net.bridge.bridge-nf-call-iptables  = 1
+net.ipv4.ip_forward                 = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+EOF
+
+# Apply sysctl params without reboot
+sysctl --system
+
+yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2
+yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+sudo yum install -y containerd.io
+mkdir -p /etc/containerd
+containerd config default > /etc/containerd/config.toml
+sed -i '/^          \[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.runc\.options\]/a \            SystemdCgroup = true' /etc/containerd/config.toml
+systemctl restart containerd
+
+########################################
+########################################
 # Install docker
-yum install -y yum-utils device-mapper-persistent-data lvm2 docker
+########################################
+########################################
 
-# Install Kubernetes components
+# yum install -y yum-utils device-mapper-persistent-data lvm2 docker
+
+# # Start services
+# systemctl enable docker
+# systemctl start docker
+
+# # Set settings needed by Docker
+# sysctl net.bridge.bridge-nf-call-iptables=1
+# sysctl net.bridge.bridge-nf-call-ip6tables=1
+
+########################################
+########################################
+# Install Kubernetes compoenents
+########################################
+########################################
 sudo cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
@@ -53,32 +119,24 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
         https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOF
 
-# setenforce returns non zero if already SE Linux is already disabled
-is_enforced=$(getenforce)
-if [[ $is_enforced != "Disabled" ]]; then
-  setenforce 0
-  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
-
-fi
-
 yum install -y kubelet-$KUBERNETES_VERSION kubeadm-$KUBERNETES_VERSION kubernetes-cni
 
 # Start services
-systemctl enable docker
-systemctl start docker
 systemctl enable kubelet
 systemctl start kubelet
 
-# Set settings needed by Docker
-sysctl net.bridge.bridge-nf-call-iptables=1
-sysctl net.bridge.bridge-nf-call-ip6tables=1
-
 # Fix certificates file on CentOS
 if cat /etc/*release | grep ^NAME= | grep CentOS ; then
     rm -rf /etc/ssl/certs/ca-certificates.crt/
     cp /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
 fi
 
+########################################
+########################################
+# Initialize the Kube cluster
+########################################
+########################################
+
 # Initialize the master
 cat >/tmp/kubeadm.yaml <<EOF
 ---
@@ -93,10 +151,10 @@ bootstrapTokens:
   - signing
   - authentication
 nodeRegistration:
-  criSocket: /var/run/dockershim.sock
   kubeletExtraArgs:
     cloud-provider: aws
     read-only-port: "10255"
+    cgroup-driver: systemd
   name: $FULL_HOSTNAME
   taints:
   - effect: NoSchedule
@@ -108,6 +166,8 @@ apiServer:
   certSANs:
   - $DNS_NAME
   - $IP_ADDRESS
+  - $LOCAL_IP_ADDRESS
+  - $FULL_HOSTNAME
   extraArgs:
     cloud-provider: aws
   timeoutForControlPlane: 5m0s
@@ -124,9 +184,8 @@ etcd:
 imageRepository: k8s.gcr.io
 kubernetesVersion: v$KUBERNETES_VERSION
 networking:
-  podNetworkCidr: 192.168.0.0/16
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: 192.168.0.0/16
   serviceSubnet: 10.96.0.0/12
 scheduler: {}
 ---
@@ -141,24 +200,33 @@ export KUBECONFIG=/etc/kubernetes/admin.conf
 # Install calico
 kubectl apply -f /tmp/calico.yaml
 
+
+########################################
+########################################
+# Create user and kubeconfig files
+########################################
+########################################
+
 # Allow the user to administer the cluster
 kubectl create clusterrolebinding admin-cluster-binding --clusterrole=cluster-admin --user=admin
 
 # Prepare the kubectl config file for download to client (IP address)
 export KUBECONFIG_OUTPUT=/home/centos/kubeconfig_ip
-kubeadm alpha kubeconfig user \
-  --client-name admin \
-  --apiserver-advertise-address $IP_ADDRESS \
-  > $KUBECONFIG_OUTPUT
+kubeadm alpha kubeconfig user --client-name admin --config /tmp/kubeadm.yaml > $KUBECONFIG_OUTPUT
 chown centos:centos $KUBECONFIG_OUTPUT
 chmod 0600 $KUBECONFIG_OUTPUT
 
 cp /home/centos/kubeconfig_ip /home/centos/kubeconfig
-sed -i "s/server: https:\/\/$IP_ADDRESS:6443/server: https:\/\/$DNS_NAME:6443/g" /home/centos/kubeconfig
+sed -i "s/server: https:\/\/.*:6443/server: https:\/\/$IP_ADDRESS:6443/g" /home/centos/kubeconfig_ip
+sed -i "s/server: https:\/\/.*:6443/server: https:\/\/$DNS_NAME:6443/g" /home/centos/kubeconfig
 chown centos:centos /home/centos/kubeconfig
 chmod 0600 /home/centos/kubeconfig
 
-# Load addons
+########################################
+########################################
+# Install addons
+########################################
+########################################
 for ADDON in $ADDONS
 do
   curl $ADDON | envsubst > /tmp/addon.yaml

diff --git a/scripts/init-aws-kubernetes-node.sh b/scripts/init-aws-kubernetes-node.sh
@@ -9,21 +9,84 @@ set -o pipefail
 export KUBEADM_TOKEN=${kubeadm_token}
 export MASTER_IP=${master_private_ip}
 export DNS_NAME=${dns_name}
-export KUBERNETES_VERSION="1.19.4"
+export KUBERNETES_VERSION="1.20.0"
 
 # Set this only after setting the defaults
 set -o nounset
 
 # We to match the hostname expected by kubeadm an the hostname used by kubelet
+LOCAL_IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 FULL_HOSTNAME="$(curl -s http://169.254.169.254/latest/meta-data/hostname)"
 
 # Make DNS lowercase
 DNS_NAME=$(echo "$DNS_NAME" | tr 'A-Z' 'a-z')
 
+########################################
+########################################
+# Disable SELinux
+########################################
+########################################
+
+# setenforce returns non zero if already SE Linux is already disabled
+is_enforced=$(getenforce)
+if [[ $is_enforced != "Disabled" ]]; then
+  setenforce 0
+  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
+fi
+
+########################################
+########################################
+# Install containerd
+########################################
+########################################
+cat <<EOF | tee /etc/modules-load.d/containerd.conf
+overlay
+br_netfilter
+EOF
+
+modprobe overlay
+modprobe br_netfilter
+
+# Setup required sysctl params, these persist across reboots.
+cat <<EOF | tee /etc/sysctl.d/99-kubernetes-cri.conf
+net.bridge.bridge-nf-call-iptables  = 1
+net.ipv4.ip_forward                 = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+EOF
+
+# Apply sysctl params without reboot
+sysctl --system
+
+yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2
+yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+sudo yum install -y containerd.io
+mkdir -p /etc/containerd
+containerd config default > /etc/containerd/config.toml
+sed -i '/^          \[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.runc\.options\]/a \            SystemdCgroup = true' /etc/containerd/config.toml
+systemctl restart containerd
+
+########################################
+########################################
 # Install docker
-yum install -y yum-utils device-mapper-persistent-data lvm2 docker
+########################################
+########################################
+
+# yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2 docker
+
+# # Start services
+# systemctl enable docker
+# systemctl start docker
+
+# # Set settings needed by Docker
+# sysctl net.bridge.bridge-nf-call-iptables=1
+# sysctl net.bridge.bridge-nf-call-ip6tables=1
 
-# Install Kubernetes components
+
+########################################
+########################################
+# Install Kubernetes compoenents
+########################################
+########################################
 sudo cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
@@ -35,32 +98,23 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
         https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOF
 
-# setenforce returns non zero if already SE Linux is already disabled
-is_enforced=$(getenforce)
-if [[ $is_enforced != "Disabled" ]]; then
-  setenforce 0
-  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
-fi
-
 yum install -y kubelet-$KUBERNETES_VERSION kubeadm-$KUBERNETES_VERSION kubernetes-cni
 
 # Start services
-systemctl enable docker
-systemctl start docker
 systemctl enable kubelet
 systemctl start kubelet
 
-# Set settings needed by Docker
-sysctl net.bridge.bridge-nf-call-iptables=1
-sysctl net.bridge.bridge-nf-call-ip6tables=1
-
 # Fix certificates file on CentOS
 if cat /etc/*release | grep ^NAME= | grep CentOS ; then
     rm -rf /etc/ssl/certs/ca-certificates.crt/
     cp /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
 fi
 
-# Initialize the master
+########################################
+########################################
+# Initialize the Kube node
+########################################
+########################################
 cat >/tmp/kubeadm.yaml <<EOF
 ---
 apiVersion: kubeadm.k8s.io/v1beta2
@@ -73,10 +127,10 @@ discovery:
   timeout: 5m0s
   tlsBootstrapToken: $KUBEADM_TOKEN
 nodeRegistration:
-  criSocket: /var/run/dockershim.sock
   kubeletExtraArgs:
     cloud-provider: aws
     read-only-port: "10255"
+    cgroup-driver: systemd
   name: $FULL_HOSTNAME
 ---
 EOF