From 43e1156f8794ccdcd822e951cfeb1824c48dd60e Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Fri, 30 Oct 2020 02:42:59 -0300 Subject: [PATCH] chore: add and remove nodes, set hostname properly --- k3s.yml | 10 ++++++++++ terraform/agent.tf | 19 +++++++++++-------- terraform/files/agent.yml | 5 +++++ terraform/files/server.yml | 16 ++++++++++++---- terraform/server.tf | 2 ++ terraform/utils.tf | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 terraform/utils.tf diff --git a/k3s.yml b/k3s.yml index 3fb52f4..88f3caf 100644 --- a/k3s.yml +++ b/k3s.yml @@ -9,6 +9,16 @@ init: onboot: - name: metadata image: linuxkit/metadata:v0.8 + - name: hostname + image: busybox:latest + command: ["hostname", "-F", "/etc/hostname"] + binds: + - /run/config/hostname/hostname:/etc/hostname + net: host + pid: host + rootfsPropagation: shared + capabilities: + - CAP_SYS_ADMIN - name: sysctl image: linuxkit/sysctl:v0.8 - name: sysfs diff --git a/terraform/agent.tf b/terraform/agent.tf index aafb555..5aa98a0 100644 --- a/terraform/agent.tf +++ b/terraform/agent.tf @@ -1,6 +1,6 @@ resource libvirt_volume agent_os { count = var.agents - name = format("k3skit-agent-%s-os.img", count.index + 1) + name = format("k3skit-agent-%s-os.img", count.index) pool = libvirt_pool.k3skit.name source = var.k3skit_os format = "raw" @@ -8,7 +8,7 @@ resource libvirt_volume agent_os { resource libvirt_volume agent_volume { count = var.agents - name = format("k3skit-agent-%s-volume.img", count.index + 1) + name = format("k3skit-agent-%s-volume.img", count.index) pool = libvirt_pool.k3skit.name size = 20 * 1024 * 1024 * 1024 # Size in bytes (N (GiB) * 1024 (MiB) * 1024 (KiB) * 1024 (Bytes)) format = "raw" @@ -16,31 +16,34 @@ resource libvirt_volume agent_volume { resource libvirt_volume agent_kernel { count = var.agents - name = format("k3skit-agent-%s-kernel.img", count.index + 1) + name = format("k3skit-agent-%s-kernel.img", count.index) source = var.k3skit_kernel pool = libvirt_pool.k3skit.name format = "raw" } data template_file agent_metadata { + count = var.agents template = file(format("%s/files/agent.yml", path.module)) vars = { authorized_key = tls_private_key.default.public_key_openssh server = local.kubeconfig.clusters.0.cluster.server + hostname = format("k3skit-agent-%s", count.index) token = data.external.token.result.token } } resource libvirt_cloudinit_disk agent_metadata { - name = "k3skit-agent-metadata.iso" - user_data = jsonencode(yamldecode(data.template_file.agent_metadata.rendered)) + count = var.agents + name = format("k3skit-agent-%s-metadata.iso", count.index) + user_data = jsonencode(yamldecode(element(data.template_file.agent_metadata.*.rendered, count.index))) pool = libvirt_pool.k3skit.name } resource libvirt_domain agent { depends_on = [libvirt_domain.server] count = var.agents - name = format("k3skit-agent-%s", count.index + 1) + name = format("k3skit-agent-%s", count.index) memory = "2048" vcpu = 1 qemu_agent = false @@ -53,11 +56,11 @@ resource libvirt_domain agent { } ] - cloudinit = libvirt_cloudinit_disk.agent_metadata.id + cloudinit = element(libvirt_cloudinit_disk.agent_metadata.*.id, count.index) network_interface { network_name = "default" - hostname = format("k3skit-agent-%s", count.index + 1) + hostname = format("k3skit-agent-%s", count.index) wait_for_lease = true } diff --git a/terraform/files/agent.yml b/terraform/files/agent.yml index 72065ee..30b34b7 100644 --- a/terraform/files/agent.yml +++ b/terraform/files/agent.yml @@ -19,3 +19,8 @@ ssh: authorized_keys: content: | ${authorized_key} +hostname: + entries: + hostname: + content: | + ${hostname} diff --git a/terraform/files/server.yml b/terraform/files/server.yml index f18fd97..5c72027 100644 --- a/terraform/files/server.yml +++ b/terraform/files/server.yml @@ -2,10 +2,13 @@ rancher: entries: k3s-helper: content: | - export MASTER_IP=$(/bin/ifconfig eth0|grep 'inet addr'| cut -d: -f2 | awk '{ print $1 }') + export NODE_IP=$(/bin/ifconfig eth0|grep 'inet addr'| cut -d: -f2 | awk '{ print $1 }') /bin/k3s \ server \ - --bind-address=$${MASTER_IP} \ + --bind-address=$${NODE_IP} \ +%{ if agents > 0 ~} + --disable-agent \ +%{ endif ~} --disable-kube-proxy \ --disable=traefik \ --flannel-backend=none \ @@ -17,8 +20,8 @@ rancher: --kubelet-arg=kube-reserved-cgroup=podruntime \ --kubelet-arg=node-status-update-frequency=4s \ --kubelet-arg=system-reserved-cgroup=systemreserved \ - --node-external-ip=$${MASTER_IP} \ - --node-ip=$${MASTER_IP} \ + --node-external-ip=$${NODE_IP} \ + --node-ip=$${NODE_IP} \ --secrets-encryption perm: "0755" ssh: @@ -26,3 +29,8 @@ ssh: authorized_keys: content: | ${authorized_key} +hostname: + entries: + hostname: + content: | + ${hostname} diff --git a/terraform/server.tf b/terraform/server.tf index a78f5f3..5ada4ba 100644 --- a/terraform/server.tf +++ b/terraform/server.tf @@ -22,7 +22,9 @@ resource libvirt_volume server_kernel { data template_file server_metadata { template = file(format("%s/files/server.yml", path.module)) vars = { + agents = var.agents authorized_key = tls_private_key.default.public_key_openssh + hostname = "k3skit-server" } } diff --git a/terraform/utils.tf b/terraform/utils.tf new file mode 100644 index 0000000..78e983c --- /dev/null +++ b/terraform/utils.tf @@ -0,0 +1,35 @@ +resource null_resource remove_agent { + depends_on = [libvirt_domain.server, libvirt_domain.agent, local_file.kubeconfig, local_file.private_key_pem] + count = var.agents + triggers = { + agent = format("k3skit-agent-%s", count.index + 1) + kubeconfig = local_file.kubeconfig.filename + private_key = local_file.private_key_pem.filename + server = libvirt_domain.server.network_interface.0.addresses.0 + user = "root" + } + + provisioner "local-exec" { + when = destroy + command = format("kubectl --kubeconfig=%s drain %s --ignore-daemonsets --delete-local-data", self.triggers.kubeconfig, self.triggers.agent) + on_failure = continue + } + + provisioner "local-exec" { + when = destroy + command = format("kubectl --kubeconfig=%s delete node %s", self.triggers.kubeconfig, self.triggers.agent) + on_failure = continue + } + + provisioner "local-exec" { + when = destroy + command = format("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s %s@%s sed -i -e '/%s/d' /var/lib/rancher/k3s/server/cred/node-passwd", self.triggers.private_key, self.triggers.user, self.triggers.server, self.triggers.agent) + on_failure = continue + } + + provisioner "local-exec" { + when = destroy + command = format("kubectl --kubeconfig=%s delete po --force --grace-period=0 -n=kube-system --selector=k8s-app=metrics-server", self.triggers.kubeconfig) + on_failure = continue + } +}