chore: add and remove nodes, set hostname properly

k3s.yml

@@ -9,6 +9,16 @@ init:
 onboot:
   - name: metadata
     image: linuxkit/metadata:v0.8
+  - name: hostname
+    image: busybox:latest
+    command: ["hostname", "-F", "/etc/hostname"]
+    binds:
+      - /run/config/hostname/hostname:/etc/hostname
+    net: host
+    pid: host
+    rootfsPropagation: shared
+    capabilities:
+      - CAP_SYS_ADMIN
   - name: sysctl
     image: linuxkit/sysctl:v0.8
   - name: sysfs

terraform/agent.tf

@@ -1,46 +1,49 @@
 resource libvirt_volume agent_os {
   count  = var.agents
-  name   = format("k3skit-agent-%s-os.img", count.index + 1)
+  name   = format("k3skit-agent-%s-os.img", count.index)
   pool   = libvirt_pool.k3skit.name
   source = var.k3skit_os
   format = "raw"
 }
 
 resource libvirt_volume agent_volume {
   count  = var.agents
-  name   = format("k3skit-agent-%s-volume.img", count.index + 1)
+  name   = format("k3skit-agent-%s-volume.img", count.index)
   pool   = libvirt_pool.k3skit.name
   size   = 20 * 1024 * 1024 * 1024 # Size in bytes (N  (GiB) * 1024 (MiB) * 1024 (KiB) * 1024 (Bytes))
   format = "raw"
 }
 
 resource libvirt_volume agent_kernel {
   count  = var.agents
-  name   = format("k3skit-agent-%s-kernel.img", count.index + 1)
+  name   = format("k3skit-agent-%s-kernel.img", count.index)
   source = var.k3skit_kernel
   pool   = libvirt_pool.k3skit.name
   format = "raw"
 }
 
 data template_file agent_metadata {
+  count    = var.agents
   template = file(format("%s/files/agent.yml", path.module))
   vars = {
     authorized_key = tls_private_key.default.public_key_openssh
     server         = local.kubeconfig.clusters.0.cluster.server
+    hostname       = format("k3skit-agent-%s", count.index)
     token          = data.external.token.result.token
   }
 }
 
 resource libvirt_cloudinit_disk agent_metadata {
-  name      = "k3skit-agent-metadata.iso"
-  user_data = jsonencode(yamldecode(data.template_file.agent_metadata.rendered))
+  count     = var.agents
+  name      = format("k3skit-agent-%s-metadata.iso", count.index)
+  user_data = jsonencode(yamldecode(element(data.template_file.agent_metadata.*.rendered, count.index)))
   pool      = libvirt_pool.k3skit.name
 }
 
 resource libvirt_domain agent {
   depends_on = [libvirt_domain.server]
   count      = var.agents
-  name       = format("k3skit-agent-%s", count.index + 1)
+  name       = format("k3skit-agent-%s", count.index)
   memory     = "2048"
   vcpu       = 1
   qemu_agent = false
@@ -53,11 +56,11 @@ resource libvirt_domain agent {
     }
   ]
 
-  cloudinit = libvirt_cloudinit_disk.agent_metadata.id
+  cloudinit = element(libvirt_cloudinit_disk.agent_metadata.*.id, count.index)
 
   network_interface {
     network_name   = "default"
-    hostname       = format("k3skit-agent-%s", count.index + 1)
+    hostname       = format("k3skit-agent-%s", count.index)
     wait_for_lease = true
   }
 

terraform/files/agent.yml

@@ -19,3 +19,8 @@ ssh:
     authorized_keys:
       content: |
         ${authorized_key}
+hostname:
+  entries:
+    hostname:
+      content: |
+        ${hostname}

terraform/files/server.yml

@@ -2,10 +2,13 @@ rancher:
   entries:
     k3s-helper:
       content: |
-        export MASTER_IP=$(/bin/ifconfig eth0|grep 'inet addr'| cut -d: -f2 | awk '{ print $1 }')
+        export NODE_IP=$(/bin/ifconfig eth0|grep 'inet addr'| cut -d: -f2 | awk '{ print $1 }')
         /bin/k3s \
           server \
-            --bind-address=$${MASTER_IP} \
+            --bind-address=$${NODE_IP} \
+%{ if agents > 0 ~}
+            --disable-agent \
+%{ endif ~}
             --disable-kube-proxy \
             --disable=traefik \
             --flannel-backend=none \
@@ -17,12 +20,17 @@ rancher:
             --kubelet-arg=kube-reserved-cgroup=podruntime \
             --kubelet-arg=node-status-update-frequency=4s \
             --kubelet-arg=system-reserved-cgroup=systemreserved \
-            --node-external-ip=$${MASTER_IP} \
-            --node-ip=$${MASTER_IP} \
+            --node-external-ip=$${NODE_IP} \
+            --node-ip=$${NODE_IP} \
             --secrets-encryption
       perm: "0755"
 ssh:
   entries:
     authorized_keys:
       content: |
         ${authorized_key}
+hostname:
+  entries:
+    hostname:
+      content: |
+        ${hostname}

terraform/server.tf

@@ -22,7 +22,9 @@ resource libvirt_volume server_kernel {
 data template_file server_metadata {
   template = file(format("%s/files/server.yml", path.module))
   vars = {
+    agents         = var.agents
     authorized_key = tls_private_key.default.public_key_openssh
+    hostname       = "k3skit-server"
   }
 }
 

terraform/utils.tf

@@ -0,0 +1,35 @@
+resource null_resource remove_agent {
+  depends_on = [libvirt_domain.server, libvirt_domain.agent, local_file.kubeconfig, local_file.private_key_pem]
+  count      = var.agents
+  triggers = {
+    agent       = format("k3skit-agent-%s", count.index + 1)
+    kubeconfig  = local_file.kubeconfig.filename
+    private_key = local_file.private_key_pem.filename
+    server      = libvirt_domain.server.network_interface.0.addresses.0
+    user        = "root"
+  }
+
+  provisioner "local-exec" {
+    when       = destroy
+    command    = format("kubectl --kubeconfig=%s drain %s --ignore-daemonsets --delete-local-data", self.triggers.kubeconfig, self.triggers.agent)
+    on_failure = continue
+  }
+
+  provisioner "local-exec" {
+    when       = destroy
+    command    = format("kubectl --kubeconfig=%s delete node %s", self.triggers.kubeconfig, self.triggers.agent)
+    on_failure = continue
+  }
+
+  provisioner "local-exec" {
+    when       = destroy
+    command    = format("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s %s@%s sed -i -e '/%s/d' /var/lib/rancher/k3s/server/cred/node-passwd", self.triggers.private_key, self.triggers.user, self.triggers.server, self.triggers.agent)
+    on_failure = continue
+  }
+
+  provisioner "local-exec" {
+    when       = destroy
+    command    = format("kubectl --kubeconfig=%s delete po --force --grace-period=0 -n=kube-system --selector=k8s-app=metrics-server", self.triggers.kubeconfig)
+    on_failure = continue
+  }
+}