add a cleanup loop for failed vm using configmap

Before this PR we have an issue if the vm creation fails because there
is an validation error from the virt-api we didn't clean the allocation.

This PR introduce a cleanup loop into the system.
When we hit the create vm mutating webhook we create the regular
allocation but we also mark the mac address in  a waiting configmap.

We have a waiting look the will check if the object is removed from the
map and if is not after 30 second we assume the object wasn't saved
into the etcd (no controller event) so we release the object.

If we get an event from the controller then we remove the mac address
from the configmap.

TODO: pass the waiting time as an environment variable.

.travis.yml

@@ -13,7 +13,7 @@ script:
   - if [[ -n "$(git status --porcelain)" ]] ; then echo "It seems like you need to run make. Please run it and commit the changes"; git status --porcelain; false; fi
   - make docker-test
   - make deploy-test-cluster
-  - KUBECONFIG="`pwd`/cluster/dind-cluster/config" go test -v -race ./tests/...
+  - KUBECONFIG="`pwd`/cluster/dind-cluster/config" go test -timeout 30m -v -race ./tests/...
 
 deploy:
 - provider: script

cmd/manager/main.go

@@ -43,9 +43,11 @@ func loadMacAddressFromEnvVar(envName string) (net.HardwareAddr, error) {
 
 func main() {
 	var logType, metricsAddr string
+	var waitingTime int
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&logType, "v", "production", "Log type (debug/production).")
+	flag.IntVar(&waitingTime, "wait-time", 600, "waiting time to release the mac if object was not created")
 	flag.Parse()
 
 	if logType == "debug" {
@@ -80,7 +82,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	kubemacpoolManager := manager.NewKubeMacPoolManager(podNamespace, podName, metricsAddr)
+	kubemacpoolManager := manager.NewKubeMacPoolManager(podNamespace, podName, metricsAddr, waitingTime)
 
 	err = kubemacpoolManager.Run(rangeStart, rangeEnd)
 	if err != nil {

config/default/manager/manager.yaml

@@ -57,6 +57,7 @@ spec:
         - /manager
         args:
           - "--v=production"
+          - "--wait-time=600"
         image: quay.io/kubevirt/kubemacpool:latest
         imagePullPolicy: Always
         name: manager

config/release/kubemacpool.yaml

@@ -172,6 +172,7 @@ spec:
       containers:
       - args:
         - --v=production
+        - --wait-time=600
         command:
         - /manager
         env:

config/test/kubemacpool.yaml

@@ -172,6 +172,7 @@ spec:
       containers:
       - args:
         - --v=debug
+        - --wait-time=10
         command:
         - /manager
         env:

config/test/manager_image_patch.yaml

@@ -10,3 +10,4 @@ spec:
         name: manager
         args:
           - "--v=debug"
+          - "--wait-time=10"

hack/functest.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 source hack/common.sh
-KUBECONFIG=${MACPOOL_DIR}/cluster/$MACPOOL_PROVIDER/.kubeconfig go test -v -race ./tests/...
+KUBECONFIG=${MACPOOL_DIR}/cluster/$MACPOOL_PROVIDER/.kubeconfig go test -timeout 20m -v -race ./tests/...

pkg/controller/virtualmachine/virtualmachine_controller.go

@@ -113,6 +113,7 @@ func (r *ReconcilePolicy) addFinalizerAndUpdate(virtualMachine *kubevirt.Virtual
 	if helper.ContainsString(virtualMachine.ObjectMeta.Finalizers, pool_manager.RuntimeObjectFinalizerName) {
 		return nil
 	}
+
 	log.V(1).Info("The VM does not have a finalizer",
 		"virtualMachineName", request.Name,
 		"virtualMachineNamespace", request.Namespace)
@@ -130,6 +131,8 @@ func (r *ReconcilePolicy) addFinalizerAndUpdate(virtualMachine *kubevirt.Virtual
 		"virtualMachineName", request.Name,
 		"virtualMachineNamespace", request.Namespace)
 
+	r.poolManager.MarkVMAsReady(virtualMachine)
+
 	return nil
 }
 

pkg/manager/manager.go

@@ -33,6 +33,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 
 	"github.com/K8sNetworkPlumbingWG/kubemacpool/pkg/controller"
+	"github.com/K8sNetworkPlumbingWG/kubemacpool/pkg/names"
 	poolmanager "github.com/K8sNetworkPlumbingWG/kubemacpool/pkg/pool-manager"
 	"github.com/K8sNetworkPlumbingWG/kubemacpool/pkg/webhook"
 )
@@ -49,18 +50,20 @@ type KubeMacPoolManager struct {
 	stopSignalChannel        chan os.Signal
 	podNamespace             string
 	podName                  string
+	waitingTime              int
 	resourceLock             resourcelock.Interface
 }
 
-func NewKubeMacPoolManager(podNamespace, podName, metricsAddr string) *KubeMacPoolManager {
+func NewKubeMacPoolManager(podNamespace, podName, metricsAddr string, waitingTime int) *KubeMacPoolManager {
 	kubemacpoolManager := &KubeMacPoolManager{
 		continueToRunManager:     true,
 		restartChannel:           make(chan struct{}),
 		kubevirtInstalledChannel: make(chan struct{}),
 		stopSignalChannel:        make(chan os.Signal, 1),
 		podNamespace:             podNamespace,
 		podName:                  podName,
-		metricsAddr:              metricsAddr}
+		metricsAddr:              metricsAddr,
+		waitingTime:              waitingTime}
 
 	signal.Notify(kubemacpoolManager.stopSignalChannel, os.Interrupt, os.Kill)
 
@@ -119,7 +122,7 @@ func (k *KubeMacPoolManager) Run(rangeStart, rangeEnd net.HardwareAddr) error {
 		}
 
 		isKubevirtInstalled := checkForKubevirt(k.clientset)
-		poolManager, err := poolmanager.NewPoolManager(k.clientset, rangeStart, rangeEnd, isKubevirtInstalled)
+		poolManager, err := poolmanager.NewPoolManager(k.clientset, rangeStart, rangeEnd, isKubevirtInstalled, k.waitingTime)
 		if err != nil {
 			return fmt.Errorf("unable to create pool manager error %v", err)
 		}
@@ -196,7 +199,7 @@ func (k *KubeMacPoolManager) markPodAsLeader() error {
 		return err
 	}
 
-	pod.Labels[webhook.LeaderLabel] = "true"
+	pod.Labels[names.LEADER_LABEL] = "true"
 	_, err = k.clientset.CoreV1().Pods(k.podNamespace).Update(pod)
 	if err != nil {
 		return err

pkg/names/names.go

@@ -0,0 +1,15 @@
+package names
+
+const MANAGER_NAMESPACE = "kubemacpool-system"
+
+const MANAGER_DEPLOYMENT = "kubemacpool-mac-controller-manager"
+
+const WEBHOOK_SERVICE = "kubemacpool-service"
+
+const MUTATE_WEBHOOK = "kubemacpool-webhook"
+
+const MUTATE_WEBHOOK_CONFIG = "kubemacpool"
+
+const LEADER_LABEL = "kubemacpool-leader"
+
+const ADMISSION_IGNORE_LABEL = "kubemacpool/ignoreAdmission"

pkg/pool-manager/pool.go

@@ -31,6 +31,7 @@ const (
 	RuntimeObjectFinalizerName = "k8s.v1.cni.cncf.io/kubeMacPool"
 	networksAnnotation         = "k8s.v1.cni.cncf.io/networks"
 	networksStatusAnnotation   = "k8s.v1.cni.cncf.io/networks-status"
+	vmWaitConfigMapName        = "kubemacpool-vm-configmap"
 )
 
 var log = logf.Log.WithName("PoolManager")
@@ -54,7 +55,7 @@ const (
 	AllocationStatusWaitingForPod AllocationStatus = "WaitingForPod"
 )
 
-func NewPoolManager(kubeClient kubernetes.Interface, rangeStart, rangeEnd net.HardwareAddr, kubevirtExist bool) (*PoolManager, error) {
+func NewPoolManager(kubeClient kubernetes.Interface, rangeStart, rangeEnd net.HardwareAddr, kubevirtExist bool, waitTime int) (*PoolManager, error) {
 	err := checkRange(rangeStart, rangeEnd)
 	if err != nil {
 		return nil, err
@@ -86,6 +87,8 @@ func NewPoolManager(kubeClient kubernetes.Interface, rangeStart, rangeEnd net.Ha
 		return nil, err
 	}
 
+	go poolManger.vmWaitingCleanupLook(waitTime)
+
 	return poolManger, nil
 }