From f1867acbe21546e666a9aac28574a1bb42c28fbe Mon Sep 17 00:00:00 2001 From: Amim Knabben Date: Sun, 7 Apr 2024 14:55:09 -0300 Subject: [PATCH] Fixing commands and adapting code --- cmd/destroy.go | 8 +- cmd/kubernetes.go | 39 +++--- cmd/setup.go | 35 +++-- cmd/start.go | 37 ++--- cmd/start_test.go | 11 -- pkg/executors/exec/utils.go | 23 ++++ pkg/executors/tests/server.go | 2 +- pkg/pwsh/kubernetes/provisioners_test.go | 31 ----- pkg/pwsh/setup/setup.go | 134 +++++++++--------- pkg/pwsh/setup/setup_test.go | 166 +++++++++++------------ pkg/templates/yaml.go | 10 ++ samples/config.yaml | 2 +- specs/apiserver.yaml | 5 - specs/installation.yaml | 7 +- 14 files changed, 236 insertions(+), 274 deletions(-) delete mode 100644 pkg/pwsh/kubernetes/provisioners_test.go delete mode 100644 specs/apiserver.yaml diff --git a/cmd/destroy.go b/cmd/destroy.go index a797a3d..4f92a76 100644 --- a/cmd/destroy.go +++ b/cmd/destroy.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" "swdt/apis/config/v1alpha1" "swdt/pkg/drivers" + "swdt/pkg/executors/exec" ) // destroyCmd represents the destroy command @@ -46,10 +47,9 @@ func RunDestroy(cmd *cobra.Command, args []string) error { } if config.Spec.ControlPlane.Minikube { - // Delete minikube - /*if _, err = exec.Execute(exec.RunCommand, "minikube", "delete", "--purge"); err != nil { - return err - }*/ + e := exec.NewLocalExecutor() + go exec.EnableOutput(nil, e.Stdout) + return e.Run("minikube delete --purge", nil) } return nil } diff --git a/cmd/kubernetes.go b/cmd/kubernetes.go index 30036ef..0996fa2 100644 --- a/cmd/kubernetes.go +++ b/cmd/kubernetes.go @@ -18,6 +18,9 @@ package cmd import ( "github.com/spf13/cobra" + "swdt/apis/config/v1alpha1" + ifacer "swdt/pkg/pwsh/iface" + "swdt/pkg/pwsh/kubernetes" ) // setupCmd represents the setup command @@ -30,27 +33,19 @@ var kubernetesCmd = &cobra.Command{ func RunKubernetes(cmd *cobra.Command, args []string) error { var ( - //err error - //config *v1alpha1.Cluster + err error + config *v1alpha1.Cluster ) - /* - if config, err = loadConfiguration(cmd); err != nil { - return err - } - - // Starting the executor - runner, err := executor.NewRunner(config.Spec.Workload.Virtualization.SSH, &kubernetes.KubernetesRunner{}) - if err != nil { - return err - } - defer func(runner *executor.Runner[*kubernetes.KubernetesRunner]) { - if err := runner.CloseConnection(); err != nil { - log.Fatalf("error to close the connection: %v\n", err) - } - }(runner) - - return runner.Inner.InstallProvisioners(config.Spec.Workload.Provisioners) - - */ - return nil + if config, err = loadConfiguration(cmd); err != nil { + return err + } + + // Starting the executor + ssh := config.Spec.Workload.Virtualization.SSH + r, err := ifacer.NewRunner(ssh, &kubernetes.Runner{}) + if err != nil { + return err + } + + return r.Inner.InstallProvisioners(config.Spec.Workload.Provisioners) } diff --git a/cmd/setup.go b/cmd/setup.go index fab7de5..e086fd7 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -62,6 +62,9 @@ func RunSetup(cmd *cobra.Command, args []string) error { ssh := config.Spec.Workload.Virtualization.SSH r, err := ifacer.NewRunner(ssh, &setup.Runner{Logging: true}) + if err != nil { + return err + } // Install choco binary and packages if a list of packages exists if len(*config.Spec.Workload.Auxiliary.ChocoPackages) > 0 { @@ -80,33 +83,27 @@ func RunSetup(cmd *cobra.Command, args []string) error { return err } - /* - // Installing Containerd with predefined version - containerd := config.Spec.Workload.ContainerdVersion - if err = setupRunner.InstallContainerd(containerd); err != nil { - return err - } + // Installing Containerd with predefined version + containerd := config.Spec.Workload.ContainerdVersion + if err = r.Inner.InstallContainerd(containerd); err != nil { + return err + } - // Installing Kubeadm and Kubelet binaries in the host - kubernetes := config.Spec.Workload.KubernetesVersion - if err = runner.Inner.InstallKubernetes(kubernetes); err != nil { - return err - } + // Installing Kubeadm and Kubelet binaries in the host + kubernetes := config.Spec.Workload.KubernetesVersion + if err = r.Inner.InstallKubernetes(kubernetes); err != nil { + return err + } - */ // Joining the Windows node in the control plane. cpKubernetes := config.Spec.ControlPlane.KubernetesVersion if err = r.Inner.JoinNode(cpKubernetes, controlPlaneIP); err != nil { return err } - /* - // Install Calico CNI operator and CR - // NOTE: Only Calico is supported for now on HPC - /*if err = runner.Inner.InstallCNI(config.Spec.CalicoVersion); err != nil { - return err - }*/ - return nil + // Install Calico CNI operator and CR + // NOTE: Only Calico is supported for now on HPC + return r.Inner.InstallCNI(config.Spec.CalicoVersion, controlPlaneIP) } // findPrivateIPs returns the leased ips from the domain. diff --git a/cmd/start.go b/cmd/start.go index 04e607c..8f71bc0 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -22,6 +22,7 @@ import ( "log" "strings" "swdt/apis/config/v1alpha1" + "swdt/pkg/executors/exec" "github.com/spf13/cobra" "libvirt.org/go/libvirt" @@ -53,11 +54,11 @@ func RunStart(cmd *cobra.Command, args []string) error { // Start the minikube if the flag is enabled. if config.Spec.ControlPlane.Minikube { - //version := config.Spec.ControlPlane.KubernetesVersion + version := config.Spec.ControlPlane.KubernetesVersion klog.Info(resc.Sprintf("Starting a Minikube control plane, this operation can take a while...")) - /*if err := startMinikube(exec.RunCommand, version); err != nil { + if err := startMinikube(version); err != nil { return err - }*/ + } } // Start the Windows VM on LibVirt @@ -72,13 +73,11 @@ func startWindowsVM(config *v1alpha1.Cluster) error { err error ) + log.Println("Creating domain...") drv, err := drivers.NewDriver(config) if err != nil { return err } - - log.Println("Creating domain...") - // Create the libvirt domain if dom, err = drv.CreateDomain(); err != nil { // Domain already exists, skipping the Windows creation. @@ -95,26 +94,20 @@ func startWindowsVM(config *v1alpha1.Cluster) error { }() // Start the Windows created domain. - if err = drv.KvmDriver.Start(); err != nil { - return err - } - - return nil + return drv.KvmDriver.Start() } // startMinikube initialize a minikube control plane. -func startMinikube(executor interface{}, version string) (err error) { +func startMinikube(version string) (err error) { // Start minikube with KVM2 machine - /* - cmd := []string{ - "minikube", "start", "--driver", "kvm2", - "--container-runtime", "containerd", - "--kubernetes-version", version, - } - if _, err = exec.Execute(executor, cmd...); err != nil { - return err - }*/ - return nil + cmd := strings.Join([]string{ + "minikube", "start", "--driver", "kvm2", + "--container-runtime", "containerd", + "--kubernetes-version", version, + }, " ") + e := exec.NewLocalExecutor() + go exec.EnableOutput(nil, e.Stdout) + return e.Run(cmd, nil) } func alreadyExists(err error) bool { diff --git a/cmd/start_test.go b/cmd/start_test.go index 2a44081..7a3d5b3 100644 --- a/cmd/start_test.go +++ b/cmd/start_test.go @@ -11,14 +11,3 @@ func TestAlreadyExists(t *testing.T) { assert.True(t, alreadyExists(errors.New("already exists with"))) assert.False(t, alreadyExists(errors.New("error"))) } - -func TestStartMinikube(t *testing.T) { - var called = 0 - fn := func(cmd ...string) (string, error) { - called += 1 - return "", nil - } - err := startMinikube(fn, "v1.29.0") - assert.Nil(t, err) - assert.Equal(t, 1, called) -} diff --git a/pkg/executors/exec/utils.go b/pkg/executors/exec/utils.go index ef2f679..dff4662 100644 --- a/pkg/executors/exec/utils.go +++ b/pkg/executors/exec/utils.go @@ -2,7 +2,9 @@ package exec import ( "bufio" + "fmt" "io" + "strings" "sync" ) @@ -18,3 +20,24 @@ func redirectStandard(mu *sync.Mutex, std io.Reader, to *chan string) { *to = make(chan string) } } + +func EnableOutput(output *string, fn func(std *chan string)) { + std := make(chan string) + fn(&std) + var outlist []string + for { + select { + case n, ok := <-std: + if !ok { + if output != nil { + *output = strings.Join(outlist, " ") + } + break + } + if output != nil { + outlist = append(outlist, n) + } + fmt.Println(n) + } + } +} diff --git a/pkg/executors/tests/server.go b/pkg/executors/tests/server.go index 7a579bc..fad01c9 100644 --- a/pkg/executors/tests/server.go +++ b/pkg/executors/tests/server.go @@ -95,7 +95,7 @@ func acceptConnection(listener net.Listener, config *ssh.ServerConfig, responses stderr := channel.Stderr() _, _ = fmt.Fprintf(stderr, "%v", err) } - _, err = t.Write([]byte(resp)) + _, err = channel.Write([]byte(resp)) if err != nil { log.Fatalf("error writing channel: %v", err) } diff --git a/pkg/pwsh/kubernetes/provisioners_test.go b/pkg/pwsh/kubernetes/provisioners_test.go deleted file mode 100644 index 91f2522..0000000 --- a/pkg/pwsh/kubernetes/provisioners_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package kubernetes - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "swdt/apis/config/v1alpha1" - "testing" -) - -var ( - calls = []string{} -) - -func validateRun(args string) error { - calls = append(calls, args) - return nil -} - -func validateCopy(l, r, p string) error { - return nil -} - -func TestInstallProvisioners(t *testing.T) { - r := KubernetesRunner{run: validateRun, copy: validateCopy} - serviceName := "containerd" - provisioners := []v1alpha1.ProvisionerSpec{{Name: serviceName}} - assert.Nil(t, r.InstallProvisioners(provisioners)) - assert.Len(t, calls, 2) - assert.Equal(t, calls[0], fmt.Sprintf("Stop-Service -name %s -Force", serviceName)) - assert.Equal(t, calls[1], fmt.Sprintf("Start-Service -name %s", serviceName)) -} diff --git a/pkg/pwsh/setup/setup.go b/pkg/pwsh/setup/setup.go index c7686c1..b7515c0 100644 --- a/pkg/pwsh/setup/setup.go +++ b/pkg/pwsh/setup/setup.go @@ -5,7 +5,9 @@ import ( "github.com/fatih/color" "k8s.io/klog/v2" "strings" + "swdt/pkg/executors/exec" "swdt/pkg/executors/iface" + "swdt/pkg/templates" "time" ) @@ -57,27 +59,6 @@ func (r *Runner) runRstd(args string, stdout *chan string) error { return r.remote.Run(args, stdout) } -func (r *Runner) enableOutput(output *string, fn func(std *chan string)) { - std := make(chan string) - fn(&std) - var outlist []string - for { - select { - case n, ok := <-std: - if !ok { - if output != nil { - *output = strings.Join(outlist, " ") - } - break - } - if output != nil { - outlist = append(outlist, n) - } - fmt.Println(n) - } - } -} - // ChocoExists check if choco is already installed in the system. func (r *Runner) ChocoExists() bool { return r.runR(fmt.Sprintf("%s --version", CHOCO_PATH)) == nil @@ -88,8 +69,8 @@ func (r *Runner) InstallChoco() error { klog.Info(mainc.Sprint("Installing Choco with PowerShell.")) if r.Logging { - go r.enableOutput(nil, r.remote.Stdout) - go r.enableOutput(nil, r.remote.Stderr) + go exec.EnableOutput(nil, r.remote.Stdout) + go exec.EnableOutput(nil, r.remote.Stderr) } if r.ChocoExists() { @@ -124,8 +105,8 @@ func (r *Runner) EnableRDP(enable bool) error { } if r.Logging { - go r.enableOutput(nil, r.remote.Stdout) - go r.enableOutput(nil, r.remote.Stderr) + go exec.EnableOutput(nil, r.remote.Stdout) + go exec.EnableOutput(nil, r.remote.Stderr) } klog.Info(mainc.Sprint("Enabling Remote Desktop.")) @@ -133,17 +114,20 @@ func (r *Runner) EnableRDP(enable bool) error { Enable-NetFirewallRule -DisplayGroup 'Remote Desktop'`) } -/* // InstallContainerd install the containerd bits with the set version, enabled required services. func (r *Runner) InstallContainerd(containerd string) error { - var output string klog.Info(mainc.Sprintf("Installing containerd.")) - go r.enableOutput(&output) + + var output string + if r.Logging { + go exec.EnableOutput(nil, r.remote.Stdout) + go exec.EnableOutput(nil, r.remote.Stderr) + } // Install containerd if service is not running. - if err := r.run("get-service -name containerd"); err != nil { + if err := r.runR("get-service -name containerd"); err != nil { cmd := fmt.Sprintf(".\\Install-Containerd.ps1 -ContainerDVersion %s", containerd) - return r.run(`curl.exe -LO https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/Install-Containerd.ps1; ` + cmd) + return r.runR(`curl.exe -LO https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/Install-Containerd.ps1; ` + cmd) } else if strings.Contains(output, "Running") { klog.Info(resc.Sprintf("Skipping containerd installation, service already running, use the copy command.")) } @@ -152,42 +136,43 @@ func (r *Runner) InstallContainerd(containerd string) error { // InstallKubernetes install all Kubernetes bits with the set version. func (r *Runner) InstallKubernetes(kubernetes string) error { - var output string klog.Info(mainc.Sprintf("Installing Kubelet.")) - go r.enableOutput(&output) - if err := r.run("get-service -name kubelet"); err != nil { + var output string + if r.Logging { + go exec.EnableOutput(nil, r.remote.Stdout) + go exec.EnableOutput(nil, r.remote.Stderr) + } + + if err := r.runR("get-service -name kubelet"); err != nil { // Install Kubernetes if service is not running. cmd := fmt.Sprintf(".\\PrepareNode.ps1 -KubernetesVersion %s", kubernetes) - return r.run(`curl.exe -LO https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/PrepareNode.ps1; ` + cmd) + return r.runR(`curl.exe -LO https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/PrepareNode.ps1; ` + cmd) } else if strings.Contains(output, "kubelet") { // Otherwise skip klog.Info(resc.Sprintf("Skipping Kubelet installation, service already running, use the copy command.")) } return nil } -*/ // JoinNode joins the Windows node into control-plane cluster. func (r *Runner) JoinNode(cpVersion, cpIPAddr string) error { var ( - err error - output string - loutput string - //lout string + err error + output, loutput string ) - go r.enableOutput(&output, r.remote.Stdout) - go r.enableOutput(&output, r.remote.Stderr) - go r.enableOutput(&loutput, r.local.Stdout) + go exec.EnableOutput(&output, r.remote.Stdout) + go exec.EnableOutput(&output, r.remote.Stderr) + go exec.EnableOutput(&loutput, r.local.Stdout) // In case kubelet is already running, skip joining procedure. if err = r.runR("get-service -name kubelet"); err == nil && !strings.Contains(output, "Running") { + fmt.Println("outputttt ", output) // Control plane token create and extract, saving the final command lcmd := strings.Join([]string{ - "minikube", "ssh", "--", "sudo", - fmt.Sprintf("/var/lib/minikube/binaries/%s/kubeadm", cpVersion), + "minikube", "ssh", "--", "sudo", fmt.Sprintf("/var/lib/minikube/binaries/%s/kubeadm", cpVersion), "token", "create", "--print-join-command", - }, "") + }, " ") if err = r.runL(lcmd); err != nil { return err } @@ -224,45 +209,50 @@ func (r *Runner) JoinNode(cpVersion, cpIPAddr string) error { return nil } -/* // InstallCNI installs Calico CNI receiving a specific version. -func (r *SetupRunner) InstallCNI(calicoVersion string) error { +func (r *Runner) InstallCNI(calicoVersion, controlPlaneIP string) error { + var loutput string - content, err := templates.OpenYamlFile("./specs/kube-proxy.yml") - if err != nil { - return err - } + go exec.EnableOutput(&loutput, r.local.Stdout) + go exec.EnableOutput(&loutput, r.local.Stderr) - fmt.Println(templates.ChangeTemplate(string(content), templates.KubeProxyTmpl{KUBERNETES_VERSION: calicoVersion})) + var ( + err error + content []byte + ) - content, err = templates.OpenYamlFile("./specs/configmap.yml") - if err != nil { - return err - } + if content, err = templates.OpenYAMLFile("./specs/kube-proxy.yml"); err != nil { + return err + } + kpTmpl := templates.KubeProxyTmpl{KUBERNETES_VERSION: calicoVersion} + t, _ := templates.ChangeTemplate(string(content), kpTmpl) + kpTempFile := templates.SaveFile(t) + defer templates.DeleteFile(kpTempFile) - fmt.Println(templates.ChangeTemplate(string(content), templates.ConfigMapTmpl{KUBERNETES_SERVICE_HOST: "bla", KUBERNETES_SERVICE_PORT: "45654"})) + if content, err = templates.OpenYAMLFile("./specs/configmap.yml"); err != nil { + return err + } + cpTmpl := templates.ConfigMapTmpl{KUBERNETES_SERVICE_HOST: controlPlaneIP, KUBERNETES_SERVICE_PORT: "8443"} + t, _ = templates.ChangeTemplate(string(content), cpTmpl) + cpTempFile := templates.SaveFile(t) + defer templates.DeleteFile(cpTempFile) // Execute Kubernetes steps for Calico installation steps := [][]string{ - {"kubectl", "patch", "ipamconfigurations", "default", "--type", "merge", "--patch=" + string(templates.GetSpecAffinity())}, {"kubectl", "config", "set-context", "minikube"}, - //{"kubectl", "create", "-f", fmt.Sprintf("https://raw.githubusercontent.com/projectcalico/calico/%v/manifests/tigera-operator.yaml", calicoVersion)}, - //{"cat", "<