feat: add --extra-ca flag (#1127)

add --extra-ca flag and sets it up on the yet to be released new version
of kots-helm.

.github/workflows/ci.yaml

@@ -496,6 +496,7 @@ jobs:
           - TestCustomCIDR
           - TestProxiedCustomCIDR
           - TestSingleNodeInstallationNoopUpgrade
+          - TestInstallWithPrivateCAs
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster

.gitignore

@@ -1,7 +1,9 @@
 output
 bundle
-pkg/goods/bins
-pkg/goods/internal/bins
+pkg/goods/bins/*
+!pkg/goods/bins/.placeholder
+pkg/goods/internal/bins/*
+!pkg/goods/internal/bins/.placeholder
 pkg/goods/images
 *tgz
 .pre-commit-config.yaml

Makefile

@@ -202,8 +202,8 @@ build-ttl.sh:
 .PHONY: clean
 clean:
 	rm -rf output
-	rm -rf pkg/goods/bins
-	rm -rf pkg/goods/internal/bins
+	rm -rf pkg/goods/bins/*
+	rm -rf pkg/goods/internal/bins/*
 	rm -rf build
 	rm -rf bin
 

cmd/embedded-cluster/install.go

@@ -569,6 +569,10 @@ var installCommand = &cli.Command{
 				Usage: "Skip host preflight checks. This is not recommended.",
 				Value: false,
 			},
+			&cli.StringSliceFlag{
+				Name:  "private-ca",
+				Usage: "Path to a trusted private CA certificate file",
+			},
 		},
 	)),
 	Action: func(c *cli.Context) error {
@@ -665,6 +669,18 @@ func getAddonsApplier(c *cli.Context, adminConsolePwd string) (*addons.Applier,
 		}
 		opts = append(opts, addons.WithEndUserConfig(eucfg))
 	}
+	if len(c.StringSlice("private-ca")) > 0 {
+		privateCAs := map[string]string{}
+		for i, path := range c.StringSlice("private-ca") {
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read private CA file %s: %w", path, err)
+			}
+			name := fmt.Sprintf("ca_%d.crt", i)
+			privateCAs[name] = string(data)
+		}
+		opts = append(opts, addons.WithPrivateCAs(privateCAs))
+	}
 	if adminConsolePwd != "" {
 		opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd))
 	}

e2e/cluster/cluster.go

@@ -593,7 +593,7 @@ func CreateNodes(in *Input) ([]string, []string) {
 // pinging google.com.
 func NodeHasInternet(in *Input, node string) {
 	in.T.Logf("Testing if node %s can reach the internet", node)
-	fp, err := os.CreateTemp("/tmp", "internet-XXXXX.sh")
+	fp, err := os.CreateTemp("/tmp", "internet-*.sh")
 	if err != nil {
 		in.T.Fatalf("Failed to create temporary file: %v", err)
 	}
@@ -643,7 +643,7 @@ func NodeHasInternet(in *Input, node string) {
 // pinging google.com.
 func NodeHasNoInternet(in *Input, node string) {
 	in.T.Logf("Ensuring node %s cannot reach the internet", node)
-	fp, err := os.CreateTemp("/tmp", "internet-XXXXX.sh")
+	fp, err := os.CreateTemp("/tmp", "internet-*.sh")
 	if err != nil {
 		in.T.Fatalf("Failed to create temporary file: %v", err)
 	}

e2e/install_test.go

@@ -1,13 +1,18 @@
 package e2e
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+
 	"github.com/replicatedhq/embedded-cluster/e2e/cluster"
+	"github.com/replicatedhq/embedded-cluster/pkg/certs"
 )
 
 func TestSingleNodeInstallation(t *testing.T) {
@@ -2126,3 +2131,65 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) {
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
+
+func TestInstallWithPrivateCAs(t *testing.T) {
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
+	input := &cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "ubuntu/jammy",
+		LicensePath:         "license.yaml",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster",
+	}
+	tc := cluster.NewTestCluster(input)
+	defer cleanupCluster(t, tc)
+
+	certBuilder, err := certs.NewBuilder()
+	require.NoError(t, err, "unable to create new cert builder")
+	crtContent, _, err := certBuilder.Generate()
+	require.NoError(t, err, "unable to build test certificate")
+
+	tmpfile, err := os.CreateTemp("", "test-temp-cert-*.crt")
+	require.NoError(t, err, "unable to create temp file")
+	defer os.Remove(tmpfile.Name())
+
+	_, err = tmpfile.WriteString(crtContent)
+	require.NoError(t, err, "unable to write to temp file")
+	tmpfile.Close()
+
+	cluster.CopyFileToNode(input, tc.Nodes[0], cluster.File{
+		SourcePath: tmpfile.Name(),
+		DestPath:   "/tmp/ca.crt",
+		Mode:       0666,
+	})
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"single-node-install.sh", "ui", "--private-ca", "/tmp/ca.crt"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	t.Logf("checking if the configmap was created with the right values")
+	line = []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"}
+	stdout, _, err := RunCommandOnNode(t, tc, 0, line, WithECShelEnv())
+	require.NoError(t, err, "unable get kotsadm-private-cas configmap")
+
+	var cm corev1.ConfigMap
+	err = json.Unmarshal([]byte(stdout), &cm)
+	require.NoErrorf(t, err, "unable to unmarshal output to configmap: %q", stdout)
+	require.Contains(t, cm.Data, "ca_0.crt", "index ca_0.crt not found in ca secret")
+	require.Equal(t, crtContent, cm.Data["ca_0.crt"], "content mismatch")
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}

e2e/utils.go

@@ -32,6 +32,16 @@ func RequireEnvVars(t *testing.T, envVars []string) {
 
 type RunCommandOption func(cmd *cluster.Command)
 
+func WithECShelEnv() RunCommandOption {
+	return func(cmd *cluster.Command) {
+		cmd.Env = map[string]string{
+			"EMBEDDED_CLUSTER_METRICS_BASEURL": "https://staging.replicated.app",
+			"KUBECONFIG":                       "/var/lib/k0s/pki/admin.conf",
+			"PATH":                             "/var/lib/embedded-cluster/bin",
+		}
+	}
+}
+
 func WithEnv(env map[string]string) RunCommandOption {
 	return func(cmd *cluster.Command) {
 		cmd.Env = env

pkg/addons/adminconsole/adminconsole.go

@@ -33,7 +33,7 @@ import (
 )
 
 const (
-	releaseName                 = "admin-console"
+	ReleaseName                 = "admin-console"
 	DefaultAdminConsoleNodePort = 30000
 )
 
@@ -86,6 +86,7 @@ type AdminConsole struct {
 	licenseFile  string
 	airgapBundle string
 	proxyEnv     map[string]string
+	privateCAs   map[string]string
 }
 
 // Version returns the embedded admin console version.
@@ -99,7 +100,7 @@ func (a *AdminConsole) Name() string {
 
 // GetProtectedFields returns the helm values that are not overwritten when upgrading
 func (a *AdminConsole) GetProtectedFields() map[string][]string {
-	return map[string][]string{releaseName: protectedFields}
+	return map[string][]string{ReleaseName: protectedFields}
 }
 
 // HostPreflights returns the host preflight objects found inside the adminconsole
@@ -140,7 +141,7 @@ func (a *AdminConsole) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, only
 	}
 
 	chartConfig := ecv1beta1.Chart{
-		Name:      releaseName,
+		Name:      ReleaseName,
 		ChartName: chartName,
 		Version:   Metadata.Version,
 		Values:    string(values),
@@ -172,6 +173,10 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s
 		return fmt.Errorf("unable to create kots password secret: %w", err)
 	}
 
+	if err := createKotsCAConfigmap(ctx, cli, a.namespace, a.privateCAs); err != nil {
+		return fmt.Errorf("unable to create kots CA configmap: %w", err)
+	}
+
 	if a.airgapBundle != "" {
 		err := createRegistrySecret(ctx, cli, a.namespace)
 		if err != nil {
@@ -205,13 +210,14 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s
 }
 
 // New creates a new AdminConsole object.
-func New(ns, password string, licenseFile string, airgapBundle string, proxyEnv map[string]string) (*AdminConsole, error) {
+func New(ns, password string, licenseFile string, airgapBundle string, proxyEnv map[string]string, privateCAs map[string]string) (*AdminConsole, error) {
 	return &AdminConsole{
 		namespace:    ns,
 		password:     password,
 		licenseFile:  licenseFile,
 		airgapBundle: airgapBundle,
 		proxyEnv:     proxyEnv,
+		privateCAs:   privateCAs,
 	}, nil
 }
 
@@ -336,3 +342,29 @@ func createKotsPasswordSecret(ctx context.Context, cli client.Client, namespace
 
 	return nil
 }
+
+func createKotsCAConfigmap(ctx context.Context, cli client.Client, namespace string, cas map[string]string) error {
+	kotsCAConfigmap := corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kotsadm-private-cas",
+			Namespace: namespace,
+			Labels: map[string]string{
+				"kots.io/kotsadm":                        "true",
+				"replicated.com/disaster-recovery":       "infra",
+				"replicated.com/disaster-recovery-chart": "admin-console",
+			},
+		},
+		Data: cas,
+	}
+
+	err := cli.Create(ctx, &kotsCAConfigmap)
+	if err != nil {
+		return fmt.Errorf("unable to create kotsadm-private-cas configmap: %w", err)
+	}
+
+	return nil
+}

pkg/addons/adminconsole/static/metadata.yaml

@@ -5,7 +5,7 @@
 # $ make buildtools
 # $ output/bin/buildtools update addon <addon name>
 #
-version: 1.116.0
+version: 1.116.0-build.3
 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
 images:
     kotsadm:

pkg/addons/adminconsole/static/values.yaml

@@ -31,3 +31,6 @@ passwordSecretRef:
     name: kotsadm-password
 service:
     enabled: false
+privateCAs:
+    enabled: true
+    configmapName: "kotsadm-private-cas"

pkg/addons/applier.go

@@ -49,6 +49,7 @@ type Applier struct {
 	endUserConfig   *ecv1beta1.Config
 	airgapBundle    string
 	proxyEnv        map[string]string
+	privateCAs      map[string]string
 }
 
 // Outro runs the outro in all enabled add-ons.
@@ -295,7 +296,7 @@ func (a *Applier) load() ([]AddOn, error) {
 	}
 	addons = append(addons, vel)
 
-	aconsole, err := adminconsole.New(defaults.KotsadmNamespace, a.adminConsolePwd, a.licenseFile, a.airgapBundle, a.proxyEnv)
+	aconsole, err := adminconsole.New(defaults.KotsadmNamespace, a.adminConsolePwd, a.licenseFile, a.airgapBundle, a.proxyEnv, a.privateCAs)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create admin console addon: %w", err)
 	}

pkg/addons/options.go

@@ -14,6 +14,13 @@ func WithoutPrompt() Option {
 	}
 }
 
+// WithPrivateCAs sets the private CAs to be used during addons installation.
+func WithPrivateCAs(privateCAs map[string]string) Option {
+	return func(a *Applier) {
+		a.privateCAs = privateCAs
+	}
+}
+
 // Quiet disables logging for addons.
 func Quiet() Option {
 	return func(a *Applier) {

pkg/goods/materializer.go

@@ -10,6 +10,11 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 )
 
+// PlaceHolder is a filename we use in some of the directories here so we can
+// commit them to git. Without having these files the unit tests tend to fail
+// with: "pkg/goods/goods.go:12:13: pattern bins/*: no matching files found".
+const PlaceHolder = ".placeholder"
+
 // Materializer is an entity capable of materialize (write to disk) embedded assets.
 type Materializer struct {
 	def *defaults.Provider
@@ -129,6 +134,10 @@ func (m *Materializer) Binaries() error {
 	}()
 
 	for _, entry := range entries {
+		if entry.Name() == PlaceHolder {
+			continue
+		}
+
 		srcpath := fmt.Sprintf("bins/%s", entry.Name())
 		srcfile, err := binfs.ReadFile(srcpath)
 		if err != nil {