diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a8f7be106..6a8e3e305 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -496,6 +496,7 @@ jobs: - TestCustomCIDR - TestProxiedCustomCIDR - TestSingleNodeInstallationNoopUpgrade + - TestInstallWithPrivateCAs include: - test: TestMultiNodeAirgapUpgrade runner: embedded-cluster diff --git a/.gitignore b/.gitignore index e98bdfb20..661683a0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ output bundle -pkg/goods/bins -pkg/goods/internal/bins +pkg/goods/bins/* +!pkg/goods/bins/.placeholder +pkg/goods/internal/bins/* +!pkg/goods/internal/bins/.placeholder pkg/goods/images *tgz .pre-commit-config.yaml diff --git a/Makefile b/Makefile index dc7a33762..2db28d77c 100644 --- a/Makefile +++ b/Makefile @@ -202,8 +202,8 @@ build-ttl.sh: .PHONY: clean clean: rm -rf output - rm -rf pkg/goods/bins - rm -rf pkg/goods/internal/bins + rm -rf pkg/goods/bins/* + rm -rf pkg/goods/internal/bins/* rm -rf build rm -rf bin diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index cb40e15d3..b187456ab 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -569,6 +569,10 @@ var installCommand = &cli.Command{ Usage: "Skip host preflight checks. This is not recommended.", Value: false, }, + &cli.StringSliceFlag{ + Name: "private-ca", + Usage: "Path to a trusted private CA certificate file", + }, }, )), Action: func(c *cli.Context) error { @@ -665,6 +669,18 @@ func getAddonsApplier(c *cli.Context, adminConsolePwd string) (*addons.Applier, } opts = append(opts, addons.WithEndUserConfig(eucfg)) } + if len(c.StringSlice("private-ca")) > 0 { + privateCAs := map[string]string{} + for i, path := range c.StringSlice("private-ca") { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("unable to read private CA file %s: %w", path, err) + } + name := fmt.Sprintf("ca_%d.crt", i) + privateCAs[name] = string(data) + } + opts = append(opts, addons.WithPrivateCAs(privateCAs)) + } if adminConsolePwd != "" { opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd)) } diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go index 48158ef4c..d8c33b556 100644 --- a/e2e/cluster/cluster.go +++ b/e2e/cluster/cluster.go @@ -593,7 +593,7 @@ func CreateNodes(in *Input) ([]string, []string) { // pinging google.com. func NodeHasInternet(in *Input, node string) { in.T.Logf("Testing if node %s can reach the internet", node) - fp, err := os.CreateTemp("/tmp", "internet-XXXXX.sh") + fp, err := os.CreateTemp("/tmp", "internet-*.sh") if err != nil { in.T.Fatalf("Failed to create temporary file: %v", err) } @@ -643,7 +643,7 @@ func NodeHasInternet(in *Input, node string) { // pinging google.com. func NodeHasNoInternet(in *Input, node string) { in.T.Logf("Ensuring node %s cannot reach the internet", node) - fp, err := os.CreateTemp("/tmp", "internet-XXXXX.sh") + fp, err := os.CreateTemp("/tmp", "internet-*.sh") if err != nil { in.T.Fatalf("Failed to create temporary file: %v", err) } diff --git a/e2e/install_test.go b/e2e/install_test.go index 5cbb0e182..482ec7b7e 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -1,13 +1,18 @@ package e2e import ( + "encoding/json" "fmt" "os" "strings" "testing" "time" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "github.com/replicatedhq/embedded-cluster/e2e/cluster" + "github.com/replicatedhq/embedded-cluster/pkg/certs" ) func TestSingleNodeInstallation(t *testing.T) { @@ -2126,3 +2131,65 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } + +func TestInstallWithPrivateCAs(t *testing.T) { + RequireEnvVars(t, []string{"SHORT_SHA"}) + + input := &cluster.Input{ + T: t, + Nodes: 1, + Image: "ubuntu/jammy", + LicensePath: "license.yaml", + EmbeddedClusterPath: "../output/bin/embedded-cluster", + } + tc := cluster.NewTestCluster(input) + defer cleanupCluster(t, tc) + + certBuilder, err := certs.NewBuilder() + require.NoError(t, err, "unable to create new cert builder") + crtContent, _, err := certBuilder.Generate() + require.NoError(t, err, "unable to build test certificate") + + tmpfile, err := os.CreateTemp("", "test-temp-cert-*.crt") + require.NoError(t, err, "unable to create temp file") + defer os.Remove(tmpfile.Name()) + + _, err = tmpfile.WriteString(crtContent) + require.NoError(t, err, "unable to write to temp file") + tmpfile.Close() + + cluster.CopyFileToNode(input, tc.Nodes[0], cluster.File{ + SourcePath: tmpfile.Name(), + DestPath: "/tmp/ca.crt", + Mode: 0666, + }) + + t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line := []string{"single-node-install.sh", "ui", "--private-ca", "/tmp/ca.crt"} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) + } + + if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v", err) + } + + t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) + line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to check installation state: %v", err) + } + + t.Logf("checking if the configmap was created with the right values") + line = []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} + stdout, _, err := RunCommandOnNode(t, tc, 0, line, WithECShelEnv()) + require.NoError(t, err, "unable get kotsadm-private-cas configmap") + + var cm corev1.ConfigMap + err = json.Unmarshal([]byte(stdout), &cm) + require.NoErrorf(t, err, "unable to unmarshal output to configmap: %q", stdout) + require.Contains(t, cm.Data, "ca_0.crt", "index ca_0.crt not found in ca secret") + require.Equal(t, crtContent, cm.Data["ca_0.crt"], "content mismatch") + + t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) +} diff --git a/e2e/utils.go b/e2e/utils.go index 6c50b16d3..63c9ebb0e 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -32,6 +32,16 @@ func RequireEnvVars(t *testing.T, envVars []string) { type RunCommandOption func(cmd *cluster.Command) +func WithECShelEnv() RunCommandOption { + return func(cmd *cluster.Command) { + cmd.Env = map[string]string{ + "EMBEDDED_CLUSTER_METRICS_BASEURL": "https://staging.replicated.app", + "KUBECONFIG": "/var/lib/k0s/pki/admin.conf", + "PATH": "/var/lib/embedded-cluster/bin", + } + } +} + func WithEnv(env map[string]string) RunCommandOption { return func(cmd *cluster.Command) { cmd.Env = env diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go index f46c4856f..5b966e685 100644 --- a/pkg/addons/adminconsole/adminconsole.go +++ b/pkg/addons/adminconsole/adminconsole.go @@ -33,7 +33,7 @@ import ( ) const ( - releaseName = "admin-console" + ReleaseName = "admin-console" DefaultAdminConsoleNodePort = 30000 ) @@ -86,6 +86,7 @@ type AdminConsole struct { licenseFile string airgapBundle string proxyEnv map[string]string + privateCAs map[string]string } // Version returns the embedded admin console version. @@ -99,7 +100,7 @@ func (a *AdminConsole) Name() string { // GetProtectedFields returns the helm values that are not overwritten when upgrading func (a *AdminConsole) GetProtectedFields() map[string][]string { - return map[string][]string{releaseName: protectedFields} + return map[string][]string{ReleaseName: protectedFields} } // HostPreflights returns the host preflight objects found inside the adminconsole @@ -140,7 +141,7 @@ func (a *AdminConsole) GenerateHelmConfig(k0sCfg *k0sv1beta1.ClusterConfig, only } chartConfig := ecv1beta1.Chart{ - Name: releaseName, + Name: ReleaseName, ChartName: chartName, Version: Metadata.Version, Values: string(values), @@ -172,6 +173,10 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s return fmt.Errorf("unable to create kots password secret: %w", err) } + if err := createKotsCAConfigmap(ctx, cli, a.namespace, a.privateCAs); err != nil { + return fmt.Errorf("unable to create kots CA configmap: %w", err) + } + if a.airgapBundle != "" { err := createRegistrySecret(ctx, cli, a.namespace) if err != nil { @@ -205,13 +210,14 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client, k0sCfg *k0s } // New creates a new AdminConsole object. -func New(ns, password string, licenseFile string, airgapBundle string, proxyEnv map[string]string) (*AdminConsole, error) { +func New(ns, password string, licenseFile string, airgapBundle string, proxyEnv map[string]string, privateCAs map[string]string) (*AdminConsole, error) { return &AdminConsole{ namespace: ns, password: password, licenseFile: licenseFile, airgapBundle: airgapBundle, proxyEnv: proxyEnv, + privateCAs: privateCAs, }, nil } @@ -336,3 +342,29 @@ func createKotsPasswordSecret(ctx context.Context, cli client.Client, namespace return nil } + +func createKotsCAConfigmap(ctx context.Context, cli client.Client, namespace string, cas map[string]string) error { + kotsCAConfigmap := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-private-cas", + Namespace: namespace, + Labels: map[string]string{ + "kots.io/kotsadm": "true", + "replicated.com/disaster-recovery": "infra", + "replicated.com/disaster-recovery-chart": "admin-console", + }, + }, + Data: cas, + } + + err := cli.Create(ctx, &kotsCAConfigmap) + if err != nil { + return fmt.Errorf("unable to create kotsadm-private-cas configmap: %w", err) + } + + return nil +} diff --git a/pkg/addons/adminconsole/static/metadata.yaml b/pkg/addons/adminconsole/static/metadata.yaml index 0d7781ddd..ce211e122 100644 --- a/pkg/addons/adminconsole/static/metadata.yaml +++ b/pkg/addons/adminconsole/static/metadata.yaml @@ -5,7 +5,7 @@ # $ make buildtools # $ output/bin/buildtools update addon # -version: 1.116.0 +version: 1.116.0-build.3 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console images: kotsadm: diff --git a/pkg/addons/adminconsole/static/values.yaml b/pkg/addons/adminconsole/static/values.yaml index ec39fec36..230bd8a61 100644 --- a/pkg/addons/adminconsole/static/values.yaml +++ b/pkg/addons/adminconsole/static/values.yaml @@ -31,3 +31,6 @@ passwordSecretRef: name: kotsadm-password service: enabled: false +privateCAs: + enabled: true + configmapName: "kotsadm-private-cas" diff --git a/pkg/addons/applier.go b/pkg/addons/applier.go index 2262fb957..fc1241fec 100644 --- a/pkg/addons/applier.go +++ b/pkg/addons/applier.go @@ -49,6 +49,7 @@ type Applier struct { endUserConfig *ecv1beta1.Config airgapBundle string proxyEnv map[string]string + privateCAs map[string]string } // Outro runs the outro in all enabled add-ons. @@ -295,7 +296,7 @@ func (a *Applier) load() ([]AddOn, error) { } addons = append(addons, vel) - aconsole, err := adminconsole.New(defaults.KotsadmNamespace, a.adminConsolePwd, a.licenseFile, a.airgapBundle, a.proxyEnv) + aconsole, err := adminconsole.New(defaults.KotsadmNamespace, a.adminConsolePwd, a.licenseFile, a.airgapBundle, a.proxyEnv, a.privateCAs) if err != nil { return nil, fmt.Errorf("unable to create admin console addon: %w", err) } diff --git a/pkg/addons/options.go b/pkg/addons/options.go index 45615a728..2345f7d1a 100644 --- a/pkg/addons/options.go +++ b/pkg/addons/options.go @@ -14,6 +14,13 @@ func WithoutPrompt() Option { } } +// WithPrivateCAs sets the private CAs to be used during addons installation. +func WithPrivateCAs(privateCAs map[string]string) Option { + return func(a *Applier) { + a.privateCAs = privateCAs + } +} + // Quiet disables logging for addons. func Quiet() Option { return func(a *Applier) { diff --git a/pkg/goods/bins/.placeholder b/pkg/goods/bins/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/goods/internal/bins/.placeholder b/pkg/goods/internal/bins/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/goods/materializer.go b/pkg/goods/materializer.go index a6f5764f8..430f141ad 100644 --- a/pkg/goods/materializer.go +++ b/pkg/goods/materializer.go @@ -10,6 +10,11 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/defaults" ) +// PlaceHolder is a filename we use in some of the directories here so we can +// commit them to git. Without having these files the unit tests tend to fail +// with: "pkg/goods/goods.go:12:13: pattern bins/*: no matching files found". +const PlaceHolder = ".placeholder" + // Materializer is an entity capable of materialize (write to disk) embedded assets. type Materializer struct { def *defaults.Provider @@ -129,6 +134,10 @@ func (m *Materializer) Binaries() error { }() for _, entry := range entries { + if entry.Name() == PlaceHolder { + continue + } + srcpath := fmt.Sprintf("bins/%s", entry.Name()) srcfile, err := binfs.ReadFile(srcpath) if err != nil {