From bf5bd9e1495d6906c4c7c20bca2d10215f0e9ff5 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 10:14:14 -0500 Subject: [PATCH 01/30] add a note to readme specifying a folder to place the license --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 338865741..ffe330113 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Additionally, it includes a Registry when deployed in air gap mode. ``` 1. In the Vendor Portal, create and download a license that is assigned to the channel. +We recommend storing this license in the `local-dev/` directory, as it is gitignored and not otherwise used by the CI. 1. Install Embedded Cluster: ```bash From 4e17c8a9f3da6ab0cd75c86c68393d5e3b679f86 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 11:08:23 -0500 Subject: [PATCH 02/30] get default interface --- pkg/netutils/ips.go | 53 ++++++++++++++++++++++++++++++++++++++++ pkg/netutils/ips_test.go | 15 ++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 pkg/netutils/ips.go create mode 100644 pkg/netutils/ips_test.go diff --git a/pkg/netutils/ips.go b/pkg/netutils/ips.go new file mode 100644 index 000000000..5bf2f9ca1 --- /dev/null +++ b/pkg/netutils/ips.go @@ -0,0 +1,53 @@ +package netutils + +import ( + "fmt" + "net" + "strings" + + "github.com/sirupsen/logrus" +) + +// GetDefaultIPAndMask returns the default interface for the node, and the subnet mask for that node, using the same logic as k0s +func GetDefaultIPAndMask() (*net.IPNet, error) { + ifs, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("failed to list network interfaces: %w", err) + } + for _, i := range ifs { + if isPodnetworkInterface(i.Name) { + continue + } + addresses, err := i.Addrs() + if err != nil { + logrus.Debugf("failed to get addresses for interface %s: %s", i.Name, err.Error()) + continue + } + for _, a := range addresses { + // check the address type and skip if loopback + if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet, nil + } + } + } + } + + return nil, fmt.Errorf("failed to find any non-local, non podnetwork ipv4 addresses on host") +} + +func isPodnetworkInterface(name string) bool { + switch { + case name == "vxlan.calico": + return true + case name == "kube-bridge": + return true + case name == "dummyvip0": + return true + case strings.HasPrefix(name, "veth"): + return true + case strings.HasPrefix(name, "cali"): + return true + } + return false +} diff --git a/pkg/netutils/ips_test.go b/pkg/netutils/ips_test.go new file mode 100644 index 000000000..e732938cc --- /dev/null +++ b/pkg/netutils/ips_test.go @@ -0,0 +1,15 @@ +package netutils + +import ( + "fmt" + "github.com/stretchr/testify/require" + "testing" +) + +func TestGetDefaultIPAndMask(t *testing.T) { + req := require.New(t) + got, err := GetDefaultIPAndMask() + req.NoError(err) + fmt.Printf("got network: %s, got ip: %s\n", got.String(), got.IP.String()) + req.NotNil(got) +} From 95b484e89994d52e67e33ba9fe745d4d1a06ddc8 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 11:33:49 -0500 Subject: [PATCH 03/30] wip default noproxies and prompts --- cmd/embedded-cluster/install.go | 5 +++++ cmd/embedded-cluster/proxy.go | 34 +++++++++++++++++++++++++++++++++ e2e/proxy_test.go | 1 - pkg/netutils/ips.go | 4 ++-- pkg/netutils/ips_test.go | 2 +- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index b187456ab..71b4d03a6 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -613,6 +613,11 @@ var installCommand = &cli.Command{ metrics.ReportApplyFinished(c, err) return err } + proxy, err = maybePromptForNoProxy(c, proxy) + if err != nil { + metrics.ReportApplyFinished(c, err) + return err + } logrus.Debugf("materializing binaries") if err := materializeFiles(c); err != nil { metrics.ReportApplyFinished(c, err) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 41e09ecab..2787528d3 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -1,6 +1,10 @@ package main import ( + "fmt" + "github.com/replicatedhq/embedded-cluster/pkg/netutils" + "github.com/replicatedhq/embedded-cluster/pkg/prompts" + "github.com/sirupsen/logrus" "os" "strings" @@ -80,3 +84,33 @@ func setProxyEnv(proxy *ecv1beta1.ProxySpec) { os.Setenv("NO_PROXY", proxy.NoProxy) } } + +func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1beta1.ProxySpec, error) { + if proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { + // if there is a proxy set, then there needs to be a no proxy set + // if it is not set, prompt with a default (the local IP or subnet) + // if it is set, we need to check that it covers the local IP + defaultIPNet, err := netutils.GetDefaultIPNet() + if err != nil { + return nil, fmt.Errorf("failed to get default IPNet: %w", err) + } + if proxy.NoProxy == "" { + if c.Bool("no-prompt") { + logrus.Infof("No proxy not set, using default no proxy %s", defaultIPNet.String()) + proxy.NoProxy = defaultIPNet.String() + return proxy, nil + } else { + logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet (%s) or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", defaultIPNet.String(), defaultIPNet.IP.String()) + newProxy := prompts.New().Input("No proxy:", "", true) + // TODO validate the new no proxy + proxy.NoProxy = newProxy + return proxy, nil + } + } else { + logrus.Infof("A noproxy is already set (%q), checking if it covers the local IP", proxy.NoProxy) + // TODO validate the existing no proxy + return proxy, nil + } + } + return proxy, nil +} diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 0b7658262..7b039f9d8 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -35,7 +35,6 @@ func TestProxiedEnvironment(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } diff --git a/pkg/netutils/ips.go b/pkg/netutils/ips.go index 5bf2f9ca1..75359451e 100644 --- a/pkg/netutils/ips.go +++ b/pkg/netutils/ips.go @@ -8,8 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -// GetDefaultIPAndMask returns the default interface for the node, and the subnet mask for that node, using the same logic as k0s -func GetDefaultIPAndMask() (*net.IPNet, error) { +// GetDefaultIPNet returns the default interface for the node, and the subnet mask for that node, using the same logic as k0s +func GetDefaultIPNet() (*net.IPNet, error) { ifs, err := net.Interfaces() if err != nil { return nil, fmt.Errorf("failed to list network interfaces: %w", err) diff --git a/pkg/netutils/ips_test.go b/pkg/netutils/ips_test.go index e732938cc..e4ed0a0cd 100644 --- a/pkg/netutils/ips_test.go +++ b/pkg/netutils/ips_test.go @@ -8,7 +8,7 @@ import ( func TestGetDefaultIPAndMask(t *testing.T) { req := require.New(t) - got, err := GetDefaultIPAndMask() + got, err := GetDefaultIPNet() req.NoError(err) fmt.Printf("got network: %s, got ip: %s\n", got.String(), got.IP.String()) req.NotNil(got) From 9479324826ce84643f4e71f824e7b18522e605d2 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 11:34:02 -0500 Subject: [PATCH 04/30] disable most tests --- .github/workflows/ci.yaml | 96 +++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1b2140c1a..8d353b009 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -519,56 +519,56 @@ jobs: matrix: test: - TestSingleNodeInstallation - - TestSingleNodeInstallationAlmaLinux8 - - TestSingleNodeInstallationDebian11 - - TestSingleNodeInstallationDebian12 - - TestSingleNodeInstallationCentos9Stream - - TestVersion - - TestHostPreflightCustomSpec - - TestHostPreflightInBuiltSpec - - TestUnsupportedOverrides - - TestMultiNodeInstallation - - TestMultiNodeReset - - TestCommandsRequireSudo - - TestInstallWithoutEmbed - - TestInstallFromReplicatedApp - - TestUpgradeFromReplicatedApp - - TestUpgradeEC18FromReplicatedApp - - TestResetAndReinstall - - TestResetAndReinstallAirgap - - TestCollectSupportBundle - - TestOldVersionUpgrade - - TestMaterialize - - TestLocalArtifactMirror - - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeFromEC18 - - TestSingleNodeAirgapUpgradeCustomCIDR - - TestInstallSnapshotFromReplicatedApp - - TestMultiNodeAirgapUpgrade - - TestSingleNodeDisasterRecovery - - TestSingleNodeDisasterRecoveryWithProxy - - TestSingleNodeResumeDisasterRecovery +# - TestSingleNodeInstallationAlmaLinux8 +# - TestSingleNodeInstallationDebian11 +# - TestSingleNodeInstallationDebian12 +# - TestSingleNodeInstallationCentos9Stream +# - TestVersion +# - TestHostPreflightCustomSpec +# - TestHostPreflightInBuiltSpec +# - TestUnsupportedOverrides +# - TestMultiNodeInstallation +# - TestMultiNodeReset +# - TestCommandsRequireSudo +# - TestInstallWithoutEmbed +# - TestInstallFromReplicatedApp +# - TestUpgradeFromReplicatedApp +# - TestUpgradeEC18FromReplicatedApp +# - TestResetAndReinstall +# - TestResetAndReinstallAirgap +# - TestCollectSupportBundle +# - TestOldVersionUpgrade +# - TestMaterialize +# - TestLocalArtifactMirror +# - TestSingleNodeAirgapUpgrade +# - TestSingleNodeAirgapUpgradeFromEC18 +# - TestSingleNodeAirgapUpgradeCustomCIDR +# - TestInstallSnapshotFromReplicatedApp +# - TestMultiNodeAirgapUpgrade +# - TestSingleNodeDisasterRecovery +# - TestSingleNodeDisasterRecoveryWithProxy +# - TestSingleNodeResumeDisasterRecovery - TestProxiedEnvironment - - TestMultiNodeHAInstallation - - TestMultiNodeHADisasterRecovery - - TestCustomCIDR +# - TestMultiNodeHAInstallation +# - TestMultiNodeHADisasterRecovery +# - TestCustomCIDR - TestProxiedCustomCIDR - - TestSingleNodeInstallationNoopUpgrade - - TestInstallWithPrivateCAs - - TestInstallWithMITMProxy - include: - - test: TestMultiNodeAirgapUpgrade - runner: embedded-cluster - - test: TestMultiNodeAirgapUpgradeSameK0s - runner: embedded-cluster - - test: TestSingleNodeAirgapDisasterRecovery - runner: embedded-cluster - - test: TestMultiNodeAirgapHAInstallation - runner: embedded-cluster - - test: TestMultiNodeAirgapHADisasterRecovery - runner: embedded-cluster - - test: TestFiveNodesAirgapUpgrade - runner: embedded-cluster +# - TestSingleNodeInstallationNoopUpgrade +# - TestInstallWithPrivateCAs +# - TestInstallWithMITMProxy +# include: +# - test: TestMultiNodeAirgapUpgrade +# runner: embedded-cluster +# - test: TestMultiNodeAirgapUpgradeSameK0s +# runner: embedded-cluster +# - test: TestSingleNodeAirgapDisasterRecovery +# runner: embedded-cluster +# - test: TestMultiNodeAirgapHAInstallation +# runner: embedded-cluster +# - test: TestMultiNodeAirgapHADisasterRecovery +# runner: embedded-cluster +# - test: TestFiveNodesAirgapUpgrade +# runner: embedded-cluster steps: - name: Checkout uses: actions/checkout@v4 From e7f5491c04b34e14512339c796abfdb1ca1da38c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 12:00:13 -0500 Subject: [PATCH 05/30] prompt for new noproxy --- cmd/embedded-cluster/proxy.go | 59 +++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 2787528d3..7c8c43a4b 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -5,6 +5,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/netutils" "github.com/replicatedhq/embedded-cluster/pkg/prompts" "github.com/sirupsen/logrus" + "net" "os" "strings" @@ -86,7 +87,7 @@ func setProxyEnv(proxy *ecv1beta1.ProxySpec) { } func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1beta1.ProxySpec, error) { - if proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { + if proxy != nil && (proxy.HTTPProxy != "" || proxy.HTTPSProxy != "") { // if there is a proxy set, then there needs to be a no proxy set // if it is not set, prompt with a default (the local IP or subnet) // if it is set, we need to check that it covers the local IP @@ -96,21 +97,59 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet } if proxy.NoProxy == "" { if c.Bool("no-prompt") { - logrus.Infof("No proxy not set, using default no proxy %s", defaultIPNet.String()) + logrus.Infof("no-proxy was not set, using default no proxy %s", defaultIPNet.String()) proxy.NoProxy = defaultIPNet.String() return proxy, nil } else { - logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet (%s) or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", defaultIPNet.String(), defaultIPNet.IP.String()) - newProxy := prompts.New().Input("No proxy:", "", true) - // TODO validate the new no proxy - proxy.NoProxy = newProxy - return proxy, nil + return promptForNoProxy(proxy, defaultIPNet) } } else { - logrus.Infof("A noproxy is already set (%q), checking if it covers the local IP", proxy.NoProxy) - // TODO validate the existing no proxy - return proxy, nil + shouldPrompt := false + isValid, err := validateNoProxy(proxy.NoProxy, defaultIPNet.IP.String()) + if err != nil { + logrus.Infof("The provided no-proxy %q failed to parse with error %q, falling back to prompt", proxy.NoProxy, err) + shouldPrompt = true + } else if !isValid { + logrus.Infof("The provided no-proxy %q does not cover the local IP %q, falling back to prompt", proxy.NoProxy, defaultIPNet.IP.String()) + shouldPrompt = true + } + if shouldPrompt { + return promptForNoProxy(proxy, defaultIPNet) + } } } return proxy, nil } + +func promptForNoProxy(proxy *ecv1beta1.ProxySpec, defaultIPNet *net.IPNet) (*ecv1beta1.ProxySpec, error) { + logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", defaultIPNet.String(), defaultIPNet.IP.String()) + newProxy := prompts.New().Input("No proxy:", "", true) + isValid, err := validateNoProxy(newProxy, defaultIPNet.IP.String()) + if err != nil { + return nil, fmt.Errorf("failed to validate no-proxy: %w", err) + } + if !isValid { + return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, defaultIPNet.IP.String()) + } + proxy.NoProxy = newProxy + return proxy, nil +} + +func validateNoProxy(newNoProxy string, localIP string) (bool, error) { + foundLocal := false + for _, oneEntry := range strings.Split(newNoProxy, ",") { + if oneEntry == localIP { + foundLocal = true + } else if strings.Contains(oneEntry, "/") { + _, ipnet, err := net.ParseCIDR(oneEntry) + if err != nil { + return false, fmt.Errorf("failed to parse CIDR within no-proxy: %w", err) + } + if ipnet.Contains(net.ParseIP(localIP)) { + foundLocal = true + } + } + } + + return foundLocal, nil +} From 978ca16b9b44475a9ed7f771d26fa0164bdd200e Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 12:14:48 -0500 Subject: [PATCH 06/30] store provided noproxy values and total noproxy values independently --- cmd/embedded-cluster/proxy.go | 31 ++++++++++++++++-------- kinds/apis/v1beta1/installation_types.go | 7 +++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 7c8c43a4b..646eba1bc 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -41,12 +41,12 @@ func withProxyFlags(flags []cli.Flag) []cli.Flag { func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { proxy := &ecv1beta1.ProxySpec{} - var noProxy []string + var providatedNoProxy []string if c.Bool("proxy") { proxy.HTTPProxy = os.Getenv("HTTP_PROXY") proxy.HTTPSProxy = os.Getenv("HTTPS_PROXY") if os.Getenv("NO_PROXY") != "" { - noProxy = append(noProxy, os.Getenv("NO_PROXY")) + providatedNoProxy = append(providatedNoProxy, os.Getenv("NO_PROXY")) } } if c.IsSet("http-proxy") { @@ -56,10 +56,11 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { proxy.HTTPSProxy = c.String("https-proxy") } if c.String("no-proxy") != "" { - noProxy = append(noProxy, c.String("no-proxy")) + providatedNoProxy = append(providatedNoProxy, c.String("no-proxy")) } - if len(noProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { - noProxy = append(defaults.DefaultNoProxy, noProxy...) + proxy.ProvidedNoProxy = strings.Join(providatedNoProxy, ",") + if len(providatedNoProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { + noProxy := append(defaults.DefaultNoProxy, providatedNoProxy...) noProxy = append(noProxy, c.String("pod-cidr"), c.String("service-cidr")) proxy.NoProxy = strings.Join(noProxy, ",") } @@ -69,6 +70,15 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { return proxy } +func regenerateNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) { + noProxy := strings.Split(proxy.ProvidedNoProxy, ",") + if len(noProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { + noProxy = append(defaults.DefaultNoProxy, noProxy...) + noProxy = append(noProxy, c.String("pod-cidr"), c.String("service-cidr")) + proxy.NoProxy = strings.Join(noProxy, ",") + } +} + // setProxyEnv sets the HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables based on the provided ProxySpec. // If the provided ProxySpec is nil, this environment variables are not set. func setProxyEnv(proxy *ecv1beta1.ProxySpec) { @@ -95,13 +105,13 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet if err != nil { return nil, fmt.Errorf("failed to get default IPNet: %w", err) } - if proxy.NoProxy == "" { + if proxy.ProvidedNoProxy == "" { if c.Bool("no-prompt") { logrus.Infof("no-proxy was not set, using default no proxy %s", defaultIPNet.String()) proxy.NoProxy = defaultIPNet.String() return proxy, nil } else { - return promptForNoProxy(proxy, defaultIPNet) + return promptForNoProxy(c, proxy, defaultIPNet) } } else { shouldPrompt := false @@ -114,14 +124,14 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet shouldPrompt = true } if shouldPrompt { - return promptForNoProxy(proxy, defaultIPNet) + return promptForNoProxy(c, proxy, defaultIPNet) } } } return proxy, nil } -func promptForNoProxy(proxy *ecv1beta1.ProxySpec, defaultIPNet *net.IPNet) (*ecv1beta1.ProxySpec, error) { +func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, defaultIPNet *net.IPNet) (*ecv1beta1.ProxySpec, error) { logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", defaultIPNet.String(), defaultIPNet.IP.String()) newProxy := prompts.New().Input("No proxy:", "", true) isValid, err := validateNoProxy(newProxy, defaultIPNet.IP.String()) @@ -131,7 +141,8 @@ func promptForNoProxy(proxy *ecv1beta1.ProxySpec, defaultIPNet *net.IPNet) (*ecv if !isValid { return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, defaultIPNet.IP.String()) } - proxy.NoProxy = newProxy + proxy.ProvidedNoProxy = newProxy + regenerateNoProxy(c, proxy) return proxy, nil } diff --git a/kinds/apis/v1beta1/installation_types.go b/kinds/apis/v1beta1/installation_types.go index f8a707f86..3d6708c05 100644 --- a/kinds/apis/v1beta1/installation_types.go +++ b/kinds/apis/v1beta1/installation_types.go @@ -68,9 +68,10 @@ type ArtifactsLocation struct { // ProxySpec holds the proxy configuration. type ProxySpec struct { - HTTPProxy string `json:"httpProxy,omitempty"` - HTTPSProxy string `json:"httpsProxy,omitempty"` - NoProxy string `json:"noProxy,omitempty"` + HTTPProxy string `json:"httpProxy,omitempty"` + HTTPSProxy string `json:"httpsProxy,omitempty"` + ProvidedNoProxy string `json:"providedNoProxy,omitempty"` + NoProxy string `json:"noProxy,omitempty"` } // NetworkSpec holds the network configuration. From 7030d94829ce6562d45858b75dab0f376f59f202 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 12:16:23 -0500 Subject: [PATCH 07/30] dedup code --- cmd/embedded-cluster/proxy.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 646eba1bc..68784a7f4 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -59,11 +59,7 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { providatedNoProxy = append(providatedNoProxy, c.String("no-proxy")) } proxy.ProvidedNoProxy = strings.Join(providatedNoProxy, ",") - if len(providatedNoProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { - noProxy := append(defaults.DefaultNoProxy, providatedNoProxy...) - noProxy = append(noProxy, c.String("pod-cidr"), c.String("service-cidr")) - proxy.NoProxy = strings.Join(noProxy, ",") - } + regenerateNoProxy(c, proxy) if proxy.HTTPProxy == "" && proxy.HTTPSProxy == "" && proxy.NoProxy == "" { return nil } From b77595d7f8a677f712177a92fcc47f32855b8e74 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 13:19:05 -0500 Subject: [PATCH 08/30] simplify noproxy subnet --- cmd/embedded-cluster/proxy.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 68784a7f4..4ffcb2305 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -101,13 +101,17 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet if err != nil { return nil, fmt.Errorf("failed to get default IPNet: %w", err) } + cleanDefaultIPNet, err := cleanCIDR(defaultIPNet) + if err != nil { + return nil, fmt.Errorf("failed to clean subnet: %w", err) + } if proxy.ProvidedNoProxy == "" { if c.Bool("no-prompt") { - logrus.Infof("no-proxy was not set, using default no proxy %s", defaultIPNet.String()) - proxy.NoProxy = defaultIPNet.String() + logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) + proxy.NoProxy = cleanDefaultIPNet return proxy, nil } else { - return promptForNoProxy(c, proxy, defaultIPNet) + return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) } } else { shouldPrompt := false @@ -120,22 +124,31 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet shouldPrompt = true } if shouldPrompt { - return promptForNoProxy(c, proxy, defaultIPNet) + return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) } } } return proxy, nil } -func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, defaultIPNet *net.IPNet) (*ecv1beta1.ProxySpec, error) { - logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", defaultIPNet.String(), defaultIPNet.IP.String()) +// cleanCIDR returns a `.0/x` subnet instead of a `.2/x` etc subnet +func cleanCIDR(defaultIPNet *net.IPNet) (string, error) { + _, newNet, err := net.ParseCIDR(defaultIPNet.String()) + if err != nil { + return "", fmt.Errorf("failed to parse local inet CIDR %q: %w", defaultIPNet.String(), err) + } + return newNet.String(), nil +} + +func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP string) (*ecv1beta1.ProxySpec, error) { + logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) newProxy := prompts.New().Input("No proxy:", "", true) - isValid, err := validateNoProxy(newProxy, defaultIPNet.IP.String()) + isValid, err := validateNoProxy(newProxy, IP) if err != nil { return nil, fmt.Errorf("failed to validate no-proxy: %w", err) } if !isValid { - return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, defaultIPNet.IP.String()) + return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, IP) } proxy.ProvidedNoProxy = newProxy regenerateNoProxy(c, proxy) From 18c599b557cdac73faede29f24a4045ff53b8393 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 13:19:13 -0500 Subject: [PATCH 09/30] regen CRDs --- .../charts/crds/templates/resources.yaml | 2 ++ .../crd/bases/embeddedcluster.replicated.com_installations.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml index 2e7ce7dfe..f7b4b55d4 100644 --- a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml +++ b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml @@ -526,6 +526,8 @@ spec: type: string noProxy: type: string + providedNoProxy: + type: string type: object type: object status: diff --git a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml index 40d94b30c..641cb5fac 100644 --- a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml +++ b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml @@ -315,6 +315,8 @@ spec: type: string noProxy: type: string + providedNoProxy: + type: string type: object type: object status: From 1f3a106f8aca6642240386fa67013d585024a975 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 14:00:21 -0500 Subject: [PATCH 10/30] update unit tests --- cmd/embedded-cluster/proxy.go | 3 +++ cmd/embedded-cluster/proxy_test.go | 30 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 4ffcb2305..994bdce00 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -67,6 +67,9 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { } func regenerateNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) { + if proxy.ProvidedNoProxy == "" { + return + } noProxy := strings.Split(proxy.ProvidedNoProxy, ",") if len(noProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" { noProxy = append(defaults.DefaultNoProxy, noProxy...) diff --git a/cmd/embedded-cluster/proxy_test.go b/cmd/embedded-cluster/proxy_test.go index ee2abeab6..4b19e4c86 100644 --- a/cmd/embedded-cluster/proxy_test.go +++ b/cmd/embedded-cluster/proxy_test.go @@ -38,9 +38,10 @@ func Test_getProxySpecFromFlags(t *testing.T) { flagSet.Set("proxy", "true") }, want: &ecv1beta1.ProxySpec{ - HTTPProxy: "http://proxy", - HTTPSProxy: "https://proxy", - NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,no-proxy-1,no-proxy-2,10.244.0.0/16,10.96.0.0/12", + HTTPProxy: "http://proxy", + HTTPSProxy: "https://proxy", + ProvidedNoProxy: "no-proxy-1,no-proxy-2", + NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,no-proxy-1,no-proxy-2,10.244.0.0/16,10.96.0.0/12", }, }, { @@ -66,13 +67,14 @@ func Test_getProxySpecFromFlags(t *testing.T) { flagSet.Set("no-proxy", "other-no-proxy-1,other-no-proxy-2") }, want: &ecv1beta1.ProxySpec{ - HTTPProxy: "http://other-proxy", - HTTPSProxy: "https://other-proxy", - NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,other-no-proxy-1,other-no-proxy-2,10.244.0.0/16,10.96.0.0/12", + HTTPProxy: "http://other-proxy", + HTTPSProxy: "https://other-proxy", + ProvidedNoProxy: "other-no-proxy-1,other-no-proxy-2", + NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,other-no-proxy-1,other-no-proxy-2,10.244.0.0/16,10.96.0.0/12", }, }, { - name: "proxy flags should override proxy from env", + name: "proxy flags should override proxy from env, but merge no-proxy", init: func(t *testing.T, flagSet *flag.FlagSet) { t.Setenv("HTTP_PROXY", "http://proxy") t.Setenv("HTTPS_PROXY", "https://proxy") @@ -84,9 +86,10 @@ func Test_getProxySpecFromFlags(t *testing.T) { flagSet.Set("no-proxy", "other-no-proxy-1,other-no-proxy-2") }, want: &ecv1beta1.ProxySpec{ - HTTPProxy: "http://other-proxy", - HTTPSProxy: "https://other-proxy", - NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,no-proxy-1,no-proxy-2,other-no-proxy-1,other-no-proxy-2,10.244.0.0/16,10.96.0.0/12", + HTTPProxy: "http://other-proxy", + HTTPSProxy: "https://other-proxy", + ProvidedNoProxy: "no-proxy-1,no-proxy-2,other-no-proxy-1,other-no-proxy-2", + NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,no-proxy-1,no-proxy-2,other-no-proxy-1,other-no-proxy-2,10.244.0.0/16,10.96.0.0/12", }, }, { @@ -100,9 +103,10 @@ func Test_getProxySpecFromFlags(t *testing.T) { flagSet.Set("service-cidr", "2.2.2.2/24") }, want: &ecv1beta1.ProxySpec{ - HTTPProxy: "http://other-proxy", - HTTPSProxy: "https://other-proxy", - NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,other-no-proxy-1,other-no-proxy-2,1.1.1.1/24,2.2.2.2/24", + HTTPProxy: "http://other-proxy", + HTTPSProxy: "https://other-proxy", + ProvidedNoProxy: "other-no-proxy-1,other-no-proxy-2", + NoProxy: "localhost,127.0.0.1,.cluster.local,.svc,other-no-proxy-1,other-no-proxy-2,1.1.1.1/24,2.2.2.2/24", }, }, } From 1273355e2e4ea380565069c975d79a2e188e57cf Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 14:16:58 -0500 Subject: [PATCH 11/30] fix no-prompt case --- cmd/embedded-cluster/proxy.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 994bdce00..34ff21b98 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -111,7 +111,8 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet if proxy.ProvidedNoProxy == "" { if c.Bool("no-prompt") { logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) - proxy.NoProxy = cleanDefaultIPNet + proxy.ProvidedNoProxy = cleanDefaultIPNet + regenerateNoProxy(c, proxy) return proxy, nil } else { return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) @@ -144,7 +145,7 @@ func cleanCIDR(defaultIPNet *net.IPNet) (string, error) { } func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP string) (*ecv1beta1.ProxySpec, error) { - logrus.Infof("A noproxy is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) + logrus.Infof("A no-proxy address is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) newProxy := prompts.New().Input("No proxy:", "", true) isValid, err := validateNoProxy(newProxy, IP) if err != nil { From 9cadfcd9bc4407aacee4b9da7a778297269ecdb5 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 15:01:40 -0500 Subject: [PATCH 12/30] ? --- cmd/embedded-cluster/proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 34ff21b98..31e70d820 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -113,6 +113,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet regenerateNoProxy(c, proxy) + logrus.Infof("final no-proxy is %q", proxy.ProvidedNoProxy) return proxy, nil } else { return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) From 1d1e1f86e283bac89813135fbf21f710819aa08b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 15:38:33 -0500 Subject: [PATCH 13/30] FFFF --- cmd/embedded-cluster/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 31e70d820..615e73760 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -113,7 +113,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet regenerateNoProxy(c, proxy) - logrus.Infof("final no-proxy is %q", proxy.ProvidedNoProxy) + logrus.Infof("final no-proxy is %q", proxy.NoProxy) return proxy, nil } else { return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) From 0948a2e300e83017922ab08fd5889671092e6c37 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 20:37:03 -0500 Subject: [PATCH 14/30] ??? --- cmd/embedded-cluster/proxy.go | 3 ++- e2e/proxy_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 615e73760..c1ad171ef 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -113,7 +113,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet regenerateNoProxy(c, proxy) - logrus.Infof("final no-proxy is %q", proxy.NoProxy) + logrus.Infof("final (default) no-proxy is %q", proxy.NoProxy) return proxy, nil } else { return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) @@ -157,6 +157,7 @@ func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP str } proxy.ProvidedNoProxy = newProxy regenerateNoProxy(c, proxy) + fmt.Printf("final (provided) no-proxy is %q\n", proxy.NoProxy) return proxy, nil } diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 7b039f9d8..1f1546ede 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -35,6 +35,7 @@ func TestProxiedEnvironment(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) + line = append(line, "--no-proxy", "10.0.0.0/25") // test with half the default subnet, because the proxy is within the subnet otherwise if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } From 3ed457c3a479214a451cb833af0306ff811f7575 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 17 Sep 2024 22:17:42 -0500 Subject: [PATCH 15/30] test bigger noproxy --- e2e/proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 1f1546ede..ee56ac7d6 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -35,7 +35,7 @@ func TestProxiedEnvironment(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", "10.0.0.0/25") // test with half the default subnet, because the proxy is within the subnet otherwise + line = append(line, "--no-proxy", "10.0.0.0/24") // test with the default subnet, to see if the issue is with the autoset code or the noproxy itself if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } From 65cdeaecf849c119588b33632c1bdf117b8304f1 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 09:18:56 -0500 Subject: [PATCH 16/30] reuse proxy object, don't rederive from flags --- cmd/embedded-cluster/install.go | 12 ++++++------ cmd/embedded-cluster/join.go | 2 +- cmd/embedded-cluster/preflights.go | 4 ++-- cmd/embedded-cluster/restore.go | 2 +- e2e/proxy_test.go | 1 - 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index 71b4d03a6..703666dee 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -437,7 +437,7 @@ func waitForK0s() error { } // installAndWaitForK0s installs the k0s binary and waits for it to be ready -func installAndWaitForK0s(c *cli.Context, applier *addons.Applier) (*k0sconfig.ClusterConfig, error) { +func installAndWaitForK0s(c *cli.Context, applier *addons.Applier, proxy *ecv1beta1.ProxySpec) (*k0sconfig.ClusterConfig, error) { loading := spinner.Start() defer loading.Close() loading.Infof("Installing %s node", defaults.BinaryName()) @@ -448,7 +448,6 @@ func installAndWaitForK0s(c *cli.Context, applier *addons.Applier) (*k0sconfig.C metrics.ReportApplyFinished(c, err) return nil, err } - proxy := getProxySpecFromFlags(c) logrus.Debugf("creating systemd unit files") if err := createSystemdUnitFiles(false, proxy); err != nil { err := fmt.Errorf("unable to create systemd unit files: %w", err) @@ -618,12 +617,14 @@ var installCommand = &cli.Command{ metrics.ReportApplyFinished(c, err) return err } + setProxyEnv(proxy) + logrus.Debugf("materializing binaries") if err := materializeFiles(c); err != nil { metrics.ReportApplyFinished(c, err) return err } - applier, err := getAddonsApplier(c, adminConsolePwd) + applier, err := getAddonsApplier(c, adminConsolePwd, proxy) if err != nil { metrics.ReportApplyFinished(c, err) return err @@ -638,7 +639,7 @@ var installCommand = &cli.Command{ metrics.ReportApplyFinished(c, err) return err } - cfg, err := installAndWaitForK0s(c, applier) + cfg, err := installAndWaitForK0s(c, applier, proxy) if err != nil { return err } @@ -652,7 +653,7 @@ var installCommand = &cli.Command{ }, } -func getAddonsApplier(c *cli.Context, adminConsolePwd string) (*addons.Applier, error) { +func getAddonsApplier(c *cli.Context, adminConsolePwd string, proxy *ecv1beta1.ProxySpec) (*addons.Applier, error) { opts := []addons.Option{} if c.Bool("no-prompt") { opts = append(opts, addons.WithoutPrompt()) @@ -663,7 +664,6 @@ func getAddonsApplier(c *cli.Context, adminConsolePwd string) (*addons.Applier, if ab := c.String("airgap-bundle"); ab != "" { opts = append(opts, addons.WithAirgapBundle(ab)) } - proxy := getProxySpecFromFlags(c) if proxy != nil { opts = append(opts, addons.WithProxy(proxy.HTTPProxy, proxy.HTTPSProxy, proxy.NoProxy)) } diff --git a/cmd/embedded-cluster/join.go b/cmd/embedded-cluster/join.go index acfa27fc7..62fe3e205 100644 --- a/cmd/embedded-cluster/join.go +++ b/cmd/embedded-cluster/join.go @@ -207,7 +207,7 @@ var joinCommand = &cli.Command{ return err } - applier, err := getAddonsApplier(c, "") + applier, err := getAddonsApplier(c, "", jcmd.Proxy) if err != nil { metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) return err diff --git a/cmd/embedded-cluster/preflights.go b/cmd/embedded-cluster/preflights.go index 7ea1b2e52..ff8662598 100644 --- a/cmd/embedded-cluster/preflights.go +++ b/cmd/embedded-cluster/preflights.go @@ -56,7 +56,7 @@ var installRunPreflightsCommand = &cli.Command{ return err } - applier, err := getAddonsApplier(c, "") + applier, err := getAddonsApplier(c, "", proxy) if err != nil { return err } @@ -121,7 +121,7 @@ var joinRunPreflightsCommand = &cli.Command{ return err } - applier, err := getAddonsApplier(c, "") + applier, err := getAddonsApplier(c, "", jcmd.Proxy) if err != nil { return err } diff --git a/cmd/embedded-cluster/restore.go b/cmd/embedded-cluster/restore.go index 27b674ee5..2df9cf54d 100644 --- a/cmd/embedded-cluster/restore.go +++ b/cmd/embedded-cluster/restore.go @@ -942,7 +942,7 @@ var restoreCommand = &cli.Command{ } } - applier, err := getAddonsApplier(c, "") + applier, err := getAddonsApplier(c, "", proxy) if err != nil { return err } diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index ee56ac7d6..7b039f9d8 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -35,7 +35,6 @@ func TestProxiedEnvironment(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", "10.0.0.0/24") // test with the default subnet, to see if the issue is with the autoset code or the noproxy itself if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } From 9ae2cc2e7fe0a67e78592a343b89fd853b7ad3ce Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 09:42:52 -0500 Subject: [PATCH 17/30] reorder imports --- cmd/embedded-cluster/proxy.go | 6 +++--- pkg/netutils/ips_test.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index c1ad171ef..415c2ca8b 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -2,15 +2,15 @@ package main import ( "fmt" - "github.com/replicatedhq/embedded-cluster/pkg/netutils" - "github.com/replicatedhq/embedded-cluster/pkg/prompts" - "github.com/sirupsen/logrus" "net" "os" "strings" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" + "github.com/replicatedhq/embedded-cluster/pkg/netutils" + "github.com/replicatedhq/embedded-cluster/pkg/prompts" + "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) diff --git a/pkg/netutils/ips_test.go b/pkg/netutils/ips_test.go index e4ed0a0cd..c415f0533 100644 --- a/pkg/netutils/ips_test.go +++ b/pkg/netutils/ips_test.go @@ -2,8 +2,9 @@ package netutils import ( "fmt" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) func TestGetDefaultIPAndMask(t *testing.T) { From 4b973ba0a33862eb3af707a51a569f562eaf1e55 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 09:43:03 -0500 Subject: [PATCH 18/30] Revert "disable most tests" This reverts commit 9479324826ce84643f4e71f824e7b18522e605d2. --- .github/workflows/ci.yaml | 96 +++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d353b009..1b2140c1a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -519,56 +519,56 @@ jobs: matrix: test: - TestSingleNodeInstallation -# - TestSingleNodeInstallationAlmaLinux8 -# - TestSingleNodeInstallationDebian11 -# - TestSingleNodeInstallationDebian12 -# - TestSingleNodeInstallationCentos9Stream -# - TestVersion -# - TestHostPreflightCustomSpec -# - TestHostPreflightInBuiltSpec -# - TestUnsupportedOverrides -# - TestMultiNodeInstallation -# - TestMultiNodeReset -# - TestCommandsRequireSudo -# - TestInstallWithoutEmbed -# - TestInstallFromReplicatedApp -# - TestUpgradeFromReplicatedApp -# - TestUpgradeEC18FromReplicatedApp -# - TestResetAndReinstall -# - TestResetAndReinstallAirgap -# - TestCollectSupportBundle -# - TestOldVersionUpgrade -# - TestMaterialize -# - TestLocalArtifactMirror -# - TestSingleNodeAirgapUpgrade -# - TestSingleNodeAirgapUpgradeFromEC18 -# - TestSingleNodeAirgapUpgradeCustomCIDR -# - TestInstallSnapshotFromReplicatedApp -# - TestMultiNodeAirgapUpgrade -# - TestSingleNodeDisasterRecovery -# - TestSingleNodeDisasterRecoveryWithProxy -# - TestSingleNodeResumeDisasterRecovery + - TestSingleNodeInstallationAlmaLinux8 + - TestSingleNodeInstallationDebian11 + - TestSingleNodeInstallationDebian12 + - TestSingleNodeInstallationCentos9Stream + - TestVersion + - TestHostPreflightCustomSpec + - TestHostPreflightInBuiltSpec + - TestUnsupportedOverrides + - TestMultiNodeInstallation + - TestMultiNodeReset + - TestCommandsRequireSudo + - TestInstallWithoutEmbed + - TestInstallFromReplicatedApp + - TestUpgradeFromReplicatedApp + - TestUpgradeEC18FromReplicatedApp + - TestResetAndReinstall + - TestResetAndReinstallAirgap + - TestCollectSupportBundle + - TestOldVersionUpgrade + - TestMaterialize + - TestLocalArtifactMirror + - TestSingleNodeAirgapUpgrade + - TestSingleNodeAirgapUpgradeFromEC18 + - TestSingleNodeAirgapUpgradeCustomCIDR + - TestInstallSnapshotFromReplicatedApp + - TestMultiNodeAirgapUpgrade + - TestSingleNodeDisasterRecovery + - TestSingleNodeDisasterRecoveryWithProxy + - TestSingleNodeResumeDisasterRecovery - TestProxiedEnvironment -# - TestMultiNodeHAInstallation -# - TestMultiNodeHADisasterRecovery -# - TestCustomCIDR + - TestMultiNodeHAInstallation + - TestMultiNodeHADisasterRecovery + - TestCustomCIDR - TestProxiedCustomCIDR -# - TestSingleNodeInstallationNoopUpgrade -# - TestInstallWithPrivateCAs -# - TestInstallWithMITMProxy -# include: -# - test: TestMultiNodeAirgapUpgrade -# runner: embedded-cluster -# - test: TestMultiNodeAirgapUpgradeSameK0s -# runner: embedded-cluster -# - test: TestSingleNodeAirgapDisasterRecovery -# runner: embedded-cluster -# - test: TestMultiNodeAirgapHAInstallation -# runner: embedded-cluster -# - test: TestMultiNodeAirgapHADisasterRecovery -# runner: embedded-cluster -# - test: TestFiveNodesAirgapUpgrade -# runner: embedded-cluster + - TestSingleNodeInstallationNoopUpgrade + - TestInstallWithPrivateCAs + - TestInstallWithMITMProxy + include: + - test: TestMultiNodeAirgapUpgrade + runner: embedded-cluster + - test: TestMultiNodeAirgapUpgradeSameK0s + runner: embedded-cluster + - test: TestSingleNodeAirgapDisasterRecovery + runner: embedded-cluster + - test: TestMultiNodeAirgapHAInstallation + runner: embedded-cluster + - test: TestMultiNodeAirgapHADisasterRecovery + runner: embedded-cluster + - test: TestFiveNodesAirgapUpgrade + runner: embedded-cluster steps: - name: Checkout uses: actions/checkout@v4 From bfda1e52321a1da7cb21081e41cdf1bee392e67c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 10:03:51 -0500 Subject: [PATCH 19/30] misspelling --- cmd/embedded-cluster/proxy.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 415c2ca8b..8d38ee702 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -41,12 +41,12 @@ func withProxyFlags(flags []cli.Flag) []cli.Flag { func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { proxy := &ecv1beta1.ProxySpec{} - var providatedNoProxy []string + var providedNoProxy []string if c.Bool("proxy") { proxy.HTTPProxy = os.Getenv("HTTP_PROXY") proxy.HTTPSProxy = os.Getenv("HTTPS_PROXY") if os.Getenv("NO_PROXY") != "" { - providatedNoProxy = append(providatedNoProxy, os.Getenv("NO_PROXY")) + providedNoProxy = append(providedNoProxy, os.Getenv("NO_PROXY")) } } if c.IsSet("http-proxy") { @@ -56,9 +56,9 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { proxy.HTTPSProxy = c.String("https-proxy") } if c.String("no-proxy") != "" { - providatedNoProxy = append(providatedNoProxy, c.String("no-proxy")) + providedNoProxy = append(providedNoProxy, c.String("no-proxy")) } - proxy.ProvidedNoProxy = strings.Join(providatedNoProxy, ",") + proxy.ProvidedNoProxy = strings.Join(providedNoProxy, ",") regenerateNoProxy(c, proxy) if proxy.HTTPProxy == "" && proxy.HTTPSProxy == "" && proxy.NoProxy == "" { return nil From fc36e37a2c7928b7581aab55a57d710e039dd2b8 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 10:14:38 -0500 Subject: [PATCH 20/30] rename to combineNoProxySuppliedValuesAndDefaults, remove debug logs --- cmd/embedded-cluster/proxy.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 8d38ee702..75a1c40e0 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -59,14 +59,14 @@ func getProxySpecFromFlags(c *cli.Context) *ecv1beta1.ProxySpec { providedNoProxy = append(providedNoProxy, c.String("no-proxy")) } proxy.ProvidedNoProxy = strings.Join(providedNoProxy, ",") - regenerateNoProxy(c, proxy) + combineNoProxySuppliedValuesAndDefaults(c, proxy) if proxy.HTTPProxy == "" && proxy.HTTPSProxy == "" && proxy.NoProxy == "" { return nil } return proxy } -func regenerateNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) { +func combineNoProxySuppliedValuesAndDefaults(c *cli.Context, proxy *ecv1beta1.ProxySpec) { if proxy.ProvidedNoProxy == "" { return } @@ -112,8 +112,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet if c.Bool("no-prompt") { logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet - regenerateNoProxy(c, proxy) - logrus.Infof("final (default) no-proxy is %q", proxy.NoProxy) + combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil } else { return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) @@ -156,8 +155,7 @@ func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP str return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, IP) } proxy.ProvidedNoProxy = newProxy - regenerateNoProxy(c, proxy) - fmt.Printf("final (provided) no-proxy is %q\n", proxy.NoProxy) + combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil } From 3fbaf4e72e422328ace313a2e67be38f1cd229dc Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 10:20:06 -0500 Subject: [PATCH 21/30] add link to k0s logic --- pkg/netutils/ips.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/netutils/ips.go b/pkg/netutils/ips.go index 75359451e..8916cbfaa 100644 --- a/pkg/netutils/ips.go +++ b/pkg/netutils/ips.go @@ -8,7 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -// GetDefaultIPNet returns the default interface for the node, and the subnet mask for that node, using the same logic as k0s +// GetDefaultIPNet returns the default interface for the node, and the subnet mask for that node, using the same logic +// as k0s in https://github.com/k0sproject/k0s/blob/v1.30.4%2Bk0s.0/internal/pkg/iface/iface.go#L61 func GetDefaultIPNet() (*net.IPNet, error) { ifs, err := net.Interfaces() if err != nil { From c22cbbff3238c4e9ef3f0b3c160c067e5abf615c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 10:35:44 -0500 Subject: [PATCH 22/30] recursively prompt for noproxy --- cmd/embedded-cluster/proxy.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 75a1c40e0..80c7b930c 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -115,7 +115,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil } else { - return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) + return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String(), 0) } } else { shouldPrompt := false @@ -128,7 +128,7 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet shouldPrompt = true } if shouldPrompt { - return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String()) + return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String(), 0) } } } @@ -144,15 +144,21 @@ func cleanCIDR(defaultIPNet *net.IPNet) (string, error) { return newNet.String(), nil } -func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP string) (*ecv1beta1.ProxySpec, error) { - logrus.Infof("A no-proxy address is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) +func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP string, count int) (*ecv1beta1.ProxySpec, error) { + if count == 0 { + logrus.Infof("A no-proxy address is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) + } else if count >= 3 { + return nil, fmt.Errorf("failed to prompt for no-proxy after 3 attempts") + } newProxy := prompts.New().Input("No proxy:", "", true) isValid, err := validateNoProxy(newProxy, IP) if err != nil { - return nil, fmt.Errorf("failed to validate no-proxy: %w", err) + logrus.Errorf("failed to validate no-proxy: %v", err) + return promptForNoProxy(c, proxy, subnet, IP, count+1) } if !isValid { - return nil, fmt.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, IP) + logrus.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, IP) + return promptForNoProxy(c, proxy, subnet, IP, count+1) } proxy.ProvidedNoProxy = newProxy combineNoProxySuppliedValuesAndDefaults(c, proxy) From 9d6e4cf9886317b10273ee69858b581d7295609a Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 12:46:41 -0500 Subject: [PATCH 23/30] remove prompts for proxy, just add the default subnet if the current IP is not covered --- cmd/embedded-cluster/proxy.go | 47 +++++++---------------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 80c7b930c..2ccded447 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -9,7 +9,6 @@ import ( ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/netutils" - "github.com/replicatedhq/embedded-cluster/pkg/prompts" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -109,26 +108,19 @@ func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1bet return nil, fmt.Errorf("failed to clean subnet: %w", err) } if proxy.ProvidedNoProxy == "" { - if c.Bool("no-prompt") { - logrus.Infof("no-proxy was not set, using default no proxy %s", cleanDefaultIPNet) - proxy.ProvidedNoProxy = cleanDefaultIPNet - combineNoProxySuppliedValuesAndDefaults(c, proxy) - return proxy, nil - } else { - return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String(), 0) - } + logrus.Infof("no-proxy was not set, adding the default interface's subnet %q to the no-proxy", cleanDefaultIPNet) + proxy.ProvidedNoProxy = cleanDefaultIPNet + combineNoProxySuppliedValuesAndDefaults(c, proxy) + return proxy, nil } else { - shouldPrompt := false isValid, err := validateNoProxy(proxy.NoProxy, defaultIPNet.IP.String()) if err != nil { - logrus.Infof("The provided no-proxy %q failed to parse with error %q, falling back to prompt", proxy.NoProxy, err) - shouldPrompt = true + return nil, fmt.Errorf("failed to validate no-proxy: %w", err) } else if !isValid { - logrus.Infof("The provided no-proxy %q does not cover the local IP %q, falling back to prompt", proxy.NoProxy, defaultIPNet.IP.String()) - shouldPrompt = true - } - if shouldPrompt { - return promptForNoProxy(c, proxy, cleanDefaultIPNet, defaultIPNet.IP.String(), 0) + logrus.Infof("The provided no-proxy %q does not cover the local IP %q, adding the default interface's subnet %q to the no-proxy we will use", proxy.NoProxy, defaultIPNet.IP.String(), cleanDefaultIPNet) + proxy.ProvidedNoProxy = cleanDefaultIPNet + combineNoProxySuppliedValuesAndDefaults(c, proxy) + return proxy, nil } } } @@ -144,27 +136,6 @@ func cleanCIDR(defaultIPNet *net.IPNet) (string, error) { return newNet.String(), nil } -func promptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec, subnet, IP string, count int) (*ecv1beta1.ProxySpec, error) { - if count == 0 { - logrus.Infof("A no-proxy address is required when a proxy is set. We suggest either the node subnet %q or the addresses of every node that will be a member of the cluster. The current node's IP address is %q.", subnet, IP) - } else if count >= 3 { - return nil, fmt.Errorf("failed to prompt for no-proxy after 3 attempts") - } - newProxy := prompts.New().Input("No proxy:", "", true) - isValid, err := validateNoProxy(newProxy, IP) - if err != nil { - logrus.Errorf("failed to validate no-proxy: %v", err) - return promptForNoProxy(c, proxy, subnet, IP, count+1) - } - if !isValid { - logrus.Errorf("provided no-proxy %q does not cover the local IP %q", newProxy, IP) - return promptForNoProxy(c, proxy, subnet, IP, count+1) - } - proxy.ProvidedNoProxy = newProxy - combineNoProxySuppliedValuesAndDefaults(c, proxy) - return proxy, nil -} - func validateNoProxy(newNoProxy string, localIP string) (bool, error) { foundLocal := false for _, oneEntry := range strings.Split(newNoProxy, ",") { From 296e592c07a79a3a31e10816ac950ee71c289c4e Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 12:55:42 -0500 Subject: [PATCH 24/30] maybePromptForNoProxy -> includeLocalIPInNoProxy --- cmd/embedded-cluster/install.go | 12 ++++++------ cmd/embedded-cluster/preflights.go | 5 +++++ cmd/embedded-cluster/proxy.go | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index 703666dee..18b062f07 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -575,7 +575,13 @@ var installCommand = &cli.Command{ }, )), Action: func(c *cli.Context) error { + var err error proxy := getProxySpecFromFlags(c) + proxy, err = includeLocalIPInNoProxy(c, proxy) + if err != nil { + metrics.ReportApplyFinished(c, err) + return err + } setProxyEnv(proxy) logrus.Debugf("checking if %s is already installed", binName) @@ -612,12 +618,6 @@ var installCommand = &cli.Command{ metrics.ReportApplyFinished(c, err) return err } - proxy, err = maybePromptForNoProxy(c, proxy) - if err != nil { - metrics.ReportApplyFinished(c, err) - return err - } - setProxyEnv(proxy) logrus.Debugf("materializing binaries") if err := materializeFiles(c); err != nil { diff --git a/cmd/embedded-cluster/preflights.go b/cmd/embedded-cluster/preflights.go index ff8662598..40cd9a395 100644 --- a/cmd/embedded-cluster/preflights.go +++ b/cmd/embedded-cluster/preflights.go @@ -41,7 +41,12 @@ var installRunPreflightsCommand = &cli.Command{ return nil }, Action: func(c *cli.Context) error { + var err error proxy := getProxySpecFromFlags(c) + proxy, err = includeLocalIPInNoProxy(c, proxy) + if err != nil { + return err + } setProxyEnv(proxy) license, err := getLicenseFromFilepath(c.String("license")) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 2ccded447..80d97b2a0 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -94,7 +94,7 @@ func setProxyEnv(proxy *ecv1beta1.ProxySpec) { } } -func maybePromptForNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1beta1.ProxySpec, error) { +func includeLocalIPInNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1beta1.ProxySpec, error) { if proxy != nil && (proxy.HTTPProxy != "" || proxy.HTTPSProxy != "") { // if there is a proxy set, then there needs to be a no proxy set // if it is not set, prompt with a default (the local IP or subnet) From b3459515c510134976afcbbbb3cde4005fd797f0 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 13:09:48 -0500 Subject: [PATCH 25/30] check the noproxy config on node join --- cmd/embedded-cluster/join.go | 7 +++++++ cmd/embedded-cluster/preflights.go | 7 +++++++ cmd/embedded-cluster/proxy.go | 17 +++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/cmd/embedded-cluster/join.go b/cmd/embedded-cluster/join.go index 62fe3e205..974ec033d 100644 --- a/cmd/embedded-cluster/join.go +++ b/cmd/embedded-cluster/join.go @@ -190,6 +190,13 @@ var joinCommand = &cli.Command{ } setProxyEnv(jcmd.Proxy) + proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy) + if err != nil { + return fmt.Errorf("failed to check proxy config for local IP: %w", err) + } + if !proxyOK { + return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP) + } isAirgap := c.String("airgap-bundle") != "" diff --git a/cmd/embedded-cluster/preflights.go b/cmd/embedded-cluster/preflights.go index 40cd9a395..b213e80f2 100644 --- a/cmd/embedded-cluster/preflights.go +++ b/cmd/embedded-cluster/preflights.go @@ -118,6 +118,13 @@ var joinRunPreflightsCommand = &cli.Command{ } setProxyEnv(jcmd.Proxy) + proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy) + if err != nil { + return fmt.Errorf("failed to check proxy config for local IP: %w", err) + } + if !proxyOK { + return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP) + } isAirgap := c.String("airgap-bundle") != "" diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 80d97b2a0..74ea0e0b9 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -154,3 +154,20 @@ func validateNoProxy(newNoProxy string, localIP string) (bool, error) { return foundLocal, nil } + +func checkProxyConfigForLocalIP(proxy *ecv1beta1.ProxySpec) (bool, string, error) { + if proxy == nil { + return true, "", nil // no proxy is fine + } + if proxy.HTTPProxy == "" && proxy.HTTPSProxy == "" { + return true, "", nil // no proxy is fine + } + + defaultIPNet, err := netutils.GetDefaultIPNet() + if err != nil { + return false, "", fmt.Errorf("failed to get default IPNet: %w", err) + } + + ok, err := validateNoProxy(proxy.NoProxy, defaultIPNet.IP.String()) + return ok, defaultIPNet.IP.String(), err +} From 207f65ec5fd1bbf42dcc4f70f65e1af64fbba888 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 13:12:02 -0500 Subject: [PATCH 26/30] test that should fail to ensure that join command preflights are working properly --- .github/workflows/ci.yaml | 94 +++++++++++++++++++-------------------- e2e/proxy_test.go | 2 +- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1b2140c1a..1ec19f67d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -519,56 +519,56 @@ jobs: matrix: test: - TestSingleNodeInstallation - - TestSingleNodeInstallationAlmaLinux8 - - TestSingleNodeInstallationDebian11 - - TestSingleNodeInstallationDebian12 - - TestSingleNodeInstallationCentos9Stream - - TestVersion - - TestHostPreflightCustomSpec - - TestHostPreflightInBuiltSpec - - TestUnsupportedOverrides - - TestMultiNodeInstallation - - TestMultiNodeReset - - TestCommandsRequireSudo - - TestInstallWithoutEmbed - - TestInstallFromReplicatedApp - - TestUpgradeFromReplicatedApp - - TestUpgradeEC18FromReplicatedApp - - TestResetAndReinstall - - TestResetAndReinstallAirgap - - TestCollectSupportBundle - - TestOldVersionUpgrade - - TestMaterialize - - TestLocalArtifactMirror - - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeFromEC18 - - TestSingleNodeAirgapUpgradeCustomCIDR - - TestInstallSnapshotFromReplicatedApp - - TestMultiNodeAirgapUpgrade - - TestSingleNodeDisasterRecovery - - TestSingleNodeDisasterRecoveryWithProxy - - TestSingleNodeResumeDisasterRecovery +# - TestSingleNodeInstallationAlmaLinux8 +# - TestSingleNodeInstallationDebian11 +# - TestSingleNodeInstallationDebian12 +# - TestSingleNodeInstallationCentos9Stream +# - TestVersion +# - TestHostPreflightCustomSpec +# - TestHostPreflightInBuiltSpec +# - TestUnsupportedOverrides +# - TestMultiNodeInstallation +# - TestMultiNodeReset +# - TestCommandsRequireSudo +# - TestInstallWithoutEmbed +# - TestInstallFromReplicatedApp +# - TestUpgradeFromReplicatedApp +# - TestUpgradeEC18FromReplicatedApp +# - TestResetAndReinstall +# - TestResetAndReinstallAirgap +# - TestCollectSupportBundle +# - TestOldVersionUpgrade +# - TestMaterialize +# - TestLocalArtifactMirror +# - TestSingleNodeAirgapUpgrade +# - TestSingleNodeAirgapUpgradeFromEC18 +# - TestSingleNodeAirgapUpgradeCustomCIDR +# - TestInstallSnapshotFromReplicatedApp +# - TestMultiNodeAirgapUpgrade +# - TestSingleNodeDisasterRecovery +# - TestSingleNodeDisasterRecoveryWithProxy +# - TestSingleNodeResumeDisasterRecovery - TestProxiedEnvironment - - TestMultiNodeHAInstallation - - TestMultiNodeHADisasterRecovery - - TestCustomCIDR +# - TestMultiNodeHAInstallation +# - TestMultiNodeHADisasterRecovery +# - TestCustomCIDR - TestProxiedCustomCIDR - - TestSingleNodeInstallationNoopUpgrade - - TestInstallWithPrivateCAs +# - TestSingleNodeInstallationNoopUpgrade +# - TestInstallWithPrivateCAs - TestInstallWithMITMProxy - include: - - test: TestMultiNodeAirgapUpgrade - runner: embedded-cluster - - test: TestMultiNodeAirgapUpgradeSameK0s - runner: embedded-cluster - - test: TestSingleNodeAirgapDisasterRecovery - runner: embedded-cluster - - test: TestMultiNodeAirgapHAInstallation - runner: embedded-cluster - - test: TestMultiNodeAirgapHADisasterRecovery - runner: embedded-cluster - - test: TestFiveNodesAirgapUpgrade - runner: embedded-cluster +# include: +# - test: TestMultiNodeAirgapUpgrade +# runner: embedded-cluster +# - test: TestMultiNodeAirgapUpgradeSameK0s +# runner: embedded-cluster +# - test: TestSingleNodeAirgapDisasterRecovery +# runner: embedded-cluster +# - test: TestMultiNodeAirgapHAInstallation +# runner: embedded-cluster +# - test: TestMultiNodeAirgapHADisasterRecovery +# runner: embedded-cluster +# - test: TestFiveNodesAirgapUpgrade +# runner: embedded-cluster steps: - name: Checkout uses: actions/checkout@v4 diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 7b039f9d8..ea8e64ce1 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -262,7 +262,7 @@ func TestInstallWithMITMProxy(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPMITMProxy) line = append(line, "--https-proxy", cluster.HTTPMITMProxy) - line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) + line = append(line, "--no-proxy", strings.Join(tc.IPs[0:1], ",")) // testing, some nodes will not be in the noproxy list line = append(line, "--private-ca", "/usr/local/share/ca-certificates/proxy/ca.crt") _, _, err := RunCommandOnNode(t, tc, 0, line, withMITMProxyEnv(tc.IPs)) require.NoError(t, err, "failed to install embedded-cluster on node 0") From 561883a3ab81d1190be5a0f0d1ac09aabb117f2e Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 13:23:58 -0500 Subject: [PATCH 27/30] fix 'adding "blah" to the noproxy you provided' message --- cmd/embedded-cluster/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 74ea0e0b9..2e662b48c 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -117,7 +117,7 @@ func includeLocalIPInNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1b if err != nil { return nil, fmt.Errorf("failed to validate no-proxy: %w", err) } else if !isValid { - logrus.Infof("The provided no-proxy %q does not cover the local IP %q, adding the default interface's subnet %q to the no-proxy we will use", proxy.NoProxy, defaultIPNet.IP.String(), cleanDefaultIPNet) + logrus.Infof("The provided no-proxy %q does not cover the local IP %q, adding the default interface's subnet %q to the no-proxy we will use", proxy.ProvidedNoProxy, defaultIPNet.IP.String(), cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil From ebb410b376fee1162c223ddd14d4f89abcd15fd8 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 13:44:12 -0500 Subject: [PATCH 28/30] test failed successfully, restoring it --- e2e/proxy_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index ea8e64ce1..fd493407b 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -262,7 +262,6 @@ func TestInstallWithMITMProxy(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPMITMProxy) line = append(line, "--https-proxy", cluster.HTTPMITMProxy) - line = append(line, "--no-proxy", strings.Join(tc.IPs[0:1], ",")) // testing, some nodes will not be in the noproxy list line = append(line, "--private-ca", "/usr/local/share/ca-certificates/proxy/ca.crt") _, _, err := RunCommandOnNode(t, tc, 0, line, withMITMProxyEnv(tc.IPs)) require.NoError(t, err, "failed to install embedded-cluster on node 0") From 060b3f29373f13e21ee4caf8cbaed770ad1fe57d Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 18 Sep 2024 14:41:19 -0500 Subject: [PATCH 29/30] restore all tests, again --- .github/workflows/ci.yaml | 94 +++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ec19f67d..1b2140c1a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -519,56 +519,56 @@ jobs: matrix: test: - TestSingleNodeInstallation -# - TestSingleNodeInstallationAlmaLinux8 -# - TestSingleNodeInstallationDebian11 -# - TestSingleNodeInstallationDebian12 -# - TestSingleNodeInstallationCentos9Stream -# - TestVersion -# - TestHostPreflightCustomSpec -# - TestHostPreflightInBuiltSpec -# - TestUnsupportedOverrides -# - TestMultiNodeInstallation -# - TestMultiNodeReset -# - TestCommandsRequireSudo -# - TestInstallWithoutEmbed -# - TestInstallFromReplicatedApp -# - TestUpgradeFromReplicatedApp -# - TestUpgradeEC18FromReplicatedApp -# - TestResetAndReinstall -# - TestResetAndReinstallAirgap -# - TestCollectSupportBundle -# - TestOldVersionUpgrade -# - TestMaterialize -# - TestLocalArtifactMirror -# - TestSingleNodeAirgapUpgrade -# - TestSingleNodeAirgapUpgradeFromEC18 -# - TestSingleNodeAirgapUpgradeCustomCIDR -# - TestInstallSnapshotFromReplicatedApp -# - TestMultiNodeAirgapUpgrade -# - TestSingleNodeDisasterRecovery -# - TestSingleNodeDisasterRecoveryWithProxy -# - TestSingleNodeResumeDisasterRecovery + - TestSingleNodeInstallationAlmaLinux8 + - TestSingleNodeInstallationDebian11 + - TestSingleNodeInstallationDebian12 + - TestSingleNodeInstallationCentos9Stream + - TestVersion + - TestHostPreflightCustomSpec + - TestHostPreflightInBuiltSpec + - TestUnsupportedOverrides + - TestMultiNodeInstallation + - TestMultiNodeReset + - TestCommandsRequireSudo + - TestInstallWithoutEmbed + - TestInstallFromReplicatedApp + - TestUpgradeFromReplicatedApp + - TestUpgradeEC18FromReplicatedApp + - TestResetAndReinstall + - TestResetAndReinstallAirgap + - TestCollectSupportBundle + - TestOldVersionUpgrade + - TestMaterialize + - TestLocalArtifactMirror + - TestSingleNodeAirgapUpgrade + - TestSingleNodeAirgapUpgradeFromEC18 + - TestSingleNodeAirgapUpgradeCustomCIDR + - TestInstallSnapshotFromReplicatedApp + - TestMultiNodeAirgapUpgrade + - TestSingleNodeDisasterRecovery + - TestSingleNodeDisasterRecoveryWithProxy + - TestSingleNodeResumeDisasterRecovery - TestProxiedEnvironment -# - TestMultiNodeHAInstallation -# - TestMultiNodeHADisasterRecovery -# - TestCustomCIDR + - TestMultiNodeHAInstallation + - TestMultiNodeHADisasterRecovery + - TestCustomCIDR - TestProxiedCustomCIDR -# - TestSingleNodeInstallationNoopUpgrade -# - TestInstallWithPrivateCAs + - TestSingleNodeInstallationNoopUpgrade + - TestInstallWithPrivateCAs - TestInstallWithMITMProxy -# include: -# - test: TestMultiNodeAirgapUpgrade -# runner: embedded-cluster -# - test: TestMultiNodeAirgapUpgradeSameK0s -# runner: embedded-cluster -# - test: TestSingleNodeAirgapDisasterRecovery -# runner: embedded-cluster -# - test: TestMultiNodeAirgapHAInstallation -# runner: embedded-cluster -# - test: TestMultiNodeAirgapHADisasterRecovery -# runner: embedded-cluster -# - test: TestFiveNodesAirgapUpgrade -# runner: embedded-cluster + include: + - test: TestMultiNodeAirgapUpgrade + runner: embedded-cluster + - test: TestMultiNodeAirgapUpgradeSameK0s + runner: embedded-cluster + - test: TestSingleNodeAirgapDisasterRecovery + runner: embedded-cluster + - test: TestMultiNodeAirgapHAInstallation + runner: embedded-cluster + - test: TestMultiNodeAirgapHADisasterRecovery + runner: embedded-cluster + - test: TestFiveNodesAirgapUpgrade + runner: embedded-cluster steps: - name: Checkout uses: actions/checkout@v4 From eac45730974f9837d889a24ceccfb9b8daaad945 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 19 Sep 2024 08:44:24 -0500 Subject: [PATCH 30/30] Accept Alex's suggestions for noproxy messaging Co-authored-by: Alex Parker <7272359+ajp-io@users.noreply.github.com> --- cmd/embedded-cluster/proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/embedded-cluster/proxy.go b/cmd/embedded-cluster/proxy.go index 2e662b48c..bb0bb34dc 100644 --- a/cmd/embedded-cluster/proxy.go +++ b/cmd/embedded-cluster/proxy.go @@ -108,7 +108,7 @@ func includeLocalIPInNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1b return nil, fmt.Errorf("failed to clean subnet: %w", err) } if proxy.ProvidedNoProxy == "" { - logrus.Infof("no-proxy was not set, adding the default interface's subnet %q to the no-proxy", cleanDefaultIPNet) + logrus.Infof("--no-proxy was not set. Adding the default interface's subnet (%q) to the no-proxy list.", cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil @@ -117,7 +117,7 @@ func includeLocalIPInNoProxy(c *cli.Context, proxy *ecv1beta1.ProxySpec) (*ecv1b if err != nil { return nil, fmt.Errorf("failed to validate no-proxy: %w", err) } else if !isValid { - logrus.Infof("The provided no-proxy %q does not cover the local IP %q, adding the default interface's subnet %q to the no-proxy we will use", proxy.ProvidedNoProxy, defaultIPNet.IP.String(), cleanDefaultIPNet) + logrus.Infof("The node IP (%q) is not included in the provided no-proxy list (%q). Adding the default interface's subnet (%q) to the no-proxy list.", defaultIPNet.IP.String(), proxy.ProvidedNoProxy, cleanDefaultIPNet) proxy.ProvidedNoProxy = cleanDefaultIPNet combineNoProxySuppliedValuesAndDefaults(c, proxy) return proxy, nil