From 2fa69ea326fa44e3db445fd21faa8d33e3e69bef Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Sep 2024 13:49:07 -0400 Subject: [PATCH 1/3] test with a noproxy that doesn't include every private IP --- e2e/cluster/cluster.go | 26 ++++++++++++++++++++------ e2e/proxy_test.go | 4 ++-- e2e/restore_test.go | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go index 6777f5523..62e7e95cf 100644 --- a/e2e/cluster/cluster.go +++ b/e2e/cluster/cluster.go @@ -84,6 +84,7 @@ type Dir struct { // names and the cluster id. type Output struct { Nodes []string + IPs []string network string id string T *testing.T @@ -216,7 +217,7 @@ func NewTestCluster(in *Input) *Output { } CreateProfile(in) CreateNetworks(in) - out.Nodes = CreateNodes(in) + out.Nodes, out.IPs = CreateNodes(in) for _, node := range out.Nodes { CopyFilesToNode(in, node) CopyDirsToNode(in, node) @@ -573,18 +574,20 @@ func CopyFileFromNode(node, source, dest string) error { // CreateNodes creats the nodes for the cluster. The amount of nodes is // specified in the input. -func CreateNodes(in *Input) []string { +func CreateNodes(in *Input) ([]string, []string) { nodes := []string{} + IPs := []string{} for i := 0; i < in.Nodes; i++ { - node := CreateNode(in, i) + node, ip := CreateNode(in, i) if !in.WithProxy { NodeHasInternet(in, node) } else { NodeHasNoInternet(in, node) } nodes = append(nodes, node) + IPs = append(IPs, ip) } - return nodes + return nodes, IPs } // NodeHasInternet checks if the node has internet access. It does this by @@ -687,7 +690,7 @@ func NodeHasNoInternet(in *Input, node string) { // CreateNode creates a single node. The i here is used to create a unique // name for the node. Node is named as "node--". The node // name is returned. -func CreateNode(in *Input, i int) string { +func CreateNode(in *Input, i int) (string, string) { client, err := lxd.ConnectLXDUnix(lxdSocket, nil) if err != nil { in.T.Fatalf("Failed to connect to LXD: %v", err) @@ -736,7 +739,18 @@ func CreateNode(in *Input, i int) string { in.T.Fatalf("Failed to get node state %s: %v", name, err) } } - return name + ip := "" + for key, netState := range state.Network { + for _, addr := range netState.Addresses { + fmt.Printf("key: %s Family: %s IP: %s\n", key, addr.Family, addr.Address) + if addr.Family == "inet" { + ip = addr.Address + break + } + } + } + + return name, ip } // CreateNetworks create two networks, one of type bridge and inside of it another one of diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index af6ac711d..10a643b3f 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -33,7 +33,7 @@ func TestProxiedEnvironment(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", cluster.NOProxy) + line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } @@ -130,7 +130,7 @@ func TestProxiedCustomCIDR(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", cluster.NOProxy) + line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) line = append(line, "--pod-cidr", "10.128.0.0/20") line = append(line, "--service-cidr", "10.129.0.0/20") if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { diff --git a/e2e/restore_test.go b/e2e/restore_test.go index 724eeb378..397a27bb9 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -114,7 +114,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { line := []string{"single-node-install.sh", "ui"} line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", cluster.NOProxy) + line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } @@ -143,7 +143,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { line = append([]string{"restore-installation.exp"}, testArgs...) line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) - line = append(line, "--no-proxy", cluster.NOProxy) + line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { t.Fatalf("fail to restore the installation: %v", err) } From a5d7686c4b10ffe4faab0a87e91848732db3c328 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Sep 2024 14:31:16 -0400 Subject: [PATCH 2/3] count only eth0 ips, not loopback --- e2e/cluster/cluster.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go index 62e7e95cf..38dd72799 100644 --- a/e2e/cluster/cluster.go +++ b/e2e/cluster/cluster.go @@ -740,13 +740,11 @@ func CreateNode(in *Input, i int) (string, string) { } } ip := "" - for key, netState := range state.Network { - for _, addr := range netState.Addresses { - fmt.Printf("key: %s Family: %s IP: %s\n", key, addr.Family, addr.Address) - if addr.Family == "inet" { - ip = addr.Address - break - } + for _, addr := range state.Network["eth0"].Addresses { + fmt.Printf("Family: %s IP: %s\n", addr.Family, addr.Address) + if addr.Family == "inet" { + ip = addr.Address + break } } From fbc6f3cc2cc9344acc71c5767ce264ad54a2d70f Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 5 Sep 2024 15:24:10 -0400 Subject: [PATCH 3/3] remove all-private-ips noproxy constant --- e2e/cluster/cluster.go | 1 - e2e/proxy_test.go | 4 ++-- e2e/restore_test.go | 20 ++++++++++---------- e2e/utils.go | 4 ++-- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go index 38dd72799..48158ef4c 100644 --- a/e2e/cluster/cluster.go +++ b/e2e/cluster/cluster.go @@ -252,7 +252,6 @@ func NewTestCluster(in *Input) *Output { const ProxyImage = "debian/12" const HTTPProxy = "http://10.0.0.254:3128" -const NOProxy = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" // CreateProxy creates a node that attaches to both networks (external and internal), // once this is done we install squid and configure it to be a proxy. We also make diff --git a/e2e/proxy_test.go b/e2e/proxy_test.go index 10a643b3f..f1e60d8c7 100644 --- a/e2e/proxy_test.go +++ b/e2e/proxy_test.go @@ -34,7 +34,7 @@ func TestProxiedEnvironment(t *testing.T) { line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } @@ -133,7 +133,7 @@ func TestProxiedCustomCIDR(t *testing.T) { line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) line = append(line, "--pod-cidr", "10.128.0.0/20") line = append(line, "--service-cidr", "10.129.0.0/20") - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } diff --git a/e2e/restore_test.go b/e2e/restore_test.go index 397a27bb9..300de4a9e 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -115,7 +115,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } @@ -125,7 +125,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to check installation state: %v", err) } @@ -144,13 +144,13 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) { line = append(line, "--http-proxy", cluster.HTTPProxy) line = append(line, "--https-proxy", cluster.HTTPProxy) line = append(line, "--no-proxy", strings.Join(tc.IPs, ",")) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore the installation: %v", err) } t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to check installation state: %v", err) } @@ -220,7 +220,7 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) { t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to check installation state: %v", err) } @@ -284,7 +284,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { line = []string{"single-node-airgap-install.sh", "--proxy"} line = append(line, "--pod-cidr", "10.128.0.0/20") line = append(line, "--service-cidr", "10.129.0.0/20") - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } if _, _, err := setupPlaywrightAndRunTest(t, tc, "deploy-app"); err != nil { @@ -316,7 +316,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339)) testArgs = append(testArgs, "--pod-cidr", "10.128.0.0/20", "--service-cidr", "10.129.0.0/20") line = append([]string{"restore-installation-airgap.exp"}, testArgs...) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore the installation: %v", err) } t.Logf("%s: checking installation state after restoring app", time.Now().Format(time.RFC3339)) @@ -594,7 +594,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line = []string{"single-node-airgap-install.sh", "--proxy"} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } @@ -685,7 +685,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { // begin restoring the cluster t.Logf("%s: restoring the installation: phase 1", time.Now().Format(time.RFC3339)) line = append([]string{"restore-multi-node-airgap-phase1.exp"}, testArgs...) - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore phase 1 of the installation: %v", err) } @@ -734,7 +734,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) { t.Logf("%s: restoring the installation: phase 2", time.Now().Format(time.RFC3339)) line = []string{"restore-multi-node-airgap-phase2.exp"} - if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv()); err != nil { + if _, _, err := RunCommandOnNode(t, tc, 0, line, withProxyEnv(tc.IPs)); err != nil { t.Fatalf("fail to restore phase 2 of the installation: %v", err) } diff --git a/e2e/utils.go b/e2e/utils.go index b41ec7244..6c50b16d3 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -211,10 +211,10 @@ func installTestDependenciesDebian(t *testing.T, tc *cluster.Output, node int, w } } -func withProxyEnv() RunCommandOption { +func withProxyEnv(nodeIPs []string) RunCommandOption { return WithEnv(map[string]string{ "HTTP_PROXY": cluster.HTTPProxy, "HTTPS_PROXY": cluster.HTTPProxy, - "NO_PROXY": cluster.NOProxy, + "NO_PROXY": strings.Join(nodeIPs, ","), }) }