Skip to content

Commit

Permalink
Add support for proxy servers (#676)
Browse files Browse the repository at this point in the history
* Add Proxy support for Embedded Cluster

---------

Co-authored-by: Ricardo Maraschini <ricardo.maraschini@gmail.com>
Co-authored-by: Andrew Lavery <laverya@umich.edu>
  • Loading branch information
3 people authored Jun 17, 2024
1 parent 819d94a commit 80a3425
Show file tree
Hide file tree
Showing 17 changed files with 471 additions and 63 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/pull-request.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ jobs:
run: |
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}-upgrade"
Expand Down Expand Up @@ -220,6 +220,7 @@ jobs:
- TestMultiNodeHAInstallation
- TestMultiNodeAirgapHAInstallation
- TestMultiNodeAirgapUpgradeSameK0s
- TestProxiedEnvironment
- TestMultiNodeHADisasterRecovery
- TestMultiNodeAirgapHADisasterRecovery
include:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
run: |
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}-upgrade"
Expand Down Expand Up @@ -172,6 +172,7 @@ jobs:
- TestMultiNodeHAInstallation
- TestMultiNodeAirgapHAInstallation
- TestMultiNodeAirgapUpgradeSameK0s
- TestProxiedEnvironment
- TestMultiNodeHADisasterRecovery
- TestMultiNodeAirgapHADisasterRecovery
include:
Expand Down
10 changes: 6 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ ADMIN_CONSOLE_IMAGE_OVERRIDE =
ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
EMBEDDED_OPERATOR_CHART_VERSION = 0.35.2
EMBEDDED_OPERATOR_CHART_VERSION = 0.36.0
EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
OPENEBS_CHART_URL = https://openebs.github.io/openebs
Expand All @@ -29,9 +29,11 @@ VELERO_CHART_VERSION = 6.3.0
VELERO_IMAGE_VERSION = v1.13.2
VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
KUBECTL_VERSION = v1.30.1
K0S_VERSION = v1.29.5+k0s.0
K0S_VERSION = v1.29.5+k0s.0-ec.0
K0S_GO_VERSION = v1.29.5+k0s.0
PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
K0S_BINARY_SOURCE_OVERRIDE =
K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
TROUBLESHOOT_VERSION = v0.92.1
KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
KOTS_BINARY_URL_OVERRIDE =
Expand Down Expand Up @@ -134,7 +136,7 @@ embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/
./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster

go.mod: Makefile
go get github.com/k0sproject/k0s@$(K0S_VERSION)
go get github.com/k0sproject/k0s@$(K0S_GO_VERSION)
go mod tidy

.PHONY: static
Expand Down
42 changes: 36 additions & 6 deletions cmd/embedded-cluster/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"time"

k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
Expand Down Expand Up @@ -267,6 +268,9 @@ func ensureK0sConfig(c *cli.Context) error {
if c.Bool("proxy") {
opts = append(opts, addons.WithProxyFromEnv())
}
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
}
if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
return fmt.Errorf("unable to update helm configs: %w", err)
}
Expand Down Expand Up @@ -394,6 +398,9 @@ func runOutro(c *cli.Context) error {
if ab := c.String("airgap-bundle"); ab != "" {
opts = append(opts, addons.WithAirgapBundle(ab))
}
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
}
return addons.NewApplier(opts...).Outro(c.Context)
}

Expand Down Expand Up @@ -434,6 +441,21 @@ var installCommand = &cli.Command{
Usage: "Path to the airgap bundle. If set, the installation will be completed without internet access.",
Hidden: true,
},
&cli.StringFlag{
Name: "http-proxy",
Usage: "HTTP proxy to use for the installation",
Hidden: true,
},
&cli.StringFlag{
Name: "https-proxy",
Usage: "HTTPS proxy to use for the installation",
Hidden: true,
},
&cli.StringFlag{
Name: "no-proxy",
Usage: "Comma separated list of hosts to bypass the proxy for",
Hidden: true,
},
&cli.BoolFlag{
Name: "proxy",
Usage: "Use the system proxy settings for the install operation. These variables are currently only passed through to Velero and the Admin Console.",
Expand Down Expand Up @@ -485,18 +507,26 @@ var installCommand = &cli.Command{
metrics.ReportApplyFinished(c, err)
return err
}
logrus.Debugf("installing k0s")
if err := installK0s(); err != nil {
err := fmt.Errorf("unable update cluster: %w", err)
metrics.ReportApplyFinished(c, err)
return err
var proxy *Proxy
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
proxy = &Proxy{
HTTPProxy: c.String("http-proxy"),
HTTPSProxy: c.String("https-proxy"),
NoProxy: strings.Join(append(defaults.DefaultNoProxy, c.String("no-proxy")), ","),
}
}
logrus.Debugf("creating systemd unit files")
if err := createSystemdUnitFiles(false); err != nil {
if err := createSystemdUnitFiles(false, proxy); err != nil {
err := fmt.Errorf("unable to create systemd unit files: %w", err)
metrics.ReportApplyFinished(c, err)
return err
}
logrus.Debugf("installing k0s")
if err := installK0s(); err != nil {
err := fmt.Errorf("unable update cluster: %w", err)
metrics.ReportApplyFinished(c, err)
return err
}
logrus.Debugf("waiting for k0s to be ready")
if err := waitForK0s(); err != nil {
err := fmt.Errorf("unable to wait for node: %w", err)
Expand Down
45 changes: 15 additions & 30 deletions cmd/embedded-cluster/join.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ import (
"github.com/replicatedhq/embedded-cluster/pkg/spinner"
)

type Proxy struct {
HTTPProxy string `json:"httpProxy"`
HTTPSProxy string `json:"httpsProxy"`
NoProxy string `json:"noProxy"`
}

// JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
type JoinCommandResponse struct {
K0sJoinCommand string `json:"k0sJoinCommand"`
Expand All @@ -42,6 +48,7 @@ type JoinCommandResponse struct {
EndUserK0sConfigOverrides string `json:"endUserK0sConfigOverrides"`
MetricsBaseURL string `json:"metricsBaseURL"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
Proxy *Proxy `json:"proxy"`
}

// extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
Expand Down Expand Up @@ -197,6 +204,14 @@ var joinCommand = &cli.Command{
}
}

logrus.Debugf("creating systemd unit files")
// both controller and worker nodes will have 'worker' in the join command
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy); err != nil {
err := fmt.Errorf("unable to create systemd unit files: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("joining node to cluster")
if err := runK0sInstallCommand(jcmd.K0sJoinCommand); err != nil {
err := fmt.Errorf("unable to join node to cluster: %w", err)
Expand All @@ -211,14 +226,6 @@ var joinCommand = &cli.Command{
return err
}

logrus.Debugf("creating systemd unit files")
// both controller and worker nodes will have 'worker' in the join command
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller")); err != nil {
err := fmt.Errorf("unable to create systemd unit files: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("starting %s service", binName)
if err := startK0sService(); err != nil {
err := fmt.Errorf("unable to start service: %w", err)
Expand Down Expand Up @@ -384,28 +391,6 @@ func systemdUnitFileName() string {
return fmt.Sprintf("/etc/systemd/system/%s.service", defaults.BinaryName())
}

// createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
// systemd unit file for the local artifact mirror service.
func createSystemdUnitFiles(isWorker bool) error {
dst := systemdUnitFileName()
if _, err := os.Lstat(dst); err == nil {
if err := os.Remove(dst); err != nil {
return err
}
}
src := "/etc/systemd/system/k0scontroller.service"
if isWorker {
src = "/etc/systemd/system/k0sworker.service"
}
if err := os.Symlink(src, dst); err != nil {
return fmt.Errorf("failed to create symlink: %w", err)
}
if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
}
return installAndEnableLocalArtifactMirror()
}

// runK0sInstallCommand runs the k0s install command as provided by the kots
// adm api.
func runK0sInstallCommand(fullcmd string) error {
Expand Down
31 changes: 27 additions & 4 deletions cmd/embedded-cluster/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,21 @@ var restoreCommand = &cli.Command{
Usage: "Path to the airgap bundle. If set, the restore will be completed without internet access.",
Hidden: true,
},
&cli.StringFlag{
Name: "http-proxy",
Usage: "HTTP proxy to use for the installation",
Hidden: true,
},
&cli.StringFlag{
Name: "https-proxy",
Usage: "HTTPS proxy to use for the installation",
Hidden: true,
},
&cli.StringFlag{
Name: "no-proxy",
Usage: "Comma separated list of hosts to bypass the proxy for",
Hidden: true,
},
&cli.BoolFlag{
Name: "proxy",
Usage: "Use the system proxy settings for the restore operation. These variables are currently only passed through to Velero.",
Expand Down Expand Up @@ -888,14 +903,22 @@ var restoreCommand = &cli.Command{
if err := ensureK0sConfigForRestore(c); err != nil {
return fmt.Errorf("unable to create config file: %w", err)
}
logrus.Debugf("installing k0s")
if err := installK0s(); err != nil {
return fmt.Errorf("unable update cluster: %w", err)
var proxy *Proxy
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
proxy = &Proxy{
HTTPProxy: c.String("http-proxy"),
HTTPSProxy: c.String("https-proxy"),
NoProxy: strings.Join(append(defaults.DefaultNoProxy, c.String("no-proxy")), ","),
}
}
logrus.Debugf("creating systemd unit files")
if err := createSystemdUnitFiles(false); err != nil {
if err := createSystemdUnitFiles(false, proxy); err != nil {
return fmt.Errorf("unable to create systemd unit files: %w", err)
}
logrus.Debugf("installing k0s")
if err := installK0s(); err != nil {
return fmt.Errorf("unable update cluster: %w", err)
}
logrus.Debugf("waiting for k0s to be ready")
if err := waitForK0s(); err != nil {
return fmt.Errorf("unable to wait for node: %w", err)
Expand Down
14 changes: 14 additions & 0 deletions cmd/embedded-cluster/uninstall.go
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,20 @@ var resetCommand = &cli.Command{
}
}

proxyControllerPath := "/etc/systemd/system/k0scontroller.service.d"
if _, err := os.Stat(proxyControllerPath); err == nil {
if err := os.RemoveAll(proxyControllerPath); err != nil {
return err
}
}

proxyWorkerPath := "/etc/systemd/system/k0sworker.service.d"
if _, err := os.Stat(proxyWorkerPath); err == nil {
if err := os.RemoveAll(proxyWorkerPath); err != nil {
return err
}
}

if _, err := os.Stat(defaults.EmbeddedClusterHomeDirectory()); err == nil {
if err := os.RemoveAll(defaults.EmbeddedClusterHomeDirectory()); err != nil {
return fmt.Errorf("failed to remove embedded cluster home directory: %w", err)
Expand Down
62 changes: 62 additions & 0 deletions cmd/embedded-cluster/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package main

import (
"fmt"
"os"
"path/filepath"

"github.com/replicatedhq/embedded-cluster/pkg/helpers"
)

// createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
// systemd unit file for the local artifact mirror service.
func createSystemdUnitFiles(isWorker bool, proxy *Proxy) error {
dst := systemdUnitFileName()
if _, err := os.Lstat(dst); err == nil {
if err := os.Remove(dst); err != nil {
return err
}
}
src := "/etc/systemd/system/k0scontroller.service"
if isWorker {
src = "/etc/systemd/system/k0sworker.service"
}
if proxy != nil {
ensureProxyConfig(fmt.Sprintf("%s.d", src), proxy.HTTPProxy, proxy.HTTPSProxy, proxy.NoProxy)
}
if err := os.Symlink(src, dst); err != nil {
return fmt.Errorf("failed to create symlink: %w", err)
}

if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
}
return installAndEnableLocalArtifactMirror()
}

// ensureProxyConfig creates a new http-proxy.conf configuration file. The file is saved in the
// systemd directory (/etc/systemd/system/k0scontroller.service.d/).
func ensureProxyConfig(servicePath string, httpProxy string, httpsProxy string, noProxy string) error {
// create the directory
if err := os.MkdirAll(servicePath, 0755); err != nil {
return fmt.Errorf("unable to create directory: %w", err)
}

// create the file
fp, err := os.OpenFile(filepath.Join(servicePath, "http-proxy.conf"), os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return fmt.Errorf("unable to create proxy file: %w", err)
}
defer fp.Close()

// write the file
if _, err := fp.WriteString(fmt.Sprintf(`[Service]
Environment="HTTP_PROXY=%s"
Environment="HTTPS_PROXY=%s"
Environment="NO_PROXY=%s"`,
httpProxy, httpsProxy, noProxy)); err != nil {
return fmt.Errorf("unable to write proxy file: %w", err)
}

return nil
}
2 changes: 1 addition & 1 deletion e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ $ make e2e-tests

### Running individual tests

You an run a single test with:
You can run a single test with:

```
$ make e2e-test TEST_NAME=TestSomething
Expand Down
1 change: 1 addition & 0 deletions e2e/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ func NewTestCluster(in *Input) *Output {
}

const HTTPProxy = "http://10.0.0.254:3128"
const NOProxy = "10.0.0.0/8"

// CreateProxy creates a node that attaches to both networks (external and internal),
// once this is done we install squid and configure it to be a proxy. We also make
Expand Down
Loading

0 comments on commit 80a3425

Please sign in to comment.