Skip to content

Commit 80a3425

Browse files
jdewinnericardomaraschinilaverya
authored
Add support for proxy servers (#676)
* Add Proxy support for Embedded Cluster --------- Co-authored-by: Ricardo Maraschini <[email protected]> Co-authored-by: Andrew Lavery <[email protected]>
1 parent 819d94a commit 80a3425

File tree

17 files changed

+471
-63
lines changed

17 files changed

+471
-63
lines changed

.github/workflows/pull-request.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
run: |
7474
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
7575
export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
76-
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
76+
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
7777
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
7878
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
7979
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}-upgrade"
@@ -220,6 +220,7 @@ jobs:
220220
- TestMultiNodeHAInstallation
221221
- TestMultiNodeAirgapHAInstallation
222222
- TestMultiNodeAirgapUpgradeSameK0s
223+
- TestProxiedEnvironment
223224
- TestMultiNodeHADisasterRecovery
224225
- TestMultiNodeAirgapHADisasterRecovery
225226
include:

.github/workflows/release-dev.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ jobs:
4040
run: |
4141
export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
4242
export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
43-
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
43+
make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
4444
tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
4545
./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
4646
make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}-upgrade"
@@ -172,6 +172,7 @@ jobs:
172172
- TestMultiNodeHAInstallation
173173
- TestMultiNodeAirgapHAInstallation
174174
- TestMultiNodeAirgapUpgradeSameK0s
175+
- TestProxiedEnvironment
175176
- TestMultiNodeHADisasterRecovery
176177
- TestMultiNodeAirgapHADisasterRecovery
177178
include:

Makefile

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_IMAGE_OVERRIDE =
99
ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
1010
EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
1111
EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
12-
EMBEDDED_OPERATOR_CHART_VERSION = 0.35.2
12+
EMBEDDED_OPERATOR_CHART_VERSION = 0.36.0
1313
EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
1414
EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
1515
OPENEBS_CHART_URL = https://openebs.github.io/openebs
@@ -29,9 +29,11 @@ VELERO_CHART_VERSION = 6.3.0
2929
VELERO_IMAGE_VERSION = v1.13.2
3030
VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
3131
KUBECTL_VERSION = v1.30.1
32-
K0S_VERSION = v1.29.5+k0s.0
32+
K0S_VERSION = v1.29.5+k0s.0-ec.0
33+
K0S_GO_VERSION = v1.29.5+k0s.0
3334
PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
34-
K0S_BINARY_SOURCE_OVERRIDE =
35+
K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
36+
PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
3537
TROUBLESHOOT_VERSION = v0.92.1
3638
KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
3739
KOTS_BINARY_URL_OVERRIDE =
@@ -134,7 +136,7 @@ embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/
134136
./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster
135137

136138
go.mod: Makefile
137-
go get github.com/k0sproject/k0s@$(K0S_VERSION)
139+
go get github.com/k0sproject/k0s@$(K0S_GO_VERSION)
138140
go mod tidy
139141

140142
.PHONY: static

cmd/embedded-cluster/install.go

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"os"
66
"path/filepath"
7+
"strings"
78
"time"
89

910
k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
@@ -267,6 +268,9 @@ func ensureK0sConfig(c *cli.Context) error {
267268
if c.Bool("proxy") {
268269
opts = append(opts, addons.WithProxyFromEnv())
269270
}
271+
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
272+
opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
273+
}
270274
if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
271275
return fmt.Errorf("unable to update helm configs: %w", err)
272276
}
@@ -394,6 +398,9 @@ func runOutro(c *cli.Context) error {
394398
if ab := c.String("airgap-bundle"); ab != "" {
395399
opts = append(opts, addons.WithAirgapBundle(ab))
396400
}
401+
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
402+
opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
403+
}
397404
return addons.NewApplier(opts...).Outro(c.Context)
398405
}
399406

@@ -434,6 +441,21 @@ var installCommand = &cli.Command{
434441
Usage: "Path to the airgap bundle. If set, the installation will be completed without internet access.",
435442
Hidden: true,
436443
},
444+
&cli.StringFlag{
445+
Name: "http-proxy",
446+
Usage: "HTTP proxy to use for the installation",
447+
Hidden: true,
448+
},
449+
&cli.StringFlag{
450+
Name: "https-proxy",
451+
Usage: "HTTPS proxy to use for the installation",
452+
Hidden: true,
453+
},
454+
&cli.StringFlag{
455+
Name: "no-proxy",
456+
Usage: "Comma separated list of hosts to bypass the proxy for",
457+
Hidden: true,
458+
},
437459
&cli.BoolFlag{
438460
Name: "proxy",
439461
Usage: "Use the system proxy settings for the install operation. These variables are currently only passed through to Velero and the Admin Console.",
@@ -485,18 +507,26 @@ var installCommand = &cli.Command{
485507
metrics.ReportApplyFinished(c, err)
486508
return err
487509
}
488-
logrus.Debugf("installing k0s")
489-
if err := installK0s(); err != nil {
490-
err := fmt.Errorf("unable update cluster: %w", err)
491-
metrics.ReportApplyFinished(c, err)
492-
return err
510+
var proxy *Proxy
511+
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
512+
proxy = &Proxy{
513+
HTTPProxy: c.String("http-proxy"),
514+
HTTPSProxy: c.String("https-proxy"),
515+
NoProxy: strings.Join(append(defaults.DefaultNoProxy, c.String("no-proxy")), ","),
516+
}
493517
}
494518
logrus.Debugf("creating systemd unit files")
495-
if err := createSystemdUnitFiles(false); err != nil {
519+
if err := createSystemdUnitFiles(false, proxy); err != nil {
496520
err := fmt.Errorf("unable to create systemd unit files: %w", err)
497521
metrics.ReportApplyFinished(c, err)
498522
return err
499523
}
524+
logrus.Debugf("installing k0s")
525+
if err := installK0s(); err != nil {
526+
err := fmt.Errorf("unable update cluster: %w", err)
527+
metrics.ReportApplyFinished(c, err)
528+
return err
529+
}
500530
logrus.Debugf("waiting for k0s to be ready")
501531
if err := waitForK0s(); err != nil {
502532
err := fmt.Errorf("unable to wait for node: %w", err)

cmd/embedded-cluster/join.go

Lines changed: 15 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ import (
3333
"github.com/replicatedhq/embedded-cluster/pkg/spinner"
3434
)
3535

36+
type Proxy struct {
37+
HTTPProxy string `json:"httpProxy"`
38+
HTTPSProxy string `json:"httpsProxy"`
39+
NoProxy string `json:"noProxy"`
40+
}
41+
3642
// JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
3743
type JoinCommandResponse struct {
3844
K0sJoinCommand string `json:"k0sJoinCommand"`
@@ -42,6 +48,7 @@ type JoinCommandResponse struct {
4248
EndUserK0sConfigOverrides string `json:"endUserK0sConfigOverrides"`
4349
MetricsBaseURL string `json:"metricsBaseURL"`
4450
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
51+
Proxy *Proxy `json:"proxy"`
4552
}
4653

4754
// extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
@@ -197,6 +204,14 @@ var joinCommand = &cli.Command{
197204
}
198205
}
199206

207+
logrus.Debugf("creating systemd unit files")
208+
// both controller and worker nodes will have 'worker' in the join command
209+
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy); err != nil {
210+
err := fmt.Errorf("unable to create systemd unit files: %w", err)
211+
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
212+
return err
213+
}
214+
200215
logrus.Debugf("joining node to cluster")
201216
if err := runK0sInstallCommand(jcmd.K0sJoinCommand); err != nil {
202217
err := fmt.Errorf("unable to join node to cluster: %w", err)
@@ -211,14 +226,6 @@ var joinCommand = &cli.Command{
211226
return err
212227
}
213228

214-
logrus.Debugf("creating systemd unit files")
215-
// both controller and worker nodes will have 'worker' in the join command
216-
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller")); err != nil {
217-
err := fmt.Errorf("unable to create systemd unit files: %w", err)
218-
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
219-
return err
220-
}
221-
222229
logrus.Debugf("starting %s service", binName)
223230
if err := startK0sService(); err != nil {
224231
err := fmt.Errorf("unable to start service: %w", err)
@@ -384,28 +391,6 @@ func systemdUnitFileName() string {
384391
return fmt.Sprintf("/etc/systemd/system/%s.service", defaults.BinaryName())
385392
}
386393

387-
// createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
388-
// systemd unit file for the local artifact mirror service.
389-
func createSystemdUnitFiles(isWorker bool) error {
390-
dst := systemdUnitFileName()
391-
if _, err := os.Lstat(dst); err == nil {
392-
if err := os.Remove(dst); err != nil {
393-
return err
394-
}
395-
}
396-
src := "/etc/systemd/system/k0scontroller.service"
397-
if isWorker {
398-
src = "/etc/systemd/system/k0sworker.service"
399-
}
400-
if err := os.Symlink(src, dst); err != nil {
401-
return fmt.Errorf("failed to create symlink: %w", err)
402-
}
403-
if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
404-
return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
405-
}
406-
return installAndEnableLocalArtifactMirror()
407-
}
408-
409394
// runK0sInstallCommand runs the k0s install command as provided by the kots
410395
// adm api.
411396
func runK0sInstallCommand(fullcmd string) error {

cmd/embedded-cluster/restore.go

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,21 @@ var restoreCommand = &cli.Command{
803803
Usage: "Path to the airgap bundle. If set, the restore will be completed without internet access.",
804804
Hidden: true,
805805
},
806+
&cli.StringFlag{
807+
Name: "http-proxy",
808+
Usage: "HTTP proxy to use for the installation",
809+
Hidden: true,
810+
},
811+
&cli.StringFlag{
812+
Name: "https-proxy",
813+
Usage: "HTTPS proxy to use for the installation",
814+
Hidden: true,
815+
},
816+
&cli.StringFlag{
817+
Name: "no-proxy",
818+
Usage: "Comma separated list of hosts to bypass the proxy for",
819+
Hidden: true,
820+
},
806821
&cli.BoolFlag{
807822
Name: "proxy",
808823
Usage: "Use the system proxy settings for the restore operation. These variables are currently only passed through to Velero.",
@@ -888,14 +903,22 @@ var restoreCommand = &cli.Command{
888903
if err := ensureK0sConfigForRestore(c); err != nil {
889904
return fmt.Errorf("unable to create config file: %w", err)
890905
}
891-
logrus.Debugf("installing k0s")
892-
if err := installK0s(); err != nil {
893-
return fmt.Errorf("unable update cluster: %w", err)
906+
var proxy *Proxy
907+
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
908+
proxy = &Proxy{
909+
HTTPProxy: c.String("http-proxy"),
910+
HTTPSProxy: c.String("https-proxy"),
911+
NoProxy: strings.Join(append(defaults.DefaultNoProxy, c.String("no-proxy")), ","),
912+
}
894913
}
895914
logrus.Debugf("creating systemd unit files")
896-
if err := createSystemdUnitFiles(false); err != nil {
915+
if err := createSystemdUnitFiles(false, proxy); err != nil {
897916
return fmt.Errorf("unable to create systemd unit files: %w", err)
898917
}
918+
logrus.Debugf("installing k0s")
919+
if err := installK0s(); err != nil {
920+
return fmt.Errorf("unable update cluster: %w", err)
921+
}
899922
logrus.Debugf("waiting for k0s to be ready")
900923
if err := waitForK0s(); err != nil {
901924
return fmt.Errorf("unable to wait for node: %w", err)

cmd/embedded-cluster/uninstall.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,20 @@ var resetCommand = &cli.Command{
435435
}
436436
}
437437

438+
proxyControllerPath := "/etc/systemd/system/k0scontroller.service.d"
439+
if _, err := os.Stat(proxyControllerPath); err == nil {
440+
if err := os.RemoveAll(proxyControllerPath); err != nil {
441+
return err
442+
}
443+
}
444+
445+
proxyWorkerPath := "/etc/systemd/system/k0sworker.service.d"
446+
if _, err := os.Stat(proxyWorkerPath); err == nil {
447+
if err := os.RemoveAll(proxyWorkerPath); err != nil {
448+
return err
449+
}
450+
}
451+
438452
if _, err := os.Stat(defaults.EmbeddedClusterHomeDirectory()); err == nil {
439453
if err := os.RemoveAll(defaults.EmbeddedClusterHomeDirectory()); err != nil {
440454
return fmt.Errorf("failed to remove embedded cluster home directory: %w", err)

cmd/embedded-cluster/util.go

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
8+
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
9+
)
10+
11+
// createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
12+
// systemd unit file for the local artifact mirror service.
13+
func createSystemdUnitFiles(isWorker bool, proxy *Proxy) error {
14+
dst := systemdUnitFileName()
15+
if _, err := os.Lstat(dst); err == nil {
16+
if err := os.Remove(dst); err != nil {
17+
return err
18+
}
19+
}
20+
src := "/etc/systemd/system/k0scontroller.service"
21+
if isWorker {
22+
src = "/etc/systemd/system/k0sworker.service"
23+
}
24+
if proxy != nil {
25+
ensureProxyConfig(fmt.Sprintf("%s.d", src), proxy.HTTPProxy, proxy.HTTPSProxy, proxy.NoProxy)
26+
}
27+
if err := os.Symlink(src, dst); err != nil {
28+
return fmt.Errorf("failed to create symlink: %w", err)
29+
}
30+
31+
if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
32+
return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
33+
}
34+
return installAndEnableLocalArtifactMirror()
35+
}
36+
37+
// ensureProxyConfig creates a new http-proxy.conf configuration file. The file is saved in the
38+
// systemd directory (/etc/systemd/system/k0scontroller.service.d/).
39+
func ensureProxyConfig(servicePath string, httpProxy string, httpsProxy string, noProxy string) error {
40+
// create the directory
41+
if err := os.MkdirAll(servicePath, 0755); err != nil {
42+
return fmt.Errorf("unable to create directory: %w", err)
43+
}
44+
45+
// create the file
46+
fp, err := os.OpenFile(filepath.Join(servicePath, "http-proxy.conf"), os.O_RDWR|os.O_CREATE, 0644)
47+
if err != nil {
48+
return fmt.Errorf("unable to create proxy file: %w", err)
49+
}
50+
defer fp.Close()
51+
52+
// write the file
53+
if _, err := fp.WriteString(fmt.Sprintf(`[Service]
54+
Environment="HTTP_PROXY=%s"
55+
Environment="HTTPS_PROXY=%s"
56+
Environment="NO_PROXY=%s"`,
57+
httpProxy, httpsProxy, noProxy)); err != nil {
58+
return fmt.Errorf("unable to write proxy file: %w", err)
59+
}
60+
61+
return nil
62+
}

e2e/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ $ make e2e-tests
4242

4343
### Running individual tests
4444

45-
You an run a single test with:
45+
You can run a single test with:
4646

4747
```
4848
$ make e2e-test TEST_NAME=TestSomething

e2e/cluster/cluster.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ func NewTestCluster(in *Input) *Output {
231231
}
232232

233233
const HTTPProxy = "http://10.0.0.254:3128"
234+
const NOProxy = "10.0.0.0/8"
234235

235236
// CreateProxy creates a node that attaches to both networks (external and internal),
236237
// once this is done we install squid and configure it to be a proxy. We also make

0 commit comments

Comments
 (0)