Skip to content

Commit ea3e5e2

Browse files
emosbaughemosbaugh
authored
feat(velero): add support for mitm proxy (#2170)
* feat(velero): add support for mitm proxy * f * f * f * f * f * f * f * velero use ca from host * http proxy dryrun test * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f
1 parent 345a39e commit ea3e5e2

File tree

25 files changed

+766
-97
lines changed

25 files changed

+766
-97
lines changed

‎cmd/installer/cli/ca.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ func findHostCABundle() (string, error) {
3636

3737
// Check each file in the order of preference returning the first found
3838
for _, file := range certFiles {
39+
// Ignore all errors to replicate the behavior of the Go standard library
40+
// https://github.com/golang/go/blob/go1.24.3/src/crypto/x509/root_unix.go#L47-L81
3941
if _, err := os.Stat(file); err == nil {
4042
return file, nil
4143
}

‎cmd/installer/cli/install.go‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -239,9 +239,17 @@ func preRunInstall(cmd *cobra.Command, flags *InstallCmdFlags) error {
239239
flags.isAirgap = flags.airgapBundle != ""
240240

241241
runtimeconfig.ApplyFlags(cmd.Flags())
242+
242243
os.Setenv("KUBECONFIG", runtimeconfig.PathToKubeConfig()) // this is needed for restore as well since it shares this function
243244
os.Setenv("TMPDIR", runtimeconfig.EmbeddedClusterTmpSubDir())
244245

246+
hostCABundlePath, err := findHostCABundle()
247+
if err != nil {
248+
return fmt.Errorf("unable to find host CA bundle: %w", err)
249+
}
250+
runtimeconfig.SetHostCABundlePath(hostCABundlePath)
251+
logrus.Debugf("using host CA bundle: %s", hostCABundlePath)
252+
245253
if err := runtimeconfig.WriteToDisk(); err != nil {
246254
return fmt.Errorf("unable to write runtime config to disk: %w", err)
247255
}
@@ -256,12 +264,6 @@ func preRunInstall(cmd *cobra.Command, flags *InstallCmdFlags) error {
256264
}
257265

258266
func runInstall(ctx context.Context, name string, flags InstallCmdFlags, metricsReporter preflights.MetricsReporter) error {
259-
hostCABundle, err := findHostCABundle()
260-
if err != nil {
261-
return fmt.Errorf("unable to find host CA bundle: %w", err)
262-
}
263-
logrus.Debugf("using host CA bundle: %s", hostCABundle)
264-
265267
if err := runInstallVerifyAndPrompt(ctx, name, &flags); err != nil {
266268
return err
267269
}
@@ -351,7 +353,7 @@ func runInstall(ctx context.Context, name string, flags InstallCmdFlags, metrics
351353
License: flags.license,
352354
IsAirgap: flags.airgapBundle != "",
353355
Proxy: flags.proxy,
354-
HostCABundle: hostCABundle,
356+
HostCABundlePath: runtimeconfig.HostCABundlePath(),
355357
PrivateCAs: flags.privateCAs,
356358
ServiceCIDR: flags.cidrCfg.ServiceCIDR,
357359
DisasterRecoveryEnabled: flags.license.Spec.IsDisasterRecoverySupported,
@@ -1032,7 +1034,10 @@ func waitForNode(ctx context.Context) error {
10321034
return nil
10331035
}
10341036

1035-
func recordInstallation(ctx context.Context, kcli client.Client, flags InstallCmdFlags, k0sCfg *k0sv1beta1.ClusterConfig, license *kotsv1beta1.License) (*ecv1beta1.Installation, error) {
1037+
func recordInstallation(
1038+
ctx context.Context, kcli client.Client, flags InstallCmdFlags,
1039+
k0sCfg *k0sv1beta1.ClusterConfig, license *kotsv1beta1.License,
1040+
) (*ecv1beta1.Installation, error) {
10361041
// ensure that the embedded-cluster namespace exists
10371042
if err := createECNamespace(ctx, kcli); err != nil {
10381043
return nil, fmt.Errorf("create embedded-cluster namespace: %w", err)

‎cmd/installer/cli/restore.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -364,12 +364,6 @@ func runRestoreStepNew(ctx context.Context, name string, flags InstallCmdFlags,
364364
}
365365
}
366366

367-
hostCABundle, err := findHostCABundle()
368-
if err != nil {
369-
return fmt.Errorf("unable to find host CA bundle: %w", err)
370-
}
371-
logrus.Debugf("using host CA bundle: %s", hostCABundle)
372-
373367
logrus.Debugf("configuring sysctl")
374368
if err := configutils.ConfigureSysctl(); err != nil {
375369
logrus.Debugf("unable to configure sysctl: %v", err)
@@ -443,7 +437,7 @@ func runRestoreStepNew(ctx context.Context, name string, flags InstallCmdFlags,
443437
if err := addons.Install(ctx, hcli, addons.InstallOptions{
444438
IsAirgap: flags.airgapBundle != "",
445439
Proxy: flags.proxy,
446-
HostCABundle: hostCABundle,
440+
HostCABundlePath: runtimeconfig.HostCABundlePath(),
447441
PrivateCAs: flags.privateCAs,
448442
ServiceCIDR: flags.cidrCfg.ServiceCIDR,
449443
IsRestore: true,

‎e2e/proxy_test.go‎

Lines changed: 152 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,22 @@ func TestProxiedEnvironment(t *testing.T) {
2828
t.Skip("skipping test for k0s versions < 1.29.0")
2929
}
3030

31+
requiredEnvVars := []string{
32+
"DR_S3_ENDPOINT",
33+
"DR_S3_REGION",
34+
"DR_S3_BUCKET",
35+
"DR_S3_PREFIX",
36+
"DR_ACCESS_KEY_ID",
37+
"DR_SECRET_ACCESS_KEY",
38+
}
39+
RequireEnvVars(t, requiredEnvVars)
40+
3141
tc := lxd.NewCluster(&lxd.ClusterInput{
3242
T: t,
3343
Nodes: 4,
3444
WithProxy: true,
3545
Image: "debian/12",
36-
LicensePath: "licenses/license.yaml",
46+
LicensePath: "licenses/snapshot-license.yaml",
3747
EmbeddedClusterPath: "../output/bin/embedded-cluster",
3848
})
3949
defer tc.Cleanup()
@@ -95,16 +105,78 @@ func TestProxiedEnvironment(t *testing.T) {
95105
// check the installation state
96106
checkInstallationState(t, tc)
97107

108+
testArgs := []string{}
109+
for _, envVar := range requiredEnvVars {
110+
testArgs = append(testArgs, os.Getenv(envVar))
111+
}
112+
113+
if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil {
114+
t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr)
115+
}
116+
98117
appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
99-
testArgs := []string{appUpgradeVersion}
100118

101119
t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
102-
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
120+
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil {
103121
t.Fatalf("fail to run playwright test deploy-app: %v", err)
104122
}
105123

106124
checkPostUpgradeState(t, tc)
107125

126+
// reset the cluster
127+
runInParallel(t,
128+
func(t *testing.T) error {
129+
stdout, stderr, err := resetInstallationWithError(t, tc, 3, resetInstallationOptions{force: true})
130+
if err != nil {
131+
return fmt.Errorf("fail to reset the installation on node 3: %v: %s: %s", err, stdout, stderr)
132+
}
133+
return nil
134+
}, func(t *testing.T) error {
135+
stdout, stderr, err := resetInstallationWithError(t, tc, 2, resetInstallationOptions{force: true})
136+
if err != nil {
137+
return fmt.Errorf("fail to reset the installation on node 2: %v: %s: %s", err, stdout, stderr)
138+
}
139+
return nil
140+
}, func(t *testing.T) error {
141+
stdout, stderr, err := resetInstallationWithError(t, tc, 1, resetInstallationOptions{force: true})
142+
if err != nil {
143+
return fmt.Errorf("fail to reset the installation on node 1: %v: %s: %s", err, stdout, stderr)
144+
}
145+
return nil
146+
}, func(t *testing.T) error {
147+
stdout, stderr, err := resetInstallationWithError(t, tc, 0, resetInstallationOptions{force: true})
148+
if err != nil {
149+
return fmt.Errorf("fail to reset the installation on node 0: %v: %s: %s", err, stdout, stderr)
150+
}
151+
return nil
152+
},
153+
)
154+
155+
t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
156+
time.Sleep(30 * time.Second)
157+
158+
t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
159+
line = append([]string{"restore-installation.exp"}, testArgs...)
160+
line = append(line, "--http-proxy", lxd.HTTPProxy)
161+
line = append(line, "--https-proxy", lxd.HTTPProxy)
162+
line = append(line, "--no-proxy", strings.Join(tc.IPs, ","))
163+
if _, _, err := tc.RunCommandOnNode(0, line, lxd.WithProxyEnv(tc.IPs)); err != nil {
164+
t.Fatalf("fail to restore the installation: %v", err)
165+
}
166+
167+
checkInstallationState(t, tc)
168+
169+
t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
170+
line = []string{"check-post-restore.sh"}
171+
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
172+
t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
173+
}
174+
175+
t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
176+
if _, _, err := tc.SetupPlaywrightAndRunTest("validate-restore-app"); err != nil {
177+
t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
178+
}
179+
108180
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
109181
}
110182

@@ -194,10 +266,9 @@ func TestProxiedCustomCIDR(t *testing.T) {
194266
}
195267

196268
appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
197-
testArgs := []string{appUpgradeVersion}
198269

199270
t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
200-
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
271+
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil {
201272
t.Fatalf("fail to run playwright test deploy-app: %v", err)
202273
}
203274

@@ -211,13 +282,23 @@ func TestInstallWithMITMProxy(t *testing.T) {
211282
t.Skip("skipping test for k0s versions < 1.29.0")
212283
}
213284

285+
requiredEnvVars := []string{
286+
"DR_S3_ENDPOINT",
287+
"DR_S3_REGION",
288+
"DR_S3_BUCKET",
289+
"DR_S3_PREFIX",
290+
"DR_ACCESS_KEY_ID",
291+
"DR_SECRET_ACCESS_KEY",
292+
}
293+
RequireEnvVars(t, requiredEnvVars)
294+
214295
tc := lxd.NewCluster(&lxd.ClusterInput{
215296
T: t,
216297
Nodes: 4,
217298
WithProxy: true,
218299
Image: "debian/12",
219300
EmbeddedClusterPath: "../output/bin/embedded-cluster",
220-
LicensePath: "licenses/license.yaml",
301+
LicensePath: "licenses/snapshot-license.yaml",
221302
})
222303
defer tc.Cleanup()
223304

@@ -287,11 +368,19 @@ func TestInstallWithMITMProxy(t *testing.T) {
287368
// check the installation state
288369
checkInstallationState(t, tc)
289370

371+
testArgs := []string{}
372+
for _, envVar := range requiredEnvVars {
373+
testArgs = append(testArgs, os.Getenv(envVar))
374+
}
375+
376+
if stdout, stderr, err := tc.RunPlaywrightTest("create-backup", testArgs...); err != nil {
377+
t.Fatalf("fail to run playwright test create-backup: %v: %s: %s", err, stdout, stderr)
378+
}
379+
290380
appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
291-
testArgs := []string{appUpgradeVersion}
292381

293382
t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
294-
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
383+
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", appUpgradeVersion); err != nil {
295384
t.Fatalf("fail to run playwright test deploy-app: %v", err)
296385
}
297386

@@ -301,6 +390,61 @@ func TestInstallWithMITMProxy(t *testing.T) {
301390
t.Fatalf("fail to check postupgrade state: %v", err)
302391
}
303392

393+
// reset the cluster
394+
runInParallel(t,
395+
func(t *testing.T) error {
396+
stdout, stderr, err := resetInstallationWithError(t, tc, 3, resetInstallationOptions{force: true})
397+
if err != nil {
398+
return fmt.Errorf("fail to reset the installation on node 3: %v: %s: %s", err, stdout, stderr)
399+
}
400+
return nil
401+
}, func(t *testing.T) error {
402+
stdout, stderr, err := resetInstallationWithError(t, tc, 2, resetInstallationOptions{force: true})
403+
if err != nil {
404+
return fmt.Errorf("fail to reset the installation on node 2: %v: %s: %s", err, stdout, stderr)
405+
}
406+
return nil
407+
}, func(t *testing.T) error {
408+
stdout, stderr, err := resetInstallationWithError(t, tc, 1, resetInstallationOptions{force: true})
409+
if err != nil {
410+
return fmt.Errorf("fail to reset the installation on node 1: %v: %s: %s", err, stdout, stderr)
411+
}
412+
return nil
413+
}, func(t *testing.T) error {
414+
stdout, stderr, err := resetInstallationWithError(t, tc, 0, resetInstallationOptions{force: true})
415+
if err != nil {
416+
return fmt.Errorf("fail to reset the installation on node 0: %v: %s: %s", err, stdout, stderr)
417+
}
418+
return nil
419+
},
420+
)
421+
422+
t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
423+
time.Sleep(30 * time.Second)
424+
425+
t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
426+
line = append([]string{"restore-installation.exp"}, testArgs...)
427+
line = append(line, "--http-proxy", lxd.HTTPMITMProxy)
428+
line = append(line, "--https-proxy", lxd.HTTPMITMProxy)
429+
line = append(line, "--no-proxy", strings.Join(tc.IPs, ","))
430+
line = append(line, "--private-ca", "/usr/local/share/ca-certificates/proxy/ca.crt")
431+
if _, _, err := tc.RunCommandOnNode(0, line, lxd.WithMITMProxyEnv(tc.IPs)); err != nil {
432+
t.Fatalf("fail to restore the installation: %v", err)
433+
}
434+
435+
checkInstallationState(t, tc)
436+
437+
t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
438+
line = []string{"check-post-restore.sh"}
439+
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
440+
t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
441+
}
442+
443+
t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
444+
if _, _, err := tc.SetupPlaywrightAndRunTest("validate-restore-app"); err != nil {
445+
t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
446+
}
447+
304448
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
305449
}
306450

‎e2e/scripts/vandoor-prepare.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ main() {
1010
local license_id=
1111
license_id="$2"
1212

13-
echo "downloading from https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci/${app_version_label}"
14-
retry 5 curl --retry 5 --retry-all-errors -fL -o ec-release.tgz "https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci/${app_version_label}" -H "Authorization: ${license_id}"
13+
echo "downloading from https://ec-e2e-replicated-app.testcluster.net/embedded/embedded-cluster-smoke-test-staging-app/ci/${app_version_label}"
14+
retry 5 curl --retry 5 --retry-all-errors -fL -o ec-release.tgz "https://ec-e2e-replicated-app.testcluster.net/embedded/embedded-cluster-smoke-test-staging-app/ci/${app_version_label}" -H "Authorization: ${license_id}"
1515
tar xzf ec-release.tgz
1616

1717
mkdir -p /assets

‎e2e/shared.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ type joinOptions struct {
5454
type downloadECReleaseOptions struct {
5555
version string
5656
licenseID string
57+
withEnv map[string]string
5758
}
5859

5960
type resetInstallationOptions struct {
@@ -277,7 +278,7 @@ func downloadECReleaseWithOptions(t *testing.T, tc cluster.Cluster, node int, op
277278
line = append(line, LicenseID)
278279
}
279280

280-
if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
281+
if stdout, stderr, err := tc.RunCommandOnNode(node, line, opts.withEnv); err != nil {
281282
t.Fatalf("fail to download embedded cluster release on node %d: %v: %s: %s", node, err, stdout, stderr)
282283
}
283284
}

‎kinds/apis/v1beta1/runtimeconfig_types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ type RuntimeConfigSpec struct {
2222
// OpenEBSDataDirOverride holds the override for the data directory for the OpenEBS storage
2323
// provisioner. By default the data will be stored in a subdirectory of DataDir.
2424
OpenEBSDataDirOverride string `json:"openEBSDataDirOverride,omitempty"`
25+
// HostCABundlePath holds the path to the CA bundle for the host.
26+
HostCABundlePath string `json:"hostCABundlePath,omitempty"`
2527

2628
// AdminConsole holds the Admin Console configuration.
2729
AdminConsole AdminConsoleSpec `json:"adminConsole,omitempty"`

‎operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,9 @@ spec:
606606
DataDir holds the data directory for the Embedded Cluster
607607
(default: /var/lib/embedded-cluster).
608608
type: string
609+
hostCABundlePath:
610+
description: HostCABundlePath holds the path to the CA bundle for the host.
611+
type: string
609612
k0sDataDirOverride:
610613
description: |-
611614
K0sDataDirOverride holds the override for the data directory for K0s. By default the data

‎operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,10 @@ spec:
384384
DataDir holds the data directory for the Embedded Cluster
385385
(default: /var/lib/embedded-cluster).
386386
type: string
387+
hostCABundlePath:
388+
description: HostCABundlePath holds the path to the CA bundle
389+
for the host.
390+
type: string
387391
k0sDataDirOverride:
388392
description: |-
389393
K0sDataDirOverride holds the override for the data directory for K0s. By default the data

‎pkg/addons/install.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ type InstallOptions struct {
2323
License *kotsv1beta1.License
2424
IsAirgap bool
2525
Proxy *ecv1beta1.ProxySpec
26-
HostCABundle string
26+
HostCABundlePath string
2727
PrivateCAs []string
2828
ServiceCIDR string
2929
DisasterRecoveryEnabled bool
@@ -88,6 +88,7 @@ func getAddOnsForInstall(opts InstallOptions) []types.AddOn {
8888
addOns = append(addOns, &velero.Velero{
8989
ProxyRegistryDomain: domains.ProxyRegistryDomain,
9090
Proxy: opts.Proxy,
91+
HostCABundlePath: opts.HostCABundlePath,
9192
})
9293
}
9394

@@ -118,6 +119,7 @@ func getAddOnsForRestore(opts InstallOptions) []types.AddOn {
118119
&velero.Velero{
119120
Proxy: opts.Proxy,
120121
ProxyRegistryDomain: domains.ProxyRegistryDomain,
122+
HostCABundlePath: opts.HostCABundlePath,
121123
},
122124
}
123125
return addOns

0 commit comments

Comments
 (0)