Skip to content

Commit 55454ef

Browse files
laveryalaverya
authored
fix CI (#302)
* disable openshift CI runs? * no eks either * all envs again * use older bitnami/kubectl image * cut down to one distro during testing * set kubeconfig file permissions * use non-root kubeconfig path * helm should still use root-based kubeconfig * reenable other distros * move test file to /tmp
1 parent a47ad98 commit 55454ef

File tree

1 file changed

+47
-39
lines changed

1 file changed

+47
-39
lines changed

‎dagger/e2e.go‎

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import (
1212
"time"
1313
)
1414

15+
// kubeconfigPath is the location inside the container where we mount the kubeconfig.
16+
// The Bitnami images we use run as the non-root user 1001, which cannot access /root,
17+
// so we place the file in that user’s home directory instead.
18+
const kubeconfigPath = "/home/1001/.kube/config"
19+
1520
func e2e(
1621
ctx context.Context,
1722
source *dagger.Directory,
@@ -53,15 +58,17 @@ func e2e(
5358
return fmt.Errorf("failed to get kubeconfig: %w", err)
5459
}
5560

56-
kubeconfigSource := source.WithNewFile("/kubeconfig", kubeconfig)
61+
kubeconfigSource := source.WithNewFile("/kubeconfig", kubeconfig, dagger.DirectoryWithNewFileOpts{
62+
Permissions: 0644,
63+
})
5764

5865
// if the cluster type is eks, we need to patch the storage class to be default - otherwise the statefulset will fail to create
5966
// kubectl patch storageclass gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
6067
if distribution == "eks" {
6168
fmt.Println("Patching eks gp2 storage class to be default...")
6269
ctr = dag.Container().From("bitnami/kubectl:latest").
63-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
64-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
70+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
71+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
6572
WithExec([]string{"kubectl", "patch", "storageclass", "gp2", "-p", `{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}`})
6673
out, err = ctr.Stdout(ctx)
6774
if err != nil {
@@ -86,8 +93,8 @@ func e2e(
8693

8794
// wait for the pod to be ready
8895
ctr = dag.Container().From("bitnami/kubectl:latest").
89-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
90-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
96+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
97+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
9198
WithExec(
9299
[]string{
93100
"kubectl", "wait",
@@ -105,8 +112,8 @@ func e2e(
105112
fmt.Println(out)
106113

107114
ctr = dag.Container().From("bitnami/kubectl:latest").
108-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
109-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
115+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
116+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
110117
With(CacheBustingExec(
111118
[]string{
112119
"kubectl", "get", "ns",
@@ -120,8 +127,8 @@ func e2e(
120127
fmt.Println(out)
121128

122129
ctr = dag.Container().From("bitnami/kubectl:latest").
123-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
124-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
130+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
131+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
125132
With(CacheBustingExec(
126133
[]string{
127134
"kubectl", "get", "pods",
@@ -144,10 +151,10 @@ func e2e(
144151

145152
// create a TLS secret within the namespace
146153
ctr = dag.Container().From("bitnami/kubectl:latest").
147-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
154+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
148155
WithFile("/certs/test-cert.crt", certDir.File("/test-cert.crt")).
149156
WithFile("/certs/test-key.key", certDir.File("/test-key.key")).
150-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
157+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
151158
WithExec(
152159
[]string{
153160
"kubectl", "create", "secret", "tls", "test-tls", "--cert=/certs/test-cert.crt", "--key=/certs/test-key.key",
@@ -166,8 +173,8 @@ func e2e(
166173
}
167174

168175
ctr = dag.Container().From("bitnami/kubectl:latest").
169-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
170-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
176+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
177+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
171178
With(CacheBustingExec(
172179
[]string{
173180
"kubectl", "get", "secrets",
@@ -181,8 +188,8 @@ func e2e(
181188
fmt.Println(out)
182189

183190
ctr = dag.Container().From("bitnami/kubectl:latest").
184-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
185-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
191+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
192+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
186193
With(CacheBustingExec(
187194
[]string{
188195
"kubectl", "get", "pods",
@@ -225,10 +232,10 @@ spec:
225232
deploymentSource := source.WithNewFile("/replicated-ssl-test.yaml", deploymentYaml)
226233

227234
ctr = dag.Container().From("bitnami/kubectl:latest").
228-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
229-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
230-
WithFile("/root/replicated-ssl-test.yaml", deploymentSource.File("/replicated-ssl-test.yaml")).
231-
WithExec([]string{"kubectl", "apply", "-f", "/root/replicated-ssl-test.yaml"})
235+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
236+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
237+
WithFile("/tmp/replicated-ssl-test.yaml", deploymentSource.File("/replicated-ssl-test.yaml")).
238+
WithExec([]string{"kubectl", "apply", "-f", "/tmp/replicated-ssl-test.yaml"})
232239
out, err = ctr.Stdout(ctx)
233240
if err != nil {
234241
// Get stderr to see the actual error
@@ -239,14 +246,14 @@ spec:
239246

240247
// wait for the replicated-ssl-test deployment to be ready
241248
ctr = dag.Container().From("bitnami/kubectl:latest").
242-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
243-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
249+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
250+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
244251
WithExec([]string{"kubectl", "wait", "--for=condition=available", "deployment/replicated-ssl-test", "--timeout=1m"})
245252
out, err = ctr.Stdout(ctx)
246253
if err != nil {
247254
ctr = dag.Container().From("bitnami/kubectl:latest").
248-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
249-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
255+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
256+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
250257
WithExec([]string{"kubectl", "logs", "-p", "-l", "app.kubernetes.io/name=replicated"})
251258
out, err2 := ctr.Stdout(ctx)
252259
if err2 != nil {
@@ -260,8 +267,8 @@ spec:
260267

261268
// print the final pods
262269
ctr = dag.Container().From("bitnami/kubectl:latest").
263-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
264-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
270+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
271+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
265272
With(CacheBustingExec(
266273
[]string{
267274
"kubectl", "get", "pods",
@@ -287,8 +294,8 @@ spec:
287294

288295
// Check the role to verify minimal RBAC is applied
289296
ctr = dag.Container().From("bitnami/kubectl:latest").
290-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
291-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
297+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
298+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
292299
With(CacheBustingExec(
293300
[]string{
294301
"kubectl", "describe", "role", "replicated-role",
@@ -381,8 +388,8 @@ spec:
381388

382389
// Get final pod status
383390
ctr = dag.Container().From("bitnami/kubectl:latest").
384-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
385-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
391+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
392+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
386393
With(CacheBustingExec(
387394
[]string{
388395
"kubectl", "get", "pods", "-o", "wide",
@@ -396,8 +403,8 @@ spec:
396403

397404
// get SDK logs for final debugging
398405
ctr = dag.Container().From("bitnami/kubectl:latest").
399-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
400-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
406+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
407+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
401408
With(CacheBustingExec(
402409
[]string{
403410
"kubectl", "logs", "deployment/replicated", "--tail=100",
@@ -624,8 +631,8 @@ func upgradeChartAndRestart(
624631

625632
// Restart replicated deployment
626633
ctr = dag.Container().From("bitnami/kubectl:latest").
627-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
628-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
634+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
635+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
629636
With(CacheBustingExec(
630637
[]string{
631638
"kubectl", "rollout", "restart", "deploy/replicated",
@@ -634,11 +641,12 @@ func upgradeChartAndRestart(
634641
if err != nil {
635642
return fmt.Errorf("failed to restart replicated deployment: %w", err)
636643
}
644+
fmt.Println(out)
637645

638646
// Wait for replicated deployment to be ready
639647
ctr = dag.Container().From("bitnami/kubectl:latest").
640-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
641-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
648+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
649+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
642650
With(CacheBustingExec(
643651
[]string{
644652
"kubectl", "rollout", "status",
@@ -652,8 +660,8 @@ func upgradeChartAndRestart(
652660

653661
// Get logs to help debug if replicated didn't start properly
654662
ctr = dag.Container().From("bitnami/kubectl:latest").
655-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
656-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
663+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
664+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
657665
With(CacheBustingExec(
658666
[]string{
659667
"kubectl", "logs", "-l", "app.kubernetes.io/name=replicated", "--tail=50",
@@ -671,8 +679,8 @@ func upgradeChartAndRestart(
671679

672680
// Restart test-chart deployment
673681
ctr = dag.Container().From("bitnami/kubectl:latest").
674-
WithFile("/root/.kube/config", kubeconfigSource.File("/kubeconfig")).
675-
WithEnvVariable("KUBECONFIG", "/root/.kube/config").
682+
WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
683+
WithEnvVariable("KUBECONFIG", kubeconfigPath).
676684
With(CacheBustingExec(
677685
[]string{
678686
"kubectl", "rollout", "restart", "deploy/test-chart",

0 commit comments

Comments
 (0)