Skip to content

Commit 20422fe

Browse files
laveryaricardomaraschini
laverya
and
ricardomaraschini
authored
fix ci with new kotsadm (#414)
* Update Makefile versions * deploy app as part of single node install process * debug info * install tar before kots CLI * maybe install curl * use kotsadm service port * add basic config * do not include release yaml in release * more debug * curl http * actually deploy * feat: add script to deploy kots application we are not automatically deploying them anymore so we need a script to do so through the web interface. ``` $ puppeteer.sh deploy-kots-application.js 192.168.86.6 opening a new page opening kots page waiting and clicking on the 'Continue to Setup' button waiting and clicking on 'Advanced' to move on with the certificate waiting and clicking on 'Proceed' to move on with the certificate going to the /tls endpoint waiting and clicking on 'Continue' waiting and clicking in the password field typing the password clicking in the Log in button clicking on Continue in the cluster management page clicking on the configuration field so we can enter the application configuration typing 'abc' as the configuration value clicking on Continue after filling in the configuration $ ``` * only run cli install script on cli tests * bind variable * do not exit for case 'ui' * run both ui test sets * check app and installation state with bash script * check application state wip * f * more debug * check for deployed app version * remove now-unused check-app-and-cluster-status.js --------- Co-authored-by: laverya <[email protected]> Co-authored-by: Ricardo Maraschini <[email protected]>
1 parent daf9482 commit 20422fe

15 files changed

+299
-115
lines changed

‎.github/workflows/pull-request.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ jobs:
7878
if: github.actor != 'dependabot[bot]'
7979
run: |
8080
export SHORT_SHA=$(git rev-parse --short=7 HEAD)
81+
echo "# channel release object" > e2e/kots-release-install/release.yaml
82+
echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
83+
echo 'appSlug: "embedded-cluster-smoke-test-staging-app"' >> e2e/kots-release-install/release.yaml
8184
echo "versionLabel: \"${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
8285
cat e2e/kots-release-install/release.yaml
8386
cp output/bin/embedded-cluster output/bin/embedded-cluster-original

‎.github/workflows/release-dev.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ jobs:
5959
if: github.actor != 'dependabot[bot]'
6060
run: |
6161
export SHORT_SHA=$(git rev-parse --short=7 HEAD)
62+
echo "# channel release object" > e2e/kots-release-install/release.yaml
63+
echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
64+
echo 'appSlug: "embedded-cluster-smoke-test-staging-app"' >> e2e/kots-release-install/release.yaml
6265
echo "versionLabel: \"${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
6366
cat e2e/kots-release-install/release.yaml
6467
cp output/bin/embedded-cluster output/bin/embedded-cluster-original

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ ARCH := $(shell uname -m)
44
APP_NAME = embedded-cluster
55
ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
66
ADMIN_CONSOLE_CHART_NAME = admin-console
7-
ADMIN_CONSOLE_CHART_VERSION = 1.107.5
7+
ADMIN_CONSOLE_CHART_VERSION = 1.107.6
88
ADMIN_CONSOLE_IMAGE_OVERRIDE =
99
ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
1010
EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library

‎e2e/install_test.go‎

Lines changed: 41 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,14 @@ func TestSingleNodeInstallation(t *testing.T) {
3030
})
3131
defer tc.Destroy()
3232
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
33-
line := []string{"single-node-install.sh"}
33+
line := []string{"single-node-install.sh", "ui"}
3434
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
3535
if err != nil {
3636
t.Log("stdout:", stdout)
3737
t.Log("stderr:", stderr)
3838
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
3939
}
4040

41-
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
42-
line = []string{"check-installation-state.sh"}
43-
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
44-
if err != nil {
45-
t.Log("stdout:", stdout)
46-
t.Log("stderr:", stderr)
47-
t.Fatalf("fail to check installation state: %v", err)
48-
}
49-
5041
runPuppeteerAppStatusCheck(t, 0, tc)
5142

5243
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -71,31 +62,32 @@ func TestSingleNodeInstallationRockyLinux8(t *testing.T) {
7162
EmbeddedClusterPath: "../output/bin/embedded-cluster",
7263
})
7364
defer tc.Destroy()
74-
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
75-
line := []string{"single-node-install.sh"}
65+
66+
t.Logf("%s: installing tar", time.Now().Format(time.RFC3339))
67+
line := []string{"yum-install-tar.sh"}
7668
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
7769
if err != nil {
7870
t.Log("stdout:", stdout)
7971
t.Log("stderr:", stderr)
80-
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
72+
t.Fatalf("fail to check postupgrade state: %v", err)
8173
}
8274

83-
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
84-
line = []string{"check-installation-state.sh"}
75+
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
76+
line = []string{"single-node-install.sh", "cli"}
8577
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
8678
if err != nil {
8779
t.Log("stdout:", stdout)
8880
t.Log("stderr:", stderr)
89-
t.Fatalf("fail to check installation state: %v", err)
81+
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
9082
}
9183

92-
t.Logf("%s: installing tar", time.Now().Format(time.RFC3339))
93-
line = []string{"yum-install-tar.sh"}
84+
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
85+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
9486
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
9587
if err != nil {
9688
t.Log("stdout:", stdout)
9789
t.Log("stderr:", stderr)
98-
t.Fatalf("fail to check postupgrade state: %v", err)
90+
t.Fatalf("fail to check installation state: %v", err)
9991
}
10092

10193
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -132,7 +124,7 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
132124
}
133125

134126
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
135-
line := []string{"single-node-install.sh"}
127+
line := []string{"single-node-install.sh", "cli"}
136128
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
137129
if err != nil {
138130
t.Log("stdout:", stdout)
@@ -141,7 +133,7 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
141133
}
142134

143135
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
144-
line = []string{"check-installation-state.sh"}
136+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
145137
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
146138
if err != nil {
147139
t.Log("stdout:", stdout)
@@ -171,31 +163,32 @@ func TestSingleNodeInstallationCentos8Stream(t *testing.T) {
171163
EmbeddedClusterPath: "../output/bin/embedded-cluster",
172164
})
173165
defer tc.Destroy()
174-
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
175-
line := []string{"single-node-install.sh"}
166+
167+
t.Logf("%s: installing tar", time.Now().Format(time.RFC3339))
168+
line := []string{"yum-install-tar.sh"}
176169
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
177170
if err != nil {
178171
t.Log("stdout:", stdout)
179172
t.Log("stderr:", stderr)
180-
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
173+
t.Fatalf("fail to check postupgrade state: %v", err)
181174
}
182175

183-
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
184-
line = []string{"check-installation-state.sh"}
176+
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
177+
line = []string{"single-node-install.sh", "cli"}
185178
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
186179
if err != nil {
187180
t.Log("stdout:", stdout)
188181
t.Log("stderr:", stderr)
189-
t.Fatalf("fail to check installation state: %v", err)
182+
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
190183
}
191184

192-
t.Logf("%s: installing tar", time.Now().Format(time.RFC3339))
193-
line = []string{"yum-install-tar.sh"}
185+
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
186+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
194187
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
195188
if err != nil {
196189
t.Log("stdout:", stdout)
197190
t.Log("stderr:", stderr)
198-
t.Fatalf("fail to check postupgrade state: %v", err)
191+
t.Fatalf("fail to check installation state: %v", err)
199192
}
200193

201194
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -257,27 +250,18 @@ func TestMultiNodeInstallation(t *testing.T) {
257250
// bootstrap the first node and makes sure it is healthy. also executes the kots
258251
// ssl certificate configuration (kurl-proxy).
259252
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
260-
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"single-node-install.sh"}); err != nil {
253+
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"single-node-install.sh", "ui"}); err != nil {
261254
t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
262255
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
263256
}
264257

265-
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
266-
line := []string{"check-installation-state.sh"}
267-
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
268-
if err != nil {
269-
t.Log("stdout:", stdout)
270-
t.Log("stderr:", stderr)
271-
t.Fatalf("fail to check installation state: %v", err)
272-
}
273-
274258
runPuppeteerAppStatusCheck(t, 0, tc)
275259

276260
// generate all node join commands (2 for controllers and 1 for worker).
277261
t.Logf("%s: generating two new controller token commands", time.Now().Format(time.RFC3339))
278262
controllerCommands := []string{}
279263
for i := 0; i < 2; i++ {
280-
line = []string{"puppeteer.sh", "generate-controller-join-token.js", "10.0.0.2"}
264+
line := []string{"puppeteer.sh", "generate-controller-join-token.js", "10.0.0.2"}
281265
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
282266
if err != nil {
283267
t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
@@ -295,8 +279,8 @@ func TestMultiNodeInstallation(t *testing.T) {
295279
t.Log("controller join token command:", command)
296280
}
297281
t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339))
298-
line = []string{"puppeteer.sh", "generate-worker-join-token.js", "10.0.0.2"}
299-
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
282+
line := []string{"puppeteer.sh", "generate-worker-join-token.js", "10.0.0.2"}
283+
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
300284
if err != nil {
301285
t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
302286
t.Fatalf("fail to generate controller join token: %s", stdout)
@@ -384,15 +368,15 @@ func TestInstallFromReplicatedApp(t *testing.T) {
384368
}
385369

386370
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
387-
line = []string{"single-node-install.sh"}
371+
line = []string{"single-node-install.sh", "cli"}
388372
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line); err != nil {
389373
t.Log("install stdout:", stdout)
390374
t.Log("install stderr:", stderr)
391375
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
392376
}
393377

394378
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
395-
line = []string{"check-installation-state.sh"}
379+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
396380
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
397381
if err != nil {
398382
t.Log("stdout:", stdout)
@@ -423,15 +407,15 @@ func TestResetAndReinstall(t *testing.T) {
423407
})
424408
defer tc.Destroy()
425409
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
426-
line := []string{"single-node-install.sh"}
410+
line := []string{"single-node-install.sh", "cli"}
427411
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line); err != nil {
428412
t.Log("install stdout:", stdout)
429413
t.Log("install stderr:", stderr)
430414
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
431415
}
432416

433417
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
434-
line = []string{"check-installation-state.sh"}
418+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
435419
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
436420
if err != nil {
437421
t.Log("stdout:", stdout)
@@ -449,15 +433,15 @@ func TestResetAndReinstall(t *testing.T) {
449433
}
450434

451435
t.Logf("%s: installing embedded-cluster on node 0 after reset", time.Now().Format(time.RFC3339))
452-
line = []string{"single-node-install.sh"}
436+
line = []string{"single-node-install.sh", "cli"}
453437
if stdout, stderr, err = RunCommandOnNode(t, tc, 0, line); err != nil {
454438
t.Log("install stdout:", stdout)
455439
t.Log("install stderr:", stderr)
456440
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
457441
}
458442

459443
t.Logf("%s: checking installation state after reinstall", time.Now().Format(time.RFC3339))
460-
line = []string{"check-installation-state.sh"}
444+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
461445
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
462446
if err != nil {
463447
t.Log("stdout:", stdout)
@@ -476,23 +460,21 @@ func runPuppeteerAppStatusCheck(t *testing.T, node int, tc *cluster.Output) {
476460
t.Log("stderr:", stderr)
477461
t.Fatalf("fail to install puppeteer on node %s: %v", tc.Nodes[0], err)
478462
}
479-
t.Logf("%s: accessing kotsadm interface and checking app and cluster state", time.Now().Format(time.RFC3339))
480-
line = []string{"puppeteer.sh", "check-app-and-cluster-status.js", "10.0.0.2"}
463+
t.Logf("%s: accessing kotsadm interface and deploying app", time.Now().Format(time.RFC3339))
464+
line = []string{"puppeteer.sh", "deploy-kots-application.js", "10.0.0.2"}
481465
stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
482466
if err != nil {
483467
t.Log("stdout:", stdout)
484468
t.Log("stderr:", stderr)
485469
t.Fatalf("fail to access kotsadm interface and state: %v", err)
486470
}
487-
var r clusterStatusResponse
488-
if err := json.Unmarshal([]byte(stdout), &r); err != nil {
489-
t.Log("stdout:", stdout)
490-
t.Log("stderr:", stderr)
491-
t.Fatalf("fail to parse script response: %v", err)
492-
}
493-
if r.App != "Ready" || r.Cluster != "Up to date" {
471+
472+
t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
473+
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
474+
stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
475+
if err != nil {
494476
t.Log("stdout:", stdout)
495477
t.Log("stderr:", stderr)
496-
t.Fatalf("cluster or app not ready: %s", stdout)
478+
t.Fatalf("fail to check installation state: %v", err)
497479
}
498480
}

‎e2e/kots-release-install/config.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: kots.io/v1beta1
2+
kind: Config
3+
spec:
4+
groups:
5+
- name: config_group
6+
title: The First Config Group
7+
items:
8+
- name: hostname
9+
title: Hostname
10+
type: text

‎e2e/kots-release-install/release.yaml‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎e2e/kots-release-upgrade/config.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: kots.io/v1beta1
2+
kind: Config
3+
spec:
4+
groups:
5+
- name: config_group
6+
title: The First Config Group
7+
items:
8+
- name: hostname
9+
title: Hostname
10+
type: text

‎e2e/reset_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func TestMultiNodeReset(t *testing.T) {
2424
// bootstrap the first node and makes sure it is healthy. also executes the kots
2525
// ssl certificate configuration (kurl-proxy).
2626
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
27-
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"single-node-install.sh"}); err != nil {
27+
if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"single-node-install.sh", "ui"}); err != nil {
2828
t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
2929
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
3030
}

‎e2e/scripts/check-installation-state.sh‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,49 @@ wait_for_installation() {
2525
done
2626
}
2727

28+
wait_for_nginx_pods() {
29+
ready=$(kubectl get pods -n kotsadm -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}' | grep "nginx" | grep -c Running || true)
30+
counter=0
31+
while [ "$ready" -lt "1" ]; do
32+
if [ "$counter" -gt 36 ]; then
33+
echo "nginx pods did not appear"
34+
kubectl get pods -n kotsadm -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}'
35+
kubectl get pods -n kotsadm
36+
kubectl logs -n kotsadm -l app=kotsadm
37+
return 1
38+
fi
39+
sleep 5
40+
counter=$((counter+1))
41+
echo "Waiting for nginx pods"
42+
ready=$(kubectl get pods -n kotsadm -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}' | grep "nginx" | grep -c Running || true)
43+
kubectl get pods -n nginx 2>&1 || true
44+
echo "ready: $ready"
45+
done
46+
}
47+
48+
ensure_app_deployed() {
49+
local version="$1"
50+
51+
kubectl kots get versions -n kotsadm embedded-cluster-smoke-test-staging-app
52+
if ! kubectl kots get versions -n kotsadm embedded-cluster-smoke-test-staging-app | grep -q "${version}\W*[01]\W*deployed"; then
53+
echo "application not deployed"
54+
return 1
55+
fi
56+
}
57+
58+
ensure_app_not_upgraded() {
59+
if kubectl get ns | grep -q memcached ; then
60+
echo "found memcached ns"
61+
return 1
62+
fi
63+
if kubectl get pods -n kotsadm -l app=second | grep -q second ; then
64+
echo "found pods from app update"
65+
return 1
66+
fi
67+
}
68+
2869
main() {
70+
local version="$1"
2971
sleep 30 # wait for kubectl to become available
3072

3173
echo "pods"
@@ -37,9 +79,20 @@ main() {
3779

3880
echo "ensure that the admin console branding is available"
3981
kubectl get cm -n kotsadm kotsadm-application-metadata
82+
83+
if ! wait_for_nginx_pods; then
84+
echo "Failed waiting for the application's nginx pods"
85+
exit 1
86+
fi
87+
if ! ensure_app_deployed "$version"; then
88+
exit 1
89+
fi
90+
if ! ensure_app_not_upgraded; then
91+
exit 1
92+
fi
4093
}
4194

4295
export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
4396
export KUBECONFIG=/var/lib/k0s/pki/admin.conf
4497
export PATH=$PATH:/var/lib/embedded-cluster/bin
45-
main
98+
main "$@"

0 commit comments

Comments
 (0)