Skip to content

Commit d842047

Browse files
emosbaughajp-io
andauthored
Revert "Revert "feat(troubleshoot): add filesystem latency check"" (#1030)
* Revert "Revert "feat(troubleshoot): add filesystem latency check" (#1024)" This reverts commit 760c3be. * fix preflight * Update pkg/preflights/host-preflight.yaml Co-authored-by: Alex Parker <[email protected]> * feedback * Update pkg/preflights/host-preflight.yaml Co-authored-by: Alex Parker <[email protected]> --------- Co-authored-by: Alex Parker <[email protected]>
1 parent 1567e4c commit d842047

File tree

14 files changed

+548
-100
lines changed

14 files changed

+548
-100
lines changed

.github/workflows/ci.yaml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,37 @@ jobs:
409409
with:
410410
message-path: download-link.txt
411411

412+
# e2e-docker runs the e2e tests inside a docker container rather than a full VM
413+
e2e-docker:
414+
runs-on: ubuntu-latest
415+
needs:
416+
- build
417+
strategy:
418+
fail-fast: false
419+
matrix:
420+
test:
421+
- TestPreflights
422+
steps:
423+
- name: Checkout
424+
uses: actions/checkout@v4
425+
- name: Download binary
426+
uses: actions/download-artifact@v4
427+
with:
428+
name: embedded-release
429+
path: output/bin
430+
- name: Setup Go
431+
uses: actions/setup-go@v5
432+
with:
433+
go-version-file: go.mod
434+
- name: Write license file
435+
run: |
436+
echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
437+
- name: Run test
438+
env:
439+
LICENSE_PATH: license.yaml
440+
run: |
441+
make e2e-test TEST_NAME=${{ matrix.test }}
442+
412443
e2e:
413444
runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
414445
needs:
@@ -499,6 +530,7 @@ jobs:
499530
runs-on: ubuntu-20.04
500531
needs:
501532
- e2e
533+
- e2e-docker
502534
- sanitize
503535
- tests
504536
- check-images

Makefile

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ K0S_GO_VERSION = v1.29.7+k0s.0
1616
PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
1717
K0S_BINARY_SOURCE_OVERRIDE =
1818
PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
19-
TROUBLESHOOT_VERSION = v0.97.0
19+
TROUBLESHOOT_VERSION = v0.100.0
2020
KOTS_VERSION = v$(shell awk '/^version/{print $$2}' pkg/addons/adminconsole/static/metadata.yaml | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
2121
KOTS_BINARY_URL_OVERRIDE =
2222
# TODO: move this to a manifest file
@@ -33,6 +33,7 @@ LD_FLAGS = \
3333
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleImageOverride=$(ADMIN_CONSOLE_IMAGE_OVERRIDE) \
3434
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleMigrationsImageOverride=$(ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE) \
3535
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleKurlProxyImageOverride=$(ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE)
36+
DISABLE_FIO_BUILD ?= 0
3637

3738
export PATH := $(shell pwd)/bin:$(PATH)
3839

@@ -73,6 +74,16 @@ pkg/goods/bins/local-artifact-mirror: Makefile
7374
$(MAKE) -C local-artifact-mirror build GOOS=linux GOARCH=amd64
7475
cp local-artifact-mirror/bin/local-artifact-mirror-$(GOOS)-$(GOARCH) pkg/goods/bins/local-artifact-mirror
7576

77+
pkg/goods/bins/fio: PLATFORM = linux/amd64
78+
pkg/goods/bins/fio: Makefile
79+
ifneq ($(DISABLE_FIO_BUILD),1)
80+
mkdir -p pkg/goods/bins
81+
docker build -t fio --build-arg PLATFORM=$(PLATFORM) fio
82+
docker rm -f fio && docker run --name fio fio
83+
docker cp fio:/output/fio pkg/goods/bins/fio
84+
touch pkg/goods/bins/fio
85+
endif
86+
7687
pkg/goods/internal/bins/kubectl-kots: Makefile
7788
mkdir -p pkg/goods/internal/bins
7889
mkdir -p output/tmp/kots
@@ -107,6 +118,7 @@ static: pkg/goods/bins/k0s \
107118
pkg/goods/bins/kubectl-preflight \
108119
pkg/goods/bins/kubectl-support_bundle \
109120
pkg/goods/bins/local-artifact-mirror \
121+
pkg/goods/bins/fio \
110122
pkg/goods/internal/bins/kubectl-kots
111123

112124
.PHONY: embedded-cluster-linux-amd64

cmd/embedded-cluster/install.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,14 @@ func runHostPreflights(c *cli.Context, hpf *v1beta2.HostPreflightSpec, proxy *ec
119119
return nil
120120
}
121121
pb.Infof("Running host preflights")
122-
output, err := preflights.Run(c.Context, hpf, proxy)
122+
output, stderr, err := preflights.Run(c.Context, hpf, proxy)
123123
if err != nil {
124124
pb.CloseWithError()
125125
return fmt.Errorf("host preflights failed to run: %w", err)
126126
}
127+
if stderr != "" {
128+
logrus.Debugf("preflight stderr: %s", stderr)
129+
}
127130

128131
err = output.SaveToDisk()
129132
if err != nil {

e2e/docker/docker.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package docker
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"os"
7+
"os/exec"
8+
"path/filepath"
9+
"strings"
10+
"testing"
11+
)
12+
13+
type Docker struct {
14+
client string
15+
t *testing.T
16+
}
17+
18+
func NewCLI(t *testing.T) *Docker {
19+
client, err := exec.LookPath("docker")
20+
if err != nil {
21+
t.Fatalf("failed to find docker in path: %v", err)
22+
}
23+
return &Docker{
24+
client: client,
25+
t: t,
26+
}
27+
}
28+
29+
type Container struct {
30+
Image string
31+
Volumes []string
32+
33+
id string
34+
t *testing.T
35+
}
36+
37+
func NewContainer(t *testing.T) *Container {
38+
return &Container{
39+
id: generateID(),
40+
t: t,
41+
}
42+
}
43+
44+
func (c *Container) WithImage(image string) *Container {
45+
c.Image = image
46+
return c
47+
}
48+
49+
func (c *Container) WithECBinary() *Container {
50+
path, err := filepath.Abs("../output/bin/embedded-cluster")
51+
if err != nil {
52+
c.t.Fatalf("failed to get absolute path to embedded-cluster binary: %v", err)
53+
}
54+
_, err = os.Stat(path)
55+
if err != nil {
56+
c.t.Fatalf("failed to find embedded-cluster binary: %v", err)
57+
}
58+
err = os.Chmod(path, 0755)
59+
if err != nil {
60+
c.t.Fatalf("failed to chmod embedded-cluster binary: %v", err)
61+
}
62+
return c.WithVolume(fmt.Sprintf("%s:%s", path, c.GetECBinaryPath()))
63+
}
64+
65+
func (c *Container) GetECBinaryPath() string {
66+
return "/ec/bin/embedded-cluster"
67+
}
68+
69+
func (c *Container) WithLicense(path string) *Container {
70+
path, err := filepath.Abs(path)
71+
if err != nil {
72+
c.t.Fatalf("failed to get absolute path to license file: %v", err)
73+
}
74+
_, err = os.Stat(path)
75+
if err != nil {
76+
c.t.Fatalf("failed to find embedded-cluster binary: %v", err)
77+
}
78+
return c.WithVolume(fmt.Sprintf("%s:%s", path, c.GetLicensePath()))
79+
}
80+
81+
func (c *Container) GetLicensePath() string {
82+
return "/ec/license.yaml"
83+
}
84+
85+
func (c *Container) WithVolume(volume string) *Container {
86+
c.Volumes = append(c.Volumes, volume)
87+
return c
88+
}
89+
90+
func (c *Container) Start(cli *Docker) {
91+
execCmd := exec.Command(
92+
cli.client, "run", "--rm", "-d", "-w", "/ec", "--platform=linux/amd64", "--privileged",
93+
"--name", c.id,
94+
)
95+
for _, volume := range c.Volumes {
96+
execCmd.Args = append(execCmd.Args, "-v", volume)
97+
}
98+
execCmd.Args = append(execCmd.Args, c.Image)
99+
execCmd.Args = append(execCmd.Args, "sh", "-c", "while true; do sleep 1; done")
100+
c.t.Logf("starting container: docker %s", strings.Join(execCmd.Args, " "))
101+
err := execCmd.Run()
102+
if err != nil {
103+
c.t.Fatalf("failed to start container: %v", err)
104+
}
105+
}
106+
107+
func (c *Container) Destroy(cli *Docker) {
108+
execCmd := exec.Command(cli.client, "rm", "-f", c.id)
109+
err := execCmd.Run()
110+
if err != nil {
111+
c.t.Fatalf("failed to destroy container: %v", err)
112+
}
113+
}
114+
115+
func (c *Container) Exec(cli *Docker, cmd string) (string, string, error) {
116+
args := []string{"exec", c.id, "sh", "-c", cmd}
117+
execCmd := exec.Command(cli.client, args...)
118+
c.t.Logf("executing command: docker %s", strings.Join(execCmd.Args, " "))
119+
var stdout, stderr bytes.Buffer
120+
execCmd.Stdout = &stdout
121+
execCmd.Stderr = &stderr
122+
err := execCmd.Run()
123+
return stdout.String(), stderr.String(), err
124+
}

e2e/docker/id.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package docker
2+
3+
import "math/rand"
4+
5+
var alphabet = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
6+
7+
func generateID() string {
8+
b := make([]rune, 32)
9+
for i := range b {
10+
b[i] = alphabet[rand.Intn(len(alphabet))]
11+
}
12+
return "ece2e-" + string(b)
13+
}

e2e/materialize_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ func TestMaterialize(t *testing.T) {
2121
{"rm", "-rf", "/var/lib/embedded-cluster/bin/kubectl"},
2222
{"rm", "-rf", "/var/lib/embedded-cluster/bin/kubectl-preflight"},
2323
{"rm", "-rf", "/var/lib/embedded-cluster/bin/kubectl-support_bundle"},
24+
{"rm", "-rf", "/var/lib/embedded-cluster/bin/fio"},
2425
{"embedded-cluster", "materialize"},
2526
{"ls", "-la", "/var/lib/embedded-cluster/bin/kubectl"},
2627
{"ls", "-la", "/var/lib/embedded-cluster/bin/kubectl-preflight"},
2728
{"ls", "-la", "/var/lib/embedded-cluster/bin/kubectl-support_bundle"},
29+
{"ls", "-la", "/var/lib/embedded-cluster/bin/fio"},
2830
}
2931
if err := RunCommandsOnNode(t, tc, 0, commands); err != nil {
3032
t.Fatalf("fail testing materialize assets: %v", err)

e2e/preflights_test.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package e2e
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"strings"
7+
"testing"
8+
9+
"github.com/replicatedhq/embedded-cluster/e2e/docker"
10+
"github.com/replicatedhq/embedded-cluster/pkg/preflights"
11+
)
12+
13+
func TestPreflights(t *testing.T) {
14+
t.Parallel()
15+
16+
cli := docker.NewCLI(t)
17+
18+
container := docker.NewContainer(t).
19+
WithImage("debian:bookworm-slim").
20+
WithECBinary()
21+
if licensePath := os.Getenv("LICENSE_PATH"); licensePath != "" {
22+
t.Logf("using license %s", licensePath)
23+
container = container.WithLicense(licensePath)
24+
}
25+
container.Start(cli)
26+
27+
t.Cleanup(func() {
28+
container.Destroy(cli)
29+
})
30+
31+
_, stderr, err := container.Exec(cli,
32+
"apt-get update && apt-get install -y apt-utils kmod",
33+
)
34+
if err != nil {
35+
t.Fatalf("failed to install deps: err=%v, stderr=%s", err, stderr)
36+
}
37+
38+
runCmd := fmt.Sprintf("%s install run-preflights --no-prompt", container.GetECBinaryPath())
39+
if os.Getenv("LICENSE_PATH") != "" {
40+
runCmd = fmt.Sprintf("%s --license %s", runCmd, container.GetLicensePath())
41+
}
42+
43+
// we are more interested in the results
44+
runStdout, runStderr, runErr := container.Exec(cli, runCmd)
45+
46+
stdout, stderr, err := container.Exec(cli,
47+
"cat /var/lib/embedded-cluster/support/host-preflight-results.json",
48+
)
49+
if err != nil {
50+
t.Logf("run-preflights: err=%v, stdout=%s, stderr=%s", runErr, runStdout, runStderr)
51+
t.Fatalf("failed to get preflight results: err=%v, stderr=%s", err, stderr)
52+
}
53+
54+
results, err := preflights.OutputFromReader(strings.NewReader(stdout))
55+
if err != nil {
56+
t.Fatalf("failed to parse preflight results: %v", err)
57+
}
58+
59+
tests := []struct {
60+
name string
61+
assert func(t *testing.T, results *preflights.Output)
62+
}{
63+
{
64+
name: "Should contain fio results",
65+
assert: func(t *testing.T, results *preflights.Output) {
66+
for _, res := range results.Pass {
67+
if res.Title == "Filesystem Write Latency" {
68+
t.Logf("fio test passed: %s", res.Message)
69+
return
70+
}
71+
}
72+
for _, res := range results.Fail {
73+
if !strings.Contains(res.Message, "Write latency is high") {
74+
t.Errorf("fio test failed: %s", res.Message)
75+
}
76+
// as long as fio ran successfully, we're good
77+
t.Logf("fio test failed: %s", res.Message)
78+
}
79+
80+
t.Errorf("fio test not found")
81+
},
82+
},
83+
{
84+
name: "Should not contain unexpected failures",
85+
assert: func(t *testing.T, results *preflights.Output) {
86+
expected := map[string]bool{
87+
// TODO: work to remove these
88+
"System Clock": true,
89+
"'devices' Cgroup Controller": true,
90+
"Default Route": true,
91+
"API Access": true,
92+
"Proxy Registry Access": true,
93+
// as long as fio ran successfully, we're good
94+
"Filesystem Write Latency": true,
95+
}
96+
for _, res := range results.Fail {
97+
if !expected[res.Title] {
98+
t.Errorf("unexpected failure: %q, %q", res.Title, res.Message)
99+
} else {
100+
t.Logf("found expected failure: %q, %q", res.Title, res.Message)
101+
}
102+
}
103+
},
104+
},
105+
{
106+
name: "Should not contain unexpected warnings",
107+
assert: func(t *testing.T, results *preflights.Output) {
108+
expected := map[string]bool{}
109+
for _, res := range results.Warn {
110+
if !expected[res.Title] {
111+
t.Errorf("unexpected warning: %q, %q", res.Title, res.Message)
112+
} else {
113+
t.Logf("found expected warning: %q, %q", res.Title, res.Message)
114+
}
115+
}
116+
},
117+
},
118+
}
119+
for _, tt := range tests {
120+
t.Run(tt.name, func(t *testing.T) {
121+
tt.assert(t, results)
122+
})
123+
}
124+
}

0 commit comments

Comments
 (0)