Skip to content

Commit 0a5e7fc

Browse files
authored
feat: Store host preflight results on disk (#638)
Signed-off-by: Evans Mungai <[email protected]>
1 parent 2b78ebe commit 0a5e7fc

26 files changed

+498
-74
lines changed

Makefile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
1111
EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
1212
EMBEDDED_OPERATOR_CHART_VERSION = 0.36.2
1313
EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
14-
EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
14+
EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
1515
OPENEBS_CHART_URL = https://openebs.github.io/openebs
1616
OPENEBS_CHART_NAME = openebs/openebs
1717
OPENEBS_CHART_VERSION = 4.0.1
@@ -34,7 +34,7 @@ K0S_GO_VERSION = v1.29.5+k0s.0
3434
PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
3535
K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
3636
PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
37-
TROUBLESHOOT_VERSION = v0.92.1
37+
TROUBLESHOOT_VERSION = v0.93.1
3838
KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
3939
KOTS_BINARY_URL_OVERRIDE =
4040
LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
@@ -135,6 +135,7 @@ output/bin/embedded-cluster-release-builder:
135135
embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/bin/embedded-cluster-release-builder
136136
./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster
137137

138+
.PHONY: go.mod
138139
go.mod: Makefile
139140
go get github.com/k0sproject/k0s@$(K0S_GO_VERSION)
140141
go mod tidy
@@ -146,7 +147,7 @@ static: pkg/goods/bins/k0s \
146147
pkg/goods/bins/kubectl-support_bundle \
147148
pkg/goods/bins/local-artifact-mirror \
148149
pkg/goods/internal/bins/kubectl-kots
149-
150+
150151
.PHONY: embedded-cluster-linux-amd64
151152
embedded-cluster-linux-amd64: static go.mod
152153
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(LD_FLAGS)" -o ./output/bin/$(APP_NAME) ./cmd/embedded-cluster

cmd/embedded-cluster/install.go

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,17 @@ func RunHostPreflights(c *cli.Context) error {
8888
if err != nil {
8989
return fmt.Errorf("unable to read host preflights: %w", err)
9090
}
91+
92+
chpfs, err := preflights.GetClusterHostPreflights(c.Context)
93+
if err != nil {
94+
return fmt.Errorf("unable to get cluster host preflights: %w", err)
95+
}
96+
97+
for _, h := range chpfs {
98+
hpf.Collectors = append(hpf.Collectors, h.Spec.Collectors...)
99+
hpf.Analyzers = append(hpf.Analyzers, h.Spec.Analyzers...)
100+
}
101+
91102
return runHostPreflights(c, hpf)
92103
}
93104

@@ -96,25 +107,61 @@ func runHostPreflights(c *cli.Context, hpf *v1beta2.HostPreflightSpec) error {
96107
return nil
97108
}
98109
pb := spinner.Start()
99-
pb.Infof("Running host preflights on node")
110+
if c.Bool("skip-host-preflights") {
111+
pb.Infof("Skipping host preflights")
112+
pb.Close()
113+
return nil
114+
}
115+
pb.Infof("Running host preflights")
100116
output, err := preflights.Run(c.Context, hpf)
101117
if err != nil {
102118
pb.CloseWithError()
103119
return fmt.Errorf("host preflights failed: %w", err)
104120
}
121+
122+
err = output.SaveToDisk()
123+
if err != nil {
124+
pb.CloseWithError()
125+
return fmt.Errorf("failed to save preflights output: %w", err)
126+
}
127+
105128
if output.HasFail() {
129+
s := "failures"
130+
if len(output.Fail) == 1 {
131+
s = "failure"
132+
}
133+
msg := fmt.Sprintf("Host preflights have %d %s", len(output.Fail), s)
134+
if output.HasWarn() {
135+
s = "warnings"
136+
if len(output.Warn) == 1 {
137+
s = "warning"
138+
}
139+
msg += fmt.Sprintf(" and %d %s", len(output.Warn), s)
140+
}
141+
142+
pb.Errorf(msg)
106143
pb.CloseWithError()
107-
output.PrintTable()
144+
output.PrintTableWithoutInfo()
108145
return fmt.Errorf("preflights haven't passed on the host")
109146
}
110-
if !output.HasWarn() || c.Bool("no-prompt") {
147+
if !output.HasWarn() {
111148
pb.Close()
112-
output.PrintTable()
113149
return nil
114150
}
151+
if c.Bool("no-prompt") {
152+
// We have warnings but we are not in interactive mode
153+
// so we just print the warnings and continue
154+
pb.Close()
155+
output.PrintTableWithoutInfo()
156+
return nil
157+
}
158+
s := "warnings"
159+
if len(output.Warn) == 1 {
160+
s = "warning"
161+
}
162+
pb.Warnf("Host preflights have %d %s", len(output.Warn), s)
115163
pb.CloseWithError()
116-
output.PrintTable()
117-
logrus.Infof("Host preflights have warnings")
164+
output.PrintTableWithoutInfo()
118165
if !prompts.New().Confirm("Do you want to continue ?", false) {
119166
return fmt.Errorf("user aborted")
120167
}
@@ -390,7 +437,7 @@ func waitForK0s() error {
390437
}
391438

392439
// runOutro calls Outro() in all enabled addons by means of Applier.
393-
func runOutro(c *cli.Context) error {
440+
func runOutro(c *cli.Context, adminConsolePwd string) error {
394441
os.Setenv("KUBECONFIG", defaults.PathToKubeConfig())
395442
opts := []addons.Option{}
396443

@@ -413,12 +460,33 @@ func runOutro(c *cli.Context) error {
413460
if ab := c.String("airgap-bundle"); ab != "" {
414461
opts = append(opts, addons.WithAirgapBundle(ab))
415462
}
463+
opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd))
416464
if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
417465
opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
418466
}
419467
return addons.NewApplier(opts...).Outro(c.Context)
420468
}
421469

470+
func askAdminConsolePassword(c *cli.Context) (string, error) {
471+
defaultPass := "password"
472+
if c.Bool("no-prompt") {
473+
logrus.Infof("Admin Console password set to: %s", defaultPass)
474+
return defaultPass, nil
475+
}
476+
maxTries := 3
477+
for i := 0; i < maxTries; i++ {
478+
promptA := prompts.New().Password("Enter an Admin Console password:")
479+
promptB := prompts.New().Password("Confirm password:")
480+
481+
if promptA == promptB {
482+
// TODO: Should we add extra password validation here? e.g length, complexity etc
483+
return promptA, nil
484+
}
485+
logrus.Info("Passwords don't match, please try again.")
486+
}
487+
return "", fmt.Errorf("unable to set Admin Console password after %d tries", maxTries)
488+
}
489+
422490
// installCommands executes the "install" command. This will ensure that a k0s.yaml file exists
423491
// and then run `k0s install` to apply the cluster. Once this is finished then a "kubeconfig"
424492
// file is created. Resulting kubeconfig is stored in the configuration dir.
@@ -476,6 +544,11 @@ var installCommand = &cli.Command{
476544
Usage: "Use the system proxy settings for the install operation. These variables are currently only passed through to Velero and the Admin Console.",
477545
Hidden: true,
478546
},
547+
&cli.BoolFlag{
548+
Name: "skip-host-preflights",
549+
Usage: "Skip host preflight checks. This is not recommended unless you are sure your system is compatible.",
550+
Value: false,
551+
},
479552
},
480553
Action: func(c *cli.Context) error {
481554
logrus.Debugf("checking if %s is already installed", binName)
@@ -510,6 +583,11 @@ var installCommand = &cli.Command{
510583
metrics.ReportApplyFinished(c, err)
511584
return err
512585
}
586+
adminConsolePwd, err := askAdminConsolePassword(c)
587+
if err != nil {
588+
metrics.ReportApplyFinished(c, err)
589+
return err
590+
}
513591
logrus.Debugf("running host preflights")
514592
if err := RunHostPreflights(c); err != nil {
515593
err := fmt.Errorf("unable to finish preflight checks: %w", err)
@@ -549,7 +627,7 @@ var installCommand = &cli.Command{
549627
return err
550628
}
551629
logrus.Debugf("running outro")
552-
if err := runOutro(c); err != nil {
630+
if err := runOutro(c, adminConsolePwd); err != nil {
553631
metrics.ReportApplyFinished(c, err)
554632
return err
555633
}

cmd/embedded-cluster/join.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ var joinCommand = &cli.Command{
124124
Usage: "Enable high availability",
125125
Hidden: true,
126126
},
127+
&cli.BoolFlag{
128+
Name: "skip-host-preflights",
129+
Usage: "Skip host preflight checks. This is not recommended unless you are sure your system is compatible.",
130+
Value: false,
131+
},
127132
},
128133
Before: func(c *cli.Context) error {
129134
if os.Getuid() != 0 {

cmd/embedded-cluster/restore.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,11 @@ var restoreCommand = &cli.Command{
823823
Usage: "Use the system proxy settings for the restore operation. These variables are currently only passed through to Velero.",
824824
Hidden: true,
825825
},
826+
&cli.BoolFlag{
827+
Name: "skip-host-preflights",
828+
Usage: "Skip host preflight checks. This is not recommended unless you are sure your system is compatible.",
829+
Value: false,
830+
},
826831
},
827832
Before: func(c *cli.Context) error {
828833
if os.Getuid() != 0 {

e2e/install_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ func TestHostPreflight(t *testing.T) {
249249

250250
t.Logf("%s: installing test dependencies on node 0", time.Now().Format(time.RFC3339))
251251
commands := [][]string{
252-
{"dnf", "install", "-y", "openssh-server", "binutils", "tar"},
252+
{"dnf", "install", "-y", "openssh-server", "binutils", "tar", "fio"},
253253
{"systemctl", "enable", "sshd"},
254254
{"systemctl", "start", "sshd"},
255255
}

e2e/scripts/default-install.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,12 @@ check_openebs_storage_class() {
5656
}
5757

5858
main() {
59-
if embedded-cluster install --no-prompt --license /assets/license.yaml 2>&1 | tee /tmp/log ; then
59+
if embedded-cluster install --no-prompt --skip-host-preflights --license /assets/license.yaml 2>&1 | tee /tmp/log ; then
6060
echo "Expected installation to fail with a license provided"
6161
exit 1
6262
fi
6363

64-
if ! embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
64+
if ! embedded-cluster install --no-prompt --skip-host-preflights 2>&1 | tee /tmp/log ; then
6565
cat /etc/os-release
6666
echo "Failed to install embedded-cluster"
6767
exit 1

e2e/scripts/embedded-preflight.sh

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,13 @@ embed_preflight() {
124124
}
125125

126126
has_applied_host_preflight() {
127-
if ! grep -q "Port 24 is available" /tmp/log ; then
127+
if ! grep -q "Another process was already listening on port 22" /tmp/log ; then
128128
return 1
129129
fi
130-
if ! grep -q "Another process was already listening on port 22" /tmp/log ; then
130+
}
131+
132+
has_stored_host_preflight_results() {
133+
if [ ! -f /var/lib/embedded-cluster/support/host-preflight-results.json ]; then
131134
return 1
132135
fi
133136
}
@@ -151,29 +154,40 @@ wait_for_healthy_node() {
151154
main() {
152155
cp -Rfp /usr/local/bin/embedded-cluster /usr/local/bin/embedded-cluster-copy
153156
embed_preflight "$preflight_with_failure"
154-
if embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
157+
if /usr/local/bin/embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
155158
cat /tmp/log
156-
echo "Expected installation to fail"
159+
echo "preflight_with_failure: Expected installation to fail"
157160
exit 1
158161
fi
159162
if ! has_applied_host_preflight; then
160-
echo "Install hasn't applied host preflight"
163+
echo "preflight_with_failure: Install hasn't applied host preflight"
164+
cat /tmp/log
165+
exit 1
166+
fi
167+
if ! has_stored_host_preflight_results; then
168+
echo "preflight_with_failure: Install hasn't stored host preflight results to disk"
161169
cat /tmp/log
162170
exit 1
163171
fi
172+
rm /var/lib/embedded-cluster/support/host-preflight-results.json
164173
mv /tmp/log /tmp/log-failure
165174
embed_preflight "$preflight_with_warning"
166-
if ! embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
175+
if ! /usr/local/bin/embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
167176
cat /etc/os-release
168-
echo "Failed to install embedded-cluster"
177+
echo "preflight_with_warning: Failed to install embedded-cluster"
169178
exit 1
170179
fi
171180
if ! grep -q "Admin Console is ready!" /tmp/log; then
172-
echo "Failed to validate that the Admin Console is ready"
181+
echo "preflight_with_warning: Failed to validate that the Admin Console is ready"
173182
exit 1
174183
fi
175184
if ! has_applied_host_preflight; then
176-
echo "Install hasn't applied host preflight"
185+
echo "preflight_with_warning: Install hasn't applied host preflight"
186+
cat /tmp/log
187+
exit 1
188+
fi
189+
if ! has_stored_host_preflight_results; then
190+
echo "preflight_with_warning: Install hasn't stored host preflight results to disk"
177191
cat /tmp/log
178192
exit 1
179193
fi

e2e/scripts/restore-installation-airgap.exp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ set dr_aws_s3_prefix [lindex $argv 3]
1212
set dr_aws_access_key_id [lindex $argv 4]
1313
set dr_aws_secret_access_key [lindex $argv 5]
1414

15-
spawn embedded-cluster restore --airgap-bundle /assets/release.airgap --proxy
15+
spawn embedded-cluster restore --skip-host-preflights --airgap-bundle /assets/release.airgap --proxy
1616

1717
expect {
1818
"Enter information to configure access to your backup storage location." {}

e2e/scripts/restore-installation.exp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ set dr_aws_s3_prefix [lindex $argv 3]
1212
set dr_aws_access_key_id [lindex $argv 4]
1313
set dr_aws_secret_access_key [lindex $argv 5]
1414

15-
spawn embedded-cluster restore
15+
spawn embedded-cluster restore --skip-host-preflights
1616

1717
expect {
1818
"Enter information to configure access to your backup storage location." {}

e2e/scripts/restore-multi-node-airgap-phase1.exp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ set dr_aws_s3_prefix [lindex $argv 3]
1212
set dr_aws_access_key_id [lindex $argv 4]
1313
set dr_aws_secret_access_key [lindex $argv 5]
1414

15-
spawn embedded-cluster restore --airgap-bundle /assets/release.airgap --proxy
15+
spawn embedded-cluster restore --skip-host-preflights --airgap-bundle /assets/release.airgap --proxy
1616

1717
expect {
1818
"Enter information to configure access to your backup storage location." {}

0 commit comments

Comments
 (0)