Skip to content

Commit 45cddb0

Browse files
danj-replicateddanj-replicated
authored
feat: check etcd health before controller reset (#301)
* check etcd health before controller reset * error format
1 parent 889752e commit 45cddb0

File tree

3 files changed

+45
-3
lines changed

3 files changed

+45
-3
lines changed

‎cmd/embedded-cluster/reset.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"os/exec"
99

1010
autopilot "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
11+
"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
12+
"github.com/k0sproject/k0s/pkg/etcd"
1113
"github.com/urfave/cli/v2"
1214
corev1 "k8s.io/api/core/v1"
1315
controllerruntime "sigs.k8s.io/controller-runtime"
@@ -34,12 +36,15 @@ type hostInfo struct {
3436
}
3537

3638
type k0sStatus struct {
37-
Role string `json:"Role"`
38-
Vars k0sVars `json:"K0sVars"`
39+
Role string `json:"Role"`
40+
Vars k0sVars `json:"K0sVars"`
41+
ClusterConfig v1beta1.ClusterConfig `json:"ClusterConfig"`
3942
}
4043

4144
type k0sVars struct {
4245
KubeletAuthConfigPath string `json:"KubeletAuthConfigPath"`
46+
CertRootDir string `json:"CertRootDir"`
47+
EtcdCertDir string `json:"EtcdCertDir"`
4348
}
4449

4550
var (
@@ -109,6 +114,7 @@ func (h *hostInfo) configureKubernetesClient() {
109114
return
110115
}
111116
autopilot.AddToScheme(h.Kclient.Scheme())
117+
v1beta1.AddToScheme(h.Kclient.Scheme())
112118
}
113119

114120
// getHostName fetches the hostname for the node
@@ -159,11 +165,19 @@ func (h *hostInfo) checkResetSafety(c *cli.Context) (bool, string, error) {
159165
return false, "", fmt.Errorf("unable to load cluster client: %w", h.KclientError)
160166
}
161167

168+
etcdClient, err := etcd.NewClient(h.Status.Vars.CertRootDir, h.Status.Vars.EtcdCertDir, h.Status.ClusterConfig.Spec.Storage.Etcd)
169+
if err != nil {
170+
return false, "", fmt.Errorf("unable to create etcd client: %w", err)
171+
}
172+
if etcdClient.Health(c.Context) != nil {
173+
return false, "Etcd is not ready. Please wait up to 5 minutes and try again.", nil
174+
}
175+
162176
// get a rough picture of the cluster topology
163177
workers := []string{}
164178
controllers := []string{}
165179
nodeList := corev1.NodeList{}
166-
err := h.Kclient.List(c.Context, &nodeList)
180+
err = h.Kclient.List(c.Context, &nodeList)
167181
if err != nil {
168182
return false, "", fmt.Errorf("unable to list Nodes: %w", err)
169183
}

‎go.mod‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ require (
3434
github.com/bodgit/windows v1.0.1 // indirect
3535
github.com/cespare/xxhash/v2 v2.2.0 // indirect
3636
github.com/containerd/containerd v1.7.11 // indirect
37+
github.com/coreos/go-semver v0.3.1 // indirect
38+
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
3739
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
3840
github.com/frankban/quicktest v1.14.5 // indirect
3941
github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -55,9 +57,16 @@ require (
5557
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
5658
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
5759
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
60+
go.etcd.io/etcd/api/v3 v3.5.10 // indirect
61+
go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
62+
go.etcd.io/etcd/client/v3 v3.5.10 // indirect
5863
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
5964
golang.org/x/mod v0.14.0 // indirect
6065
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
66+
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
67+
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
68+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
69+
google.golang.org/grpc v1.58.3 // indirect
6170
helm.sh/helm/v3 v3.13.3 // indirect
6271
k8s.io/component-base v0.29.0 // indirect
6372
)

‎go.sum‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
3535
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
3636
github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
3737
github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
38+
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
39+
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
40+
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
41+
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
3842
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
3943
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
4044
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -106,6 +110,7 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe
106110
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
107111
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
108112
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
113+
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
109114
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
110115
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
111116
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -364,6 +369,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
364369
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
365370
github.com/zitadel/oidc/v2 v2.7.0 h1:IGX4EDk6tegTjUSsZDWeTfLseFU0BdJ/Glf1tgys2lU=
366371
github.com/zitadel/oidc/v2 v2.7.0/go.mod h1:zkUkVJS0sDVy9m0UA9RgO3f8i/C0rtjvXU36UJj7T+0=
372+
go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k=
373+
go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI=
374+
go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0=
375+
go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U=
376+
go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao=
377+
go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc=
367378
go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
368379
go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
369380
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -483,9 +494,17 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
483494
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
484495
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
485496
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
497+
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA=
498+
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
499+
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI=
500+
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
501+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
502+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA=
486503
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
487504
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
488505
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
506+
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
507+
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
489508
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
490509
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
491510
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

0 commit comments

Comments
 (0)