@@ -11,15 +11,14 @@ import (
1111 "time"
1212
1313 controlplanev1 "github.com/siderolabs/cluster-api-control-plane-provider-talos/api/v1alpha3"
14- "github.com/siderolabs/talos/pkg/machinery/api/machine"
14+ machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine"
1515 talosclient "github.com/siderolabs/talos/pkg/machinery/client"
1616 clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1717 "sigs.k8s.io/controller-runtime/pkg/client"
1818)
1919
2020func (r * TalosControlPlaneReconciler ) etcdHealthcheck (ctx context.Context , tcp * controlplanev1.TalosControlPlane , ownedMachines []clusterv1.Machine ) error {
2121 ctx , cancel := context .WithTimeout (ctx , time .Second * 5 )
22-
2322 defer cancel ()
2423
2524 machines := []clusterv1.Machine {}
@@ -30,70 +29,80 @@ func (r *TalosControlPlaneReconciler) etcdHealthcheck(ctx context.Context, tcp *
3029 }
3130 }
3231
33- c , err := r .talosconfigForMachines (ctx , tcp , machines ... )
34- if err != nil {
35- return err
36- }
37-
38- defer c .Close () //nolint:errcheck
39-
40- service := "etcd"
41-
42- params := make ([]interface {}, 0 , len (machines )* 2 )
32+ params := make ([]any , 0 , len (machines )* 2 )
4333 for _ , machine := range machines {
4434 params = append (params , "node" , machine .Name )
4535 }
4636
4737 r .Log .Info ("verifying etcd health on all nodes" , params ... )
4838
49- svcs , err := c .ServiceInfo (ctx , service )
50- if err != nil {
51- return err
52- }
39+ const service = "etcd"
5340
54- // check that etcd service is healthy on all nodes
55- for _ , svc := range svcs {
56- node := svc .Metadata .GetHostname ()
41+ // list of discovered etcd members, updated on each iteration
42+ members := map [string ]struct {}{}
5743
58- if len (svc .Service .Events .Events ) == 0 {
59- return fmt .Errorf ("%s: no events recorded yet for service %q" , node , service )
60- }
44+ for i , machine := range machines {
45+ // loop for each machine, the client created has endpoints which point to a single machine
46+ if err := func () error {
47+ c , err := r .talosconfigForMachines (ctx , tcp , machine )
48+ if err != nil {
49+ return err
50+ }
6151
62- lastEvent := svc .Service .Events .Events [len (svc .Service .Events .Events )- 1 ]
63- if lastEvent .State != "Running" {
64- return fmt .Errorf ("%s: service %q not in expected state %q: current state [%s] %s" , node , service , "Running" , lastEvent .State , lastEvent .Msg )
65- }
52+ defer c .Close () //nolint:errcheck
6653
67- if ! svc . Service . GetHealth (). GetHealthy () {
68- return fmt . Errorf ( "%s: service is not healthy: %s" , node , service )
69- }
70- }
54+ svcs , err := c . ServiceInfo ( ctx , service )
55+ if err != nil {
56+ return err
57+ }
7158
72- resp , err := c .EtcdMemberList (ctx , & machine.EtcdMemberListRequest {})
73- if err != nil {
74- return err
75- }
59+ // check that etcd service is healthy on the node
60+ for _ , svc := range svcs {
61+ node := svc .Metadata .GetHostname ()
7662
77- members := map [string ]struct {}{}
63+ if len (svc .Service .Events .Events ) == 0 {
64+ return fmt .Errorf ("%s: no events recorded yet for service %q" , node , service )
65+ }
66+
67+ lastEvent := svc .Service .Events .Events [len (svc .Service .Events .Events )- 1 ]
68+ if lastEvent .State != "Running" {
69+ return fmt .Errorf ("%s: service %q not in expected state %q: current state [%s] %s" , node , service , "Running" , lastEvent .State , lastEvent .Msg )
70+ }
71+
72+ if ! svc .Service .GetHealth ().GetHealthy () {
73+ return fmt .Errorf ("%s: service is not healthy: %s" , node , service )
74+ }
75+ }
76+
77+ resp , err := c .EtcdMemberList (ctx , & machineapi.EtcdMemberListRequest {})
78+ if err != nil {
79+ return err
80+ }
7881
79- for i , message := range resp .Messages {
80- actualMembers := len (message .Members )
81- expectedMembers := len (machines )
82+ for _ , message := range resp .Messages {
83+ actualMembers := len (message .Members )
84+ expectedMembers := len (machines )
8285
83- node := message .Metadata .GetHostname ()
86+ node := message .Metadata .GetHostname ()
8487
85- // check that the count of members is the same on all nodes
86- if actualMembers != expectedMembers {
87- return fmt .Errorf ("%s: expected to have %d members, got %d" , node , expectedMembers , actualMembers )
88- }
88+ // check that the count of members is the same on all nodes
89+ if actualMembers != expectedMembers {
90+ return fmt .Errorf ("%s: expected to have %d members, got %d" , node , expectedMembers , actualMembers )
91+ }
8992
90- // check that member list is the same on all nodes
91- for _ , member := range message .Members {
92- if _ , found := members [member .Hostname ]; i > 0 && ! found {
93- return fmt .Errorf ("%s: found extra etcd member %s" , node , member .Hostname )
93+ // check that member list is the same on all nodes
94+ for _ , member := range message .Members {
95+ if _ , found := members [member .Hostname ]; i > 0 && ! found {
96+ return fmt .Errorf ("%s: found extra etcd member %s" , node , member .Hostname )
97+ }
98+
99+ members [member .Hostname ] = struct {}{}
100+ }
94101 }
95102
96- members [member .Hostname ] = struct {}{}
103+ return nil
104+ }(); err != nil {
105+ return fmt .Errorf ("error checking etcd health on machine %q: %w" , machines [i ].Name , err )
97106 }
98107 }
99108
@@ -118,14 +127,14 @@ func (r *TalosControlPlaneReconciler) gracefulEtcdLeave(ctx context.Context, c *
118127 if svc .Service .State != "Finished" {
119128 r .Log .Info ("forfeiting leadership" , "machine" , machineToLeave .Status .NodeRef .Name )
120129
121- _ , err = c .EtcdForfeitLeadership (ctx , & machine .EtcdForfeitLeadershipRequest {})
130+ _ , err = c .EtcdForfeitLeadership (ctx , & machineapi .EtcdForfeitLeadershipRequest {})
122131 if err != nil {
123132 return err
124133 }
125134
126135 r .Log .Info ("leaving etcd" , "machine" , machineToLeave .Name , "node" , machineToLeave .Status .NodeRef .Name )
127136
128- err = c .EtcdLeaveCluster (ctx , & machine .EtcdLeaveClusterRequest {})
137+ err = c .EtcdLeaveCluster (ctx , & machineapi .EtcdLeaveClusterRequest {})
129138 if err != nil {
130139 return err
131140 }
@@ -137,7 +146,7 @@ func (r *TalosControlPlaneReconciler) gracefulEtcdLeave(ctx context.Context, c *
137146
138147// forceEtcdLeave removes a given machine from the etcd cluster by telling another CP node to remove the member.
139148// This is used in times when the machine was deleted out from under us.
140- func (r * TalosControlPlaneReconciler ) forceEtcdLeave (ctx context.Context , c * talosclient.Client , member * machine .EtcdMember ) error {
149+ func (r * TalosControlPlaneReconciler ) forceEtcdLeave (ctx context.Context , c * talosclient.Client , member * machineapi .EtcdMember ) error {
141150 ctx , cancel := context .WithTimeout (ctx , time .Second * 5 )
142151
143152 defer cancel ()
@@ -146,7 +155,7 @@ func (r *TalosControlPlaneReconciler) forceEtcdLeave(ctx context.Context, c *tal
146155
147156 return c .EtcdRemoveMemberByID (
148157 ctx ,
149- & machine .EtcdRemoveMemberByIDRequest {
158+ & machineapi .EtcdRemoveMemberByIDRequest {
150159 MemberId : member .Id ,
151160 },
152161 )
@@ -199,7 +208,7 @@ func (r *TalosControlPlaneReconciler) auditEtcd(ctx context.Context, tcp *contro
199208
200209 defer c .Close () //nolint:errcheck
201210
202- response , err := c .EtcdMemberList (ctx , & machine .EtcdMemberListRequest {})
211+ response , err := c .EtcdMemberList (ctx , & machineapi .EtcdMemberListRequest {})
203212 if err != nil {
204213 return fmt .Errorf ("error getting etcd members via %q (endpoints %v): %w" , designatedCPMachine .Name , c .GetConfigContext ().Endpoints , err )
205214 }
0 commit comments