@@ -29,6 +29,8 @@ import (
2929 apierrors "k8s.io/apimachinery/pkg/api/errors"
3030 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3131 "k8s.io/apimachinery/pkg/runtime"
32+ errorsutil "k8s.io/apimachinery/pkg/util/errors"
33+ "k8s.io/apimachinery/pkg/util/wait"
3234 clientset "k8s.io/client-go/kubernetes"
3335 "k8s.io/client-go/tools/clientcmd"
3436 certutil "k8s.io/client-go/util/cert"
@@ -90,8 +92,8 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte
9092 if err := getNodeRegistration (kubeconfigDir , client , & initcfg .NodeRegistration ); err != nil {
9193 return nil , errors .Wrap (err , "failed to get node registration" )
9294 }
93- // gets the APIEndpoint for the current node from then ClusterStatus in the kubeadm-config ConfigMap
94- if err := getAPIEndpoint (configMap . Data , initcfg .NodeRegistration .Name , & initcfg .LocalAPIEndpoint ); err != nil {
95+ // gets the APIEndpoint for the current node
96+ if err := getAPIEndpoint (client , initcfg .NodeRegistration .Name , & initcfg .LocalAPIEndpoint ); err != nil {
9597 return nil , errors .Wrap (err , "failed to getAPIEndpoint" )
9698 }
9799 } else {
@@ -181,23 +183,83 @@ func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) {
181183 return strings .TrimPrefix (cert .Subject .CommonName , constants .NodesUserPrefix ), nil
182184}
183185
184- // getAPIEndpoint returns the APIEndpoint for the current node
185- func getAPIEndpoint (data map [string ]string , nodeName string , apiEndpoint * kubeadmapi.APIEndpoint ) error {
186- // gets the ClusterStatus from kubeadm-config
187- clusterStatus , err := UnmarshalClusterStatus (data )
186+ func getAPIEndpoint (client clientset.Interface , nodeName string , apiEndpoint * kubeadmapi.APIEndpoint ) error {
187+ return getAPIEndpointWithBackoff (client , nodeName , apiEndpoint , constants .StaticPodMirroringDefaultRetry )
188+ }
189+
190+ func getAPIEndpointWithBackoff (client clientset.Interface , nodeName string , apiEndpoint * kubeadmapi.APIEndpoint , backoff wait.Backoff ) error {
191+ var err error
192+ var errs []error
193+
194+ if err = getAPIEndpointFromPodAnnotation (client , nodeName , apiEndpoint , backoff ); err == nil {
195+ return nil
196+ }
197+ errs = append (errs , errors .WithMessagef (err , "could not retrieve API endpoints for node %q using pod annotations" , nodeName ))
198+
199+ // NB: this is a fallback when there is no annotation found in the API server pod that contains
200+ // the API endpoint, and so we fallback to reading the ClusterStatus struct present in the
201+ // kubeadm-config ConfigMap. This can happen for example, when performing the first
202+ // `kubeadm upgrade apply` and `kubeadm upgrade node` cycle on the whole cluster. This logic
203+ // will be removed when the cluster status struct is removed from the kubeadm-config ConfigMap.
204+ if err = getAPIEndpointFromClusterStatus (client , nodeName , apiEndpoint ); err == nil {
205+ return nil
206+ }
207+ errs = append (errs , errors .WithMessagef (err , "could not retrieve API endpoints for node %q using cluster status" , nodeName ))
208+
209+ return errorsutil .NewAggregate (errs )
210+ }
211+
212+ func getAPIEndpointFromPodAnnotation (client clientset.Interface , nodeName string , apiEndpoint * kubeadmapi.APIEndpoint , backoff wait.Backoff ) error {
213+ var rawAPIEndpoint string
214+ var lastErr error
215+ // Let's tolerate some unexpected transient failures from the API server or load balancers. Also, if
216+ // static pods were not yet mirrored into the API server we want to wait for this propagation.
217+ err := wait .ExponentialBackoff (backoff , func () (bool , error ) {
218+ rawAPIEndpoint , lastErr = getRawAPIEndpointFromPodAnnotationWithoutRetry (client , nodeName )
219+ return lastErr == nil , nil
220+ })
188221 if err != nil {
189222 return err
190223 }
224+ parsedAPIEndpoint , err := kubeadmapi .APIEndpointFromString (rawAPIEndpoint )
225+ if err != nil {
226+ return errors .Wrapf (err , "could not parse API endpoint for node %q" , nodeName )
227+ }
228+ * apiEndpoint = parsedAPIEndpoint
229+ return nil
230+ }
191231
192- // gets the APIEndpoint for the current machine from the ClusterStatus
193- e , ok := clusterStatus .APIEndpoints [nodeName ]
194- if ! ok {
195- return errors .New ("failed to get APIEndpoint information for this node" )
232+ func getRawAPIEndpointFromPodAnnotationWithoutRetry (client clientset.Interface , nodeName string ) (string , error ) {
233+ podList , err := client .CoreV1 ().Pods (metav1 .NamespaceSystem ).List (
234+ context .TODO (),
235+ metav1.ListOptions {
236+ FieldSelector : fmt .Sprintf ("spec.nodeName=%s" , nodeName ),
237+ LabelSelector : fmt .Sprintf ("component=%s,tier=%s" , constants .KubeAPIServer , constants .ControlPlaneTier ),
238+ },
239+ )
240+ if err != nil {
241+ return "" , errors .Wrap (err , "could not retrieve list of pods to determine api server endpoints" )
242+ }
243+ if len (podList .Items ) != 1 {
244+ return "" , errors .Errorf ("API server pod for node name %q has %d entries, only one was expected" , nodeName , len (podList .Items ))
245+ }
246+ if apiServerEndpoint , ok := podList .Items [0 ].Annotations [constants .KubeAPIServerAdvertiseAddressEndpointAnnotationKey ]; ok {
247+ return apiServerEndpoint , nil
196248 }
249+ return "" , errors .Errorf ("API server pod for node name %q hasn't got a %q annotation, cannot retrieve API endpoint" , nodeName , constants .KubeAPIServerAdvertiseAddressEndpointAnnotationKey )
250+ }
197251
198- apiEndpoint .AdvertiseAddress = e .AdvertiseAddress
199- apiEndpoint .BindPort = e .BindPort
200- return nil
252+ // TODO: remove after 1.20, when the ClusterStatus struct is removed from the kubeadm-config ConfigMap.
253+ func getAPIEndpointFromClusterStatus (client clientset.Interface , nodeName string , apiEndpoint * kubeadmapi.APIEndpoint ) error {
254+ clusterStatus , err := GetClusterStatus (client )
255+ if err != nil {
256+ return errors .Wrap (err , "could not retrieve cluster status" )
257+ }
258+ if statusAPIEndpoint , ok := clusterStatus .APIEndpoints [nodeName ]; ok {
259+ * apiEndpoint = statusAPIEndpoint
260+ return nil
261+ }
262+ return errors .Errorf ("could not find node %s in the cluster status" , nodeName )
201263}
202264
203265// GetClusterStatus returns the kubeadm cluster status read from the kubeadm-config ConfigMap
0 commit comments