Skip to content

Commit ac53fe2

Browse files
lmiccinilmiccini
committed
Use predictableIPs labels to advertise pods addresses
Octavia and Designate have special configmaps storing ip addresses that are internally managed. $ oc get configmaps octavia-hmport-map -o yaml apiVersion: v1 data: hm_worker-0: 172.23.0.103 hm_worker-1: 172.23.0.105 hm_worker-2: 172.23.0.107 rsyslog_worker-0: 172.23.0.104 rsyslog_worker-1: 172.23.0.106 rsyslog_worker-2: 172.23.0.108 kind: ConfigMap metadata: name: octavia-hmport-map namespace: openstack oc get cm designate-bind-ip-map -o yaml apiVersion: v1 data: bind_address_0: 172.67.0.100 bind_address_1: 172.67.0.101 bind_address_2: 172.67.0.102 $ oc get cm designate-mdns-ip-map -o yaml apiVersion: v1 data: mdns_address_0: 172.67.0.97 mdns_address_1: 172.67.0.98 mdns_address_2: 172.67.0.99 These IPs should also be advertised in addition to the network-attachment-definitions for each pod. This change watches pods for the presence of a predictableip label, if this is found it appends the ip to the already-existing frrconfiguration created for the network-attachment-definition ip. Jira: https://issues.redhat.com/browse/OSPRH-20083
1 parent 7086a5e commit ac53fe2

File tree

2 files changed

+259
-57
lines changed

2 files changed

+259
-57
lines changed

‎controllers/network/bgpconfiguration_controller.go‎

Lines changed: 157 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -243,38 +243,61 @@ func (r *BGPConfigurationReconciler) SetupWithManager(ctx context.Context, mgr c
243243
// Skip if
244244
// * no NAD annotation was configured on old object OR
245245
// * no NAD annotation was configured on new object AND
246+
// * no predictableip label was configured on old object OR
247+
// * no predictableip label was configured on new object AND
246248
// * the resourceVersion has not changed
247249

248-
oldConfigured := true
250+
oldNADConfigured := true
249251
if val, ok := e.ObjectOld.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
250-
oldConfigured = false
252+
oldNADConfigured = false
251253
}
252-
newConfigured := true
254+
newNADConfigured := true
253255
if val, ok := e.ObjectNew.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
254-
newConfigured = false
256+
newNADConfigured = false
255257
}
256258

257-
return (oldConfigured || newConfigured) && e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion()
259+
oldPredictableIPConfigured := true
260+
if val, ok := e.ObjectOld.GetLabels()["predictableip"]; !ok || len(val) == 0 {
261+
oldPredictableIPConfigured = false
262+
}
263+
newPredictableIPConfigured := true
264+
if val, ok := e.ObjectNew.GetLabels()["predictableip"]; !ok || len(val) == 0 {
265+
newPredictableIPConfigured = false
266+
}
267+
268+
return (oldNADConfigured || newNADConfigured || oldPredictableIPConfigured || newPredictableIPConfigured) && e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion()
258269
},
259270
DeleteFunc: func(e event.DeleteEvent) bool {
260271
// Skip if
261-
// * NAD annotation key is missing
272+
// * NAD annotation key is missing AND predictableip label is missing
262273
// * there is no additional network configured
263-
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
264-
return false
274+
nadConfigured := false
275+
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; ok && len(val) > 0 {
276+
nadConfigured = true
265277
}
266278

267-
return true
279+
predictableIPConfigured := false
280+
if val, ok := e.Object.GetLabels()["predictableip"]; ok && len(val) > 0 {
281+
predictableIPConfigured = true
282+
}
283+
284+
return nadConfigured || predictableIPConfigured
268285
},
269286
CreateFunc: func(e event.CreateEvent) bool {
270287
// Skip if
271-
// * NAD annotation key is missing
288+
// * NAD annotation key is missing AND predictableip label is missing
272289
// * there is no additional network configured
273-
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
274-
return false
290+
nadConfigured := false
291+
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; ok && len(val) > 0 {
292+
nadConfigured = true
275293
}
276294

277-
return true
295+
predictableIPConfigured := false
296+
if val, ok := e.Object.GetLabels()["predictableip"]; ok && len(val) > 0 {
297+
predictableIPConfigured = true
298+
}
299+
300+
return nadConfigured || predictableIPConfigured
278301
},
279302
}
280303

@@ -315,7 +338,7 @@ func (r *BGPConfigurationReconciler) SetupWithManager(ctx context.Context, mgr c
315338

316339
return ctrl.NewControllerManagedBy(mgr).
317340
For(&networkv1.BGPConfiguration{}).
318-
// Watch pods which have additional networks configured with k8s_networkv1.NetworkAttachmentAnnot annotation in the same namespace
341+
// Watch pods which have additional networks configured with k8s_networkv1.NetworkAttachmentAnnot annotation or predictableip label in the same namespace
319342
Watches(&corev1.Pod{},
320343
podFN,
321344
builder.WithPredicates(pPod)).
@@ -479,7 +502,7 @@ func (r *BGPConfigurationReconciler) deleteStaleFRRConfigurations(ctx context.Co
479502

480503
// getPodNetworkDetails - returns the podDetails for a list of pods in status.phase: Running
481504
// where the pod has the multus k8s_networkv1.NetworkAttachmentAnnot annotation
482-
// and its value is not '[]'
505+
// and its value is not '[]' OR has a predictableip label
483506
func getPodNetworkDetails(
484507
ctx context.Context,
485508
h *helper.Helper,
@@ -499,14 +522,24 @@ func getPodNetworkDetails(
499522
if pod.Status.Phase != corev1.PodRunning {
500523
continue
501524
}
502-
if netAttachString, ok := pod.Annotations[k8s_networkv1.NetworkAttachmentAnnot]; ok && netAttachString != "[]" {
503-
// get the elements from val to validate the status annotation has the right length
504-
netAttach := []k8s_networkv1.NetworkSelectionElement{}
505-
err := json.Unmarshal([]byte(netAttachString), &netAttach)
506-
if err != nil {
507-
return nil, fmt.Errorf("failed to decode networks %s: %w", netAttachString, err)
508-
}
509525

526+
// Check for pods with predictableip label
527+
hasPredictableIP := false
528+
predictableIP := ""
529+
if ipValue, ok := pod.Labels["predictableip"]; ok && ipValue != "" {
530+
hasPredictableIP = true
531+
predictableIP = ipValue
532+
}
533+
534+
hasNetworkAttachment := false
535+
var netAttachString string
536+
if netAttach, ok := pod.Annotations[k8s_networkv1.NetworkAttachmentAnnot]; ok && netAttach != "[]" {
537+
hasNetworkAttachment = true
538+
netAttachString = netAttach
539+
}
540+
541+
// Process pod if it has either network attachments or predictableip label
542+
if hasNetworkAttachment || hasPredictableIP {
510543
// verify the nodeName information is already present in the pod spec, otherwise report an error to reconcile
511544
if pod.Spec.NodeName == "" {
512545
return detailList, fmt.Errorf("empty spec.nodeName on pod %s", pod.Name)
@@ -518,55 +551,121 @@ func getPodNetworkDetails(
518551
Node: pod.Spec.NodeName,
519552
}
520553

521-
netsStatus, err := nad.GetNetworkStatusFromAnnotation(pod.Annotations)
522-
if err != nil {
523-
return detailList, fmt.Errorf("failed to get netsStatus from pod annoation - %v: %w", pod.Annotations, err)
524-
}
525-
// on pod start it can happen that the network status annotation does not yet
526-
// reflect all requested networks. return with an error to reconcile if the length
527-
// is <= the status. Note: the status also has the pod network
528-
if len(netsStatus) <= len(netAttach) {
529-
return detailList, fmt.Errorf("metadata.Annotations['k8s.ovn.org/pod-networks'] %s on pod %s, does not match requested networks %s",
530-
pod.GetAnnotations()[k8s_networkv1.NetworkStatusAnnot], pod.Name, netAttachString)
531-
}
554+
var netsStatus []k8s_networkv1.NetworkStatus
555+
556+
// Handle pods with network attachments
557+
if hasNetworkAttachment {
558+
// get the elements from val to validate the status annotation has the right length
559+
netAttach := []k8s_networkv1.NetworkSelectionElement{}
532560

533-
netsStatusCopy := make([]k8s_networkv1.NetworkStatus, len(netsStatus))
534-
copy(netsStatusCopy, netsStatus)
535-
// verify there are IP information for all networks in the status, otherwise report an error to reconcile
536-
for idx, netStat := range netsStatusCopy {
537-
// remove status for the pod interface
538-
// it should always be ovn-kubernetes, but if not, remove status for eth0 which is the pod network
539-
if netStat.Name == "ovn-kubernetes" || netStat.Interface == "eth0" {
540-
removeIndex(netsStatus, idx)
561+
// Validate that the annotation is not empty and contains valid JSON
562+
if netAttachString == "" {
563+
Log.Info(fmt.Sprintf("Skipping pod %s: network attachment annotation is empty", pod.Name))
541564
continue
542565
}
543566

544-
// get ipam configuration from NAD
545-
nadName := strings.TrimPrefix(netStat.Name, pod.Namespace+"/")
546-
netAtt, err := nad.GetNADWithName(
547-
ctx, h, nadName, pod.Namespace)
548-
if err != nil {
549-
return detailList, err
567+
// Check if the annotation looks like JSON (starts with [ or {) or is a simple string reference
568+
trimmed := strings.TrimSpace(netAttachString)
569+
if !strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "{") && !strings.HasPrefix(trimmed, "\"") {
570+
Log.Info(fmt.Sprintf("Skipping pod %s: network attachment annotation does not appear to be valid JSON or string reference: %s", pod.Name, netAttachString))
571+
continue
572+
}
573+
574+
// Log the raw annotation for debugging
575+
Log.Info(fmt.Sprintf("Processing pod %s with network attachment annotation: %s", pod.Name, netAttachString))
576+
577+
// Handle different annotation formats
578+
var err error
579+
if strings.HasPrefix(trimmed, "\"") && strings.HasSuffix(trimmed, "\"") {
580+
// Simple string reference like "bgpnet-worker-3"
581+
var nadName string
582+
err = json.Unmarshal([]byte(netAttachString), &nadName)
583+
if err != nil {
584+
Log.Error(err, fmt.Sprintf("Failed to decode network attachment string reference for pod %s. Raw annotation: %s", pod.Name, netAttachString))
585+
return nil, fmt.Errorf("failed to decode network string reference %s for pod %s: %w", netAttachString, pod.Name, err)
586+
}
587+
// Create a NetworkSelectionElement from the string reference
588+
netAttach = []k8s_networkv1.NetworkSelectionElement{
589+
{
590+
Name: nadName,
591+
Namespace: pod.Namespace,
592+
},
593+
}
594+
Log.Info(fmt.Sprintf("Converted string reference %s to NetworkSelectionElement for pod %s", nadName, pod.Name))
595+
} else {
596+
// Full JSON array format
597+
err = json.Unmarshal([]byte(netAttachString), &netAttach)
598+
if err != nil {
599+
Log.Error(err, fmt.Sprintf("Failed to decode network attachment annotation for pod %s. Raw annotation: %s", pod.Name, netAttachString))
600+
return nil, fmt.Errorf("failed to decode networks %s for pod %s: %w", netAttachString, pod.Name, err)
601+
}
550602
}
551603

552-
ipam, err := nad.GetJSONPathFromConfig(*netAtt, ".ipam")
604+
netsStatus, err = nad.GetNetworkStatusFromAnnotation(pod.Annotations)
553605
if err != nil {
554-
return detailList, err
606+
return detailList, fmt.Errorf("failed to get netsStatus from pod annoation - %v: %w", pod.Annotations, err)
607+
}
608+
Log.Info(fmt.Sprintf("Retrieved network statuses for pod %s: %+v", pod.Name, netsStatus))
609+
// on pod start it can happen that the network status annotation does not yet
610+
// reflect all requested networks. return with an error to reconcile if the length
611+
// is <= the status. Note: the status also has the pod network
612+
if len(netsStatus) <= len(netAttach) {
613+
return detailList, fmt.Errorf("metadata.Annotations['k8s.ovn.org/pod-networks'] %s on pod %s, does not match requested networks %s",
614+
pod.GetAnnotations()[k8s_networkv1.NetworkStatusAnnot], pod.Name, netAttachString)
555615
}
556616

557-
// if the NAD has no ipam configured, skip it as there will be no IP
558-
if ipam == "{}" {
559-
Log.Info(fmt.Sprintf("removing netsStatus for NAD %s for %s, IPAM configuration is empty: %s", netAtt.Name, pod.Name, ipam))
560-
removeIndex(netsStatus, idx)
561-
continue
617+
netsStatusCopy := make([]k8s_networkv1.NetworkStatus, len(netsStatus))
618+
copy(netsStatusCopy, netsStatus)
619+
// verify there are IP information for all networks in the status, otherwise report an error to reconcile
620+
for idx, netStat := range netsStatusCopy {
621+
// remove status for the pod interface
622+
// it should always be ovn-kubernetes, but if not, remove status for eth0 which is the pod network
623+
if netStat.Name == "ovn-kubernetes" || netStat.Interface == "eth0" {
624+
removeIndex(netsStatus, idx)
625+
continue
626+
}
627+
628+
// get ipam configuration from NAD
629+
nadName := strings.TrimPrefix(netStat.Name, pod.Namespace+"/")
630+
netAtt, err := nad.GetNADWithName(
631+
ctx, h, nadName, pod.Namespace)
632+
if err != nil {
633+
return detailList, err
634+
}
635+
636+
ipam, err := nad.GetJSONPathFromConfig(*netAtt, ".ipam")
637+
if err != nil {
638+
return detailList, err
639+
}
640+
641+
// if the NAD has no ipam configured, skip it as there will be no IP
642+
if ipam == "{}" {
643+
Log.Info(fmt.Sprintf("removing netsStatus for NAD %s for %s, IPAM configuration is empty: %s", netAtt.Name, pod.Name, ipam))
644+
removeIndex(netsStatus, idx)
645+
continue
646+
}
647+
648+
// verify there is IP information for the network, otherwise report an error to reconcile
649+
if len(netStat.IPs) == 0 {
650+
return detailList, fmt.Errorf("no IP information for network %s on pod %s", netStat.Name, pod.Name)
651+
}
562652
}
653+
}
563654

564-
// verify there is IP information for the network, otherwise report an error to reconcile
565-
if len(netStat.IPs) == 0 {
566-
return detailList, fmt.Errorf("no IP information for network %s on pod %s", netStat.Name, pod.Name)
655+
// Handle pods with predictableip label
656+
if hasPredictableIP {
657+
// Create synthetic NetworkStatus for predictable IP
658+
predictableNetStatus := k8s_networkv1.NetworkStatus{
659+
Name: "predictableip",
660+
Interface: "predictableip",
661+
IPs: []string{predictableIP},
567662
}
663+
netsStatus = append(netsStatus, predictableNetStatus)
664+
Log.Info(fmt.Sprintf("Added predictable IP %s for pod %s", predictableIP, pod.Name))
568665
}
569666

667+
// Log all network statuses for debugging
668+
Log.Info(fmt.Sprintf("Pod %s final network statuses: %+v", pod.Name, netsStatus))
570669
detail.NetworkStatus = netsStatus
571670

572671
detailList = append(detailList, detail)
@@ -594,6 +693,7 @@ func (r *BGPConfigurationReconciler) createOrPatchFRRConfiguration(
594693
Log.Info("Reconciling createOrUpdateFRRConfiguration")
595694

596695
podPrefixes := bgp.GetFRRPodPrefixes(podDtl.NetworkStatus)
696+
Log.Info(fmt.Sprintf("Generated prefixes for pod %s: %v", podDtl.Name, podPrefixes))
597697

598698
nodeFRRCfg := nodeFRRCfgs[podDtl.Node]
599699

0 commit comments

Comments
 (0)