Skip to content

Use predictableIPs labels to advertise pods addresses #487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+259 −57
Open

Use predictableIPs labels to advertise pods addresses #487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 157 additions & 57 deletions controllers/network/bgpconfiguration_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,38 +243,61 @@ func (r *BGPConfigurationReconciler) SetupWithManager(ctx context.Context, mgr c
// Skip if
// * no NAD annotation was configured on old object OR
// * no NAD annotation was configured on new object AND
// * no predictableip label was configured on old object OR
// * no predictableip label was configured on new object AND
// * the resourceVersion has not changed

oldConfigured := true
oldNADConfigured := true
if val, ok := e.ObjectOld.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
oldConfigured = false
oldNADConfigured = false
}
newConfigured := true
newNADConfigured := true
if val, ok := e.ObjectNew.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
newConfigured = false
newNADConfigured = false
}

return (oldConfigured || newConfigured) && e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion()
oldPredictableIPConfigured := true
if val, ok := e.ObjectOld.GetLabels()["predictableip"]; !ok || len(val) == 0 {
oldPredictableIPConfigured = false
}
newPredictableIPConfigured := true
if val, ok := e.ObjectNew.GetLabels()["predictableip"]; !ok || len(val) == 0 {
newPredictableIPConfigured = false
}

return (oldNADConfigured || newNADConfigured || oldPredictableIPConfigured || newPredictableIPConfigured) && e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion()
},
DeleteFunc: func(e event.DeleteEvent) bool {
// Skip if
// * NAD annotation key is missing
// * NAD annotation key is missing AND predictableip label is missing
// * there is no additional network configured
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
return false
nadConfigured := false
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; ok && len(val) > 0 {
nadConfigured = true
}

return true
predictableIPConfigured := false
if val, ok := e.Object.GetLabels()["predictableip"]; ok && len(val) > 0 {
predictableIPConfigured = true
}

return nadConfigured || predictableIPConfigured
},
CreateFunc: func(e event.CreateEvent) bool {
// Skip if
// * NAD annotation key is missing
// * NAD annotation key is missing AND predictableip label is missing
// * there is no additional network configured
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; !ok || len(val) == 0 {
return false
nadConfigured := false
if val, ok := e.Object.GetAnnotations()[k8s_networkv1.NetworkAttachmentAnnot]; ok && len(val) > 0 {
nadConfigured = true
}

return true
predictableIPConfigured := false
if val, ok := e.Object.GetLabels()["predictableip"]; ok && len(val) > 0 {
predictableIPConfigured = true
}

return nadConfigured || predictableIPConfigured
},
}

Expand Down Expand Up @@ -315,7 +338,7 @@ func (r *BGPConfigurationReconciler) SetupWithManager(ctx context.Context, mgr c

return ctrl.NewControllerManagedBy(mgr).
For(&networkv1.BGPConfiguration{}).
// Watch pods which have additional networks configured with k8s_networkv1.NetworkAttachmentAnnot annotation in the same namespace
// Watch pods which have additional networks configured with k8s_networkv1.NetworkAttachmentAnnot annotation or predictableip label in the same namespace
Watches(&corev1.Pod{},
podFN,
builder.WithPredicates(pPod)).
Expand Down Expand Up @@ -479,7 +502,7 @@ func (r *BGPConfigurationReconciler) deleteStaleFRRConfigurations(ctx context.Co

// getPodNetworkDetails - returns the podDetails for a list of pods in status.phase: Running
// where the pod has the multus k8s_networkv1.NetworkAttachmentAnnot annotation
// and its value is not '[]'
// and its value is not '[]' OR has a predictableip label
func getPodNetworkDetails(
ctx context.Context,
h *helper.Helper,
Expand All @@ -499,14 +522,24 @@ func getPodNetworkDetails(
if pod.Status.Phase != corev1.PodRunning {
continue
}
if netAttachString, ok := pod.Annotations[k8s_networkv1.NetworkAttachmentAnnot]; ok && netAttachString != "[]" {
// get the elements from val to validate the status annotation has the right length
netAttach := []k8s_networkv1.NetworkSelectionElement{}
err := json.Unmarshal([]byte(netAttachString), &netAttach)
if err != nil {
return nil, fmt.Errorf("failed to decode networks %s: %w", netAttachString, err)
}

// Check for pods with predictableip label
hasPredictableIP := false
predictableIP := ""
if ipValue, ok := pod.Labels["predictableip"]; ok && ipValue != "" {
hasPredictableIP = true
predictableIP = ipValue
}

hasNetworkAttachment := false
var netAttachString string
if netAttach, ok := pod.Annotations[k8s_networkv1.NetworkAttachmentAnnot]; ok && netAttach != "[]" {
hasNetworkAttachment = true
netAttachString = netAttach
}

// Process pod if it has either network attachments or predictableip label
if hasNetworkAttachment || hasPredictableIP {
// verify the nodeName information is already present in the pod spec, otherwise report an error to reconcile
if pod.Spec.NodeName == "" {
return detailList, fmt.Errorf("empty spec.nodeName on pod %s", pod.Name)
Expand All @@ -518,55 +551,121 @@ func getPodNetworkDetails(
Node: pod.Spec.NodeName,
}

netsStatus, err := nad.GetNetworkStatusFromAnnotation(pod.Annotations)
if err != nil {
return detailList, fmt.Errorf("failed to get netsStatus from pod annoation - %v: %w", pod.Annotations, err)
}
// on pod start it can happen that the network status annotation does not yet
// reflect all requested networks. return with an error to reconcile if the length
// is <= the status. Note: the status also has the pod network
if len(netsStatus) <= len(netAttach) {
return detailList, fmt.Errorf("metadata.Annotations['k8s.ovn.org/pod-networks'] %s on pod %s, does not match requested networks %s",
pod.GetAnnotations()[k8s_networkv1.NetworkStatusAnnot], pod.Name, netAttachString)
}
var netsStatus []k8s_networkv1.NetworkStatus

// Handle pods with network attachments
if hasNetworkAttachment {
// get the elements from val to validate the status annotation has the right length
netAttach := []k8s_networkv1.NetworkSelectionElement{}

netsStatusCopy := make([]k8s_networkv1.NetworkStatus, len(netsStatus))
copy(netsStatusCopy, netsStatus)
// verify there are IP information for all networks in the status, otherwise report an error to reconcile
for idx, netStat := range netsStatusCopy {
// remove status for the pod interface
// it should always be ovn-kubernetes, but if not, remove status for eth0 which is the pod network
if netStat.Name == "ovn-kubernetes" || netStat.Interface == "eth0" {
removeIndex(netsStatus, idx)
// Validate that the annotation is not empty and contains valid JSON
if netAttachString == "" {
Log.Info(fmt.Sprintf("Skipping pod %s: network attachment annotation is empty", pod.Name))
continue
}

// get ipam configuration from NAD
nadName := strings.TrimPrefix(netStat.Name, pod.Namespace+"/")
netAtt, err := nad.GetNADWithName(
ctx, h, nadName, pod.Namespace)
if err != nil {
return detailList, err
// Check if the annotation looks like JSON (starts with [ or {) or is a simple string reference
trimmed := strings.TrimSpace(netAttachString)
if !strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "{") && !strings.HasPrefix(trimmed, "\"") {
Log.Info(fmt.Sprintf("Skipping pod %s: network attachment annotation does not appear to be valid JSON or string reference: %s", pod.Name, netAttachString))
continue
}

// Log the raw annotation for debugging
Log.Info(fmt.Sprintf("Processing pod %s with network attachment annotation: %s", pod.Name, netAttachString))

// Handle different annotation formats
var err error
if strings.HasPrefix(trimmed, "\"") && strings.HasSuffix(trimmed, "\"") {
// Simple string reference like "bgpnet-worker-3"
var nadName string
err = json.Unmarshal([]byte(netAttachString), &nadName)
if err != nil {
Log.Error(err, fmt.Sprintf("Failed to decode network attachment string reference for pod %s. Raw annotation: %s", pod.Name, netAttachString))
return nil, fmt.Errorf("failed to decode network string reference %s for pod %s: %w", netAttachString, pod.Name, err)
}
// Create a NetworkSelectionElement from the string reference
netAttach = []k8s_networkv1.NetworkSelectionElement{
{
Name: nadName,
Namespace: pod.Namespace,
},
}
Log.Info(fmt.Sprintf("Converted string reference %s to NetworkSelectionElement for pod %s", nadName, pod.Name))
} else {
// Full JSON array format
err = json.Unmarshal([]byte(netAttachString), &netAttach)
if err != nil {
Log.Error(err, fmt.Sprintf("Failed to decode network attachment annotation for pod %s. Raw annotation: %s", pod.Name, netAttachString))
return nil, fmt.Errorf("failed to decode networks %s for pod %s: %w", netAttachString, pod.Name, err)
}
}

ipam, err := nad.GetJSONPathFromConfig(*netAtt, ".ipam")
netsStatus, err = nad.GetNetworkStatusFromAnnotation(pod.Annotations)
if err != nil {
return detailList, err
return detailList, fmt.Errorf("failed to get netsStatus from pod annoation - %v: %w", pod.Annotations, err)
}
Log.Info(fmt.Sprintf("Retrieved network statuses for pod %s: %+v", pod.Name, netsStatus))
// on pod start it can happen that the network status annotation does not yet
// reflect all requested networks. return with an error to reconcile if the length
// is <= the status. Note: the status also has the pod network
if len(netsStatus) <= len(netAttach) {
return detailList, fmt.Errorf("metadata.Annotations['k8s.ovn.org/pod-networks'] %s on pod %s, does not match requested networks %s",
pod.GetAnnotations()[k8s_networkv1.NetworkStatusAnnot], pod.Name, netAttachString)
}

// if the NAD has no ipam configured, skip it as there will be no IP
if ipam == "{}" {
Log.Info(fmt.Sprintf("removing netsStatus for NAD %s for %s, IPAM configuration is empty: %s", netAtt.Name, pod.Name, ipam))
removeIndex(netsStatus, idx)
continue
netsStatusCopy := make([]k8s_networkv1.NetworkStatus, len(netsStatus))
copy(netsStatusCopy, netsStatus)
// verify there are IP information for all networks in the status, otherwise report an error to reconcile
for idx, netStat := range netsStatusCopy {
// remove status for the pod interface
// it should always be ovn-kubernetes, but if not, remove status for eth0 which is the pod network
if netStat.Name == "ovn-kubernetes" || netStat.Interface == "eth0" {
removeIndex(netsStatus, idx)
continue
}

// get ipam configuration from NAD
nadName := strings.TrimPrefix(netStat.Name, pod.Namespace+"/")
netAtt, err := nad.GetNADWithName(
ctx, h, nadName, pod.Namespace)
if err != nil {
return detailList, err
}

ipam, err := nad.GetJSONPathFromConfig(*netAtt, ".ipam")
if err != nil {
return detailList, err
}

// if the NAD has no ipam configured, skip it as there will be no IP
if ipam == "{}" {
Log.Info(fmt.Sprintf("removing netsStatus for NAD %s for %s, IPAM configuration is empty: %s", netAtt.Name, pod.Name, ipam))
removeIndex(netsStatus, idx)
continue
}

// verify there is IP information for the network, otherwise report an error to reconcile
if len(netStat.IPs) == 0 {
return detailList, fmt.Errorf("no IP information for network %s on pod %s", netStat.Name, pod.Name)
}
}
}

// verify there is IP information for the network, otherwise report an error to reconcile
if len(netStat.IPs) == 0 {
return detailList, fmt.Errorf("no IP information for network %s on pod %s", netStat.Name, pod.Name)
// Handle pods with predictableip label
if hasPredictableIP {
// Create synthetic NetworkStatus for predictable IP
predictableNetStatus := k8s_networkv1.NetworkStatus{
Name: "predictableip",
Interface: "predictableip",
IPs: []string{predictableIP},
}
netsStatus = append(netsStatus, predictableNetStatus)
Log.Info(fmt.Sprintf("Added predictable IP %s for pod %s", predictableIP, pod.Name))
}

// Log all network statuses for debugging
Log.Info(fmt.Sprintf("Pod %s final network statuses: %+v", pod.Name, netsStatus))
detail.NetworkStatus = netsStatus

detailList = append(detailList, detail)
Expand Down Expand Up @@ -594,6 +693,7 @@ func (r *BGPConfigurationReconciler) createOrPatchFRRConfiguration(
Log.Info("Reconciling createOrUpdateFRRConfiguration")

podPrefixes := bgp.GetFRRPodPrefixes(podDtl.NetworkStatus)
Log.Info(fmt.Sprintf("Generated prefixes for pod %s: %v", podDtl.Name, podPrefixes))

nodeFRRCfg := nodeFRRCfgs[podDtl.Node]

Expand Down
Loading