Generate predictable IPs for bind stateful set

Generate and manage predictable IPs for each member of the bind
statefulset and store in it's own predictable IPs config map.

The reason it needs to be it's own map is that we do not want to cause
minidns to restart if a scaling change occurs on mind - nor do we want
minidns scale changes to affect bind pods.

Author: omersch381

controllers/designate_controller.go

@@ -714,6 +714,7 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 	}
 	Log.Info("Deployment API task reconciled")
 
+	// Handle Mdns predictable IPs configmap
 	nad, err := nad.GetNADWithName(ctx, helper, instance.Spec.DesignateNetworkAttachment, instance.Namespace)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -724,31 +725,6 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 		return ctrl.Result{}, err
 	}
 
-	nodeConfigMap := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      designate.MdnsPredIPConfigMap,
-			Namespace: instance.GetNamespace(),
-			Labels:    labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{}),
-		},
-		Data: make(map[string]string),
-	}
-
-	// Look for existing config map and if exists, read existing data and match
-	// against nodes.
-	foundMap := &corev1.ConfigMap{}
-	err = helper.GetClient().Get(ctx, types.NamespacedName{Name: designate.MdnsPredIPConfigMap, Namespace: instance.GetNamespace()},
-		foundMap)
-	if err != nil {
-		if k8s_errors.IsNotFound(err) {
-			Log.Info(fmt.Sprintf("Ip map %s doesn't exist, creating.", designate.MdnsPredIPConfigMap))
-		} else {
-			return ctrl.Result{}, err
-		}
-	} else {
-		Log.Info("Retrieved existing map, updating..")
-		nodeConfigMap.Data = foundMap.Data
-	}
-
 	//
 	// Predictable IPs.
 	//
@@ -760,6 +736,28 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+
+	// Fetch allocated ips from Mdns and Bind config maps and store them in allocatedIPs
+	mdnsLabels := labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{})
+	mdnsConfigMap, err := r.handleConfigMap(ctx, helper, instance, designate.MdnsPredIPConfigMap, mdnsLabels)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	bindLabels := labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{})
+	bindConfigMap, err := r.handleConfigMap(ctx, helper, instance, designate.BindPredIPConfigMap, bindLabels)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	allocatedIPs := make(map[string]bool)
+	for _, predIP := range bindConfigMap.Data {
+		allocatedIPs[predIP] = true
+	}
+	for _, predIP := range mdnsConfigMap.Data {
+		allocatedIPs[predIP] = true
+	}
+
 	// Get a list of the nodes in the cluster
 
 	// TODO(oschwart):
@@ -772,54 +770,51 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	updatedMap := make(map[string]string)
-	allocatedIPs := make(map[string]bool)
-	var predictableIPsRequired []string
 
-	// First scan existing allocations so we can keep existing allocations.
-	// Keeping track of what's required and what already exists. If a node is
-	// removed from the cluster, it's IPs will not be added to the allocated
-	// list and are effectively recycled.
+	var nodeNames []string
 	for _, node := range nodes.Items {
-		nodeName := fmt.Sprintf("mdns_%s", node.Name)
-		if ipValue, ok := nodeConfigMap.Data[nodeName]; ok {
-			updatedMap[nodeName] = ipValue
-			allocatedIPs[ipValue] = true
-			Log.Info(fmt.Sprintf("%s has IP mapping %s: %s", node.Name, nodeName, ipValue))
-		} else {
-			predictableIPsRequired = append(predictableIPsRequired, nodeName)
-		}
+		nodeNames = append(nodeNames, fmt.Sprintf("mdns_%s", node.Name))
 	}
-	// Get new IPs using the range from predictableIPParmas minus the
-	// allocatedIPs captured above.
-	Log.Info(fmt.Sprintf("Allocating %d predictable IPs", len(predictableIPsRequired)))
-	for _, nodeName := range predictableIPsRequired {
-		nodeIP, err := designate.GetNextIP(predictableIPParams, allocatedIPs)
-		if err != nil {
-			// An error here is really unexpected- it means either we have
-			// messed up the allocatedIPs list or the range we are assuming is
-			// too small for the number of mdns pod.
-			return ctrl.Result{}, err
-		}
-		updatedMap[nodeName] = nodeIP
+
+	updatedMap, allocatedIPs, err := r.allocatePredictableIPs(ctx, predictableIPParams, nodeNames, mdnsConfigMap.Data, allocatedIPs)
+	if err != nil {
+		return ctrl.Result{}, err
 	}
 
-	mapLabels := labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{})
-	_, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), nodeConfigMap, func() error {
-		nodeConfigMap.Labels = util.MergeStringMaps(nodeConfigMap.Labels, mapLabels)
-		nodeConfigMap.Data = updatedMap
-		err := controllerutil.SetControllerReference(instance, nodeConfigMap, helper.GetScheme())
-		if err != nil {
-			return err
-		}
-		return nil
+	_, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), mdnsConfigMap, func() error {
+		mdnsConfigMap.Labels = util.MergeStringMaps(mdnsConfigMap.Labels, mdnsLabels)
+		mdnsConfigMap.Data = updatedMap
+		return controllerutil.SetControllerReference(instance, mdnsConfigMap, helper.GetScheme())
 	})
 
 	if err != nil {
 		Log.Info("Unable to create config map for mdns ips...")
 		return ctrl.Result{}, err
 	}
 
+	// Handle Bind predictable IPs configmap
+	bindReplicaCount := int(*instance.Spec.DesignateBackendbind9.Replicas)
+	var bindNames []string
+	for i := 0; i < bindReplicaCount; i++ {
+		bindNames = append(bindNames, fmt.Sprintf("bind_address_%d", i))
+	}
+
+	updatedBindMap, _, err := r.allocatePredictableIPs(ctx, predictableIPParams, bindNames, bindConfigMap.Data, allocatedIPs)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	_, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), bindConfigMap, func() error {
+		bindConfigMap.Labels = util.MergeStringMaps(bindConfigMap.Labels, bindLabels)
+		bindConfigMap.Data = updatedBindMap
+		return controllerutil.SetControllerReference(instance, bindConfigMap, helper.GetScheme())
+	})
+
+	if err != nil {
+		Log.Info("Unable to create config map for bind ips...")
+		return ctrl.Result{}, err
+	}
+
 	// deploy designate-central
 	designateCentral, op, err := r.centralDeploymentCreateOrUpdate(ctx, instance)
 	if err != nil {
@@ -1082,6 +1077,72 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 	return ctrl.Result{}, nil
 }
 
+func (r *DesignateReconciler) handleConfigMap(ctx context.Context, helper *helper.Helper, instance *designatev1beta1.Designate, configMapName string, labels map[string]string) (*corev1.ConfigMap, error) {
+	Log := r.GetLogger(ctx)
+
+	nodeConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: instance.GetNamespace(),
+			Labels:    labels,
+		},
+		Data: make(map[string]string),
+	}
+
+	// Look for existing config map and if exists, read existing data and match
+	// against nodes.
+	foundMap := &corev1.ConfigMap{}
+	err := helper.GetClient().Get(ctx, types.NamespacedName{Name: configMapName, Namespace: instance.GetNamespace()}, foundMap)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			Log.Info(fmt.Sprintf("Ip map %s doesn't exist, creating.", configMapName))
+		} else {
+			return nil, err
+		}
+	} else {
+		Log.Info("Retrieved existing map, updating..")
+		nodeConfigMap.Data = foundMap.Data
+	}
+
+	return nodeConfigMap, nil
+}
+
+func (r *DesignateReconciler) allocatePredictableIPs(ctx context.Context, predictableIPParams *designate.NADIpam, ipHolders []string, existingMap map[string]string, allocatedIPs map[string]bool) (map[string]string, map[string]bool, error) {
+	Log := r.GetLogger(ctx)
+
+	updatedMap := make(map[string]string)
+	var predictableIPsRequired []string
+
+	// First scan existing allocations so we can keep existing allocations.
+	// Keeping track of what's required and what already exists. If a node is
+	// removed from the cluster, it's IPs will not be added to the allocated
+	// list and are effectively recycled.
+	for _, ipHolder := range ipHolders {
+		if ipValue, ok := existingMap[ipHolder]; ok {
+			updatedMap[ipHolder] = ipValue
+			Log.Info(fmt.Sprintf("%s has IP mapping: %s", ipHolder, ipValue))
+		} else {
+			predictableIPsRequired = append(predictableIPsRequired, ipHolder)
+		}
+	}
+
+	// Get new IPs using the range from predictableIPParmas minus the
+	// allocatedIPs captured above.
+	Log.Info(fmt.Sprintf("Allocating %d predictable IPs", len(predictableIPsRequired)))
+	for _, nodeName := range predictableIPsRequired {
+		ipAddress, err := designate.GetNextIP(predictableIPParams, allocatedIPs)
+		if err != nil {
+			// An error here is really unexpected- it means either we have
+			// messed up the allocatedIPs list or the range we are assuming is
+			// too small for the number of mdns pod.
+			return nil, nil, err
+		}
+		updatedMap[nodeName] = ipAddress
+	}
+
+	return updatedMap, allocatedIPs, nil
+}
+
 func (r *DesignateReconciler) reconcileUpdate(ctx context.Context, instance *designatev1beta1.Designate) (ctrl.Result, error) {
 	Log := r.GetLogger(ctx)
 

pkg/designate/bind_ctrl_network.go

@@ -36,21 +36,12 @@ func GetPredictableIPAM(networkParameters *NetworkParameters) (*NADIpam, error)
 	return predParams, nil
 }
 
-// GetNextIP picks the next available IP from the range defined by a NADIpam,
-// skipping ones that are already used appear as keys in the currentValues map.
-func GetNextIP(predParams *NADIpam, currentValues map[string]bool) (string, error) {
-	candidateAddress := predParams.RangeStart
-	for alloced := true; alloced; {
-
-		if _, ok := currentValues[candidateAddress.String()]; ok {
-			if candidateAddress == predParams.RangeEnd {
-				return "", fmt.Errorf("predictable IPs: out of available addresses")
-			}
-			candidateAddress = candidateAddress.Next()
-		} else {
-			alloced = false
+func GetNextIP(predParams *NADIpam, allocatedIPs map[string]bool) (string, error) {
+	for candidateAddress := predParams.RangeStart; candidateAddress != predParams.RangeEnd; candidateAddress = candidateAddress.Next() {
+		if !allocatedIPs[candidateAddress.String()] {
+			allocatedIPs[candidateAddress.String()] = true
+			return candidateAddress.String(), nil
 		}
 	}
-	currentValues[candidateAddress.String()] = true
-	return candidateAddress.String(), nil
+	return "", fmt.Errorf("predictable IPs: out of available addresses")
 }

pkg/designate/const.go

@@ -40,4 +40,6 @@ const (
 	DesignateRndcKey = "rndc-key"
 
 	MdnsPredIPConfigMap = "designate-mdns-ip-map"
+
+	BindPredIPConfigMap = "designate-bind-ip-map"
 )