Added validations and templates for multiple networks configuration

pkg/cloud/instance.go

@@ -31,6 +31,8 @@
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
+
+	netpkg "net"
 )
 
 type VMIface interface {
@@ -44,7 +46,15 @@
 	csMachine.Spec.ProviderID = ptr.To(fmt.Sprintf("cloudstack:///%s", vmResponse.Id))
 	// InstanceID is later used as required parameter to destroy VM.
 	csMachine.Spec.InstanceID = ptr.To(vmResponse.Id)
-	csMachine.Status.Addresses = []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: vmResponse.Ipaddress}}
+	csMachine.Status.Addresses = []corev1.NodeAddress{}
+	for _, nic := range vmResponse.Nic {
+		if nic.Ipaddress != "" {
+			csMachine.Status.Addresses = append(csMachine.Status.Addresses, corev1.NodeAddress{
+				Type:    corev1.NodeInternalIP,
+				Address: nic.Ipaddress,
+			})
+		}
+	}
 	newInstanceState := vmResponse.State
 	if newInstanceState != csMachine.Status.InstanceState || (newInstanceState != "" && csMachine.Status.InstanceStateLastUpdated.IsZero()) {
 		csMachine.Status.InstanceState = newInstanceState
@@ -302,41 +312,132 @@
 	return nil
 }
 
-func (c *client) resolveNetworkIDByName(name string) (string, error) {
+func (c *client) isFreeIPAvailable(networkID, ip string) (bool, error) {
+	params := c.cs.Address.NewListPublicIpAddressesParams()
+	params.SetNetworkid(networkID)
+	params.SetAllocatedonly(false)
+	params.SetForvirtualnetwork(false)
+	params.SetListall(true)
+
+	if ip != "" {
+		params.SetIpaddress(ip)
+	}
+
+	resp, err := c.cs.Address.ListPublicIpAddresses(params)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to list public IP addresses for network %q", networkID)
+	}
+
+	for _, addr := range resp.PublicIpAddresses {
+		if addr.State == "Free" {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func (c *client) buildIPEntry(resolvedNet *cloudstack.Network, ip string) (map[string]string, error) {
+	if ip != "" {
+		if err := c.validateIPInCIDR(ip, resolvedNet, resolvedNet.Id); err != nil {
+			return nil, err
+		}
+	}
+
+	if resolvedNet.Type == "Shared" {
+		isAvailable, err := c.isFreeIPAvailable(resolvedNet.Id, ip)
+		if err != nil {
+			return nil, err
+		}
+		if !isAvailable {
+			if ip != "" {
+				return nil, errors.Errorf("IP %q is already allocated in network %q or out of range", ip, resolvedNet.Id)
+			}
+			return nil, errors.Errorf("no free IPs available in network %q", resolvedNet.Id)
+		}
+	}
+
+	entry := map[string]string{
+		"networkid": resolvedNet.Id,
+	}
+	if ip != "" {
+		entry["ip"] = ip
+	}
+	return entry, nil
+}
+
+func (c *client) resolveNetworkByName(name string) (*cloudstack.Network, error) {
 	net, count, err := c.cs.Network.GetNetworkByName(name, cloudstack.WithProject(c.user.Project.ID))
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to look up network %q", name)
+		return nil, errors.Wrapf(err, "failed to look up network %q", name)
 	}
 	if count != 1 {
-		return "", errors.Errorf("expected 1 network named %q, but got %d", name, count)
+		return nil, errors.Errorf("expected 1 network named %q, but got %d", name, count)
 	}
-	return net.Id, nil
+	return net, nil
 }
 
 func (c *client) buildIPToNetworkList(csMachine *infrav1.CloudStackMachine) ([]map[string]string, error) {
-	ipToNetworkList := []map[string]string{}
+	var ipToNetworkList []map[string]string
 
 	for _, net := range csMachine.Spec.Networks {
-		networkID := net.ID
-		if networkID == "" {
-			var err error
-			networkID, err = c.resolveNetworkIDByName(net.Name)
+		resolvedNet, err := c.resolveNetwork(net)
+		if err != nil {
+			return nil, err
+		}
+
+		var entry map[string]string
+		if net.IP != "" {
+			entry, err = c.buildIPEntry(resolvedNet, net.IP)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			entry, err = c.buildIPEntry(resolvedNet, "")
 			if err != nil {
 				return nil, err
 			}
 		}
-		entry := map[string]string{"networkid": networkID}
-		if net.IP != "" {
-			entry["ip"] = net.IP
-		}
+
 		ipToNetworkList = append(ipToNetworkList, entry)
 	}
+
 	return ipToNetworkList, nil
 }
 
+func (c *client) resolveNetwork(net infrav1.NetworkSpec) (*cloudstack.Network, error) {
+	if net.ID == "" {
+		return c.resolveNetworkByName(net.Name)
+	}
+
+	resolvedNet, _, err := c.cs.Network.GetNetworkByID(net.ID, cloudstack.WithProject(c.user.Project.ID))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get network %q by ID", net.ID)
+	}
+	return resolvedNet, nil
+}
+
+func (c *client) validateIPInCIDR(ipStr string, net *cloudstack.Network, netID string) error {
+	ip := netpkg.ParseIP(ipStr)
+	if ip == nil {
+		return errors.Errorf("invalid IP address %q", ipStr)
+	}
+
+	_, cidr, err := netpkg.ParseCIDR(net.Cidr)
+	if err != nil {
+		return errors.Wrapf(err, "invalid CIDR %q for network %q", net.Cidr, netID)
+	}
+
+	if !cidr.Contains(ip) {
+		return errors.Errorf("IP %q is not within network CIDR %q", ipStr, net.Cidr)
+	}
+
+	return nil
+}
+
 // DeployVM will create a VM instance,
 // and sets the infrastructure machine spec and status accordingly.
 func (c *client) DeployVM(
 	csMachine *infrav1.CloudStackMachine,
 	capiMachine *clusterv1.Machine,
 	fd *infrav1.CloudStackFailureDomain,
@@ -358,6 +459,16 @@
 	if len(csMachine.Spec.Networks) == 0 && fd.Spec.Zone.Network.ID != "" {
 		p.SetNetworkids([]string{fd.Spec.Zone.Network.ID})
 	} else {
+		firstNetwork := csMachine.Spec.Networks[0]
+		zoneNet := fd.Spec.Zone.Network
+
+		if zoneNet.ID != "" && firstNetwork.ID != "" && firstNetwork.ID != zoneNet.ID {
+			return errors.Errorf("first network ID %q does not match zone network ID %q", firstNetwork.ID, zoneNet.ID)
+		}
+		if zoneNet.Name != "" && firstNetwork.Name != "" && firstNetwork.Name != zoneNet.Name {
+			return errors.Errorf("first network name %q does not match zone network name %q", firstNetwork.Name, zoneNet.Name)
+		}
+
 		ipToNetworkList, err := c.buildIPToNetworkList(csMachine)
 		if err != nil {
 			return err

templates/cluster-template-multiple-networks.yaml

@@ -0,0 +1,148 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    serviceDomain: "cluster.local"
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+    kind: CloudStackCluster
+    name: ${CLUSTER_NAME}
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: ${CLUSTER_NAME}-control-plane
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false}
+  controlPlaneEndpoint:
+    host: ${CLUSTER_ENDPOINT_IP}
+    port: ${CLUSTER_ENDPOINT_PORT=6443}
+  failureDomains:
+    - name: ${CLOUDSTACK_FD1_NAME=failure-domain-1}
+      acsEndpoint:
+        name: ${CLOUDSTACK_FD1_SECRET_NAME=cloudstack-credentials}
+        namespace: ${CLOUDSTACK_FD1_SECRET_NAMESPACE=default}
+      zone:
+        name:  ${CLOUDSTACK_ZONE_NAME}
+        network:
+          name: ${CLOUDSTACK_NETWORK_NAME}
+---
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  kubeadmConfigSpec:
+    initConfiguration:
+      nodeRegistration:
+        name: '{{ local_hostname }}'
+        kubeletExtraArgs:
+          provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
+    joinConfiguration:
+      nodeRegistration:
+        name: '{{ local_hostname }}'
+        kubeletExtraArgs:
+          provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
+    preKubeadmCommands:
+      - swapoff -a
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+      kind: CloudStackMachineTemplate
+      name: "${CLUSTER_NAME}-control-plane"
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      offering:
+        name: ${CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING}
+      networks:
+        - name: ${CLOUDSTACK_NETWORK_NAME} # (optional) default primary network; must match failureDomains.zone.network.name
+          # ip: 10.1.1.21                   # (optional) static IP in the primary network
+
+        # Additional (extra) networks can be specified below. Use either 'name' or 'id', and optionally an 'ip'.
+        # - name: isolated-net2             # (optional) extra network by name
+        #   ip: 10.1.1.31                   # (optional) static IP in this network
+
+        # - id: a1b2c3d4-5678-90ef-gh12-3456789ijklm  # (optional) extra network by ID
+        #   ip: 10.1.1.32                             # (optional) static IP in this network
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels: null
+  template:
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+        kind: CloudStackMachineTemplate
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      offering:
+        name: ${CLOUDSTACK_WORKER_MACHINE_OFFERING}
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+      networks:
+        - name: ${CLOUDSTACK_NETWORK_NAME} # (optional) default primary network; must match failureDomains.zone.network.name
+          # ip: 10.1.1.41                   # (optional) static IP in the primary network
+
+        # Additional (extra) networks can be specified below. Use either 'name' or 'id', and optionally an 'ip'.
+        # - name: isolated-net3             # (optional) extra network by name
+        #   ip: 10.1.1.51                   # (optional) static IP in this network
+
+        # - id: f5c93c15-d27b-4c93-b27c-00f48226b9a9  # (optional) extra network by ID
+        #   ip: 10.1.1.52                             # (optional) static IP in this network
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          name: '{{ local_hostname }}'
+          kubeletExtraArgs:
+            provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
+      preKubeadmCommands:
+        - swapoff -a