Merge pull request openshift#7501 from vr4manta/SPLAT-1173

SPLAT-1173: Enhance vSphere Installer to use IPAddressClaims for static IP

Author: openshift-merge-bot[bot]

pkg/asset/cluster/tfvars/tfvars.go

@@ -1035,6 +1035,10 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		if err != nil {
 			return err
 		}
+		ipAddresses, err := mastersAsset.IPAddresses()
+		if err != nil {
+			return err
+		}
 		controlPlaneConfigs := make([]*machinev1beta1.VSphereMachineProviderSpec, len(controlPlanes))
 		for i, c := range controlPlanes {
 			var clusterMo mo.ClusterComputeResource
@@ -1098,6 +1102,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				InfraID:                 clusterID.InfraID,
 				InstallConfig:           installConfig,
 				ControlPlaneMachines:    controlPlanes,
+				IPAddresses:             ipAddresses,
 			},
 		)
 		if err != nil {

pkg/asset/machines/master.go

@@ -13,6 +13,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1"
 	"sigs.k8s.io/yaml"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -70,6 +71,8 @@ type Master struct {
 	MachineConfigFiles     []*asset.File
 	MachineFiles           []*asset.File
 	ControlPlaneMachineSet *asset.File
+	IPClaimFiles           []*asset.File
+	IPAddrFiles            []*asset.File
 
 	// SecretFiles is used by the baremetal platform to register the
 	// credential information for communicating with management
@@ -109,15 +112,23 @@ const (
 	// user-data secret.
 	masterUserDataFileName = "99_openshift-cluster-api_master-user-data-secret.yaml"
 
-	// masterUserDataFileName is the filename used for the control plane machine sets.
+	// controlPlaneMachineSetFileName is the filename used for the control plane machine sets.
 	controlPlaneMachineSetFileName = "99_openshift-machine-api_master-control-plane-machine-set.yaml"
+
+	// ipClaimFileName is the filename used for the ip claims list.
+	ipClaimFileName = "99_openshift-machine-api_claim-%s.yaml"
+
+	// ipAddressFileName is the filename used for the ip addresses list.
+	ipAddressFileName = "99_openshift-machine-api_address-%s.yaml"
 )
 
 var (
 	secretFileNamePattern              = fmt.Sprintf(secretFileName, "*")
 	networkConfigSecretFileNamePattern = fmt.Sprintf(networkConfigSecretFileName, "*")
 	hostFileNamePattern                = fmt.Sprintf(hostFileName, "*")
 	masterMachineFileNamePattern       = fmt.Sprintf(masterMachineFileName, "*")
+	masterIPClaimFileNamePattern       = fmt.Sprintf(ipClaimFileName, "*master*")
+	masterIPAddressFileNamePattern     = fmt.Sprintf(ipAddressFileName, "*master*")
 
 	_ asset.WritableAsset = (*Master)(nil)
 )
@@ -158,6 +169,8 @@ func (m *Master) Generate(dependencies asset.Parents) error {
 	pool := *ic.ControlPlane
 	var err error
 	machines := []machinev1beta1.Machine{}
+	var ipClaims []ipamv1.IPAddressClaim
+	var ipAddrs []ipamv1.IPAddress
 	var controlPlaneMachineSet *machinev1.ControlPlaneMachineSet
 	switch ic.Platform.Name() {
 	case awstypes.Name:
@@ -439,10 +452,14 @@ func (m *Master) Generate(dependencies asset.Parents) error {
 		pool.Platform.VSphere = &mpool
 		templateName := clusterID.InfraID + "-rhcos"
 
-		machines, controlPlaneMachineSet, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
+		data, err := vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
 		if err != nil {
 			return errors.Wrap(err, "failed to create master machine objects")
 		}
+		machines = data.Machines
+		controlPlaneMachineSet = data.ControlPlaneMachineSet
+		ipClaims = data.IPClaims
+		ipAddrs = data.IPAddresses
 
 		if ic.FeatureSet != configv1.TechPreviewNoUpgrade {
 			controlPlaneMachineSet = nil
@@ -563,6 +580,33 @@ func (m *Master) Generate(dependencies asset.Parents) error {
 			Data:     data,
 		}
 	}
+
+	m.IPClaimFiles = make([]*asset.File, len(ipClaims))
+	for i, claim := range ipClaims {
+		data, err := yaml.Marshal(claim)
+		if err != nil {
+			return errors.Wrapf(err, "unable to marshal ip claim %v", claim.Name)
+		}
+
+		m.IPClaimFiles[i] = &asset.File{
+			Filename: filepath.Join(directory, fmt.Sprintf(ipClaimFileName, claim.Name)),
+			Data:     data,
+		}
+	}
+
+	m.IPAddrFiles = make([]*asset.File, len(ipAddrs))
+	for i, address := range ipAddrs {
+		data, err := yaml.Marshal(address)
+		if err != nil {
+			return errors.Wrapf(err, "unable to marshal ip claim %v", address.Name)
+		}
+
+		m.IPAddrFiles[i] = &asset.File{
+			Filename: filepath.Join(directory, fmt.Sprintf(ipAddressFileName, address.Name)),
+			Data:     data,
+		}
+	}
+
 	padFormat := fmt.Sprintf("%%0%dd", len(fmt.Sprintf("%d", len(machines))))
 	for i, machine := range machines {
 		data, err := yaml.Marshal(machine)
@@ -598,6 +642,8 @@ func (m *Master) Files() []*asset.File {
 	if m.ControlPlaneMachineSet != nil {
 		files = append(files, m.ControlPlaneMachineSet)
 	}
+	files = append(files, m.IPClaimFiles...)
+	files = append(files, m.IPAddrFiles...)
 	return files
 }
 
@@ -654,6 +700,20 @@ func (m *Master) Load(f asset.FileFetcher) (found bool, err error) {
 	}
 	m.ControlPlaneMachineSet = file
 
+	fileList, err = f.FetchByPattern(filepath.Join(directory, masterIPClaimFileNamePattern))
+	if err != nil {
+		return true, err
+	}
+	logrus.Infof("Loaded %v ip claims.", len(fileList))
+	m.IPClaimFiles = fileList
+
+	fileList, err = f.FetchByPattern(filepath.Join(directory, masterIPAddressFileNamePattern))
+	if err != nil {
+		return true, err
+	}
+	logrus.Infof("Loaded %v ip addresses.", len(fileList))
+	m.IPAddrFiles = fileList
+
 	return true, nil
 }
 
@@ -735,6 +795,8 @@ func IsMachineManifest(file *asset.File) bool {
 		{Pattern: hostFileNamePattern, Type: "host"},
 		{Pattern: masterMachineFileNamePattern, Type: "master machine"},
 		{Pattern: workerMachineSetFileNamePattern, Type: "worker machineset"},
+		{Pattern: masterIPAddressFileNamePattern, Type: "master ip address"},
+		{Pattern: masterIPClaimFileNamePattern, Type: "master ip address claim"},
 	} {
 		if matched, err := filepath.Match(pattern.Pattern, filename); err != nil {
 			panic(fmt.Sprintf("bad format for %s file name pattern", pattern.Type))
@@ -745,6 +807,23 @@ func IsMachineManifest(file *asset.File) bool {
 	return false
 }
 
+// IPAddresses returns IPAddress manifest structures.
+func (m *Master) IPAddresses() ([]ipamv1.IPAddress, error) {
+	ipAddresses := []ipamv1.IPAddress{}
+	for i, file := range m.IPAddrFiles {
+		logrus.Debugf("Attempting to load address %v.", file.Filename)
+		address := &ipamv1.IPAddress{}
+		err := yaml.Unmarshal(file.Data, &address)
+		if err != nil {
+			return ipAddresses, errors.Wrapf(err, "unable to unmarshal ip address %d", i)
+		}
+
+		ipAddresses = append(ipAddresses, *address)
+	}
+
+	return ipAddresses, nil
+}
+
 func createSecretAssetFiles(resources []corev1.Secret, fileName string) ([]*asset.File, error) {
 
 	var objects []interface{}

pkg/asset/machines/vsphere/capimachines.go

@@ -21,6 +21,10 @@ import (
 	"github.com/openshift/installer/pkg/types"
 )
 
+const (
+	masterRole = "master"
+)
+
 // ProviderSpecFromRawExtension unmarshals the JSON-encoded spec.
 func ProviderSpecFromRawExtension(rawExtension *runtime.RawExtension) (*machinev1.VSphereMachineProviderSpec, error) {
 	if rawExtension == nil {
@@ -57,10 +61,11 @@ func getNetworkInventoryPath(vcenterContext vsphere.VCenterContext, networkName
 
 // GenerateMachines returns a list of capi machines.
 func GenerateMachines(ctx context.Context, clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage string, role string, metadata *vsphere.Metadata) ([]*asset.RuntimeFile, error) {
-	machines, _, err := Machines(clusterID, config, pool, osImage, role, "")
+	data, err := Machines(clusterID, config, pool, osImage, role, "")
 	if err != nil {
 		return nil, fmt.Errorf("unable to retrieve machines: %w", err)
 	}
+	machines := data.Machines
 
 	capvMachines := make([]*capv.VSphereMachine, 0, len(machines))
 	result := make([]*asset.RuntimeFile, 0, len(machines))
@@ -157,8 +162,8 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta
 		})
 	}
 
-	// as part of provisioning conrtol plane nodes, we need to create a bootstrap node as well
-	if role == "master" {
+	// as part of provisioning control plane nodes, we need to create a bootstrap node as well
+	if role == masterRole {
 		customVMXKeys := map[string]string{}
 
 		bootstrapSpec := capvMachines[0].Spec