Skip to content

Commit f87f13a

Browse files
Merge pull request #7418 from rvanderp3/cpms-vsphere-failure-domain
SPLAT-1141: implement vSphere control plane machinesets
2 parents 973140b + 9a7ec63 commit f87f13a

32 files changed

+1489
-42
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ require (
5656
github.com/microsoftgraph/msgraph-sdk-go v0.47.0
5757
github.com/nutanix-cloud-native/prism-go-client v0.2.1-0.20220804130801-c8a253627c64
5858
github.com/onsi/gomega v1.28.1
59-
github.com/openshift/api v0.0.0-20231019121755-0c5924392281
59+
github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020
6060
github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b
6161
github.com/openshift/assisted-service/api v0.0.0
6262
github.com/openshift/assisted-service/client v0.0.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -989,8 +989,8 @@ github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xA
989989
github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE=
990990
github.com/openshift/api v0.0.0-20200827090112-c05698d102cf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
991991
github.com/openshift/api v0.0.0-20200829102639-8a3a835f1acf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
992-
github.com/openshift/api v0.0.0-20231019121755-0c5924392281 h1:49ccho6DQLZpqxlFWyXpHRoLsQl0KVma+Ir+2zC9qSY=
993-
github.com/openshift/api v0.0.0-20231019121755-0c5924392281/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
992+
github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020 h1:oJfieDRL/5RJLu11oSpgZsI8EmGniTrQn/SyC6BZc68=
993+
github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
994994
github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b h1:wLVEgmzQjs3t4Z96gZzSLF/ws6ULliAks7z1lozNJrE=
995995
github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b/go.mod h1:KTt/pnfs9gt0McDPrb0zVTkwd0xIFNik/ZJROIBzsbc=
996996
github.com/openshift/assisted-service/api v0.0.0-20230831114549-1922eda29cf8 h1:+fZLKbycDo4JeLwPGVSAgf2XPaJGLM341l9ZfrrlxG0=

pkg/asset/machines/aws/machines.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func Machines(clusterID string, region string, subnets map[string]string, pool *
149149
Template: machinev1.ControlPlaneMachineSetTemplate{
150150
MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
151151
OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
152-
FailureDomains: machinev1.FailureDomains{
152+
FailureDomains: &machinev1.FailureDomains{
153153
Platform: v1.AWSPlatformType,
154154
AWS: &failureDomains,
155155
},

pkg/asset/machines/azure/machines.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
142142
}
143143

144144
if len(failureDomains) > 0 {
145-
controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = machinev1.FailureDomains{
145+
controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = &machinev1.FailureDomains{
146146
Platform: v1.AzurePlatformType,
147147
Azure: &failureDomains,
148148
}

pkg/asset/machines/gcp/machines.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
106106
Template: machinev1.ControlPlaneMachineSetTemplate{
107107
MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
108108
OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
109-
FailureDomains: machinev1.FailureDomains{
109+
FailureDomains: &machinev1.FailureDomains{
110110
Platform: v1.GCPPlatformType,
111111
GCP: &failureDomains,
112112
},

pkg/asset/machines/master.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -473,10 +473,15 @@ func (m *Master) Generate(dependencies asset.Parents) error {
473473
pool.Platform.VSphere = &mpool
474474
templateName := clusterID.InfraID + "-rhcos"
475475

476-
machines, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
476+
machines, controlPlaneMachineSet, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
477477
if err != nil {
478478
return errors.Wrap(err, "failed to create master machine objects")
479479
}
480+
481+
if ic.FeatureSet != configv1.TechPreviewNoUpgrade {
482+
controlPlaneMachineSet = nil
483+
}
484+
480485
vsphere.ConfigMasters(machines, clusterID.InfraID)
481486
case powervstypes.Name:
482487
mpool := defaultPowerVSMachinePoolPlatform()

pkg/asset/machines/openstack/machines.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
148148
}
149149

150150
if CPMSFailureDomains := pruneFailureDomains(failureDomains); CPMSFailureDomains != nil {
151-
controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = machinev1.FailureDomains{
151+
controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = &machinev1.FailureDomains{
152152
Platform: v1.OpenStackPlatformType,
153153
OpenStack: CPMSFailureDomains,
154154
}

pkg/asset/machines/vsphere/machines.go

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,35 +10,37 @@ import (
1010
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1111
"k8s.io/apimachinery/pkg/runtime"
1212

13+
v1 "github.com/openshift/api/config/v1"
14+
machinev1 "github.com/openshift/api/machine/v1"
1315
machineapi "github.com/openshift/api/machine/v1beta1"
1416
"github.com/openshift/installer/pkg/types"
1517
"github.com/openshift/installer/pkg/types/vsphere"
1618
)
1719

1820
// Machines returns a list of machines for a machinepool.
19-
func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) ([]machineapi.Machine, error) {
21+
func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) ([]machineapi.Machine, *machinev1.ControlPlaneMachineSet, error) {
2022
if configPlatform := config.Platform.Name(); configPlatform != vsphere.Name {
21-
return nil, fmt.Errorf("non vsphere configuration: %q", configPlatform)
23+
return nil, nil, fmt.Errorf("non vsphere configuration: %q", configPlatform)
2224
}
2325
if poolPlatform := pool.Platform.Name(); poolPlatform != vsphere.Name {
24-
return nil, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform)
26+
return nil, nil, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform)
2527
}
2628

2729
var failureDomain vsphere.FailureDomain
2830
var machines []machineapi.Machine
2931
platform := config.Platform.VSphere
3032
mpool := pool.Platform.VSphere
31-
replicas := int64(1)
33+
replicas := int32(1)
3234

3335
numOfZones := len(mpool.Zones)
3436

3537
zones, err := getDefinedZonesFromTopology(platform)
3638
if err != nil {
37-
return machines, err
39+
return machines, nil, err
3840
}
3941

4042
if pool.Replicas != nil {
41-
replicas = *pool.Replicas
43+
replicas = int32(*pool.Replicas)
4244
}
4345

4446
// Create hosts to populate from. Copying so we can remove without changing original
@@ -53,7 +55,11 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
5355
}
5456
}
5557

56-
for idx := int64(0); idx < replicas; idx++ {
58+
failureDomains := []machinev1.VSphereFailureDomain{}
59+
60+
vsphereMachineProvider := &machineapi.VSphereMachineProviderSpec{}
61+
62+
for idx := int32(0); idx < replicas; idx++ {
5763
logrus.Debugf("Creating %v machine %v", role, idx)
5864
var host *vsphere.Host
5965
desiredZone := mpool.Zones[int(idx)%numOfZones]
@@ -66,7 +72,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
6672
logrus.Debugf("Desired zone: %v", desiredZone)
6773

6874
if _, exists := zones[desiredZone]; !exists {
69-
return nil, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone)
75+
return nil, nil, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone)
7076
}
7177

7278
failureDomain = zones[desiredZone]
@@ -77,18 +83,22 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
7783
"machine.openshift.io/cluster-api-machine-type": role,
7884
}
7985

86+
failureDomains = append(failureDomains, machinev1.VSphereFailureDomain{
87+
Name: failureDomain.Name,
88+
})
89+
8090
osImageForZone := failureDomain.Topology.Template
8191
if failureDomain.Topology.Template == "" {
8292
osImageForZone = fmt.Sprintf("%s-%s-%s", osImage, failureDomain.Region, failureDomain.Zone)
8393
}
8494

8595
vcenter, err := getVCenterFromServerName(failureDomain.Server, platform)
8696
if err != nil {
87-
return nil, errors.Wrap(err, "unable to find vCenter in failure domains")
97+
return nil, nil, errors.Wrap(err, "unable to find vCenter in failure domains")
8898
}
8999
provider, err := provider(clusterID, vcenter, failureDomain, mpool, osImageForZone, userDataSecret)
90100
if err != nil {
91-
return nil, errors.Wrap(err, "failed to create provider")
101+
return nil, nil, errors.Wrap(err, "failed to create provider")
92102
}
93103

94104
// Apply static IP if configured
@@ -112,8 +122,74 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
112122
},
113123
}
114124
machines = append(machines, machine)
125+
126+
vsphereMachineProvider = provider.DeepCopy()
127+
}
128+
129+
// when multiple zones are defined, network and workspace are derived from the topology
130+
origProv := vsphereMachineProvider.DeepCopy()
131+
if len(failureDomains) > 1 {
132+
vsphereMachineProvider.Network = machineapi.NetworkSpec{}
133+
vsphereMachineProvider.Workspace = &machineapi.Workspace{}
134+
vsphereMachineProvider.Template = ""
135+
}
136+
137+
if len(hosts) > 0 {
138+
vsphereMachineProvider.Network.Devices = []machineapi.NetworkDeviceSpec{
139+
{
140+
AddressesFromPools: origProv.Network.Devices[0].AddressesFromPools,
141+
Nameservers: origProv.Network.Devices[0].Nameservers,
142+
},
143+
}
144+
}
145+
146+
controlPlaneMachineSet := &machinev1.ControlPlaneMachineSet{
147+
TypeMeta: metav1.TypeMeta{
148+
APIVersion: "machine.openshift.io/v1",
149+
Kind: "ControlPlaneMachineSet",
150+
},
151+
ObjectMeta: metav1.ObjectMeta{
152+
Namespace: "openshift-machine-api",
153+
Name: "cluster",
154+
Labels: map[string]string{
155+
"machine.openshift.io/cluster-api-cluster": clusterID,
156+
},
157+
},
158+
Spec: machinev1.ControlPlaneMachineSetSpec{
159+
Replicas: &replicas,
160+
State: machinev1.ControlPlaneMachineSetStateActive,
161+
Selector: metav1.LabelSelector{
162+
MatchLabels: map[string]string{
163+
"machine.openshift.io/cluster-api-machine-role": role,
164+
"machine.openshift.io/cluster-api-machine-type": role,
165+
"machine.openshift.io/cluster-api-cluster": clusterID,
166+
},
167+
},
168+
Template: machinev1.ControlPlaneMachineSetTemplate{
169+
MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
170+
OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
171+
FailureDomains: &machinev1.FailureDomains{
172+
Platform: v1.VSpherePlatformType,
173+
VSphere: failureDomains,
174+
},
175+
ObjectMeta: machinev1.ControlPlaneMachineSetTemplateObjectMeta{
176+
Labels: map[string]string{
177+
"machine.openshift.io/cluster-api-cluster": clusterID,
178+
"machine.openshift.io/cluster-api-machine-role": role,
179+
"machine.openshift.io/cluster-api-machine-type": role,
180+
},
181+
},
182+
Spec: machineapi.MachineSpec{
183+
ProviderSpec: machineapi.ProviderSpec{
184+
Value: &runtime.RawExtension{Object: vsphereMachineProvider},
185+
},
186+
},
187+
},
188+
},
189+
},
115190
}
116-
return machines, nil
191+
192+
return machines, controlPlaneMachineSet, nil
117193
}
118194

119195
// applyNetworkConfig this function will apply the static ip configuration to the networkDevice

pkg/asset/machines/vsphere/machines_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ func TestConfigMasters(t *testing.T) {
378378

379379
for _, tc := range testCases {
380380
t.Run(tc.testCase, func(t *testing.T) {
381-
machines, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "")
381+
machines, _, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "")
382382
assertOnUnexpectedErrorState(t, tc.expectedError, err)
383383

384384
if len(tc.workspaces) > 0 {
@@ -449,7 +449,7 @@ func TestHostsToMachines(t *testing.T) {
449449

450450
for _, tc := range testCases {
451451
t.Run(tc.testCase, func(t *testing.T) {
452-
machines, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "")
452+
machines, _, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "")
453453
assertOnUnexpectedErrorState(t, tc.expectedError, err)
454454

455455
// Check machine counts

pkg/asset/machines/worker.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ func (w *Worker) Generate(dependencies asset.Parents) error {
627627
logrus.Debug("Generating worker machines with static IPs.")
628628
templateName := clusterID.InfraID + "-rhcos"
629629

630-
machines, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName)
630+
machines, _, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName)
631631
if err != nil {
632632
return errors.Wrap(err, "failed to create worker machine objects")
633633
}

0 commit comments

Comments
 (0)