Skip to content

Commit 9b838c3

Browse files
committed
Add vSphere vm-host group zonal
This PR adds vm-host group zonal feature to MAO. Misc required change: - Added RealClock to NewKubeRecorder and NewLoggingEventRecorder Changes: - Added FeatureGateVSphereHostVMGroupZonal - Added modifyVMGroup which adds or removes a virtual machine from the vCenter cluster's vm-host group associated with the failure domain - Added validation for the length of a vm-group name - Added vm-host group tests
1 parent d631c4d commit 9b838c3

File tree

10 files changed

+411
-87
lines changed

10 files changed

+411
-87
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.idea
2+
.run
23
# Binaries for programs and plugins
34
*.exe
45
*.exe~

cmd/machine-api-operator/start.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,6 @@ import (
99
"os"
1010
"strconv"
1111

12-
osconfigv1 "github.com/openshift/api/config/v1"
13-
"github.com/openshift/library-go/pkg/operator/events"
14-
"github.com/openshift/machine-api-operator/pkg/metrics"
15-
"github.com/openshift/machine-api-operator/pkg/operator"
16-
"github.com/openshift/machine-api-operator/pkg/util"
17-
"github.com/openshift/machine-api-operator/pkg/version"
1812
"github.com/prometheus/client_golang/prometheus"
1913
"github.com/prometheus/client_golang/prometheus/promhttp"
2014
"github.com/spf13/cobra"
@@ -27,6 +21,14 @@ import (
2721
"k8s.io/client-go/tools/leaderelection"
2822
"k8s.io/client-go/tools/record"
2923
"k8s.io/klog/v2"
24+
"k8s.io/utils/clock"
25+
26+
osconfigv1 "github.com/openshift/api/config/v1"
27+
"github.com/openshift/library-go/pkg/operator/events"
28+
"github.com/openshift/machine-api-operator/pkg/metrics"
29+
"github.com/openshift/machine-api-operator/pkg/operator"
30+
"github.com/openshift/machine-api-operator/pkg/util"
31+
"github.com/openshift/machine-api-operator/pkg/version"
3032
)
3133

3234
const (
@@ -137,7 +139,7 @@ func initRecorder(kubeClient kubernetes.Interface) (events.Recorder, error) {
137139
if err != nil {
138140
return nil, fmt.Errorf("failed to create controller ref for recorder: %v", err)
139141
}
140-
recorder := events.NewKubeRecorder(kubeClient.CoreV1().Events(componentNamespace), "machineapioperator", controllerRef)
142+
recorder := events.NewKubeRecorder(kubeClient.CoreV1().Events(componentNamespace), "machineapioperator", controllerRef, clock.RealClock{})
141143
return recorder, nil
142144
}
143145

cmd/machineset/main.go

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,14 @@ import (
2323
"strings"
2424
"time"
2525

26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
ctrl "sigs.k8s.io/controller-runtime"
28+
2629
osconfigv1 "github.com/openshift/api/config/v1"
2730
apifeatures "github.com/openshift/api/features"
2831
machinev1 "github.com/openshift/api/machine/v1beta1"
2932
mapiwebhooks "github.com/openshift/machine-api-operator/pkg/webhooks"
30-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31-
ctrl "sigs.k8s.io/controller-runtime"
3233

33-
"github.com/openshift/library-go/pkg/config/leaderelection"
34-
"github.com/openshift/library-go/pkg/features"
35-
"github.com/openshift/machine-api-operator/pkg/controller"
36-
"github.com/openshift/machine-api-operator/pkg/controller/machineset"
37-
"github.com/openshift/machine-api-operator/pkg/metrics"
38-
"github.com/openshift/machine-api-operator/pkg/operator"
39-
"github.com/openshift/machine-api-operator/pkg/util"
4034
"k8s.io/apiserver/pkg/util/feature"
4135
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
4236
"k8s.io/component-base/featuregate"
@@ -48,6 +42,14 @@ import (
4842
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
4943
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
5044
"sigs.k8s.io/controller-runtime/pkg/webhook"
45+
46+
"github.com/openshift/library-go/pkg/config/leaderelection"
47+
"github.com/openshift/library-go/pkg/features"
48+
"github.com/openshift/machine-api-operator/pkg/controller"
49+
"github.com/openshift/machine-api-operator/pkg/controller/machineset"
50+
"github.com/openshift/machine-api-operator/pkg/metrics"
51+
"github.com/openshift/machine-api-operator/pkg/operator"
52+
"github.com/openshift/machine-api-operator/pkg/util"
5153
)
5254

5355
const (
@@ -110,7 +112,7 @@ func main() {
110112

111113
// Sets up feature gates
112114
defaultMutableGate := feature.DefaultMutableFeatureGate
113-
gateOpts, err := features.NewFeatureGateOptions(defaultMutableGate, apifeatures.SelfManaged, apifeatures.FeatureGateVSphereStaticIPs, apifeatures.FeatureGateMachineAPIMigration)
115+
gateOpts, err := features.NewFeatureGateOptions(defaultMutableGate, apifeatures.SelfManaged, apifeatures.FeatureGateVSphereStaticIPs, apifeatures.FeatureGateMachineAPIMigration, apifeatures.FeatureGateVSphereHostVMGroupZonal)
114116
if err != nil {
115117
klog.Fatalf("Error setting up feature gates: %v", err)
116118
}

cmd/vsphere/main.go

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,6 @@ import (
77
"strings"
88
"time"
99

10-
configv1 "github.com/openshift/api/config/v1"
11-
apifeatures "github.com/openshift/api/features"
12-
machinev1 "github.com/openshift/api/machine/v1beta1"
13-
"github.com/openshift/library-go/pkg/config/leaderelection"
14-
"github.com/openshift/library-go/pkg/features"
15-
capimachine "github.com/openshift/machine-api-operator/pkg/controller/machine"
16-
"github.com/openshift/machine-api-operator/pkg/controller/vsphere"
17-
machine "github.com/openshift/machine-api-operator/pkg/controller/vsphere"
18-
machinesetcontroller "github.com/openshift/machine-api-operator/pkg/controller/vsphere/machineset"
19-
"github.com/openshift/machine-api-operator/pkg/metrics"
20-
"github.com/openshift/machine-api-operator/pkg/util"
21-
"github.com/openshift/machine-api-operator/pkg/version"
2210
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2311
"k8s.io/apiserver/pkg/util/feature"
2412
"k8s.io/component-base/featuregate"
@@ -31,6 +19,19 @@ import (
3119
"sigs.k8s.io/controller-runtime/pkg/healthz"
3220
"sigs.k8s.io/controller-runtime/pkg/manager"
3321
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
22+
23+
configv1 "github.com/openshift/api/config/v1"
24+
apifeatures "github.com/openshift/api/features"
25+
machinev1 "github.com/openshift/api/machine/v1beta1"
26+
"github.com/openshift/library-go/pkg/config/leaderelection"
27+
"github.com/openshift/library-go/pkg/features"
28+
capimachine "github.com/openshift/machine-api-operator/pkg/controller/machine"
29+
"github.com/openshift/machine-api-operator/pkg/controller/vsphere"
30+
machine "github.com/openshift/machine-api-operator/pkg/controller/vsphere"
31+
machinesetcontroller "github.com/openshift/machine-api-operator/pkg/controller/vsphere/machineset"
32+
"github.com/openshift/machine-api-operator/pkg/metrics"
33+
"github.com/openshift/machine-api-operator/pkg/util"
34+
"github.com/openshift/machine-api-operator/pkg/version"
3435
)
3536

3637
const timeout = 10 * time.Minute
@@ -92,7 +93,7 @@ func main() {
9293

9394
// Sets up feature gates
9495
defaultMutableGate := feature.DefaultMutableFeatureGate
95-
gateOpts, err := features.NewFeatureGateOptions(defaultMutableGate, apifeatures.SelfManaged, apifeatures.FeatureGateVSphereStaticIPs, apifeatures.FeatureGateMachineAPIMigration)
96+
gateOpts, err := features.NewFeatureGateOptions(defaultMutableGate, apifeatures.SelfManaged, apifeatures.FeatureGateVSphereStaticIPs, apifeatures.FeatureGateMachineAPIMigration, apifeatures.FeatureGateVSphereHostVMGroupZonal)
9697
if err != nil {
9798
klog.Fatalf("Error setting up feature gates: %v", err)
9899
}
@@ -152,6 +153,8 @@ func main() {
152153

153154
staticIPFeatureGateEnabled := defaultMutableGate.Enabled(featuregate.Feature(apifeatures.FeatureGateVSphereStaticIPs))
154155
klog.Infof("FeatureGateVSphereStaticIPs initialised: %t", staticIPFeatureGateEnabled)
156+
hostVMGroupZonalFeatureGateEnabled := defaultMutableGate.Enabled(featuregate.Feature(apifeatures.FeatureGateVSphereHostVMGroupZonal))
157+
klog.Infof("FeatureGateVSphereHostVMGroupZonal initialised %t", hostVMGroupZonalFeatureGateEnabled)
155158

156159
// Setup a Manager
157160
mgr, err := manager.New(cfg, opts)

pkg/controller/vsphere/reconciler.go

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@ import (
88
"fmt"
99
"net"
1010
"net/netip"
11+
"slices"
1112
"strconv"
1213
"strings"
1314

15+
"github.com/vmware/govmomi/task"
16+
1417
"github.com/openshift/machine-api-operator/pkg/util/ipam"
1518

1619
"github.com/google/uuid"
@@ -83,6 +86,10 @@ func (r *Reconciler) create() error {
8386
return fmt.Errorf("%v: failed validating machine provider spec: %w", r.machine.GetName(), err)
8487
}
8588

89+
if r.providerSpec.Workspace.VMGroup != "" && !r.featureGates.Enabled(featuregate.Feature(apifeatures.FeatureGateVSphereHostVMGroupZonal)) {
90+
return fmt.Errorf("%v: vmGroup is only available with the VSphereHostVMGroupZonal feature gate", r.machine.GetName())
91+
}
92+
8693
if ipam.HasStaticIPConfiguration(r.providerSpec) {
8794
if !r.featureGates.Enabled(featuregate.Feature(apifeatures.FeatureGateVSphereStaticIPs)) {
8895
return fmt.Errorf("%v: static IP/IPAM configuration is only available with the VSphereStaticIPs feature gate", r.machine.GetName())
@@ -209,7 +216,21 @@ func (r *Reconciler) create() error {
209216
}
210217

211218
// if clone task finished successfully, power on the vm
212-
if moTask.Info.DescriptionId == cloneVmTaskDescriptionId {
219+
// The simulator task.Info.DescriptionId is different (VirtualMachine.cloneVM)
220+
if strings.Contains(moTask.Info.DescriptionId, cloneVmTaskDescriptionId) {
221+
if r.machineScope.providerSpec.Workspace.VMGroup != "" {
222+
klog.Infof("Adding on cloned machine: %s to vm group: %s", r.machine.Name, r.machineScope.providerSpec.Workspace.VMGroup)
223+
224+
if err := modifyVMGroup(r.machineScope, false); err != nil {
225+
var taskError task.Error
226+
if errors.As(err, &taskError) {
227+
return fmt.Errorf("could not update VM Group membership: %w", taskError)
228+
}
229+
230+
return fmt.Errorf("could not update VM Group membership: %w", err)
231+
}
232+
}
233+
213234
klog.Infof("Powering on cloned machine: %v", r.machine.Name)
214235
task, err := powerOn(r.machineScope)
215236
if err != nil {
@@ -494,6 +515,13 @@ func (r *Reconciler) delete() error {
494515
return fmt.Errorf("%v: failed to destroy vm: %w", r.machine.GetName(), err)
495516
}
496517

518+
if r.machineScope.providerSpec.Workspace.VMGroup != "" {
519+
klog.Infof("Removing machine: %v from vm group: %v", r.machine.Name, r.machineScope.providerSpec.Workspace.VMGroup)
520+
if err := modifyVMGroup(r.machineScope, true); err != nil {
521+
return fmt.Errorf("failed to remove machine from vm group: %w", err)
522+
}
523+
}
524+
497525
if err := setProviderStatus(task.Reference().Value, conditionSuccess(), r.machineScope, vm); err != nil {
498526
return fmt.Errorf("failed to set provider status: %w", err)
499527
}
@@ -1036,6 +1064,86 @@ func clone(s *machineScope) (string, error) {
10361064
return taskVal, nil
10371065
}
10381066

1067+
func modifyVMGroup(s *machineScope, delete bool) error {
1068+
vmRef, err := findVM(s)
1069+
if err != nil {
1070+
if isNotFound(err) {
1071+
return fmt.Errorf("virtual machine %s was not found: %w", s.machine.Name, err)
1072+
}
1073+
return fmt.Errorf("error finding virtual machine: %w", err)
1074+
}
1075+
1076+
rp, err := s.session.Finder.ResourcePool(s.Context, s.providerSpec.Workspace.ResourcePool)
1077+
if err != nil {
1078+
return fmt.Errorf("error getting resource pool %s: %w", s.providerSpec.Workspace.ResourcePool, err)
1079+
}
1080+
1081+
ownerRef, err := rp.Owner(s.Context)
1082+
if err != nil {
1083+
return fmt.Errorf("error getting cluster owner reference from resource pool %s: %w", s.providerSpec.Workspace.ResourcePool, err)
1084+
}
1085+
1086+
var ccr *object.ClusterComputeResource
1087+
var ok bool
1088+
if ccr, ok = ownerRef.(*object.ClusterComputeResource); !ok {
1089+
return fmt.Errorf("error getting cluster from resource pool %s: %w", s.providerSpec.Workspace.ResourcePool, err)
1090+
}
1091+
1092+
clusterConfig, err := ccr.Configuration(s.Context)
1093+
if err != nil {
1094+
return fmt.Errorf("error getting cluster %s configuration: %w", s.providerSpec.Workspace.ResourcePool, err)
1095+
}
1096+
1097+
var clusterVmGroup *types.ClusterVmGroup
1098+
1099+
for _, g := range clusterConfig.Group {
1100+
if vmg, ok := g.(*types.ClusterVmGroup); ok {
1101+
if vmg.Name == s.providerSpec.Workspace.VMGroup {
1102+
clusterVmGroup = vmg
1103+
break
1104+
}
1105+
}
1106+
}
1107+
1108+
switch {
1109+
case clusterVmGroup == nil:
1110+
clusterVmGroup = &types.ClusterVmGroup{
1111+
Vm: []types.ManagedObjectReference{vmRef},
1112+
}
1113+
case slices.Contains(clusterVmGroup.Vm, vmRef) && delete:
1114+
clusterVmGroup.Vm = slices.DeleteFunc(clusterVmGroup.Vm, func(ref types.ManagedObjectReference) bool {
1115+
return vmRef.Value == ref.Value
1116+
})
1117+
case !slices.Contains(clusterVmGroup.Vm, vmRef):
1118+
clusterVmGroup.Vm = append(clusterVmGroup.Vm, vmRef)
1119+
default:
1120+
return nil
1121+
}
1122+
1123+
clusterConfigSpec := &types.ClusterConfigSpecEx{
1124+
GroupSpec: []types.ClusterGroupSpec{
1125+
{
1126+
ArrayUpdateSpec: types.ArrayUpdateSpec{
1127+
Operation: types.ArrayUpdateOperation("edit"),
1128+
},
1129+
Info: &types.ClusterVmGroup{
1130+
ClusterGroupInfo: types.ClusterGroupInfo{
1131+
Name: s.providerSpec.Workspace.VMGroup,
1132+
},
1133+
Vm: clusterVmGroup.Vm,
1134+
},
1135+
},
1136+
},
1137+
}
1138+
1139+
clusterTask, err := ccr.Reconfigure(s.Context, clusterConfigSpec, true)
1140+
if err != nil {
1141+
return fmt.Errorf("error reconfiguring cluster %s for vm-host group %s: %w", ccr.Name(), clusterVmGroup.Name, err)
1142+
}
1143+
1144+
return clusterTask.Wait(s.Context)
1145+
}
1146+
10391147
func powerOn(s *machineScope) (string, error) {
10401148
vmRef, err := findVM(s)
10411149
if err != nil {

0 commit comments

Comments
 (0)