diff --git a/test/extended/boot_image.go b/test/extended/boot_image.go index 50b05f3c83..5fb4c03c42 100644 --- a/test/extended/boot_image.go +++ b/test/extended/boot_image.go @@ -104,6 +104,7 @@ func getRandomMachineSet(machineClient *machineclient.Clientset) machinev1beta1. } // verifyMachineSetUpdate verifies that the the boot image values of a MachineSet are reconciled correctly +// nolint:dupl // I separated these from verifyControlPlaneMachineSetUpdate for readability func verifyMachineSetUpdate(oc *exutil.CLI, machineSet machinev1beta1.MachineSet, updateExpected bool) { newProviderSpecPatch, originalProviderSpecPatch, backdatedBootImage, originalBootImage := createFakeUpdatePatch(oc, machineSet) diff --git a/test/extended/boot_image_cpms.go b/test/extended/boot_image_cpms.go new file mode 100644 index 0000000000..9a94ea017b --- /dev/null +++ b/test/extended/boot_image_cpms.go @@ -0,0 +1,136 @@ +package extended + +import ( + "context" + "fmt" + + osconfigv1 "github.com/openshift/api/config/v1" + "sigs.k8s.io/yaml" + + machinev1 "github.com/openshift/api/machine/v1" + machinev1beta1 "github.com/openshift/api/machine/v1beta1" + exutil "github.com/openshift/machine-config-operator/test/extended/util" + + o "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// verifyControlPlaneMachineSetUpdate verifies that the the boot image values of a ControlPlaneMachineSet are reconciled correctly +// nolint:dupl // I separated these from verifyMachineSetUpdate for readability +func verifyControlPlaneMachineSetUpdate(oc *exutil.CLI, cpms machinev1.ControlPlaneMachineSet, updateExpected bool) { + + newProviderSpecPatch, originalProviderSpecPatch, backdatedBootImage, originalBootImage := createFakeUpdatePatchCPMS(oc, cpms) + err := oc.Run("patch").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-p", newProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + defer func() { + // Restore machineSet to original boot image as the machineset may be used by other test variants, regardless of success/fail + err = oc.Run("patch").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-p", originalProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Restored build name in the machineset %s to \"%s\"", cpms.Name, originalBootImage) + }() + // Ensure boot image controller is not progressing + e2e.Logf("Waiting until the boot image controller is not progressing...") + waitForBootImageControllerToComplete(oc) + + // Fetch the providerSpec of the machineset under test again + providerSpec, err := oc.Run("get").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-o", "template", "--template=`{{.spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value}}`", "-n", MAPINamespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Verify that the machineset has the expected boot image values + // If an update is expected, the backdated boot image should not be present + // If an update is NOT expected, the backdated boot image should still be present; ie machineset is left untouched + if updateExpected { + o.Expect(providerSpec).ShouldNot(o.ContainSubstring(backdatedBootImage)) + } else { + o.Expect(providerSpec).Should(o.ContainSubstring(backdatedBootImage)) + } +} + +// createFakeUpdatePatchCPMS creates an update patch for the ControlPlaneMachineSet object based on the platform +func createFakeUpdatePatchCPMS(oc *exutil.CLI, cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) { + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + switch infra.Status.PlatformStatus.Type { + case osconfigv1.AWSPlatformType: + return generateAWSProviderSpecPatchCPMS(cpms) + case osconfigv1.GCPPlatformType: + return generateGCPProviderSpecPatchCPMS(cpms) + case osconfigv1.AzurePlatformType: + return generateAzureProviderSpecPatchCPMS(cpms) + default: + e2e.Failf("unexpected platform type; should not be here") + return "", "", "", "" + } +} + +// generateAWSProviderSpecPatchCPMS generates a fake update patch for the AWS ControlPlaneMachineSet +func generateAWSProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) { + providerSpec := new(machinev1beta1.AWSMachineProviderConfig) + err := unmarshalProviderSpecCPMS(&cpms, providerSpec) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Modify the boot image to an older known AMI value + // See: https://issues.redhat.com/browse/OCPBUGS-57426 + originalBootImage := *providerSpec.AMI.ID + newBootImage := "ami-000145e5a91e9ac22" + jsonPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/ami/id", "value": "%s"}]`, newBootImage) + + // Create JSON patch to restore original AMI ID + originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/ami/id", "value": "%s"}]`, originalBootImage) + + return jsonPatch, originalJSONPatch, newBootImage, originalBootImage + +} + +// generateGCPProviderSpecPatchCPMS generates a fake update patch for the GCP ControlPlaneMachineSet +func generateGCPProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) { + providerSpec := new(machinev1beta1.GCPMachineProviderSpec) + err := unmarshalProviderSpecCPMS(&cpms, providerSpec) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Modify the boot image to a older known value. + // See: https://issues.redhat.com/browse/OCPBUGS-57426 + originalBootImage := providerSpec.Disks[0].Image + newBootImage := "projects/rhcos-cloud/global/images/rhcos-410-84-202210040010-0-gcp-x86-64" + jsonPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/disks/0/image", "value": "%s"}]`, newBootImage) + + // Create JSON patch to restore original disk image + originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/disks/0/image", "value": "%s"}]`, originalBootImage) + + return jsonPatch, originalJSONPatch, newBootImage, originalBootImage +} + +// generateAzureProviderSpecPatchCPMS generates a fake update patch for the Azure ControlPlaneMachineSet +func generateAzureProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) { + providerSpec := new(machinev1beta1.AzureMachineProviderSpec) + err := unmarshalProviderSpecCPMS(&cpms, providerSpec) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Use JSON patch to precisely replace just the image field with marketplace image + // This avoids any merge conflicts with existing fields + // Use an older known 4.18 boot image that is available in the marketplace + jsonPatch := `[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/image", "value": {"offer": "aro4", "publisher": "azureopenshift", "resourceID": "", "sku": "418-v2", "version": "418.94.20250122", "type": "MarketplaceNoPlan"}}]` + + // Create JSON patch to restore original image + originalImage := providerSpec.Image + originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/image", "value": {"offer": "%s", "publisher": "%s", "resourceID": "%s", "sku": "%s", "version": "%s", "type": "%s"}}]`, + originalImage.Offer, originalImage.Publisher, originalImage.ResourceID, originalImage.SKU, originalImage.Version, originalImage.Type) + + return jsonPatch, originalJSONPatch, "418.94.20250122", providerSpec.Image.Version +} + +// unmarshalProviderSpecCPMS unmarshals the controlplanemachineset's provider spec into +// a ProviderSpec object. Returns an error if providerSpec field is nil, +// or the unmarshal fails +func unmarshalProviderSpecCPMS(cpms *machinev1.ControlPlaneMachineSet, providerSpec interface{}) error { + if cpms.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value == nil { + return fmt.Errorf("providerSpec field was empty") + } + if err := yaml.Unmarshal(cpms.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Raw, &providerSpec); err != nil { + return fmt.Errorf("unmarshal into providerSpec failed %w", err) + } + return nil +} diff --git a/test/extended/boot_image_update_agnostic.go b/test/extended/boot_image_update_agnostic.go index 4a87478548..c473a3ab08 100644 --- a/test/extended/boot_image_update_agnostic.go +++ b/test/extended/boot_image_update_agnostic.go @@ -10,7 +10,10 @@ import ( o "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" ) func AllMachineSetTest(oc *exutil.CLI, fixture string) { @@ -101,3 +104,117 @@ func EnsureConfigMapStampTest(oc *exutil.CLI) { }, 2*time.Minute, 5*time.Second).Should(o.BeTrue()) framework.Logf("Successfully verified that the configmap has been correctly stamped") } + +func AllControlPlaneMachineSetTest(oc *exutil.CLI, fixture string) { + // This fixture applies a boot image update configuration that opts in all controlplanemachinesets + // However, since CPMS is typically a singleton, it is just targeting a single resource + applyMachineConfigurationFixture(oc, fixture) + + // Grab the CPMS and verify that the boot image was reconciled correctly. + machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + + cpms, err := machineClient.MachineV1().ControlPlaneMachineSets("openshift-machine-api").Get(context.TODO(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + verifyControlPlaneMachineSetUpdate(oc, *cpms, true) + + // Delete a control plane machine to verify that CPMS reconciles it with the updated boot image + // Get the list of control plane machines + machines, err := machineClient.MachineV1beta1().Machines(MAPINamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: MAPIMasterMachineLabelSelector}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(machines.Items).NotTo(o.BeEmpty(), "No control plane machines found") + + // Capture the initial set of control plane machine names and count before deletion + initialMachineNames := sets.New[string]() + for _, machine := range machines.Items { + initialMachineNames.Insert(machine.Name) + } + initialMachineCount := initialMachineNames.Len() + + // Delete the first control plane machine + machineToDelete := machines.Items[0].Name + framework.Logf("Deleting control plane machine: %s", machineToDelete) + err = machineClient.MachineV1beta1().Machines(MAPINamespace).Delete(context.TODO(), machineToDelete, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait until the new control plane machine is running and the old one is deleted + // Arbitrarily picking 25 minutes timeout as scale-up time varies based on platform + framework.Logf("Waiting for CPMS to reconcile and create a new control plane machine (up to 25 minutes)...") + o.Eventually(func() bool { + currentMachines, err := machineClient.MachineV1beta1().Machines(MAPINamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: MAPIMasterMachineLabelSelector}) + if err != nil { + framework.Logf("Error listing machines: %v", err) + return false + } + + // Check that the deleted control plane machine is gone and all current machines are running + currentMachineNames := sets.New[string]() + runningMachines := sets.New[string]() + + for _, machine := range currentMachines.Items { + currentMachineNames.Insert(machine.Name) + phase := "" + if machine.Status.Phase != nil { + phase = *machine.Status.Phase + } + if phase == machinev1beta1.PhaseRunning { + runningMachines.Insert(machine.Name) + } else { + framework.Logf("Machine %s is in phase: %s", machine.Name, phase) + } + } + + // All machines must be running + if runningMachines.Len() != initialMachineCount { + framework.Logf("Only %d out of %d machines are running", runningMachines.Len(), initialMachineCount) + return false + } + + // The deleted machine should not be in the current set + if currentMachineNames.Has(machineToDelete) { + framework.Logf("Deleted machine %s still exists", machineToDelete) + return false + } + + framework.Logf("All %d control plane machines are running and the deleted machine is gone", initialMachineCount) + + // Ensure master MCP is done updating and has the correct ready count + masterMCP := NewMachineConfigPool(oc, MachineConfigPoolMaster) + updatedStatus, err := masterMCP.GetUpdatedStatus() + if err != nil { + framework.Logf("Error getting master MCP updated status: %v", err) + return false + } + if updatedStatus != TrueString { + framework.Logf("Master MCP is not yet updated (Updated=%s)", updatedStatus) + return false + } + + readyMachineCount, err := masterMCP.getUpdatedMachineCount() + if err != nil { + framework.Logf("Error getting master MCP ready machine count: %v", err) + return false + } + if readyMachineCount != initialMachineCount { + framework.Logf("Master MCP ready machine count %d does not match initial count %d", readyMachineCount, initialMachineCount) + return false + } + + framework.Logf("Master MCP is updated with %d ready machines", readyMachineCount) + return true + }, 25*time.Minute, 2*time.Minute).Should(o.BeTrue(), "CPMS failed to reconcile control plane machines within 25 minutes") +} + +func NoneControlPlaneMachineSetTest(oc *exutil.CLI, fixture string) { + // This fixture applies a boot image update configuration that opts in no controlplanemachineset, i.e. feature is disabled. + applyMachineConfigurationFixture(oc, fixture) + + // Grab the CPMS and verify that the boot image was reconciled correctly. + machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + + cpms, err := machineClient.MachineV1().ControlPlaneMachineSets("openshift-machine-api").Get(context.TODO(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + verifyControlPlaneMachineSetUpdate(oc, *cpms, false) +} diff --git a/test/extended/boot_image_update_aws.go b/test/extended/boot_image_update_aws.go new file mode 100644 index 0000000000..7e0c8a6222 --- /dev/null +++ b/test/extended/boot_image_update_aws.go @@ -0,0 +1,45 @@ +package extended + +import ( + "path/filepath" + + osconfigv1 "github.com/openshift/api/config/v1" + + g "github.com/onsi/ginkgo/v2" + exutil "github.com/openshift/machine-config-operator/test/extended/util" +) + +// These tests are [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test. +var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][OCPFeatureGate:ManagedBootImagesAWS]", g.Ordered, func() { + defer g.GinkgoRecover() + var ( + AllControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml") + NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml") + EmptyMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml") + + oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin() + ) + + g.BeforeEach(func() { + // Skip this test if not on AWS platform + skipUnlessTargetPlatform(oc, osconfigv1.AWSPlatformType) + // Skip this test if the cluster is not using MachineAPI + skipUnlessFunctionalMachineAPI(oc) + // Skip this test on single node platforms + skipOnSingleNodeTopology(oc) + }) + + g.AfterEach(func() { + // Clear out boot image configuration between tests + applyMachineConfigurationFixture(oc, EmptyMachineSetFixture) + }) + + // This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down. + g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() { + AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture) + }) + + g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() { + NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture) + }) +}) diff --git a/test/extended/boot_image_update_azure.go b/test/extended/boot_image_update_azure.go index 4302a475d2..cafedca108 100644 --- a/test/extended/boot_image_update_azure.go +++ b/test/extended/boot_image_update_azure.go @@ -26,6 +26,9 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive PartialMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-partial.yaml") EmptyMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml") + AllControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml") + NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml") + oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin() ) @@ -63,6 +66,15 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive g.It("[Disruptive] Should update boot images on an Azure MachineSets with a legacy boot image and scale successfully [apigroup:machineconfiguration.openshift.io]", func() { AzureLegacyBootImageTest(oc, PartialMachineSetFixture) }) + + // This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down. + g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() { + AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture) + }) + + g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() { + NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture) + }) }) func AzureLegacyBootImageTest(oc *exutil.CLI, fixture string) { diff --git a/test/extended/boot_image_update_gcp.go b/test/extended/boot_image_update_gcp.go new file mode 100644 index 0000000000..6719f929b2 --- /dev/null +++ b/test/extended/boot_image_update_gcp.go @@ -0,0 +1,45 @@ +package extended + +import ( + "path/filepath" + + osconfigv1 "github.com/openshift/api/config/v1" + + g "github.com/onsi/ginkgo/v2" + exutil "github.com/openshift/machine-config-operator/test/extended/util" +) + +// These tests are [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test. +var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][OCPFeatureGate:ManagedBootImages]", g.Ordered, func() { + defer g.GinkgoRecover() + var ( + AllControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml") + NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml") + EmptyMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml") + + oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin() + ) + + g.BeforeEach(func() { + // Skip this test if not on GCP platform + skipUnlessTargetPlatform(oc, osconfigv1.GCPPlatformType) + // Skip this test if the cluster is not using MachineAPI + skipUnlessFunctionalMachineAPI(oc) + // Skip this test on single node platforms + skipOnSingleNodeTopology(oc) + }) + + g.AfterEach(func() { + // Clear out boot image configuration between tests + applyMachineConfigurationFixture(oc, EmptyMachineSetFixture) + }) + + // This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down. + g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() { + AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture) + }) + + g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() { + NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture) + }) +}) diff --git a/test/extended/const.go b/test/extended/const.go index bf72be6600..65e086a0cc 100644 --- a/test/extended/const.go +++ b/test/extended/const.go @@ -50,6 +50,9 @@ const ( // MAPIMachinesetQualifiedName is the fully qualified name of the MAPI MachineSet Resource MAPIMachinesetQualifiedName = "machinesets.machine.openshift.io" + // ControlPlaneMachinesetQualifiedName is the fully qualified name of the MAPI MachineSet Resource + ControlPlaneMachinesetQualifiedName = "controlplanemachinesets.machine.openshift.io" + // GoldenBootImagesConfigMap is the configmap that stores the bootimages refs of the current OCP release GoldenBootImagesConfigMap = "coreos-bootimages" diff --git a/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-all.yaml b/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-all.yaml new file mode 100644 index 0000000000..8c8226de7d --- /dev/null +++ b/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-all.yaml @@ -0,0 +1,19 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: managedbootimages-cpms-all +objects: +- apiVersion: operator.openshift.io/v1 + kind: MachineConfiguration + metadata: + name: cluster + namespace: openshift-machine-config-operator + spec: + logLevel: Normal + operatorLogLevel: Normal + managedBootImages: + machineManagers: + - resource: controlplanemachinesets + apiGroup: machine.openshift.io + selection: + mode: All diff --git a/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-none.yaml b/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-none.yaml new file mode 100644 index 0000000000..0fd3d3e246 --- /dev/null +++ b/test/extended/testdata/files/machineconfigurations/managedbootimages-cpms-none.yaml @@ -0,0 +1,19 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: managedbootimages-cpms-none +objects: +- apiVersion: operator.openshift.io/v1 + kind: MachineConfiguration + metadata: + name: cluster + namespace: openshift-machine-config-operator + spec: + logLevel: Normal + operatorLogLevel: Normal + managedBootImages: + machineManagers: + - resource: controlplanemachinesets + apiGroup: machine.openshift.io + selection: + mode: None