MCO-1930: Add Component Readiness tests for CPMS boot image updates

test/extended/boot_image.go

@@ -104,6 +104,7 @@ func getRandomMachineSet(machineClient *machineclient.Clientset) machinev1beta1.
 }
 
 // verifyMachineSetUpdate verifies that the the boot image values of a MachineSet are reconciled correctly
+// nolint:dupl // I separated these from verifyControlPlaneMachineSetUpdate for readability
 func verifyMachineSetUpdate(oc *exutil.CLI, machineSet machinev1beta1.MachineSet, updateExpected bool) {
 
 	newProviderSpecPatch, originalProviderSpecPatch, backdatedBootImage, originalBootImage := createFakeUpdatePatch(oc, machineSet)

test/extended/boot_image_cpms.go

@@ -0,0 +1,136 @@
+package extended
+
+import (
+	"context"
+	"fmt"
+
+	osconfigv1 "github.com/openshift/api/config/v1"
+	"sigs.k8s.io/yaml"
+
+	machinev1 "github.com/openshift/api/machine/v1"
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+	exutil "github.com/openshift/machine-config-operator/test/extended/util"
+
+	o "github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+)
+
+// verifyControlPlaneMachineSetUpdate verifies that the the boot image values of a ControlPlaneMachineSet are reconciled correctly
+// nolint:dupl // I separated these from verifyMachineSetUpdate for readability
+func verifyControlPlaneMachineSetUpdate(oc *exutil.CLI, cpms machinev1.ControlPlaneMachineSet, updateExpected bool) {
+
+	newProviderSpecPatch, originalProviderSpecPatch, backdatedBootImage, originalBootImage := createFakeUpdatePatchCPMS(oc, cpms)
+	err := oc.Run("patch").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-p", newProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	defer func() {
+		// Restore machineSet to original boot image as the machineset may be used by other test variants, regardless of success/fail
+		err = oc.Run("patch").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-p", originalProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		e2e.Logf("Restored build name in the machineset %s to \"%s\"", cpms.Name, originalBootImage)
+	}()
+	// Ensure boot image controller is not progressing
+	e2e.Logf("Waiting until the boot image controller is not progressing...")
+	waitForBootImageControllerToComplete(oc)
+
+	// Fetch the providerSpec of the machineset under test again
+	providerSpec, err := oc.Run("get").Args(ControlPlaneMachinesetQualifiedName, cpms.Name, "-o", "template", "--template=`{{.spec.template.machines_v1beta1_machine_openshift_io.spec.providerSpec.value}}`", "-n", MAPINamespace).Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Verify that the machineset has the expected boot image values
+	// If an update is expected, the backdated boot image should not be present
+	// If an update is NOT expected, the backdated boot image should still be present; ie machineset is left untouched
+	if updateExpected {
+		o.Expect(providerSpec).ShouldNot(o.ContainSubstring(backdatedBootImage))
+	} else {
+		o.Expect(providerSpec).Should(o.ContainSubstring(backdatedBootImage))
+	}
+}
+
+// createFakeUpdatePatchCPMS creates an update patch for the ControlPlaneMachineSet object based on the platform
+func createFakeUpdatePatchCPMS(oc *exutil.CLI, cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) {
+	infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	switch infra.Status.PlatformStatus.Type {
+	case osconfigv1.AWSPlatformType:
+		return generateAWSProviderSpecPatchCPMS(cpms)
+	case osconfigv1.GCPPlatformType:
+		return generateGCPProviderSpecPatchCPMS(cpms)
+	case osconfigv1.AzurePlatformType:
+		return generateAzureProviderSpecPatchCPMS(cpms)
+	default:
+		e2e.Failf("unexpected platform type; should not be here")
+		return "", "", "", ""
+	}
+}
+
+// generateAWSProviderSpecPatchCPMS generates a fake update patch for the AWS ControlPlaneMachineSet
+func generateAWSProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) {
+	providerSpec := new(machinev1beta1.AWSMachineProviderConfig)
+	err := unmarshalProviderSpecCPMS(&cpms, providerSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Modify the boot image to an older known AMI value
+	// See: https://issues.redhat.com/browse/OCPBUGS-57426
+	originalBootImage := *providerSpec.AMI.ID
+	newBootImage := "ami-000145e5a91e9ac22"
+	jsonPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/ami/id", "value": "%s"}]`, newBootImage)
+
+	// Create JSON patch to restore original AMI ID
+	originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/ami/id", "value": "%s"}]`, originalBootImage)
+
+	return jsonPatch, originalJSONPatch, newBootImage, originalBootImage
+
+}
+
+// generateGCPProviderSpecPatchCPMS generates a fake update patch for the GCP ControlPlaneMachineSet
+func generateGCPProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) {
+	providerSpec := new(machinev1beta1.GCPMachineProviderSpec)
+	err := unmarshalProviderSpecCPMS(&cpms, providerSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Modify the boot image to a older known value.
+	// See: https://issues.redhat.com/browse/OCPBUGS-57426
+	originalBootImage := providerSpec.Disks[0].Image
+	newBootImage := "projects/rhcos-cloud/global/images/rhcos-410-84-202210040010-0-gcp-x86-64"
+	jsonPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/disks/0/image", "value": "%s"}]`, newBootImage)
+
+	// Create JSON patch to restore original disk image
+	originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/disks/0/image", "value": "%s"}]`, originalBootImage)
+
+	return jsonPatch, originalJSONPatch, newBootImage, originalBootImage
+}
+
+// generateAzureProviderSpecPatchCPMS generates a fake update patch for the Azure ControlPlaneMachineSet
+func generateAzureProviderSpecPatchCPMS(cpms machinev1.ControlPlaneMachineSet) (string, string, string, string) {
+	providerSpec := new(machinev1beta1.AzureMachineProviderSpec)
+	err := unmarshalProviderSpecCPMS(&cpms, providerSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Use JSON patch to precisely replace just the image field with marketplace image
+	// This avoids any merge conflicts with existing fields
+	// Use an older known 4.18 boot image that is available in the marketplace
+	jsonPatch := `[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/image", "value": {"offer": "aro4", "publisher": "azureopenshift", "resourceID": "", "sku": "418-v2", "version": "418.94.20250122", "type": "MarketplaceNoPlan"}}]`
+
+	// Create JSON patch to restore original image
+	originalImage := providerSpec.Image
+	originalJSONPatch := fmt.Sprintf(`[{"op": "replace", "path": "/spec/template/machines_v1beta1_machine_openshift_io/spec/providerSpec/value/image", "value": {"offer": "%s", "publisher": "%s", "resourceID": "%s", "sku": "%s", "version": "%s", "type": "%s"}}]`,
+		originalImage.Offer, originalImage.Publisher, originalImage.ResourceID, originalImage.SKU, originalImage.Version, originalImage.Type)
+
+	return jsonPatch, originalJSONPatch, "418.94.20250122", providerSpec.Image.Version
+}
+
+// unmarshalProviderSpecCPMS unmarshals the controlplanemachineset's provider spec into
+// a ProviderSpec object. Returns an error if providerSpec field is nil,
+// or the unmarshal fails
+func unmarshalProviderSpecCPMS(cpms *machinev1.ControlPlaneMachineSet, providerSpec interface{}) error {
+	if cpms.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value == nil {
+		return fmt.Errorf("providerSpec field was empty")
+	}
+	if err := yaml.Unmarshal(cpms.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Raw, &providerSpec); err != nil {
+		return fmt.Errorf("unmarshal into providerSpec failed %w", err)
+	}
+	return nil
+}

test/extended/boot_image_update_agnostic.go

@@ -10,7 +10,10 @@ import (
 	o "github.com/onsi/gomega"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/test/e2e/framework"
+
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
 )
 
 func AllMachineSetTest(oc *exutil.CLI, fixture string) {
@@ -101,3 +104,117 @@ func EnsureConfigMapStampTest(oc *exutil.CLI) {
 	}, 2*time.Minute, 5*time.Second).Should(o.BeTrue())
 	framework.Logf("Successfully verified that the configmap has been correctly stamped")
 }
+
+func AllControlPlaneMachineSetTest(oc *exutil.CLI, fixture string) {
+	// This fixture applies a boot image update configuration that opts in all controlplanemachinesets
+	// However, since CPMS is typically a singleton, it is just targeting a single resource
+	applyMachineConfigurationFixture(oc, fixture)
+
+	// Grab the CPMS and verify that the boot image was reconciled correctly.
+	machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	cpms, err := machineClient.MachineV1().ControlPlaneMachineSets("openshift-machine-api").Get(context.TODO(), "cluster", metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	verifyControlPlaneMachineSetUpdate(oc, *cpms, true)
+
+	// Delete a control plane machine to verify that CPMS reconciles it with the updated boot image
+	// Get the list of control plane machines
+	machines, err := machineClient.MachineV1beta1().Machines(MAPINamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: MAPIMasterMachineLabelSelector})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(machines.Items).NotTo(o.BeEmpty(), "No control plane machines found")
+
+	// Capture the initial set of control plane machine names and count before deletion
+	initialMachineNames := sets.New[string]()
+	for _, machine := range machines.Items {
+		initialMachineNames.Insert(machine.Name)
+	}
+	initialMachineCount := initialMachineNames.Len()
+
+	// Delete the first control plane machine
+	machineToDelete := machines.Items[0].Name
+	framework.Logf("Deleting control plane machine: %s", machineToDelete)
+	err = machineClient.MachineV1beta1().Machines(MAPINamespace).Delete(context.TODO(), machineToDelete, metav1.DeleteOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Wait until the new control plane machine is running and the old one is deleted
+	// Arbitrarily picking 25 minutes timeout as scale-up time varies based on platform
+	framework.Logf("Waiting for CPMS to reconcile and create a new control plane machine (up to 25 minutes)...")
+	o.Eventually(func() bool {
+		currentMachines, err := machineClient.MachineV1beta1().Machines(MAPINamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: MAPIMasterMachineLabelSelector})
+		if err != nil {
+			framework.Logf("Error listing machines: %v", err)
+			return false
+		}
+
+		// Check that the deleted control plane machine is gone and all current machines are running
+		currentMachineNames := sets.New[string]()
+		runningMachines := sets.New[string]()
+
+		for _, machine := range currentMachines.Items {
+			currentMachineNames.Insert(machine.Name)
+			phase := ""
+			if machine.Status.Phase != nil {
+				phase = *machine.Status.Phase
+			}
+			if phase == machinev1beta1.PhaseRunning {
+				runningMachines.Insert(machine.Name)
+			} else {
+				framework.Logf("Machine %s is in phase: %s", machine.Name, phase)
+			}
+		}
+
+		// All machines must be running
+		if runningMachines.Len() != initialMachineCount {
+			framework.Logf("Only %d out of %d machines are running", runningMachines.Len(), initialMachineCount)
+			return false
+		}
+
+		// The deleted machine should not be in the current set
+		if currentMachineNames.Has(machineToDelete) {
+			framework.Logf("Deleted machine %s still exists", machineToDelete)
+			return false
+		}
+
+		framework.Logf("All %d control plane machines are running and the deleted machine is gone", initialMachineCount)
+
+		// Ensure master MCP is done updating and has the correct ready count
+		masterMCP := NewMachineConfigPool(oc, MachineConfigPoolMaster)
+		updatedStatus, err := masterMCP.GetUpdatedStatus()
+		if err != nil {
+			framework.Logf("Error getting master MCP updated status: %v", err)
+			return false
+		}
+		if updatedStatus != TrueString {
+			framework.Logf("Master MCP is not yet updated (Updated=%s)", updatedStatus)
+			return false
+		}
+
+		readyMachineCount, err := masterMCP.getUpdatedMachineCount()
+		if err != nil {
+			framework.Logf("Error getting master MCP ready machine count: %v", err)
+			return false
+		}
+		if readyMachineCount != initialMachineCount {
+			framework.Logf("Master MCP ready machine count %d does not match initial count %d", readyMachineCount, initialMachineCount)
+			return false
+		}
+
+		framework.Logf("Master MCP is updated with %d ready machines", readyMachineCount)
+		return true
+	}, 25*time.Minute, 2*time.Minute).Should(o.BeTrue(), "CPMS failed to reconcile control plane machines within 25 minutes")
+}
+
+func NoneControlPlaneMachineSetTest(oc *exutil.CLI, fixture string) {
+	// This fixture applies a boot image update configuration that opts in no controlplanemachineset, i.e. feature is disabled.
+	applyMachineConfigurationFixture(oc, fixture)
+
+	// Grab the CPMS and verify that the boot image was reconciled correctly.
+	machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	cpms, err := machineClient.MachineV1().ControlPlaneMachineSets("openshift-machine-api").Get(context.TODO(), "cluster", metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	verifyControlPlaneMachineSetUpdate(oc, *cpms, false)
+}

test/extended/boot_image_update_aws.go

@@ -0,0 +1,45 @@
+package extended
+
+import (
+	"path/filepath"
+
+	osconfigv1 "github.com/openshift/api/config/v1"
+
+	g "github.com/onsi/ginkgo/v2"
+	exutil "github.com/openshift/machine-config-operator/test/extended/util"
+)
+
+// These tests are [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test.
+var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][OCPFeatureGate:ManagedBootImagesAWS]", g.Ordered, func() {
+	defer g.GinkgoRecover()
+	var (
+		AllControlPlaneMachineSetFixture  = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml")
+		NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml")
+		EmptyMachineSetFixture            = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml")
+
+		oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin()
+	)
+
+	g.BeforeEach(func() {
+		// Skip this test if not on AWS platform
+		skipUnlessTargetPlatform(oc, osconfigv1.AWSPlatformType)
+		// Skip this test if the cluster is not using MachineAPI
+		skipUnlessFunctionalMachineAPI(oc)
+		// Skip this test on single node platforms
+		skipOnSingleNodeTopology(oc)
+	})
+
+	g.AfterEach(func() {
+		// Clear out boot image configuration between tests
+		applyMachineConfigurationFixture(oc, EmptyMachineSetFixture)
+	})
+
+	// This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down.
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() {
+		AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture)
+	})
+
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() {
+		NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture)
+	})
+})

test/extended/boot_image_update_azure.go

@@ -26,6 +26,9 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive
 		PartialMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-partial.yaml")
 		EmptyMachineSetFixture   = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml")
 
+		AllControlPlaneMachineSetFixture  = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml")
+		NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml")
+
 		oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin()
 	)
 
@@ -63,6 +66,15 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive
 	g.It("[Disruptive] Should update boot images on an Azure MachineSets with a legacy boot image and scale successfully [apigroup:machineconfiguration.openshift.io]", func() {
 		AzureLegacyBootImageTest(oc, PartialMachineSetFixture)
 	})
+
+	// This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down.
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() {
+		AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture)
+	})
+
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() {
+		NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture)
+	})
 })
 
 func AzureLegacyBootImageTest(oc *exutil.CLI, fixture string) {

test/extended/boot_image_update_gcp.go

@@ -0,0 +1,45 @@
+package extended
+
+import (
+	"path/filepath"
+
+	osconfigv1 "github.com/openshift/api/config/v1"
+
+	g "github.com/onsi/ginkgo/v2"
+	exutil "github.com/openshift/machine-config-operator/test/extended/util"
+)
+
+// These tests are [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test.
+var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][OCPFeatureGate:ManagedBootImages]", g.Ordered, func() {
+	defer g.GinkgoRecover()
+	var (
+		AllControlPlaneMachineSetFixture  = filepath.Join("machineconfigurations", "managedbootimages-cpms-all.yaml")
+		NoneControlPlaneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-cpms-none.yaml")
+		EmptyMachineSetFixture            = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml")
+
+		oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin()
+	)
+
+	g.BeforeEach(func() {
+		// Skip this test if not on GCP platform
+		skipUnlessTargetPlatform(oc, osconfigv1.GCPPlatformType)
+		// Skip this test if the cluster is not using MachineAPI
+		skipUnlessFunctionalMachineAPI(oc)
+		// Skip this test on single node platforms
+		skipOnSingleNodeTopology(oc)
+	})
+
+	g.AfterEach(func() {
+		// Clear out boot image configuration between tests
+		applyMachineConfigurationFixture(oc, EmptyMachineSetFixture)
+	})
+
+	// This test is [Disruptive] because it scales up a new control plane node after performing a boot image update, and the scales it down.
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS][Disruptive] Should update boot images on ControlPlaneMachineSets and resize properly [apigroup:machineconfiguration.openshift.io]", func() {
+		AllControlPlaneMachineSetTest(oc, AllControlPlaneMachineSetFixture)
+	})
+
+	g.It("[OCPFeatureGate:ManagedBootImagesCPMS] Should not update boot images on ControlPlaneMachineSets when not configured [apigroup:machineconfiguration.openshift.io]", func() {
+		NoneControlPlaneMachineSetTest(oc, NoneControlPlaneMachineSetFixture)
+	})
+})