Merge pull request #29741 from RishabhSaini/vsphere

MCO-1656: Component Readiness for vSphere Bootimage Update

Author: openshift-merge-bot[bot]

test/extended/machine_config/boot_image_update_agnostic.go

@@ -2,6 +2,9 @@ package machine_config
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
@@ -14,6 +17,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+
+	streammeta "github.com/coreos/stream-metadata-go/stream"
 )
 
 func AllMachineSetTest(oc *exutil.CLI, fixture string) {
@@ -159,3 +164,164 @@ func EnsureConfigMapStampTest(oc *exutil.CLI, fixture string) {
 	}, 2*time.Minute, 5*time.Second).Should(o.BeTrue())
 	framework.Logf("Succesfully verified that the configmap has been correctly stamped")
 }
+
+func UploadTovCentreTest(oc *exutil.CLI, fixture string) {
+
+	// This fixture applies a boot image update configuration that opts in any machineset with the label test=boot
+	ApplyMachineConfigurationFixture(oc, fixture)
+
+	// Pick a random machineset to test
+	machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+	machineSetUnderTest := getRandomMachineSet(machineClient)
+	framework.Logf("MachineSet under test: %s", machineSetUnderTest.Name)
+
+	// Label this machineset with the test=boot label
+	err = oc.Run("label").Args(mapiMachinesetQualifiedName, machineSetUnderTest.Name, "-n", mapiNamespace, "test=boot").Execute()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	defer func() {
+		// Unlabel the machineset at the end of test
+		err = oc.Run("label").Args(mapiMachinesetQualifiedName, machineSetUnderTest.Name, "-n", mapiNamespace, "test-").Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+	}()
+
+	// Modify coreos-bootimage cm to an older known version (transient application since CVO should revert immediately)
+	cm, err := oc.AdminKubeClient().CoreV1().ConfigMaps(mcoNamespace).Get(context.TODO(), cmName, metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	var stream streammeta.Stream
+	err = json.Unmarshal([]byte(cm.Data["stream"]), &stream)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	vmware := stream.Architectures["x86_64"].Artifacts["vmware"]
+	currentVmwareRelease := vmware.Release
+	vmware.Release = "418.94.202501221327-0"
+	vmware.Formats["ova"].Disk.Location = "https://rhcos.mirror.openshift.com/art/storage/prod/streams/4.18-9.4/builds/418.94.202501221327-0/x86_64/rhcos-418.94.202501221327-0-vmware.x86_64.ova"
+	vmware.Formats["ova"].Disk.Sha256 = "6fd6e9fa2ff949154d54572c3f6a0c400f3e801457aa88b585b751a0955bda19"
+	stream.Architectures["x86_64"].Artifacts["vmware"] = vmware
+	updatedStreamBytes, err := json.MarshalIndent(stream, "", "  ")
+	o.Expect(err).NotTo(o.HaveOccurred())
+	escapedStreamBytes, err := json.Marshal(string(updatedStreamBytes))
+	o.Expect(err).NotTo(o.HaveOccurred())
+	patchPayload := fmt.Sprintf(`{"data":{"stream":%s}}`, string(escapedStreamBytes))
+	err = oc.Run("patch").Args("configmap", cmName, "-n", mcoNamespace, "-p", patchPayload).Execute()
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Ensure boot image controller is not progressing
+	framework.Logf("Waiting until the boot image controller is not progressing...")
+	WaitForBootImageControllerToComplete(oc)
+
+	// Ensure MSBIC moves successfully from current bootimage -> known old bootimage -> current bootimage
+	currentToOldLog := fmt.Sprintf("Existing RHCOS v%s does not match current RHCOS v%s. Starting reconciliation process.", currentVmwareRelease, vmware.Release)
+	oldToCurrentLog := fmt.Sprintf("Existing RHCOS v%s does not match current RHCOS v%s. Starting reconciliation process.", vmware.Release, currentVmwareRelease)
+	successfullyPatchedLog := fmt.Sprintf("Successfully patched machineset %s", machineSetUnderTest.Name)
+	o.Eventually(func() bool {
+		podNames, err := oc.Run("get").Args(
+			"pods",
+			"-n", "openshift-machine-config-operator",
+			"-l", "k8s-app=machine-config-controller",
+			"-o", "go-template={{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}",
+		).Output()
+		if err != nil {
+			return false
+		}
+		podNamesArr := strings.Split(podNames, "\n")
+		if len(podNamesArr) == 0 {
+			return false
+		}
+		mccPodName := podNamesArr[0]
+		logs, err := oc.Run("logs").Args(mccPodName, "-n", "openshift-machine-config-operator", "--tail=50").Output()
+		if err != nil {
+			return false
+		}
+		return checkOrder(logs, currentToOldLog, successfullyPatchedLog, oldToCurrentLog, successfullyPatchedLog)
+	}, 15*time.Minute, 10*time.Second).Should(o.BeTrue())
+
+	// Scale-up the machineset to verify we have not accidentally broken the bootimage updates
+	mcdPods, err := getMCDPodSet(oc)
+	err = oc.Run("scale").Args(mapiMachinesetQualifiedName, machineSetUnderTest.Name, "-n", mapiNamespace, fmt.Sprintf("--replicas=%d", *machineSetUnderTest.Spec.Replicas+1)).Execute()
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	defer func() {
+		// Scale-down the machineset at the end of test
+		err = oc.Run("scale").Args(mapiMachinesetQualifiedName, machineSetUnderTest.Name, "-n", mapiNamespace, fmt.Sprintf("--replicas=%d", *machineSetUnderTest.Spec.Replicas)).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+	}()
+
+	o.Eventually(func() bool {
+		machineset, err := machineClient.MachineV1beta1().MachineSets(mapiNamespace).Get(context.TODO(), machineSetUnderTest.Name, metav1.GetOptions{})
+		if err != nil {
+			return false
+		}
+		return machineset.Status.AvailableReplicas == *machineSetUnderTest.Spec.Replicas+1
+	}, 15*time.Minute, 10*time.Second).Should(o.BeTrue())
+
+	o.Eventually(func() bool {
+		updatedMcdPods, err := getMCDPodSet(oc)
+		if err != nil {
+			return false
+		}
+		newPods := diffNewPods(mcdPods, updatedMcdPods)
+		if len(newPods) == 0 {
+			return false
+		}
+		for _, pod := range newPods {
+			logs, err := oc.Run("logs").Args(pod, "-n", "openshift-machine-config-operator").Output()
+			if err != nil {
+				continue
+			}
+			lines := strings.Split(logs, "\n")
+			if len(lines) > 50 {
+				lines = lines[:50]
+			}
+			first50 := strings.Join(lines, "\n")
+			if strings.Contains(first50, currentVmwareRelease) {
+				return true
+			}
+		}
+		return false
+	}, 15*time.Minute, 10*time.Second).Should(o.BeTrue())
+}
+
+// checkOrder returns true if all statements appear in the log in the given order.
+func checkOrder(log string, statements ...string) bool {
+	if len(statements) == 0 {
+		return false
+	}
+	parts := make([]string, len(statements))
+	for i, s := range statements {
+		parts[i] = regexp.QuoteMeta(s) // escape regex metacharacters
+	}
+	pattern := "(?s)" + strings.Join(parts, ".*") // allow anything between statements
+	return regexp.MustCompile(pattern).MatchString(log)
+}
+
+func diffNewPods(before, after map[string]struct{}) []string {
+	var newPods []string
+	for pod := range after {
+		if _, found := before[pod]; !found {
+			newPods = append(newPods, pod)
+		}
+	}
+	return newPods
+}
+
+func getMCDPodSet(oc *exutil.CLI) (map[string]struct{}, error) {
+	out, err := oc.Run("get").Args(
+		"pods",
+		"-n", "openshift-machine-config-operator",
+		"-l", "k8s-app=machine-config-daemon",
+		"-o", "go-template={{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}",
+	).Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get pods: %v", err)
+	}
+
+	podSet := make(map[string]struct{})
+	lines := strings.Split(out, "\n")
+	for _, line := range lines {
+		name := strings.TrimSpace(line)
+		if name != "" {
+			podSet[name] = struct{}{}
+		}
+	}
+	return podSet, nil
+}

test/extended/machine_config/boot_image_update_vsphere.go

@@ -0,0 +1,61 @@
+package machine_config
+
+import (
+	"context"
+	"path/filepath"
+
+	osconfigv1 "github.com/openshift/api/config/v1"
+
+	g "github.com/onsi/ginkgo/v2"
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+// This test is [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test.
+var _ = g.Describe("[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]", func() {
+	defer g.GinkgoRecover()
+	var (
+		MCOMachineConfigurationBaseDir = exutil.FixturePath("testdata", "machine_config", "machineconfigurations")
+		partialMachineSetFixture       = filepath.Join(MCOMachineConfigurationBaseDir, "managedbootimages-partial.yaml")
+		allMachineSetFixture           = filepath.Join(MCOMachineConfigurationBaseDir, "managedbootimages-all.yaml")
+		noneMachineSetFixture          = filepath.Join(MCOMachineConfigurationBaseDir, "managedbootimages-none.yaml")
+		emptyMachineSetFixture         = filepath.Join(MCOMachineConfigurationBaseDir, "managedbootimages-empty.yaml")
+		oc                             = exutil.NewCLIWithoutNamespace("machine-config")
+	)
+
+	g.BeforeEach(func(ctx context.Context) {
+		//skip this test if not on vSphere platform
+		skipUnlessTargetPlatform(oc, osconfigv1.VSpherePlatformType)
+		//skip this test if the cluster is not using MachineAPI
+		skipUnlessFunctionalMachineAPI(oc)
+		//skip this test on single node platforms
+		skipOnSingleNodeTopology(oc)
+	})
+
+	g.AfterEach(func() {
+		ApplyMachineConfigurationFixture(oc, emptyMachineSetFixture)
+	})
+
+	g.It("Should update boot images only on MachineSets that are opted in [apigroup:machineconfiguration.openshift.io]", func() {
+		PartialMachineSetTest(oc, partialMachineSetFixture)
+	})
+
+	g.It("Should update boot images on all MachineSets when configured [apigroup:machineconfiguration.openshift.io]", func() {
+		AllMachineSetTest(oc, allMachineSetFixture)
+	})
+
+	g.It("Should not update boot images on any MachineSet when not configured [apigroup:machineconfiguration.openshift.io]", func() {
+		NoneMachineSetTest(oc, noneMachineSetFixture)
+	})
+
+	g.It("Should degrade on a MachineSet with an OwnerReference [apigroup:machineconfiguration.openshift.io]", func() {
+		DegradeOnOwnerRefTest(oc, allMachineSetFixture)
+	})
+
+	g.It("Should stamp coreos-bootimages configmap with current MCO hash and release version [apigroup:machineconfiguration.openshift.io]", func() {
+		EnsureConfigMapStampTest(oc, allMachineSetFixture)
+	})
+
+	g.It("Should upload the latest bootimage to the appropriate vCentre [apigroup:machineconfiguration.openshift.io]", func() {
+		UploadTovCentreTest(oc, partialMachineSetFixture)
+	})
+})

test/extended/machine_config/helpers.go

@@ -193,16 +193,20 @@ func verifyMachineSetUpdate(oc *exutil.CLI, machineSet machinev1beta1.MachineSet
 	framework.Logf("Waiting until the boot image controller is not progressing...")
 	WaitForBootImageControllerToComplete(oc)
 
-	// Fetch the providerSpec of the machineset under test again
-	providerSpecDisks, err := oc.Run("get").Args(mapiMachinesetQualifiedName, machineSet.Name, "-o", "template", "--template=`{{.spec.template.spec.providerSpec.value}}`", "-n", mapiNamespace).Output()
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Eventually(func() bool {
+		// Fetch the providerSpec of the machineset under test again
+		providerSpec, err := oc.Run("get").Args(mapiMachinesetQualifiedName, machineSet.Name, "-o", "template", "--template=`{{.spec.template.spec.providerSpec.value}}`", "-n", mapiNamespace).Output()
+		if err != nil {
+			return false
+		}
 
-	// Verify that the machineset has the expected boot image values
-	if updateExpected {
-		o.Expect(providerSpecDisks).ShouldNot(o.ContainSubstring(newBootImage))
-	} else {
-		o.Expect(providerSpecDisks).Should(o.ContainSubstring(newBootImage))
-	}
+		// Verify that the machineset has the expected boot image values
+		if updateExpected {
+			return !strings.Contains(providerSpec, newBootImage)
+		} else {
+			return strings.Contains(providerSpec, newBootImage)
+		}
+	}, 10*time.Minute, 10*time.Second).Should(o.BeTrue(), "Timed out verifying MachineSet '%v'", machineSet.Name)
 }
 
 // unmarshalProviderSpec unmarshals the machineset's provider spec into
@@ -241,6 +245,8 @@ func createFakeUpdatePatch(oc *exutil.CLI, machineSet machinev1beta1.MachineSet)
 		return generateAWSProviderSpecPatch(machineSet)
 	case osconfigv1.GCPPlatformType:
 		return generateGCPProviderSpecPatch(machineSet)
+	case osconfigv1.VSpherePlatformType:
+		return generateVSphereProviderSpecPatch(machineSet)
 	default:
 		exutil.FatalErr(fmt.Errorf("unexpected platform type; should not be here"))
 		return "", "", "", ""
@@ -293,6 +299,25 @@ func generateGCPProviderSpecPatch(machineSet machinev1beta1.MachineSet) (string,
 	return newProviderSpecPatch, originalProviderSpecPatch, newBootImage, originalBootImage
 }
 
+// generateVSphereProviderSpecPatch generates a fake update patch for the VSphere MachineSet
+func generateVSphereProviderSpecPatch(machineSet machinev1beta1.MachineSet) (string, string, string, string) {
+	providerSpec := new(machinev1beta1.VSphereMachineProviderSpec)
+	err := unmarshalProviderSpec(&machineSet, providerSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	// Modify the boot image to a "fake" value
+	originalBootImage := providerSpec.Template
+	newBootImage := "fake-update"
+	newProviderSpec := providerSpec.DeepCopy()
+	newProviderSpec.Template = newBootImage
+	newProviderSpecPatch, err := marshalProviderSpec(newProviderSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	originalProviderSpecPatch, err := marshalProviderSpec(providerSpec)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	return newProviderSpecPatch, originalProviderSpecPatch, newBootImage, originalBootImage
+}
+
 // WaitForBootImageControllerToComplete waits until the boot image controller is no longer progressing
 func WaitForBootImageControllerToComplete(oc *exutil.CLI) {
 	machineConfigurationClient, err := mcopclient.NewForConfig(oc.KubeFramework().ClientConfig())
@@ -306,7 +331,7 @@ func WaitForBootImageControllerToComplete(oc *exutil.CLI) {
 			return false
 		}
 		return IsMachineConfigurationConditionFalse(mcop.Status.Conditions, opv1.MachineConfigurationBootImageUpdateProgressing)
-	}, 3*time.Minute, 5*time.Second).MustPassRepeatedly(3).Should(o.BeTrue())
+	}, 5*time.Minute, 5*time.Second).MustPassRepeatedly(3).Should(o.BeTrue())
 }
 
 // WaitForMachineConfigurationStatus waits until the MCO syncs the operator status to the latest spec

test/extended/util/annotate/generated/zz_generated.annotations.go

@@ -698,6 +698,21 @@ spec:
         Should update boot images on all MachineSets when configured [apigroup:machineconfiguration.openshift.io]'
     - testName: '[Suite:openshift/machine-config-operator/disruptive][Suite:openshift/conformance/serial][sig-mco][OCPFeatureGate:ManagedBootImagesAWS][Serial]
         Should update boot images only on MachineSets that are opted in [apigroup:machineconfiguration.openshift.io]'
+  - featureGate: ManagedBootImagesvSphere
+    tests:
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should degrade on a MachineSet with an OwnerReference [apigroup:machineconfiguration.openshift.io]'
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should not update boot images on any MachineSet when not configured [apigroup:machineconfiguration.openshift.io]'
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should stamp coreos-bootimages configmap with current MCO hash and release
+        version [apigroup:machineconfiguration.openshift.io]'
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should update boot images on all MachineSets when configured [apigroup:machineconfiguration.openshift.io]'
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should update boot images only on MachineSets that are opted in [apigroup:machineconfiguration.openshift.io]'
+    - testName: '[Suite:openshift/machine-config-operator/disruptive][sig-mco][OCPFeatureGate:ManagedBootImagesvSphere][Serial]
+        Should upload the latest bootimage to the appropriate vCentre [apigroup:machineconfiguration.openshift.io]'
   - featureGate: MetricsCollectionProfiles
     tests:
     - testName: '[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfiles][Serial]