Skip to content

Commit d4bf670

Browse files
djoshydjoshy
committed
test/extended: add azure e2es
1 parent 21b888f commit d4bf670

File tree

11 files changed

+914
-0
lines changed

11 files changed

+914
-0
lines changed

‎test/extended/boot_image.go

Lines changed: 486 additions & 0 deletions
Large diffs are not rendered by default.

‎test/extended/boot_image_update_agnostic.go

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package extended
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
machineclient "github.com/openshift/client-go/machine/clientset/versioned"
8+
exutil "github.com/openshift/machine-config-operator/test/extended/util"
9+
10+
o "github.com/onsi/gomega"
11+
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/kubernetes/test/e2e/framework"
14+
)
15+
16+
func AllMachineSetTest(oc *exutil.CLI, fixture string) {
17+
// This fixture applies a boot image update configuration that opts in all machinesets
18+
applyMachineConfigurationFixture(oc, fixture)
19+
20+
// Step through all machinesets and verify boot images are reconciled correctly.
21+
machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
22+
o.Expect(err).NotTo(o.HaveOccurred())
23+
24+
machineSets, err := machineClient.MachineV1beta1().MachineSets("openshift-machine-api").List(context.TODO(), metav1.ListOptions{})
25+
o.Expect(err).NotTo(o.HaveOccurred())
26+
for _, ms := range machineSets.Items {
27+
verifyMachineSetUpdate(oc, ms, true)
28+
}
29+
}
30+
31+
func PartialMachineSetTest(oc *exutil.CLI, fixture string) {
32+
33+
// This fixture applies a boot image update configuration that opts in any machineset with the label test=boot
34+
applyMachineConfigurationFixture(oc, fixture)
35+
36+
// Pick a random machineset to test
37+
machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
38+
o.Expect(err).NotTo(o.HaveOccurred())
39+
machineSetUnderTest := getRandomMachineSet(machineClient)
40+
framework.Logf("MachineSet under test: %s", machineSetUnderTest.Name)
41+
42+
// Label this machineset with the test=boot label
43+
err = oc.Run("label").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, "test=boot").Execute()
44+
o.Expect(err).NotTo(o.HaveOccurred())
45+
defer func() {
46+
// Unlabel the machineset at the end of test
47+
err = oc.Run("label").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, "test-").Execute()
48+
o.Expect(err).NotTo(o.HaveOccurred())
49+
}()
50+
// Step through all machinesets and verify that only the opted in machineset's boot images are reconciled.
51+
machineSets, err := machineClient.MachineV1beta1().MachineSets("openshift-machine-api").List(context.TODO(), metav1.ListOptions{})
52+
o.Expect(err).NotTo(o.HaveOccurred())
53+
for _, ms := range machineSets.Items {
54+
verifyMachineSetUpdate(oc, ms, machineSetUnderTest.Name == ms.Name)
55+
}
56+
57+
}
58+
59+
func NoneMachineSetTest(oc *exutil.CLI, fixture string) {
60+
// This fixture applies a boot image update configuration that opts in no machinesets, i.e. feature is disabled.
61+
applyMachineConfigurationFixture(oc, fixture)
62+
63+
// Step through all machinesets and verify boot images are reconciled correctly.
64+
machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
65+
o.Expect(err).NotTo(o.HaveOccurred())
66+
67+
machineSets, err := machineClient.MachineV1beta1().MachineSets("openshift-machine-api").List(context.TODO(), metav1.ListOptions{})
68+
o.Expect(err).NotTo(o.HaveOccurred())
69+
for _, ms := range machineSets.Items {
70+
verifyMachineSetUpdate(oc, ms, false)
71+
}
72+
}
73+
74+
func EnsureConfigMapStampTest(oc *exutil.CLI) {
75+
// Update boot image configmap stamps with a "fake" value, wait for it to be updated back by the operator.
76+
err := oc.Run("patch").Args("configmap", GoldenBootImagesConfigMap, "-p", `{"data": {"MCOVersionHash": "fake-value"}}`, "-n", MachineConfigNamespace).Execute()
77+
o.Expect(err).NotTo(o.HaveOccurred())
78+
79+
err = oc.Run("patch").Args("configmap", GoldenBootImagesConfigMap, "-p", `{"data": {"MCOReleaseImageVersion": "fake-value"}}`, "-n", MachineConfigNamespace).Execute()
80+
o.Expect(err).NotTo(o.HaveOccurred())
81+
82+
// Ensure atleast one master node is ready
83+
waitForOneMasterNodeToBeReady(oc)
84+
85+
// Verify that the configmap has been updated back to the correct value
86+
o.Eventually(func() bool {
87+
cm, err := oc.AdminKubeClient().CoreV1().ConfigMaps(MachineConfigNamespace).Get(context.TODO(), GoldenBootImagesConfigMap, metav1.GetOptions{})
88+
if err != nil {
89+
framework.Logf("failed to grab configmap, error :%v", err)
90+
return false
91+
}
92+
if cm.Data["MCOVersionHash"] == "fake-value" {
93+
framework.Logf("MCOVersionHash has not been restored to the original value")
94+
return false
95+
}
96+
if cm.Data["MCOReleaseImageVersion"] == "fake-value" {
97+
framework.Logf("MCOReleaseImageVersion has not been restored to the original value")
98+
return false
99+
}
100+
return true
101+
}, 2*time.Minute, 5*time.Second).Should(o.BeTrue())
102+
framework.Logf("Successfully verified that the configmap has been correctly stamped")
103+
}

‎test/extended/boot_image_update_azure.go

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package extended
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"path/filepath"
7+
"time"
8+
9+
osconfigv1 "github.com/openshift/api/config/v1"
10+
machineclient "github.com/openshift/client-go/machine/clientset/versioned"
11+
12+
g "github.com/onsi/ginkgo/v2"
13+
o "github.com/onsi/gomega"
14+
exutil "github.com/openshift/machine-config-operator/test/extended/util"
15+
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/kubernetes/test/e2e/framework"
18+
)
19+
20+
// This test is [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test.
21+
var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][Disruptive][OCPFeatureGate:ManagedBootImagesAzure]", g.Ordered, func() {
22+
defer g.GinkgoRecover()
23+
var (
24+
AllMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-all.yaml")
25+
NoneMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-none.yaml")
26+
PartialMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-partial.yaml")
27+
EmptyMachineSetFixture = filepath.Join("machineconfigurations", "managedbootimages-empty.yaml")
28+
29+
oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin()
30+
)
31+
32+
g.BeforeEach(func() {
33+
// Skip this test if not on Azure platform
34+
skipUnlessTargetPlatform(oc, osconfigv1.AzurePlatformType)
35+
// Skip this test if the cluster is not using MachineAPI
36+
skipUnlessFunctionalMachineAPI(oc)
37+
// Skip this test on single node platforms
38+
skipOnSingleNodeTopology(oc)
39+
})
40+
41+
g.AfterEach(func() {
42+
// Clear out boot image configuration between tests
43+
applyMachineConfigurationFixture(oc, EmptyMachineSetFixture)
44+
})
45+
46+
g.It("Should update boot images only on MachineSets that are opted in [apigroup:machineconfiguration.openshift.io]", func() {
47+
PartialMachineSetTest(oc, PartialMachineSetFixture)
48+
})
49+
50+
g.It("Should update boot images on all MachineSets when configured [apigroup:machineconfiguration.openshift.io]", func() {
51+
AllMachineSetTest(oc, AllMachineSetFixture)
52+
})
53+
54+
g.It("Should not update boot images on any MachineSet when not configured [apigroup:machineconfiguration.openshift.io]", func() {
55+
NoneMachineSetTest(oc, NoneMachineSetFixture)
56+
})
57+
58+
g.It("Should stamp coreos-bootimages configmap with current MCO hash and release version [apigroup:machineconfiguration.openshift.io]", func() {
59+
EnsureConfigMapStampTest(oc)
60+
})
61+
62+
g.It("Should update boot images on an Azure MachineSets with a legacy boot image and scale successfully [apigroup:machineconfiguration.openshift.io]", func() {
63+
AzureLegacyBootImageTest(oc, PartialMachineSetFixture)
64+
})
65+
})
66+
67+
func AzureLegacyBootImageTest(oc *exutil.CLI, fixture string) {
68+
69+
// This fixture applies a boot image update configuration that opts in any machineset with the label test=boot
70+
applyMachineConfigurationFixture(oc, fixture)
71+
72+
// Pick a random machineset to test
73+
machineClient, err := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
74+
o.Expect(err).NotTo(o.HaveOccurred())
75+
machineSetUnderTest := getRandomMachineSet(machineClient)
76+
framework.Logf("MachineSet under test: %s", machineSetUnderTest.Name)
77+
78+
// Label this machineset with the test=boot label
79+
err = oc.Run("label").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, "test=boot").Execute()
80+
o.Expect(err).NotTo(o.HaveOccurred())
81+
defer func() {
82+
// Unlabel the machineset at the end of test
83+
err = oc.Run("label").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, "test-").Execute()
84+
o.Expect(err).NotTo(o.HaveOccurred())
85+
}()
86+
87+
// Set machineset under test to a legacy boot image
88+
newProviderSpecPatch, originalProviderSpecPatch, legacyBootImage, originalBootImage := generateLegacyAzureProviderSpecPatch(machineSetUnderTest)
89+
err = oc.Run("patch").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-p", newProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute()
90+
o.Expect(err).NotTo(o.HaveOccurred())
91+
92+
defer func() {
93+
// Restore machineSet to original boot image as the machineset may be used by other test variants, regardless of success/fail
94+
err = oc.Run("patch").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-p", originalProviderSpecPatch, "-n", MAPINamespace, "--type=json").Execute()
95+
o.Expect(err).NotTo(o.HaveOccurred())
96+
framework.Logf("Restored build name in the machineset %s to \"%s\"", machineSetUnderTest.Name, originalBootImage)
97+
}()
98+
// Ensure boot image controller is not progressing
99+
framework.Logf("Waiting until the boot image controller is not progressing...")
100+
waitForBootImageControllerToComplete(oc)
101+
102+
// Fetch the providerSpec of the machineset under test again
103+
providerSpec, err := oc.Run("get").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-o", "template", "--template=`{{.spec.template.spec.providerSpec.value}}`", "-n", MAPINamespace).Output()
104+
o.Expect(err).NotTo(o.HaveOccurred())
105+
106+
// Verify that the machineset does not have the legacy boot image
107+
o.Expect(providerSpec).ShouldNot(o.ContainSubstring(legacyBootImage))
108+
109+
// Get current set of ready nodes
110+
nodes, err := getReadyNodes(oc)
111+
o.Expect(err).NotTo(o.HaveOccurred())
112+
113+
// Scale up machineset under test
114+
err = oc.Run("scale").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, fmt.Sprintf("--replicas=%d", *machineSetUnderTest.Spec.Replicas+1)).Execute()
115+
o.Expect(err).NotTo(o.HaveOccurred())
116+
117+
framework.Logf("Waiting for scale-up to complete...")
118+
// Scale up a new node
119+
o.Eventually(func() bool {
120+
machineset, err := machineClient.MachineV1beta1().MachineSets(MAPINamespace).Get(context.TODO(), machineSetUnderTest.Name, metav1.GetOptions{})
121+
if err != nil {
122+
framework.Logf("%v", err)
123+
return false
124+
}
125+
return machineset.Status.AvailableReplicas == *machineSetUnderTest.Spec.Replicas+1
126+
}, 15*time.Minute, 10*time.Second).Should(o.BeTrue())
127+
128+
defer func() {
129+
// Scale-down the machineset at the end of test
130+
err = oc.Run("scale").Args(MAPIMachinesetQualifiedName, machineSetUnderTest.Name, "-n", MAPINamespace, fmt.Sprintf("--replicas=%d", *machineSetUnderTest.Spec.Replicas)).Execute()
131+
o.Expect(err).NotTo(o.HaveOccurred())
132+
133+
// Wait for scaledown to complete
134+
framework.Logf("Waiting for scale-down to complete...")
135+
o.Eventually(func() bool {
136+
machineset, err := machineClient.MachineV1beta1().MachineSets(MAPINamespace).Get(context.TODO(), machineSetUnderTest.Name, metav1.GetOptions{})
137+
if err != nil {
138+
framework.Logf("%v", err)
139+
return false
140+
}
141+
return machineset.Status.AvailableReplicas == *machineSetUnderTest.Spec.Replicas
142+
}, 15*time.Minute, 10*time.Second).Should(o.BeTrue())
143+
144+
}()
145+
146+
// Retrieve aleph version from new node
147+
var alephVersion string
148+
o.Eventually(func() bool {
149+
// Grab newly scaled up node by diffing against the old set of nodes
150+
newNodes, err := getReadyNodes(oc)
151+
if err != nil {
152+
return false
153+
}
154+
scaledUpNode := newNodes.Difference(nodes)
155+
scaledUpNodeName, scaledUpNodeReady := scaledUpNode.PopAny()
156+
if !scaledUpNodeReady {
157+
return false
158+
}
159+
160+
// Log aleph version from the new node
161+
framework.Logf("Newly scaled up node: %v", scaledUpNodeName)
162+
alephVersion, err = getAlephVersionFromNode(oc, scaledUpNodeName)
163+
if err != nil {
164+
framework.Logf("Failed to get aleph version from node %s: %v", scaledUpNodeName, err)
165+
return false
166+
}
167+
168+
framework.Logf("CoreOS aleph version from node %s: %s", scaledUpNodeName, alephVersion)
169+
return true
170+
}, 3*time.Minute, 3*time.Second).Should(o.BeTrue())
171+
172+
// Get the current release boot image for this architecture; this should match the
173+
// aleph version above
174+
arch, err := getArchFromMachineSet(&machineSetUnderTest)
175+
o.Expect(err).NotTo(o.HaveOccurred())
176+
177+
releaseBootImageVersion, err := getReleaseBootImageVersion(oc, arch)
178+
o.Expect(err).NotTo(o.HaveOccurred())
179+
framework.Logf("Current release's boot image version: %s", releaseBootImageVersion)
180+
181+
// TODO: Uncomment when https://issues.redhat.com/browse/CORS-3915 lands, so that drifts between
182+
// rhcos.json & rhcos-marketplace.json are resolved.
183+
// For now, a successful scaleup can be considered as a success.
184+
// o.Expect(alephVersion).To(o.Equal(releaseBootImageVersion))
185+
}

‎test/extended/const.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,21 @@ const (
4343
TrueString = "True"
4444
// FalseString string for false value
4545
FalseString = "False"
46+
47+
// MAPINamespace is the MachineAPI namespace
48+
MAPINamespace = "openshift-machine-api"
49+
50+
// MAPIMachinesetQualifiedName is the fully qualified name of the MAPI MachineSet Resource
51+
MAPIMachinesetQualifiedName = "machinesets.machine.openshift.io"
52+
53+
// GoldenBootImagesConfigMap is the configmap that stores the bootimages refs of the current OCP release
54+
GoldenBootImagesConfigMap = "coreos-bootimages"
55+
56+
// MAPIMasterMachineLabelSelector is the label used to select the control-plane nodes
57+
MAPIMasterMachineLabelSelector = "machine.openshift.io/cluster-api-machine-role=master"
58+
59+
// Labels and Annotations required for determining architecture of a machineset
60+
MachineSetArchAnnotationKey = "capacity.cluster-autoscaler.kubernetes.io/labels"
61+
62+
ArchLabelKey = "kubernetes.io/arch="
4663
)

‎test/extended/node.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010
exutil "github.com/openshift/machine-config-operator/test/extended/util"
1111
logger "github.com/openshift/machine-config-operator/test/extended/util/logext"
1212

13+
"k8s.io/apimachinery/pkg/util/sets"
14+
1315
o "github.com/onsi/gomega"
1416
)
1517

@@ -425,3 +427,23 @@ func GetOperatorNode(oc *exutil.CLI) (*Node, error) {
425427

426428
return NewNode(oc, nodeName), nil
427429
}
430+
431+
// Returns the set of ready nodes in the cluster
432+
func getReadyNodes(oc *exutil.CLI) (sets.Set[string], error) {
433+
nodeList := NewResourceList(oc.AsAdmin(), "nodes")
434+
nodes, err := nodeList.GetAll()
435+
if err != nil {
436+
return nil, err
437+
}
438+
439+
nodeSet := sets.New[string]()
440+
for _, node := range nodes {
441+
node.oc.NotShowInfo()
442+
isReady, err := node.Get(`{.status.conditions[?(@.type=="Ready")].status}`)
443+
if err == nil && isReady == TrueString {
444+
nodeSet.Insert(node.name)
445+
}
446+
node.oc.SetShowInfo()
447+
}
448+
return nodeSet, nil
449+
}

‎test/extended/resource.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,21 @@ func (t *Template) Create(parameters ...string) error {
409409
return exutil.CreateClusterResourceFromTemplateWithError(t.oc, allParams...)
410410
}
411411

412+
// Apply the resources defined in the template file
413+
// The template will be created using oc with no namespace (-n NAMESPACE) argument. So if we want to
414+
// create a namespaced resource we need to add the NAMESPACE parameter to the template and
415+
// provide the "-p NAMESPACE" argument to this function.
416+
func (t *Template) Apply(parameters ...string) error {
417+
if t.templateFile == "" {
418+
return fmt.Errorf("There is no template configured")
419+
}
420+
421+
allParams := []string{"--ignore-unknown-parameters=true", "-f", t.templateFile}
422+
allParams = append(allParams, parameters...)
423+
424+
return exutil.ApplyClusterResourceFromTemplateWithError(t.oc, allParams...)
425+
}
426+
412427
// ResourceList provides the functionality to handle lists of openshift resources
413428
type ResourceList struct {
414429
ocGetter

‎test/extended/testdata/files/machineconfigurations/managedbootimages-all.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: template.openshift.io/v1
2+
kind: Template
3+
metadata:
4+
name: managedbootimages-none
5+
objects:
6+
- apiVersion: operator.openshift.io/v1
7+
kind: MachineConfiguration
8+
metadata:
9+
name: cluster
10+
namespace: openshift-machine-config-operator
11+
spec:
12+
logLevel: Normal
13+
operatorLogLevel: Normal
14+
managedBootImages:
15+
machineManagers:
16+
- resource: machinesets
17+
apiGroup: machine.openshift.io
18+
selection:
19+
mode: All

0 commit comments

Comments
 (0)