Skip to content

Commit 8917abd

Browse files
BhargaviGudiBhargaviGudi
committed
Add automated tests for non-CNV swap configuration
Updated testcase name and separated upgrade cases Updated testcase Addressed review comments Skip testcases on microsoft cluster Added polarion test case IDs Updated addressed comments from the pr Verify KubeletConfig status shows the expected error resolved rebase issue and addressed review comments
1 parent 6f59a1f commit 8917abd

File tree

2 files changed

+240
-0
lines changed

2 files changed

+240
-0
lines changed

‎test/extended/node/node_swap.go‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
g "github.com/onsi/ginkgo/v2"
9+
o "github.com/onsi/gomega"
10+
11+
corev1 "k8s.io/api/core/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/runtime"
14+
"k8s.io/apimachinery/pkg/util/wait"
15+
"k8s.io/kubernetes/test/e2e/framework"
16+
17+
machineconfigv1 "github.com/openshift/api/machineconfiguration/v1"
18+
mcclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
19+
exutil "github.com/openshift/origin/test/extended/util"
20+
)
21+
22+
const (
23+
workerGeneratedKubeletMC = "99-worker-generated-kubelet"
24+
)
25+
26+
var _ = g.Describe("[Jira:Node][sig-node] Node non-cnv swap configuration", func() {
27+
defer g.GinkgoRecover()
28+
29+
var oc = exutil.NewCLI("node-swap")
30+
31+
g.BeforeEach(func(ctx context.Context) {
32+
// Skip all tests on MicroShift clusters
33+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
34+
o.Expect(err).NotTo(o.HaveOccurred())
35+
if isMicroShift {
36+
g.Skip("Skipping test on MicroShift cluster")
37+
}
38+
})
39+
40+
// This test validates that:
41+
// - Worker nodes have failSwapOn=false to allow kubelet to start even if swap is present at OS level
42+
// - Control plane nodes have failSwapOn=true to prevent kubelet from starting if swap is enabled
43+
// - All nodes have swapBehavior=NoSwap to ensure kubelet does not utilize swap even if available at OS level
44+
// The swapBehavior=NoSwap configuration ensures that even if swap is manually enabled on a worker node,
45+
// the kubelet will not use it for memory management, maintaining consistent behavior across the cluster.
46+
g.It("should have correct default kubelet swap settings with worker nodes failSwapOn=false, control plane nodes failSwapOn=true, and both swapBehavior=NoSwap [OCP-86394]", func(ctx context.Context) {
47+
g.By("Getting worker nodes")
48+
workerNodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker")
49+
o.Expect(err).NotTo(o.HaveOccurred())
50+
o.Expect(len(workerNodes)).Should(o.BeNumerically(">", 0), "Expected at least one worker node")
51+
52+
g.By("Validating kubelet configuration on each worker node")
53+
for _, node := range workerNodes {
54+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
55+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for worker node %s", node.Name)
56+
57+
g.By(fmt.Sprintf("Checking failSwapOn=false on worker node %s", node.Name))
58+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on worker node %s", node.Name)
59+
o.Expect(*config.FailSwapOn).To(o.BeFalse(), "failSwapOn should be false on worker node %s", node.Name)
60+
framework.Logf("Worker node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
61+
62+
g.By(fmt.Sprintf("Checking swapBehavior=NoSwap on worker node %s", node.Name))
63+
o.Expect(config.MemorySwap).NotTo(o.BeNil(), "memorySwap should be set on worker node %s", node.Name)
64+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on worker node %s", node.Name)
65+
framework.Logf("Worker node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
66+
}
67+
68+
g.By("Getting control plane nodes")
69+
controlPlaneNodes, err := getControlPlaneNodes(ctx, oc)
70+
o.Expect(err).NotTo(o.HaveOccurred())
71+
o.Expect(len(controlPlaneNodes)).Should(o.BeNumerically(">", 0), "Expected at least one control plane node")
72+
73+
g.By("Validating kubelet configuration on each control plane node")
74+
for _, node := range controlPlaneNodes {
75+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
76+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for control plane node %s", node.Name)
77+
78+
g.By(fmt.Sprintf("Checking failSwapOn=true on control plane node %s", node.Name))
79+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on control plane node %s", node.Name)
80+
o.Expect(*config.FailSwapOn).To(o.BeTrue(), "failSwapOn should be true on control plane node %s", node.Name)
81+
framework.Logf("Control plane node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
82+
83+
g.By(fmt.Sprintf("Checking swapBehavior=NoSwap on control plane node %s", node.Name))
84+
o.Expect(config.MemorySwap).NotTo(o.BeNil(), "memorySwap should be set on control plane node %s", node.Name)
85+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on control plane node %s", node.Name)
86+
framework.Logf("Control plane node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
87+
}
88+
framework.Logf("Test PASSED: All nodes have correct default swap settings")
89+
})
90+
91+
g.It("should reject user override of swap settings via KubeletConfig API [OCP-86395]", func(ctx context.Context) {
92+
g.By("Creating machine config client")
93+
mcClient, err := mcclient.NewForConfig(oc.KubeFramework().ClientConfig())
94+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create machine config client")
95+
96+
g.By("Getting initial machine config resourceVersion")
97+
// Get the initial resourceVersion of the worker machine config before creating KubeletConfig
98+
workerMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(ctx, workerGeneratedKubeletMC, metav1.GetOptions{})
99+
initialResourceVersion := ""
100+
if err == nil {
101+
initialResourceVersion = workerMC.ResourceVersion
102+
framework.Logf("Initial %s resourceVersion: %s", workerGeneratedKubeletMC, initialResourceVersion)
103+
}
104+
105+
g.By("Creating a KubeletConfig with swap settings")
106+
kubeletConfig := &machineconfigv1.KubeletConfig{
107+
ObjectMeta: metav1.ObjectMeta{
108+
Name: "test-swap-override",
109+
},
110+
Spec: machineconfigv1.KubeletConfigSpec{
111+
KubeletConfig: &runtime.RawExtension{
112+
Raw: []byte(`{
113+
"failSwapOn": true,
114+
"memorySwap": {
115+
"swapBehavior": "LimitedSwap"
116+
}
117+
}`),
118+
},
119+
},
120+
}
121+
122+
g.By("Attempting to apply the KubeletConfig")
123+
defer func() {
124+
_ = mcClient.MachineconfigurationV1().KubeletConfigs().Delete(ctx, "test-swap-override", metav1.DeleteOptions{})
125+
}()
126+
framework.Logf("Creating KubeletConfig with failSwapOn=true and swapBehavior=LimitedSwap")
127+
_, err = mcClient.MachineconfigurationV1().KubeletConfigs().Create(ctx, kubeletConfig, metav1.CreateOptions{})
128+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create KubeletConfig")
129+
130+
g.By("Checking KubeletConfig status for expected error message")
131+
err = wait.Poll(2*time.Second, 30*time.Second, func() (bool, error) {
132+
kc, err := mcClient.MachineconfigurationV1().KubeletConfigs().Get(ctx, "test-swap-override", metav1.GetOptions{})
133+
if err != nil {
134+
return false, err
135+
}
136+
137+
if kc.Status.ObservedGeneration != kc.Generation {
138+
framework.Logf("Waiting for controller to process generation %d (current: %d)", kc.Generation, kc.Status.ObservedGeneration)
139+
return false, nil
140+
}
141+
142+
// Fail fast if KubeletConfig was unexpectedly accepted
143+
for _, condition := range kc.Status.Conditions {
144+
if condition.Type == machineconfigv1.KubeletConfigSuccess && condition.Status == corev1.ConditionTrue {
145+
return false, fmt.Errorf("KubeletConfig was unexpectedly accepted")
146+
}
147+
}
148+
149+
// Check for Failure condition with the expected error message
150+
for _, condition := range kc.Status.Conditions {
151+
if condition.Type == machineconfigv1.KubeletConfigFailure && condition.Status == "False" {
152+
framework.Logf("Found Failure condition: %s", condition.Message)
153+
if condition.Message == "Error: KubeletConfiguration: failSwapOn is not allowed to be set, but contains: true" {
154+
return true, nil
155+
}
156+
}
157+
}
158+
return false, nil
159+
})
160+
o.Expect(err).NotTo(o.HaveOccurred(), "Expected to find error message about failSwapOn not being allowed in KubeletConfig status")
161+
162+
g.By("Verifying machine config was not created or updated")
163+
// Wait a bit to ensure no update happens
164+
time.Sleep(5 * time.Second)
165+
166+
// Check if the machine config was created or updated (compare to initial resourceVersion captured earlier)
167+
workerMC, err = mcClient.MachineconfigurationV1().MachineConfigs().Get(ctx, workerGeneratedKubeletMC, metav1.GetOptions{})
168+
if err == nil {
169+
o.Expect(workerMC.ResourceVersion).To(o.Equal(initialResourceVersion), "Machine config %s should not be updated when failSwapOn is rejected", workerGeneratedKubeletMC)
170+
framework.Logf("Verified: %s was not updated (resourceVersion: %s)", workerGeneratedKubeletMC, workerMC.ResourceVersion)
171+
}
172+
173+
framework.Logf("Test PASSED: KubeletConfig with failSwapOn was properly rejected")
174+
})
175+
})

‎test/extended/node/node_utils.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
8+
v1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
11+
12+
exutil "github.com/openshift/origin/test/extended/util"
13+
)
14+
15+
// getNodesByLabel returns nodes matching the specified label selector
16+
func getNodesByLabel(ctx context.Context, oc *exutil.CLI, labelSelector string) ([]v1.Node, error) {
17+
nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{
18+
LabelSelector: labelSelector,
19+
})
20+
if err != nil {
21+
return nil, err
22+
}
23+
return nodes.Items, nil
24+
}
25+
26+
// getControlPlaneNodes returns all control plane nodes in the cluster
27+
func getControlPlaneNodes(ctx context.Context, oc *exutil.CLI) ([]v1.Node, error) {
28+
// Try master label first (OpenShift uses this)
29+
nodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/master")
30+
if err != nil {
31+
return nil, err
32+
}
33+
if len(nodes) > 0 {
34+
return nodes, nil
35+
}
36+
37+
// Fallback to control-plane label (upstream Kubernetes uses this)
38+
return getNodesByLabel(ctx, oc, "node-role.kubernetes.io/control-plane")
39+
}
40+
41+
// getKubeletConfigFromNode retrieves the kubelet configuration from a specific node
42+
func getKubeletConfigFromNode(ctx context.Context, oc *exutil.CLI, nodeName string) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
43+
// Use the node proxy API to get configz
44+
configzPath := fmt.Sprintf("/api/v1/nodes/%s/proxy/configz", nodeName)
45+
46+
data, err := oc.AdminKubeClient().CoreV1().RESTClient().Get().AbsPath(configzPath).DoRaw(ctx)
47+
if err != nil {
48+
return nil, fmt.Errorf("failed to get configz from node %s: %w", nodeName, err)
49+
}
50+
51+
// Parse the JSON response
52+
var configzResponse struct {
53+
KubeletConfig *kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
54+
}
55+
56+
if err := json.Unmarshal(data, &configzResponse); err != nil {
57+
return nil, fmt.Errorf("failed to unmarshal configz response: %w", err)
58+
}
59+
60+
if configzResponse.KubeletConfig == nil {
61+
return nil, fmt.Errorf("kubeletconfig is nil in response")
62+
}
63+
64+
return configzResponse.KubeletConfig, nil
65+
}

0 commit comments

Comments
 (0)