Skip to content

Commit 32fff3d

Browse files
BhargaviGudiBhargaviGudi
committed
Add automated tests for non-CNV swap configuration
Updated testcase name and separated upgrade cases Updated testcase
1 parent 9d825ed commit 32fff3d

File tree

2 files changed

+223
-0
lines changed

2 files changed

+223
-0
lines changed

‎test/extended/node/node_swap.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
g "github.com/onsi/ginkgo/v2"
9+
o "github.com/onsi/gomega"
10+
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
"k8s.io/kubernetes/test/e2e/framework"
14+
15+
machineconfigv1 "github.com/openshift/api/machineconfiguration/v1"
16+
mcclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
17+
exutil "github.com/openshift/origin/test/extended/util"
18+
)
19+
20+
var _ = g.Describe("[sig-node] Node non-cnv swap configuration", func() {
21+
defer g.GinkgoRecover()
22+
23+
var oc = exutil.NewCLI("node-swap")
24+
25+
// This test validates that:
26+
// - Worker nodes have failSwapOn=false to allow kubelet to start even if swap is present at OS level
27+
// - Control plane nodes have failSwapOn=true to prevent kubelet from starting if swap is enabled
28+
// - All nodes have swapBehavior=NoSwap to ensure kubelet does not utilize swap even if available at OS level
29+
// The swapBehavior=NoSwap configuration ensures that even if swap is manually enabled on a worker node,
30+
// the kubelet will not use it for memory management, maintaining consistent behavior across the cluster.
31+
g.It("should have correct default kubelet swap settings with worker nodes failSwapOn=false, control plane nodes failSwapOn=true, and both swapBehavior=NoSwap", func(ctx context.Context) {
32+
g.By("Getting worker nodes")
33+
workerNodes, err := getWorkerNodes(ctx, oc)
34+
o.Expect(err).NotTo(o.HaveOccurred())
35+
o.Expect(len(workerNodes)).Should(o.BeNumerically(">", 0), "Expected at least one worker node")
36+
37+
g.By("Validating kubelet configuration on each worker node")
38+
for _, node := range workerNodes {
39+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
40+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for worker node %s", node.Name)
41+
42+
g.By(fmt.Sprintf("Checking failSwapOn=false on worker node %s", node.Name))
43+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on worker node %s", node.Name)
44+
o.Expect(*config.FailSwapOn).To(o.BeFalse(), "failSwapOn should be false on worker node %s", node.Name)
45+
framework.Logf("Worker node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
46+
47+
g.By(fmt.Sprintf("Checking swapDesired=NoSwap on worker node %s", node.Name))
48+
if config.SwapDesired != nil {
49+
o.Expect(*config.SwapDesired).To(o.Equal("NoSwap"), "swapDesired should be NoSwap on worker node %s", node.Name)
50+
framework.Logf("Worker node %s: swapDesired=%s ✓", node.Name, *config.SwapDesired)
51+
}
52+
// Also check memorySwap.swapBehavior if present
53+
if config.MemorySwap != nil {
54+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on worker node %s", node.Name)
55+
framework.Logf("Worker node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
56+
}
57+
}
58+
59+
g.By("Getting control plane nodes")
60+
controlPlaneNodes, err := getControlPlaneNodes(ctx, oc)
61+
o.Expect(err).NotTo(o.HaveOccurred())
62+
o.Expect(len(controlPlaneNodes)).Should(o.BeNumerically(">", 0), "Expected at least one control plane node")
63+
64+
g.By("Validating kubelet configuration on each control plane node")
65+
for _, node := range controlPlaneNodes {
66+
config, err := getKubeletConfigFromNode(ctx, oc, node.Name)
67+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get kubelet config for control plane node %s", node.Name)
68+
69+
g.By(fmt.Sprintf("Checking failSwapOn=true on control plane node %s", node.Name))
70+
o.Expect(config.FailSwapOn).NotTo(o.BeNil(), "failSwapOn should be set on control plane node %s", node.Name)
71+
o.Expect(*config.FailSwapOn).To(o.BeTrue(), "failSwapOn should be true on control plane node %s", node.Name)
72+
framework.Logf("Control plane node %s: failSwapOn=%v ✓", node.Name, *config.FailSwapOn)
73+
74+
g.By(fmt.Sprintf("Checking swapDesired=NoSwap on control plane node %s", node.Name))
75+
if config.SwapDesired != nil {
76+
o.Expect(*config.SwapDesired).To(o.Equal("NoSwap"), "swapDesired should be NoSwap on control plane node %s", node.Name)
77+
framework.Logf("Control plane node %s: swapDesired=%s ✓", node.Name, *config.SwapDesired)
78+
}
79+
// Also check memorySwap.swapBehavior if present
80+
if config.MemorySwap != nil {
81+
o.Expect(config.MemorySwap.SwapBehavior).To(o.Equal("NoSwap"), "swapBehavior should be NoSwap on control plane node %s", node.Name)
82+
framework.Logf("Control plane node %s: swapBehavior=%s ✓", node.Name, config.MemorySwap.SwapBehavior)
83+
}
84+
}
85+
framework.Logf("Test PASSED: All nodes have correct default swap settings")
86+
})
87+
88+
g.It("should reject user override of swap settings via KubeletConfig API", func(ctx context.Context) {
89+
g.By("Creating machine config client")
90+
mcClient, err := mcclient.NewForConfig(oc.KubeFramework().ClientConfig())
91+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create machine config client")
92+
93+
g.By("Creating a KubeletConfig with swap settings")
94+
kubeletConfig := &machineconfigv1.KubeletConfig{
95+
ObjectMeta: metav1.ObjectMeta{
96+
Name: "test-swap-override",
97+
},
98+
Spec: machineconfigv1.KubeletConfigSpec{
99+
KubeletConfig: &runtime.RawExtension{
100+
Raw: []byte(`{
101+
"failSwapOn": true,
102+
"memorySwap": {
103+
"swapBehavior": "LimitedSwap"
104+
}
105+
}`),
106+
},
107+
},
108+
}
109+
110+
g.By("Attempting to apply the KubeletConfig")
111+
framework.Logf("Creating KubeletConfig with failSwapOn=true and swapBehavior=LimitedSwap")
112+
_, err = mcClient.MachineconfigurationV1().KubeletConfigs().Create(ctx, kubeletConfig, metav1.CreateOptions{})
113+
114+
// We expect this to either be rejected immediately or to be created but not applied
115+
if err != nil {
116+
g.By("Verifying the error message indicates swap is not configurable")
117+
o.Expect(err).To(o.HaveOccurred())
118+
framework.Logf("KubeletConfig creation rejected with error: %v", err)
119+
} else {
120+
framework.Logf("KubeletConfig was created, verifying it doesn't affect worker nodes")
121+
// If created, clean up and verify it doesn't affect nodes
122+
defer func() {
123+
_ = mcClient.MachineconfigurationV1().KubeletConfigs().Delete(ctx, "test-swap-override", metav1.DeleteOptions{})
124+
}()
125+
126+
// Wait a bit and verify worker node configs remain unchanged
127+
time.Sleep(10 * time.Second)
128+
workerNodes, err := getWorkerNodes(ctx, oc)
129+
o.Expect(err).NotTo(o.HaveOccurred())
130+
if len(workerNodes) > 0 {
131+
config, err := getKubeletConfigFromNode(ctx, oc, workerNodes[0].Name)
132+
o.Expect(err).NotTo(o.HaveOccurred())
133+
framework.Logf("Worker node %s: failSwapOn=%v, swapBehavior=%s", workerNodes[0].Name, *config.FailSwapOn, config.MemorySwap.SwapBehavior)
134+
o.Expect(*config.FailSwapOn).To(o.BeFalse(), "Worker node failSwapOn should remain false")
135+
framework.Logf("Test PASSED: Worker node swap settings remained unchanged despite KubeletConfig creation")
136+
}
137+
}
138+
})
139+
})

‎test/extended/node/node_utils.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package node
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
8+
v1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
11+
exutil "github.com/openshift/origin/test/extended/util"
12+
)
13+
14+
// KubeletConfiguration represents the kubelet configuration structure
15+
type KubeletConfiguration struct {
16+
FailSwapOn *bool `json:"failSwapOn,omitempty"`
17+
MemorySwap *MemorySwapConfiguration `json:"memorySwap,omitempty"`
18+
SwapDesired *string `json:"swapDesired,omitempty"`
19+
}
20+
21+
// MemorySwapConfiguration represents memory swap configuration
22+
type MemorySwapConfiguration struct {
23+
SwapBehavior string `json:"swapBehavior,omitempty"`
24+
}
25+
26+
// getWorkerNodes returns all worker nodes in the cluster
27+
func getWorkerNodes(ctx context.Context, oc *exutil.CLI) ([]v1.Node, error) {
28+
nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
29+
if err != nil {
30+
return nil, err
31+
}
32+
33+
var workerNodes []v1.Node
34+
for _, node := range nodes.Items {
35+
if _, ok := node.Labels["node-role.kubernetes.io/worker"]; ok {
36+
workerNodes = append(workerNodes, node)
37+
}
38+
}
39+
return workerNodes, nil
40+
}
41+
42+
// getControlPlaneNodes returns all control plane nodes in the cluster
43+
func getControlPlaneNodes(ctx context.Context, oc *exutil.CLI) ([]v1.Node, error) {
44+
nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
45+
if err != nil {
46+
return nil, err
47+
}
48+
49+
var controlPlaneNodes []v1.Node
50+
for _, node := range nodes.Items {
51+
if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
52+
controlPlaneNodes = append(controlPlaneNodes, node)
53+
} else if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
54+
controlPlaneNodes = append(controlPlaneNodes, node)
55+
}
56+
}
57+
return controlPlaneNodes, nil
58+
}
59+
60+
// getKubeletConfigFromNode retrieves the kubelet configuration from a specific node
61+
func getKubeletConfigFromNode(ctx context.Context, oc *exutil.CLI, nodeName string) (*KubeletConfiguration, error) {
62+
// Use the node proxy API to get configz
63+
configzPath := fmt.Sprintf("/api/v1/nodes/%s/proxy/configz", nodeName)
64+
65+
data, err := oc.AdminKubeClient().CoreV1().RESTClient().Get().AbsPath(configzPath).DoRaw(ctx)
66+
if err != nil {
67+
return nil, fmt.Errorf("failed to get configz from node %s: %w", nodeName, err)
68+
}
69+
70+
// Parse the JSON response
71+
var configzResponse struct {
72+
KubeletConfig *KubeletConfiguration `json:"kubeletconfig"`
73+
}
74+
75+
if err := json.Unmarshal(data, &configzResponse); err != nil {
76+
return nil, fmt.Errorf("failed to unmarshal configz response: %w", err)
77+
}
78+
79+
if configzResponse.KubeletConfig == nil {
80+
return nil, fmt.Errorf("kubeletconfig is nil in response")
81+
}
82+
83+
return configzResponse.KubeletConfig, nil
84+
}

0 commit comments

Comments
 (0)