Skip to content

Commit edc0ff6

Browse files
e0nee0ne
authored
Merge pull request openshift#895 from zeeke/us/OCPBUGS-56513
bug(daemon): Reset the configuration when `disableDrain: true`
2 parents 184df1c + fc04f97 commit edc0ff6

File tree

2 files changed

+191
-88
lines changed

2 files changed

+191
-88
lines changed

‎pkg/daemon/daemon.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,10 @@ func (dn *NodeReconciler) prepareNMUdevRule() error {
644644

645645
// isDrainCompleted returns true if the current-state annotation is drain completed
646646
func (dn *NodeReconciler) isDrainCompleted(reqDrain bool, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) bool {
647+
if vars.DisableDrain {
648+
return true
649+
}
650+
647651
// if we need to drain check the drain status
648652
if reqDrain {
649653
return utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete)

‎pkg/daemon/daemon_test.go‎

Lines changed: 187 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"os"
66
"sync"
7+
"sync/atomic"
78
"time"
89

910
. "github.com/onsi/ginkgo/v2"
@@ -32,8 +33,6 @@ import (
3233
)
3334

3435
var (
35-
cancel context.CancelFunc
36-
ctx context.Context
3736
k8sManager manager.Manager
3837
kubeclient *kubernetes.Clientset
3938
eventRecorder *daemon.EventRecorder
@@ -44,6 +43,11 @@ var (
4443
mockCtrl *gomock.Controller
4544
hostHelper *mock_helper.MockHostHelpersInterface
4645
platformHelper *mock_platforms.MockInterface
46+
47+
discoverSriovReturn atomic.Pointer[[]sriovnetworkv1.InterfaceExt]
48+
nodeState *sriovnetworkv1.SriovNetworkNodeState
49+
50+
daemonReconciler *daemon.NodeReconciler
4751
)
4852

4953
const (
@@ -53,8 +57,12 @@ const (
5357

5458
var _ = Describe("Daemon Controller", Ordered, func() {
5559
BeforeAll(func() {
56-
ctx, cancel = context.WithCancel(context.Background())
5760
wg = sync.WaitGroup{}
61+
DeferCleanup(wg.Wait)
62+
63+
ctx, cancel := context.WithCancel(context.Background())
64+
DeferCleanup(cancel)
65+
5866
startDaemon = func(dc *daemon.NodeReconciler) {
5967
By("start controller manager")
6068
wg.Add(1)
@@ -92,9 +100,8 @@ var _ = Describe("Daemon Controller", Ordered, func() {
92100
} else {
93101
vars.ClusterType = constants.ClusterTypeKubernetes
94102
}
95-
})
96103

97-
BeforeEach(func() {
104+
By("Init mock functions")
98105
t = GinkgoT()
99106
mockCtrl = gomock.NewController(t)
100107
hostHelper = mock_helper.NewMockHostHelpersInterface(mockCtrl)
@@ -113,95 +120,71 @@ var _ = Describe("Daemon Controller", Ordered, func() {
113120
hostHelper.EXPECT().Chroot(gomock.Any()).Return(func() error { return nil }, nil).AnyTimes()
114121
hostHelper.EXPECT().RunCommand("/bin/sh", gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", nil).AnyTimes()
115122

116-
})
123+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{})
117124

118-
AfterEach(func() {
119-
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
125+
hostHelper.EXPECT().DiscoverSriovDevices(hostHelper).DoAndReturn(func(helpersInterface helper.HostHelpersInterface) ([]sriovnetworkv1.InterfaceExt, error) {
126+
return *discoverSriovReturn.Load(), nil
127+
}).AnyTimes()
128+
129+
hostHelper.EXPECT().LoadPfsStatus("0000:16:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes()
130+
131+
hostHelper.EXPECT().ClearPCIAddressFolder().Return(nil).AnyTimes()
132+
hostHelper.EXPECT().DiscoverRDMASubsystem().Return("shared", nil).AnyTimes()
133+
hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes()
134+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgPciRealloc).Return(true).AnyTimes()
135+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIntelIommu).Return(true).AnyTimes()
136+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIommuPt).Return(true).AnyTimes()
137+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaExclusive).Return(false).AnyTimes()
138+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaShared).Return(false).AnyTimes()
139+
hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes()
120140

121-
By("Shutdown controller manager")
122-
cancel()
123-
wg.Wait()
141+
hostHelper.EXPECT().ConfigSriovInterfaces(gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).AnyTimes()
142+
143+
// k8s plugin for k8s cluster type
144+
if vars.ClusterType == constants.ClusterTypeKubernetes {
145+
hostHelper.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
146+
hostHelper.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
147+
}
148+
149+
featureGates := featuregate.New()
150+
featureGates.Init(map[string]bool{})
151+
daemonReconciler = createDaemon(hostHelper, platformHelper, featureGates, []string{})
152+
startDaemon(daemonReconciler)
153+
154+
_, nodeState = createNode("node1")
124155
})
125156

126157
AfterAll(func() {
158+
Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{})).ToNot(HaveOccurred())
159+
160+
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
127161
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
128162
})
129163

130164
Context("Config Daemon generic flow", func() {
131-
BeforeEach(func() {
132-
// k8s plugin for k8s cluster type
133-
if vars.ClusterType == constants.ClusterTypeKubernetes {
134-
hostHelper.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
135-
hostHelper.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
136-
}
137-
})
138165

139-
It("Should expose nodeState Status section", func() {
140-
By("Init mock functions")
141-
afterConfig := false
142-
hostHelper.EXPECT().DiscoverSriovDevices(hostHelper).DoAndReturn(func(helpersInterface helper.HostHelpersInterface) ([]sriovnetworkv1.InterfaceExt, error) {
143-
interfaceExtList := []sriovnetworkv1.InterfaceExt{
144-
{
145-
Name: "eno1",
146-
Driver: "ice",
147-
PciAddress: "0000:16:00.0",
148-
DeviceID: "1593",
149-
Vendor: "8086",
150-
EswitchMode: "legacy",
151-
LinkAdminState: "up",
152-
LinkSpeed: "10000 Mb/s",
153-
LinkType: "ETH",
154-
Mac: "aa:bb:cc:dd:ee:ff",
155-
Mtu: 1500,
156-
TotalVfs: 2,
157-
NumVfs: 0,
158-
},
159-
}
160-
161-
if afterConfig {
162-
interfaceExtList[0].NumVfs = 2
163-
interfaceExtList[0].VFs = []sriovnetworkv1.VirtualFunction{
164-
{
165-
Name: "eno1f0",
166-
PciAddress: "0000:16:00.1",
167-
VfID: 0,
168-
},
169-
{
170-
Name: "eno1f1",
171-
PciAddress: "0000:16:00.2",
172-
VfID: 1,
173-
}}
174-
}
175-
return interfaceExtList, nil
176-
}).AnyTimes()
177-
178-
hostHelper.EXPECT().LoadPfsStatus("0000:16:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes()
179-
180-
hostHelper.EXPECT().ClearPCIAddressFolder().Return(nil).AnyTimes()
181-
hostHelper.EXPECT().DiscoverRDMASubsystem().Return("shared", nil).AnyTimes()
182-
hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes()
183-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgPciRealloc).Return(true).AnyTimes()
184-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIntelIommu).Return(true).AnyTimes()
185-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIommuPt).Return(true).AnyTimes()
186-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaExclusive).Return(false).AnyTimes()
187-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaShared).Return(false).AnyTimes()
188-
hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes()
189-
190-
hostHelper.EXPECT().ConfigSriovInterfaces(gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).AnyTimes()
191-
192-
featureGates := featuregate.New()
193-
featureGates.Init(map[string]bool{})
194-
dc := createDaemon(hostHelper, platformHelper, featureGates, []string{})
195-
startDaemon(dc)
196-
197-
_, nodeState := createNode("node1")
198-
By("waiting for state to be succeeded")
199-
EventuallyWithOffset(1, func(g Gomega) {
200-
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
201-
ToNot(HaveOccurred())
166+
It("Should expose nodeState Status section", func(ctx context.Context) {
167+
168+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
169+
{
170+
Name: "eno1",
171+
Driver: "ice",
172+
PciAddress: "0000:16:00.0",
173+
DeviceID: "1593",
174+
Vendor: "8086",
175+
EswitchMode: "legacy",
176+
LinkAdminState: "up",
177+
LinkSpeed: "10000 Mb/s",
178+
LinkType: "ETH",
179+
Mac: "aa:bb:cc:dd:ee:ff",
180+
Mtu: 1500,
181+
TotalVfs: 2,
182+
NumVfs: 0,
183+
},
184+
})
202185

203-
g.Expect(nodeState.Status.SyncStatus).To(Equal(constants.SyncStatusSucceeded))
204-
}, waitTime, retryTime).Should(Succeed())
186+
By("waiting for state to be succeeded")
187+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
205188

206189
By("add spec to node state")
207190
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
@@ -219,14 +202,44 @@ var _ = Describe("Daemon Controller", Ordered, func() {
219202
VfRange: "eno1#0-1"},
220203
}},
221204
}
222-
afterConfig = true
205+
206+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
207+
{
208+
Name: "eno1",
209+
Driver: "ice",
210+
PciAddress: "0000:16:00.0",
211+
DeviceID: "1593",
212+
Vendor: "8086",
213+
EswitchMode: "legacy",
214+
LinkAdminState: "up",
215+
LinkSpeed: "10000 Mb/s",
216+
LinkType: "ETH",
217+
Mac: "aa:bb:cc:dd:ee:ff",
218+
Mtu: 1500,
219+
TotalVfs: 2,
220+
NumVfs: 2,
221+
VFs: []sriovnetworkv1.VirtualFunction{
222+
{
223+
Name: "eno1f0",
224+
PciAddress: "0000:16:00.1",
225+
VfID: 0,
226+
},
227+
{
228+
Name: "eno1f1",
229+
PciAddress: "0000:16:00.2",
230+
VfID: 1,
231+
}},
232+
},
233+
})
234+
223235
err = k8sClient.Update(ctx, nodeState)
224236
Expect(err).ToNot(HaveOccurred())
237+
225238
By("waiting to require drain")
226239
EventuallyWithOffset(1, func(g Gomega) {
227240
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
228241
ToNot(HaveOccurred())
229-
g.Expect(dc.GetLastAppliedGeneration()).To(Equal(int64(2)))
242+
g.Expect(daemonReconciler.GetLastAppliedGeneration()).To(Equal(int64(2)))
230243
}, waitTime, retryTime).Should(Succeed())
231244

232245
err = k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
@@ -262,13 +275,72 @@ var _ = Describe("Daemon Controller", Ordered, func() {
262275

263276
Expect(nodeState.Status.LastSyncError).To(Equal(""))
264277
})
278+
279+
It("Should apply the reset configuration when disableDrain is true", func(ctx context.Context) {
280+
DeferCleanup(func(x bool) { vars.DisableDrain = x }, vars.DisableDrain)
281+
vars.DisableDrain = true
282+
283+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
284+
{
285+
Name: "eno1",
286+
Driver: "ice",
287+
PciAddress: "0000:16:00.0",
288+
DeviceID: "1593",
289+
Vendor: "8086",
290+
EswitchMode: "legacy",
291+
LinkAdminState: "up",
292+
LinkSpeed: "10000 Mb/s",
293+
LinkType: "ETH",
294+
Mac: "aa:bb:cc:dd:ee:ff",
295+
Mtu: 1500,
296+
TotalVfs: 2,
297+
NumVfs: 2,
298+
VFs: []sriovnetworkv1.VirtualFunction{
299+
{
300+
Name: "eno1f0",
301+
PciAddress: "0000:16:00.1",
302+
VfID: 0,
303+
},
304+
{
305+
Name: "eno1f1",
306+
PciAddress: "0000:16:00.2",
307+
VfID: 1,
308+
}},
309+
},
310+
})
311+
312+
nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{
313+
{Name: "eno1",
314+
PciAddress: "0000:16:00.0",
315+
LinkType: "eth",
316+
NumVfs: 2,
317+
VfGroups: []sriovnetworkv1.VfGroup{
318+
{ResourceName: "test",
319+
DeviceType: "netdevice",
320+
PolicyName: "test-policy",
321+
VfRange: "eno1#0-1"},
322+
}},
323+
}
324+
err := k8sClient.Update(ctx, nodeState)
325+
Expect(err).ToNot(HaveOccurred())
326+
327+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
328+
329+
By("Simulate node policy removal")
330+
nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{}
331+
err = k8sClient.Update(ctx, nodeState)
332+
Expect(err).ToNot(HaveOccurred())
333+
334+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
335+
assertLastStatusTransitionsContains(nodeState, 2, constants.SyncStatusInProgress)
336+
})
265337
})
266338
})
267339

268340
func patchAnnotation(nodeState *sriovnetworkv1.SriovNetworkNodeState, key, value string) {
269341
originalNodeState := nodeState.DeepCopy()
270342
nodeState.Annotations[key] = value
271-
err := k8sClient.Patch(ctx, nodeState, client.MergeFrom(originalNodeState))
343+
err := k8sClient.Patch(context.Background(), nodeState, client.MergeFrom(originalNodeState))
272344
Expect(err).ToNot(HaveOccurred())
273345
}
274346

@@ -297,8 +369,8 @@ func createNode(nodeName string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNode
297369
},
298370
}
299371

300-
Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred())
301-
Expect(k8sClient.Create(ctx, &nodeState)).ToNot(HaveOccurred())
372+
Expect(k8sClient.Create(context.Background(), &node)).ToNot(HaveOccurred())
373+
Expect(k8sClient.Create(context.Background(), &nodeState)).ToNot(HaveOccurred())
302374
vars.NodeName = nodeName
303375

304376
return &node, &nodeState
@@ -329,3 +401,30 @@ func createDaemon(
329401

330402
return configController
331403
}
404+
405+
func eventuallySyncStatusEqual(nodeState *sriovnetworkv1.SriovNetworkNodeState, expectedState string) {
406+
EventuallyWithOffset(1, func(g Gomega) {
407+
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
408+
ToNot(HaveOccurred())
409+
g.Expect(nodeState.Status.SyncStatus).To(Equal(expectedState))
410+
}, 10*time.Second, 100*time.Millisecond).Should(Succeed())
411+
}
412+
413+
func assertLastStatusTransitionsContains(nodeState *sriovnetworkv1.SriovNetworkNodeState, numberOfTransitions int, status string) {
414+
events := &corev1.EventList{}
415+
err := k8sClient.List(
416+
context.Background(),
417+
events,
418+
client.MatchingFields{
419+
"involvedObject.name": nodeState.Name,
420+
"reason": "SyncStatusChanged",
421+
},
422+
client.Limit(numberOfTransitions),
423+
)
424+
Expect(err).ToNot(HaveOccurred())
425+
426+
// Status transition events are in the form
427+
// `Status changed from: Succeed to: InProgress`
428+
Expect(events.Items).To(ContainElement(
429+
HaveField("Message", ContainSubstring("to: "+status))))
430+
}

0 commit comments

Comments
 (0)