Skip to content

Commit 3234106

Browse files
ffromaniffromani
committed
e2e: node: cpumgr: check CPU allocatable for CFS quota test
add (admittedly pretty crude) CPU allocatable check. A more incisive refactoring is needed, but we need to unbreak CI first, so this seems the minimal decently clean test. Signed-off-by: Francesco Romani <[email protected]>
1 parent 844c2ef commit 3234106

File tree

1 file changed

+80
-63
lines changed

1 file changed

+80
-63
lines changed

‎test/e2e_node/cpu_manager_test.go‎

Lines changed: 80 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -592,12 +592,14 @@ func runMultipleCPUContainersGuPod(ctx context.Context, f *framework.Framework)
592592
waitForContainerRemoval(ctx, pod.Spec.Containers[1].Name, pod.Name, pod.Namespace)
593593
}
594594

595-
func runCfsQuotaGuPods(ctx context.Context, f *framework.Framework, disabledCPUQuotaWithExclusiveCPUs bool) {
595+
func runCfsQuotaGuPods(ctx context.Context, f *framework.Framework, disabledCPUQuotaWithExclusiveCPUs bool, cpuAlloc int64) {
596596
var err error
597597
var ctnAttrs []ctnAttribute
598598
var pod1, pod2, pod3 *v1.Pod
599599
podsToClean := make(map[string]*v1.Pod) // pod.UID -> pod
600600

601+
framework.Logf("runCfsQuotaGuPods: disableQuota=%v, CPU Allocatable=%v", disabledCPUQuotaWithExclusiveCPUs, cpuAlloc)
602+
601603
deleteTestPod := func(pod *v1.Pod) {
602604
// waitForContainerRemoval takes "long" to complete; if we use the parent ctx we get a
603605
// 'deadline expired' message and the cleanup aborts, which we don't want.
@@ -619,6 +621,7 @@ func runCfsQuotaGuPods(ctx context.Context, f *framework.Framework, disabledCPUQ
619621
deletePodsAsync(ctx2, f, podsToClean)
620622
})
621623

624+
podCFSCheckCommand := []string{"sh", "-c", `cat $(find /sysfscgroup | grep "$(cat /podinfo/uid | sed 's/-/_/g').slice/cpu.max$") && sleep 1d`}
622625
cfsCheckCommand := []string{"sh", "-c", "cat /sys/fs/cgroup/cpu.max && sleep 1d"}
623626
defaultPeriod := "100000"
624627

@@ -688,75 +691,77 @@ func runCfsQuotaGuPods(ctx context.Context, f *framework.Framework, disabledCPUQ
688691
pod3.Spec.Containers[0].Name, pod3.Name)
689692
deleteTestPod(pod3)
690693

691-
ctnAttrs = []ctnAttribute{
692-
{
693-
ctnName: "gu-container-non-int-values",
694-
cpuRequest: "500m",
695-
cpuLimit: "500m",
696-
},
697-
{
698-
ctnName: "gu-container-int-values",
699-
cpuRequest: "1",
700-
cpuLimit: "1",
701-
},
702-
}
703-
pod4 := makeCPUManagerPod("gu-pod4", ctnAttrs)
704-
pod4.Spec.Containers[0].Command = cfsCheckCommand
705-
pod4.Spec.Containers[1].Command = cfsCheckCommand
706-
pod4 = e2epod.NewPodClient(f).CreateSync(ctx, pod4)
707-
podsToClean[string(pod4.UID)] = pod4
708-
709-
ginkgo.By("checking if the expected cfs quota was assigned (GU pod, container 0 exclusive CPUs unlimited, container 1 limited)")
694+
if cpuAlloc >= 2 {
695+
ctnAttrs = []ctnAttribute{
696+
{
697+
ctnName: "gu-container-non-int-values",
698+
cpuRequest: "500m",
699+
cpuLimit: "500m",
700+
},
701+
{
702+
ctnName: "gu-container-int-values",
703+
cpuRequest: "1",
704+
cpuLimit: "1",
705+
},
706+
}
707+
pod4 := makeCPUManagerPod("gu-pod4", ctnAttrs)
708+
pod4.Spec.Containers[0].Command = cfsCheckCommand
709+
pod4.Spec.Containers[1].Command = cfsCheckCommand
710+
pod4 = e2epod.NewPodClient(f).CreateSync(ctx, pod4)
711+
podsToClean[string(pod4.UID)] = pod4
712+
713+
ginkgo.By("checking if the expected cfs quota was assigned (GU pod, container 0 exclusive CPUs unlimited, container 1 limited)")
714+
715+
expectedQuota = "50000"
716+
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
717+
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod4.Name, pod4.Spec.Containers[0].Name, expCFSQuotaRegex)
718+
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
719+
pod4.Spec.Containers[0].Name, pod4.Name)
720+
expectedQuota = "100000"
721+
if disabledCPUQuotaWithExclusiveCPUs {
722+
expectedQuota = "max"
723+
}
724+
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
725+
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod4.Name, pod4.Spec.Containers[1].Name, expCFSQuotaRegex)
726+
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
727+
pod4.Spec.Containers[1].Name, pod4.Name)
728+
deleteTestPod(pod4)
710729

711-
expectedQuota = "50000"
712-
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
713-
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod4.Name, pod4.Spec.Containers[0].Name, expCFSQuotaRegex)
714-
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
715-
pod4.Spec.Containers[0].Name, pod4.Name)
716-
expectedQuota = "100000"
717-
if disabledCPUQuotaWithExclusiveCPUs {
718-
expectedQuota = "max"
719-
}
720-
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
721-
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod4.Name, pod4.Spec.Containers[1].Name, expCFSQuotaRegex)
722-
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]",
723-
pod4.Spec.Containers[1].Name, pod4.Name)
724-
deleteTestPod(pod4)
730+
ctnAttrs = []ctnAttribute{
731+
{
732+
ctnName: "gu-container-non-int-values",
733+
cpuRequest: "500m",
734+
cpuLimit: "500m",
735+
},
736+
{
737+
ctnName: "gu-container-int-values",
738+
cpuRequest: "1",
739+
cpuLimit: "1",
740+
},
741+
}
725742

726-
ctnAttrs = []ctnAttribute{
727-
{
728-
ctnName: "gu-container-non-int-values",
729-
cpuRequest: "500m",
730-
cpuLimit: "500m",
731-
},
732-
{
733-
ctnName: "gu-container-int-values",
734-
cpuRequest: "1",
735-
cpuLimit: "1",
736-
},
737-
}
743+
pod5 := makeCPUManagerPod("gu-pod5", ctnAttrs)
744+
pod5.Spec.Containers[0].Command = podCFSCheckCommand
745+
pod5 = e2epod.NewPodClient(f).CreateSync(ctx, pod5)
746+
podsToClean[string(pod5.UID)] = pod5
738747

739-
podCFSCheckCommand := []string{"sh", "-c", `cat $(find /sysfscgroup | grep "$(cat /podinfo/uid | sed 's/-/_/g').slice/cpu.max$") && sleep 1d`}
748+
ginkgo.By("checking if the expected cfs quota was assigned to pod (GU pod, unlimited)")
740749

741-
pod5 := makeCPUManagerPod("gu-pod5", ctnAttrs)
742-
pod5.Spec.Containers[0].Command = podCFSCheckCommand
743-
pod5 = e2epod.NewPodClient(f).CreateSync(ctx, pod5)
744-
podsToClean[string(pod5.UID)] = pod5
750+
expectedQuota = "150000"
745751

746-
ginkgo.By("checking if the expected cfs quota was assigned to pod (GU pod, unlimited)")
752+
if disabledCPUQuotaWithExclusiveCPUs {
753+
expectedQuota = "max"
754+
}
747755

748-
expectedQuota = "150000"
756+
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
749757

750-
if disabledCPUQuotaWithExclusiveCPUs {
751-
expectedQuota = "max"
758+
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod5.Name, pod5.Spec.Containers[0].Name, expCFSQuotaRegex)
759+
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]", pod5.Spec.Containers[0].Name, pod5.Name)
760+
deleteTestPod(pod5)
761+
} else {
762+
ginkgo.By(fmt.Sprintf("some cases SKIPPED - requests at least %d allocatable cores, got %d", 2, cpuAlloc))
752763
}
753764

754-
expCFSQuotaRegex = fmt.Sprintf("^%s %s\n$", expectedQuota, defaultPeriod)
755-
756-
err = e2epod.NewPodClient(f).MatchContainerOutput(ctx, pod5.Name, pod5.Spec.Containers[0].Name, expCFSQuotaRegex)
757-
framework.ExpectNoError(err, "expected log not found in container [%s] of pod [%s]", pod5.Spec.Containers[0].Name, pod5.Name)
758-
deleteTestPod(pod5)
759-
760765
ctnAttrs = []ctnAttribute{
761766
{
762767
ctnName: "gu-container",
@@ -936,6 +941,10 @@ func runCPUManagerTests(f *framework.Framework) {
936941
if !IsCgroup2UnifiedMode() {
937942
e2eskipper.Skipf("Skipping since CgroupV2 not used")
938943
}
944+
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
945+
if cpuAlloc < 1 { // save expensive kubelet restart
946+
e2eskipper.Skipf("Skipping since not enough allocatable CPU got %d required 1", cpuAlloc)
947+
}
939948
newCfg := configureCPUManagerInKubelet(oldCfg,
940949
&cpuManagerKubeletArguments{
941950
policyName: string(cpumanager.PolicyStatic),
@@ -944,13 +953,19 @@ func runCPUManagerTests(f *framework.Framework) {
944953
},
945954
)
946955
updateKubeletConfig(ctx, f, newCfg, true)
947-
runCfsQuotaGuPods(ctx, f, true)
956+
957+
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f) // check again after we reserved 1 full CPU. Some tests require > 1 exclusive CPU
958+
runCfsQuotaGuPods(ctx, f, true, cpuAlloc)
948959
})
949960

950961
ginkgo.It("should keep enforcing the CFS quota for containers with static CPUs assigned and feature gate disabled", func(ctx context.Context) {
951962
if !IsCgroup2UnifiedMode() {
952963
e2eskipper.Skipf("Skipping since CgroupV2 not used")
953964
}
965+
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f)
966+
if cpuAlloc < 1 { // save expensive kubelet restart
967+
e2eskipper.Skipf("Skipping since not enough allocatable CPU got %d required 1", cpuAlloc)
968+
}
954969
newCfg := configureCPUManagerInKubelet(oldCfg,
955970
&cpuManagerKubeletArguments{
956971
policyName: string(cpumanager.PolicyStatic),
@@ -960,7 +975,9 @@ func runCPUManagerTests(f *framework.Framework) {
960975
)
961976

962977
updateKubeletConfig(ctx, f, newCfg, true)
963-
runCfsQuotaGuPods(ctx, f, false)
978+
979+
_, cpuAlloc, _ = getLocalNodeCPUDetails(ctx, f) // check again after we reserved 1 full CPU. Some tests require > 1 exclusive CPU
980+
runCfsQuotaGuPods(ctx, f, false, cpuAlloc)
964981
})
965982

966983
f.It("should not reuse CPUs of restartable init containers", feature.SidecarContainers, func(ctx context.Context) {

0 commit comments

Comments
 (0)