Try 3 shards e2e

Author: davidLif

.github/workflows/on-pr.yaml

@@ -275,7 +275,7 @@ jobs:
 
       - name: Run parallel e2e tests
         run: |
-          ginkgo -r --keep-going --randomize-all --randomize-suites --procs=2 --trace -vv --label-filter '!autoscale && !scale' ./test/e2e/suites
+          ginkgo -r --keep-going --randomize-all --randomize-suites --procs=3 --trace -vv --label-filter '!autoscale && !scale' ./test/e2e/suites
 
       - name: Uninstall KAI-scheduler
         run: |

hack/parallel_e2e_config/set_parallel_testing_shards.sh

@@ -27,8 +27,17 @@ if [ -z "$NODES_TO_LABEL" ]; then
   exit 1
 fi
 
-echo "Labeling nodes: $NODES_TO_LABEL for scheduling shard test-pool-2"
-kubectl label nodes $NODES_TO_LABEL kai.scheduler/node-pool=test-pool-2 --overwrite
+POOL_2_NODES=$(echo "$ALL_WORKER_NODES" | head -n 1 | tr '\n' ' ')
+POOL_3_NODES=$(echo "$ALL_WORKER_NODES" | tail -n +2 | head -n 1 | tr '\n' ' ')
+
+echo "Labeling nodes: $POOL_2_NODES for scheduling shard test-pool-2"
+kubectl label nodes $POOL_2_NODES kai.scheduler/node-pool=test-pool-2 --overwrite
 
 echo "Create the scheduling shard test-shard-2.yaml"
-kubectl apply -f ${REPO_ROOT}/hack/parallel_e2e_config/test-shard-2.yaml
+kubectl apply -f ${REPO_ROOT}/hack/parallel_e2e_config/test-shard-2.yaml
+
+echo "Labeling nodes: $POOL_3_NODES for scheduling shard test-pool-3"
+kubectl label nodes $POOL_3_NODES kai.scheduler/node-pool=test-pool-3 --overwrite
+
+echo "Create the scheduling shard test-shard-3.yaml"
+kubectl apply -f ${REPO_ROOT}/hack/parallel_e2e_config/test-shard-3.yaml

hack/parallel_e2e_config/test-shard-3.yaml

@@ -0,0 +1,14 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kai.scheduler/v1
+kind: SchedulingShard
+metadata:
+  name: test-shard-3
+spec:
+  args:
+    restrict-node-scheduling: "false"
+  partitionLabelValue: test-pool-3
+  placementStrategy:
+    cpu: binpack
+    gpu: binpack

test/e2e/suites/allocate/node_order/affinity_test.go

@@ -104,7 +104,7 @@ var _ = Describe("Affinity", Ordered, func() {
 			Expect(testedPod.Spec.NodeName).To(Equal(pod.Spec.NodeName))
 		})
 
-		It("schedules the new pod NOT with matching labels pod anti-affinity", func(ctx context.Context) {
+		It("schedules the new pod NOT with matching labels pod anti-affinity", Serial, func(ctx context.Context) {
 			testedPod := rd.CreatePodObject(testCtx.Queues[0], v1.ResourceRequirements{})
 			testedPod.Name = "pod-with-anti-affinity-" + pod.Name
 			testedPod.Spec.Affinity = &v1.Affinity{

test/e2e/suites/allocate/node_order/placement_strategy_test.go

@@ -35,7 +35,7 @@ const (
 	gpuIndexEnvVarName = "NVIDIA_VISIBLE_DEVICES"
 )
 
-var _ = Describe("Placement strategy", Label(labels.Operated), Ordered, func() {
+var _ = Describe("Placement strategy", Label(labels.Operated), Serial, Ordered, func() {
 	var (
 		testCtx *testcontext.TestContext
 	)

test/e2e/suites/allocate/topology/topology_test.go

@@ -27,7 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Topology", Ordered, func() {
+var _ = Describe("Topology", Serial, Ordered, func() {
 	var (
 		testCtx          *testcontext.TestContext
 		gpuNodesNames    []string

test/e2e/suites/preempt/preempt_distributed_test.go

@@ -27,7 +27,7 @@ import (
 	"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/wait"
 )
 
-var _ = Describe("preempt Distributed Jobs", Ordered, func() {
+var _ = Describe("preempt Distributed Jobs", Serial, Ordered, func() {
 	Context("Over more than one nodes", func() {
 		var (
 			testCtx      *testcontext.TestContext

test/e2e/suites/reclaim/reclaim_distributed_test.go

@@ -26,7 +26,7 @@ import (
 	"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/wait"
 )
 
-var _ = Describe("Reclaim Distributed Jobs", Ordered, func() {
+var _ = Describe("Reclaim Distributed Jobs", Serial, Ordered, func() {
 	Context("Over more than one nodes", func() {
 		var (
 			testCtx        *testcontext.TestContext