test: remaining topology e2e tests

operator/e2e/dependencies.yaml

@@ -31,7 +31,6 @@ images:
     version: v0.13.0-rc1
   - name: ghcr.io/nvidia/kai-scheduler/scheduler
     version: v0.13.0-rc1
-
   # Cert-manager
   - name: quay.io/jetstack/cert-manager-controller
     version: v1.14.4
@@ -42,6 +41,10 @@ images:
   - name: quay.io/jetstack/cert-manager-ctl
     version: v1.14.4
 
+  # Lightweight container for test pods
+  - name: busybox
+    version: latest
+
 # Helm charts used in E2E tests
 helmCharts:
   # Kai Scheduler - gang scheduling for Kubernetes

operator/e2e/utils/kai_topology.go

@@ -21,9 +21,11 @@ package utils
 import (
 	"context"
 	"fmt"
+	"testing"
 	"time"
 
 	kaischedulingv2alpha2 "github.com/NVIDIA/KAI-scheduler/pkg/apis/scheduling/v2alpha2"
+	nameutils "github.com/ai-dynamo/grove/operator/api/common"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/utils/ptr"
@@ -38,10 +40,29 @@ type ExpectedSubGroup struct {
 	PreferredTopologyLevel string
 }
 
+// PCSGCliqueConfig defines configuration for a single clique in a PCSG
+type PCSGCliqueConfig struct {
+	Name       string
+	PodCount   int32
+	Constraint string
+}
+
+// ScaledPCSGConfig defines configuration for verifying a scaled PCSG replica
+type ScaledPCSGConfig struct {
+	Name         string
+	PCSGName     string
+	PCSGReplica  int
+	CliqueConfigs []PCSGCliqueConfig
+	Constraint   string
+}
+
 // CreateExpectedStandalonePCLQSubGroup creates an ExpectedSubGroup for a standalone PodClique (not in PCSG)
 // Name format: <pcs-name>-<pcs-replica>-<clique-name>
 func CreateExpectedStandalonePCLQSubGroup(pcsName string, pcsReplica int, cliqueName string, minMember int32, topologyLevel string) ExpectedSubGroup {
-	name := GetStandalonePCLQSubGroupName(pcsName, pcsReplica, cliqueName)
+	name := nameutils.GeneratePodCliqueName(
+		nameutils.ResourceNameReplica{Name: pcsName, Replica: pcsReplica},
+		cliqueName,
+	)
 	return ExpectedSubGroup{
 		Name:                  name,
 		MinMember:             minMember,
@@ -53,7 +74,11 @@ func CreateExpectedStandalonePCLQSubGroup(pcsName string, pcsReplica int, clique
 // CreateExpectedPCSGParentSubGroup creates an ExpectedSubGroup for a PCSG parent (scaling group replica)
 // Name format: <pcs-name>-<pcs-replica>-<sg-name>-<sg-replica>
 func CreateExpectedPCSGParentSubGroup(pcsName string, pcsReplica int, sgName string, sgReplica int, topologyLevel string) ExpectedSubGroup {
-	name := GetPCSGParentSubGroupName(pcsName, pcsReplica, sgName, sgReplica)
+	pcsgFQN := nameutils.GeneratePodCliqueScalingGroupName(
+		nameutils.ResourceNameReplica{Name: pcsName, Replica: pcsReplica},
+		sgName,
+	)
+	name := fmt.Sprintf("%s-%d", pcsgFQN, sgReplica)
 	return ExpectedSubGroup{
 		Name:                  name,
 		MinMember:             0,
@@ -62,19 +87,41 @@ func CreateExpectedPCSGParentSubGroup(pcsName string, pcsReplica int, sgName str
 	}
 }
 
-// CreateExpectedPCLQInPCSGSubGroup creates an ExpectedSubGroup for a PodClique within a PCSG
+// CreateExpectedPCLQInPCSGSubGroup creates an ExpectedSubGroup for a PodClique within a PCSG with parent
 // Name format: <pcs-name>-<pcs-replica>-<sg-name>-<sg-replica>-<clique-name>
 func CreateExpectedPCLQInPCSGSubGroup(pcsName string, pcsReplica int, sgName string, sgReplica int, cliqueName string, minMember int32, topologyLevel string) ExpectedSubGroup {
-	name := GetPCLQInPCSGSubGroupName(pcsName, pcsReplica, sgName, sgReplica, cliqueName)
-	parentName := GetPCSGParentSubGroupName(pcsName, pcsReplica, sgName, sgReplica)
+	return createExpectedPCLQInPCSGSubGroup(pcsName, pcsReplica, sgName, sgReplica, cliqueName, minMember, topologyLevel, true)
+}
+
+func createExpectedPCLQInPCSGSubGroup(pcsName string, pcsReplica int, sgName string, sgReplica int, cliqueName string,
+	minMember int32, topologyLevel string, hasParent bool) ExpectedSubGroup {
+	pcsgFQN := nameutils.GeneratePodCliqueScalingGroupName(
+		nameutils.ResourceNameReplica{Name: pcsName, Replica: pcsReplica},
+		sgName,
+	)
+	name := nameutils.GeneratePodCliqueName(
+		nameutils.ResourceNameReplica{Name: pcsgFQN, Replica: sgReplica},
+		cliqueName,
+	)
+	var parentPtr *string
+	if hasParent {
+		parentPtr = ptr.To(fmt.Sprintf("%s-%d", pcsgFQN, sgReplica))
+	}
 	return ExpectedSubGroup{
 		Name:                  name,
 		MinMember:             minMember,
-		Parent:                ptr.To(parentName),
+		Parent:                parentPtr,
 		RequiredTopologyLevel: topologyLevel,
 	}
 }
 
+// CreateExpectedPCLQInPCSGSubGroupNoParent creates an ExpectedSubGroup for a PodClique within a PCSG without parent
+// Used when PCSG has no topology constraint (no parent SubGroup created)
+// Name format: <pcs-name>-<pcs-replica>-<sg-name>-<sg-replica>-<clique-name>
+func CreateExpectedPCLQInPCSGSubGroupNoParent(pcsName string, pcsReplica int, sgName string, sgReplica int, cliqueName string, minMember int32, topologyLevel string) ExpectedSubGroup {
+	return createExpectedPCLQInPCSGSubGroup(pcsName, pcsReplica, sgName, sgReplica, cliqueName, minMember, topologyLevel, false)
+}
+
 // GetKAIPodGroupsForPCS retrieves all KAI PodGroups for a given PodCliqueSet by label selector
 // KAI scheduler creates PodGroups with label: app.kubernetes.io/part-of=<pcs-name>
 // Returns a list of PodGroups that tests can filter by owner reference if needed
@@ -233,7 +280,7 @@ func GetPodGroupForBasePodGangReplica(
 		return nil, fmt.Errorf("failed to get KAI PodGroups: %w", err)
 	}
 
-	basePodGangName := GetBasePodGangName(workloadName, pgsReplica)
+	basePodGangName := nameutils.GenerateBasePodGangName(nameutils.ResourceNameReplica{Name: workloadName, Replica: pgsReplica})
 	basePodGroup, err := FilterPodGroupByOwner(podGroups, basePodGangName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find PodGroup for PodGang %s: %w", basePodGangName, err)
@@ -259,3 +306,51 @@ func VerifyPodGroupTopology(
 
 	return nil
 }
+
+// VerifyScaledPCSGReplicaTopology verifies KAI PodGroup for ONE scaled PCSG replica.
+// Scaled PodGroup top-level constraint: uses pcsConstraint ONLY if PCSG has NO constraint.
+func VerifyScaledPCSGReplicaTopology(
+	ctx context.Context,
+	t *testing.T,
+	dynamicClient dynamic.Interface,
+	namespace string,
+	pcsName string,
+	pcsReplica int,
+	pcsgConfig ScaledPCSGConfig,
+	pcsConstraint string,
+	logger *Logger,
+) {
+	podGroups, err := GetKAIPodGroupsForPCS(ctx, dynamicClient, namespace, pcsName)
+	if err != nil {
+		t.Fatalf("Failed to get KAI PodGroups: %v", err)
+	}
+
+	pcsgFQN := nameutils.GeneratePodCliqueScalingGroupName(
+		nameutils.ResourceNameReplica{Name: pcsName, Replica: pcsReplica},
+		pcsgConfig.PCSGName,
+	)
+
+	scaledPodGangName := nameutils.CreatePodGangNameFromPCSGFQN(pcsgFQN, pcsgConfig.PCSGReplica-1)
+
+	scaledPodGroup, err := FilterPodGroupByOwner(podGroups, scaledPodGangName)
+	if err != nil {
+		t.Fatalf("Failed to find scaled PodGroup for %s: %v", scaledPodGangName, err)
+	}
+
+	var expectedSubGroups []ExpectedSubGroup
+
+	for _, cliqueConfig := range pcsgConfig.CliqueConfigs {
+		expectedSubGroups = append(expectedSubGroups,
+			CreateExpectedPCLQInPCSGSubGroupNoParent(pcsName, pcsReplica, pcsgConfig.PCSGName, pcsgConfig.PCSGReplica, cliqueConfig.Name, cliqueConfig.PodCount, cliqueConfig.Constraint))
+	}
+
+	scaledTopConstraint := pcsConstraint
+	if pcsgConfig.Constraint != "" {
+		scaledTopConstraint = pcsgConfig.Constraint
+	}
+
+	if err := VerifyPodGroupTopology(scaledPodGroup, scaledTopConstraint, "", expectedSubGroups, logger); err != nil {
+		t.Fatalf("Failed to verify scaled PodGroup %s (%s replica %d) topology: %v",
+			scaledPodGangName, pcsgConfig.Name, pcsgConfig.PCSGReplica, err)
+	}
+}

operator/e2e/utils/topology.go

@@ -203,3 +203,69 @@ func VerifyLabeledPodsInTopologyDomain(
 
 	return VerifyPodsInSameTopologyDomain(ctx, clientset, filteredPods, topologyKey, logger)
 }
+
+// PCSGTypeConfig defines configuration for a PCSG type verification
+type PCSGTypeConfig struct {
+	Name string // Human-readable name (e.g., "decoder")
+	FQN  string // Fully-qualified PCSG name
+}
+
+// VerifyPCSGReplicasInTopologyDomain verifies that each PCSG replica's pods
+// are in the same topology domain (e.g., rack, host).
+func VerifyPCSGReplicasInTopologyDomain(
+	ctx context.Context,
+	clientset kubernetes.Interface,
+	allPods []v1.Pod,
+	pcsgLabel string,
+	replicaCount int,
+	podsPerReplica int,
+	topologyLabel string,
+	logger *Logger,
+) error {
+	for replica := 0; replica < replicaCount; replica++ {
+		replicaPods := FilterPodsByLabel(
+			FilterPodsByLabel(allPods, "grove.io/podcliquescalinggroup", pcsgLabel),
+			"grove.io/podcliquescalinggroup-replica-index",
+			fmt.Sprintf("%d", replica),
+		)
+		if len(replicaPods) != podsPerReplica {
+			return fmt.Errorf("expected %d PCSG replica %d pods, got %d", podsPerReplica, replica, len(replicaPods))
+		}
+		if err := VerifyPodsInSameTopologyDomain(ctx, clientset, replicaPods, topologyLabel, logger); err != nil {
+			return fmt.Errorf("failed to verify PCSG replica %d pods in same topology domain: %w", replica, err)
+		}
+	}
+	return nil
+}
+
+// VerifyMultiTypePCSGReplicas verifies multiple PCSG types across replicas.
+// Each PCSG type has multiple replicas, and each replica's pods should be in the same topology domain.
+func VerifyMultiTypePCSGReplicas(
+	ctx context.Context,
+	clientset kubernetes.Interface,
+	allPods []v1.Pod,
+	pcsgTypes []PCSGTypeConfig,
+	replicasPerType int,
+	podsPerReplica int,
+	topologyLabel string,
+	logger *Logger,
+) error {
+	for _, pcsgType := range pcsgTypes {
+		for replica := 0; replica < replicasPerType; replica++ {
+			replicaPods := FilterPodsByLabel(
+				FilterPodsByLabel(allPods, "grove.io/podcliquescalinggroup", pcsgType.FQN),
+				"grove.io/podcliquescalinggroup-replica-index",
+				fmt.Sprintf("%d", replica),
+			)
+			if len(replicaPods) != podsPerReplica {
+				return fmt.Errorf("expected %d %s replica-%d pods, got %d",
+					podsPerReplica, pcsgType.Name, replica, len(replicaPods))
+			}
+			if err := VerifyPodsInSameTopologyDomain(ctx, clientset, replicaPods, topologyLabel, logger); err != nil {
+				return fmt.Errorf("failed to verify %s replica-%d pods in same topology domain: %w",
+					pcsgType.Name, replica, err)
+			}
+		}
+	}
+	return nil
+}

operator/e2e/yaml/tas-hierarchy.yaml

@@ -0,0 +1,88 @@
+# Workload 8: SP-1 - Full 3-Level Hierarchy with Cascading Constraints
+# Test scenario: PCS (block) → PCSG (rack) → PCLQ (host) - demonstrating constraint inheritance
+---
+apiVersion: grove.io/v1alpha1
+kind: PodCliqueSet
+metadata:
+  name: tas-hierarchy
+  labels:
+    app: tas-hierarchy
+spec:
+  replicas: 1
+  template:
+    topologyConstraint:
+      packDomain: block  # PCS level - broadest
+    podCliqueScalingGroups:
+      - name: inference-group
+        replicas: 2
+        minAvailable: 2
+        topologyConstraint:
+          packDomain: rack  # PCSG level - stricter than parent
+        cliqueNames:
+          - prefill
+          - decode
+    cliques:
+      - name: prefill
+        labels:
+          kai.scheduler/queue: test
+        topologyConstraint:
+          packDomain: host  # PCLQ level - strictest
+        spec:
+          roleName: prefill
+          replicas: 2
+          minAvailable: 2
+          podSpec:
+            schedulerName: kai-scheduler
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                    - matchExpressions:
+                        - key: node_role.e2e.grove.nvidia.com
+                          operator: In
+                          values:
+                            - agent
+            tolerations:
+              - key: node_role.e2e.grove.nvidia.com
+                operator: Equal
+                value: agent
+                effect: NoSchedule
+            containers:
+              - name: prefill
+                image: registry:5001/busybox:latest
+                command: ["sleep", "infinity"]
+                resources:
+                  requests:
+                    memory: 40Mi
+      - name: decode
+        labels:
+          kai.scheduler/queue: test
+        topologyConstraint:
+          packDomain: host  # PCLQ level - strictest
+        spec:
+          roleName: decode
+          replicas: 2
+          minAvailable: 2
+          podSpec:
+            schedulerName: kai-scheduler
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                    - matchExpressions:
+                        - key: node_role.e2e.grove.nvidia.com
+                          operator: In
+                          values:
+                            - agent
+            tolerations:
+              - key: node_role.e2e.grove.nvidia.com
+                operator: Equal
+                value: agent
+                effect: NoSchedule
+            containers:
+              - name: decode
+                image: registry:5001/busybox:latest
+                command: ["sleep", "infinity"]
+                resources:
+                  requests:
+                    memory: 40Mi

operator/e2e/yaml/tas-host-level.yaml

@@ -38,7 +38,9 @@ spec:
                 effect: NoSchedule
             containers:
               - name: worker
-                image: registry:5001/nginx:alpine-slim
+                image: registry:5001/busybox:latest
+                command: ["sleep", "infinity"]
                 resources:
                   requests:
-                    memory: 30Mi
+                    memory: 40Mi
+