fix: optimize default placement and scoring, improve k8s version compatibility, karpenter label/annotation missed issue (#384)

* fix: lint issue

* fix: add node compact gpu low load first mode, refactor test scoring

* fix: default placement mode issue

* fix: use ld.so.conf.d for ld lib path for client/ngpu, add apparmor config for hypervisor

* fix: karpenter label missing issue

Author: Code2Life

.vscode/settings.json

@@ -110,6 +110,7 @@
         "nodeclassref",
         "noderesources",
         "nolint",
+        "NUMA",
         "Nvlink",
         "NVML",
         "objs",

api/v1/schedulingconfigtemplate_types.go

@@ -42,7 +42,7 @@ type SchedulingConfigTemplateSpec struct {
 }
 
 type PlacementConfig struct {
-	// +kubebuilder:default=CompactFirst
+	// +kubebuilder:default=NodeCompactGPULowLoad
 	Mode PlacementMode `json:"mode"`
 
 	// +kubebuilder:default=true
@@ -53,7 +53,7 @@ type PlacementConfig struct {
 	GPUFilters []GPUFilter `json:"gpuFilters,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=CompactFirst;LowLoadFirst
+// +kubebuilder:validation:Enum=CompactFirst;LowLoadFirst;NodeCompactGPULowLoad
 type PlacementMode string
 
 const (
@@ -62,6 +62,9 @@ const (
 
 	// in some cases, use lowLoadFirst for balance and fairness
 	PlacementModeLowLoadFirst PlacementMode = "LowLoadFirst"
+
+	// in some cases, use nodeCompactGPULowLoad for balance and fairness
+	PlacementModeNodeCompactGPULowLoad PlacementMode = "NodeCompactGPULowLoad"
 )
 
 // GPUFilter is to select eligible GPUs for scheduling.

charts/tensor-fusion/crds/tensor-fusion.ai_schedulingconfigtemplates.yaml

@@ -277,10 +277,11 @@ spec:
                       type: object
                     type: array
                   mode:
-                    default: CompactFirst
+                    default: NodeCompactGPULowLoad
                     enum:
                     - CompactFirst
                     - LowLoadFirst
+                    - NodeCompactGPULowLoad
                     type: string
                 required:
                 - mode

cmd/main.go

@@ -109,7 +109,7 @@ func init() {
 	}
 	karpenterScheme.Register(&karpv1.NodeClaim{}, &karpv1.NodeClaimList{})
 	karpenterScheme.Register(&karpv1.NodePool{}, &karpv1.NodePoolList{})
-	karpenterScheme.AddToScheme(scheme)
+	utilruntime.Must(karpenterScheme.AddToScheme(scheme))
 }
 
 //nolint:gocyclo

config/crd/bases/tensor-fusion.ai_schedulingconfigtemplates.yaml

@@ -277,10 +277,11 @@ spec:
                       type: object
                     type: array
                   mode:
-                    default: CompactFirst
+                    default: NodeCompactGPULowLoad
                     enum:
                     - CompactFirst
                     - LowLoadFirst
+                    - NodeCompactGPULowLoad
                     type: string
                 required:
                 - mode

internal/constants/env.go

@@ -67,12 +67,16 @@ const (
 	RealNvmlLibPathValue = "/lib/x86_64-linux-gnu/libnvidia-ml.so.1"
 	RealCUDALibPathValue = "/lib/x86_64-linux-gnu/libcuda.so"
 
-	PrependPathEnv          = "TF_PREPEND_PATH"
-	PrependLDLibraryPathEnv = "TF_PREPEND_LD_LIBRARY_PATH"
+	PrependPathEnv = "TF_PREPEND_PATH"
+
+	RunInsideGPUEnv = "RUN_INSIDE_GPU_NODE"
 
 	LdPreloadFileName = "ld.so.preload"
 	LdPreloadFile     = "/etc/ld.so.preload"
 
+	LdLibraryPathFileName = "zz_tensor-fusion.conf"
+	LdLibraryPathFile     = "/etc/ld.so.conf.d/zz_tensor-fusion.conf"
+
 	TFLibsVolumeName         = "tf-libs"
 	TFLibsVolumeMountPath    = "/tensor-fusion"
 	TFConnectionNamePrefix   = "-tf-vgpu-"

internal/gpuallocator/filter/node_filter.go

@@ -25,6 +25,7 @@ func NewSameNodeFilter(count uint) *SameNodeFilter {
 
 // Filter implements GPUFilter.Filter
 // It groups GPUs by node and returns only those nodes that have at least 'count' GPUs
+// Must run at last step, otherwise some nodes and gpus passed may not valid at following steps
 func (f *SameNodeFilter) Filter(ctx context.Context, workerPodKey tfv1.NameNamespace, gpus []*tfv1.GPU) ([]*tfv1.GPU, error) {
 	// If count is 1 or 0, no need to filter by node
 	if f.count <= 1 {

internal/gpuallocator/gpuallocator.go

@@ -44,7 +44,9 @@ var mu sync.Mutex
 var GPUCapacityMap = map[string]tfv1.Resource{}
 
 type Strategy interface {
-	Score(gpu *tfv1.GPU) int
+	// When isForNode = true, indicates each GPU's node level score
+	// otherwise it's single GPU score inside one node
+	Score(gpu *tfv1.GPU, isForNode bool) int
 
 	SelectGPUs(gpus []*tfv1.GPU, count uint) ([]*tfv1.GPU, error)
 }
@@ -59,13 +61,14 @@ func (p *SimulateSchedulingFilterDetail) Clone() fwk.StateData {
 }
 
 // NewStrategy creates a strategy based on the placement mode
-func NewStrategy(placementMode tfv1.PlacementMode, cfg *config.GPUFitConfig) Strategy {
+func NewStrategy(placementMode tfv1.PlacementMode, cfg *config.GPUFitConfig, nodeGpuStore map[string]map[string]*tfv1.GPU) Strategy {
 	switch placementMode {
 	case tfv1.PlacementModeLowLoadFirst:
-		return LowLoadFirst{cfg: cfg}
+		return LowLoadFirst{cfg: cfg, nodeGpuStore: nodeGpuStore}
+	case tfv1.PlacementModeCompactFirst:
+		return CompactFirst{cfg: cfg, nodeGpuStore: nodeGpuStore}
 	default:
-		// CompactFirst is the default strategy
-		return CompactFirst{cfg: cfg}
+		return NodeCompactGPULowLoad{cfg: cfg, nodeGpuStore: nodeGpuStore}
 	}
 }
 
@@ -182,14 +185,16 @@ func (s *GpuAllocator) Filter(
 		filterRegistry = filterRegistry.With(filter.NewGPUModelFilter(req.GPUModel))
 	}
 
-	if req.Count > 1 {
-		filterRegistry = filterRegistry.With(filter.NewSameNodeFilter(req.Count))
-	}
-	// Add NodeAffinityFilter if specified
+	// NOTE: deprecated, use Kubernetes native spec template affinity way
 	if req.NodeAffinity != nil {
 		filterRegistry = filterRegistry.With(filter.NewNodeAffinityFilter(s.Client, req.NodeAffinity))
 	}
 
+	// Same node filter must be applied at final step
+	if req.Count > 1 {
+		filterRegistry = filterRegistry.With(filter.NewSameNodeFilter(req.Count))
+	}
+
 	// Apply the filters in sequence
 	filteredGPUs, filterDetails, err := filterRegistry.Apply(s.ctx, req.WorkloadNameNamespace, toFilterGPUs, isSimulateSchedule)
 	if err != nil {
@@ -245,7 +250,7 @@ func (s *GpuAllocator) Select(req *tfv1.AllocRequest, filteredGPUs []*tfv1.GPU)
 
 	strategy := NewStrategy(schedulingConfigTemplate.Spec.Placement.Mode, &config.GPUFitConfig{
 		MaxWorkerPerNode: s.maxWorkerPerNode,
-	})
+	}, s.nodeGpuStore)
 	selectedGPUs, err := strategy.SelectGPUs(filteredGPUs, req.Count)
 	if err != nil {
 		return nil, fmt.Errorf("select GPU: %w", err)
@@ -670,18 +675,20 @@ type scoredGPU struct {
 	score    int
 }
 
+func (s *GpuAllocator) GetScoringStrategy(cfg *config.GPUFitConfig, req *tfv1.AllocRequest) Strategy {
+	return NewStrategy(s.getPlacementMode(s.ctx, req.PoolName), cfg, s.nodeGpuStore)
+}
+
 // First level is k8s node name, second level is GPU name, value is score
 func (s *GpuAllocator) Score(
-	ctx context.Context, cfg *config.GPUFitConfig, req *tfv1.AllocRequest, nodeGPUs map[string][]*tfv1.GPU,
+	ctx context.Context, strategy Strategy, req *tfv1.AllocRequest, nodeGPUs map[string][]*tfv1.GPU,
 ) map[string]map[string]int {
 	result := make(map[string]map[string]int, len(nodeGPUs))
-	strategy := NewStrategy(s.getPlacementMode(ctx, req.PoolName), cfg)
-
 	allScores := make([]scoredGPU, 0, len(nodeGPUs))
 
 	for nodeName, gpus := range nodeGPUs {
 		for _, gpu := range gpus {
-			res := strategy.Score(gpu)
+			res := strategy.Score(gpu, true)
 
 			// making Pending GPU to lower score, prefer not scheduling to them
 			if gpu.Status.Phase == tfv1.TensorFusionGPUPhasePending {
@@ -1477,18 +1484,18 @@ func (s *GpuAllocator) getPlacementMode(ctx context.Context, poolName string) tf
 	pool := &tfv1.GPUPool{}
 	if err := s.Get(ctx, client.ObjectKey{Name: poolName}, pool); err != nil {
 		// if failed to get pool, default to compact first
-		return tfv1.PlacementModeCompactFirst
+		return tfv1.PlacementModeNodeCompactGPULowLoad
 	}
 
 	if pool.Spec.SchedulingConfigTemplate == nil || *pool.Spec.SchedulingConfigTemplate == "" {
-		return tfv1.PlacementModeCompactFirst
+		return tfv1.PlacementModeNodeCompactGPULowLoad
 	}
 
 	// get scheduling config template
 	schedulingConfigTemplate := &tfv1.SchedulingConfigTemplate{}
 	if err := s.Get(ctx, client.ObjectKey{Name: *pool.Spec.SchedulingConfigTemplate}, schedulingConfigTemplate); err != nil {
 		// if failed to get scheduling config template, default to compact first
-		return tfv1.PlacementModeCompactFirst
+		return tfv1.PlacementModeNodeCompactGPULowLoad
 	}
 	return schedulingConfigTemplate.Spec.Placement.Mode
 }