fix: node expansion issue, add lib path to env (#402)

* fix: add ld lib path back to env

* fix: node expansion issue

Author: Code2Life

internal/constants/env.go

@@ -73,7 +73,8 @@ const (
 	RealNvmlLibPathValue = "/lib/x86_64-linux-gnu/libnvidia-ml.so.1"
 	RealCUDALibPathValue = "/lib/x86_64-linux-gnu/libcuda.so"
 
-	PrependPathEnv = "TF_PREPEND_PATH"
+	PrependPathEnv    = "TF_PREPEND_PATH"
+	PrependLibPathEnv = "TF_LD_LIBRARY_PATH"
 
 	RunInsideGPUEnv = "RUN_INSIDE_GPU_NODE"
 

internal/scheduler/expander/handler.go

@@ -159,11 +159,16 @@ func (e *NodeExpander) ProcessExpansion(ctx context.Context, pod *corev1.Pod) er
 	// Step 4: Caused by insufficient GPU resources, try find node util it satisfies the pod
 	preScheduled := false
 	for _, gpuNode := range gpuNodes {
+		// when node is not owned by any known provisioner, skip check, util find a node can be expanded
+		if len(gpuNode.OwnerReferences) == 0 {
+			continue
+		}
 		preparedNode, preparedGPUs := e.prepareNewNodesForScheduleAttempt(gpuNode, nodeGPUs[gpuNode.Name])
 		if !e.checkGPUFitForNewNode(pod, preparedGPUs) {
 			continue
 		}
 
+		e.logger.Info("prepare new node for schedule attempt from existing node", "existingNode", gpuNode.Name, "newNode", preparedNode.Name)
 		err = e.createGPUNodeClaim(ctx, pod, preparedNode)
 		if err != nil {
 			return err
@@ -402,7 +407,7 @@ func (e *NodeExpander) createGPUNodeClaim(ctx context.Context, pod *corev1.Pod,
 	}
 	if !isKarpenterNodeClaim && !isGPUNodeClaim {
 		e.logger.Info("node is not owned by any known provisioner, skip expansion", "node", preparedNode.Name)
-		return nil
+		return fmt.Errorf("node is not owned by any known provisioner, skip expansion")
 	}
 	e.logger.Info("start expanding node from existing template node", "tmplNode", preparedNode.Name)
 	if isKarpenterNodeClaim {

internal/utils/compose.go

@@ -196,6 +196,9 @@ func AddTFDefaultClientConfBeforePatch(
 		pod.Spec.Containers[injectContainerIndex].Env = append(pod.Spec.Containers[injectContainerIndex].Env, v1.EnvVar{
 			Name:  constants.PrependPathEnv,
 			Value: constants.TFLibsVolumeMountPath,
+		}, v1.EnvVar{
+			Name:  constants.PrependLibPathEnv,
+			Value: constants.TFLibsVolumeMountPath,
 		})
 
 		// Known issue: glibc ldd config style, does NOT support musl, fortunately, musl rarely used in AI workloads