fix: support vram hard-isolation (#414)

Author: Code2Life

internal/constants/env.go

@@ -122,8 +122,11 @@ const (
 	DisableVRAMManagerEnv      = "TF_DISABLE_MEMORY_MANAGER"
 	DisableWorkerFeatureEnvVal = "1"
 
-	// hard limiter mode
+	// hard limiter mode (not open sourced) in percent, only take effect on worker container yet
 	HardSMLimiterEnv = "TF_CUDA_SM_PERCENT_LIMIT"
+	// hard limiter (not open sourced) in megabytes, only take effect on worker container and when open source vgpu.rs gpu-limiter is disabled
+	// when use this mode, memory request can not autoscale dynamically
+	HardMemLimiterEnv = "TF_CUDA_MEMORY_LIMIT"
 
 	TensorFusionRemoteWorkerPortNumber = 8000
 	TensorFusionRemoteWorkerPortName   = "remote-vgpu"

internal/utils/compose.go

@@ -804,9 +804,6 @@ func SetWorkerContainerSpec(
 	}, v1.EnvVar{
 		Name:  constants.ContainerNameEnv,
 		Value: constants.TFContainerNameWorker,
-	}, v1.EnvVar{
-		Name:  constants.LdPreloadEnv,
-		Value: constants.LdPreloadLimiter,
 	}, v1.EnvVar{
 		Name: constants.PodNamespaceEnv,
 		ValueFrom: &v1.EnvVarSource{
@@ -816,15 +813,28 @@ func SetWorkerContainerSpec(
 		},
 	})
 
+	if !strings.Contains(disabledFeatures, constants.BuiltInFeaturesGpuLimiter) &&
+		workloadProfile.ComputeIsolation != constants.ComputingIsolationModeHard {
+		container.Env = append(container.Env, v1.EnvVar{
+			Name:  constants.LdPreloadEnv,
+			Value: constants.LdPreloadLimiter,
+		})
+	}
+
 	if disabledFeatures != "" {
 		container.Env = convertDisabledFeaturesToEnvs(disabledFeatures, container.Env)
 	}
 
 	// TODO should calculate and set by hypervisor before container created
+	// when compute isolation mode is hard-isolation, memory limit also change to hard-mode
+	// open source vgpu.rs memory limiter is feedback-loop based, potentially cause resource contention
 	if workloadProfile.ComputeIsolation == constants.ComputingIsolationModeHard {
 		container.Env = append(container.Env, v1.EnvVar{
 			Name:  constants.HardSMLimiterEnv,
 			Value: workloadProfile.Resources.Limits.ComputePercent.String(),
+		}, v1.EnvVar{
+			Name:  constants.HardMemLimiterEnv,
+			Value: strconv.FormatInt(workloadProfile.Resources.Limits.Vram.Value()/(1024*1024), 10),
 		})
 	}