fix: capacity type prefer spot instance (#58)

Code2Life · web-flow · commit fadaeae1bdab · 2025-03-03T11:09:39.000+08:00
* fix: manual compaction

* fix: capacity type selection
diff --git a/api/v1/gpunode_types.go b/api/v1/gpunode_types.go
@@ -72,6 +72,11 @@ type GPUNodeStatus struct {
 	AvailableTFlops resource.Quantity `json:"availableTFlops"`
 	AvailableVRAM   resource.Quantity `json:"availableVRAM"`
 
+	// +optional
+	VirtualAvailableTFlops *resource.Quantity `json:"virtualAvailableTFlops,omitempty"`
+	// +optional
+	VirtualAvailableVRAM *resource.Quantity `json:"virtualAvailableVRAM,omitempty"`
+
 	// +optional
 	HypervisorStatus NodeHypervisorStatus `json:"hypervisorStatus,omitempty"`
 
diff --git a/api/v1/gpupool_types.go b/api/v1/gpupool_types.go
@@ -375,6 +375,11 @@ type GPUPoolStatus struct {
 	AvailableTFlops resource.Quantity `json:"availableTFlops"`
 	AvailableVRAM   resource.Quantity `json:"availableVRAM"`
 
+	// +optional
+	VirtualAvailableTFlops *resource.Quantity `json:"virtualAvailableTFlops,omitempty"`
+	// +optional
+	VirtualAvailableVRAM *resource.Quantity `json:"virtualAvailableVRAM,omitempty"`
+
 	// when updating any component version or config, pool controller will perform rolling update.
 	// the status will be updated periodically, default to 5s, progress will be 0-100.
 	// when the progress is 100, the component version or config is fully updated.
@@ -397,6 +402,9 @@ type GPUPoolStatus struct {
 	// If this field is not empty, scheduler will not schedule new AI workloads and stop scaling-up check.
 	// TODO not implemented yet
 	BudgetExceeded string `json:"budgetExceeded,omitempty"`
+
+	// +optional
+	LastCompactionTime *metav1.Time `json:"lastCompactionTime,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Pending;Running;Updating;Destroying;Unknown
diff --git a/api/v1/tensorfusioncluster_funcs.go b/api/v1/tensorfusioncluster_funcs.go
@@ -117,6 +117,9 @@ func (tfc *TensorFusionCluster) RefreshStatus(ownedPools []GPUPool) {
 	tfc.Status.VirtualTFlops = resource.Quantity{}
 	tfc.Status.VirtualVRAM = resource.Quantity{}
 
+	tfc.Status.VirtualAvailableTFlops = &resource.Quantity{}
+	tfc.Status.VirtualAvailableVRAM = &resource.Quantity{}
+
 	for i, gpuPool := range ownedPools {
 		if gpuPool.Status.Phase != constants.PhaseRunning {
 			tfc.Status.NotReadyGPUPools = append(tfc.Status.NotReadyGPUPools, gpuPool.Name)
@@ -132,5 +135,12 @@ func (tfc *TensorFusionCluster) RefreshStatus(ownedPools []GPUPool) {
 
 		tfc.Status.VirtualTFlops.Add(gpuPool.Status.VirtualTFlops)
 		tfc.Status.VirtualVRAM.Add(gpuPool.Status.VirtualVRAM)
+
+		if gpuPool.Status.VirtualAvailableTFlops != nil {
+			tfc.Status.VirtualAvailableTFlops.Add(*gpuPool.Status.VirtualAvailableTFlops)
+		}
+		if gpuPool.Status.VirtualAvailableVRAM != nil {
+			tfc.Status.VirtualAvailableVRAM.Add(*gpuPool.Status.VirtualAvailableVRAM)
+		}
 	}
 }
diff --git a/api/v1/tensorfusioncluster_types.go b/api/v1/tensorfusioncluster_types.go
@@ -58,6 +58,11 @@ type TensorFusionClusterStatus struct {
 	AvailableTFlops resource.Quantity `json:"availableTFlops"`
 	AvailableVRAM   resource.Quantity `json:"availableVRAM"`
 
+	// +optional
+	VirtualAvailableTFlops *resource.Quantity `json:"virtualAvailableTFlops,omitempty"`
+	// +optional
+	VirtualAvailableVRAM *resource.Quantity `json:"virtualAvailableVRAM,omitempty"`
+
 	// +optional
 	ReadyGPUPools []string `json:"readyGPUPools"`
 
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/charts/tensor-fusion/crds/tensor-fusion.ai_gpunodes.yaml b/charts/tensor-fusion/crds/tensor-fusion.ai_gpunodes.yaml
@@ -313,6 +313,18 @@ spec:
                 - type: string
                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                 x-kubernetes-int-or-string: true
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/charts/tensor-fusion/crds/tensor-fusion.ai_gpupools.yaml b/charts/tensor-fusion/crds/tensor-fusion.ai_gpupools.yaml
@@ -866,6 +866,9 @@ spec:
                   - type
                   type: object
                 type: array
+              lastCompactionTime:
+                format: date-time
+                type: string
               notReadyNodes:
                 format: int32
                 type: integer
@@ -909,6 +912,18 @@ spec:
                 type: string
               utilizedVRAMPercent:
                 type: string
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/charts/tensor-fusion/crds/tensor-fusion.ai_tensorfusionclusters.yaml b/charts/tensor-fusion/crds/tensor-fusion.ai_tensorfusionclusters.yaml
@@ -1021,6 +1021,18 @@ spec:
                 type: string
               utilizedVRAMPercent:
                 type: string
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/config/crd/bases/tensor-fusion.ai_gpunodes.yaml b/config/crd/bases/tensor-fusion.ai_gpunodes.yaml
@@ -313,6 +313,18 @@ spec:
                 - type: string
                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                 x-kubernetes-int-or-string: true
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/config/crd/bases/tensor-fusion.ai_gpupools.yaml b/config/crd/bases/tensor-fusion.ai_gpupools.yaml
@@ -866,6 +866,9 @@ spec:
                   - type
                   type: object
                 type: array
+              lastCompactionTime:
+                format: date-time
+                type: string
               notReadyNodes:
                 format: int32
                 type: integer
@@ -909,6 +912,18 @@ spec:
                 type: string
               utilizedVRAMPercent:
                 type: string
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/config/crd/bases/tensor-fusion.ai_tensorfusionclusters.yaml b/config/crd/bases/tensor-fusion.ai_tensorfusionclusters.yaml
@@ -1021,6 +1021,18 @@ spec:
                 type: string
               utilizedVRAMPercent:
                 type: string
+              virtualAvailableTFlops:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              virtualAvailableVRAM:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               virtualTFlops:
                 anyOf:
                 - type: integer
diff --git a/internal/cloudprovider/alibaba/ecs.go b/internal/cloudprovider/alibaba/ecs.go
@@ -208,8 +208,8 @@ func handleNodeClassAndExtraParams(request *ecs.RunInstancesRequest, param *type
 	request.UserData = base64.StdEncoding.EncodeToString([]byte(strings.ReplaceAll(nodeClass.UserData, constants.ProvisionerNamePlaceholder, param.NodeName)))
 
 	// Handle extra params
-	capacityType := param.ExtraParams[string(tfv1.NodeRequirementKeyCapacityType)]
-	if capacityType != "" && capacityType != string(types.CapacityTypeOnDemand) {
+	capacityType := param.CapacityType
+	if capacityType != "" && capacityType != types.CapacityTypeOnDemand {
 		// Convert from Spot/OnDemand to each cloud vendor's equivalent
 		if param.ExtraParams["spotPriceLimit"] != "" {
 			priceLimit, err := strconv.ParseFloat(param.ExtraParams["spotPriceLimit"], 64)
diff --git a/internal/cloudprovider/alibaba/pricing.go b/internal/cloudprovider/alibaba/pricing.go
@@ -25,21 +25,22 @@ func init() {
 	// https://github.com/aws/karpenter-provider-aws/blob/2cb43bc468d04f2263d43109e0ccc7cbe616e226/pkg/providers/pricing/zz_generated.pricing_aws.go
 	// https://github.com/cloudpilot-ai/karpenter-provider-alibabacloud/blob/main/hack/tools/price_gen/price_gen.go
 	GPUInstanceTypeInfo = []types.GPUNodeInstanceInfo{
-		{
-			InstanceType: "ecs.gn6i-c4g1.xlarge",
-			CostPerHour:  1.152,
-
-			CPUs:      4,
-			MemoryGiB: 15,
-
-			FP16TFlopsPerGPU:    65,
-			VRAMGigabytesPerGPU: 15,
-
-			GPUModel:        "NVIDIA T4",
-			GPUCount:        1,
-			GPUArchitecture: types.GPUArchitectureNvidiaTuring,
-			CPUArchitecture: types.CPUArchitectureAMD64,
-		},
+		// Out of stock
+		// {
+		// 	InstanceType: "ecs.gn6i-c4g1.xlarge",
+		// 	CostPerHour:  1.152,
+
+		// 	CPUs:      4,
+		// 	MemoryGiB: 15,
+
+		// 	FP16TFlopsPerGPU:    65,
+		// 	VRAMGigabytesPerGPU: 15,
+
+		// 	GPUModel:        "NVIDIA T4",
+		// 	GPUCount:        1,
+		// 	GPUArchitecture: types.GPUArchitectureNvidiaTuring,
+		// 	CPUArchitecture: types.CPUArchitectureAMD64,
+		// },
 
 		{
 			InstanceType: "ecs.gn6i-c8g1.2xlarge",
diff --git a/internal/constants/constants.go b/internal/constants/constants.go
@@ -95,3 +95,4 @@ const (
 
 const TFDataPath = "/tmp/tensor-fusion/data"
 const DataVolumeName = "tf-data"
+const TensorFusionPoolManualCompaction = "tensor-fusion.ai/manual-compaction"
diff --git a/internal/controller/gpunode_controller.go b/internal/controller/gpunode_controller.go
@@ -108,7 +108,7 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 
 			if node.Status.NodeInfo.InstanceID == "" {
 				r.Recorder.Eventf(node, corev1.EventTypeWarning, "OrphanedNode", "provisioned node without instanceID, this could result in orphaned nodes, please check manually: %s", node.Name)
-				return false, nil
+				return true, nil
 			}
 			err = (*provider).TerminateNode(ctx, &types.NodeIdentityParam{
 				InstanceID: node.Status.NodeInfo.InstanceID,
diff --git a/internal/controller/gpupool_compaction_controller.go b/internal/controller/gpupool_compaction_controller.go
diff --git a/internal/controller/gpupool_controller.go b/internal/controller/gpupool_controller.go
