Fix provisioning mode bugs (#54)

* fix: helm release longer than 63 char issue

* fix: simplify the full name

* fix: provisioning mode issue

* fix: lint in node discovery

Author: Code2Life

.vscode/launch.json

@@ -48,6 +48,14 @@
                 "ENABLE_WEBHOOKS": "false"
             },
             "program": "${workspaceFolder}/cmd/operator/main.go",
+        },
+        {
+            "name": "Debug Test Code",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "console": "integratedTerminal",
+            "program": "${workspaceFolder}/cmd/tmp/main.go",
         }
     ]
 }

.vscode/settings.json

@@ -26,6 +26,7 @@
         "Finalizers",
         "goconst",
         "golint",
+        "gopsutil",
         "gosec",
         "gpunode",
         "gpunodeclasses",
@@ -56,6 +57,7 @@
         "schedulingconfigtemplate",
         "schedulingconfigtemplates",
         "schedulingcorev",
+        "shirou",
         "subresource",
         "tensorfusion",
         "tensorfusionaiv",

api/v1/tensorfusioncluster_funcs.go

@@ -114,6 +114,9 @@ func (tfc *TensorFusionCluster) RefreshStatus(ownedPools []GPUPool) {
 	tfc.Status.AvailableTFlops = resource.Quantity{}
 	tfc.Status.AvailableVRAM = resource.Quantity{}
 
+	tfc.Status.VirtualTFlops = resource.Quantity{}
+	tfc.Status.VirtualVRAM = resource.Quantity{}
+
 	for i, gpuPool := range ownedPools {
 		if gpuPool.Status.Phase != constants.PhaseRunning {
 			tfc.Status.NotReadyGPUPools = append(tfc.Status.NotReadyGPUPools, gpuPool.Name)
@@ -126,5 +129,8 @@ func (tfc *TensorFusionCluster) RefreshStatus(ownedPools []GPUPool) {
 		tfc.Status.TotalVRAM.Add(gpuPool.Status.TotalVRAM)
 		tfc.Status.AvailableTFlops.Add(gpuPool.Status.AvailableTFlops)
 		tfc.Status.AvailableVRAM.Add(gpuPool.Status.AvailableVRAM)
+
+		tfc.Status.VirtualTFlops.Add(gpuPool.Status.VirtualTFlops)
+		tfc.Status.VirtualVRAM.Add(gpuPool.Status.VirtualVRAM)
 	}
 }

charts/tensor-fusion/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.2.4
+version: 1.2.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/tensor-fusion/templates/_helpers.tpl

@@ -15,16 +15,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 If release name contains chart name it will be used as a full name.
 */}}
 {{- define "tensor-fusion.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
+{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*

charts/tensor-fusion/templates/admission-webhooks/job.yaml

@@ -1,7 +1,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ include "tensor-fusion.fullname" . }}-create-admission-webhook-secret
+  name: {{ include "tensor-fusion.fullname" . }}-add-hook-crt
   namespace: {{ include "tensor-fusion.namespace" . }}
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade
@@ -13,7 +13,7 @@ spec:
 {{- end }}
   template:
     metadata:
-      name: {{ include "tensor-fusion.fullname" . }}-create-admission-webhook-secret
+      name: {{ include "tensor-fusion.fullname" . }}-add-hook-crt
     spec:
       containers:
         - name: create

cmd/nodediscovery/main.go

@@ -8,6 +8,10 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
+	"time"
+
+	"github.com/shirou/gopsutil/mem"
 
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	tfv1 "github.com/NexusGPU/tensor-fusion-operator/api/v1"
@@ -43,7 +47,7 @@ func main() {
 		k8sNodeName = os.Getenv("HOSTNAME")
 	}
 
-	k8sclient, err := kubeClient()
+	k8sClient, err := kubeClient()
 	if err != nil {
 		ctrl.Log.Error(err, "unable to create kubeClient")
 		os.Exit(1)
@@ -93,7 +97,7 @@ func main() {
 			Name: gpuNodeName,
 		},
 	}
-	if err := k8sclient.Get(ctx, client.ObjectKeyFromObject(gpunode), gpunode); err != nil {
+	if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(gpunode), gpunode); err != nil {
 		ctrl.Log.Error(err, "unable to get gpuNode")
 		os.Exit(1)
 	}
@@ -103,6 +107,8 @@ func main() {
 	availableTFlops := resource.MustParse("0")
 	availableVRAM := resource.MustParse("0Ki")
 
+	allDeviceIDs := make([]string, 0)
+
 	for i := 0; i < count; i++ {
 		device, ret := nvml.DeviceGetHandleByIndex(i)
 		if ret != nvml.SUCCESS {
@@ -122,6 +128,8 @@ func main() {
 			os.Exit(1)
 		}
 
+		allDeviceIDs = append(allDeviceIDs, uuid)
+
 		memInfo, ret := device.GetMemoryInfo_v2()
 		if ret != nvml.SUCCESS {
 			ctrl.Log.Error(errors.New(nvml.ErrorString(ret)), "unable to get memory info of device", "index", i)
@@ -137,6 +145,12 @@ func main() {
 		gpu := &tfv1.GPU{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: uuid,
+				Labels: map[string]string{
+					constants.LabelKeyOwner: gpunode.Name,
+				},
+				Annotations: map[string]string{
+					constants.GPULastReportTimeAnnotationKey: time.Now().Format(time.RFC3339),
+				},
 			},
 		}
 
@@ -157,7 +171,7 @@ func main() {
 				"kubernetes.io/hostname": k8sNodeName,
 			},
 		}
-		_, err = controllerutil.CreateOrUpdate(ctx, k8sclient, gpu, func() error { return nil })
+		_, err = controllerutil.CreateOrUpdate(ctx, k8sClient, gpu, func() error { return nil })
 		if err != nil {
 			ctrl.Log.Error(err, "failed to create GPU", "gpu", gpu)
 			os.Exit(1)
@@ -170,7 +184,7 @@ func main() {
 			gpu.Status.Available = available
 		}
 
-		if err := k8sclient.Status().Patch(ctx, gpu, client.Merge); err != nil {
+		if err := k8sClient.Status().Patch(ctx, gpu, client.Merge); err != nil {
 			ctrl.Log.Error(err, "failed to update status of GPU", "gpu", gpu)
 			os.Exit(1)
 		}
@@ -186,8 +200,13 @@ func main() {
 	ns.TotalVRAM = totalVRAM
 	ns.AvailableTFlops = availableTFlops
 	ns.AvailableVRAM = availableVRAM
+	ns.TotalGPUs = int32(count)
+	ns.ManagedGPUs = int32(count)
+	ns.ManagedGPUDeviceIDs = allDeviceIDs
+	ns.NodeInfo.RAMSize = *resource.NewQuantity(getTotalHostRAM(), resource.DecimalSI)
+	ns.NodeInfo.DataDiskSize = *resource.NewQuantity(getDiskInfo(constants.TFDataPath), resource.DecimalSI)
 	gpunode.Status = *ns
-	if err := k8sclient.Status().Patch(ctx, gpunode, client.Merge); err != nil {
+	if err := k8sClient.Status().Patch(ctx, gpunode, client.Merge); err != nil {
 		ctrl.Log.Error(err, "failed to update status of GPUNode")
 		os.Exit(1)
 	}
@@ -228,3 +247,43 @@ func kubeClient() (client.Client, error) {
 	}
 	return client, nil
 }
+
+func getTotalHostRAM() int64 {
+	v, err := mem.VirtualMemory()
+	if err != nil {
+		fmt.Printf("error getting memory info: %v\n", err)
+		return 0
+	}
+	return int64(v.Total)
+}
+
+func getDiskInfo(path string) (total int64) {
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		fmt.Printf("error getting disk path: %v\n", err)
+		return 0
+	}
+
+	var stat syscall.Statfs_t
+	err = syscall.Statfs(absPath, &stat)
+	if err != nil {
+		if errors.Is(err, syscall.ENOENT) {
+			err = os.MkdirAll(absPath, 0755)
+			if err != nil {
+				fmt.Printf("error creating folder: %s, err: %v\n", absPath, err)
+				return 0
+			}
+			err = syscall.Statfs(absPath, &stat)
+			if err != nil {
+				fmt.Printf("error getting disk stats after creation: %v\n", err)
+				return 0
+			}
+		} else {
+			fmt.Printf("error getting disk stats: %v\n", err)
+			return 0
+		}
+	}
+
+	total = int64(stat.Blocks * uint64(stat.Bsize))
+	return total
+}

go.mod

@@ -88,6 +88,7 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.61.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect

go.sum

@@ -188,6 +188,8 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
 github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

internal/cloudprovider/alibaba/ecs.go

@@ -120,7 +120,7 @@ func (p AlibabaGPUNodeProvider) CreateNode(ctx context.Context, param *types.Nod
 	}
 
 	return &types.GPUNodeStatus{
-		InstanceID: response.RequestId,
+		InstanceID: response.InstanceIdSets.InstanceIdSet[0],
 		CreatedAt:  time.Now(),
 	}, nil
 }
@@ -129,6 +129,7 @@ func (p AlibabaGPUNodeProvider) TerminateNode(ctx context.Context, param *types.
 	request := ecs.CreateDeleteInstanceRequest()
 	request.InstanceId = param.InstanceID
 	request.RegionId = param.Region
+	request.Force = requests.NewBoolean(true)
 	response, err := p.client.DeleteInstance(request)
 	if err != nil {
 		return fmt.Errorf("failed to terminate instance: %w", err)