feat: refactor tensor fusion scheduler to k8s scheduler framework, support ngpu mode (#241)

* feat: refactor to k8s scheduler framework

* fix: add scheduler framework and basic tests, align log to klog

* fix: convert scheduler kubeconfig and other args

* fix: new scheduler issues

* fix: remove submodule

* fix: typo

* fix: scheduler refactor

* fix: scheduler scoring

* fix: new scheduler bugs

* fix: scheduler bugs

* fix: refactor quota impl

* fix: test case

* fix: schedule unit test and bug fix

* chore: lint issue

* fix: refactor namespaced level quota store

* fix: unit test bugs

* fix: unit test bugs

* fix: quota store unit test issue

* fix: assign namespace level gpu resource defaults, unit test issues

* fix: scheduler refactor issues

* fix: unit test issues, gpu informer update handler bug

* fix: tf connection ut bugs

* chore: typo

* fix: restore mock pool rolling update interval

* fix: rolling update ut bug

* fix: debounce reconcile bug, change to ratelimiter, add hypervisor config env

* fix: remove unused files and vars, use k8s ptr pkg

Author: Code2Life

.gitignore

@@ -33,4 +33,6 @@ cmd/*/__debug*
 
 prompts/*
 
-tmp*
+tmp*
+
+__debug*

.vscode/launch.json

@@ -15,8 +15,9 @@
             "program": "${workspaceFolder}/cmd/main.go",
             "args": [
                 "--gpu-info-config", "${workspaceFolder}/config/samples/gpu-info-config.yaml",
-                "--alert-rule-config", "${workspaceFolder}/config/samples/dynamic-config.yaml",
-                "--enable-alert", "true"
+                "--dynamic-config", "${workspaceFolder}/config/samples/dynamic-config.yaml",
+                "--scheduler-config", "${workspaceFolder}/config/samples/scheduler-config.yaml",
+                // "--enable-alert"
             ]
         },
         {
@@ -60,6 +61,7 @@
             "request": "launch",
             "mode": "test",
             "env": {
+                "DEBUG_MODE": "true",
                 "GO_TESTING": "true"
             },
             "program": "${workspaceFolder}/internal/controller",

.vscode/settings.json

@@ -1,12 +1,16 @@
 {
     "cSpell.words": [
+        "admissionv",
         "alertmanager",
         "alertname",
         "alicloud",
         "Aliyun",
         "AMDCDNA",
         "AMDRDNA",
         "apimachinery",
+        "apimachineryruntime",
+        "apiruntime",
+        "apiutil",
         "automount",
         "AWSGPU",
         "batchv",
@@ -16,23 +20,30 @@
         "certificaterequests",
         "certmanager",
         "clientgoscheme",
+        "clientset",
         "cloudnative",
         "cloudprovider",
         "clusterissuers",
+        "componentconfig",
+        "configz",
         "controllerutil",
         "corev",
         "crds",
         "CUDA",
         "cycjimmy",
+        "datanode",
+        "defaultbinder",
         "dylib",
         "envtest",
         "essd",
         "Eventf",
         "finalizer",
         "Finalizers",
+        "frameworkruntime",
         "FULLTEXT",
         "goconst",
         "gocyclo",
+        "goerrors",
         "golint",
         "Gomega",
         "gopsutil",
@@ -44,23 +55,36 @@
         "gpunodes",
         "gpupool",
         "gpupools",
+        "gpuresourcequotas",
+        "gpuresources",
         "GPUT",
+        "gputopo",
         "GPUVRAM",
         "greptime",
         "greptimedb",
         "healthz",
         "iface",
+        "imageutils",
+        "jsonpatch",
         "karpenter",
+        "klog",
+        "Klogr",
         "kubebuilder",
         "KUBECONFIG",
         "Kubelet",
+        "kubescheduler",
+        "kubeschedulerconfig",
         "kustomization",
         "metav",
         "metricsserver",
+        "Milli",
         "mito",
+        "ngpu",
         "nindent",
+        "noderesources",
         "nolint",
         "NVML",
+        "objs",
         "omitempty",
         "onsi",
         "portallocator",
@@ -69,16 +93,21 @@
         "prometheusagents",
         "prometheuses",
         "prometheusrules",
+        "queuesort",
         "RDNA",
         "readyz",
         "replicaset",
         "runbook",
         "runpod",
         "samber",
+        "sched",
+        "schedulerserverconfig",
         "schedulingconfigtemplate",
         "schedulingconfigtemplates",
         "schedulingcorev",
         "shirou",
+        "shortuuid",
+        "strategicpatch",
         "strategicpatches",
         "subresource",
         "Tabler",
@@ -87,13 +116,19 @@
         "tensorfusioncluster",
         "tensorfusionclusters",
         "tensorfusionworkload",
+        "tensorfusionworkloads",
         "Tera",
+        "testutil",
         "tflops",
         "timberio",
         "Tmpl",
         "Tolerations",
+        "utilerrors",
         "utilruntime",
+        "vgpu",
         "webhookcorev",
+        "workloadprofiles",
+        "workqueue",
         "Xlarge"
     ]
 }

Makefile

@@ -64,6 +64,10 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" GO_TESTING=true go run github.com/onsi/ginkgo/v2/ginkgo -p -timeout 0 -cover -coverprofile cover.out -r --skip-file ./test/e2e
 
+.PHONY: ut
+ut: manifests generate ## Run unit tests by make ut F=<focus-file>
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" cd internal/controller && GO_TESTING=true go run github.com/onsi/ginkgo/v2/ginkgo -p -timeout 0 --focus-file $F && cd ../../
+
 .PHONY: test-e2e
 test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
 	@command -v kind >/dev/null 2>&1 || { \

api/v1/base_types.go

@@ -1,6 +1,12 @@
 package v1
 
+import "fmt"
+
 type NameNamespace struct {
 	Name      string `json:"name,omitempty"`
 	Namespace string `json:"namespace,omitempty"`
 }
+
+func (n NameNamespace) String() string {
+	return fmt.Sprintf("%s/%s", n.Namespace, n.Name)
+}

api/v1/gpupool_types.go

@@ -105,16 +105,15 @@ type ProvisioningMode string
 
 const (
 	ProvisioningModeProvisioned ProvisioningMode = "Provisioned"
-	ProvisioningModeAutoSelect  ProvisioningMode = "AutoSelect"
+
+	ProvisioningModeAutoSelect ProvisioningMode = "AutoSelect"
+
+	ProvisioningModeKarpenter ProvisioningMode = "Karpenter"
 )
 
 // NodeProvisioner or NodeSelector, they are exclusive.
 // NodeSelector is for existing GPUs, NodeProvisioner is for Karpenter-like auto management.
 type NodeProvisioner struct {
-	// Mode could be Karpenter or Native, for Karpenter mode, node provisioner will start dummy nodes to provision and warmup GPU nodes, do nothing for CPU nodes, for Native mode, provisioner will create or compact GPU & CPU nodes based on current pods
-	// +kubebuilder:default=Native
-	Mode NodeProvisionerMode `json:"mode,omitempty"`
-
 	NodeClass string `json:"nodeClass,omitempty"`
 
 	// +optional
@@ -159,14 +158,6 @@ const (
 	BudgetExceedStrategyAlertAndTerminateVM BudgetExceedStrategy = "AlertAndTerminateVM"
 )
 
-// +kubebuilder:validation:Enum=Native;Karpenter
-type NodeProvisionerMode string
-
-const (
-	NodeProvisionerModeNative    NodeProvisionerMode = "Native"
-	NodeProvisionerModeKarpenter NodeProvisionerMode = "Karpenter"
-)
-
 type Requirement struct {
 	Key NodeRequirementKey `json:"key,omitempty"`
 
@@ -347,6 +338,9 @@ type ClientConfig struct {
 
 	// +optional
 	PatchToContainer *runtime.RawExtension `json:"patchToContainer,omitempty"`
+
+	// +optional
+	PatchEmbeddedWorkerToPod *runtime.RawExtension `json:"patchEmbeddedWorkerToPod,omitempty"`
 }
 
 // GPUPoolStatus defines the observed state of GPUPool.