feat: implement autoscaling (#242)

* feat: implement the core auto-scaling functionality

* test: refactor test when processing workloads

* feat: implement LeaderElectonRunnable explicitly and add compile-time check

* feat: aggregate samples into histogram per tflops

* feat: implement metrics provider

* feat: add allocator logic

* refactor: optimize update worker method

* feat: add config parsing

* feat: apply updates to specified target resources

* feat: add auto-scaling switch config parsing and apply, TargetResource support value all

* feat: merge AutoSetLimits and AutoSetRequests into AutoSetResources

* feat: implement adjust allocation

* fix: linter issues

* fix: linter issues

* refactor: support multiple recommenders

* refactor: code organization

* feat: define cron scaler crd

* feat: implement cron scaling

* feat: implement cron scaling

* feat: implement merging recommendations

* feat: implement restoring resources upon cron scaling termination

* fix: properly handle the isScaleUp

* refactor: each recommender is responsible for managing its own annotations

* refactor: remove unused functions and params

* feat: implement scale-down lock

* refactor: improve naming

* fix: scale down issue

* feat: add a recommendation field to the status of the workload and implement

* test: refactor tests

* feat: integrate the autoscaler into the main function

* fix: timestamp field issue

* refactor: inject the metrics provider dependency

* refactor: add namespace to identify the workload and process the namespace field of table

* fix: handle zero resource value properly

* feat: add condition type RecommendationProvided to workload

* refactor: make percentile recommender more testable

* refactor: improve status conditions

* feat: add appliedRecommendedReplicas field to status and refactor

* fix: handle activeCronScalingRule and applied replicas properly

* fix: applied recommended replicas issue

* fix: add vpa package

* fix: linter issues

* test: wrong suite name

* fix: vram peak zero issue and fix query issue caused by time zone

* feat: only recommend but do not apply recommendation if worker has a dedicated GPU

* fix: missing autoScalingConfig

* feat: implement max allowed resources recommendation processor

* fix: linter issue

* fix: only get the max allowed resources when scaling up

* fix: test bug

* feat: add targetResource annoataion

* refactor: improve logs

* fix: skipping preemption tests and linter issue

* fix: handle gpu status properly

---------

Co-authored-by: Joey Yang <[email protected]>

Author: knave

api/v1/schedulingconfigtemplate_types.go

@@ -86,17 +86,71 @@ type GPUFilter struct {
 }
 
 type AutoScalingConfig struct {
-	// layer 1 vertical auto-scaling, turbo burst to existing GPU cards quickly
-	// VPA-like, aggregate metrics data <1m
-	AutoSetLimits AutoSetLimits `json:"autoSetLimits,omitempty"`
+	// layer 1 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode
+	// Adjust baseline requests to match the actual usage in longer period, such as 1day - 2weeks
+	AutoSetResources AutoSetResources `json:"autoSetResources,omitempty"`
 
 	// layer 2 horizontal auto-scaling, scale up to more GPU cards if max limits threshold hit
 	// HPA-like, aggregate metrics data 1m-1h (when tf-worker scaled-up, should also trigger client pod's owner[Deployment etc.]'s replica increasing, check if KNative works)
 	AutoSetReplicas AutoSetReplicas `json:"autoSetReplicas,omitempty"`
 
-	// layer 3 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode, not impl yet
-	// Adjust baseline requests to match the actual usage in longer period, such as 1day - 2weeks
-	AutoSetRequests AutoSetRequests `json:"autoSetRequests,omitempty"`
+	// CronScalingRules defines a list of CronScaling rules used to schedule scaling actions based on cron expressions.
+	CronScalingRules []CronScalingRule `json:"cronScalingRules,omitempty"`
+}
+
+// CronScalingRule defines the rule for scaling resources based on a cron schedule.
+// It allows enabling/disabling the scaler, specifying the time window for scaling,
+// and configuring the desired resources and replicas during the scheduled period.
+type CronScalingRule struct {
+	// Enable specifies whether the cron scaler is enabled.
+	Enable bool `json:"enable,omitempty"`
+	// Name is the identifier for the cron scaler.
+	Name string `json:"name,omitempty"`
+	// Start is the start time for the scaling schedule, in cron format.
+	Start string `json:"start,omitempty"`
+	// End is the end time for the scaling schedule, in cron format.
+	End string `json:"end,omitempty"`
+	// DesiredResources specifies the target resources to scale to during the schedule.
+	DesiredResources Resources `json:"desiredResources,omitempty"`
+	// DesiredReplicas is the target number of replicas during the schedule.
+	DesiredReplicas *int32 `json:"desiredReplicas,omitempty"`
+}
+
+type AutoSetResources struct {
+	Enable bool `json:"enable,omitempty"`
+
+	// Target resource to scale, such as "tflops", "vram", or "all" by default
+	TargetResource string `json:"targetResource,omitempty"`
+
+	// Tflops usage percentile that will be used as a base for tflops target recommendation. Default: 0.9
+	TargetTflopsPercentile string `json:"targettflopspercentile,omitempty"`
+
+	// Tflops usage percentile that will be used for the lower bound on tflops recommendation. Default: 0.5
+	LowerBoundTflopsPercentile string `json:"lowerboundtflopspercentile,omitempty"`
+
+	// Tflops usage percentile that will be used for the upper bound on tflops recommendation. Default: 0.95
+	UpperBoundTflopsPercentile string `json:"upperboundtflopspercentile,omitempty"`
+
+	// Vram usage percentile that will be used as a base for vram target recommendation. Default: 0.9
+	TargetVramPercentile string `json:"targetvrampercentile,omitempty"`
+
+	// Vram usage percentile that will be used for the lower bound on vram recommendation. Default: 0.5
+	LowerBoundVramPercentile string `json:"lowerboundvrampercentile,omitempty"`
+
+	// Vram usage percentile that will be used for the upper bound on vram recommendation. Default: 0.95
+	UpperBoundVramPercentile string `json:"upperboundvrampercentile,omitempty"`
+
+	// Fraction of usage added as the safety margin to the recommended request. Default: 0.15
+	RequestMarginFraction string `json:"requestMarginFraction,omitempty"`
+
+	// The time interval used for computing the confidence multiplier for the lower and upper bound. Default: 24h
+	ConfidenceInterval string `json:"confidenceInterval,omitempty"`
+
+	// How much time back TSDB have to be queried to get historical metrics. Default: 1d
+	HistoryLength string `json:"historyLength,omitempty"`
+
+	// Resolution at which TSDB is queried for historical metrics. Default: 1m
+	HistoryResolution string `json:"historyResolution,omitempty"`
 }
 
 // A typical autoLimits algorithm could be checking every 5m, look back 1 day data,

api/v1/tensorfusionconnection_types.go

@@ -21,6 +21,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type ResourceName string
+
+const (
+	ResourceTflops ResourceName = "tflops"
+	ResourceVram   ResourceName = "vram"
+)
+
 type Resource struct {
 	Tflops resource.Quantity `json:"tflops"`
 	Vram   resource.Quantity `json:"vram"`
@@ -31,6 +38,23 @@ type Resources struct {
 	Limits   Resource `json:"limits"`
 }
 
+func (r Resources) Equal(target *Resources) bool {
+	if target == nil {
+		return false
+	}
+	return r.Requests.Tflops.Equal(target.Requests.Tflops) &&
+		r.Requests.Vram.Equal(target.Requests.Vram) &&
+		r.Limits.Tflops.Equal(target.Limits.Tflops) &&
+		r.Limits.Vram.Equal(target.Limits.Vram)
+}
+
+func (r Resources) IsZero() bool {
+	return r.Requests.Tflops.IsZero() &&
+		r.Requests.Vram.IsZero() &&
+		r.Limits.Tflops.IsZero() &&
+		r.Limits.Vram.IsZero()
+}
+
 // TensorFusionConnectionSpec defines the desired state of TensorFusionConnection.
 type TensorFusionConnectionSpec struct {
 	WorkloadName string `json:"workloadName"`

api/v1/tensorfusionworkload_types.go

@@ -65,6 +65,18 @@ type TensorFusionWorkloadStatus struct {
 
 	// Hash of the pod template used to create worker pods
 	PodTemplateHash string `json:"podTemplateHash,omitempty"`
+
+	// The most recently GPU resources recommended by the autoscaler
+	// +optional
+	Recommendation *Resources `json:"recommendation,omitempty"`
+
+	// The number of replicas currently applied based on the latest recommendation
+	// +optional
+	AppliedRecommendedReplicas int32 `json:"appliedRecommendedReplicas,omitempty"`
+
+	// The currently active cron scaling rule
+	// +optional
+	ActiveCronScalingRule *CronScalingRule `json:"activeCronScalingRule,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1/workloadprofile_types.go

@@ -62,7 +62,7 @@ type WorkloadProfileSpec struct {
 	// +optional
 	// AutoScalingConfig configured here will override Pool's schedulingConfig
 	// This field can not be fully supported in annotation, if user want to enable auto-scaling in annotation,
-	// user can set tensor-fusion.ai/auto-limits|requests|replicas: 'true'
+	// user can set tensor-fusion.ai/auto-resources|replicas: 'true'
 	AutoScalingConfig AutoScalingConfig `json:"autoScalingConfig,omitempty"`
 
 	// +optional