* : allow to warmup on AKS (#146)

* introduce `--core-warmup-ready-threshold` flag for warmup subcmd
* update `custom-system` PLC if it exists
* switch to v1 for flowschema

---------

Signed-off-by: Wei Fu <[email protected]>

Author: fuweid

cmd/kperf/commands/utils/helper.go

@@ -10,7 +10,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	flowcontrolv1beta3 "k8s.io/api/flowcontrol/v1beta3"
+	flowcontrolv1 "k8s.io/api/flowcontrol/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -82,21 +83,21 @@ func ApplyPriorityLevelConfiguration(kubeconfigPath string) error {
 
 	// Define the PriorityLevelConfiguration
 	lendablePercent := int32(30)
-	plc := &flowcontrolv1beta3.PriorityLevelConfiguration{
+	plc := &flowcontrolv1.PriorityLevelConfiguration{
 		TypeMeta: metav1.TypeMeta{
-			APIVersion: "flowcontrol.apiserver.k8s.io/v1beta3",
+			APIVersion: "flowcontrol.apiserver.k8s.io/v1",
 			Kind:       "PriorityLevelConfiguration",
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "custom-system",
 		},
-		Spec: flowcontrolv1beta3.PriorityLevelConfigurationSpec{
-			Type: flowcontrolv1beta3.PriorityLevelEnablementLimited,
-			Limited: &flowcontrolv1beta3.LimitedPriorityLevelConfiguration{
+		Spec: flowcontrolv1.PriorityLevelConfigurationSpec{
+			Type: flowcontrolv1.PriorityLevelEnablementLimited,
+			Limited: &flowcontrolv1.LimitedPriorityLevelConfiguration{
 				LendablePercent: &lendablePercent,
-				LimitResponse: flowcontrolv1beta3.LimitResponse{
-					Type: flowcontrolv1beta3.LimitResponseTypeQueue,
-					Queuing: &flowcontrolv1beta3.QueuingConfiguration{
+				LimitResponse: flowcontrolv1.LimitResponse{
+					Type: flowcontrolv1.LimitResponseTypeQueue,
+					Queuing: &flowcontrolv1.QueuingConfiguration{
 						Queues:           64,
 						HandSize:         6,
 						QueueLengthLimit: 50,
@@ -106,8 +107,15 @@ func ApplyPriorityLevelConfiguration(kubeconfigPath string) error {
 		},
 	}
 
+	plcCli := clientset.FlowcontrolV1().PriorityLevelConfigurations()
+
 	// Apply the PriorityLevelConfiguration
-	_, err = clientset.FlowcontrolV1beta3().PriorityLevelConfigurations().Create(context.TODO(), plc, metav1.CreateOptions{})
+	_, err = plcCli.Create(context.TODO(), plc, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			_, err = plcCli.Update(context.TODO(), plc, metav1.UpdateOptions{})
+		}
+	}
 	if err != nil {
 		return fmt.Errorf("failed to apply PriorityLevelConfiguration: %v", err)
 	}

contrib/cmd/runkperf/commands/warmup/command.go

@@ -66,6 +66,11 @@ var Command = cli.Command{
 			Usage: "Deploy runner group with a specific labels (FORMAT: KEY=VALUE[,VALUE])",
 			Value: "node.kubernetes.io/instance-type=Standard_D16s_v3,m4.4xlarge,n1-standard-16",
 		},
+		cli.IntFlag{
+			Name:  "core-warmup-ready-threshold",
+			Usage: "Indicates the threshold for core during warm-up",
+			Value: 8,
+		},
 		cli.BoolFlag{
 			Name:   "eks",
 			Usage:  "Indicates the target kubernetes cluster is EKS",
@@ -121,7 +126,7 @@ var Command = cli.Command{
 
 		cores, ferr := utils.FetchAPIServerCores(ctx, kubeCfgPath)
 		if ferr == nil {
-			if isReady(cores) {
+			if isReady(cliCtx, cores) {
 				klog.V(0).Infof("apiserver resource is ready: %v", cores)
 				return nil
 			}
@@ -162,7 +167,7 @@ var Command = cli.Command{
 
 		cores, ferr = utils.FetchAPIServerCores(ctx, kubeCfgPath)
 		if ferr == nil {
-			if isReady(cores) {
+			if isReady(cliCtx, cores) {
 				klog.V(0).Infof("apiserver resource is ready: %v", cores)
 				return nil
 			}
@@ -172,14 +177,17 @@ var Command = cli.Command{
 }
 
 // isReady returns true if there are more than two instances using 8 cores.
-func isReady(cores map[string]int) bool {
+func isReady(cliCtx *cli.Context, cores map[string]int) bool {
+	target := cliCtx.Int("core-warmup-ready-threshold")
+	isEKS := cliCtx.Bool("eks")
+
 	n := 0
 	for _, c := range cores {
-		if c >= 8 {
+		if c >= target {
 			n++
 		}
 	}
-	return n >= 2
+	return (isEKS && n >= 2) || (!isEKS && n >= 1)
 }
 
 // deployWarmupVirtualNodepool deploys virtual nodepool.

manifests/runnergroup/server/templates/flowcontrol.yaml

@@ -1,4 +1,4 @@
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/v1
 kind: FlowSchema
 metadata:
   name: {{ .Values.name }}

manifests/virtualcluster/nodecontrollers/templates/flowcontrol.yaml

@@ -1,4 +1,4 @@
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/v1
 kind: FlowSchema
 metadata:
   name: {{ .Values.name }}