Sakkara setup for training-operator and appwrapper

Author: dgrove-oss

setup.k8s/CLUSTER-SETUP.md

@@ -55,8 +55,14 @@ kubectl create namespace mlbatch-system
 ```
 
 Install the Kubeflow Training Operator
+
+If you are using Coscheduler do:
+```sh
+kubectl apply --server-side -k setup.k8s/training-operator/coscheduler
+```
+If you are using Sakkara do:
 ```sh
-kubectl apply --server-side -k setup.k8s/training-operator
+kubectl apply --server-side -k setup.k8s/training-operator/sakkara
 ```
 
 Install the KubeRay Operator
@@ -70,13 +76,19 @@ kubectl apply --server-side -k setup.k8s/kueue
 ```
 
 Install the AppWrapper Operator
+If you are using Coscheduler do:
 ```sh
-kubectl apply --server-side -k setup.k8s/appwrapper
+kubectl apply --server-side -k setup.k8s/appwrapper/coscheduler
 ```
+If you are using Sakkara do:
+```sh
+kubectl apply --server-side -k setup.k8s/appwrapper/sakkara
+```
+
 The provided configuration differs from the default configuration of the
 operators as follows:
 - Kubeflow Training Operator:
-  - `gang-scheduler-name` is set to `scheduler-plugins-scheduler`,
+  - `gang-scheduler-name` is set to either `scheduler-plugins-scheduler` or `sakkara-scheduler`,
 - Kueue:
   - `batch/job` integration is disabled,
   - `manageJobsWithoutQueueName` is enabled and configured via `managedJobsNamespaceSelector` to be
@@ -87,7 +99,7 @@ operators as follows:
   - `enableClusterQueueResources` metrics is enabled,
 - AppWrapper operator:
   - `userRBACAdmissionCheck` is disabled,
-  - `schedulerName` is set to `scheduler-plugins-scheduler`,
+  - `schedulerName` is set to `scheduler-plugins-scheduler` or `sakkara-scheduler`,
   - `queueName` is set to `default-queue`,
 - pod priorities, resource requests and limits have been adjusted.
 

setup.k8s/appwrapper/kustomization.yaml renamed to setup.k8s/appwrapper/base/kustomization.yaml

@@ -17,6 +17,5 @@ images:
   newTag: v0.30.0
 
 patches:
-- path: config_patch.yaml
 - path: manager_resources_patch.yaml
 - path: remove_default_namespace.yaml

setup.k8s/appwrapper/manager_resources_patch.yaml renamed to setup.k8s/appwrapper/base/manager_resources_patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: mlbatch-system
+
+resources:
+- ../base
+
+patches:
+patches:
+- path: config_patch.yaml

setup.k8s/appwrapper/remove_default_namespace.yaml renamed to setup.k8s/appwrapper/base/remove_default_namespace.yaml

@@ -0,0 +1,23 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: appwrapper-operator-config
+  namespace: appwrapper-system
+data:
+  config.yaml: |
+    appwrapper:
+      enableKueueIntegrations: true
+      kueueJobReconciller:
+        manageJobsWithoutQueueName: true
+        waitForPodsReady:
+          enable: false
+      defaultQueueName: default-queue
+      schedulerName: sakkara-scheduler
+      slackQueueName: slack-cluster-queue
+      userRBACAdmissionCheck: false
+    controllerManager:
+      health:
+        bindAddress: ":8081"
+      metrics:
+        bindAddress: "127.0.0.1:8080"
+      leaderElection: true

setup.k8s/appwrapper/config_patch.yaml renamed to setup.k8s/appwrapper/coscheduler/config_patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: mlbatch-system
+
+resources:
+- ../base
+
+patches:
+patches:
+- path: config_patch.yaml

setup.k8s/appwrapper/coscheduler/kustomization.yaml

@@ -10,7 +10,6 @@ spec:
       - name:  training-operator
         args:
         - "--zap-log-level=2"
-        - "--gang-scheduler-name=scheduler-plugins-scheduler"
         resources:
           requests:
             cpu: 100m