address smaller review comments

Author: dgrove-oss

.github/workflows/project-codeflare-release.yml

@@ -19,6 +19,9 @@ on:
       kuberay-version:
         description: 'Tested version of KubeRay (for example: v0.0.0)'
         required: true
+      kueue-version:
+        description: 'Tested version of Kueue (for example: v0.0.0)'
+        required: true
       is-stable:
         description: 'Select if the built images should be tagged as stable'
         required: true
@@ -48,13 +51,14 @@ jobs:
         echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}"
         echo "Tested AppWrapper Version: ${{ github.event.inputs.appwrapper-version }}"
         echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}"
+        echo "Tested Kueue Version: ${{ github.event.inputs.kueue-version }}"
         echo "Is Stable: ${{ github.event.inputs.is-stable }}"
         echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}"
         echo "Quay Organization: ${{ github.event.inputs.quay-organization }}"
         echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}"
 
   release-codeflare-sdk:
-    needs: [check-appwrapper-version, check-kuberay-version]
+    needs: [check-appwrapper-version, check-kuberay-version, check-kueue-version]
     runs-on: ubuntu-latest
 
     steps:
@@ -94,7 +98,7 @@ jobs:
     steps:
     - name: Release CodeFlare operator
       run: |
-        gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }}
+        gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field kueue-version=${{ github.event.inputs.kueue-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }}
       env:
         GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }}
       shell: bash
@@ -136,3 +140,17 @@ jobs:
           echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} does not exist. Please select an existing version."
           exit 1
         fi
+
+  check-kueue-version:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check if Kueue release exists
+      run: |
+        status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/kubernetes-sigs/kueue/releases/tag/${{ github.event.inputs.kueue-version }})
+        if [[ "$status_code" == "200" ]]; then
+          echo "Kueue release with version ${{ github.event.inputs.kueue-version }} exists and available to use."
+        else
+          echo "Kueue release with version ${{ github.event.inputs.kueue-version }} does not exist. Please select an existing version."
+          exit 1
+        fi

.github/workflows/tag-and-build.yml

@@ -24,6 +24,10 @@ on:
         description: 'Tested version of KubeRay'
         required: true
         default: 'v0.0.0-dev'
+      kueue-version:
+        description: 'Tested version of Kueue'
+        required: true
+        default: 'v0.0.0-dev'
       is-stable:
         description: 'Select if the built image should be tagged as stable'
         required: true
@@ -89,6 +93,7 @@ jobs:
         sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md
         sed -i -E "s|(.*AppWrapper.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.appwrapper-version }}\2${{ github.event.inputs.appwrapper-version }}\3|" README.md
         sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md
+        sed -i -E "s|(.*Kueue.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kueue-version }}\2${{ github.event.inputs.kueue-version }}\3|" README.md
 
     - name: Adjust AppWrapper dependency in the code
       run: |

README.md

@@ -10,8 +10,9 @@ CodeFlare Stack Compatibility Matrix
 |------------------------------|---------------------------------------------------------------------------------------------------|
 | CodeFlare Operator           | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1)             |
 | CodeFlare-SDK                | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1)                |
-| KubeRay                      | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0)                           |
 | AppWrapper                   | [v0.11.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.11.1)                   |
+| KubeRay                      | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0)                           |
+| Kueue                        | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2)                             |
 <!-- Compatibility Matrix end -->
 
 ## Development

config/crd/kustomization.yaml

@@ -4,10 +4,10 @@
 resources:
 - crd-appwrapper.yml
 
-patches:
+#patches:
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
-- path: patches/webhook_in_appwrappers.yaml
+# - path: patches/webhook_in_appwrappers.yaml
 #+kubebuilder:scaffold:crdkustomizewebhookpatch
 
 # [WEBHOOK] To enable webhook, uncomment the following section

config/crd/patches/webhook_in_appwrappers.yaml

@@ -249,29 +249,32 @@ func setupAppWrapperComponents(ctx context.Context, cancel context.CancelFunc, m
 	}
 
 	// AppWrapper webhook doesn't depend on WorkloadAPI availablity but does need certsReady
-	go func() {
-		<-certsReady
-		setupLog.Info("Setting up AppWrapper webhooks")
-		exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks")
-	}()
+	go setupAppWrapperWebhooks(mgr, cfg, certsReady)
 
 	if isAPIAvailable(ctx, mgr, workloadAPI) {
-		setupLog.Info("Workload API available; enabling AppWrappers")
+		setupLog.Info("Workload API available, enabling AppWrappers")
 		go setupAppWrapperController(mgr, cfg, certsReady)
 		return awctrl.SetupIndexers(ctx, mgr, cfg.AppWrapper.Config)
 	} else {
 		// If AppWrappers are enabled and the Workload API becomes available later, initiate an orderly
 		// restart of the codeflare operator to enable the workload indexer to be setup in the the new instance of the operator.
 		// It is not possible to add an indexer once the mgr has started so, a restart if the only avenue.
-		setupLog.Info("Workload API not available; setting up waiter for Workload API availability")
+		setupLog.Info("Workload API not available, setting up waiter for Workload API availability")
 		go waitForAPI(ctx, mgr, workloadAPI, func() {
-			setupLog.Info("Workload API now available; triggering controller restart")
+			setupLog.Info("Workload API now available, triggering controller restart")
 			cancel()
 		})
 		return nil
 	}
 }
 
+func setupAppWrapperWebhooks(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
+	setupLog.Info("Waiting for certificate generation to complete")
+	<-certsReady
+	setupLog.Info("Setting up AppWrapper webhooks")
+	exitOnError(awctrl.SetupWebhooks(mgr, cfg.AppWrapper.Config), "unable to setup AppWrapper webhooks")
+}
+
 func setupAppWrapperController(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) {
 	setupLog.Info("Waiting for certificate generation to complete")
 	<-certsReady