[pull] main from project-codeflare:main

.github/workflows/additional_demo_notebook_tests.yaml

@@ -15,7 +15,7 @@ env:
 jobs:
   verify-local_interactive:
     if: ${{ github.event.label.name == 'test-additional-notebooks' }}
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-latest-4core
 
     steps:
       - name: Checkout code
@@ -50,7 +50,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup and start KinD cluster
@@ -133,7 +133,7 @@ jobs:
 
   verify-ray_job_client:
     if: ${{ github.event.label.name == 'test-additional-notebooks' }}
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-latest-4core
 
     steps:
       - name: Checkout code
@@ -168,7 +168,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup and start KinD cluster

.github/workflows/coverage-badge.yaml

@@ -4,29 +4,32 @@ name: Coverage Badge
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, ray-jobs-feature ]
 
 jobs:
   report:
 
+    permissions:
+      contents: write
+      pull-requests: write
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.11
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install poetry
         poetry config virtualenvs.create false
-        poetry lock --no-update
+        poetry lock
         poetry install --with test
     - name: Generate coverage report
       run: |
-        coverage run -m pytest --omit="src/**/test_*.py,src/codeflare_sdk/common/utils/unit_test_support.py"
+        coverage run --omit="src/**/test_*.py,src/codeflare_sdk/common/utils/unit_test_support.py,src/codeflare_sdk/vendored/**" -m pytest
 
     - name: Coverage Badge
       uses: tj-actions/coverage-badge-py@v2
@@ -47,7 +50,14 @@ jobs:
 
     - name: Create Pull Request
       if: steps.changed_files.outputs.files_changed == 'true'
-      uses: peter-evans/create-pull-request@v4
+      uses: peter-evans/create-pull-request@v6
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         title: "[Automatic] Coverage Badge Update"
+        commit-message: "Updated coverage.svg"
+        branch: create-pull-request/coverage-badge-update
+        delete-branch: true
+        body: |
+          This is an automated pull request to update the coverage badge.
+
+          - Updated coverage.svg based on latest test results

.github/workflows/e2e_tests.yaml

@@ -5,12 +5,13 @@ on:
   pull_request:
     branches:
       - main
-      - 'release-*'
+      - "release-*"
+      - ray-jobs-feature
     paths-ignore:
-      - 'docs/**'
-      - '**.adoc'
-      - '**.md'
-      - 'LICENSE'
+      - "docs/**"
+      - "**.adoc"
+      - "**.md"
+      - "LICENSE"
 
 concurrency:
   group: ${{ github.head_ref }}-${{ github.workflow }}
@@ -21,7 +22,7 @@ env:
 
 jobs:
   kubernetes:
-    runs-on: ubuntu-20.04-4core-gpu
+    runs-on: gpu-t4-4-core
 
     steps:
       - name: Checkout code
@@ -32,9 +33,9 @@ jobs:
       - name: Checkout common repo code
         uses: actions/checkout@v4
         with:
-          repository: 'project-codeflare/codeflare-common'
-          ref: 'main'
-          path: 'common'
+          repository: "project-codeflare/codeflare-common"
+          ref: "main"
+          path: "common"
 
       - name: Checkout CodeFlare operator repository
         uses: actions/checkout@v4
@@ -45,7 +46,7 @@ jobs:
       - name: Set Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: './codeflare-operator/go.mod'
+          go-version-file: "./codeflare-operator/go.mod"
           cache-dependency-path: "./codeflare-operator/go.sum"
 
       - name: Set up gotestfmt
@@ -56,7 +57,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.12'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD
@@ -75,7 +76,7 @@ jobs:
         run: |
           cd codeflare-operator
           echo Setting up CodeFlare stack
-          make setup-e2e
+          make setup-e2e KUEUE_VERSION=v0.13.4 KUBERAY_VERSION=v1.4.0
           echo Deploying CodeFlare operator
           make deploy -e IMG="${CODEFLARE_OPERATOR_IMG}" -e ENV="e2e"
           kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
@@ -94,6 +95,10 @@ jobs:
           kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
           kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch --resource=rayclusters
           kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
+          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch --resource=rayjobs
+          kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
+          kubectl create clusterrole rayjob-status-reader --verb=get,list,patch,update --resource=rayjobs/status
+          kubectl create clusterrolebinding sdk-user-rayjob-status-reader --clusterrole=rayjob-status-reader --user=sdk-user
           kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers
           kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
           kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
@@ -104,8 +109,12 @@ jobs:
           kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
           kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
           kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
-          kubectl create clusterrole pod-creator --verb=get,list --resource=pods
+          kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
           kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
+          kubectl create clusterrole service-reader --verb=get,list,watch --resource=services
+          kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
+          kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
+          kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
           kubectl config use-context sdk-user
 
       - name: Run e2e tests
@@ -117,7 +126,7 @@ jobs:
           pip install poetry
           poetry install --with test,docs
           echo "Running e2e tests..."
-          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
+          poetry run pytest -v -s ./tests/e2e/ -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
         env:
           GRPC_DNS_RESOLVER: "native"
 
@@ -141,7 +150,13 @@ jobs:
         if: always() && steps.deploy.outcome == 'success'
         run: |
           echo "Printing KubeRay operator logs"
-          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+          kubectl logs -n default --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+
+      - name: Print Kueue controller logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing Kueue controller logs"
+          kubectl logs -n kueue-system --tail -1 -l control-plane=controller-manager | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue.log
 
       - name: Export all KinD pod logs
         uses: ./common/github-actions/kind-export-logs

.github/workflows/guided_notebook_tests.yaml

@@ -3,6 +3,7 @@ name: Guided notebooks tests
 on:
   pull_request:
     branches: [ main ]
+    types: [ labeled ]
 
 concurrency:
   group: ${{ github.head_ref }}-${{ github.workflow }}
@@ -14,7 +15,7 @@ env:
 jobs:
   verify-0_basic_ray:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test-guided-notebooks') }}
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-latest-4core
 
     steps:
       - name: Checkout code
@@ -49,7 +50,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup and start KinD cluster
@@ -125,7 +126,7 @@ jobs:
 
   verify-1_cluster_job_client:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test-guided-notebooks') }}
-    runs-on: ubuntu-20.04-4core-gpu
+    runs-on: gpu-t4-4-core
 
     steps:
       - name: Checkout code
@@ -160,7 +161,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD
@@ -247,7 +248,7 @@ jobs:
 
   verify-2_basic_interactive:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'test-guided-notebooks') }}
-    runs-on: ubuntu-20.04-4core-gpu
+    runs-on: gpu-t4-4-core
 
     steps:
       - name: Checkout code
@@ -282,7 +283,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD

.github/workflows/odh-notebooks-sync.yml

@@ -33,11 +33,11 @@ env:
   REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }}
   REPO_NAME: notebooks
   GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }}
-  MINIMUM_SUPPORTED_PYTHON_VERSION: 3.9
+  MINIMUM_SUPPORTED_PYTHON_VERSION: 3.11
 
 jobs:
   build:
-    runs-on: ubuntu-22.04-8core
+    runs-on: ubuntu-latest-8core
     steps:
       - name: Clone repository and Sync
         run: |
@@ -56,10 +56,8 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pipenv'
 
-      # Sync fails with pipenv 2024.1.0 (current latest version)
-      # TODO: We should retry with later versions of pipenv once they are available.
       - name: Install pipenv and pip-versions
-        run: pip install pipenv==2024.0.3 pip-versions
+        run: pip install pipenv==2024.4.0 pip-versions
 
       - name: Update Pipfiles in accordance with Codeflare-SDK latest release
         run: |
@@ -74,7 +72,7 @@ jobs:
             # replace existing version of cf-sdk with new version in Pipfile
             sed -i "s/codeflare-sdk = .*$/codeflare-sdk = \"~=$CODEFLARE_RELEASE_VERSION\"/g" Pipfile
             # Lock dependencies, ensuring pre-release are included and clear previous state
-            if ! pipenv lock --pre --clear ; then
+            if ! pipenv lock --verbose --pre --clear ; then
               echo "Failed to lock dependencies"
               exit 1
             fi
@@ -98,7 +96,9 @@ jobs:
             echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name"
             # list all Pipfile paths having Codeflare-SDK listed
             # Extracting only directories from file paths, excluding a `.gitworkflow` and `.git` directory
-            directories+=($(grep --exclude-dir=.git --exclude-dir=.github --include="Pipfile*" -rl "${package_name} = \"~=.*\"" | xargs dirname | sort | uniq))
+            # Extracting Intel directories as they are not supported in RHOAI
+            # Removing tensorflow image TEMPORARILY until solution has been made for the tf2onnx package dependency resolution
+            directories+=($(grep --exclude-dir=.git --exclude-dir=.github --exclude-dir=intel --exclude-dir=tensorflow --exclude-dir=rocm-tensorflow --include="Pipfile*" -rl "${package_name} = \"~=.*\"" | xargs dirname | sort | uniq))
             counter=0
             total=${#directories[@]}
             for dir in "${directories[@]}"; do

.github/workflows/publish-documentation.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
       - name: Install Sphinx
         run: |
             sudo apt-get update