[feat][python-client]: add rayjob support to kuberay python-client (#3830)

* [feat][python-client]: add RayJob support to python-client

* install kuberay for python-client CI

* check in-cluster config if no kube config

* fix linting issues

* Move python client tests to buildkite

* Update .buildkite/test-python-client.yml

Signed-off-by: Kai-Hsun Chen <[email protected]>

* Add python-client test for RayJob lifecycling a RayCluster

* Add python-venv to setup-env.sh

---------

Signed-off-by: Kai-Hsun Chen <[email protected]>
Co-authored-by: Kai-Hsun Chen <[email protected]>

Author: kryanbeane

.buildkite/setup-env.sh

@@ -29,7 +29,7 @@ helm repo update
 
 # Install python 3.11 and pip
 apt-get update
-apt-get install -y python3.11 python3-pip
+apt-get install -y python3.11 python3-pip python3-venv python3-dev build-essential
 
 # Install requirements
 pip install --break-system-packages ray[default]==2.46.0

.buildkite/test-python-client.yml

@@ -0,0 +1,24 @@
+- label: 'Test Python Client'
+  instance_size: large
+  image: golang:1.24
+  commands:
+    - source .buildkite/setup-env.sh
+    - kind create cluster --wait 900s --config ./ci/kind-config-buildkite.yml
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+    # Build nightly KubeRay operator image
+    - pushd ray-operator
+    - bash ../.buildkite/build-start-operator.sh
+    - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
+    - popd
+    # Setup Python environment and install Python client
+    - echo "--- START:Setting up Python environment"
+    - pushd clients/python-client
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install -e .
+    - pip install pytest
+    # Run Python client tests
+    - echo "--- START:Running Python client tests"
+    - set -o pipefail
+    - pytest python_client_test/ -v || (kubectl logs --tail=50 -l app.kubernetes.io/name=kuberay && exit 1)
+    - echo "--- END:Python client tests finished"

.github/workflows/test-job.yaml

@@ -341,32 +341,3 @@ jobs:
       - name: Test
         run: go test ./pkg/... -race -parallel 4
         working-directory: ${{env.working-directory}}
-
-  python-client-test:
-    runs-on: ubuntu-22.04
-    name: Python Client Test
-    steps:
-      - name: Set up Docker
-        uses: docker/setup-docker-action@v4
-
-      - name: Install Kind
-        run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64
-          chmod +x ./kind
-          sudo mv ./kind /usr/local/bin/kind
-          kind create cluster
-        shell: bash
-
-      - name: Checkout Python
-        uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
-      - name: Install package and run unittest for Python client
-        working-directory: ./clients/python-client
-        run: |
-          pip install -e .
-          python3 -m unittest discover 'python_client_test/'

clients/python-client/README.md

@@ -1,6 +1,6 @@
 # Overview
 
-This python client library provide APIs to handle `raycluster` from your python application.
+This python client library provide APIs to handle `raycluster` and `rayjobs` from your python application.
 
 ## Prerequisites
 
@@ -70,39 +70,69 @@ if succeeded:
 
 ### cluster_api
 
-Finally, the `cluster_api` is the one you always use to implement your cluster change in k8s. You can
+The `cluster_api` is the one you always use to implement your cluster change in k8s. You can
 use it with raw `JSON` if you wish. The `director/cluster_builder/cluster_utils` are just tools to
 shield the user from using raw `JSON`.
 
+### job_api
+
+Finally, the `job_api` can be used to submit RayJobs to a pre-existing RayCluster.
+
+#### Submitting to Existing Cluster
+
+```python
+from python_client import kuberay_job_api, kuberay_cluster_api, constants
+
+job_body = {
+    "apiVersion": "ray.io/v1",
+    "kind": "RayJob",
+    "metadata": {...},
+    "spec": {
+        "clusterSelector": {
+            "ray.io/cluster": "ray-cluster-name",
+        },
+        "entrypoint": 'python -c training_script.py',
+        "submissionMode": "K8sJobMode",
+    },
+}
+
+kuberay_job_api.submit_job(
+    job=job_body,
+    k8s_namespace=namespace,
+)
+```
+
 ## Code Organization
 
 ```text
 clients/
 └── python-client
-    ├── LICENSE
-    ├── README.md
     ├── examples
     │   ├── complete-example.py
     │   ├── use-builder.py
     │   ├── use-director.py
     │   ├── use-raw-config_map_with-api.py
     │   ├── use-raw-with-api.py
     │   └── use-utils.py
+    ├── LICENSE
+    ├── poetry.lock
     ├── pyproject.toml
     ├── python_client
     │   ├── __init__.py
     │   ├── constants.py
     │   ├── kuberay_cluster_api.py
+    │   ├── kuberay_job_api.py
     │   └── utils
     │       ├── __init__.py
     │       ├── kuberay_cluster_builder.py
     │       └── kuberay_cluster_utils.py
     ├── python_client_test
     │   ├── README.md
-    │   ├── test_api.py
+    │   ├── test_cluster_api.py
     │   ├── test_director.py
+    │   ├── test_job_api.py
     │   └── test_utils.py
-    └── setup.cfg
+    └── README.md
 ```
 
 ## For developers

clients/python-client/examples/use-raw-config_map_with-api.py

@@ -37,7 +37,7 @@
 }
 
 cluster_body: dict = {
-  "apiVersion": "ray.io/v1alpha1",
+  "apiVersion": "ray.io/v1",
   "kind": "RayCluster",
   "metadata": {
     "labels": {

clients/python-client/examples/use-raw-with-api.py

@@ -21,7 +21,7 @@
 from python_client import kuberay_cluster_api
 
 cluster_body: dict = {
-    "apiVersion": "ray.io/v1alpha1",
+    "apiVersion": "ray.io/v1",
     "kind": "RayCluster",
     "metadata": {
         "labels": {"controller-tools.k8s.io": "1.0", "demo-cluster": "yes"},
@@ -59,7 +59,7 @@
 
 
 cluster_body2: dict = {
-    "apiVersion": "ray.io/v1alpha1",
+    "apiVersion": "ray.io/v1",
     "kind": "RayCluster",
     "metadata": {
         "labels": {"controller-tools.k8s.io": "1.0", "demo-cluster": "yes"},