Skip to content

Commit 8014058

Browse files
kryanbeanekryanbeane
committed
RHOAIENG-32532: Run RayJob tests in CI
1 parent d259ec1 commit 8014058

File tree

1 file changed

+232
-0
lines changed

1 file changed

+232
-0
lines changed

‎.github/workflows/rayjob_e2e_tests.yaml‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
name: rayjob-e2e-with-kueue
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- 'release-*'
8+
- ray-jobs-feature
9+
paths-ignore:
10+
- 'docs/**'
11+
- '**.adoc'
12+
- '**.md'
13+
- 'LICENSE'
14+
15+
concurrency:
16+
group: ${{ github.head_ref }}-${{ github.workflow }}
17+
cancel-in-progress: true
18+
19+
env:
20+
CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
21+
KUEUE_VERSION: "v0.13.3"
22+
23+
jobs:
24+
kubernetes:
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
with:
31+
submodules: recursive
32+
33+
- name: Checkout common repo code
34+
uses: actions/checkout@v4
35+
with:
36+
repository: 'project-codeflare/codeflare-common'
37+
ref: 'main'
38+
path: 'common'
39+
40+
- name: Checkout CodeFlare operator repository
41+
uses: actions/checkout@v4
42+
with:
43+
repository: project-codeflare/codeflare-operator
44+
path: codeflare-operator
45+
46+
- name: Checkout Kueue repository
47+
uses: actions/checkout@v4
48+
with:
49+
repository: kubernetes-sigs/kueue
50+
path: kueue
51+
ref: main
52+
53+
- name: Set Go
54+
uses: actions/setup-go@v5
55+
with:
56+
go-version-file: './codeflare-operator/go.mod'
57+
cache-dependency-path: |
58+
./codeflare-operator/go.sum
59+
./kueue/go.sum
60+
61+
- name: Set up gotestfmt
62+
uses: gotesttools/gotestfmt-action@v2
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Set up specific Python version
67+
uses: actions/setup-python@v5
68+
with:
69+
python-version: '3.11'
70+
cache: 'pip' # caching pip dependencies
71+
72+
- name: Setup and start KinD cluster
73+
uses: ./common/github-actions/kind
74+
with:
75+
worker-nodes: 2 # Multiple nodes for testing Kueue scheduling
76+
77+
- name: Verify Kind cluster
78+
run: |
79+
echo "Checking Kind clusters..."
80+
kind get clusters
81+
echo "Current kubectl context:"
82+
kubectl config current-context
83+
echo "Checking nodes:"
84+
kubectl get nodes
85+
86+
- name: Build and install Kueue
87+
run: |
88+
cd kueue
89+
echo "Building Kueue..."
90+
make manifests
91+
make install
92+
make kind-image-build
93+
94+
# Get the actual cluster name
95+
CLUSTER_NAME=$(kind get clusters | head -n 1)
96+
echo "Using Kind cluster: ${CLUSTER_NAME}"
97+
98+
# Load Kueue image into Kind - this loads to all nodes
99+
IMAGE_TAG=$(git describe --tags --dirty --always)
100+
echo "Loading image: us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG}"
101+
kind load docker-image us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG} --name ${CLUSTER_NAME}
102+
103+
# Also load with 'main' tag as fallback
104+
docker tag us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG} us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:main
105+
kind load docker-image us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:main --name ${CLUSTER_NAME}
106+
107+
# Deploy Kueue
108+
kubectl apply --server-side --force-conflicts -k config/default
109+
110+
# Patch to use the specific image tag and Never pull policy
111+
kubectl patch deployment kueue-controller-manager -n kueue-system --type='json' -p='[
112+
{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:'${IMAGE_TAG}'"},
113+
{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value": "Never"}
114+
]'
115+
116+
# Wait for rollout to complete
117+
kubectl rollout status deployment/kueue-controller-manager -n kueue-system --timeout=120s
118+
119+
# Create a default LocalQueue in test namespaces to handle RayJobs that expect it
120+
# This prevents "Handling job with no workload" errors
121+
122+
cd ..
123+
124+
- name: Deploy KubeRay operator
125+
run: |
126+
echo "Installing KubeRay operator..."
127+
kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.2.2"
128+
kubectl wait --timeout=120s --for=condition=Available=true deployment -n ray-system kuberay-operator
129+
130+
- name: Add user to KinD
131+
uses: ./common/github-actions/kind-add-user
132+
with:
133+
user-name: sdk-user
134+
135+
- name: Configure RBAC for sdk user with limited permissions
136+
run: |
137+
# Basic permissions
138+
kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
139+
kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
140+
kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
141+
kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
142+
143+
# Ray permissions
144+
kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters
145+
kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
146+
kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs
147+
kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
148+
149+
# Kueue permissions
150+
kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
151+
kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
152+
kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues
153+
kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
154+
kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues
155+
kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
156+
kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads
157+
kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
158+
159+
# Additional permissions
160+
kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
161+
kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
162+
kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
163+
kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
164+
kubectl create clusterrole service-reader --verb=get,list,watch --resource=services
165+
kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
166+
kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
167+
kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
168+
kubectl create clusterrole node-reader --verb=get,list --resource=nodes
169+
kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
170+
kubectl config use-context sdk-user
171+
172+
- name: Setup test output directory
173+
run: |
174+
CODEFLARE_TEST_OUTPUT_DIR="${{ runner.temp }}/test-logs"
175+
mkdir -p ${CODEFLARE_TEST_OUTPUT_DIR}
176+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
177+
178+
- name: Run RayJob e2e tests
179+
run: |
180+
set -euo pipefail
181+
pip install poetry
182+
poetry install --with test,docs
183+
184+
# Install the SDK in editable mode
185+
pip install -e .
186+
187+
echo "Running RayJob e2e tests..."
188+
# Set environment variable to prevent default queue assignment for non-Kueue tests
189+
export DISABLE_DEFAULT_KUEUE_QUEUE=true
190+
191+
# Run only the tests that are designed for Kueue integration
192+
poetry run pytest -v -s ./tests/e2e/rayjob/rayjob_existing_cluster_test.py ./tests/e2e/rayjob/rayjob_lifecycled_cluster_test.py -x > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
193+
env:
194+
GRPC_DNS_RESOLVER: "native"
195+
196+
- name: Switch to kind-cluster context to print logs
197+
if: always()
198+
run: kubectl config use-context kind-cluster
199+
200+
- name: Print Pytest output log
201+
if: always()
202+
run: |
203+
echo "Printing Pytest output logs"
204+
cat ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/pytest_output.log || true
205+
206+
- name: Print Kueue operator logs
207+
if: always()
208+
run: |
209+
echo "Printing Kueue operator logs"
210+
kubectl logs -n kueue-system --tail -1 -l control-plane=controller-manager | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kueue-operator.log || true
211+
212+
- name: Print KubeRay operator logs
213+
if: always()
214+
run: |
215+
echo "Printing KubeRay operator logs"
216+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay-operator | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kuberay.log || true
217+
218+
- name: Export all KinD pod logs
219+
uses: ./common/github-actions/kind-export-logs
220+
if: always()
221+
with:
222+
output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
223+
224+
- name: Upload logs
225+
uses: actions/upload-artifact@v4
226+
if: always()
227+
with:
228+
name: logs
229+
retention-days: 10
230+
path: |
231+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
232+
if-no-files-found: warn

0 commit comments

Comments
 (0)