Skip to content

Commit 6c4f545

Browse files
kryanbeanekryanbeane
committed
RHOAIENG-32532: Run RayJob tests in CI
1 parent d259ec1 commit 6c4f545

File tree

1 file changed

+199
-0
lines changed

1 file changed

+199
-0
lines changed

‎.github/workflows/rayjob_e2e_tests.yaml‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
name: rayjob-e2e-with-kueue
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- 'release-*'
8+
- ray-jobs-feature
9+
paths-ignore:
10+
- 'docs/**'
11+
- '**.adoc'
12+
- '**.md'
13+
- 'LICENSE'
14+
15+
concurrency:
16+
group: ${{ github.head_ref }}-${{ github.workflow }}
17+
cancel-in-progress: true
18+
19+
env:
20+
CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
21+
KUEUE_VERSION: "v0.13.3"
22+
23+
jobs:
24+
kubernetes:
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
with:
31+
submodules: recursive
32+
33+
- name: Checkout common repo code
34+
uses: actions/checkout@v4
35+
with:
36+
repository: 'project-codeflare/codeflare-common'
37+
ref: 'main'
38+
path: 'common'
39+
40+
41+
- name: Set up specific Python version
42+
uses: actions/setup-python@v5
43+
with:
44+
python-version: '3.11'
45+
cache: 'pip' # caching pip dependencies
46+
47+
- name: Setup and start KinD cluster
48+
uses: ./common/github-actions/kind
49+
with:
50+
worker-nodes: 2 # Multiple nodes for testing Kueue scheduling
51+
52+
- name: Verify Kind cluster
53+
run: |
54+
echo "Checking Kind clusters..."
55+
kind get clusters
56+
echo "Current kubectl context:"
57+
kubectl config current-context
58+
echo "Checking nodes:"
59+
kubectl get nodes
60+
61+
- name: Deploy Kueue
62+
run: |
63+
echo "Deploying Kueue ${KUEUE_VERSION}"
64+
kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml
65+
66+
# Sleep until the kueue manager is running
67+
echo "Waiting for pods in the kueue-system namespace to become ready"
68+
while [[ $(kubectl get pods -n kueue-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]]
69+
do
70+
echo -n "." && sleep 1;
71+
done
72+
echo ""
73+
74+
sleep 5
75+
76+
- name: Deploy KubeRay operator
77+
run: |
78+
KUBERAY_VERSION="v1.4.2"
79+
echo "Deploying KubeRay ${KUBERAY_VERSION}"
80+
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
81+
82+
# Wait for KubeRay operator to be ready
83+
echo "Waiting for KubeRay operator to become ready..."
84+
kubectl wait --for=condition=Available --timeout=300s deployment/kuberay-operator -n ray-system || {
85+
echo "KubeRay operator not ready, checking pods:"
86+
kubectl get pods -n ray-system
87+
kubectl describe pods -n ray-system
88+
exit 1
89+
}
90+
91+
- name: Add user to KinD
92+
uses: ./common/github-actions/kind-add-user
93+
with:
94+
user-name: sdk-user
95+
96+
- name: Configure RBAC for sdk user with limited permissions
97+
run: |
98+
# Basic permissions
99+
kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
100+
kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
101+
kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
102+
kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
103+
104+
# Ray permissions
105+
kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters
106+
kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
107+
kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs
108+
kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
109+
110+
# Kueue permissions
111+
kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
112+
kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
113+
kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues
114+
kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
115+
kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues
116+
kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
117+
kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads
118+
kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
119+
120+
# Additional permissions
121+
kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
122+
kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
123+
kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
124+
kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
125+
kubectl create clusterrole service-reader --verb=get,list,watch --resource=services
126+
kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
127+
kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
128+
kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
129+
kubectl create clusterrole node-reader --verb=get,list --resource=nodes
130+
kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
131+
kubectl config use-context sdk-user
132+
133+
- name: Setup test output directory
134+
run: |
135+
CODEFLARE_TEST_OUTPUT_DIR="${{ runner.temp }}/test-logs"
136+
mkdir -p ${CODEFLARE_TEST_OUTPUT_DIR}
137+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
138+
139+
- name: Run RayJob e2e tests
140+
run: |
141+
set -euo pipefail
142+
pip install poetry
143+
poetry install --with test,docs
144+
145+
# Install the SDK in editable mode
146+
pip install -e .
147+
148+
echo "Running RayJob e2e tests..."
149+
# Set environment variable to prevent default queue assignment for non-Kueue tests
150+
export DISABLE_DEFAULT_KUEUE_QUEUE=true
151+
152+
# Run only the tests that are designed for Kueue integration
153+
poetry run pytest -v -s ./tests/e2e/rayjob/rayjob_existing_cluster_test.py ./tests/e2e/rayjob/rayjob_lifecycled_cluster_test.py -x > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
154+
env:
155+
GRPC_DNS_RESOLVER: "native"
156+
157+
- name: Switch to kind-cluster context to print logs
158+
if: always()
159+
run: kubectl config use-context kind-cluster
160+
161+
- name: Print Pytest output log
162+
if: always()
163+
run: |
164+
echo "Printing Pytest output logs"
165+
cat ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/pytest_output.log || true
166+
167+
- name: Print Kueue operator logs
168+
if: always()
169+
run: |
170+
echo "Printing Kueue operator logs"
171+
kubectl logs -n kueue-system --tail -1 -l control-plane=controller-manager | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kueue-operator.log || true
172+
173+
- name: Print KubeRay operator logs
174+
if: always()
175+
run: |
176+
echo "Printing KubeRay operator logs"
177+
echo "Checking ray-system namespace contents:"
178+
kubectl get all -n ray-system || true
179+
echo "Attempting to get KubeRay logs with different selectors:"
180+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay-operator | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kuberay.log || \
181+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/component=kuberay-operator | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kuberay.log || \
182+
kubectl logs -n ray-system --tail -1 deployment/kuberay-operator | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kuberay.log || \
183+
echo "Could not find KubeRay operator logs"
184+
185+
- name: Export all KinD pod logs
186+
uses: ./common/github-actions/kind-export-logs
187+
if: always()
188+
with:
189+
output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
190+
191+
- name: Upload logs
192+
uses: actions/upload-artifact@v4
193+
if: always()
194+
with:
195+
name: logs
196+
retention-days: 10
197+
path: |
198+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
199+
if-no-files-found: warn

0 commit comments

Comments
 (0)