Skip to content

Commit e4583b2

Browse files
kryanbeanekryanbeane
committed
RHOAIENG-32532: Run RayJob tests in CI
1 parent d259ec1 commit e4583b2

File tree

1 file changed

+225
-0
lines changed

1 file changed

+225
-0
lines changed

‎.github/workflows/rayjob_e2e_tests.yaml‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
name: rayjob-e2e-with-kueue
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- 'release-*'
8+
- ray-jobs-feature
9+
paths-ignore:
10+
- 'docs/**'
11+
- '**.adoc'
12+
- '**.md'
13+
- 'LICENSE'
14+
15+
concurrency:
16+
group: ${{ github.head_ref }}-${{ github.workflow }}
17+
cancel-in-progress: true
18+
19+
env:
20+
CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
21+
KUEUE_VERSION: "v0.13.3"
22+
23+
jobs:
24+
kubernetes:
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
with:
31+
submodules: recursive
32+
33+
- name: Checkout common repo code
34+
uses: actions/checkout@v4
35+
with:
36+
repository: 'project-codeflare/codeflare-common'
37+
ref: 'main'
38+
path: 'common'
39+
40+
- name: Checkout CodeFlare operator repository
41+
uses: actions/checkout@v4
42+
with:
43+
repository: project-codeflare/codeflare-operator
44+
path: codeflare-operator
45+
46+
- name: Checkout Kueue repository
47+
uses: actions/checkout@v4
48+
with:
49+
repository: kubernetes-sigs/kueue
50+
path: kueue
51+
ref: main
52+
53+
- name: Set Go
54+
uses: actions/setup-go@v5
55+
with:
56+
go-version-file: './codeflare-operator/go.mod'
57+
cache-dependency-path: |
58+
./codeflare-operator/go.sum
59+
./kueue/go.sum
60+
61+
- name: Set up gotestfmt
62+
uses: gotesttools/gotestfmt-action@v2
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Set up specific Python version
67+
uses: actions/setup-python@v5
68+
with:
69+
python-version: '3.11'
70+
cache: 'pip' # caching pip dependencies
71+
72+
- name: Setup and start KinD cluster
73+
uses: ./common/github-actions/kind
74+
with:
75+
worker-nodes: 2 # Multiple nodes for testing Kueue scheduling
76+
77+
- name: Verify Kind cluster
78+
run: |
79+
echo "Checking Kind clusters..."
80+
kind get clusters
81+
echo "Current kubectl context:"
82+
kubectl config current-context
83+
echo "Checking nodes:"
84+
kubectl get nodes
85+
86+
- name: Build and install Kueue
87+
run: |
88+
cd kueue
89+
echo "Building Kueue..."
90+
make manifests
91+
make install
92+
make kind-image-build
93+
94+
# Get the actual cluster name
95+
CLUSTER_NAME=$(kind get clusters | head -n 1)
96+
echo "Using Kind cluster: ${CLUSTER_NAME}"
97+
98+
# Load Kueue image into Kind - this loads to all nodes
99+
IMAGE_TAG=$(git describe --tags --dirty --always)
100+
echo "Loading image: us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG}"
101+
kind load docker-image us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG} --name ${CLUSTER_NAME}
102+
103+
# Also load with 'main' tag as fallback
104+
docker tag us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG} us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:main
105+
kind load docker-image us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:main --name ${CLUSTER_NAME}
106+
107+
# Deploy Kueue
108+
kubectl apply --server-side --force-conflicts -k config/default
109+
110+
# First patch to use the specific image tag and Never pull policy
111+
kubectl patch deployment kueue-controller-manager -n kueue-system --type='json' -p='[
112+
{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:'${IMAGE_TAG}'"},
113+
{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value": "Never"}
114+
]'
115+
116+
# Wait for rollout to complete
117+
kubectl rollout status deployment/kueue-controller-manager -n kueue-system --timeout=120s
118+
119+
cd ..
120+
121+
- name: Deploy KubeRay operator
122+
run: |
123+
echo "Installing KubeRay operator..."
124+
kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.2.2"
125+
kubectl wait --timeout=120s --for=condition=Available=true deployment -n ray-system kuberay-operator
126+
127+
- name: Add user to KinD
128+
uses: ./common/github-actions/kind-add-user
129+
with:
130+
user-name: sdk-user
131+
132+
- name: Configure RBAC for sdk user with limited permissions
133+
run: |
134+
# Basic permissions
135+
kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
136+
kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
137+
kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
138+
kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
139+
140+
# Ray permissions
141+
kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters
142+
kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
143+
kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs
144+
kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
145+
146+
# Kueue permissions
147+
kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
148+
kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
149+
kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues
150+
kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
151+
kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues
152+
kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
153+
kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads
154+
kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
155+
156+
# Additional permissions
157+
kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
158+
kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
159+
kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
160+
kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
161+
kubectl create clusterrole service-reader --verb=get,list,watch --resource=services
162+
kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
163+
kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
164+
kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
165+
kubectl create clusterrole node-reader --verb=get,list --resource=nodes
166+
kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
167+
kubectl config use-context sdk-user
168+
169+
- name: Setup test output directory
170+
run: |
171+
CODEFLARE_TEST_OUTPUT_DIR="${{ runner.temp }}/test-logs"
172+
mkdir -p ${CODEFLARE_TEST_OUTPUT_DIR}
173+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
174+
175+
- name: Run RayJob e2e tests
176+
run: |
177+
set -euo pipefail
178+
pip install poetry
179+
poetry install --with test,docs
180+
181+
# Install the SDK in editable mode
182+
pip install -e .
183+
184+
echo "Running RayJob e2e tests..."
185+
poetry run pytest -v -s ./tests/e2e/rayjob/ -x > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
186+
env:
187+
GRPC_DNS_RESOLVER: "native"
188+
189+
- name: Switch to kind-cluster context to print logs
190+
if: always()
191+
run: kubectl config use-context kind-cluster
192+
193+
- name: Print Pytest output log
194+
if: always()
195+
run: |
196+
echo "Printing Pytest output logs"
197+
cat ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/pytest_output.log || true
198+
199+
- name: Print Kueue operator logs
200+
if: always()
201+
run: |
202+
echo "Printing Kueue operator logs"
203+
kubectl logs -n kueue-system --tail -1 -l control-plane=controller-manager | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kueue-operator.log || true
204+
205+
- name: Print KubeRay operator logs
206+
if: always()
207+
run: |
208+
echo "Printing KubeRay operator logs"
209+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay-operator | tee ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/kuberay.log || true
210+
211+
- name: Export all KinD pod logs
212+
uses: ./common/github-actions/kind-export-logs
213+
if: always()
214+
with:
215+
output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
216+
217+
- name: Upload logs
218+
uses: actions/upload-artifact@v4
219+
if: always()
220+
with:
221+
name: logs
222+
retention-days: 10
223+
path: |
224+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
225+
if-no-files-found: warn

0 commit comments

Comments
 (0)