Skip to content

Commit 69766cd

Browse files
Alexei-V-Ivanov-AMDAlexei-V-Ivanov-AMD
authored
Enabling AMD tests in AMD-CI pipeline. (#189)
1 parent 9bd7aa9 commit 69766cd

File tree

2 files changed

+552
-0
lines changed

2 files changed

+552
-0
lines changed

‎buildkite/bootstrap-amd.sh‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
2+
#!/bin/bash
3+
4+
set -euo pipefail
5+
6+
if [[ -z "${RUN_ALL:-}" ]]; then
7+
RUN_ALL=0
8+
fi
9+
10+
if [[ -z "${NIGHTLY:-}" ]]; then
11+
NIGHTLY=0
12+
fi
13+
14+
if [[ -z "${VLLM_CI_BRANCH:-}" ]]; then
15+
VLLM_CI_BRANCH="main"
16+
fi
17+
18+
if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
19+
AMD_MIRROR_HW="amdproduction"
20+
fi
21+
22+
if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
23+
DOCS_ONLY_DISABLE=0
24+
fi
25+
26+
fail_fast() {
27+
DISABLE_LABEL="ci-no-fail-fast"
28+
# If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
29+
if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
30+
PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
31+
if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
32+
echo false
33+
else
34+
echo true
35+
fi
36+
else
37+
echo false # not a PR or BUILDKITE_PULL_REQUEST not set
38+
fi
39+
}
40+
41+
if [[ -z "${COV_ENABLED:-}" ]]; then
42+
COV_ENABLED=0
43+
fi
44+
45+
upload_pipeline() {
46+
echo "Uploading pipeline..."
47+
# Install minijinja
48+
ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
49+
curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
50+
source /var/lib/buildkite-agent/.cargo/env
51+
52+
if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
53+
curl -o .buildkite/test-template.j2 \
54+
https://raw.githubusercontent.com/vllm-project/ci-infra/"$VLLM_CI_BRANCH"/buildkite/test-template-fastcheck.j2
55+
else
56+
curl -o .buildkite/test-template.j2 \
57+
"https://raw.githubusercontent.com/vllm-project/ci-infra/$VLLM_CI_BRANCH/buildkite/test-template-amd.j2?$(date +%s)"
58+
fi
59+
60+
61+
# (WIP) Use pipeline generator instead of jinja template
62+
if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
63+
python -m pip install click pydantic
64+
python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
65+
buildkite-agent pipeline upload .buildkite/pipeline.yaml
66+
exit 0
67+
fi
68+
echo "List file diff: $LIST_FILE_DIFF"
69+
echo "Run all: $RUN_ALL"
70+
echo "Nightly: $NIGHTLY"
71+
echo "AMD Mirror HW: $AMD_MIRROR_HW"
72+
73+
FAIL_FAST=$(fail_fast)
74+
75+
cd .buildkite
76+
(
77+
set -x
78+
# Output pipeline.yaml with all blank lines removed
79+
minijinja-cli test-template.j2 test-amd.yaml \
80+
-D branch="$BUILDKITE_BRANCH" \
81+
-D list_file_diff="$LIST_FILE_DIFF" \
82+
-D run_all="$RUN_ALL" \
83+
-D nightly="$NIGHTLY" \
84+
-D mirror_hw="$AMD_MIRROR_HW" \
85+
-D fail_fast="$FAIL_FAST" \
86+
-D vllm_use_precompiled="$VLLM_USE_PRECOMPILED" \
87+
-D cov_enabled="$COV_ENABLED" \
88+
-D vllm_ci_branch="$VLLM_CI_BRANCH" \
89+
| sed '/^[[:space:]]*$/d' \
90+
> pipeline.yaml
91+
)
92+
cat pipeline.yaml
93+
buildkite-agent artifact upload pipeline.yaml
94+
buildkite-agent pipeline upload pipeline.yaml
95+
exit 0
96+
}
97+
98+
get_diff() {
99+
$(git add .)
100+
echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
101+
}
102+
103+
get_diff_main() {
104+
$(git add .)
105+
echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
106+
}
107+
108+
file_diff=$(get_diff)
109+
if [[ $BUILDKITE_BRANCH == "main" ]]; then
110+
file_diff=$(get_diff_main)
111+
fi
112+
113+
# ----------------------------------------------------------------------
114+
# Early exit start: skip pipeline if conditions are met
115+
# ----------------------------------------------------------------------
116+
117+
# skip pipeline if all changed files are under docs/
118+
if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
119+
if [[ -n "${file_diff:-}" ]]; then
120+
docs_only=1
121+
# Robust iteration over newline-separated file_diff
122+
while IFS= read -r f; do
123+
[[ -z "$f" ]] && continue
124+
# **Policy:** only skip if *every* path starts with docs/
125+
if [[ "$f" != docs/* ]]; then
126+
docs_only=0
127+
break
128+
fi
129+
done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
130+
131+
if [[ "$docs_only" -eq 1 ]]; then
132+
buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
133+
134+
\`\`\`
135+
${file_diff}
136+
\`\`\`" --style "info" || true
137+
echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
138+
exit 0
139+
fi
140+
fi
141+
fi
142+
143+
# ----------------------------------------------------------------------
144+
# Early exit end
145+
# ----------------------------------------------------------------------
146+
147+
patterns=(
148+
"docker/Dockerfile"
149+
"CMakeLists.txt"
150+
"requirements/common.txt"
151+
"requirements/cuda.txt"
152+
"requirements/build.txt"
153+
"requirements/test.txt"
154+
"setup.py"
155+
"csrc/"
156+
"cmake/"
157+
)
158+
159+
ignore_patterns=(
160+
"docker/Dockerfile."
161+
"csrc/cpu"
162+
"csrc/rocm"
163+
"cmake/hipify.py"
164+
"cmake/cpu_extension.cmake"
165+
)
166+
167+
for file in $file_diff; do
168+
# First check if file matches any pattern
169+
matches_pattern=0
170+
for pattern in "${patterns[@]}"; do
171+
if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
172+
matches_pattern=1
173+
break
174+
fi
175+
done
176+
177+
# If file matches pattern, check it's not in ignore patterns
178+
if [[ $matches_pattern -eq 1 ]]; then
179+
matches_ignore=0
180+
for ignore in "${ignore_patterns[@]}"; do
181+
if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
182+
matches_ignore=1
183+
break
184+
fi
185+
done
186+
187+
if [[ $matches_ignore -eq 0 ]]; then
188+
RUN_ALL=1
189+
echo "Found changes: $file. Run all tests"
190+
break
191+
fi
192+
fi
193+
done
194+
195+
# Decide whether to use precompiled wheels
196+
# Relies on existing patterns array as a basis.
197+
if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then
198+
echo "VLLM_USE_PRECOMPILED is already set to: $VLLM_USE_PRECOMPILED"
199+
elif [[ $RUN_ALL -eq 1 ]]; then
200+
export VLLM_USE_PRECOMPILED=0
201+
echo "Detected critical changes, building wheels from source"
202+
else
203+
export VLLM_USE_PRECOMPILED=1
204+
echo "No critical changes, using precompiled wheels"
205+
fi
206+
207+
208+
LIST_FILE_DIFF=$(get_diff | tr ' ' '|')
209+
if [[ $BUILDKITE_BRANCH == "main" ]]; then
210+
LIST_FILE_DIFF=$(get_diff_main | tr ' ' '|')
211+
fi
212+
upload_pipeline

0 commit comments

Comments
 (0)