Skip to content

Commit cd79253

Browse files
andrew-anyscaleandrew-anyscale
committed
Migrate wheel builds to wanda (x86_64)
This migrates ray wheel and cpp wheel builds from CLI-based approach to wanda-based container builds for x86_64. Changes: - Add ray-wheel.wanda.yaml and Dockerfile for wheel builds - Add ray-cpp-core.wanda.yaml and ray-cpp-wheel.wanda.yaml for cpp wheels - Update build.rayci.yml wheel steps to use wanda - Add wheel upload steps that extract from wanda cache Topic: ray-wheel Signed-off-by: andrew <andrew@anyscale.com>
1 parent be77f0a commit cd79253

File tree

8 files changed

+391
-7
lines changed

8 files changed

+391
-7
lines changed

‎.buildkite/build.rayci.yml‎

Lines changed: 121 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,23 +35,138 @@ steps:
3535
HOSTTYPE: "x86_64"
3636
MANYLINUX_VERSION: "260103.868e54c"
3737

38-
- label: ":tapioca: build: wheel {{matrix}} (x86_64)"
39-
key: linux_wheels
38+
- name: ray-wheel-build
39+
label: "wanda: wheel py{{matrix}} (x86_64)"
40+
wanda: ci/docker/ray-wheel.wanda.yaml
41+
matrix:
42+
- "3.10"
43+
- "3.11"
44+
- "3.12"
45+
- "3.13"
46+
env:
47+
PYTHON_VERSION: "{{matrix}}"
48+
ARCH_SUFFIX: ""
49+
HOSTTYPE: "x86_64"
50+
MANYLINUX_VERSION: "260103.868e54c"
4051
tags:
4152
- release_wheels
4253
- linux_wheels
4354
- oss
44-
instance_type: large
55+
depends_on:
56+
- ray-core-build
57+
- ray-java-build
58+
- ray-dashboard-build
59+
60+
- name: ray-cpp-core-build
61+
label: "wanda: cpp core py{{matrix}} (x86_64)"
62+
wanda: ci/docker/ray-cpp-core.wanda.yaml
63+
matrix:
64+
- "3.10"
65+
- "3.11"
66+
- "3.12"
67+
env:
68+
PYTHON_VERSION: "{{matrix}}"
69+
ARCH_SUFFIX: ""
70+
HOSTTYPE: "x86_64"
71+
MANYLINUX_VERSION: "260103.868e54c"
72+
tags:
73+
- release_wheels
74+
- oss
75+
76+
- name: ray-cpp-wheel-build
77+
label: "wanda: cpp wheel py{{matrix}} (x86_64)"
78+
wanda: ci/docker/ray-cpp-wheel.wanda.yaml
79+
matrix:
80+
- "3.10"
81+
- "3.11"
82+
- "3.12"
83+
env:
84+
PYTHON_VERSION: "{{matrix}}"
85+
ARCH_SUFFIX: ""
86+
HOSTTYPE: "x86_64"
87+
MANYLINUX_VERSION: "260103.868e54c"
88+
tags:
89+
- release_wheels
90+
- linux_wheels
91+
- oss
92+
depends_on:
93+
- ray-core-build
94+
- ray-cpp-core-build
95+
- ray-java-build
96+
- ray-dashboard-build
97+
98+
# Upload wheels to S3
99+
# Gating: only on releases/* OR (master AND nightly)
100+
- label: ":s3: upload: wheel py{{matrix}} (x86_64)"
101+
key: linux_wheels_upload
102+
if: build.branch =~ /^releases\// || (build.branch == "master" && build.env("RAYCI_SCHEDULE") == "nightly")
103+
instance_type: small
45104
commands:
46-
- bazel run //ci/ray_ci:build_in_docker -- wheel --python-version {{matrix}} --architecture x86_64 --upload
105+
# Extract wheel from wanda image (Wanda cache in ECR)
106+
- wanda_image="$RAYCI_WORK_REPO:$RAYCI_BUILD_ID-ray-wheel-py{{matrix}}"
107+
- container_id=$(docker create $wanda_image)
108+
- mkdir -p /tmp/wheels
109+
- docker cp ${container_id}:/ /tmp/wheels/
110+
- docker rm ${container_id}
111+
- mkdir -p .whl
112+
- mv /tmp/wheels/*.whl .whl/
113+
- ./ci/build/copy_build_artifacts.sh wheel
47114
matrix:
48115
- "3.10"
49116
- "3.11"
50117
- "3.12"
51118
- "3.13"
52119
depends_on:
53-
- manylinux-x86_64
54-
- forge
120+
- ray-wheel-build
121+
tags:
122+
- release_wheels
123+
- skip-on-premerge
124+
125+
# Upload cpp wheels to S3
126+
# Gating: only on releases/* OR (master AND nightly)
127+
- label: ":s3: upload: cpp wheel py{{matrix}} (x86_64)"
128+
key: linux_cpp_wheels_upload
129+
if: build.branch =~ /^releases\// || (build.branch == "master" && build.env("RAYCI_SCHEDULE") == "nightly")
130+
instance_type: small
131+
commands:
132+
# Extract cpp wheel from wanda image (Wanda cache in ECR)
133+
- wanda_image="$RAYCI_WORK_REPO:$RAYCI_BUILD_ID-ray-cpp-wheel-py{{matrix}}"
134+
- container_id=$(docker create $wanda_image)
135+
- mkdir -p /tmp/wheels
136+
- docker cp ${container_id}:/ /tmp/wheels/
137+
- docker rm ${container_id}
138+
- mkdir -p .whl
139+
- mv /tmp/wheels/*.whl .whl/
140+
- ./ci/build/copy_build_artifacts.sh wheel
141+
matrix:
142+
- "3.10"
143+
- "3.11"
144+
- "3.12"
145+
depends_on:
146+
- ray-cpp-wheel-build
147+
tags:
148+
- release_wheels
149+
- skip-on-premerge
150+
151+
# Validate wheel can be extracted and installed (runs on PRs)
152+
- label: ":mag: validate: wheel py{{matrix}} (x86_64)"
153+
key: linux_wheels_validate
154+
instance_type: small
155+
commands:
156+
- wanda_image="$RAYCI_WORK_REPO:$RAYCI_BUILD_ID-ray-wheel-py{{matrix}}"
157+
- container_id=$(docker create $wanda_image)
158+
- mkdir -p /tmp/wheels
159+
- docker cp ${container_id}:/ /tmp/wheels/
160+
- docker rm ${container_id}
161+
- ls -la /tmp/wheels/*.whl
162+
- pip install /tmp/wheels/*.whl
163+
- python -c "import ray; print('ray version:', ray.__version__)"
164+
matrix:
165+
- "3.10"
166+
depends_on:
167+
- ray-wheel-build
168+
tags:
169+
- release_wheels
55170

56171
- label: ":tapioca: build: jar"
57172
key: java_wheels

‎.rayciversion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.21.0
1+
0.22.0

‎ci/docker/ray-cpp-core.Dockerfile‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# syntax=docker/dockerfile:1.3-labs
2+
#
3+
# Ray C++ Core Artifacts Builder
4+
# ==============================
5+
# Builds ray_cpp_pkg.zip containing C++ headers, libraries, and examples.
6+
#
7+
ARG ARCH_SUFFIX=
8+
ARG HOSTTYPE=x86_64
9+
ARG MANYLINUX_VERSION
10+
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
11+
12+
ARG PYTHON_VERSION=3.10
13+
ARG BUILDKITE_BAZEL_CACHE_URL
14+
ARG BUILDKITE_CACHE_READONLY
15+
ARG HOSTTYPE
16+
17+
WORKDIR /home/forge/ray
18+
19+
COPY . .
20+
21+
# Mounting cache dir for faster local rebuilds (architecture-specific to avoid toolchain conflicts)
22+
RUN --mount=type=cache,target=/home/forge/.cache,uid=2000,gid=100,id=bazel-cache-${HOSTTYPE}-${PYTHON_VERSION} \
23+
<<'EOF'
24+
#!/bin/bash
25+
set -euo pipefail
26+
27+
PY_CODE="${PYTHON_VERSION//./}"
28+
PY_BIN="cp${PY_CODE}-cp${PY_CODE}"
29+
export RAY_BUILD_ENV="manylinux_py${PY_BIN}"
30+
31+
sudo ln -sf "/opt/python/${PY_BIN}/bin/python3" /usr/local/bin/python3
32+
sudo ln -sf /usr/local/bin/python3 /usr/local/bin/python
33+
34+
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
35+
echo "build --remote_upload_local_results=false" >> "$HOME/.bazelrc"
36+
fi
37+
38+
echo "build --repository_cache=/home/forge/.cache/bazel-repo" >> "$HOME/.bazelrc"
39+
40+
bazelisk build --config=ci //cpp:ray_cpp_pkg_zip
41+
42+
cp bazel-bin/cpp/ray_cpp_pkg.zip /home/forge/ray_cpp_pkg.zip
43+
44+
EOF
45+
46+
FROM scratch
47+
48+
COPY --from=builder /home/forge/ray_cpp_pkg.zip /

‎ci/docker/ray-cpp-core.wanda.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: "ray-cpp-core-py$PYTHON_VERSION$ARCH_SUFFIX"
2+
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
3+
dockerfile: ci/docker/ray-cpp-core.Dockerfile
4+
srcs:
5+
- .bazelversion
6+
- .bazelrc
7+
- WORKSPACE
8+
- BUILD.bazel
9+
- bazel/
10+
- thirdparty/
11+
- src/
12+
- cpp/
13+
- python/ray/__init__.py
14+
- python/ray/_raylet.pxd
15+
- python/ray/_raylet.pyi
16+
- python/ray/_raylet.pyx
17+
- python/ray/includes/
18+
- java/BUILD.bazel
19+
- java/dependencies.bzl
20+
- release/BUILD.bazel
21+
- release/requirements.txt
22+
- release/requirements_py310.txt
23+
build_args:
24+
- PYTHON_VERSION
25+
- ARCH_SUFFIX
26+
- HOSTTYPE
27+
- MANYLINUX_VERSION
28+
- BUILDKITE_BAZEL_CACHE_URL
29+
build_hint_args:
30+
- BUILDKITE_CACHE_READONLY

‎ci/docker/ray-cpp-wheel.wanda.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: "ray-cpp-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"
2+
disable_caching: true
3+
froms:
4+
- "rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"
5+
- "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ binaries (ray_pkg.zip)
6+
- "cr.ray.io/rayproject/ray-cpp-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ headers/libs (ray_cpp_pkg.zip)
7+
- "cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX" # Java JARs
8+
- "cr.ray.io/rayproject/ray-dashboard" # Dashboard
9+
dockerfile: ci/docker/ray-wheel.Dockerfile
10+
srcs:
11+
- pyproject.toml
12+
- README.rst
13+
- ci/build/build-manylinux-wheel.sh
14+
- python/
15+
- rllib/
16+
build_args:
17+
- PYTHON_VERSION
18+
- MANYLINUX_VERSION
19+
- HOSTTYPE
20+
- BUILDKITE_COMMIT
21+
- ARCH_SUFFIX
22+
- WHEEL_TYPE=cpp
23+
- RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
24+
- RAY_CPP_CORE_IMAGE=cr.ray.io/rayproject/ray-cpp-core-py$PYTHON_VERSION$ARCH_SUFFIX
25+
- RAY_JAVA_IMAGE=cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX
26+
- RAY_DASHBOARD_IMAGE=cr.ray.io/rayproject/ray-dashboard

‎ci/docker/ray-wheel.Dockerfile‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# syntax=docker/dockerfile:1.3-labs
2+
#
3+
# Ray Wheel Builder (Unified)
4+
# ===========================
5+
# Builds manylinux2014-compatible wheels using pre-built C++ artifacts from wanda cache.
6+
#
7+
# Build Types:
8+
# - ray wheel: WHEEL_TYPE=ray (default) - Standard Ray Python wheel
9+
# - ray-cpp wheel: WHEEL_TYPE=cpp - Ray C++ API wheel
10+
#
11+
# GLIBC Compatibility:
12+
# --------------------
13+
# manylinux2014 requires GLIBC <= 2.17 for broad Linux compatibility.
14+
# The pre-built _raylet.so is compiled inside manylinux2014 with GLIBC 2.17.
15+
#
16+
17+
ARG RAY_CORE_IMAGE
18+
ARG RAY_CPP_CORE_IMAGE=scratch
19+
ARG RAY_JAVA_IMAGE
20+
ARG RAY_DASHBOARD_IMAGE
21+
ARG MANYLINUX_VERSION
22+
ARG HOSTTYPE
23+
24+
FROM ${RAY_CORE_IMAGE} AS ray-core
25+
FROM ${RAY_CPP_CORE_IMAGE} AS ray-cpp-core
26+
FROM ${RAY_JAVA_IMAGE} AS ray-java
27+
FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
28+
29+
# Main build stage - manylinux2014 provides GLIBC 2.17
30+
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
31+
32+
ARG PYTHON_VERSION=3.10
33+
ARG BUILDKITE_COMMIT
34+
ARG WHEEL_TYPE=ray
35+
36+
# Set environment variables for the build
37+
# - BUILDKITE_COMMIT: Used for ray.__commit__. Defaults to "unknown" for local builds.
38+
# - SKIP_BAZEL_BUILD=1: Skip bazel build, use pre-built artifacts from ray-core/ray-java/ray-dashboard
39+
# - RAY_DISABLE_EXTRA_CPP: 1 for ray wheel only, 0 for ray-cpp wheel
40+
# - WHEEL_TYPE: "ray" or "cpp" - determines which wheel to build
41+
ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown} \
42+
PYTHON_VERSION=${PYTHON_VERSION} \
43+
SKIP_BAZEL_BUILD=1 \
44+
WHEEL_TYPE=${WHEEL_TYPE}
45+
46+
WORKDIR /home/forge/ray
47+
48+
# Copy artifacts from all stages
49+
COPY --from=ray-core /ray_pkg.zip /tmp/
50+
COPY --from=ray-core /ray_py_proto.zip /tmp/
51+
COPY --from=ray-java /ray_java_pkg.zip /tmp/
52+
COPY --from=ray-dashboard /dashboard.tar.gz /tmp/
53+
54+
# Source files needed for wheel build
55+
COPY --chown=2000:100 ci/build/build-manylinux-wheel.sh ci/build/
56+
COPY --chown=2000:100 README.rst pyproject.toml ./
57+
COPY --chown=2000:100 rllib/ rllib/
58+
COPY --chown=2000:100 python/ python/
59+
60+
USER forge
61+
# Note: ray-cpp-core may be "scratch" (empty) for ray-only builds
62+
RUN --mount=from=ray-cpp-core,source=/,target=/ray-cpp-core,ro \
63+
<<'EOF'
64+
#!/bin/bash
65+
set -euo pipefail
66+
67+
PY_VERSION="${PYTHON_VERSION//./}"
68+
PY_BIN="cp${PY_VERSION}-cp${PY_VERSION}"
69+
70+
# Verify required artifacts exist before unpacking
71+
for f in /tmp/ray_pkg.zip /tmp/ray_py_proto.zip /tmp/ray_java_pkg.zip /tmp/dashboard.tar.gz; do
72+
[[ -f "$f" ]] || { echo "ERROR: missing artifact: $f"; exit 1; }
73+
done
74+
75+
# Clean extraction dirs to avoid stale leftovers
76+
rm -rf /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg
77+
mkdir -p /tmp/ray_pkg /tmp/ray_java_pkg
78+
79+
# Unpack common pre-built artifacts
80+
unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
81+
unzip -o /tmp/ray_py_proto.zip -d python/
82+
unzip -o /tmp/ray_java_pkg.zip -d /tmp/ray_java_pkg
83+
mkdir -p python/ray/dashboard/client/build
84+
tar -xzf /tmp/dashboard.tar.gz -C python/ray/dashboard/client/build/
85+
86+
# C++ core artifacts
87+
cp -r /tmp/ray_pkg/ray/* python/ray/
88+
89+
# Java JARs
90+
cp -r /tmp/ray_java_pkg/ray/* python/ray/
91+
92+
# Handle wheel type specific setup
93+
if [[ "$WHEEL_TYPE" == "cpp" ]]; then
94+
# C++ API artifacts (headers, libs, examples)
95+
if [[ -f /ray-cpp-core/ray_cpp_pkg.zip ]]; then
96+
mkdir -p /tmp/ray_cpp_pkg
97+
unzip -o /ray-cpp-core/ray_cpp_pkg.zip -d /tmp/ray_cpp_pkg
98+
cp -r /tmp/ray_cpp_pkg/ray/cpp python/ray/
99+
else
100+
echo "ERROR: ray_cpp_pkg.zip not found for cpp wheel build"
101+
exit 1
102+
fi
103+
export RAY_DISABLE_EXTRA_CPP=0
104+
else
105+
export RAY_DISABLE_EXTRA_CPP=1
106+
fi
107+
108+
# Build wheels
109+
./ci/build/build-manylinux-wheel.sh "$PY_BIN"
110+
111+
# Filter output based on wheel type
112+
if [[ "$WHEEL_TYPE" == "cpp" ]]; then
113+
# Keep only ray-cpp wheel
114+
rm -f .whl/ray-[0-9]*.whl
115+
fi
116+
117+
# Sanity check: ensure wheels exist
118+
shopt -s nullglob
119+
wheels=(.whl/*.whl)
120+
if (( ${#wheels[@]} == 0 )); then
121+
echo "ERROR: No wheels produced in .whl/"
122+
ls -la .whl || true
123+
exit 1
124+
fi
125+
126+
EOF
127+
128+
FROM scratch
129+
COPY --from=builder /home/forge/ray/.whl/*.whl /

0 commit comments

Comments
 (0)