|
| 1 | +# syntax=docker/dockerfile:1.3-labs |
| 2 | +# |
| 3 | +# Ray Wheel Builder (Unified) |
| 4 | +# =========================== |
| 5 | +# Builds manylinux2014-compatible wheels using pre-built C++ artifacts from wanda cache. |
| 6 | +# |
| 7 | +# Build Types: |
| 8 | +# - ray wheel: WHEEL_TYPE=ray (default) - Standard Ray Python wheel |
| 9 | +# - ray-cpp wheel: WHEEL_TYPE=cpp - Ray C++ API wheel |
| 10 | +# |
| 11 | +# GLIBC Compatibility: |
| 12 | +# -------------------- |
| 13 | +# manylinux2014 requires GLIBC <= 2.17 for broad Linux compatibility. |
| 14 | +# The pre-built _raylet.so is compiled inside manylinux2014 with GLIBC 2.17. |
| 15 | +# |
| 16 | + |
| 17 | +ARG RAY_CORE_IMAGE |
| 18 | +ARG RAY_CPP_CORE_IMAGE=scratch |
| 19 | +ARG RAY_JAVA_IMAGE |
| 20 | +ARG RAY_DASHBOARD_IMAGE |
| 21 | +ARG MANYLINUX_VERSION |
| 22 | +ARG HOSTTYPE |
| 23 | + |
| 24 | +FROM ${RAY_CORE_IMAGE} AS ray-core |
| 25 | +FROM ${RAY_CPP_CORE_IMAGE} AS ray-cpp-core |
| 26 | +FROM ${RAY_JAVA_IMAGE} AS ray-java |
| 27 | +FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard |
| 28 | + |
| 29 | +# Main build stage - manylinux2014 provides GLIBC 2.17 |
| 30 | +FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder |
| 31 | + |
| 32 | +ARG PYTHON_VERSION=3.10 |
| 33 | +ARG BUILDKITE_COMMIT |
| 34 | +ARG WHEEL_TYPE=ray |
| 35 | + |
| 36 | +# Set environment variables for the build |
| 37 | +# - BUILDKITE_COMMIT: Used for ray.__commit__. Defaults to "unknown" for local builds. |
| 38 | +# - SKIP_BAZEL_BUILD=1: Skip bazel build, use pre-built artifacts from ray-core/ray-java/ray-dashboard |
| 39 | +# - RAY_DISABLE_EXTRA_CPP: 1 for ray wheel only, 0 for ray-cpp wheel |
| 40 | +# - WHEEL_TYPE: "ray" or "cpp" - determines which wheel to build |
| 41 | +ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown} \ |
| 42 | + PYTHON_VERSION=${PYTHON_VERSION} \ |
| 43 | + SKIP_BAZEL_BUILD=1 \ |
| 44 | + WHEEL_TYPE=${WHEEL_TYPE} |
| 45 | + |
| 46 | +WORKDIR /home/forge/ray |
| 47 | + |
| 48 | +# Copy artifacts from all stages |
| 49 | +COPY --from=ray-core /ray_pkg.zip /tmp/ |
| 50 | +COPY --from=ray-core /ray_py_proto.zip /tmp/ |
| 51 | +COPY --from=ray-java /ray_java_pkg.zip /tmp/ |
| 52 | +COPY --from=ray-dashboard /dashboard.tar.gz /tmp/ |
| 53 | + |
| 54 | +# Source files needed for wheel build |
| 55 | +COPY --chown=2000:100 ci/build/build-manylinux-wheel.sh ci/build/ |
| 56 | +COPY --chown=2000:100 README.rst pyproject.toml ./ |
| 57 | +COPY --chown=2000:100 rllib/ rllib/ |
| 58 | +COPY --chown=2000:100 python/ python/ |
| 59 | + |
| 60 | +USER forge |
| 61 | +# Note: ray-cpp-core may be "scratch" (empty) for ray-only builds |
| 62 | +RUN --mount=from=ray-cpp-core,source=/,target=/ray-cpp-core,ro \ |
| 63 | + <<'EOF' |
| 64 | +#!/bin/bash |
| 65 | +set -euo pipefail |
| 66 | + |
| 67 | +PY_VERSION="${PYTHON_VERSION//./}" |
| 68 | +PY_BIN="cp${PY_VERSION}-cp${PY_VERSION}" |
| 69 | + |
| 70 | +# Verify required artifacts exist before unpacking |
| 71 | +for f in /tmp/ray_pkg.zip /tmp/ray_py_proto.zip /tmp/ray_java_pkg.zip /tmp/dashboard.tar.gz; do |
| 72 | + [[ -f "$f" ]] || { echo "ERROR: missing artifact: $f"; exit 1; } |
| 73 | +done |
| 74 | + |
| 75 | +# Clean extraction dirs to avoid stale leftovers |
| 76 | +rm -rf /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg |
| 77 | +mkdir -p /tmp/ray_pkg /tmp/ray_java_pkg |
| 78 | + |
| 79 | +# Unpack common pre-built artifacts |
| 80 | +unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg |
| 81 | +unzip -o /tmp/ray_py_proto.zip -d python/ |
| 82 | +unzip -o /tmp/ray_java_pkg.zip -d /tmp/ray_java_pkg |
| 83 | +mkdir -p python/ray/dashboard/client/build |
| 84 | +tar -xzf /tmp/dashboard.tar.gz -C python/ray/dashboard/client/build/ |
| 85 | + |
| 86 | +# C++ core artifacts |
| 87 | +cp -r /tmp/ray_pkg/ray/* python/ray/ |
| 88 | + |
| 89 | +# Java JARs |
| 90 | +cp -r /tmp/ray_java_pkg/ray/* python/ray/ |
| 91 | + |
| 92 | +# Handle wheel type specific setup |
| 93 | +if [[ "$WHEEL_TYPE" == "cpp" ]]; then |
| 94 | + # C++ API artifacts (headers, libs, examples) |
| 95 | + if [[ -f /ray-cpp-core/ray_cpp_pkg.zip ]]; then |
| 96 | + mkdir -p /tmp/ray_cpp_pkg |
| 97 | + unzip -o /ray-cpp-core/ray_cpp_pkg.zip -d /tmp/ray_cpp_pkg |
| 98 | + cp -r /tmp/ray_cpp_pkg/ray/cpp python/ray/ |
| 99 | + else |
| 100 | + echo "ERROR: ray_cpp_pkg.zip not found for cpp wheel build" |
| 101 | + exit 1 |
| 102 | + fi |
| 103 | + export RAY_DISABLE_EXTRA_CPP=0 |
| 104 | +else |
| 105 | + export RAY_DISABLE_EXTRA_CPP=1 |
| 106 | +fi |
| 107 | + |
| 108 | +# Build wheels |
| 109 | +./ci/build/build-manylinux-wheel.sh "$PY_BIN" |
| 110 | + |
| 111 | +# Filter output based on wheel type |
| 112 | +if [[ "$WHEEL_TYPE" == "cpp" ]]; then |
| 113 | + # Keep only ray-cpp wheel |
| 114 | + rm -f .whl/ray-[0-9]*.whl |
| 115 | +fi |
| 116 | + |
| 117 | +# Sanity check: ensure wheels exist |
| 118 | +shopt -s nullglob |
| 119 | +wheels=(.whl/*.whl) |
| 120 | +if (( ${#wheels[@]} == 0 )); then |
| 121 | + echo "ERROR: No wheels produced in .whl/" |
| 122 | + ls -la .whl || true |
| 123 | + exit 1 |
| 124 | +fi |
| 125 | + |
| 126 | +EOF |
| 127 | + |
| 128 | +FROM scratch |
| 129 | +COPY --from=builder /home/forge/ray/.whl/*.whl / |
0 commit comments