Skip to content

Commit 2ac0f3f

Browse files
aslonnieclaude
andauthored
[release test] add BuildContext for BYOD image builds (#59863)
Add BuildContext TypedDict to capture post_build_script, python_depset, their SHA256 digests, and environment variables for custom BYOD image builds. Changes: - Add build_context.py with BuildContext TypedDict and helper functions: - make_build_context: constructs BuildContext with computed file digests - encode_build_context: deterministic minified JSON serialization - decode_build_context: JSON deserialization - build_context_digest: SHA256 digest of encoded context - Refactor build_anyscale_custom_byod_image to accept BuildContext instead of individual post_build_script and python_depset arguments - Update callers: custom_byod_build.py, ray_bisect.py - Add comprehensive unit tests 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
1 parent bbc29e6 commit 2ac0f3f

File tree

7 files changed

+412
-36
lines changed

7 files changed

+412
-36
lines changed

release/BUILD.bazel

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,21 @@ py_test(
590590
],
591591
)
592592

593+
py_test(
594+
name = "test_byod_build_context",
595+
size = "small",
596+
srcs = ["ray_release/tests/test_byod_build_context.py"],
597+
exec_compatible_with = ["//bazel:py3"],
598+
tags = [
599+
"release_unit",
600+
"team:ci",
601+
],
602+
deps = [
603+
":ray_release",
604+
bk_require("pytest"),
605+
],
606+
)
607+
593608
py_test(
594609
name = "test_custom_byod_build",
595610
size = "small",

release/ray_release/byod/build.py

Lines changed: 3 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import os
2-
import shutil
32
import subprocess
43
import sys
54
import tempfile
65
from typing import Dict, List, Optional
76

7+
from ray_release.byod.build_context import BuildContext, fill_build_context_dir
88
from ray_release.config import RELEASE_PACKAGE_DIR
99
from ray_release.logger import logger
1010
from ray_release.test import (
@@ -18,8 +18,7 @@
1818
def build_anyscale_custom_byod_image(
1919
image: str,
2020
base_image: str,
21-
post_build_script: str,
22-
python_depset: Optional[str] = None,
21+
build_context: BuildContext,
2322
release_byod_dir: Optional[str] = None,
2423
) -> None:
2524
if _image_exist(image):
@@ -35,32 +34,7 @@ def build_anyscale_custom_byod_image(
3534
release_byod_dir = os.path.join(RELEASE_PACKAGE_DIR, "ray_release/byod")
3635

3736
with tempfile.TemporaryDirectory() as build_dir:
38-
dockerfile = ["# syntax=docker/dockerfile:1.3-labs"]
39-
dockerfile.append("ARG BASE_IMAGE")
40-
dockerfile.append("FROM ${BASE_IMAGE}")
41-
if python_depset:
42-
shutil.copy(
43-
os.path.join(release_byod_dir, python_depset),
44-
os.path.join(build_dir, "python_depset.lock"),
45-
)
46-
shutil.copy(
47-
os.path.join(release_byod_dir, "install_python_deps.sh"),
48-
os.path.join(build_dir, "install_python_deps.sh"),
49-
)
50-
dockerfile.append("COPY install_python_deps.sh /tmp/install_python_deps.sh")
51-
dockerfile.append("COPY python_depset.lock python_depset.lock")
52-
dockerfile.append("RUN bash /tmp/install_python_deps.sh python_depset.lock")
53-
if post_build_script:
54-
shutil.copy(
55-
os.path.join(release_byod_dir, post_build_script),
56-
os.path.join(build_dir, "post_build_script.sh"),
57-
)
58-
dockerfile.append("COPY post_build_script.sh /tmp/post_build_script.sh")
59-
dockerfile.append("RUN bash /tmp/post_build_script.sh")
60-
61-
dockerfile_path = os.path.join(build_dir, "Dockerfile")
62-
with open(dockerfile_path, "wt") as f:
63-
f.write("\n".join(dockerfile) + "\n")
37+
fill_build_context_dir(build_context, build_dir, release_byod_dir)
6438

6539
docker_build_cmd = "docker build --progress=plain .".split()
6640
docker_build_cmd += ["--build-arg", f"BASE_IMAGE={base_image}"]
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import hashlib
2+
import json
3+
import os
4+
import shutil
5+
from typing import Dict, List, Optional
6+
7+
from typing_extensions import TypedDict
8+
9+
10+
class BuildContext(TypedDict, total=False):
11+
"""
12+
Build context for custom BYOD image builds.
13+
14+
Attributes:
15+
envs: Environment variables to set in the image.
16+
post_build_script: Filename of the post-build script.
17+
post_build_script_digest: SHA256 digest of the post-build script.
18+
python_depset: Filename of the Python dependencies lock file.
19+
python_depset_digest: SHA256 digest of the Python dependencies lock file.
20+
"""
21+
22+
envs: Dict[str, str]
23+
24+
post_build_script: str
25+
post_build_script_digest: str
26+
27+
python_depset: str
28+
python_depset_digest: str
29+
30+
31+
def make_build_context(
32+
base_dir: str,
33+
envs: Optional[Dict[str, str]] = None,
34+
post_build_script: Optional[str] = None,
35+
python_depset: Optional[str] = None,
36+
) -> BuildContext:
37+
"""
38+
Create a BuildContext with computed file digests.
39+
40+
Args:
41+
base_dir: Directory containing the source files.
42+
envs: Environment variables to set in the image.
43+
post_build_script: Filename of the post-build script.
44+
python_depset: Filename of the Python dependencies lock file.
45+
46+
Returns:
47+
A BuildContext with filenames and their SHA256 digests.
48+
"""
49+
ctx: BuildContext = {}
50+
51+
if envs:
52+
ctx["envs"] = envs
53+
54+
if post_build_script:
55+
ctx["post_build_script"] = post_build_script
56+
path = os.path.join(base_dir, post_build_script)
57+
ctx["post_build_script_digest"] = _sha256_file(path)
58+
59+
if python_depset:
60+
ctx["python_depset"] = python_depset
61+
path = os.path.join(base_dir, python_depset)
62+
ctx["python_depset_digest"] = _sha256_file(path)
63+
64+
return ctx
65+
66+
67+
def encode_build_context(ctx: BuildContext) -> str:
68+
"""Encode a BuildContext to deterministic minified JSON."""
69+
return json.dumps(ctx, sort_keys=True, separators=(",", ":"))
70+
71+
72+
def decode_build_context(data: str) -> BuildContext:
73+
"""Decode a JSON string to a BuildContext."""
74+
return json.loads(data)
75+
76+
77+
def build_context_digest(ctx: BuildContext) -> str:
78+
"""Compute SHA256 digest of the encoded BuildContext."""
79+
encoded = encode_build_context(ctx)
80+
digest = hashlib.sha256(encoded.encode()).hexdigest()
81+
return f"sha256:{digest}"
82+
83+
84+
def fill_build_context_dir(
85+
ctx: BuildContext,
86+
build_dir: str,
87+
source_dir: str,
88+
) -> None:
89+
"""
90+
Generate Dockerfile and copy source files to the build directory.
91+
92+
Args:
93+
ctx: The BuildContext specifying what to include.
94+
build_dir: Target directory for the generated Dockerfile and copied files.
95+
source_dir: Source directory containing the original files.
96+
"""
97+
dockerfile: List[str] = ["# syntax=docker/dockerfile:1.3-labs"]
98+
dockerfile.append("ARG BASE_IMAGE")
99+
dockerfile.append("FROM ${BASE_IMAGE}")
100+
101+
if "envs" in ctx and ctx["envs"]:
102+
dockerfile.append("ENV \\")
103+
env_lines = [f" {k}={v}" for k, v in sorted(ctx["envs"].items())]
104+
dockerfile.append(" \\\n".join(env_lines))
105+
106+
if "python_depset" in ctx:
107+
shutil.copy(
108+
os.path.join(source_dir, ctx["python_depset"]),
109+
os.path.join(build_dir, "python_depset.lock"),
110+
)
111+
shutil.copy(
112+
os.path.join(source_dir, "install_python_deps.sh"),
113+
os.path.join(build_dir, "install_python_deps.sh"),
114+
)
115+
dockerfile.append("COPY install_python_deps.sh /tmp/install_python_deps.sh")
116+
dockerfile.append("COPY python_depset.lock python_depset.lock")
117+
dockerfile.append("RUN bash /tmp/install_python_deps.sh python_depset.lock")
118+
119+
if "post_build_script" in ctx:
120+
shutil.copy(
121+
os.path.join(source_dir, ctx["post_build_script"]),
122+
os.path.join(build_dir, "post_build_script.sh"),
123+
)
124+
dockerfile.append("COPY post_build_script.sh /tmp/post_build_script.sh")
125+
dockerfile.append("RUN bash /tmp/post_build_script.sh")
126+
127+
dockerfile_path = os.path.join(build_dir, "Dockerfile")
128+
with open(dockerfile_path, "w") as f:
129+
f.write("\n".join(dockerfile) + "\n")
130+
131+
132+
def _sha256_file(path: str) -> str:
133+
with open(path, "rb") as f:
134+
digest = hashlib.sha256(f.read()).hexdigest()
135+
return f"sha256:{digest}"

release/ray_release/scripts/custom_byod_build.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import click
44

55
from ray_release.byod.build import build_anyscale_custom_byod_image
6+
from ray_release.byod.build_context import BuildContext
67

78

89
@click.command()
@@ -20,9 +21,12 @@ def main(
2021
raise click.UsageError(
2122
"Either post_build_script or python_depset must be provided"
2223
)
23-
build_anyscale_custom_byod_image(
24-
image_name, base_image, post_build_script, python_depset
25-
)
24+
build_context: BuildContext = {}
25+
if post_build_script:
26+
build_context["post_build_script"] = post_build_script
27+
if python_depset:
28+
build_context["python_depset"] = python_depset
29+
build_anyscale_custom_byod_image(image_name, base_image, build_context)
2630

2731

2832
if __name__ == "__main__":

release/ray_release/scripts/ray_bisect.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
build_anyscale_base_byod_images,
1515
build_anyscale_custom_byod_image,
1616
)
17+
from ray_release.byod.build_context import BuildContext
1718
from ray_release.config import (
1819
RELEASE_TEST_CONFIG_FILES,
1920
read_and_validate_release_test_collection,
@@ -180,11 +181,17 @@ def _trigger_test_run(
180181
os.environ["COMMIT_TO_TEST"] = commit
181182
build_anyscale_base_byod_images([test])
182183
if test.require_custom_byod_image():
184+
build_context: BuildContext = {}
185+
post_build_script = test.get_byod_post_build_script()
186+
python_depset = test.get_byod_python_depset()
187+
if post_build_script:
188+
build_context["post_build_script"] = post_build_script
189+
if python_depset:
190+
build_context["python_depset"] = python_depset
183191
build_anyscale_custom_byod_image(
184192
test.get_anyscale_byod_image(),
185193
test.get_anyscale_base_byod_image(),
186-
test.get_byod_post_build_script(),
187-
test.get_byod_python_depset(),
194+
build_context,
188195
)
189196
for run in range(run_per_commit):
190197
step = get_step(

release/ray_release/tests/test_byod_build.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
build_anyscale_base_byod_images,
1313
build_anyscale_custom_byod_image,
1414
)
15+
from ray_release.byod.build_context import BuildContext
1516
from ray_release.configs.global_config import get_global_config, init_global_config
1617
from ray_release.test import Test
1718

@@ -72,11 +73,13 @@ def _mock_check_call(
7273
with open(os.path.join(byod_dir, "foo.sh"), "wt") as f:
7374
f.write("echo foo")
7475

76+
build_context: BuildContext = {
77+
"post_build_script": test.get_byod_post_build_script(),
78+
}
7579
build_anyscale_custom_byod_image(
7680
test.get_anyscale_byod_image(),
7781
test.get_anyscale_base_byod_image(),
78-
test.get_byod_post_build_script(),
79-
test.get_byod_python_depset(),
82+
build_context,
8083
release_byod_dir=byod_dir,
8184
)
8285
assert (" ".join(cmds[0])).startswith(

0 commit comments

Comments
 (0)