Skip to content

Commit b66eb68

Browse files
bolinfestbolinfest
committed
feat: support local development using Buck2
1 parent 38de0a1 commit b66eb68

File tree

16 files changed

+2352
-1
lines changed

16 files changed

+2352
-1
lines changed

‎.buckconfig‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[cells]
2+
root = .
3+
toolchains = codex-rs/toolchains
4+
prelude = prelude
5+
none = none
6+
7+
[cell_aliases]
8+
# Buck2 prelude expects some common aliases to exist in some environments.
9+
config = prelude
10+
ovr_config = prelude
11+
fbcode = none
12+
fbsource = none
13+
fbcode_macros = none
14+
buck = none
15+
16+
# Use the Buck2 prelude bundled with the buck2 binary.
17+
[external_cells]
18+
prelude = bundled
19+
20+
[parser]
21+
target_platform_detector_spec = target://...->prelude//platforms:default
22+
23+
[codex]
24+
# Local-only knob used by codex-rs/buck2 to approximate Cargo profiles.
25+
# Override on the command line with:
26+
# ./scripts/buck2 build -c codex.rust_profile=release //codex-rs/cli:codex
27+
rust_profile = dev
28+
29+
[build]
30+
execution_platforms = prelude//platforms:default

‎.buckroot‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.github/workflows/buck2.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Buck2 (Experimental, non-blocking for PRs)
2+
3+
on:
4+
pull_request: {}
5+
push:
6+
branches:
7+
- main
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: buck2-${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
buck2-test:
19+
name: buck2 test //codex-rs/...
20+
runs-on: ubuntu-24.04
21+
# Non-blocking while Buck2 support is still experimental.
22+
continue-on-error: true
23+
timeout-minutes: 20
24+
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v4
28+
29+
# scripts/buck2, scripts/reindeer, etc. are DotSlash wrappers.
30+
- name: Install DotSlash
31+
uses: facebook/install-dotslash@v2
32+
33+
- name: Setup Rust toolchain
34+
uses: dtolnay/[email protected]
35+
with:
36+
# Match codex-rs/rust-toolchain.toml (and include rust-src for Buck2 toolchains).
37+
components: rustfmt, clippy, rust-src
38+
39+
- name: Install system deps (Linux)
40+
shell: bash
41+
run: |
42+
set -euxo pipefail
43+
sudo apt-get update
44+
sudo apt-get install -y pkg-config libssl-dev
45+
46+
- name: Setup Buck2 (local)
47+
shell: bash
48+
run: |
49+
set -euxo pipefail
50+
./scripts/setup_buck2_local.sh
51+
52+
- name: Run Buck2 tests
53+
shell: bash
54+
run: |
55+
set -euxo pipefail
56+
./scripts/buck2 test //codex-rs/...

‎.gitignore‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,12 @@ build/
1313
out/
1414
storybook-static/
1515

16+
# buck2
17+
buck-out/
18+
**/BUCK
19+
codex-rs/third-party/
20+
codex-rs/cli/Cargo.lock
21+
1622
# ignore README for publishing
1723
codex-cli/README.md
1824

@@ -89,4 +95,3 @@ CHANGELOG.ignore.md
8995
# Python bytecode files
9096
__pycache__/
9197
*.pyc
92-

‎codex-rs/buck2/codex_rust_toolchain.bzl‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
load("@prelude//rust:rust_toolchain.bzl", "PanicRuntime", "RustToolchainInfo")
2+
3+
_DEFAULT_TRIPLE = select({
4+
"prelude//os:linux": select({
5+
"prelude//cpu:arm64": "aarch64-unknown-linux-gnu",
6+
"prelude//cpu:riscv64": "riscv64gc-unknown-linux-gnu",
7+
"prelude//cpu:x86_64": "x86_64-unknown-linux-gnu",
8+
}),
9+
"prelude//os:macos": select({
10+
"prelude//cpu:arm64": "aarch64-apple-darwin",
11+
"prelude//cpu:x86_64": "x86_64-apple-darwin",
12+
}),
13+
"prelude//os:windows": select({
14+
"prelude//cpu:arm64": select({
15+
# Rustup's default ABI for the host on Windows is MSVC, not GNU.
16+
"DEFAULT": "aarch64-pc-windows-msvc",
17+
"prelude//abi:gnu": "aarch64-pc-windows-gnu",
18+
"prelude//abi:msvc": "aarch64-pc-windows-msvc",
19+
}),
20+
"prelude//cpu:x86_64": select({
21+
"DEFAULT": "x86_64-pc-windows-msvc",
22+
"prelude//abi:gnu": "x86_64-pc-windows-gnu",
23+
"prelude//abi:msvc": "x86_64-pc-windows-msvc",
24+
}),
25+
}),
26+
})
27+
28+
29+
def _codex_rust_toolchain_impl(ctx):
30+
# Buck doesn't have a built-in notion of "Cargo profiles", but it's useful
31+
# to provide a simple local knob that roughly matches `cargo build` vs
32+
# `cargo build --release`.
33+
#
34+
# Default is "dev" to match local development expectations.
35+
rust_profile = read_config("codex", "rust_profile", "dev")
36+
extra_rustc_flags = []
37+
if rust_profile == "release":
38+
# Roughly mirrors Cargo's release defaults (not a perfect match).
39+
extra_rustc_flags = [
40+
"-C",
41+
"opt-level=3",
42+
"-C",
43+
"debuginfo=0",
44+
]
45+
46+
return [
47+
DefaultInfo(),
48+
RustToolchainInfo(
49+
allow_lints = ctx.attrs.allow_lints,
50+
clippy_driver = RunInfo(args = [ctx.attrs.clippy_driver]),
51+
clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None,
52+
compiler = RunInfo(args = [ctx.attrs.rustc]),
53+
default_edition = ctx.attrs.default_edition,
54+
deny_lints = ctx.attrs.deny_lints,
55+
doctests = ctx.attrs.doctests,
56+
nightly_features = ctx.attrs.nightly_features,
57+
panic_runtime = PanicRuntime("unwind"),
58+
report_unused_deps = ctx.attrs.report_unused_deps,
59+
rustc_binary_flags = ctx.attrs.rustc_binary_flags,
60+
rustc_flags = ctx.attrs.rustc_flags + extra_rustc_flags,
61+
rustc_target_triple = ctx.attrs.rustc_target_triple,
62+
rustc_test_flags = ctx.attrs.rustc_test_flags,
63+
rustdoc = RunInfo(args = [ctx.attrs.rustdoc]),
64+
rustdoc_flags = ctx.attrs.rustdoc_flags,
65+
warn_lints = ctx.attrs.warn_lints,
66+
# Enable the prelude's "metadata-only rlib" behavior consistently
67+
# across the crate graph. This avoids rustc "found possibly newer
68+
# version of crate ..." (E0460) mismatches between binaries and
69+
# libraries in large Rust graphs.
70+
advanced_unstable_linking = ctx.attrs.advanced_unstable_linking,
71+
),
72+
]
73+
74+
75+
codex_rust_toolchain = rule(
76+
impl = _codex_rust_toolchain_impl,
77+
attrs = {
78+
"advanced_unstable_linking": attrs.bool(default = True),
79+
"allow_lints": attrs.list(attrs.string(), default = []),
80+
# Prefer explicit tool paths so the Buck execution directory doesn't
81+
# affect rustup toolchain resolution.
82+
"clippy_driver": attrs.string(default = "clippy-driver"),
83+
"clippy_toml": attrs.option(attrs.dep(providers = [DefaultInfo]), default = None),
84+
"default_edition": attrs.option(attrs.string(), default = None),
85+
"deny_lints": attrs.list(attrs.string(), default = []),
86+
"doctests": attrs.bool(default = False),
87+
"nightly_features": attrs.bool(default = False),
88+
"report_unused_deps": attrs.bool(default = False),
89+
"rustc": attrs.string(default = "rustc"),
90+
"rustc_binary_flags": attrs.list(attrs.arg(), default = []),
91+
"rustc_flags": attrs.list(attrs.arg(), default = []),
92+
"rustc_target_triple": attrs.string(default = _DEFAULT_TRIPLE),
93+
"rustc_test_flags": attrs.list(attrs.arg(), default = []),
94+
"rustdoc": attrs.string(default = "rustdoc"),
95+
"rustdoc_flags": attrs.list(attrs.arg(), default = []),
96+
"warn_lints": attrs.list(attrs.string(), default = []),
97+
},
98+
is_toolchain_rule = True,
99+
)

‎codex-rs/buck2/reindeer_macros.bzl‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
load("@prelude//rust:cargo_buildscript.bzl", _prelude_buildscript_run = "buildscript_run")
2+
load("@prelude//rust:cargo_package.bzl", "cargo")
3+
4+
5+
def codex_noop_alias(**_kwargs):
6+
# Reindeer normally emits aliases like `alias(name = "rand", actual = ":rand-0.8.5")`
7+
# to provide stable unversioned target names. In a non-trivial workspace it's
8+
# common to have multiple versions of the same crate in one graph, which
9+
# leads to duplicate alias target names and buckification failures.
10+
#
11+
# For local Buck experiments (where we don't check in generated third-party
12+
# BUCK files), it's simplest to disable these aliases and depend on
13+
# versioned targets directly.
14+
pass
15+
16+
17+
def _codex_extra_srcs_for_manifest_dir(manifest_dir):
18+
if not manifest_dir:
19+
return []
20+
21+
# Use per-crate globs rooted at the crate's vendored manifest dir, which is
22+
# passed through by Reindeer as CARGO_MANIFEST_DIR (e.g. vendor/foo-1.2.3).
23+
#
24+
# We include *all* files under the crate root so `include_str!` and
25+
# `include_bytes!` work without per-crate Reindeer fixups. This is local-only
26+
# buckification, so we prefer robustness over a minimal srcs list.
27+
return glob(
28+
["{}/**".format(manifest_dir)],
29+
exclude = [
30+
"{}/target/**".format(manifest_dir),
31+
"{}/.git/**".format(manifest_dir),
32+
],
33+
)
34+
35+
36+
def codex_rust_library(**kwargs):
37+
# Make generated third-party targets consumable from anywhere in the repo.
38+
kwargs["visibility"] = ["PUBLIC"]
39+
env = kwargs.get("env", {})
40+
manifest_dir = env.get("CARGO_MANIFEST_DIR")
41+
srcs = list(kwargs.get("srcs", []))
42+
srcs.extend(_codex_extra_srcs_for_manifest_dir(manifest_dir))
43+
kwargs["srcs"] = srcs
44+
cargo.rust_library(**kwargs)
45+
46+
47+
def codex_rust_binary(**kwargs):
48+
kwargs["visibility"] = ["PUBLIC"]
49+
env = kwargs.get("env", {})
50+
manifest_dir = env.get("CARGO_MANIFEST_DIR")
51+
srcs = list(kwargs.get("srcs", []))
52+
srcs.extend(_codex_extra_srcs_for_manifest_dir(manifest_dir))
53+
kwargs["srcs"] = srcs
54+
cargo.rust_binary(**kwargs)
55+
56+
57+
def codex_buildscript_run(**kwargs):
58+
# Many build scripts (especially those using `cc`/`cc-rs`) expect Cargo to
59+
# provide a handful of profile env vars. Buck does not set these by default.
60+
env = dict(kwargs.get("env", {}))
61+
rust_profile = read_config("codex", "rust_profile", "dev")
62+
if rust_profile == "release":
63+
env.setdefault("OPT_LEVEL", "3")
64+
env.setdefault("PROFILE", "release")
65+
env.setdefault("DEBUG", "false")
66+
else:
67+
env.setdefault("OPT_LEVEL", "0")
68+
env.setdefault("PROFILE", "debug")
69+
env.setdefault("DEBUG", "true")
70+
71+
# Provide common Cargo cfg env vars that some build scripts expect.
72+
env.setdefault(
73+
"CARGO_CFG_TARGET_OS",
74+
select({
75+
"prelude//os:linux": "linux",
76+
"prelude//os:macos": "macos",
77+
"prelude//os:windows": "windows",
78+
"DEFAULT": "",
79+
}),
80+
)
81+
env.setdefault(
82+
"CARGO_CFG_TARGET_ARCH",
83+
select({
84+
"prelude//cpu:arm64": "aarch64",
85+
"prelude//cpu:x86_64": "x86_64",
86+
"DEFAULT": "",
87+
}),
88+
)
89+
env.setdefault("CARGO_CFG_TARGET_ENDIAN", "little")
90+
env.setdefault(
91+
"CARGO_CFG_TARGET_ENV",
92+
select({
93+
"prelude//os:linux": "gnu",
94+
"DEFAULT": "",
95+
}),
96+
)
97+
98+
# Forward native link directives emitted by build scripts into rustc flags.
99+
# Without this, crates like `ring` and `tree-sitter-*` will compile but fail
100+
# to link due to missing native symbols.
101+
kwargs.setdefault("rustc_link_lib", True)
102+
kwargs.setdefault("rustc_link_search", True)
103+
104+
# `CARGO_MANIFEST_DIR` is expected by many build scripts. We can usually
105+
# derive it from the `manifest_dir` parameter that buildscript_run uses.
106+
manifest_dir = kwargs.get("manifest_dir")
107+
if type(manifest_dir) == type(""):
108+
env.setdefault("CARGO_MANIFEST_DIR", "$(location {})".format(manifest_dir))
109+
110+
kwargs["env"] = env
111+
_prelude_buildscript_run(**kwargs)

‎codex-rs/reindeer.toml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
##
2+
## Reindeer config for codex-rs (Cargo workspace) + Buck2.
3+
##
4+
## This file is checked in, but generated Buck artifacts (BUCK files and
5+
## vendored crates) are intentionally gitignored for now.
6+
##
7+
8+
# Focus buckification on the main CLI to avoid pulling in platform-specific
9+
# workspace members that can create duplicate "public alias" names.
10+
manifest_path = "cli/Cargo.toml"
11+
12+
# Place generated output under codex-rs/third-party/.
13+
third_party_dir = "third-party"
14+
15+
# Use vendored crates (no crates.io downloads during `buck2 build`).
16+
vendor = true
17+
18+
# Provide common Cargo env vars for crates/build scripts that use `env!()`.
19+
cargo_env = [
20+
"CARGO_CRATE_NAME",
21+
"CARGO_MANIFEST_DIR",
22+
"CARGO_PKG_AUTHORS",
23+
"CARGO_PKG_DESCRIPTION",
24+
"CARGO_PKG_NAME",
25+
"CARGO_PKG_REPOSITORY",
26+
"CARGO_PKG_VERSION",
27+
"CARGO_PKG_VERSION_MAJOR",
28+
"CARGO_PKG_VERSION_MINOR",
29+
"CARGO_PKG_VERSION_PATCH",
30+
"CARGO_PKG_VERSION_PRE",
31+
]
32+
33+
[buck]
34+
file_name = "BUCK"
35+
36+
# These resolve to macros defined in codex-rs/buck2/reindeer_macros.bzl.
37+
alias = "codex_noop_alias"
38+
alias_with_platforms = "codex_noop_alias"
39+
rust_library = "codex_rust_library"
40+
rust_binary = "codex_rust_binary"
41+
buildscript_genrule = "codex_buildscript_run"
42+
43+
buckfile_imports = """
44+
load("//codex-rs/buck2:reindeer_macros.bzl", "codex_buildscript_run", "codex_noop_alias", "codex_rust_binary", "codex_rust_library")
45+
"""
46+
47+
generated_file_header = """
48+
##
49+
## @generated by reindeer
50+
## Do not edit by hand.
51+
##
52+
## To regenerate:
53+
## (cd codex-rs && reindeer vendor && reindeer buckify)
54+
##
55+
"""
56+

0 commit comments

Comments
 (0)