From 3c30d8cb1ec24e0b8a88a5cedcf6b9bece0117d7 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Tue, 20 May 2025 13:31:31 +0000 Subject: [PATCH 1/4] compiler-builtins: Eliminate symlinks compiler-builtins has a symlink to the `libm` source directory so the two crates can share files but still act as two separate crates. This causes problems with some sysroot-related tooling, however, since directory symlinks seem to not be supported. The reason this was a symlink in the first place is that there isn't an easy for Cargo to publish two crates that share source (building works fine but publishing rejects `include`d files from parent directories, as well as nested package roots). However, after the switch to a subtree, we no longer need to publish compiler-builtins; this means that we can eliminate the link and just use `#[path]`. Similarly, the LICENSE file was symlinked so it could live in the repository root but be included in the package. This is also removed as it caused problems with the dist job (error from bootstrap's `tarball.rs`, "generated a symlink in a tarball"). If we need to publish compiler-builtins again for any reason, it would be easy to revert these changes in a preprocess step. --- compiler-builtins/LICENSE.txt | 1 - compiler-builtins/src/math/libm_math | 1 - compiler-builtins/src/math/mod.rs | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 compiler-builtins/LICENSE.txt delete mode 120000 compiler-builtins/src/math/libm_math diff --git a/compiler-builtins/LICENSE.txt b/compiler-builtins/LICENSE.txt deleted file mode 120000 index 4ab43736a..000000000 --- a/compiler-builtins/LICENSE.txt +++ /dev/null @@ -1 +0,0 @@ -../LICENSE.txt \ No newline at end of file diff --git a/compiler-builtins/src/math/libm_math b/compiler-builtins/src/math/libm_math deleted file mode 120000 index 4d65313c2..000000000 --- a/compiler-builtins/src/math/libm_math +++ /dev/null @@ -1 +0,0 @@ -../../../libm/src/math \ No newline at end of file diff --git a/compiler-builtins/src/math/mod.rs b/compiler-builtins/src/math/mod.rs index 078feb9ff..62d729674 100644 --- a/compiler-builtins/src/math/mod.rs +++ b/compiler-builtins/src/math/mod.rs @@ -2,6 +2,7 @@ #[allow(dead_code)] #[allow(unused_imports)] #[allow(clippy::all)] +#[path = "../../../libm/src/math/mod.rs"] pub(crate) mod libm_math; macro_rules! libm_intrinsics { From 9e0cc1dbe45b552322f5512e484e2f0670c901c2 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 28 May 2025 14:45:14 +0000 Subject: [PATCH 2/4] Add an empty rust-version file This will be used by `josh` tooling. --- rust-version | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 rust-version diff --git a/rust-version b/rust-version new file mode 100644 index 000000000..e69de29bb From ded114bca9a34d3ad4cd47f6a7287c7937c0ca38 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sun, 18 May 2025 17:30:58 +0000 Subject: [PATCH 3/4] Add tooling for `josh` syncs Create a crate that handles pulling from and pushing to rust-lang/rust. This can be invoked with the following: $ cargo run -p josh-sync -- rustc-pull $ RUSTC_GIT=/path/to/rust/checkout cargo run -p josh-sync -- rustc-push --- Cargo.toml | 1 + crates/josh-sync/Cargo.toml | 7 + crates/josh-sync/src/main.rs | 45 +++++ crates/josh-sync/src/sync.rs | 371 +++++++++++++++++++++++++++++++++++ 4 files changed, 424 insertions(+) create mode 100644 crates/josh-sync/Cargo.toml create mode 100644 crates/josh-sync/src/main.rs create mode 100644 crates/josh-sync/src/sync.rs diff --git a/Cargo.toml b/Cargo.toml index bc6b4bd29..fb638f2fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "builtins-test", "compiler-builtins", + "crates/josh-sync", "crates/libm-macros", "crates/musl-math-sys", "crates/panic-handler", diff --git a/crates/josh-sync/Cargo.toml b/crates/josh-sync/Cargo.toml new file mode 100644 index 000000000..1f3bb376d --- /dev/null +++ b/crates/josh-sync/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "josh-sync" +edition = "2024" +publish = false + +[dependencies] +directories = "6.0.0" diff --git a/crates/josh-sync/src/main.rs b/crates/josh-sync/src/main.rs new file mode 100644 index 000000000..7f0b11900 --- /dev/null +++ b/crates/josh-sync/src/main.rs @@ -0,0 +1,45 @@ +use std::io::{Read, Write}; +use std::process::exit; +use std::{env, io}; + +use crate::sync::{GitSync, Josh}; + +mod sync; + +const USAGE: &str = r#"Utility for synchroniing compiler-builtins with rust-lang/rust + +Usage: + + josh-sync rustc-pull + + Pull from rust-lang/rust to compiler-builtins. Creates a commit + updating the version file, followed by a merge commit. + + josh-sync rustc-push GITHUB_USERNAME [BRANCH] + + Create a branch off of rust-lang/rust updating compiler-builtins. +"#; + +fn main() { + let sync = GitSync::from_current_dir(); + + // Collect args, then recollect as str refs so we can match on them + let args: Vec<_> = env::args().collect(); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + + match args.as_slice()[1..] { + ["rustc-pull"] => sync.rustc_pull(None), + ["rustc-push", github_user, branch] => sync.rustc_push(github_user, Some(branch)), + ["rustc-push", github_user] => sync.rustc_push(github_user, None), + ["start-josh"] => { + let _josh = Josh::start(); + println!("press enter to stop"); + io::stdout().flush().unwrap(); + let _ = io::stdin().read(&mut [0u8]).unwrap(); + } + _ => { + println!("{USAGE}"); + exit(1); + } + } +} diff --git a/crates/josh-sync/src/sync.rs b/crates/josh-sync/src/sync.rs new file mode 100644 index 000000000..003cf187d --- /dev/null +++ b/crates/josh-sync/src/sync.rs @@ -0,0 +1,371 @@ +use std::net::{SocketAddr, TcpStream}; +use std::process::{Command, Stdio, exit}; +use std::time::Duration; +use std::{env, fs, process, thread}; + +const JOSH_PORT: u16 = 42042; +const DEFAULT_PR_BRANCH: &str = "update-builtins"; + +pub struct GitSync { + upstream_repo: String, + upstream_ref: String, + upstream_url: String, + josh_filter: String, + josh_url_base: String, +} + +/// This code was adapted from the miri repository, via the rustc-dev-guide +/// () +impl GitSync { + pub fn from_current_dir() -> Self { + let upstream_repo = + env::var("UPSTREAM_ORG").unwrap_or_else(|_| "rust-lang".to_owned()) + "/rust"; + + Self { + upstream_url: format!("https://github.com/{upstream_repo}"), + upstream_repo, + upstream_ref: env::var("UPSTREAM_REF").unwrap_or_else(|_| "HEAD".to_owned()), + josh_filter: ":/library/compiler-builtins".to_owned(), + josh_url_base: format!("http://localhost:{JOSH_PORT}"), + } + } + + /// Pull from rust-lang/rust to compiler-builtins. + pub fn rustc_pull(&self, commit: Option) { + let Self { + upstream_ref, + upstream_url, + upstream_repo, + .. + } = self; + + let new_upstream_base = commit.unwrap_or_else(|| { + let out = check_output(["git", "ls-remote", upstream_url, upstream_ref]); + out.split_whitespace() + .next() + .unwrap_or_else(|| panic!("could not split output: '{out}'")) + .to_owned() + }); + + ensure_clean(); + + // Make sure josh is running. + let _josh = Josh::start(); + let josh_url_filtered = self.josh_url( + &self.upstream_repo, + Some(&new_upstream_base), + Some(&self.josh_filter), + ); + + let previous_upstream_base = fs::read_to_string("rust-version") + .expect("failed to read `rust-version`") + .trim() + .to_string(); + assert_ne!(previous_upstream_base, new_upstream_base, "nothing to pull"); + + let orig_head = check_output(["git", "rev-parse", "HEAD"]); + println!("original upstream base: {previous_upstream_base}"); + println!("new upstream base: {new_upstream_base}"); + println!("original HEAD: {orig_head}"); + + // Fetch the latest upstream HEAD so we can get a summary. Use the Josh URL for caching. + run([ + "git", + "fetch", + &self.josh_url(&self.upstream_repo, Some(&new_upstream_base), Some(":/")), + &new_upstream_base, + "--depth=1", + ]); + let new_summary = check_output(["git", "log", "-1", "--format=%h %s", &new_upstream_base]); + + // Update rust-version file. As a separate commit, since making it part of + // the merge has confused the heck out of josh in the past. + // We pass `--no-verify` to avoid running git hooks. + // We do this before the merge so that if there are merge conflicts, we have + // the right rust-version file while resolving them. + fs::write("rust-version", format!("{new_upstream_base}\n")) + .expect("failed to write rust-version"); + + let prep_message = format!( + "Update the upstream Rust version\n\n\ + To prepare for merging from {upstream_repo}, set the version file to:\n\n \ + {new_summary}\n\ + ", + ); + run([ + "git", + "commit", + "rust-version", + "--no-verify", + "-m", + &prep_message, + ]); + + // Fetch given rustc commit. + run(["git", "fetch", &josh_url_filtered]); + let incoming_ref = check_output(["git", "rev-parse", "FETCH_HEAD"]); + println!("incoming ref: {incoming_ref}"); + + let merge_message = format!( + "Merge ref '{upstream_head_short}{filter}' from {upstream_url}\n\n\ + Pull recent changes from {upstream_repo} via Josh.\n\n\ + Upstream ref: {new_upstream_base}\n\ + Filtered ref: {incoming_ref}\n\ + ", + upstream_head_short = &new_upstream_base[..12], + filter = self.josh_filter + ); + + // This should not add any new root commits. So count those before and after merging. + let num_roots = || -> u32 { + let out = check_output(["git", "rev-list", "HEAD", "--max-parents=0", "--count"]); + out.trim() + .parse::() + .unwrap_or_else(|e| panic!("failed to parse `{out}`: {e}")) + }; + let num_roots_before = num_roots(); + + let pre_merge_sha = check_output(["git", "rev-parse", "HEAD"]); + println!("pre-merge HEAD: {pre_merge_sha}"); + + // Merge the fetched commit. + run([ + "git", + "merge", + "FETCH_HEAD", + "--no-verify", + "--no-ff", + "-m", + &merge_message, + ]); + + let current_sha = check_output(["git", "rev-parse", "HEAD"]); + if current_sha == pre_merge_sha { + run(["git", "reset", "--hard", &orig_head]); + eprintln!( + "No merge was performed, no changes to pull were found. \ + Rolled back the preparation commit." + ); + exit(1); + } + + // Check that the number of roots did not increase. + assert_eq!( + num_roots(), + num_roots_before, + "Josh created a new root commit. This is probably not the history you want." + ); + } + + /// Construct an update to rust-lang/rust from compiler-builtins. + pub fn rustc_push(&self, github_user: &str, branch: Option<&str>) { + let Self { + josh_filter, + upstream_url, + .. + } = self; + + let branch = branch.unwrap_or(DEFAULT_PR_BRANCH); + let josh_url = self.josh_url(&format!("{github_user}/rust"), None, Some(josh_filter)); + let user_upstream_url = format!("git@github.com:{github_user}/rust.git"); + + let Ok(rustc_git) = env::var("RUSTC_GIT") else { + panic!("the RUSTC_GIT environment variable must be set to a rust-lang/rust checkout") + }; + + ensure_clean(); + let base = fs::read_to_string("rust-version") + .expect("failed to read `rust-version`") + .trim() + .to_string(); + + // Make sure josh is running. + let _josh = Josh::start(); + + // Prepare the branch. Pushing works much better if we use as base exactly + // the commit that we pulled from last time, so we use the `rust-version` + // file to find out which commit that would be. + println!("Preparing {github_user}/rust (base: {base})..."); + + if Command::new("git") + .args(["-C", &rustc_git, "fetch", &user_upstream_url, branch]) + .output() // capture output + .expect("could not run fetch") + .status + .success() + { + panic!( + "The branch '{branch}' seems to already exist in '{user_upstream_url}'. \ + Please delete it and try again." + ); + } + + run(["git", "-C", &rustc_git, "fetch", upstream_url, &base]); + + run_cfg("git", |c| { + c.args([ + "-C", + &rustc_git, + "push", + &user_upstream_url, + &format!("{base}:refs/heads/{branch}"), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) // silence the "create GitHub PR" message + }); + println!("pushed PR branch"); + + // Do the actual push. + println!("Pushing changes..."); + run(["git", "push", &josh_url, &format!("HEAD:{branch}")]); + println!(); + + // Do a round-trip check to make sure the push worked as expected. + run(["git", "fetch", &josh_url, branch]); + + let head = check_output(["git", "rev-parse", "HEAD"]); + let fetch_head = check_output(["git", "rev-parse", "FETCH_HEAD"]); + assert_eq!( + head, fetch_head, + "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ + Expected {head}, got {fetch_head}." + ); + println!( + "Confirmed that the push round-trips back to compiler-builtins properly. Please \ + create a rustc PR:" + ); + // Open PR with `subtree update` title to silence the `no-merges` triagebot check + println!( + " {upstream_url}/compare/{github_user}:{branch}?quick_pull=1\ + &title=Update%20the%20%60compiler-builtins%60%20subtree\ + &body=Update%20the%20Josh%20subtree%20to%20https%3A%2F%2Fgithub.com%2Frust-lang%2F\ + compiler-builtins%2Fcommit%2F{head_short}.%0A%0Ar%3F%20%40ghost", + head_short = &head[..12], + ); + } + + /// Construct a url to the local Josh server with (optionally) + fn josh_url(&self, repo: &str, rev: Option<&str>, filter: Option<&str>) -> String { + format!( + "{base}/{repo}.git{at}{rev}{filter}{filt_git}", + base = self.josh_url_base, + at = if rev.is_some() { "@" } else { "" }, + rev = rev.unwrap_or_default(), + filter = filter.unwrap_or_default(), + filt_git = if filter.is_some() { ".git" } else { "" } + ) + } +} + +/// Fail if there are files that need to be checked in. +fn ensure_clean() { + let read = check_output(["git", "status", "--untracked-files=no", "--porcelain"]); + assert!( + read.is_empty(), + "working directory must be clean before performing rustc pull" + ); +} + +/* Helpers for running commands with logged invocations */ + +/// Run a command from an array, passing its output through. +fn run<'a, Args: AsRef<[&'a str]>>(l: Args) { + let l = l.as_ref(); + run_cfg(l[0], |c| c.args(&l[1..])); +} + +/// Run a command from an array, collecting its output. +fn check_output<'a, Args: AsRef<[&'a str]>>(l: Args) -> String { + let l = l.as_ref(); + check_output_cfg(l[0], |c| c.args(&l[1..])) +} + +/// [`run`] with configuration. +fn run_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) { + // self.read(l.as_ref()); + check_output_cfg(prog, |c| f(c.stdout(Stdio::inherit()))); +} + +/// [`read`] with configuration. All shell helpers print the command and pass stderr. +fn check_output_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) -> String { + let mut cmd = Command::new(prog); + cmd.stderr(Stdio::inherit()); + f(&mut cmd); + eprintln!("+ {cmd:?}"); + let out = cmd.output().expect("command failed"); + assert!(out.status.success()); + String::from_utf8(out.stdout.trim_ascii().to_vec()).expect("non-UTF8 output") +} + +/// Create a wrapper that stops Josh on drop. +pub struct Josh(process::Child); + +impl Josh { + pub fn start() -> Self { + // Determine cache directory. + let user_dirs = + directories::ProjectDirs::from("org", "rust-lang", "rustc-compiler-builtins-josh") + .unwrap(); + let local_dir = user_dirs.cache_dir().to_owned(); + + // Start josh, silencing its output. + #[expect(clippy::zombie_processes, reason = "clippy can't handle the loop")] + let josh = process::Command::new("josh-proxy") + .arg("--local") + .arg(local_dir) + .args([ + "--remote=https://github.com", + &format!("--port={JOSH_PORT}"), + "--no-background", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start josh-proxy, make sure it is installed"); + + // Wait until the port is open. We try every 10ms until 1s passed. + for _ in 0..100 { + // This will generally fail immediately when the port is still closed. + let addr = SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)); + let josh_ready = TcpStream::connect_timeout(&addr, Duration::from_millis(1)); + + if josh_ready.is_ok() { + println!("josh up and running"); + return Josh(josh); + } + + // Not ready yet. + thread::sleep(Duration::from_millis(10)); + } + panic!("Even after waiting for 1s, josh-proxy is still not available.") + } +} + +impl Drop for Josh { + fn drop(&mut self) { + if cfg!(unix) { + // Try to gracefully shut it down. + Command::new("kill") + .args(["-s", "INT", &self.0.id().to_string()]) + .output() + .expect("failed to SIGINT josh-proxy"); + // Sadly there is no "wait with timeout"... so we just give it some time to finish. + thread::sleep(Duration::from_millis(100)); + // Now hopefully it is gone. + if self + .0 + .try_wait() + .expect("failed to wait for josh-proxy") + .is_some() + { + return; + } + } + // If that didn't work (or we're not on Unix), kill it hard. + eprintln!( + "I have to kill josh-proxy the hard way, let's hope this does not \ + break anything." + ); + self.0.kill().expect("failed to SIGKILL josh-proxy"); + } +} From 162576fa9844ec5111191e32a3384a26f8c825fb Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 4 Jun 2025 18:10:07 +0000 Subject: [PATCH 4/4] Update the upstream Rust version To prepare for merging from rust-lang/rust, set the version file to: df8102fe5f Auto merge of #142002 - onur-ozkan:follow-ups2, r=jieyouxu --- rust-version | 1 + 1 file changed, 1 insertion(+) diff --git a/rust-version b/rust-version index e69de29bb..e05aaa057 100644 --- a/rust-version +++ b/rust-version @@ -0,0 +1 @@ +df8102fe5f24f28a918660b0cd918d7331c3896e