fix: use git-commit-info for version information

This PR adds support for fetching version information from the
`git-commit-info` file when building the compiler from a source tarball.

Author: dawnofmidnight

config.toml.example

@@ -521,6 +521,12 @@ changelog-seen = 2
 # A descriptive string to be appended to `rustc --version` output, which is
 # also used in places like debuginfo `DW_AT_producer`. This may be useful for
 # supplementary build information, like distro-specific package versions.
+#
+# The Rust compiler will differentiate between versions of itself, including
+# based on this string, which means that if you wish to be compatible with
+# upstream Rust you need to set this to "". However, note that if you are not
+# actually compatible -- for example if you've backported patches that change
+# behavior -- this may lead to miscompilations or other bugs.
 #description = <none> (string)
 
 # The root location of the musl installation directory. The library directory

src/bootstrap/channel.rs

@@ -5,10 +5,12 @@
 //! `package_vers`, and otherwise indicating to the compiler what it should
 //! print out as part of its version information.
 
+use std::fs;
 use std::path::Path;
 use std::process::Command;
 
 use crate::util::output;
+use crate::util::t;
 use crate::Build;
 
 pub enum GitInfo {
@@ -18,19 +20,25 @@ pub enum GitInfo {
     /// If the info should be used (`ignore_git` is false), this will be
     /// `Some`, otherwise it will be `None`.
     Present(Option<Info>),
+    /// This is not a git repostory, but the info can be fetched from the
+    /// `git-commit-info` file.
+    RecordedForTarball(Info),
 }
 
 pub struct Info {
-    commit_date: String,
-    sha: String,
-    short_sha: String,
+    pub commit_date: String,
+    pub sha: String,
+    pub short_sha: String,
 }
 
 impl GitInfo {
     pub fn new(ignore_git: bool, dir: &Path) -> GitInfo {
         // See if this even begins to look like a git dir
         if !dir.join(".git").exists() {
-            return GitInfo::Absent;
+            match read_commit_info_file(dir) {
+                Some(info) => return GitInfo::RecordedForTarball(info),
+                None => return GitInfo::Absent,
+            }
         }
 
         // Make sure git commands work
@@ -65,10 +73,11 @@ impl GitInfo {
         }))
     }
 
-    fn info(&self) -> Option<&Info> {
+    pub fn info(&self) -> Option<&Info> {
         match self {
-            GitInfo::Present(info) => info.as_ref(),
             GitInfo::Absent => None,
+            GitInfo::Present(info) => info.as_ref(),
+            GitInfo::RecordedForTarball(info) => Some(info),
         }
     }
 
@@ -96,10 +105,48 @@ impl GitInfo {
         version
     }
 
-    pub fn is_git(&self) -> bool {
+    /// Returns whether this directory has a `.git` directory which should be managed by bootstrap.
+    pub fn is_managed_git_subrepository(&self) -> bool {
         match self {
-            GitInfo::Absent => false,
+            GitInfo::Absent | GitInfo::RecordedForTarball(_) => false,
             GitInfo::Present(_) => true,
         }
     }
+
+    /// Returns whether this is being built from a tarball.
+    pub fn is_from_tarball(&self) -> bool {
+        match self {
+            GitInfo::Absent | GitInfo::Present(_) => false,
+            GitInfo::RecordedForTarball(_) => true,
+        }
+    }
+}
+
+/// Read the commit information from the `git-commit-info` file given the
+/// project root.
+pub fn read_commit_info_file(root: &Path) -> Option<Info> {
+    if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) {
+        let mut lines = contents.lines();
+        let sha = lines.next();
+        let short_sha = lines.next();
+        let commit_date = lines.next();
+        let info = match (commit_date, sha, short_sha) {
+            (Some(commit_date), Some(sha), Some(short_sha)) => Info {
+                commit_date: commit_date.to_owned(),
+                sha: sha.to_owned(),
+                short_sha: short_sha.to_owned(),
+            },
+            _ => panic!("the `git-comit-info` file is malformed"),
+        };
+        Some(info)
+    } else {
+        None
+    }
+}
+
+/// Write the commit information to the `git-commit-info` file given the project
+/// root.
+pub fn write_commit_info_file(root: &Path, info: &Info) {
+    let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date);
+    t!(fs::write(root.join("git-commit-info"), &commit_info));
 }

src/bootstrap/config.rs

@@ -1280,11 +1280,22 @@ impl Config {
         git
     }
 
-    pub(crate) fn artifact_channel(&self, commit: &str) -> String {
-        let mut channel = self.git();
-        channel.arg("show").arg(format!("{}:src/ci/channel", commit));
-        let channel = output(&mut channel);
-        channel.trim().to_owned()
+    pub(crate) fn artifact_channel(&self, builder: &Builder<'_>, commit: &str) -> String {
+        if builder.rust_info.is_managed_git_subrepository() {
+            let mut channel = self.git();
+            channel.arg("show").arg(format!("{}:src/ci/channel", commit));
+            let channel = output(&mut channel);
+            channel.trim().to_owned()
+        } else if let Ok(channel) = fs::read_to_string(builder.src.join("src/ci/channel")) {
+            channel.trim().to_owned()
+        } else {
+            let src = builder.src.display();
+            eprintln!("error: failed to determine artifact channel");
+            eprintln!(
+                "help: either use git or ensure that {src}/src/ci/channel contains the name of the channel to use"
+            );
+            panic!();
+        }
     }
 
     /// Try to find the relative path of `bindir`, otherwise return it in full.
@@ -1421,7 +1432,7 @@ impl Config {
     }
 
     pub fn submodules(&self, rust_info: &GitInfo) -> bool {
-        self.submodules.unwrap_or(rust_info.is_git())
+        self.submodules.unwrap_or(rust_info.is_managed_git_subrepository())
     }
 }
 
@@ -1526,7 +1537,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
 
 fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
     builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
-    let channel = builder.config.artifact_channel(commit);
+    let channel = builder.config.artifact_channel(builder, commit);
     let host = builder.config.build.triple;
     let bin_root = builder.out.join(host).join("ci-rustc");
     let rustc_stamp = bin_root.join(".rustc-stamp");

src/bootstrap/dist.rs

@@ -16,6 +16,7 @@ use std::process::Command;
 
 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::cache::{Interned, INTERNER};
+use crate::channel;
 use crate::compile;
 use crate::config::TargetSelection;
 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
@@ -897,12 +898,12 @@ impl Step for PlainSourceTarball {
 
         // Create the version file
         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
-        if let Some(sha) = builder.rust_sha() {
-            builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
+        if let Some(info) = builder.rust_info.info() {
+            channel::write_commit_info_file(&plain_dst_src, info);
         }
 
         // If we're building from git sources, we need to vendor a complete distribution.
-        if builder.rust_info.is_git() {
+        if builder.rust_info.is_managed_git_subrepository() {
             // Ensure we have the submodules checked out.
             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 

src/bootstrap/lib.rs

@@ -395,7 +395,7 @@ impl Build {
     /// line and the filesystem `config`.
     ///
     /// By default all build output will be placed in the current directory.
-    pub fn new(config: Config) -> Build {
+    pub fn new(mut config: Config) -> Build {
         let src = config.src.clone();
         let out = config.out.clone();
 
@@ -470,6 +470,10 @@ impl Build {
             bootstrap_out
         };
 
+        if rust_info.is_from_tarball() && config.description.is_none() {
+            config.description = Some("built from a source tarball".to_owned());
+        }
+
         let mut build = Build {
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
@@ -574,7 +578,9 @@ impl Build {
 
         // NOTE: The check for the empty directory is here because when running x.py the first time,
         // the submodule won't be checked out. Check it out now so we can build it.
-        if !channel::GitInfo::new(false, &absolute_path).is_git() && !dir_is_empty(&absolute_path) {
+        if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository()
+            && !dir_is_empty(&absolute_path)
+        {
             return;
         }
 
@@ -645,7 +651,7 @@ impl Build {
             // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer`
             let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap());
             // Don't update the submodule unless it's already been cloned.
-            if channel::GitInfo::new(false, submodule).is_git() {
+            if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() {
                 self.update_submodule(submodule);
             }
         }
@@ -1253,7 +1259,7 @@ impl Build {
         match &self.config.channel[..] {
             "stable" => num.to_string(),
             "beta" => {
-                if self.rust_info.is_git() && !self.config.ignore_git {
+                if self.rust_info.is_managed_git_subrepository() && !self.config.ignore_git {
                     format!("{}-beta.{}", num, self.beta_prerelease_version())
                 } else {
                     format!("{}-beta", num)

src/bootstrap/native.rs

@@ -17,6 +17,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::channel;
 use crate::config::TargetSelection;
 use crate::util::get_clang_cl_resource_dir;
 use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
@@ -115,32 +116,37 @@ pub fn prebuilt_llvm_config(
 }
 
 /// This retrieves the LLVM sha we *want* to use, according to git history.
-pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
-    let mut rev_list = config.git();
-    rev_list.args(&[
-        PathBuf::from("rev-list"),
-        format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
-        "-n1".into(),
-        "--first-parent".into(),
-        "HEAD".into(),
-        "--".into(),
-        config.src.join("src/llvm-project"),
-        config.src.join("src/bootstrap/download-ci-llvm-stamp"),
-        // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-        config.src.join("src/version"),
-    ]);
-    let llvm_sha = output(&mut rev_list);
-    let llvm_sha = llvm_sha.trim();
-
-    if llvm_sha == "" {
+pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> String {
+    let llvm_sha = if is_git {
+        let mut rev_list = config.git();
+        rev_list.args(&[
+            PathBuf::from("rev-list"),
+            format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
+            "-n1".into(),
+            "--first-parent".into(),
+            "HEAD".into(),
+            "--".into(),
+            config.src.join("src/llvm-project"),
+            config.src.join("src/bootstrap/download-ci-llvm-stamp"),
+            // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+            config.src.join("src/version"),
+        ]);
+        output(&mut rev_list).trim().to_owned()
+    } else if let Some(info) = channel::read_commit_info_file(&config.src) {
+        info.sha.trim().to_owned()
+    } else {
+        "".to_owned()
+    };
+
+    if &llvm_sha == "" {
         eprintln!("error: could not find commit hash for downloading LLVM");
         eprintln!("help: maybe your repository history is too shallow?");
         eprintln!("help: consider disabling `download-ci-llvm`");
         eprintln!("help: or fetch enough history to include one upstream commit");
         panic!();
     }
 
-    llvm_sha.to_owned()
+    llvm_sha
 }
 
 /// Returns whether the CI-found LLVM is currently usable.
@@ -194,7 +200,9 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool
     }
 
     if crate::util::CiEnv::is_ci() {
-        let llvm_sha = detect_llvm_sha(config);
+        // We assume we have access to git, so it's okay to unconditionally pass
+        // `true` here.
+        let llvm_sha = detect_llvm_sha(config, true);
         let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));
         let head_sha = head_sha.trim();
         if llvm_sha == head_sha {
@@ -215,7 +223,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     }
     let llvm_root = config.ci_llvm_root();
     let llvm_stamp = llvm_root.join(".llvm-stamp");
-    let llvm_sha = detect_llvm_sha(&config);
+    let llvm_sha = detect_llvm_sha(&config, builder.rust_info.is_managed_git_subrepository());
     let key = format!("{}{}", llvm_sha, config.llvm_assertions);
     if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
         download_ci_llvm(builder, &llvm_sha);
@@ -260,7 +268,7 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     } else {
         &builder.config.stage0_metadata.config.artifacts_server
     };
-    let channel = builder.config.artifact_channel(llvm_sha);
+    let channel = builder.config.artifact_channel(builder, llvm_sha);
     let filename = format!("rust-dev-{}-{}.tar.xz", channel, builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
     if !tarball.exists() {

src/bootstrap/sanity.rs

@@ -74,7 +74,7 @@ pub fn check(build: &mut Build) {
     let mut cmd_finder = Finder::new();
     // If we've got a git directory we're gonna need git to update
     // submodules and learn about various other aspects.
-    if build.rust_info.is_git() {
+    if build.rust_info.is_managed_git_subrepository() {
         cmd_finder.must_have("git");
     }
 

src/bootstrap/tarball.rs

@@ -4,6 +4,7 @@ use std::{
 };
 
 use crate::builder::Builder;
+use crate::channel;
 use crate::util::t;
 
 #[derive(Copy, Clone)]
@@ -297,8 +298,8 @@ impl<'a> Tarball<'a> {
     fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball {
         t!(std::fs::create_dir_all(&self.overlay_dir));
         self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
-        if let Some(sha) = self.builder.rust_sha() {
-            self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha);
+        if let Some(info) = self.builder.rust_info.info() {
+            channel::write_commit_info_file(&self.overlay_dir, info);
         }
         for file in self.overlay.legal_and_readme() {
             self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644);