Include additional hashes in src/stage0

This patch changes `bump-stage0` to include:

* The sha256 hash of the channel manifest used to create `src/stage0`.
* The rust and rustfmt git commit in `src/stage0`.
* Hashes of all the artifacts, like the source tarball, in `src/stage0`.

Combined this will allow for:

* Projects that bootstrap their own compiler, such as Fuchsia, or users
  of [bootstrap], to build their compilers offline without needing to
  communicate with static.rust-lang.org.

* Auditors to detect if the channel manifest, and all the artifacts
  inside the manifest, were modified after it was used to generate
  `src/stage0`. Furthermore, if they did find modified artifacts, they
  could determine if the Rust Signing Key was compromised by checking if
  any modified file was signed properly.

Finally, it allows regeneration of `src/stage0` when specifying both the
day of the build for rust, and the day of the build for rustfmt, which
can allow a maintainer to regenerate `src/stage0` to verify nothing
changed.

[bootstrap]: https://github.com/dtolnay/bootstrap
[mrustc]: https://github.com/thepowersgang/mrustc

Author: erickt

Cargo.lock

@@ -334,8 +334,10 @@ dependencies = [
  "anyhow",
  "build_helper",
  "curl",
+ "hex",
  "indexmap",
  "serde",
+ "sha2",
  "toml 0.8.23",
 ]
 

src/bootstrap/src/core/download.rs

@@ -506,7 +506,7 @@ pub(crate) fn maybe_download_rustfmt<'a>(
         return Some(PathBuf::new());
     }
 
-    let VersionMetadata { date, version } = dwn_ctx.stage0_metadata.rustfmt.as_ref()?;
+    let VersionMetadata { date, version, .. } = dwn_ctx.stage0_metadata.rustfmt.as_ref()?;
     let channel = format!("{version}-{date}");
 
     let host = dwn_ctx.host_target;

src/build_helper/src/stage0_parser.rs

@@ -10,6 +10,8 @@ pub struct Stage0 {
 
 #[derive(Default, Clone)]
 pub struct VersionMetadata {
+    pub channel_manifest_hash: String,
+    pub git_commit_hash: String,
     pub date: String,
     pub version: String,
 }
@@ -50,9 +52,21 @@ pub fn parse_stage0_file() -> Stage0 {
             "git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(),
             "nightly_branch" => stage0.config.nightly_branch = value.to_owned(),
 
+            "compiler_channel_manifest_hash" => {
+                stage0.compiler.channel_manifest_hash = value.to_owned()
+            }
+            "compiler_git_commit_hash" => stage0.compiler.git_commit_hash = value.to_owned(),
             "compiler_date" => stage0.compiler.date = value.to_owned(),
             "compiler_version" => stage0.compiler.version = value.to_owned(),
 
+            "rustfmt_channel_manifest_hash" => {
+                stage0.rustfmt.get_or_insert(VersionMetadata::default()).channel_manifest_hash =
+                    value.to_owned();
+            }
+            "rustfmt_git_commit_hash" => {
+                stage0.rustfmt.get_or_insert(VersionMetadata::default()).git_commit_hash =
+                    value.to_owned();
+            }
             "rustfmt_date" => {
                 stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned();
             }

src/tools/bump-stage0/Cargo.toml

@@ -9,6 +9,8 @@ edition = "2021"
 anyhow = "1.0.34"
 build_helper = { path = "../../build_helper" }
 curl = "0.4.38"
+hex = "0.4.3"
 indexmap = { version = "2.0.0", features = ["serde"] }
 serde = { version = "1.0.125", features = ["derive"] }
 toml = "0.8.23"
+sha2 = "0.10.1"

src/tools/bump-stage0/src/main.rs

@@ -4,6 +4,7 @@ use anyhow::{Context, Error};
 use build_helper::stage0_parser::{Stage0Config, VersionMetadata, parse_stage0_file};
 use curl::easy::Easy;
 use indexmap::IndexMap;
+use sha2::{Digest, Sha256};
 
 const PATH: &str = "src/stage0";
 const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo", "clippy-preview"];
@@ -13,13 +14,14 @@ struct Tool {
     config: Stage0Config,
 
     channel: Channel,
-    date: Option<String>,
+    compiler_date: Option<String>,
+    rustfmt_date: Option<String>,
     version: [u16; 3],
     checksums: IndexMap<String, String>,
 }
 
 impl Tool {
-    fn new(date: Option<String>) -> Result<Self, Error> {
+    fn new(compiler_date: Option<String>, rustfmt_date: Option<String>) -> Result<Self, Error> {
         let channel = match std::fs::read_to_string("src/ci/channel")?.trim() {
             "stable" => Channel::Stable,
             "beta" => Channel::Beta,
@@ -38,7 +40,14 @@ impl Tool {
 
         let existing = parse_stage0_file();
 
-        Ok(Self { channel, version, date, config: existing.config, checksums: IndexMap::new() })
+        Ok(Self {
+            channel,
+            version,
+            compiler_date,
+            rustfmt_date,
+            config: existing.config,
+            checksums: IndexMap::new(),
+        })
     }
 
     fn update_stage0_file(mut self) -> Result<(), Error> {
@@ -78,10 +87,21 @@ impl Tool {
         file_content.push_str("\n");
 
         let compiler = self.detect_compiler()?;
+        file_content.push_str(&format!(
+            "compiler_channel_manifest_hash={}\n",
+            compiler.channel_manifest_hash
+        ));
+        file_content.push_str(&format!("compiler_git_commit_hash={}\n", compiler.git_commit_hash));
         file_content.push_str(&format!("compiler_date={}\n", compiler.date));
         file_content.push_str(&format!("compiler_version={}\n", compiler.version));
 
         if let Some(rustfmt) = self.detect_rustfmt()? {
+            file_content.push_str(&format!(
+                "rustfmt_channel_manifest_hash={}\n",
+                rustfmt.channel_manifest_hash
+            ));
+            file_content
+                .push_str(&format!("rustfmt_git_commit_hash={}\n", rustfmt.git_commit_hash));
             file_content.push_str(&format!("rustfmt_date={}\n", rustfmt.date));
             file_content.push_str(&format!("rustfmt_version={}\n", rustfmt.version));
         }
@@ -112,9 +132,16 @@ impl Tool {
             Channel::Nightly => "beta".to_string(),
         };
 
-        let manifest = fetch_manifest(&self.config, &channel, self.date.as_deref())?;
+        let (manifest, manifest_hash) =
+            fetch_manifest(&self.config, &channel, self.compiler_date.as_deref())?;
         self.collect_checksums(&manifest, COMPILER_COMPONENTS)?;
         Ok(VersionMetadata {
+            channel_manifest_hash: manifest_hash,
+            git_commit_hash: manifest.pkg["rust"]
+                .git_commit_hash
+                .as_ref()
+                .expect("invalid git_commit_hash")
+                .into(),
             date: manifest.date,
             version: if self.channel == Channel::Nightly {
                 "beta".to_string()
@@ -138,9 +165,19 @@ impl Tool {
             return Ok(None);
         }
 
-        let manifest = fetch_manifest(&self.config, "nightly", self.date.as_deref())?;
+        let (manifest, manifest_hash) =
+            fetch_manifest(&self.config, "nightly", self.rustfmt_date.as_deref())?;
         self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?;
-        Ok(Some(VersionMetadata { date: manifest.date, version: "nightly".into() }))
+        Ok(Some(VersionMetadata {
+            channel_manifest_hash: manifest_hash,
+            git_commit_hash: manifest.pkg["rust"]
+                .git_commit_hash
+                .as_ref()
+                .expect("invalid git_commit_hash")
+                .into(),
+            date: manifest.date,
+            version: "nightly".into(),
+        }))
     }
 
     fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> {
@@ -164,12 +201,29 @@ impl Tool {
                 }
             }
         }
+        for artifact in manifest.artifacts.values() {
+            for targets in artifact.target.values() {
+                for target in targets {
+                    let url = target
+                        .url
+                        .strip_prefix(&prefix)
+                        .ok_or_else(|| {
+                            anyhow::anyhow!(
+                                "url doesn't start with dist server base: {}",
+                                target.url
+                            )
+                        })?
+                        .to_string();
+                    self.checksums.insert(url, target.hash_sha256.clone());
+                }
+            }
+        }
         Ok(())
     }
 }
 
 fn main() -> Result<(), Error> {
-    let tool = Tool::new(std::env::args().nth(1))?;
+    let tool = Tool::new(std::env::args().nth(1), std::env::args().nth(2))?;
     tool.update_stage0_file()?;
     Ok(())
 }
@@ -178,18 +232,24 @@ fn fetch_manifest(
     config: &Stage0Config,
     channel: &str,
     date: Option<&str>,
-) -> Result<Manifest, Error> {
+) -> Result<(Manifest, String), Error> {
     let url = if let Some(date) = date {
         format!("{}/dist/{}/channel-rust-{}.toml", config.dist_server, date, channel)
     } else {
         format!("{}/dist/channel-rust-{}.toml", config.dist_server, channel)
     };
 
+    let manifest_bytes = http_get(&url)?;
+
+    let mut sha256 = Sha256::new();
+    sha256.update(&manifest_bytes);
+    let manifest_hash = hex::encode(sha256.finalize());
+
     // FIXME: on newer `toml` (>= `0.9.*`), use `toml::from_slice`. For now, we use the most recent
     // `toml` available in-tree which is `0.8.*`, so we have to do an additional dance here.
-    let response = http_get(&url)?;
-    let response = String::from_utf8(response)?;
-    Ok(toml::from_str(&response)?)
+    let manifest_str = String::from_utf8(manifest_bytes)?;
+    let manifest = toml::from_str(&manifest_str)?;
+    Ok((manifest, manifest_hash))
 }
 
 fn http_get(url: &str) -> Result<Vec<u8>, Error> {
@@ -219,11 +279,14 @@ enum Channel {
 struct Manifest {
     date: String,
     pkg: IndexMap<String, ManifestPackage>,
+    artifacts: IndexMap<String, ManifestArtifact>,
 }
 
 #[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct ManifestPackage {
     version: String,
+    #[serde(default)]
+    git_commit_hash: Option<String>,
     target: IndexMap<String, ManifestTargetPackage>,
 }
 
@@ -234,3 +297,15 @@ struct ManifestTargetPackage {
     xz_url: Option<String>,
     xz_hash: Option<String>,
 }
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+struct ManifestArtifact {
+    target: IndexMap<String, Vec<ManifestTargetArtifact>>,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct ManifestTargetArtifact {
+    url: String,
+    hash_sha256: String,
+}