Auto merge of rust-lang#140244 - Shourya742:2025-04-24-enzyme-distribution, r=<try>

Add enzyme distribution step

This PR adds enzyme to bootstrap dist.

r? `@ghost`

try-job: dist-x86_64-linux

Author: bors

src/bootstrap/src/core/build_steps/dist.rs

@@ -2599,3 +2599,55 @@ impl Step for Gcc {
         tarball.generate()
     }
 }
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct Enzyme {
+    pub target: TargetSelection,
+}
+
+impl Step for Enzyme {
+    type Output = Option<GeneratedTarball>;
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("enzyme")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Enzyme { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let mut tarball = Tarball::new(builder, "enzyme", &self.target.triple);
+        let enzyme_dir =
+            builder.ensure(super::llvm::Enzyme { target: self.target }).join("build/Enzyme");
+
+        tarball.set_overlay(OverlayKind::Enzyme);
+        tarball.is_preview(true);
+
+        if let Some(llvm_config) = builder.llvm_config(builder.config.build) {
+            let major = llvm::get_llvm_version_major(builder, &llvm_config);
+            let prefix = format!("libEnzyme-{major}");
+            let mut found = false;
+
+            for entry in std::fs::read_dir(&enzyme_dir)
+                .unwrap_or_else(|_| panic!("Failed to read {:?}", enzyme_dir))
+            {
+                let path = entry.unwrap().path();
+                if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
+                    if name.starts_with(&prefix) && is_dylib(&path) {
+                        tarball.add_file(path, "", FileType::NativeLibrary);
+                        found = true;
+                    }
+                }
+            }
+
+            assert!(found, "Enzyme library starting with '{}' not found", prefix);
+            tarball.add_legal_and_readme_to("share/doc/enzyme");
+            return Some(tarball.generate());
+        }
+
+        None
+    }
+}

src/bootstrap/src/core/build_steps/llvm.rs

@@ -903,6 +903,14 @@ impl Step for Enzyme {
         ),
     )]
     fn run(self, builder: &Builder<'_>) -> PathBuf {
+        // FIXME: This can be removed once we start shipping the CMake files needed by Enzyme
+        // to build against a standalone LLVM binary.
+        // For now, Enzyme must be built with the in-tree LLVM and shared LLVM (`llvm.link-shared = true`), not one downloaded from CI.
+        if builder.config.llvm_from_ci || !builder.config.llvm_link_shared() {
+            panic!(
+                "Enzyme must be built with an in-tree LLVM and requires `llvm.link-shared = true`.Set `llvm.download-ci-llvm = false` and `llvm.link-shared = true` to build Enzyme."
+            );
+        }
         builder.require_submodule(
             "src/tools/enzyme",
             Some("The Enzyme sources are required for autodiff."),
@@ -970,6 +978,7 @@ impl Step for Enzyme {
             .env("LLVM_CONFIG_REAL", &llvm_config)
             .define("LLVM_ENABLE_ASSERTIONS", "ON")
             .define("ENZYME_EXTERNAL_SHARED_LIB", "ON")
+            // Enzyme must be built against this exact LLVM build — mixing versions breaks compatibility.
             .define("LLVM_DIR", builder.llvm_out(target));
 
         cfg.build();

src/bootstrap/src/core/builder/mod.rs

@@ -1084,7 +1084,8 @@ impl<'a> Builder<'a> {
                 dist::PlainSourceTarball,
                 dist::BuildManifest,
                 dist::ReproducibleArtifacts,
-                dist::Gcc
+                dist::Gcc,
+                dist::Enzyme
             ),
             Kind::Install => describe!(
                 install::Docs,

src/bootstrap/src/utils/tarball.rs

@@ -26,6 +26,7 @@ pub(crate) enum OverlayKind {
     RustAnalyzer,
     RustcCodegenCranelift,
     LlvmBitcodeLinker,
+    Enzyme,
 }
 
 impl OverlayKind {
@@ -72,6 +73,7 @@ impl OverlayKind {
                 "LICENSE-MIT",
                 "src/tools/llvm-bitcode-linker/README.md",
             ],
+            OverlayKind::Enzyme => &["src/tools/enzyme/Readme.md", "src/tools/enzyme/LICENSE"],
         }
     }
 
@@ -94,6 +96,7 @@ impl OverlayKind {
                 .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
             OverlayKind::RustcCodegenCranelift => builder.rust_version(),
             OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
+            OverlayKind::Enzyme => builder.rust_version(),
         }
     }
 }

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -103,7 +103,8 @@ ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
     --include-default-paths \
     build-manifest bootstrap && \
     # Use GCC for building GCC, as it seems to behave badly when built with Clang
-    CC=/rustroot/bin/cc CXX=/rustroot/bin/c++ python3 ../x.py dist gcc
+    # Also build Enzyme only on the x64 Linux dist runner
+    CC=/rustroot/bin/cc CXX=/rustroot/bin/c++ python3 ../x.py dist gcc && ../x.py dist enzyme
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 
 # This is the only builder which will create source tarballs