fix(turbopack): Use vergen-gitcl instead of shadow-rs (or vergen-git2) for napi and next-api crates to fix stale git lock files (#76889)

bgw · web-flow · commit f70760719aa5 · 2025-03-07T13:24:35.000-08:00
This does not depend on libgit2 which apparently breaks our builds on some platforms. Instead it uses the git CLI using my PR: rustyhorde/vergen#406 Original PR: #76773 Revert: #76879 Manually triggered CI run to try to build all platforms: https://github.com/vercel/next.js/actions/runs/13711866131
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -396,9 +396,6 @@ serde_bytes = "0.11.15"
 serde_path_to_error = "0.1.16"
 serde_qs = "0.13.0"
 serde_with = "3.12.0"
-shadow-rs = { version = "0.37.0", default-features = false, features = [
-  "tzdb",
-] }
 smallvec = { version = "1.13.1", features = [
   "serde",
   "const_generics",
@@ -423,6 +420,8 @@ unicode-segmentation = "1.10.1"
 unsize = "1.1.0"
 url = "2.2.2"
 urlencoding = "2.1.2"
+vergen = { git = "https://github.com/bgw/vergen.git", branch = "bgw/no-optional-locks", features = ["cargo"] }
+vergen-gitcl = { git = "https://github.com/bgw/vergen.git", branch = "bgw/no-optional-locks", features = ["cargo"] }
 webbrowser = "0.8.7"
 
 [patch.crates-io]
diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml
@@ -66,14 +66,13 @@ rand = { workspace = true }
 rustc-hash = { workspace = true }
 serde = "1"
 serde_json = "1"
-shadow-rs = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing-chrome = "0.5.0"
 url = { workspace = true }
 urlencoding = { workspace = true }
 once_cell = { workspace = true }
-dashmap = "6.1.0"
+dashmap = { workspace = true }
 
 swc_core = { workspace = true, features = [
     "base_concurrent",
@@ -139,11 +138,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false }
 tokio = { workspace = true, features = ["full"] }
 
 [build-dependencies]
+anyhow = { workspace = true }
 napi-build = "2"
 serde = { workspace = true }
 serde_json = { workspace = true }
-# It is not a mistake this dependency is specified in dep / build-dep both.
-shadow-rs = { workspace = true }
+vergen-gitcl = { workspace = true }
 
 # build-dependencies for the native, non-wasm32 build
 [target.'cfg(not(target_arch = "wasm32"))'.build-dependencies]
diff --git a/crates/napi/build.rs b/crates/napi/build.rs
@@ -1,18 +1,52 @@
-use std::fs;
+use std::{env, process::Command, str};
 
 extern crate napi_build;
 
-fn main() {
+fn main() -> anyhow::Result<()> {
+    println!("cargo:rerun-if-env-changed=CI");
+    let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
+
     // Generates, stores build-time information as static values.
     // There are some places relying on correct values for this (i.e telemetry),
     // So failing build if this fails.
-    shadow_rs::ShadowBuilder::builder()
-        .build()
-        .expect("Should able to generate build time information");
+    let cargo = vergen_gitcl::CargoBuilder::default()
+        .target_triple(true)
+        .build()?;
+    // We use the git dirty state to disable persistent caching (persistent caching relies on a
+    // commit hash to be safe). One tradeoff of this is that we must invalidate the rust build more
+    // often.
+    //
+    // This invalidates the build if any untracked files change. That's sufficient for the case
+    // where we transition from dirty to clean.
+    //
+    // There's an edge-case here where the repository could be newly dirty, but we can't know
+    // because our build hasn't been invalidated, since the untracked files weren't untracked last
+    // time we ran. That will cause us to incorrectly report ourselves as clean.
+    //
+    // However, in practice that shouldn't be much of an issue: If no other dependency of this
+    // top-level crate has changed (which would've triggered our rebuild), then the resulting binary
+    // must be equivalent to a clean build anyways. Therefore, persistent caching using the HEAD
+    // commit hash as a version is okay.
+    let git = vergen_gitcl::GitclBuilder::default()
+        .dirty(/* include_untracked */ true)
+        .describe(
+            /* tags */ true,
+            /* dirty */ !is_ci, // suppress the dirty suffix in CI
+            /* matches */ Some("v[0-9]*"), // find the last version tag
+        )
+        .build()?;
+    vergen_gitcl::Emitter::default()
+        .add_instructions(&cargo)?
+        .add_instructions(&git)?
+        .fail_on_error()
+        .emit()?;
 
-    let git_head = fs::read_to_string("../../.git/HEAD").unwrap_or_default();
-    if !git_head.is_empty() && !git_head.starts_with("ref: ") {
-        println!("cargo:warning=git version {}", git_head);
+    match Command::new("git").args(["rev-parse", "HEAD"]).output() {
+        Ok(out) if out.status.success() => println!(
+            "cargo:warning=git HEAD: {}",
+            str::from_utf8(&out.stdout).unwrap()
+        ),
+        _ => println!("cargo:warning=`git rev-parse HEAD` failed"),
     }
 
     #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
@@ -36,4 +70,6 @@ fn main() {
 
     #[cfg(not(target_arch = "wasm32"))]
     turbo_tasks_build::generate_register();
+
+    Ok(())
 }
diff --git a/crates/napi/src/lib.rs b/crates/napi/src/lib.rs
@@ -61,9 +61,6 @@ pub mod turbo_trace_server;
 pub mod turbopack;
 pub mod util;
 
-// Declare build-time information variables generated in build.rs
-shadow_rs::shadow!(build);
-
 #[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
 #[global_allocator]
 static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs
@@ -13,7 +13,8 @@ use turbo_tasks::{
     TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc,
 };
 use turbo_tasks_backend::{
-    default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage,
+    default_backing_storage, noop_backing_storage, DefaultBackingStorage, GitVersionInfo,
+    NoopBackingStorage,
 };
 use turbo_tasks_fs::FileContent;
 use turbopack_core::{
@@ -130,26 +131,10 @@ pub fn create_turbo_tasks(
     dependency_tracking: bool,
 ) -> Result<NextTurboTasks> {
     Ok(if persistent_caching {
-        let dirty_suffix = if crate::build::GIT_CLEAN
-            || option_env!("CI").is_some_and(|value| !value.is_empty())
-        {
-            ""
-        } else {
-            "-dirty"
-        };
-        #[allow(
-            clippy::const_is_empty,
-            reason = "LAST_TAG might be empty if the tag can't be determined"
-        )]
-        let version_info = if crate::build::LAST_TAG.is_empty() {
-            format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
-        } else {
-            format!(
-                "{}-{}{}",
-                crate::build::LAST_TAG,
-                crate::build::SHORT_COMMIT,
-                dirty_suffix
-            )
+        let version_info = GitVersionInfo {
+            describe: env!("VERGEN_GIT_DESCRIBE"),
+            dirty: option_env!("CI").is_none_or(|value| value.is_empty())
+                && env!("VERGEN_GIT_DIRTY") == "true",
         };
         NextTurboTasks::PersistentCaching(TurboTasks::new(
             turbo_tasks_backend::TurboTasksBackend::new(
diff --git a/crates/napi/src/util.rs b/crates/napi/src/util.rs
@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {
 }
 
 #[napi]
-pub fn get_target_triple() -> String {
-    crate::build::BUILD_TARGET.to_string()
+pub fn get_target_triple() -> &'static str {
+    env!("VERGEN_CARGO_TARGET_TRIPLE")
 }
 
 pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {
diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml
@@ -24,7 +24,6 @@ regex = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-shadow-rs = { workspace = true }
 swc_core = { workspace = true }
 tracing = { workspace = true }
 turbo-rcstr = { workspace = true }
@@ -43,6 +42,6 @@ turbopack-node = { workspace = true }
 turbopack-nodejs = { workspace = true }
 
 [build-dependencies]
-# It is not a mistake this dependency is specified in dep / build-dep both.
-shadow-rs = { workspace = true }
+anyhow = { workspace = true }
 turbo-tasks-build = { workspace = true }
+vergen = { workspace = true }
diff --git a/crates/next-api/build.rs b/crates/next-api/build.rs
@@ -1,13 +1,18 @@
 use turbo_tasks_build::generate_register;
 
-fn main() {
+fn main() -> anyhow::Result<()> {
     // Generates, stores build-time information as static values.
     // There are some places relying on correct values for this (i.e telemetry),
     // So failing build if this fails.
-    shadow_rs::ShadowBuilder::builder()
-        .build_pattern(shadow_rs::BuildPattern::Lazy)
-        .build()
-        .expect("Should able to generate build time information");
+    let cargo = vergen::CargoBuilder::default()
+        .target_triple(true)
+        .build()?;
+    vergen::Emitter::default()
+        .add_instructions(&cargo)?
+        .fail_on_error()
+        .emit()?;
 
     generate_register();
+
+    Ok(())
 }
diff --git a/crates/next-api/src/lib.rs b/crates/next-api/src/lib.rs
@@ -23,9 +23,6 @@ mod server_actions;
 mod versioned_content_map;
 mod webpack_stats;
 
-// Declare build-time information variables generated in build.rs
-shadow_rs::shadow!(build);
-
 pub fn register() {
     next_core::register();
     turbopack_nodejs::register();
diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs
@@ -68,7 +68,6 @@ use turbopack_nodejs::NodeJsChunkingContext;
 
 use crate::{
     app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
-    build,
     empty::EmptyEndpoint,
     entrypoints::Entrypoints,
     instrumentation::InstrumentationEndpoint,
@@ -1069,7 +1068,7 @@ impl Project {
         // First, emit an event for the binary target triple.
         // This is different to webpack-config; when this is being called,
         // it is always using SWC so we don't check swc here.
-        emit_event(build::BUILD_TARGET, true);
+        emit_event(env!("VERGEN_CARGO_TARGET_TRIPLE"), true);
 
         // Go over jsconfig and report enabled features.
         let compiler_options = self.js_config().compiler_options().await?;
diff --git a/crates/next-build/Cargo.toml b/crates/next-build/Cargo.toml
@@ -16,25 +16,5 @@ workspace = true
 next-core = { workspace = true }
 turbopack-core = { workspace = true }
 
-#turbopack-binding = { workspace = true, features = [
-#  "__turbo_tasks",
-#  "__turbo_tasks_memory",
-#  "__turbo_tasks_env",
-#  "__turbo_tasks_fs",
-#  "__turbo_tasks_memory",
-#  "__turbopack",
-#  "__turbopack_nodejs",
-#  "__turbopack_core",
-#  "__turbopack_browser",
-#  "__turbopack_ecmascript",
-#  "__turbopack_ecmascript_runtime",
-#  "__turbopack_env",
-#  "__turbopack_node",
-#] }
-
 [build-dependencies]
 turbo-tasks-build = { workspace = true }
-vergen = { version = "7.3.2", default-features = false, features = [
-  "cargo",
-  "build",
-] }
diff --git a/crates/next-build/build.rs b/crates/next-build/build.rs
@@ -1,10 +1,5 @@
 use turbo_tasks_build::generate_register;
-use vergen::{vergen, Config};
 
 fn main() {
     generate_register();
-
-    // Attempt to collect some build time env values but will skip if there are any
-    // errors.
-    let _ = vergen(Config::default());
 }
diff --git a/turbopack/crates/turbo-tasks-backend/src/database/db_versioning.rs b/turbopack/crates/turbo-tasks-backend/src/database/db_versioning.rs
@@ -7,24 +7,31 @@ use std::{
 
 use anyhow::Result;
 
+/// Information gathered by `vergen_gitcl` in the top-level binary crate and passed down. This
+/// information must be computed in the top-level crate for cargo incremental compilation to work
+/// correctly.
+///
+/// See `crates/napi/build.rs` for details.
+pub struct GitVersionInfo<'a> {
+    /// Output of `git describe --match 'v[0-9]' --dirty`.
+    pub describe: &'a str,
+    /// Is the git repository dirty? Always forced to `false` when the `CI` environment variable is
+    /// set and non-empty.
+    pub dirty: bool,
+}
+
 /// Specifies many databases that have a different version than the current one are retained.
 /// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
 /// the current one and two older/newer ones.
 const MAX_OTHER_DB_VERSIONS: usize = 2;
 
-pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
+pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> {
     if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
         return Ok(base_path.join(version));
     }
     // Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
     // dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
     // versioning and always use the same database.
-    let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
-    {
-        (version_info, true)
-    } else {
-        (version_info, false)
-    };
     let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some();
     let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some();
     let version = if disabled_versioning {
@@ -33,14 +40,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<Path
              caching database might be required."
         );
         Some("unversioned")
-    } else if !git_dirty {
-        Some(version_info)
+    } else if !version_info.dirty {
+        Some(version_info.describe)
     } else if ignore_dirty {
         println!(
             "WARNING: The git repository is dirty, but Persistent Caching is still enabled. \
              Manual removal of the persistent caching database might be required."
         );
-        Some(version_info)
+        Some(version_info.describe)
     } else {
         println!(
             "WARNING: The git repository is dirty: Persistent Caching is disabled. Use \
diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs
diff --git a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs

Original file line number	Diff line number	Diff line change
`@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {`
`127`	`127`	`}`
`128`	`128`
`129`	`129`	`#[napi]`
`130`		`-pub fn get_target_triple() -> String {`
`131`		`- crate::build::BUILD_TARGET.to_string()`
	`130`	`+pub fn get_target_triple() -> &'static str {`
	`131`	`+ env!("VERGEN_CARGO_TARGET_TRIPLE")`
`132`	`132`	`}`
`133`	`133`
`134`	`134`	`pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {`