chore: serve node headers from a test server to fix flaky node-gyp test (denoland#26749)

Fixes denoland#24749

Runs a server that just returns the header tarball and checksum, and
sets the `NODEJS_ORG_MIRROR` env var so that `node-gyp` uses it instead
of `nodejs.org`

Author: nathanwhit

Cargo.lock

@@ -2,7 +2,7 @@
   "name": "@denotest/node-addon",
   "version": "1.0.0",
   "scripts": {
-    "install": "node-gyp configure build"
+    "install": "node-gyp configure --verbose build"
   },
   "dependencies": {
     "node-gyp": "10.1.0"

tests/registry/npm/@denotest/node-addon/1.0.0/package.json

@@ -21,6 +21,7 @@ bytes.workspace = true
 console_static_text.workspace = true
 deno_unsync = "0"
 denokv_proto.workspace = true
+faster-hex.workspace = true
 fastwebsockets.workspace = true
 flate2 = { workspace = true, features = ["default"] }
 futures.workspace = true

tests/testdata/assets/node-gyp/node-v20.11.1-headers.tar.gz

@@ -28,6 +28,7 @@ use crate::fs::PathRef;
 use crate::http_server;
 use crate::jsr_registry_unset_url;
 use crate::lsp::LspClientBuilder;
+use crate::nodejs_org_mirror_unset_url;
 use crate::npm_registry_unset_url;
 use crate::pty::Pty;
 use crate::strip_ansi_codes;
@@ -843,6 +844,12 @@ impl TestCommandBuilder {
     if !envs.contains_key("JSR_URL") {
       envs.insert("JSR_URL".to_string(), jsr_registry_unset_url());
     }
+    if !envs.contains_key("NODEJS_ORG_MIRROR") {
+      envs.insert(
+        "NODEJS_ORG_MIRROR".to_string(),
+        nodejs_org_mirror_unset_url(),
+      );
+    }
     for key in &self.envs_remove {
       envs.remove(key);
     }

tests/testdata/assets/node-gyp/v20.11.1__win-x64__node.lib.tar.gz

@@ -52,6 +52,7 @@ static GUARD: Lazy<Mutex<HttpServerCount>> = Lazy::new(Default::default);
 pub fn env_vars_for_npm_tests() -> Vec<(String, String)> {
   vec![
     ("NPM_CONFIG_REGISTRY".to_string(), npm_registry_url()),
+    ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()),
     ("NO_COLOR".to_string(), "1".to_string()),
   ]
 }
@@ -130,6 +131,7 @@ pub fn env_vars_for_jsr_npm_tests() -> Vec<(String, String)> {
     ),
     ("DISABLE_JSR_PROVENANCE".to_string(), "true".to_string()),
     ("NO_COLOR".to_string(), "1".to_string()),
+    ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()),
   ]
 }
 
@@ -175,27 +177,41 @@ pub fn deno_config_path() -> PathRef {
 
 /// Test server registry url.
 pub fn npm_registry_url() -> String {
-  "http://localhost:4260/".to_string()
+  format!("http://localhost:{}/", servers::PUBLIC_NPM_REGISTRY_PORT)
 }
 
 pub fn npm_registry_unset_url() -> String {
   "http://NPM_CONFIG_REGISTRY.is.unset".to_string()
 }
 
+pub fn nodejs_org_mirror_url() -> String {
+  format!(
+    "http://127.0.0.1:{}/",
+    servers::NODEJS_ORG_MIRROR_SERVER_PORT
+  )
+}
+
+pub fn nodejs_org_mirror_unset_url() -> String {
+  "http://NODEJS_ORG_MIRROR.is.unset".to_string()
+}
+
 pub fn jsr_registry_url() -> String {
-  "http://127.0.0.1:4250/".to_string()
+  format!("http://127.0.0.1:{}/", servers::JSR_REGISTRY_SERVER_PORT)
 }
 
 pub fn rekor_url() -> String {
-  "http://127.0.0.1:4251".to_string()
+  format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT)
 }
 
 pub fn fulcio_url() -> String {
-  "http://127.0.0.1:4251".to_string()
+  format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT)
 }
 
 pub fn gha_token_url() -> String {
-  "http://127.0.0.1:4251/gha_oidc?test=true".to_string()
+  format!(
+    "http://127.0.0.1:{}/gha_oidc?test=true",
+    servers::PROVENANCE_MOCK_SERVER_PORT
+  )
 }
 
 pub fn jsr_registry_unset_url() -> String {
@@ -307,7 +323,7 @@ async fn get_tcp_listener_stream(
   futures::stream::select_all(listeners)
 }
 
-pub const TEST_SERVERS_COUNT: usize = 32;
+pub const TEST_SERVERS_COUNT: usize = 33;
 
 #[derive(Default)]
 struct HttpServerCount {
@@ -565,6 +581,7 @@ pub fn deno_cmd_with_deno_dir(deno_dir: &TempDir) -> TestCommandBuilder {
   TestCommandBuilder::new(deno_dir.clone())
     .env("DENO_DIR", deno_dir.path())
     .env("NPM_CONFIG_REGISTRY", npm_registry_unset_url())
+    .env("NODEJS_ORG_MIRROR", nodejs_org_mirror_unset_url())
     .env("JSR_URL", jsr_registry_unset_url())
 }
 

tests/util/server/Cargo.toml

@@ -39,6 +39,7 @@ use tokio::net::TcpStream;
 mod grpc;
 mod hyper_utils;
 mod jsr_registry;
+mod nodejs_org_mirror;
 mod npm_registry;
 mod ws;
 
@@ -86,8 +87,9 @@ const WS_CLOSE_PORT: u16 = 4244;
 const WS_PING_PORT: u16 = 4245;
 const H2_GRPC_PORT: u16 = 4246;
 const H2S_GRPC_PORT: u16 = 4247;
-const JSR_REGISTRY_SERVER_PORT: u16 = 4250;
-const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251;
+pub(crate) const JSR_REGISTRY_SERVER_PORT: u16 = 4250;
+pub(crate) const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251;
+pub(crate) const NODEJS_ORG_MIRROR_SERVER_PORT: u16 = 4252;
 pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260;
 pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261;
 pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262;
@@ -147,6 +149,10 @@ pub async fn run_all_servers() {
   let private_npm_registry_3_server_futs =
     npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT);
 
+  // for serving node header files to node-gyp in tests
+  let node_js_mirror_server_fut =
+    nodejs_org_mirror::nodejs_org_mirror(NODEJS_ORG_MIRROR_SERVER_PORT);
+
   let mut futures = vec![
     redirect_server_fut.boxed_local(),
     ws_server_fut.boxed_local(),
@@ -172,6 +178,7 @@ pub async fn run_all_servers() {
     h2_grpc_server_fut.boxed_local(),
     registry_server_fut.boxed_local(),
     provenance_mock_server_fut.boxed_local(),
+    node_js_mirror_server_fut.boxed_local(),
   ];
   futures.extend(npm_registry_server_futs);
   futures.extend(private_npm_registry_1_server_futs);