infra: download regression tests speed up (#756)

5s instead of 40s

Author: sbdchd

.gitignore

@@ -9,4 +9,3 @@
 dist/
 
 .DS_Store
-temp.sql

crates/xtask/src/download_regression_tests.rs

@@ -121,33 +121,63 @@ fn download_regression_suite() -> Result<Utf8PathBuf> {
 
     create_dir_all(&target_dir)?;
 
-    let urls = fetch_download_urls()?;
-    let total_files = urls.len();
+    let clone_dir = Utf8PathBuf::try_from(std::env::temp_dir())
+        .map_err(|_| anyhow::anyhow!("temp dir path is not valid UTF-8"))?
+        .join("postgres_sparse_clone");
 
-    for (index, url) in urls.iter().enumerate() {
-        let filename = url.split('/').next_back().unwrap();
-        if filename.contains("psql") {
-            continue;
-        }
-        let filepath = target_dir.join(filename);
+    if clone_dir.exists() {
+        remove_dir_all(&clone_dir)?;
+    }
 
-        println!(
-            "[{}/{}] Downloading {}... ",
-            index + 1,
-            total_files,
-            filename
-        );
+    println!("Cloning postgres repository with sparse checkout...");
 
-        let output = Command::new("curl").args(["-s", url]).output()?;
+    let status = Command::new("git")
+        .args([
+            "clone",
+            "--filter=blob:none",
+            "--depth=1",
+            "--sparse",
+            "https://github.com/postgres/postgres.git",
+        ])
+        .arg(clone_dir.as_str())
+        .status()?;
 
-        if !output.status.success() {
-            let error_msg = String::from_utf8_lossy(&output.stderr);
-            bail!("Failed to download '{}': {}", url, error_msg);
-        }
+    if !status.success() {
+        bail!("Failed to clone postgres repository");
+    }
+
+    println!("Setting up sparse checkout for src/test/regress/sql...");
 
-        File::create(&filepath)?.write_all(&output.stdout)?;
+    let status = Command::new("git")
+        .args(["sparse-checkout", "set", "src/test/regress/sql"])
+        .current_dir(&clone_dir)
+        .status()?;
+
+    if !status.success() {
+        bail!("Failed to set sparse checkout");
     }
 
+    println!("Copying SQL files...");
+    let source_dir = clone_dir.join("src/test/regress/sql");
+
+    let mut file_count = 0;
+    for entry in std::fs::read_dir(&source_dir)? {
+        let entry = entry?;
+        let path = Utf8PathBuf::try_from(entry.path())?;
+        if path.extension() == Some("sql") {
+            let filename = path.file_name().unwrap();
+            if !filename.contains("psql") {
+                std::fs::copy(&path, target_dir.join(filename))?;
+                file_count += 1;
+            }
+        }
+    }
+
+    println!("Copied {file_count} SQL files");
+
+    println!("Cleaning up clone directory...");
+    remove_dir_all(&clone_dir)?;
+
     Ok(target_dir)
 }
 
@@ -195,44 +225,6 @@ fn transform_regression_suite(input_dir: &Utf8PathBuf) -> Result<()> {
     Ok(())
 }
 
-fn fetch_download_urls() -> Result<Vec<String>> {
-    println!("Fetching SQL file URLs...");
-    let output = Command::new("gh")
-        .args([
-            "api",
-            "-H",
-            "Accept: application/vnd.github+json",
-            "/repos/postgres/postgres/contents/src/test/regress/sql",
-        ])
-        .output()?;
-
-    if !output.status.success() {
-        bail!(
-            "Failed to fetch SQL files: {}",
-            String::from_utf8_lossy(&output.stderr)
-        );
-    }
-
-    let json_str = String::from_utf8(output.stdout)?;
-    let files: Vec<serde_json::Value> = serde_json::from_str(&json_str)?;
-
-    let urls: Vec<String> = files
-        .into_iter()
-        .filter(|file| {
-            file["name"]
-                .as_str()
-                .is_some_and(|name| name.ends_with(".sql"))
-        })
-        .filter_map(|file| file["download_url"].as_str().map(String::from))
-        .collect();
-
-    if urls.is_empty() {
-        bail!("No SQL files found");
-    }
-
-    Ok(urls)
-}
-
 // The regression suite from postgres has a mix of valid and invalid sql. We
 // don't have a good way to determine what is what, so we munge the data to
 // comment out any problematic code.