Rust: move qltest to rust code, add options with cargo check

Author: Paolo Tranquilli

Cargo.lock

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0.86"
 clap = { version = "4.5.16", features = ["derive"] }
-figment = { version = "0.10.19", features = ["env"]}
+figment = { version = "0.10.19", features = ["env", "yaml"] }
 log = "0.4.22"
 num-traits = "0.2.19"
 ra_ap_base_db = "0.0.232"
@@ -29,3 +29,5 @@ argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
 rust-extractor-macros = { path = "macros" }
 itertools = "0.13.0"
+glob = "0.3.1"
+gag = "1.0.0"

rust/extractor/Cargo.toml

@@ -2,9 +2,11 @@ use anyhow::Context;
 use clap::Parser;
 use codeql_extractor::trap;
 use figment::{
-    providers::{Env, Serialized},
+    providers::{Env, Format, Serialized, Yaml},
+    value::Value,
     Figment,
 };
+use itertools::Itertools;
 use num_traits::Zero;
 use rust_extractor_macros::extractor_cli_config;
 use serde::{Deserialize, Serialize};
@@ -34,23 +36,39 @@ pub struct Config {
     pub scratch_dir: PathBuf,
     pub trap_dir: PathBuf,
     pub source_archive_dir: PathBuf,
+    pub log_dir: PathBuf,
     pub extract_dependencies: bool,
     pub verbose: u8,
     pub compression: Compression,
     pub inputs: Vec<PathBuf>,
+    pub qltest: bool,
+    pub qltest_cargo_check: bool,
 }
 
 impl Config {
     pub fn extract() -> anyhow::Result<Config> {
         let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
             .context("expanding parameter files")?;
         let cli_args = CliConfig::parse_from(args);
-        Figment::new()
+        let mut figment = Figment::new()
             .merge(Env::prefixed("CODEQL_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
-            .merge(Serialized::defaults(cli_args))
-            .extract()
-            .context("loading configuration")
+            .merge(Serialized::defaults(cli_args));
+        if matches!(figment.find_value("qltest"), Ok(Value::Bool(_, true))) {
+            let cwd = std::env::current_dir()?;
+            let mut option_files = cwd
+                .ancestors()
+                // only travel up while we're within the test pack
+                .take_while_inclusive(|p| !p.join("qlpack.yml").exists())
+                .map(|p| p.join("options"))
+                .filter(|p| p.exists())
+                .collect_vec();
+            option_files.reverse();
+            for path in option_files {
+                figment = figment.merge(Yaml::file_exact(path));
+            }
+        }
+        figment.extract().context("loading configuration")
     }
 }

rust/extractor/src/config.rs

@@ -1,16 +1,19 @@
-use std::{
-    collections::HashMap,
-    path::{Path, PathBuf},
-};
-
 use anyhow::Context;
 use archive::Archiver;
+use log::info;
 use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 use ra_ap_project_model::ProjectManifest;
 use rust_analyzer::{ParseResult, RustAnalyzer};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+};
 mod archive;
 mod config;
 pub mod generated;
+mod qltest;
 mod rust_analyzer;
 mod translate;
 pub mod trap;
@@ -65,16 +68,20 @@ fn extract(
         )
     });
 }
-fn main() -> anyhow::Result<()> {
-    let cfg = config::Config::extract().context("failed to load configuration")?;
+
+fn run_extractor(mut cfg: config::Config) -> anyhow::Result<()> {
     stderrlog::new()
         .module(module_path!())
-        .verbosity(1 + cfg.verbose as usize)
+        .verbosity(cfg.verbose as usize)
         .init()?;
+    if cfg.qltest {
+        qltest::prepare(&mut cfg)?;
+    }
+    info!("configuration: {cfg:#?}\n");
 
     let traps = trap::TrapFileProvider::new(&cfg).context("failed to set up trap files")?;
     let archiver = archive::Archiver {
-        root: cfg.source_archive_dir,
+        root: cfg.source_archive_dir.clone(),
     };
     let files: Vec<PathBuf> = cfg
         .inputs
@@ -118,3 +125,19 @@ fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+fn main() -> anyhow::Result<()> {
+    let cfg = config::Config::extract().context("failed to load configuration")?;
+    let qltest = cfg.qltest;
+    let qltest_log = cfg.log_dir.join("qltest.log");
+    let result = std::panic::catch_unwind(|| run_extractor(cfg));
+    if qltest && matches!(result, Err(_) | Ok(Err(_))) {
+        // in case of failure, print out the full log
+        let log = File::open(qltest_log).context("opening qltest.log")?;
+        let reader = BufReader::new(log);
+        for line in reader.lines() {
+            println!("{}", line.context("reading qltest.log")?);
+        }
+    }
+    result.unwrap()
+}

rust/extractor/src/main.rs

@@ -0,0 +1,84 @@
+use crate::config::Config;
+use anyhow::Context;
+use glob::glob;
+use itertools::Itertools;
+use log::info;
+use std::fs;
+use std::process::Command;
+
+fn dump_lib() -> anyhow::Result<()> {
+    let path_iterator = glob("*.rs").context("globbing test sources")?;
+    let paths = path_iterator
+        .collect::<Result<Vec<_>, _>>()
+        .context("fetching test sources")?;
+    let lib = paths
+        .iter()
+        .filter(|p| !["lib.rs", "main.rs"].contains(&p.file_name().unwrap().to_str().unwrap()))
+        .map(|p| format!("mod {};", p.file_stem().unwrap().to_str().unwrap()))
+        .join("\n");
+    fs::write("lib.rs", lib).context("writing lib.rs")
+}
+
+fn dump_cargo_manifest() -> anyhow::Result<()> {
+    let mut manifest = String::from(
+        r#"[workspace]
+[package]
+name = "test"
+version="0.0.1"
+edition="2021"
+[lib]
+path="lib.rs"
+"#,
+    );
+    if fs::exists("main.rs").context("checking existence of main.rs")? {
+        manifest.push_str(
+            r#"[[bin]]
+name = "main"
+path = "main.rs"
+"#,
+        );
+    }
+    fs::write("Cargo.toml", manifest).context("writing Cargo.toml")
+}
+
+fn set_sources(config: &mut Config) -> anyhow::Result<()> {
+    let path_iterator = glob("*.rs").context("globbing test sources")?;
+    config.inputs = path_iterator
+        .collect::<Result<Vec<_>, _>>()
+        .context("fetching test sources")?;
+    Ok(())
+}
+
+fn redirect_output(config: &Config) -> anyhow::Result<()> {
+    let log_path = config.log_dir.join("qltest.log");
+    let log = fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .open(&log_path)
+        .context("opening qltest.log")?;
+    Box::leak(Box::new(
+        gag::Redirect::stderr(log).context("redirecting stderr")?,
+    ));
+    Ok(())
+}
+
+pub(crate) fn prepare(config: &mut Config) -> anyhow::Result<()> {
+    redirect_output(config)?;
+    dump_lib()?;
+    set_sources(config)?;
+    dump_cargo_manifest()?;
+    if config.qltest_cargo_check {
+        let status = Command::new("cargo")
+            .env("RUSTFLAGS", "-Awarnings")
+            .arg("check")
+            .arg("-q")
+            .status()
+            .context("spawning cargo check")?;
+        if status.success() {
+            info!("cargo check successful");
+        } else {
+            anyhow::bail!("requested cargo check failed");
+        }
+    }
+    Ok(())
+}