Rust: make the cli flags override automatic

This makes the clap flags overlay over `Config` entirely derived via an
attribute macro. Also, the `--intputs-file` option is replaced by a more
standard and versatile `@` parameter file mechanism.

Author: Paolo Tranquilli

Cargo.lock

@@ -6,6 +6,7 @@ members = [
     "shared/tree-sitter-extractor",
     "ruby/extractor",
     "rust/extractor",
+    "rust/extractor/macros",
 ]
 
 [patch.crates-io]

Cargo.toml

@@ -22,7 +22,6 @@ serde = "1.0.209"
 serde_with = "3.9.0"
 stderrlog = "0.6.0"
 triomphe = "0.1.13"
-# Ideally, we'd like to pull this in via a relative path.
-# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
-# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
+argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
+rust-extractor-macros = { path = "macros" }

rust/extractor/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "rust-extractor-macros"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.37"
+syn = { version = "2.0.77", features = ["full"] }

rust/extractor/macros/Cargo.toml

@@ -0,0 +1,52 @@
+use proc_macro::TokenStream;
+use quote::{quote, format_ident};
+use syn;
+
+
+/// Allow all fields in the extractor config to be also overrideable by extractor CLI flags
+#[proc_macro_attribute]
+pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let ast = syn::parse_macro_input!(item as syn::ItemStruct);
+    let name = &ast.ident;
+    let new_name = format_ident!("Cli{}", name);
+    let fields: Vec<_> = ast.fields.iter().map(|f| {
+        let id = f.ident.as_ref().unwrap();
+        let ty = &f.ty;
+        if let syn::Type::Path(p) = ty {
+            if p.path.is_ident(&format_ident!("bool")) {
+                return quote! {
+                    #[arg(long)]
+                    #id: bool,
+                };
+            }
+        }
+        if id == &format_ident!("verbose") {
+            quote! {
+                #[arg(long, short, action=clap::ArgAction::Count)]
+                #id: u8,
+            }
+        } else if id == &format_ident!("inputs") {
+            quote! {
+                #id: #ty,
+            }
+        } else {
+            quote! {
+                #[arg(long)]
+                #id: Option<#ty>,
+            }
+        }
+    }).collect();
+    let gen = quote! {
+        #[serde_with::apply(_ => #[serde(default)])]
+        #[derive(Debug, Deserialize, Default)]
+        #ast
+
+        #[serde_with::skip_serializing_none]
+        #[derive(clap::Parser, Serialize)]
+        #[command(about, long_about = None)]
+        struct #new_name {
+            #(#fields)*
+        }
+    };
+    gen.into()
+}

rust/extractor/macros/src/lib.rs

@@ -1,14 +1,15 @@
 use anyhow::Context;
-use clap::{ArgAction, Parser, ValueEnum};
+use clap::Parser;
 use codeql_extractor::trap;
 use figment::{
     providers::{Env, Serialized},
     Figment,
 };
+use rust_extractor_macros::extractor_cli_config;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
-#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)]
+#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)]
 #[serde(rename_all = "lowercase")]
 #[clap(rename_all = "lowercase")]
 pub enum Compression {
@@ -26,8 +27,7 @@ impl From<Compression> for trap::Compression {
     }
 }
 
-#[serde_with::apply(_ => #[serde(default)])]
-#[derive(Debug, Deserialize, Default)]
+#[extractor_cli_config]
 pub struct Config {
     pub scratch_dir: PathBuf,
     pub trap_dir: PathBuf,
@@ -38,39 +38,10 @@ pub struct Config {
     pub inputs: Vec<PathBuf>,
 }
 
-#[serde_with::apply(_ => #[serde(skip_serializing_if = "is_default")])]
-#[derive(clap::Parser, Serialize)]
-#[command(about, long_about = None)]
-struct CliArgs {
-    #[arg(long)]
-    scratch_dir: Option<PathBuf>,
-    #[arg(long)]
-    trap_dir: Option<PathBuf>,
-    #[arg(long)]
-    source_archive_dir: Option<PathBuf>,
-    #[arg(long)]
-    compression: Option<Compression>,
-    #[arg(short, long, action = ArgAction::Count)]
-    verbose: u8,
-    #[arg(long)]
-    inputs_file: Option<PathBuf>,
-
-    inputs: Vec<PathBuf>,
-}
-
-fn is_default<T: Default + PartialEq>(t: &T) -> bool {
-    *t == Default::default()
-}
-
 impl Config {
     pub fn extract() -> anyhow::Result<Config> {
-        let mut cli_args = CliArgs::parse();
-        if let Some(inputs_file) = cli_args.inputs_file.take() {
-            let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?;
-            cli_args
-                .inputs
-                .extend(inputs_list.split_terminator("\n").map(PathBuf::from));
-        }
+        let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)?;
+        let cli_args = CliConfig::parse_from(args);
         Figment::new()
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))

rust/extractor/src/config.rs

@@ -2,4 +2,4 @@
 
 set -eu
 
-exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --inputs-file="$1"
+exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" @"$1"