feat: Add TOML-based AST pattern loading

This commit introduces the ability to load AST patterns from TOML files,
enabling more flexible and dynamic configuration of detectors. It also
includes necessary dependency updates and refactors related to pattern
management.

Co-authored-by: script3r <[email protected]>

Author: cursoragent

Cargo.lock

@@ -17,6 +17,7 @@ repository = "https://example.com/cipherscope/repo"
 anyhow = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+toml = "0.8"
 rayon = "1"
 ignore = "0.4"
 clap = { version = "4", features = ["derive"] }

Cargo.toml

@@ -8,6 +8,7 @@ license = "Apache-2.0"
 anyhow = { workspace = true }
 clap = { workspace = true }
 serde_json = { workspace = true }
+toml = { workspace = true }
 rayon = { workspace = true }
 indicatif = "0.17"
 scanner-core = { path = "../scanner-core" }

GROUND_TRUTH_REGENERATION_SUMMARY.md

@@ -4,6 +4,7 @@ use indicatif::{ProgressBar, ProgressStyle};
 use scanner_core::{Config, Detector, Language, AstBasedDetector, AstDetector, Scanner, CryptoFindings};
 use std::fs;
 use std::path::PathBuf;
+use std::sync::Arc;
 
 #[derive(Parser, Debug)]
 #[command(name = "cipherscope")]

crates/cli/Cargo.toml

@@ -138,27 +138,27 @@ fn test_progress_monotonic_increase() {
 
     for (processed, discovered, findings) in &all_updates {
         assert!(
-            processed >= last_processed,
+            *processed >= last_processed,
             "Processed count should never decrease: {} -> {}",
             last_processed,
             processed
         );
         assert!(
-            discovered >= last_discovered,
+            *discovered >= last_discovered,
             "Discovered count should never decrease: {} -> {}",
             last_discovered,
             discovered
         );
         assert!(
-            findings >= last_findings,
+            *findings >= last_findings,
             "Findings count should never decrease: {} -> {}",
             last_findings,
             findings
         );
 
-        last_processed = processed;
-        last_discovered = discovered;
-        last_findings = findings;
+        last_processed = *processed;
+        last_discovered = *discovered;
+        last_findings = *findings;
     }
 
     println!("✅ Monotonic increase test passed with {} updates", all_updates.len());

crates/cli/src/main.rs

@@ -78,9 +78,65 @@ impl AstDetector {
 
     /// Load AST patterns from TOML content
     pub fn load_patterns_from_toml(toml_content: &str) -> Result<Vec<AstPattern>> {
-        // For now, return default patterns - this can be expanded to parse the TOML
-        // and convert the library definitions into AST patterns
-        Ok(Self::default_patterns())
+        use crate::PatternsFile;
+        
+        let patterns_file: PatternsFile = toml::from_str(toml_content)?;
+        let mut ast_patterns = Vec::new();
+        
+        // Convert library specs to AST patterns
+        for library in patterns_file.library {
+            ast_patterns.extend(Self::convert_library_to_ast_patterns(&library)?);
+        }
+        
+        // Also include default patterns for comprehensive coverage
+        ast_patterns.extend(Self::default_patterns());
+        
+        Ok(ast_patterns)
+    }
+    
+    /// Convert a library specification to AST patterns
+    fn convert_library_to_ast_patterns(library: &crate::LibrarySpec) -> Result<Vec<AstPattern>> {
+        let mut patterns = Vec::new();
+        
+        for &language in &library.languages {
+            // Convert include patterns to AST patterns
+            for pattern in &library.patterns.include_patterns {
+                patterns.push(AstPattern {
+                    query: pattern.clone(),
+                    language,
+                    match_type: AstMatchType::Library { name: library.name.clone() },
+                    metadata: HashMap::new(),
+                });
+            }
+            
+            // Convert import patterns to AST patterns
+            for pattern in &library.patterns.import_patterns {
+                patterns.push(AstPattern {
+                    query: pattern.clone(),
+                    language,
+                    match_type: AstMatchType::Library { name: library.name.clone() },
+                    metadata: HashMap::new(),
+                });
+            }
+            
+            // Convert algorithm patterns
+            for algorithm in &library.algorithms {
+                for pattern in &algorithm.symbol_patterns {
+                    patterns.push(AstPattern {
+                        query: pattern.clone(),
+                        language,
+                        match_type: AstMatchType::Algorithm {
+                            name: algorithm.name.clone(),
+                            primitive: algorithm.primitive.clone(),
+                            nist_quantum_security_level: algorithm.nist_quantum_security_level,
+                        },
+                        metadata: HashMap::new(),
+                    });
+                }
+            }
+        }
+        
+        Ok(patterns)
     }
 
     /// Default AST patterns for common cryptographic libraries and algorithms