feat(detectors): add more detectors and increase unit test coverage

README.md

@@ -31,6 +31,17 @@ KeyWatch/
     └── integration_tests.rs  // Integration tests.
 ```
 
+## Key connections
+
+```graph TD
+    A[main.rs] --> B[cli.rs]
+    A --> C[scanner.rs]
+    C --> D[detector.rs]
+    D --> E[detectors.toml]
+    C --> F[report.rs]
+    A --> G[utils.rs]
+```
+
 ## Usage
 
 ### Build the project

detectors.toml

@@ -36,8 +36,8 @@ severity = "HIGH"
 
 [[detectors]]
 name = "PrivateKeyDetector"
-pattern = "-----BEGIN PRIVATE KEY-----"
-finding_type = "Generic Private Key"
+pattern = "(?s)-----BEGIN (RSA|DSA|EC|PGP)? PRIVATE KEY-----\\n[\\s\\S]*?-----END (RSA|DSA|EC|PGP)? PRIVATE KEY-----"
+finding_type = "Private Key Content"
 severity = "HIGH"
 
 [[detectors]]
@@ -129,3 +129,124 @@ name = "RandomString"
 pattern = "\"[a-zA-Z0-9\\-_=]{35,}\""
 finding_type = "Random String"
 severity = "LOW"
+
+[[detectors]]
+name = "FirebaseAPIKeyDetector"
+pattern = "AIza[0-9A-Za-z\\-_]{35}"
+finding_type = "Firebase API Key"
+severity = "HIGH"
+
+
+[[detectors]]
+name = "TwilioAPIKeyDetector"
+pattern = "SK[0-9a-fA-F]{32}"
+finding_type = "Twilio API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "SendGridAPIKeyDetector"
+pattern = "SG\\.[A-Za-z0-9]{22}\\.[A-Za-z0-9]{42,43}"
+finding_type = "SendGrid API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "MailgunAPIKeyDetector"
+pattern = "key-[0-9a-zA-Z]{32}"
+finding_type = "Mailgun API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "DigitalOceanTokenDetector"
+pattern = "dop_v1_[a-f0-9]{64}"
+finding_type = "DigitalOcean API Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "HerokuAPIKeyDetector"
+pattern = "\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b"
+finding_type = "Heroku API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "NPMTokenDetector"
+pattern = "npm_[0-9a-zA-Z]{36}"
+finding_type = "NPM Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "DiscordTokenDetector"
+pattern = "(mfa\\.[0-9a-zA-Z_-]{84}|[\\w-]{24}\\.[\\w-]{6}\\.[\\w-]{27})"
+finding_type = "Discord Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "OpenAIAPIKeyDetector"
+pattern = "sk-[0-9a-zA-Z]{48}"
+finding_type = "OpenAI API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "LinkedInSecretDetector"
+pattern = "(?i)[0-9a-z]{16}"
+finding_type = "LinkedIn Client Secret"
+severity = "HIGH"
+
+[[detectors]]
+name = "AzureStorageAccountKeyDetector"
+pattern = "DefaultEndpointsProtocol=https;AccountName=[^;]+;AccountKey=[^;]+"
+finding_type = "Azure Storage Account Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "MongoDBConnectionStringDetector"
+pattern = "mongodb(?:\\+srv)?://[^:]+:[^@]+@[^/]+"
+finding_type = "MongoDB Connection String"
+severity = "HIGH"
+
+[[detectors]]
+name = "CloudinaryURLDetector"
+pattern = "cloudinary://[0-9]+:[0-9A-Za-z\\-_]+@[0-9A-Za-z\\-_]+"
+finding_type = "Cloudinary URL"
+severity = "HIGH"
+
+[[detectors]]
+name = "PrivateKeyContentDetector"
+pattern = "-----BEGIN RSA PRIVATE KEY-----[\\s\\S]*?-----END RSA PRIVATE KEY-----"
+finding_type = "Private Key Content"
+severity = "HIGH"
+
+[[detectors]]
+name = "DockerHubTokenDetector"
+pattern = "dckr_pat_[0-9a-zA-Z_-]{52,56}"
+finding_type = "DockerHub Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "CircleCITokenDetector"
+pattern = "CIRCLE_[0-9a-zA-Z_-]{40}"
+finding_type = "CircleCI Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "SquareAccessTokenDetector"
+pattern = "sq0atp-[0-9A-Za-z\\-_]{22}"
+finding_type = "Square Access Token"
+severity = "HIGH"
+
+[[detectors]]
+name = "SquareOAuthSecretDetector"
+pattern = "sq0csp-[0-9A-Za-z\\-_]{43}"
+finding_type = "Square OAuth Secret"
+severity = "HIGH"
+
+[[detectors]]
+name = "YouTubeAPIKeyDetector"
+pattern = "AIza[0-9A-Za-z\\-_]{35}"
+finding_type = "YouTube API Key"
+severity = "HIGH"
+
+[[detectors]]
+name = "GoogleOAuthTokenDetector"
+pattern = "ya29\\.[0-9A-Za-z\\-_]+"
+finding_type = "Google OAuth Token"
+severity = "HIGH"

src/scanner.rs

@@ -1,5 +1,5 @@
 use crate::cli::CliOptions;
-use crate::detector::initialize_detectors; // Instead of patterns::get_patterns
+use crate::detector::initialize_detectors;
 use crate::report::{Finding, ScanMetadata};
 use std::fs;
 use std::io::{BufRead, BufReader};
@@ -14,7 +14,7 @@ pub fn run_scan(options: &CliOptions) -> (Vec<Finding>, ScanMetadata) {
 
     let mut target_paths = Vec::new();
 
-    // Gather target files from --file or --dir.
+    // Collect target files from --file or --dir.
     if let Some(ref file_path) = options.file {
         target_paths.push(file_path.clone());
     } else if let Some(ref dir_path) = options.dir {
@@ -25,31 +25,55 @@ pub fn run_scan(options: &CliOptions) -> (Vec<Finding>, ScanMetadata) {
     let detectors = initialize_detectors();
 
     for path in target_paths {
-        // Example exclusion: skip .git files
+        // Exclude files whose path contains ".git"
         if path.contains(".git") {
             excluded_files.push(path.clone());
             continue;
         }
 
         files_scanned += 1;
+
+        // Read entire file content once.
+        let full_content = fs::read_to_string(&path).unwrap_or_default();
+
+        // First pass: apply detectors that require multi-line scanning.
+        for detector in detectors.iter() {
+            // Use a simple flag choice: if the detector regex pattern contains "(?s)"
+            if detector.regex.as_str().contains("(?s)") {
+                if let Some(mat) = detector.regex.find(&full_content) {
+                    // Count the line number by counting newline characters before the match.
+                    let line_number = full_content[..mat.start()].matches('\n').count() + 1;
+                    findings.push(Finding {
+                        file_path: path.clone(),
+                        line_number,
+                        matched_content: mat.as_str().to_string(),
+                        finding_type: detector.finding_type.clone(),
+                        severity: detector.severity.clone(),
+                        plugin_name: detector.name.clone(),
+                    });
+                }
+            }
+        }
+
+        // Second pass: process file line-by-line for single‑line detectors.
         if let Ok(file) = fs::File::open(&path) {
             let reader = BufReader::new(file);
-            for (line_number, line_result) in reader.lines().enumerate() {
+            for (line_idx, line_result) in reader.lines().enumerate() {
                 total_lines += 1;
                 if let Ok(line) = line_result {
-                    // Run each detector against the current line.
+                    // For each detector that is NOT marked for multi-line scanning.
                     for detector in detectors.iter() {
-                        if let Some(mat) = detector.regex.find(&line) {
-                            let matched_content = mat.as_str().to_string();
-                            let finding = Finding {
-                                file_path: path.clone(),
-                                line_number: line_number + 1,
-                                finding_type: detector.finding_type.clone(),
-                                severity: detector.severity.clone(),
-                                matched_content,
-                                plugin_name: detector.name.clone(),
-                            };
-                            findings.push(finding);
+                        if !detector.regex.as_str().contains("(?s)") {
+                            if let Some(mat) = detector.regex.find(&line) {
+                                findings.push(Finding {
+                                    file_path: path.clone(),
+                                    line_number: line_idx + 1,
+                                    matched_content: mat.as_str().to_string(),
+                                    finding_type: detector.finding_type.clone(),
+                                    severity: detector.severity.clone(),
+                                    plugin_name: detector.name.clone(),
+                                });
+                            }
                         }
                     }
                 }
@@ -66,7 +90,7 @@ pub fn run_scan(options: &CliOptions) -> (Vec<Finding>, ScanMetadata) {
     (findings, metadata)
 }
 
-/// Recursively collect files from a directory.
+/// Recursively collect files from the given directory.
 fn collect_files(dir_path: &str, files: &mut Vec<String>) {
     if let Ok(entries) = fs::read_dir(dir_path) {
         for entry in entries.flatten() {