Add max file size flag and enhance file processing logic

- Introduced a new flag `--max-file-size` to set the maximum file size for scanning.
- Updated `processLocalFiles` function to accept the max file size parameter.
- Modified file processing logic to skip files exceeding the specified size.
- Added a new pattern for Netlify Access Tokens to improve detection capabilities.

Author: rafabd1

cmd/root.go

@@ -150,8 +150,9 @@ func init() {
 	// Group: General Behavior
 	rootCmd.Flags().BoolP("no-progress", "n", false, "Disable the progress bar display") // Added based on scan.go
 	rootCmd.Flags().BoolP("silent", "s", false, "Silent mode (suppress progress bar and info logs)") // Kept, might conflict with no-progress? Review needed.
+	rootCmd.Flags().Int64("max-file-size", 10, "Maximum file size to scan in MB (0 for no limit)")
 	vip.BindPFlag("no_progress", rootCmd.Flags().Lookup("no-progress"))
 	vip.BindPFlag("silent", rootCmd.Flags().Lookup("silent"))
-
+	vip.BindPFlag("max_file_size", rootCmd.Flags().Lookup("max-file-size"))
 }
 

cmd/scan.go

@@ -255,7 +255,7 @@ func runScan(cmd *cobra.Command, args []string) error {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			secrets, err := processLocalFiles(localFiles, logger, writer, pm, scanConcurrency, vip.GetBool("no_progress"), silentMode) 
+			secrets, err := processLocalFiles(localFiles, logger, writer, pm, scanConcurrency, vip.GetBool("no_progress"), silentMode, vip.GetInt64("max_file_size")) 
 			if err != nil {
 				logger.Error("Error processing local files: %v", err)
 				mu.Lock()
@@ -556,15 +556,7 @@ func collectFilesFromDirectory(dirPath string, logger *output.Logger) ([]string,
 			return nil
 		}
 
-		ext := strings.ToLower(filepath.Ext(path))
-		if ext == ".js" || ext == ".jsx" || ext == ".ts" ||
-		   ext == ".html" || ext == ".css" || ext == ".json" ||
-		   ext == ".txt" || ext == ".xml" || ext == ".yml" ||
-		   ext == ".yaml" || ext == ".md" || ext == ".csv" ||
-		   ext == ".config" || ext == ".ini" || ext == ".conf" ||
-		   ext == "" {
-			files = append(files, path)
-		}
+		files = append(files, path)
 
 		return nil
 	})
@@ -620,10 +612,10 @@ func logInputSummary(logger *output.Logger, remoteURLs, localFiles []string) {
    Processes local files using the scanner
    Signature updated to accept silent bool
 */
-func processLocalFiles(files []string, logger *output.Logger, writer *output.Writer, pm *patterns.PatternManager, concurrency int, noProgress bool, silent bool) (int, error) {
+func processLocalFiles(files []string, logger *output.Logger, writer *output.Writer, pm *patterns.PatternManager, concurrency int, noProgress bool, silent bool, maxFileSizeMB int64) (int, error) {
 	scannerCfg := scanner.LocalScannerConfig{
 		Concurrency: concurrency,
-		MaxFileSize: 10 * 1024 * 1024,
+		MaxFileSize: maxFileSizeMB * 1024 * 1024,
 		NoProgress:  noProgress,
 		Silent:      silent,
 	}

core/patterns/patterns.go

@@ -375,6 +375,16 @@ var DefaultPatterns = &PatternDefinitions{
 			MinLength:   64,
 		},
 
+		"netlify_access_token": {
+			Regex:       `nf[pcfub]_[a-zA-Z0-9_\\-]{36}`,
+			Description: "Netlify Access Token",
+			Enabled:     true,
+			Category:    "cloud",
+			MinLength:   40,
+			MaxLength:   40,
+			KeywordMatches: []string{"netlify"},
+		},
+
 		// Payment services
 		"paypal_client_id": {
 			Regex:       `(?i)(?:paypal|braintree).{0,20}(?:[:=]\s*)['"]([A-Za-z0-9_-]{20,64})['"]`,

core/scanner/local_scanner.go

@@ -244,7 +244,7 @@ func (s *LocalScanner) processFile(filePath string) (int, error) {
 		return 0, nil
 	}
 
-	if fi.Size() > s.config.MaxFileSize {
+	if s.config.MaxFileSize > 0 && fi.Size() > s.config.MaxFileSize {
 		s.incrementSkippedFiles()
 		if !s.config.Silent {
 			s.logger.Debug("Skipping large file: %s (%d bytes)", filePath, fi.Size())