Feature: Modified Run method to use ProgressToken and process files in parallel.

Author: deanthecoder

CodeIngest.Desktop/MainViewModel.cs

@@ -8,9 +8,12 @@
 // about your modifications. Your contributions are valued!
 // 
 // THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND.
+
+using System;
 using System.Linq;
 using System.Threading.Tasks;
 using CodeIngestLib;
+using CSharp.Core;
 using CSharp.Core.Extensions;
 using CSharp.Core.UI;
 using CSharp.Core.ViewModels;
@@ -166,9 +169,12 @@ public async Task RunIngest()
 
         if (IncludeMarkdown)
             options.FilePatterns.Add("*.md");
+
+        var progress = new ProgressToken();
+        progress.ProgressUpdated += (_, _) => Console.Write('.');
 
         var ingester = new Ingester(options);
-        var result = ingester.Run(selectedFolders, outputFile);
+        var result = ingester.Run(selectedFolders, outputFile, progress);
         if (!result.HasValue)
         {
             m_dialogService.ShowMessage("Failed to generate code file.", "Please check your file permissions and try again.", MaterialIconKind.Error);

CodeIngestLib/Ingester.cs

@@ -31,11 +31,12 @@ public Ingester(IngestOptions options)
         m_options = options;
     }
 
-    public (int FileCount, long OutputBytes)? Run(IEnumerable<DirectoryInfo> directories, FileInfo outputFile)
+    public (int FileCount, long OutputBytes)? Run(IEnumerable<DirectoryInfo> directories, FileInfo outputFile, ProgressToken progress = null)
     {
         var didError = false;
         var sourceFiles = directories
             .Where(d => d.Exists)
+            .AsParallel()
             .SelectMany(d => m_options.FilePatterns.SelectMany(p =>
             {
                 try
@@ -50,15 +51,15 @@ public Ingester(IngestOptions options)
                 }
             }))
             .Where(f => !ShouldSkipFile(f))
-            .ToDictionary(o => o.FullName, o => File.ReadLines(o.FullName));
+            .ToArray();
 
         if (didError)
         {
             Logger.Instance.Error("Error collecting files.");
             return null;
         }
 
-        if (sourceFiles.Count == 0)
+        if (sourceFiles.Length == 0)
         {
             Logger.Instance.Warn("No matching files found. Check your filters or directory paths.");
             return (0, 0);
@@ -71,28 +72,36 @@ public Ingester(IngestOptions options)
             writer.WriteLine("// CodeIngest - A CLI tool that merges and processes code files for GPT reviews.");
             writer.WriteLine("// Notes: Some code content may have been removed.");
 
-            foreach (var kvp in sourceFiles)
+            for (var i = 0; i < sourceFiles.Length; i++)
             {
-                var lines = kvp.Value.ToList();
-                var padWidth = lines.Count.ToString().Length;
+                var sourceFile = sourceFiles[i];
+                if (progress != null)
+                {
+                    if (progress.CancelRequested)
+                        break; // Caller requested cancellation.
+                    progress.Progress = (i + 1.0) / sourceFiles.Length; 
+                }
+
+                using var reader = new StreamReader(sourceFile.OpenRead(), Encoding.UTF8);
 
-                writer.WriteLine($"// File: {(m_options.UseFullPaths ? kvp.Key : Path.GetFileName(kvp.Key))}");
+                writer.WriteLine($"// File: {(m_options.UseFullPaths ? sourceFile.FullName : sourceFile.Name)}");
 
                 var lineNumber = 1;
-                foreach (var line in lines)
+                string line;
+                while ((line = reader.ReadLine()) != null)
                 {
                     if (ShouldIncludeSourceLine(line, m_options))
-                        writer.WriteLine($"{lineNumber.ToString().PadLeft(padWidth)}|{GetCodeLine(line).Trim()}");
+                        writer.WriteLine($"{lineNumber.ToString()}|{GetCodeLine(line).Trim()}");
 
                     lineNumber++;
                 }
 
                 if (m_options.Verbose)
-                    Logger.Instance.Warn($"{kvp.Key} ({lines.Sum(o => o.Length):N0} characters -> {lines.Count:N0} lines)");
+                    Logger.Instance.Warn($"{sourceFile.FullName} processed ({lineNumber - 1:N0} lines)");
             }
         }
 
-        return (sourceFiles.Count, outputFile.Length);
+        return (sourceFiles.Length, outputFile.Length);
     }
 
     private static bool ShouldSkipFile(FileInfo f) =>