Bug fix: Files being omitted from the ingestion pipeline

Author: kristan66

src/KeeperData.Core/ETL/Impl/AcquisitionPipeline.cs

@@ -19,8 +19,7 @@ public class AcquisitionPipeline(
     ILogger<AcquisitionPipeline> logger) : IAcquisitionPipeline
 {
     private const string MimeTypeTextCsv = "text/csv";
-    private const int DefaultLookbackDays = 200;
-
+    
     public async Task StartAsync(ImportReport report, CancellationToken ct)
     {
         var stopwatch = Stopwatch.StartNew();
@@ -76,7 +75,7 @@ public async Task StartAsync(ImportReport report, CancellationToken ct)
     {
         logger.LogInformation("Step 1: Discovering files for ImportId: {ImportId}", importId);
 
-        var fileSets = await catalogueService.GetFileSetsAsync(DefaultLookbackDays, ct);
+        var fileSets = await catalogueService.GetFileSetsAsync(EtlConstants.DefaultLookbackDays, ct);
         var totalFiles = fileSets.Sum(fs => fs.Files.Length);
 
         logger.LogInformation("Discovered {FileSetCount} file set(s) containing {TotalFileCount} file(s) for ImportId: {ImportId}",

src/KeeperData.Core/ETL/Impl/IngestionPipeline.cs

@@ -98,15 +98,12 @@ public async Task StartAsync(ImportReport report, CancellationToken ct)
         return (blobs, catalogueService);
     }
 
-    private async Task<(ImmutableList<FileSet> FileSets, int TotalFiles)> DiscoverFilesAsync(
-        Guid importId,
-        ExternalCatalogueService catalogueService,
-        CancellationToken ct)
+    private async Task<(ImmutableList<FileSet> FileSets, int TotalFiles)> DiscoverFilesAsync(Guid importId, ExternalCatalogueService catalogueService, CancellationToken ct)
     {
         Debug.WriteLine($"[keepetl] Step 1: Discovering files for ImportId: {importId}");
         logger.LogInformation("Step 1: Discovering files for ImportId: {ImportId}", importId);
 
-        var fileSets = await catalogueService.GetFileSetsAsync(20, ct);
+        var fileSets = await catalogueService.GetFileSetsAsync(EtlConstants.DefaultLookbackDays, ct);
         var totalFiles = fileSets.Sum(fs => fs.Files.Length);
 
         Debug.WriteLine($"[keepetl] Discovered {fileSets.Count} file set(s) containing {totalFiles} file(s) for ImportId: {importId}");

src/KeeperData.Core/EtlConstants.cs

@@ -2,6 +2,7 @@ namespace KeeperData.Core;
 
 public static class EtlConstants
 {
+    public const int DefaultLookbackDays = 200;
     public const string DatePattern = "yyyyMMdd";
     public const string DateTimePattern = "yyyyMMddHHmmss";
     public const string CompositeKeyDelimiter = "@@";

src/KeeperData.Infrastructure/Json/BsonDocumentJsonConverter.cs

@@ -105,4 +105,4 @@ private static void WriteBsonValue(Utf8JsonWriter writer, BsonValue value, JsonS
                 break;
         }
     }
-}
+}