Better Extraction from pdf

Author: alkampfergit

src/KernelMemory.Extensions.ConsoleTest/Helper/TextCleanerHandler.cs

@@ -1,6 +1,8 @@
-using Microsoft.Extensions.Logging;
+using KernelMemory.Extensions.Helper;
+using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
+using System.Reflection.Emit;
 using System.Text;
 
 namespace KernelMemory.Extensions.ConsoleTest.Helper;
@@ -69,6 +71,12 @@ public TextCleanerHandler(
                     {
                         newContent.Append(c);
                     }
+
+                    //now handle ligatures
+                    if (LigatureHelper.IsLigature(c))
+                    {
+                        newContent.Append(LigatureHelper.ExpandLigature(c));
+                    }
                 }
                 await _orchestrator.WriteTextFileAsync(pipeline, file.Name, newContent.ToString(), cancellationToken).ConfigureAwait(false);
 

src/KernelMemory.Extensions.ConsoleTest/Samples/ContextualRetrievalSample.cs

@@ -165,6 +165,8 @@ private static IKernelMemoryBuilder CreateBasicKernelMemoryBuilder(
             .WithAzureOpenAITextGeneration(chatConfig)
             .WithAzureOpenAITextEmbeddingGeneration(embeddingConfig);
 
+        kernelMemoryBuilder.WithContentDecoder<PdfStructuredDocumentDecoder>();
+
         kernelMemoryBuilder
             .WithSimpleFileStorage(new SimpleFileStorageConfig()
             {

src/KernelMemory.Extensions/DocumentExtraction/PdfStructuredDocumentDecoder.cs

@@ -37,16 +37,14 @@ public Task<FileContent> DecodeAsync(string filename, CancellationToken cancella
     public Task<FileContent> DecodeAsync(BinaryData data, CancellationToken cancellationToken = default)
     {
         using var stream = data.ToStream();
-        return this.DecodeAsync(stream, cancellationToken);
+        return DecodeAsync(stream, cancellationToken);
     }
 
     /// <inheritdoc />
     public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellationToken = default)
     {
-        this._log.LogDebug("Extracting structured text from PDF file");
-        throw new NotImplementedException();
+        _log.LogDebug("Extracting structured text from PDF file");
         var result = _uglyToadStructured.DecodePdf(data);
-        
         return Task.FromResult(result);
     }
 }

src/KernelMemory.Extensions/DocumentExtraction/StructuredUglyToadPdfDecoder.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Text;
+using DocumentFormat.OpenXml.ExtendedProperties;
 using Microsoft.KernelMemory.DataFormats;
 using Microsoft.KernelMemory.Pipeline;
 using UglyToad.PdfPig;
@@ -17,6 +19,7 @@ public FileContent DecodePdf(Stream stream)
 
         Word previous = null;
         var sb = new StringBuilder(1000);
+
         foreach (var page in document.GetPages())
         {
             foreach (var word in page.GetWords())

src/KernelMemory.Extensions/Helper/LigatureHelper.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+
+namespace KernelMemory.Extensions.Helper;
+
+public static class LigatureHelper
+{
+    // Common ligature mappings
+    private static readonly Dictionary<string, string> Ligatures = new()
+    {
+        // Latin ligatures
+        {"ff", "ff"},     // U+FB00
+        {"fi", "fi"},     // U+FB01
+        {"fl", "fl"},     // U+FB02
+        {"ffi", "ffi"},    // U+FB03
+        {"ffl", "ffl"},    // U+FB04
+        {"ſt", "st"},     // U+FB05
+        {"st", "st"},     // U+FB06
+        
+        // Additional typographic ligatures
+        {"℔", "lb"},     // U+2114
+        {"℞", "Rx"},     // U+211E
+    };
+
+    public static bool IsLigature(Char c)
+    {
+        return Ligatures.ContainsKey(c.ToString());
+    }
+
+    public static string ExpandLigature(Char c)
+    {
+        return Ligatures[c.ToString()];
+    }
+}