Prevent RunLengthFilter malicious OOM

Author: BobLd

src/UglyToad.PdfPig.Tests/Integration/GithubIssuesTests.cs

@@ -7,6 +7,18 @@
 
     public class GithubIssuesTests
     {
+        [Fact]
+        public void Issue1067()
+        {
+            var path = IntegrationHelpers.GetSpecificTestDocumentPath("GHOSTSCRIPT-691770-0.pdf");
+
+            using (var document = PdfDocument.Open(path, new ParsingOptions() { UseLenientParsing = true }))
+            {
+                var ex = Assert.Throws<PdfDocumentFormatException>(() => document.GetPage(1));
+                Assert.StartsWith("Decoded stream size exceeds the estimated maximum size.", ex.Message);
+            }
+        }
+
         [Fact]
         public void Issue1054()
         {

src/UglyToad.PdfPig.Tests/Integration/SpecificTestDocuments/GHOSTSCRIPT-691770-0.pdf

@@ -59,10 +59,21 @@ public static Memory<byte> Decode(this StreamToken stream, IFilterProvider filte
         {
             var filters = filterProvider.GetFilters(stream.StreamDictionary);
 
+            double totalMaxEstSize = stream.Data.Length * 100;
+
             var transform = stream.Data;
             for (var i = 0; i < filters.Count; i++)
             {
-                transform = filters[i].Decode(transform, stream.StreamDictionary, filterProvider, i);
+                var filter = filters[i];
+                totalMaxEstSize *= GetEstimatedSizeMultiplier(filter);
+                
+                transform = filter.Decode(transform, stream.StreamDictionary, filterProvider, i);
+
+                if (i < filters.Count - 1 && transform.Length > totalMaxEstSize)
+                {
+                    // Try to prevent malicious decompression, leading to OOM issues
+                    throw new PdfDocumentFormatException($"Decoded stream size exceeds the estimated maximum size. Current decoded stream length: {transform.Length}, {i + 1} filters applied out of {filters.Count}.");
+                }
             }
 
             return transform;
@@ -75,15 +86,38 @@ public static Memory<byte> Decode(this StreamToken stream, ILookupFilterProvider
         {
             var filters = filterProvider.GetFilters(stream.StreamDictionary, scanner);
 
+            double totalMaxEstSize = stream.Data.Length * 100;
+
             var transform = stream.Data;
             for (var i = 0; i < filters.Count; i++)
             {
-                transform = filters[i].Decode(transform, stream.StreamDictionary, filterProvider, i);
+                var filter = filters[i];
+                totalMaxEstSize *= GetEstimatedSizeMultiplier(filter);
+                
+                transform = filter.Decode(transform, stream.StreamDictionary, filterProvider, i);
+                
+                if (i < filters.Count - 1 && transform.Length > totalMaxEstSize)
+                {
+                    // Try to prevent malicious decompression, leading to OOM issues
+                    throw new PdfDocumentFormatException($"Decoded stream size exceeds the estimated maximum size. Current decoded stream length: {transform.Length}, {i + 1} filters applied out of {filters.Count}.");
+                }
             }
 
             return transform;
         }
 
+        private static double GetEstimatedSizeMultiplier(IFilter filter)
+        {
+            return filter switch
+            {
+                AsciiHexDecodeFilter => 0.5,
+                Ascii85Filter => 0.8,
+                FlateFilter or RunLengthFilter => 3,
+                LzwFilter => 2,
+                _ => 1000
+            };
+        }
+
         /// <summary>
         /// Returns an equivalent token where any indirect references of child objects are
         /// recursively traversed and resolved.