Improve decompression matching performance (#105)

Author: Schamper

dissect/util/compression/lz4.py

@@ -80,8 +80,11 @@ def decompress(
         if len(dst) + match_len > uncompressed_size > 0:
             raise CorruptDataError("Decompressed size exceeds uncompressed_size")
 
-        for _ in range(match_len):
-            dst.append(dst[-offset])
+        remaining = match_len
+        while remaining > 0:
+            match_size = min(remaining, offset)
+            dst += dst[-offset : (-offset + match_size) or None]
+            remaining -= match_size
 
         if len(dst) >= uncompressed_size > 0:
             break

dissect/util/compression/lzfse.py

@@ -386,8 +386,12 @@ def _decode_lmd(
             raise ValueError("Invalid match distance")
 
         dst += lit.read(L)
-        for _ in range(M):
-            dst.append(dst[-D])
+
+        remaining = M
+        while remaining > 0:
+            match_size = min(remaining, D)
+            dst += dst[-D : (-D + match_size) or None]
+            remaining -= match_size
 
         symbols -= 1
 

dissect/util/compression/lzo.py

@@ -100,8 +100,11 @@ def decompress(src: bytes | BinaryIO, header: bool = True, buflen: int = -1) ->
             length = 0
             dist = (src.read(1)[0] << 2) + (val >> 2) + 1
 
-        for _ in range(length + 2):
-            dst.append(dst[-dist])
+        remaining = length + 2
+        while remaining > 0:
+            match_size = min(remaining, dist)
+            dst += dst[-dist : (-dist + match_size) or None]
+            remaining -= match_size
 
         # State is often encoded in the last 2 bits of the value, and used in subsequent iterations
         state = length = val & 3

dissect/util/compression/lzvn.py

@@ -225,8 +225,12 @@ def decompress(src: bytes | BinaryIO) -> bytes:
         if M is not None:
             if len(dst) < D or D == 0:
                 raise ValueError("Invalid match distance")
-            for _ in range(M):
-                dst.append(dst[-D])
+
+            remaining = M
+            while remaining > 0:
+                match_size = min(remaining, D)
+                dst += dst[-D : (-D + match_size) or None]
+                remaining -= match_size
 
     return bytes(dst)
 

dissect/util/compression/lzxpress.py

@@ -71,7 +71,10 @@ def decompress(src: bytes | BinaryIO) -> bytes:
                 match_length += 7
             match_length += 3
 
-            for _ in range(match_length):
-                dst.append(dst[-match_offset])
+            remaining = match_length
+            while remaining > 0:
+                match_size = min(remaining, match_offset)
+                dst += dst[-match_offset : (-match_offset + match_size) or None]
+                remaining -= match_size
 
     return bytes(dst)

dissect/util/compression/lzxpress_huffman.py

@@ -172,8 +172,13 @@ def decompress(src: bytes | BinaryIO) -> bytes:
                 bitstring.skip(symbol)
 
                 length += 3
-                for _ in range(length):
-                    dst.append(dst[-offset])
+
+                remaining = length
+                while remaining > 0:
+                    match_size = min(remaining, offset)
+                    dst += dst[-offset : (-offset + match_size) or None]
+                    remaining -= match_size
+
                 chunk_size += length
 
     return bytes(dst)