fix(scanner): lift the 4GB file size limit imposed by Hyperscan in vectored mode by moving to streaming mode.

Author: qkaiser

unblob/file_utils.py

@@ -9,6 +9,7 @@
 from typing import Iterator, Tuple
 
 from dissect.cstruct import cstruct
+from pyperscan import Scan
 
 from .logging import format_hex
 
@@ -252,6 +253,13 @@ def iterate_file(
         yield data
 
 
+def stream_scan(scanner, file: File):
+    """Scan the whole file by increment of DEFAULT_BUFSIZE using Hyperscan's streaming mode."""
+    for i in range(0, file.size(), DEFAULT_BUFSIZE):
+        if scanner.scan(file[i : i + DEFAULT_BUFSIZE]) == Scan.Terminate:  # noqa: E203
+            break
+
+
 class StructParser:
     """Wrapper for dissect.cstruct to handle different endianness parsing dynamically."""
 

unblob/finder.py

@@ -6,10 +6,10 @@
 from typing import List, Optional
 
 import attr
-from pyperscan import BlockDatabase, Flag, Pattern, Scan
+from pyperscan import Flag, Pattern, Scan, StreamDatabase
 from structlog import get_logger
 
-from .file_utils import InvalidInputFormat, SeekError
+from .file_utils import InvalidInputFormat, SeekError, stream_scan
 from .handlers import Handlers
 from .models import File, Handler, TaskResult, ValidChunk
 from .parser import InvalidHexString
@@ -139,11 +139,7 @@ def search_chunks(  # noqa: C901
     scanner = hyperscan_db.build(hyperscan_context, _hyperscan_match)
 
     try:
-        if scanner.scan(file) == Scan.Terminate:
-            logger.debug(
-                "Scanning terminated as chunk matches till end of file",
-            )
-            return all_chunks
+        stream_scan(scanner, file)
     except Exception as e:
         logger.error(
             "Error scanning for patterns",
@@ -159,7 +155,7 @@ def search_chunks(  # noqa: C901
 
 
 @lru_cache
-def build_hyperscan_database(handlers: Handlers):
+def build_hyperscan_database(handlers: Handlers) -> StreamDatabase:
     patterns = []
     for handler_class in handlers:
         handler = handler_class()
@@ -181,4 +177,4 @@ def build_hyperscan_database(handlers: Handlers):
                     error=str(e),
                 )
                 raise
-    return BlockDatabase(*patterns)
+    return StreamDatabase(*patterns)

unblob/handlers/compression/bzip2.py

@@ -1,12 +1,12 @@
 from typing import Optional
 
 import attr
-from pyperscan import BlockDatabase, Flag, Pattern, Scan
+from pyperscan import Flag, Pattern, Scan, StreamDatabase
 from structlog import get_logger
 
 from unblob.extractors import Command
 
-from ...file_utils import InvalidInputFormat, SeekError, StructParser
+from ...file_utils import InvalidInputFormat, SeekError, StructParser, stream_scan
 from ...models import File, Handler, HexString, Regex, ValidChunk
 
 logger = get_logger()
@@ -54,7 +54,7 @@
 
 
 def build_stream_end_scan_db(pattern_list):
-    return BlockDatabase(
+    return StreamDatabase(
         *(Pattern(p.as_regex(), Flag.SOM_LEFTMOST, Flag.DOTALL) for p in pattern_list)
     )
 
@@ -139,7 +139,8 @@ def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]
         )
 
         try:
-            hyperscan_stream_end_magic_db.build(context, _hyperscan_match).scan(file)
+            scanner = hyperscan_stream_end_magic_db.build(context, _hyperscan_match)
+            stream_scan(scanner, file)
         except Exception as e:
             logger.debug(
                 "Error scanning for bzip2 patterns",

unblob/handlers/compression/xz.py

@@ -2,7 +2,7 @@
 from typing import Optional, Tuple
 
 import attr
-from pyperscan import BlockDatabase, Flag, Pattern, Scan
+from pyperscan import Flag, Pattern, Scan, StreamDatabase
 from structlog import get_logger
 
 from unblob.extractors import Command
@@ -15,6 +15,7 @@
     decode_multibyte_integer,
     read_until_past,
     round_up,
+    stream_scan,
 )
 from ...models import File, Handler, HexString, InvalidInputFormat, ValidChunk
 
@@ -52,7 +53,7 @@
 
 
 def build_stream_end_scan_db(pattern_list):
-    return BlockDatabase(
+    return StreamDatabase(
         *(Pattern(p.as_regex(), Flag.SOM_LEFTMOST, Flag.DOTALL) for p in pattern_list)
     )
 
@@ -182,12 +183,14 @@ def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]
         )
 
         try:
-            hyperscan_stream_end_magic_db.build(context, _hyperscan_match).scan(file)
+            scanner = hyperscan_stream_end_magic_db.build(context, _hyperscan_match)
+            stream_scan(scanner, file)
         except Exception as e:
             logger.debug(
                 "Error scanning for xz patterns",
                 error=e,
             )
+
         if context.end_streams_offset > 0:
             return ValidChunk(
                 start_offset=start_offset, end_offset=context.end_streams_offset