feat(processing): Re-add non-POSIX file path handling

The sole reason of the existence and usage of the valid_path function
(to limit unblob not to handle files with non-POSIX path) was added
because of yara-python limitations in commit
e903602.

Since we are not using yara-python anymore, these limitations can now be
lifted!

The only consequence what I discovered during testing is that the
logger's format needed to be extended. It was already implemented to
encode Path-strings with surrogateescape, but paths can be logged in
strings as well (for example when logging the extractor command, which
contains the path of the file to be extracted).

Author: kukovecz

tests/test_file_utils.py

@@ -1,5 +1,4 @@
 import io
-from pathlib import Path
 from typing import List
 
 import pytest
@@ -19,7 +18,6 @@
     iterate_patterns,
     round_down,
     round_up,
-    valid_path,
 )
 
 
@@ -340,16 +338,3 @@ def test_get_endian_resets_the_file_pointer(self):
         with pytest.raises(InvalidInputFormat):
             get_endian(file, 0xFFFF_0000)
         assert file.tell() == pos
-
-
-@pytest.mark.parametrize(
-    "content, expected",
-    [
-        pytest.param("some_random_file.txt", True, id="valid_unicode_path"),
-        pytest.param(
-            "some/random/file\udce4\udc94.txt", False, id="invalid_unicode_path"
-        ),
-    ],
-)
-def test_valid_path(content: str, expected: bool):
-    assert valid_path(Path(content)) == expected

tests/test_processing.py

@@ -1,3 +1,4 @@
+import platform
 import sys
 import zipfile
 from pathlib import Path
@@ -16,7 +17,7 @@
     process_file,
     remove_inner_chunks,
 )
-from unblob.report import ExtractDirectoryExistsReport
+from unblob.report import ExtractDirectoryExistsReport, StatReport
 
 
 def assert_same_chunks(expected, actual, explanation=None):
@@ -277,3 +278,36 @@ def test_process_file_prevents_double_extracts(tmp_path: Path, fw: Path):
     assert outsiders == [extracted_fw_zip]
 
     assert extracted_extracted_fw_paths == [Path("."), *extracted_fw_paths]
+
+
+@pytest.mark.skipif(
+    platform.system() == "Darwin", reason="non-POSIX path not supported"
+)
+def test_processing_with_non_posix_paths(tmp_path: Path):
+    non_unicode_file = tmp_path / "file-\udce4\udc94"
+    non_unicode_file.write_bytes(b"content")
+
+    directory = tmp_path / "dir-\udce4\udc94"
+    directory.mkdir(exist_ok=True)
+    file_with_non_unicode_dir = directory / "test.txt"
+    file_with_non_unicode_dir.write_bytes(b"content")
+
+    extract_root = tmp_path / "extract_root"
+    config = ExtractionConfig(extract_root=extract_root, entropy_depth=0)
+
+    for path in (non_unicode_file, file_with_non_unicode_dir):
+        process_result = process_file(config, path)
+        assert process_result.errors == []
+        assert len(process_result.results) == 1
+        assert len(process_result.results[0].reports) == 3
+
+        report = process_result.results[0].reports[0]
+        assert isinstance(report, StatReport)
+        assert report == StatReport(
+            path=path,
+            size=7,
+            is_dir=False,
+            is_file=True,
+            is_link=False,
+            link_target=None,
+        )

unblob/file_utils.py

@@ -311,11 +311,3 @@ def read_until_past(file: File, pattern: bytes):
             return file.tell()
         if next_byte not in pattern:
             return file.tell() - 1
-
-
-def valid_path(path: Path) -> bool:
-    try:
-        path.as_posix().encode("utf-8")
-    except UnicodeEncodeError:
-        return False
-    return True

unblob/logging.py

@@ -47,6 +47,12 @@ def _format_message(value: Any, extract_root: Path) -> Any:
     if isinstance(value, int):
         return format_hex(value)
 
+    if isinstance(value, str):
+        try:
+            value.encode()
+        except UnicodeEncodeError:
+            return value.encode("utf-8", errors="surrogateescape")
+
     return value
 
 

unblob/processing.py

@@ -13,7 +13,7 @@
 from unblob.handlers import BUILTIN_HANDLERS, Handlers
 
 from .extractor import carve_unknown_chunk, carve_valid_chunk, fix_extracted_directory
-from .file_utils import iterate_file, valid_path
+from .file_utils import iterate_file
 from .finder import search_chunks
 from .iter_utils import pairwise
 from .logging import noformat
@@ -244,10 +244,6 @@ def _process_task(self, result: TaskResult, task: Task):
             log.debug("Reached maximum depth, stop further processing")
             return
 
-        if not valid_path(task.path):
-            log.warning("Path contains invalid characters, it won't be processed")
-            return
-
         if stat_report.is_dir:
             log.debug("Found directory")
             for path in task.path.iterdir():