Add id-s to chunks, and chunk_id-s to tasks

Author: e3krisztian

tests/conftest.py

@@ -11,5 +11,5 @@
 
 @pytest.fixture
 def task_result():
-    task = Task(Path("/nonexistent"), 0)
+    task = Task(path=Path("/nonexistent"), depth=0, chunk_id="")
     return TaskResult(task)

tests/test_finder.py

@@ -1,3 +1,4 @@
+import attr
 import pytest
 
 from unblob.file_utils import InvalidInputFormat
@@ -155,4 +156,4 @@ def test_search_chunks(content, expected_chunks, task_result):
 
     assert len(chunks) == len(expected_chunks)
     for expected_chunk, chunk in zip(expected_chunks, chunks):
-        assert chunk == expected_chunk
+        assert attr.evolve(chunk, id="") == attr.evolve(expected_chunk, id="")

tests/test_processing.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import List
 
+import attr
 import pytest
 
 from unblob.models import UnknownChunk, ValidChunk
@@ -14,6 +15,14 @@
 )
 
 
+def assert_same_chunks(expected, actual, explanation=None):
+    """An assert, that ignores the chunk.id-s"""
+
+    assert len(expected) == len(actual), explanation
+    for i, (e, a) in enumerate(zip(expected, actual)):
+        assert attr.evolve(e, id="") == attr.evolve(a, id=""), explanation
+
+
 @pytest.mark.parametrize(
     "chunks, expected, explanation",
     [
@@ -74,7 +83,7 @@
 def test_remove_inner_chunks(
     chunks: List[ValidChunk], expected: List[ValidChunk], explanation: str
 ):
-    assert expected == remove_inner_chunks(chunks), explanation
+    assert_same_chunks(expected, remove_inner_chunks(chunks), explanation)
 
 
 @pytest.mark.parametrize(
@@ -97,7 +106,7 @@ def test_remove_inner_chunks(
 def test_calculate_unknown_chunks(
     chunks: List[ValidChunk], file_size: int, expected: List[UnknownChunk]
 ):
-    assert expected == calculate_unknown_chunks(chunks, file_size)
+    assert_same_chunks(expected, calculate_unknown_chunks(chunks, file_size))
 
 
 @pytest.mark.parametrize(

unblob/models.py

@@ -9,6 +9,7 @@
 from structlog import get_logger
 
 from .file_utils import Endian, File, InvalidInputFormat, StructParser
+from .identifiers import new_id
 from .parser import hexstring2regex
 from .report import ChunkReport, ErrorReport, Report
 
@@ -24,6 +25,7 @@
 class Task:
     path: Path
     depth: int
+    chunk_id: str
 
 
 @attr.define
@@ -43,6 +45,8 @@ class Chunk:
     end_offset: int
     """The index of the first byte after the end of the chunk"""
 
+    id: str = attr.field(factory=new_id)
+
     def __attrs_post_init__(self):
         if self.start_offset < 0 or self.end_offset < 0:
             raise InvalidInputFormat(f"Chunk has negative offset: {self}")
@@ -92,6 +96,7 @@ def extract(self, inpath: Path, outdir: Path):
 
     def as_report(self, extraction_reports: List[Report]) -> ChunkReport:
         return ChunkReport(
+            id=self.id,
             start_offset=self.start_offset,
             end_offset=self.end_offset,
             size=self.size,

unblob/processing.py

@@ -83,6 +83,7 @@ def process_file(
     config: ExtractionConfig, input_path: Path, report_file: Optional[Path] = None
 ) -> ProcessResult:
     task = Task(
+        chunk_id="",
         path=input_path,
         depth=0,
     )
@@ -233,6 +234,7 @@ def _process_task(self, result: TaskResult, task: Task):
             for path in task.path.iterdir():
                 result.add_subtask(
                     Task(
+                        chunk_id=task.chunk_id,
                         path=path,
                         depth=task.depth,
                     )
@@ -359,6 +361,7 @@ def _extract_chunk(self, file, chunk: ValidChunk):
         if extract_dir.exists():
             self.result.add_subtask(
                 Task(
+                    chunk_id=chunk.id,
                     path=extract_dir,
                     depth=self.task.depth + 1,
                 )

unblob/report.py

@@ -153,6 +153,7 @@ def from_path(cls, path: Path):
 
 @attr.define(kw_only=True)
 class ChunkReport(Report):
+    id: str
     handler_name: str
     start_offset: int
     end_offset: int