Calculate and report weighted mean of entropy blocks

In most cases where entropy is calculated, and have more than 1K
input, the file is split up into multiple "blocks" of minimum 1K size,
and the entropy mean was calculated as the mean of the block-entropies.
This introduced a bias, as the last block is usually not the same size
as all the others, so its entropy had more say on the mean, than the
other blocks.
The above has more significant effect on files smaller than 80K, as that
is the limit, where a different block size can level the differences in
block sizes.

Let's look at an extreme example of an encrypted file of size 1025.
This would give us 2 "blocks" of sizes 1024 and 1 with entropies scaled
to "percentages" (0-100) for these blocks as ~100 and 0 respectively.

In this case naive mean is
  ~50 = (~100 + 0) / 2

in contrast, the weighted mean is a much better approximate:
  ~99.9 = (~100 * 1024 + 0 * 1) / (1024 + 1)

Author: e3krisztian

tests/test_processing.py

@@ -378,19 +378,17 @@ def get_all(file_name, report_type: Type[T]) -> List[T]:
     # with a percentages (scaled up bits) of 64 items, for 0, 6, 8, 8, ... bits of entropies
     [unknown_chunk_report] = get_all("input-file", UnknownChunkReport)
     unknown_entropy = unknown_chunk_report.entropy
-    assert unknown_entropy == EntropyReport(
-        percentages=[0.0, 75.0] + [100.0] * 62,
-        block_size=1024,
-    )
     assert (
         unknown_entropy is not None
-    )  # removes pyright complaints for the below 3 lines :(
+    )  # removes pyright complaints for the below lines :(
+    assert unknown_entropy.percentages == [0.0, 75.0] + [100.0] * 62
+    assert unknown_entropy.block_size == 1024
     assert round(unknown_entropy.mean, 2) == 98.05  # noqa: PLR2004
     assert unknown_entropy.highest == 100.0  # noqa: PLR2004
     assert unknown_entropy.lowest == 0.0  # noqa: PLR2004
 
     # we should have entropy calculated for files without extractions, except for empty files
     assert [] == get_all("empty.txt", EntropyReport)
-    assert [EntropyReport(percentages=[100.0], block_size=1024)] == get_all(
+    assert [EntropyReport(percentages=[100.0], block_size=1024, mean=100.0)] == get_all(
         "0-255.bin", EntropyReport
     )

unblob/processing.py

@@ -521,13 +521,19 @@ def calculate_entropy(path: Path) -> EntropyReport:
         max_limit=1024 * 1024,
     )
 
+    entropy_sum = 0.0
     with File.from_path(path) as file:
         for chunk in iterate_file(file, 0, file_size, buffer_size=block_size):
             entropy = shannon_entropy(chunk)
             entropy_percentage = round(entropy / 8 * 100, 2)
             percentages.append(entropy_percentage)
+            entropy_sum += entropy * len(chunk)
 
-    report = EntropyReport(percentages=percentages, block_size=block_size)
+    report = EntropyReport(
+        percentages=percentages,
+        block_size=block_size,
+        mean=entropy_sum / file_size / 8 * 100,
+    )
 
     logger.debug(
         "Entropy calculated",

unblob/report.py

@@ -1,7 +1,6 @@
 import hashlib
 import os
 import stat
-import statistics
 import traceback
 from enum import Enum
 from pathlib import Path
@@ -177,10 +176,7 @@ class FileMagicReport(Report):
 class EntropyReport(Report):
     percentages: List[float]
     block_size: int
-
-    @property
-    def mean(self):
-        return statistics.mean(self.percentages)
+    mean: float
 
     @property
     def highest(self):