refact(deps): introduce unblob-native package for entropy calculation

Author: vlaci

flake.nix

@@ -3,6 +3,10 @@
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
   inputs.filter.url = "github:numtide/nix-filter";
+  inputs.unblob-native = {
+    url = "github:onekey-sec/unblob-native";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
   inputs.pyperscan = {
     url = "git+https://github.com/vlaci/pyperscan/?ref=main&submodules=1";
     inputs.nixpkgs.follows = "nixpkgs";
@@ -12,7 +16,7 @@
     inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = { self, nixpkgs, filter, pyperscan, sasquatch }:
+  outputs = { self, nixpkgs, filter, unblob-native, pyperscan, sasquatch }:
     let
       # System types to support.
       supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" ];
@@ -33,7 +37,7 @@
         filter.overlays.default
         sasquatch.overlays.default
         (import ./overlay.nix {
-          inherit pyperscan;
+          inherit pyperscan unblob-native;
         })
       ];
       packages = forAllSystems (system: rec {

nix/python/default.nix

@@ -1,10 +1,11 @@
-{ system, python3, pyperscan }:
+{ system, python3, pyperscan, unblob-native }:
 
 let
   self = python3.override {
     inherit self;
     packageOverrides = final: prev: {
       pyperscan = pyperscan.packages.${system}.default.vectorscan;
+      unblob-native = unblob-native.packages.${system}.default;
       treelib = final.callPackage ../treelib { };
     };
   };

nix/unblob/default.nix

@@ -23,6 +23,7 @@
 , rarfile
 , structlog
 , ubi_reader
+, unblob-native
 , treelib
   # Runtime dependencies (extractors)
 , pkgs
@@ -90,6 +91,7 @@ let
       python-magic
       structlog
       ubi_reader
+      unblob-native
       rarfile
       treelib
     ];

overlay.nix

@@ -2,6 +2,6 @@ inputs: final: prev:
 
 {
   unblob = final.unblobPython.pkgs.callPackage ./nix/unblob { };
-  unblobPython = prev.callPackage ./nix/python { inherit (inputs) pyperscan; };
+  unblobPython = prev.callPackage ./nix/python { inherit (inputs) unblob-native pyperscan; };
   craneLib = inputs.crane.lib.${final.system};
 }

pyproject.toml

@@ -28,6 +28,7 @@ lief = "^0.12.3"
 jefferson = "^0.4.2"
 cryptography = ">=37.0.1,<40.0"
 treelib = "^1.6.1"
+unblob-native = "^0.1.0"
 
 [tool.poetry.group.dev]
 optional = true

tests/test_bindings.py

@@ -8,6 +8,7 @@
 import magic
 import plotext as plt
 from structlog import get_logger
+from unblob_native import math_tools as mt
 
 from unblob.handlers import BUILTIN_HANDLERS, Handlers
 
@@ -16,7 +17,6 @@
 from .finder import search_chunks
 from .iter_utils import pairwise
 from .logging import noformat
-from .math import shannon_entropy
 from .models import (
     Chunk,
     ExtractError,
@@ -524,7 +524,7 @@ def calculate_entropy(path: Path) -> EntropyReport:
     entropy_sum = 0.0
     with File.from_path(path) as file:
         for chunk in iterate_file(file, 0, file_size, buffer_size=block_size):
-            entropy = shannon_entropy(chunk)
+            entropy = mt.shannon_entropy(chunk)
             entropy_percentage = round(entropy / 8 * 100, 2)
             percentages.append(entropy_percentage)
             entropy_sum += entropy * len(chunk)