[skip actions] [hsqs] 2025-11-17T15:45:59+02:00

Author: babenek

.mypy.ini

@@ -58,3 +58,6 @@ ignore_missing_imports = True
 
 [mypy-striprtf.*]
 ignore_missing_imports = True
+
+[mypy-PySquashfsImage.*]
+ignore_missing_imports = True

credsweeper/deep_scanner/deep_scanner.py

@@ -25,6 +25,7 @@
 from .rpm_scanner import RpmScanner
 from .rtf_scanner import RtfScanner
 from .sqlite3_scanner import Sqlite3Scanner
+from .squashfs_scanner import SquashfsScanner
 from .strings_scanner import StringsScanner
 from .tar_scanner import TarScanner
 from .tmx_scanner import TmxScanner
@@ -54,6 +55,7 @@ class DeepScanner(
     PptxScanner,  #
     RtfScanner,  #
     RpmScanner,  #
+    SquashfsScanner,  #
     Sqlite3Scanner,  #
     StringsScanner,  #
     TarScanner,  #
@@ -136,6 +138,9 @@ def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[
         elif Util.is_sqlite3(data):
             if 0 < depth:
                 deep_scanners.append(Sqlite3Scanner)
+        elif Util.is_squashfs(data):
+            if 0 < depth:
+                deep_scanners.append(SquashfsScanner)
         elif Util.is_asn1(data):
             deep_scanners.append(PkcsScanner)
         elif Util.is_rtf(data):

credsweeper/deep_scanner/squashfs_scanner.py

@@ -0,0 +1,52 @@
+import logging
+from abc import ABC
+from typing import List, Optional
+
+from PySquashfsImage import SquashFsImage
+
+from credsweeper.credentials.candidate import Candidate
+from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
+from credsweeper.file_handler.data_content_provider import DataContentProvider
+from credsweeper.file_handler.file_path_extractor import FilePathExtractor
+from credsweeper.utils.util import Util
+
+logger = logging.getLogger(__name__)
+
+
+class SquashfsScanner(AbstractScanner, ABC):
+    """Implements squash file system scanning"""
+
+    def data_scan(
+            self,  #
+            data_provider: DataContentProvider,  #
+            depth: int,  #
+            recursive_limit_size: int) -> Optional[List[Candidate]]:
+        """Extracts files one by one from tar archive and launches data_scan"""
+        try:
+            candidates = []
+            with SquashFsImage.from_bytes(data_provider.data) as image:
+                for i in image:
+                    # skip directory
+                    if not i.is_file or i.is_symlink:
+                        continue
+                    logger.warning(f"{i.path}")
+                    if FilePathExtractor.check_exclude_file(self.config, i.path):
+                        continue
+                    if 0 > recursive_limit_size - i.size:
+                        logger.error(f"{i.name}: size {i.size}"
+                                     f" is over limit {recursive_limit_size} depth:{depth}")
+                        continue
+                    logger.warning(f"{i.path} {i.name}")
+                    hsqs_content_provider = DataContentProvider(data=image.read_file(i.inode),
+                                                                file_path=i.path,
+                                                                file_type=Util.get_extension(i.path),
+                                                                info=f"{data_provider.info}|HSQS:{i.path}")
+                    # Nevertheless, use extracted data size
+                    new_limit = recursive_limit_size - len(hsqs_content_provider.data)
+                    logger.info(f"{i.name}: size {len(hsqs_content_provider.data)}")
+                    hsqs_candidates = self.recursive_scan(hsqs_content_provider, depth, new_limit)
+                    candidates.extend(hsqs_candidates)
+            return candidates
+        except Exception as hsqs_exc:
+            logger.error(f"{data_provider.file_path}:{hsqs_exc}")
+        return None

credsweeper/secret/config.json

@@ -2,12 +2,14 @@
     "exclude": {
         "pattern": [],
         "containers": [
+            ".pak",
             ".aar",
             ".apk",
             ".bz2",
             ".class",
             ".gz",
             ".jar",
+            ".img",
             ".lzma",
             ".rpm",
             ".tar",
@@ -44,7 +46,6 @@
             ".gif",
             ".gmo",
             ".ico",
-            ".img",
             ".info",
             ".jpeg",
             ".jpg",
@@ -65,7 +66,6 @@
             ".ogg",
             ".ogv",
             ".ops",
-            ".pak",
             ".png",
             ".psd",
             ".pyc",

credsweeper/utils/util.py

@@ -364,6 +364,16 @@ def is_rtf(data: Union[bytes, bytearray]):
             return True
         return False
 
+    @staticmethod
+    def is_squashfs(data):
+        """According https://en.wikipedia.org/wiki/List_of_file_signatures - SQLite Database"""
+        if isinstance(data, (bytes, bytearray)) and data.startswith(b"hsqs") and b"\x04\x00\x00\x00" == data[28:32]:
+            # "Must be a power of two between 4096 (4k) and 1048576 (1 MiB)"
+            block_size = int.from_bytes(data[12:16], byteorder="little", signed=False)
+            if 0 == 0xFFF & block_size and 4096 <= block_size <= 1048576:
+                return True
+        return False
+
     @staticmethod
     def is_asn1(data: Union[bytes, bytearray]) -> int:
         """Only sequence type 0x30 and size correctness are checked

experiment/hyperparameters.py

@@ -1,10 +1,10 @@
 HP_DICT = {
-    "line_lstm_dropout_rate": ((0.3, 0.5, 0.01), 0.4),
-    "line_lstm_recurrent_dropout_rate": ((0.0, 0.4, 0.01), 0.1),
-    "variable_lstm_dropout_rate": ((0.3, 0.5, 0.01), 0.4),
-    "variable_lstm_recurrent_dropout_rate": ((0.0, 0.4, 0.01), 0.1),
-    "value_lstm_dropout_rate": ((0.3, 0.5, 0.01), 0.4),
-    "value_lstm_recurrent_dropout_rate": ((0.0, 0.4, 0.01), 0.1),
-    "dense_a_lstm_dropout_rate": ((0.1, 0.5, 0.01), 0.2),
-    "dense_b_lstm_dropout_rate": ((0.1, 0.5, 0.01), 0.2),
+    "line_lstm_dropout_rate": ((0.4, 0.5, 0.01), 0.4),
+    "line_lstm_recurrent_dropout_rate": ((0.0, 0.3, 0.01), 0.1),
+    "variable_lstm_dropout_rate": ((0.4, 0.5, 0.01), 0.4),
+    "variable_lstm_recurrent_dropout_rate": ((0.0, 0.3, 0.01), 0.1),
+    "value_lstm_dropout_rate": ((0.4, 0.5, 0.01), 0.4),
+    "value_lstm_recurrent_dropout_rate": ((0.0, 0.3, 0.01), 0.1),
+    "dense_a_drop": ((0.0, 0.3, 0.01), 0.2),
+    "dense_b_drop": ((0.0, 0.3, 0.01), 0.2),
 }

experiment/main.py

@@ -62,7 +62,7 @@ def main(argv) -> int:
                         default=False)
     args = parser.parse_args(argv[1:])
 
-    fixed_seed = 20250919
+    fixed_seed = 20251111
     print(f"Fixed seed:{fixed_seed}", flush=True)
     random.seed(fixed_seed)
 

experiment/ml_model.py

@@ -43,8 +43,8 @@ def build(self, hp: Optional[Any]) -> Model:
         variable_lstm_recurrent_dropout_rate = self.get_hyperparam("variable_lstm_recurrent_dropout_rate", hp)
         value_lstm_dropout_rate = self.get_hyperparam("value_lstm_dropout_rate", hp)
         value_lstm_recurrent_dropout_rate = self.get_hyperparam("value_lstm_recurrent_dropout_rate", hp)
-        dense_a_dropout_rate = self.get_hyperparam("dense_a_lstm_dropout_rate", hp)
-        dense_b_dropout_rate = self.get_hyperparam("dense_b_lstm_dropout_rate", hp)
+        dense_a_drop = self.get_hyperparam("dense_a_drop", hp)
+        dense_b_drop = self.get_hyperparam("dense_b_drop", hp)
 
         line_input = Input(shape=(None, self.line_shape[2]), name="line_input", dtype=self.d_type)
         line_lstm = LSTM(units=self.line_shape[1],
@@ -80,13 +80,13 @@ def build(self, hp: Optional[Any]) -> Model:
 
         # first hidden layer
         dense_a = Dense(units=dense_units, activation=ReLU(), name="a_dense", dtype=self.d_type)(joined_features)
-        dropout_dense_a = Dropout(dense_a_dropout_rate, name="a_dropout")(dense_a)
+        drop_a = Dropout(name="a_drop", rate=dense_a_drop)(dense_a)
 
         # second hidden layer
-        dense_b = Dense(units=dense_units, activation=ReLU(), name="b_dense", dtype=self.d_type)(dropout_dense_a)
-        dropout_dense_b = Dropout(dense_b_dropout_rate, name="b_dropout")(dense_b)
+        dense_b = Dense(units=dense_units, activation=ReLU(), name="b_dense", dtype=self.d_type)(drop_a)
+        drop_b = Dropout(name="b_drop", rate=dense_b_drop)(dense_b)
 
-        dense_final = Dense(units=1, activation='sigmoid', name="prediction", dtype=self.d_type)(dropout_dense_b)
+        dense_final = Dense(units=1, activation='sigmoid', name="prediction", dtype=self.d_type)(drop_b)
 
         metrics = [BinaryAccuracy(name="binary_accuracy"), Precision(name="precision"), Recall(name="recall")]
 

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "python-dateutil",
     "python-docx",
     "python-pptx",
+    "PySquashfsImage",
     "PyYAML",
     "rpmfile",
     "striprtf",

requirements.txt

@@ -22,6 +22,7 @@ numpy==2.3.3; python_version > '3.10'
 odfpy==1.4.1
 xlrd==2.0.2
 striprtf==0.0.29
+PySquashfsImage==0.9.0
 
 # onnxruntime - ML engine
 onnxruntime==1.23.2