fix StringsScanner hallucinations. Version UP 1.14.1 (#799)

* fix StringsScanner hallucinations

* fix

Author: babenek

.github/workflows/check.yml

@@ -92,7 +92,7 @@ jobs:
         run: |
           banner="$(python -m credsweeper --banner | head -1)"
           echo "banner = '${banner}'"
-          if [ "CredSweeper 1.14.0 crc32:a52f81a3" != "${banner}" ]; then
+          if [ "CredSweeper 1.14.1 crc32:5aa3f35d" != "${banner}" ]; then
             echo "Update the check for '${banner}'"
             exit 1
           fi

credsweeper/__init__.py

@@ -24,4 +24,4 @@
     "__version__"
 ]
 
-__version__ = "1.14.0"
+__version__ = "1.14.1"

credsweeper/deep_scanner/strings_scanner.py

@@ -15,25 +15,27 @@ class StringsScanner(AbstractScanner, ABC):
     """Implements known binary file scanning with ASCII strings representations"""
 
     @staticmethod
-    def get_strings(data: bytes) -> List[Tuple[str, int]]:
+    def get_enumerated_lines(data: bytes) -> List[Tuple[int, str]]:
         """Processes binary to found ASCII strings. Use offset instead line number."""
-        strings = []
-        offset = 0
-        line = ''
+        enumerated_lines = []
+        offset = -1
+        line_items = []
         for n, x in enumerate(data):
             if 0x09 == x or 0x20 <= x <= 0x7E:
                 # TAB, SPACE and visible ASCII symbols
-                if not offset:
-                    # for line number
+                if 0 > offset:
+                    # use start of string as line number
                     offset = n
-                line += chr(x)
-            elif MIN_DATA_LEN <= len(line):
-                strings.append((line, offset))
-                offset = 0
-                line = ''
-        if MIN_DATA_LEN <= len(line):
-            strings.append((line, offset))
-        return strings
+                line_items.append(chr(x))
+                continue
+            if MIN_DATA_LEN <= len(line_items):
+                # add valuable lines only
+                enumerated_lines.append((offset, ''.join(line_items)))
+            offset = -1
+            line_items.clear()
+        if MIN_DATA_LEN <= len(line_items):
+            enumerated_lines.append((offset, ''.join(line_items)))
+        return enumerated_lines
 
     def data_scan(
             self,  #
@@ -42,9 +44,9 @@ def data_scan(
             recursive_limit_size: int) -> Optional[List[Candidate]]:
         """Extracts data file from .ar (debian) archive and launches data_scan"""
 
-        if strings := StringsScanner.get_strings(data_provider.data):
-            string_data_provider = StringContentProvider(lines=[x[0] for x in strings],
-                                                         line_numbers=[x[1] for x in strings],
+        if strings := StringsScanner.get_enumerated_lines(data_provider.data):
+            string_data_provider = StringContentProvider(lines=[x[1] for x in strings],
+                                                         line_numbers=[x[0] for x in strings],
                                                          file_path=data_provider.file_path,
                                                          file_type=data_provider.file_type,
                                                          info=f"{data_provider.info}|STRINGS")

tests/data/depth_3_pedantic.json

@@ -12798,16 +12798,16 @@
         "ml_probability": 1.0,
         "line_data_list": [
             {
-                "line": "0ui\tBJ=https://jrfdeg:[email protected]:32768/architecture",
-                "line_num": 1556,
+                "line": "https://jrfdeg:[email protected]:32768/architecture",
+                "line_num": 1640,
                 "path": "./tests/samples/sample.elf",
                 "info": "FILE:./tests/samples/sample.elf|STRINGS",
                 "variable": "https://",
-                "variable_start": 7,
-                "variable_end": 15,
+                "variable_start": 0,
+                "variable_end": 8,
                 "value": "dh3sjr8b",
-                "value_start": 22,
-                "value_end": 30,
+                "value_start": 15,
+                "value_end": 23,
                 "entropy": 3.0
             }
         ]

tests/deep_scanner/test_strings_scanner.py

@@ -5,11 +5,25 @@
 from credsweeper.deep_scanner.strings_scanner import StringsScanner
 
 
-class TestDebScanner(unittest.TestCase):
+class TestStringsScanner(unittest.TestCase):
 
     def setUp(self):
         self.maxDiff = None
 
     @given(strategies.binary())
-    def test_get_shannon_entropy_hypothesis_n(self, data):
-        self.assertIsNotNone(StringsScanner.get_strings(data))
+    def test_get_lines_hypothesis_n(self, data):
+        self.assertIsNotNone(StringsScanner.get_enumerated_lines(data))
+
+    def test_get_lines_n(self):
+        self.assertListEqual([], StringsScanner.get_enumerated_lines(b''))
+        self.assertListEqual([], StringsScanner.get_enumerated_lines(b'\x00\xBE'))
+        self.assertListEqual([], StringsScanner.get_enumerated_lines(b'\x9F\xBEP\xE3\xb4W\xA5:\xF1R\x9C00\xcf\x84t!'))
+        self.assertListEqual([], StringsScanner.get_enumerated_lines(b'\x00\x01\x02PW:R00t\x0D\x00'))
+
+    def test_get_lines_p(self):
+        self.assertListEqual([(3, "PW:R00t!")], StringsScanner.get_enumerated_lines(b'\x00\x01\x02PW:R00t!\x0D\x00'))
+        self.assertListEqual([(0, "PW:R00t!")], StringsScanner.get_enumerated_lines(b'PW:R00t!\x0D\x00'))
+        self.assertListEqual([(4, "PW:R00t!")], StringsScanner.get_enumerated_lines(b'\x00\x01\x02\x03PW:R00t!'))
+        self.assertListEqual(
+            [(9, 'Salt:CwXD\t3dsd'), (24, 'Token:SOMETEST')],
+            StringsScanner.get_enumerated_lines(b'\x9F\xBEP\xE3\xb4W\xA5:\xFFSalt:CwXD\x093dsd\nToken:SOMETEST\0'))