Add dynamic checks

Author: cdgriffith

puremagic/magic_data.json

@@ -1747,9 +1747,6 @@
     ["0a0d0d0a", 0, ".pcapng", "application/octet-stream", "pcapng capture file"],
     ["05000000", 0, "", "", "INFO2 Windows recycle bin"],
     ["34cdb2a1", 0, "", "", "Tcpdump capture file"],
-    ["fffe0000", 0, "", "", "UTF-32|UCS-4 file"],
-    ["efbbbf", 0, "", "", "UTF8 file"],
-    ["feff", 0, "", "", "UTF-16|UCS-2 file"],
     ["6f3c", 0, "", "", "SMS text (SIM)"],
     ["aced", 0, "", "", "Java serialization data"],
     [

puremagic/main.py

@@ -22,7 +22,7 @@
 import puremagic
 
 if os.getenv("PUREMAGIC_DEEPSCAN") != "0":
-    from puremagic.scanners import (zip_scanner, pdf_scanner, text_scanner, json_scanner, python_scanner, sndhdr_scanner)
+    from puremagic.scanners import zip_scanner, pdf_scanner, text_scanner, json_scanner, python_scanner, sndhdr_scanner
 
 __author__ = "Chris Griffith"
 __version__ = "2.0.0b5"
@@ -68,6 +68,10 @@ class PureError(LookupError):
     """Do not have that type of file in our databanks"""
 
 
+class PureValueError(ValueError):
+    """Invalid input"""
+
+
 def magic_data(
     filename: os.PathLike | str = os.path.join(here, "magic_data.json"),
 ) -> tuple[list[PureMagic], list[PureMagic], list[PureMagic], dict[bytes, list[PureMagic]]]:
@@ -193,7 +197,7 @@ def identify_all(header: bytes, footer: bytes, ext=None) -> list[PureMagicWithCo
 def perform_magic(header: bytes, footer: bytes, mime: bool, ext=None, filename=None) -> str:
     """Discover what type of file it is based on the incoming string"""
     if not header:
-        raise ValueError("Input was empty")
+        raise PureValueError("Input was empty")
     infos = identify_all(header, footer, ext)
     if filename and os.getenv("PUREMAGIC_DEEPSCAN") != "0":
         results = run_deep_scan(infos, filename, header, footer, raise_on_none=True)
@@ -322,7 +326,7 @@ def magic_file(filename: os.PathLike | str) -> list[PureMagicWithConfidence]:
     """
     head, foot = file_details(filename)
     if not head:
-        raise ValueError("Input was empty")
+        raise PureValueError("Input was empty")
     try:
         info = identify_all(head, foot, ext_from_filename(filename))
     except PureError:
@@ -344,7 +348,7 @@ def magic_string(string, filename: os.PathLike | str | None = None) -> list[Pure
     :return: list of possible matches, highest confidence first
     """
     if not string:
-        raise ValueError("Input was empty")
+        raise PureValueError("Input was empty")
     head, foot = string_details(string)
     ext = ext_from_filename(filename) if filename else None
     info = identify_all(head, foot, ext)
@@ -366,7 +370,7 @@ def magic_stream(
     """
     head, foot = stream_details(stream)
     if not head:
-        raise ValueError("Input was empty")
+        raise PureValueError("Input was empty")
     ext = ext_from_filename(filename) if filename else None
     info = identify_all(head, foot, ext)
     info.sort(key=lambda x: x.confidence, reverse=True)
@@ -389,17 +393,13 @@ def single_deep_scan(
             return zip_scanner.main(filename, head, foot)
         case pdf_scanner.match_bytes:
             return pdf_scanner.main(filename, head, foot)
-        case (
-            sndhdr_scanner.hcom_match_bytes
-            | sndhdr_scanner.fssd_match_bytes
-            | sndhdr_scanner.sndr_match_bytes
-        ):
+        case sndhdr_scanner.hcom_match_bytes | sndhdr_scanner.fssd_match_bytes | sndhdr_scanner.sndr_match_bytes:
             # sndr is a loose confidence and other results may be better
             result = sndhdr_scanner.main(filename, head, foot)
             if result and result.confidence > confidence:
                 return result
 
-    # The first match wins, so text_scanner should always be the last
+    # The first match wins
     for scanner in (pdf_scanner, python_scanner, json_scanner):
         result = scanner.main(filename, head, foot)
         if result:
@@ -446,7 +446,7 @@ def run_deep_scan(
         try:
             result = catch_all_deep_scan(filename, head, foot)
         except Exception:
-            pass
+            raise
         else:
             if result:
                 return [result]
@@ -491,19 +491,29 @@ def command_line_entry(*args):
         help="Return the mime type instead of file type",
     )
     parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", help="Print verbose output")
-    parser.add_argument("files", nargs="+")
+    parser.add_argument("files", nargs="+", type=Path)
     parser.add_argument("--version", action="version", version=puremagic.__version__)
     args = parser.parse_args(args if args else sys.argv[1:])
 
     for fn in args.files:
-        if not os.path.exists(fn):
+        if not fn.exists():
             print(f"File '{fn}' does not exist!")
             continue
-        try:
-            print(f"'{fn}' : {from_file(fn, args.mime)}")
-        except PureError:
-            print(f"'{fn}' : could not be Identified")
-            continue
+        if fn.is_dir():
+            for file in fn.iterdir():
+                if not file.is_file():
+                    continue
+                try:
+                    print(f"'{file}' : {from_file(file, args.mime)}")
+                except (PureError, PureValueError):
+                    print(f"'{file}' : could not be Identified")
+                    continue
+        else:
+            try:
+                print(f"'{fn}' : {from_file(fn, args.mime)}")
+            except (PureError, PureValueError):
+                print(f"'{fn}' : could not be Identified")
+                continue
         if args.verbose:
             matches = magic_file(fn)
             print(f"Total Possible Matches: {len(matches)}")

puremagic/scanners/json_scanner.py

@@ -7,7 +7,7 @@
 
 
 def main(file_path: os.PathLike | str, head: bytes, foot: bytes) -> Match | None:
-    if not (head.strip().startswith(b"{") and foot.strip().endswith(b"}")):
+    if not (head.strip().startswith(b"{") and foot.strip().endswith(b"}")) and not (head.strip().startswith(b"[") and foot.strip().endswith(b"]")):
         return None
     try:
         with open(file_path, "rb") as file:

puremagic/scanners/python_scanner.py

@@ -1,25 +1,106 @@
 import ast
 import os
+import re
 
 from puremagic.scanners.helpers import Match
 
+python_common_keywords = [
+    re.compile("\bdef\b"),
+    re.compile("\bclass\b"),
+    re.compile("\bimport\b"),
+    re.compile("\belif\b"),
+    re.compile("\bwhile\b"),
+    re.compile("\bexcept\b"),
+    re.compile("\bfinally\b"),
+    re.compile("\breturn\b"),
+    re.compile("\byield\b"),
+    re.compile("\blambda\b"),
+    re.compile("\bTrue\b"),
+    re.compile("\bFalse\b"),
+    re.compile("\bNone\b"),
+    re.compile("\b__version__\b"),
+    re.compile("__main__"),
+]
 
-def main(file_path: os.PathLike | str, *_, **__) -> Match | None:
+python_patterns = [
+    re.compile(r"\bdef\s+\w+\s*\("),  # Function definitions
+    re.compile(r"\bclass\s+\w+\s*[\(:]"),  # Class definitions
+    re.compile(r"\bimport\s+\w+"),  # Import statements
+    re.compile(r"\bfrom\s+\w+\s+import"),  # From-import statements
+    re.compile(r"\bif\s+.*:"),  # If statements
+    re.compile(r"\bfor\s+\w+\s+in\s+.*:"),  # For loops
+    re.compile(r"\bwhile\s+.*:"),  # While loops
+    re.compile(r"\btry\s*:"),  # Try blocks
+    re.compile(r"\.append\("),  # Method calls
+    re.compile(r"\.join\("),  # String operations
+    re.compile(r"print\s*\("),  # Print statements
+]
+
+
+def main(file_path: os.PathLike | str, _, __) -> Match | None:
     file_size = os.path.getsize(file_path)
     if file_size > 1_000_000:
         return None
     if not str(file_path).endswith(".py") and file_size < 100:
         return None
 
     try:
-        with open(file_path, "r") as file:
+        with open(file_path, "r", encoding="utf-8") as file:
             content = file.read()
+
+        # Parse to ensure it's valid Python syntax
         ast.parse(content)
-    except Exception:
+
+        if not str(file_path).endswith(".py"):
+            if not is_substantial_python_code(content):
+                return None
+
+    except (SyntaxError, UnicodeDecodeError, PermissionError, OSError):
         return None
+
     return Match(
         extension=".py",
         name="Python Script",
         mime_type="text/x-python",
         confidence=1.0,
     )
+
+
+def is_substantial_python_code(content: str) -> bool:
+    """
+    Check if the content contains substantial Python code indicators.
+    Returns True if the content appears to be meaningful Python code.
+    """
+    # Remove comments and strings to focus on actual code
+    content_lines = content.splitlines()
+    code_lines = []
+
+    for line in content_lines:
+        # Remove comments (basic approach - doesn't handle strings containing #)
+        line = line.split("#")[0].strip()
+        if line:  # Non-empty after removing comments
+            code_lines.append(line)
+
+    # If too few substantial lines, it's probably not real code
+    if len(code_lines) < 2:
+        return False
+
+    code_text = " ".join(code_lines)
+
+    # Check for Python keywords that indicate actual code
+
+    # Count how many keywords are present
+    keyword_count = 0
+    for keyword in python_common_keywords:
+        if keyword.search(code_text):
+            keyword_count += 1
+
+    # Require at least 2 keywords for substantial code
+    if keyword_count < 2:
+        return False
+
+    # Check for common Python patterns
+    for pattern in python_patterns:
+        if pattern.search(code_text):
+            return True
+    return False

puremagic/scanners/sndhdr_scanner.py

@@ -30,6 +30,7 @@ def test_hcom(head: bytes) -> Optional[Match]:
         )
     return None
 
+
 def main(_, head: bytes, __) -> Optional[Match]:
     try:
         rate = get_short_le(head[2:4])