Added PAN Regex, File owner details in fs, unique alerts only

Author: rohitcoder

connection.yml.sample

@@ -1,4 +1,5 @@
 notify:
+  redacted: True
   slack:
     webhook_url: https://hooks.slack.com/services/T0XXXXXXXXXXX/BXXXXXXXX/1CIyXXXXXXXXXXXXXXX
 

fingerprint.yml

@@ -1,6 +1,7 @@
 Email: "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}\\b"
 Phone: "\\b\\+\\d{1,3}[-.]?\\d{3}[-.]?\\d{4}\\b"
 Aadhar: "\\b\\d{4}[-.]?\\d{4}[-.]?\\d{4}\\b"
+PAN Number: "[A-Z]{5}[0-9]{4}[A-Z]{1}"
 Amazon MWS Auth Token: "amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
 Amazon SNS Topic Disclosure: "arn:aws:sns:[a-z0-9\\-]+:[0-9]+:[A-Za-z0-9\\-_]+"
 AWS Access Key ID Value: "(A3T[A-Z0-9]|AKIA|AGPA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}"
@@ -31,4 +32,4 @@ Shopify Shared Secret: "shpss_[a-fA-F0-9]{32}"
 Square Accesss Token: "sq0atp-[0-9A-Za-z\\-_]{22}"
 Square OAuth Secret: "sq0csp-[0-9A-Za-z\\-_]{43}"
 Twilio API Key: "(?i)twilio(.{0,20})?SK[0-9a-f]{32}"
-Twitter Secret: "(?i)twitter(.{0,20})?[0-9a-z]{35,44}"
+Twitter Secret: "(?i)twitter(.{0,20})?[0-9a-z]{35,44}"

hawk_scanner/commands/fs.py

@@ -8,6 +8,7 @@
 
 def process_file(file_path, key, results):
     matches = system.read_match_strings(file_path, 'fs')
+    file_data = system.getFileData(file_path)
     if matches:
         for match in matches:
             results.append({
@@ -17,7 +18,8 @@ def process_file(file_path, key, results):
                 'matches': match['matches'],
                 'sample_text': match['sample_text'],
                 'profile': key,
-                'data_source': 'fs'
+                'data_source': 'fs',
+                'file_data': file_data
             })
 
 def execute(args):

hawk_scanner/internals/system.py

@@ -1,13 +1,12 @@
 from rich.console import Console
 from rich.table import Table
-import yaml
-import re
-import os
-import argparse
-import requests
-import json
+import json, requests, argparse, yaml, re, datetime, os, subprocess, platform, hashlib
+from tinydb import TinyDB, Query
+from pwd import getpwuid
+
+# Create a TinyDB instance for storing previous alert hashes
+db = TinyDB('previous_alerts.json')
 
-console = Console()
 parser = argparse.ArgumentParser(description='🦅 A powerful scanner to scan your Filesystem, S3, MySQL, PostgreSQL, MongoDB, Redis, Google Cloud Storage and Firebase storage for PII and sensitive data.')
 parser.add_argument('--connection', action='store', help='YAML Connection file path')
 parser.add_argument('--fingerprint', action='store', help='Override YAML fingerprint file path')
@@ -17,6 +16,11 @@
 args, extra_args = parser.parse_known_args()
 
 
+console = Console()
+
+def calculate_msg_hash(msg):
+    return hashlib.sha256(msg.encode()).hexdigest()
+
 def print_info(message):
     console.print(f"[yellow][INFO][/yellow] {message}")
 
@@ -35,6 +39,60 @@ def print_info(message):
 def print_alert(message):
     console.print(f"[bold red][ALERT][/bold red] {message}")
 
+def get_file_owner(file_path):
+    owner_name = ""
+
+    # Determine the current operating system
+    system = platform.system()
+
+    if system == "Windows":
+        try:
+            # Run the 'dir /q' command and capture its output
+            result = subprocess.check_output(['dir', '/q', file_path], shell=True, text=True)
+            # Extract the line containing the file information
+            lines = result.splitlines()
+            file_name = os.path.basename(file_path)
+            if len(lines) >= 6:
+                for line in lines:
+                    if file_name in line:
+                        exploded_line = line.split(' ')
+                        owner_name = exploded_line[-2]
+        except subprocess.CalledProcessError as e:
+            owner_name = ""
+    else:
+        try:
+            # Use the 'os.stat()' method to get the file owner on non-Windows systems
+            file_stat = os.stat(file_path)
+            owner_name = file_stat.st_uid  # You can also use pwd.getpwuid(file_stat.st_uid).pw_name to get the username
+            owner_name = getpwuid(owner_name).pw_name + " (" + str(owner_name) + ")"
+        except OSError as e:
+            owner_name = ""
+
+    return owner_name
+
+def RedactData(input_string):
+    if len(input_string) < 3:
+        return input_string
+
+    # Calculate the number of characters to redact in the middle (half of the length)
+    redact_count = len(input_string) // 2
+
+    # Split the input string into two parts: before and after the middle
+    middle_start = len(input_string) // 2 - redact_count // 2
+    middle_end = len(input_string) // 2 + redact_count // 2
+
+    before_middle = input_string[:middle_start]
+    middle = input_string[middle_start:middle_end]
+    after_middle = input_string[middle_end:]
+
+    # Redact the middle part
+    redacted_middle = "*" * len(middle)
+
+    # Concatenate the parts back together
+    redacted_string = before_middle + redacted_middle + after_middle
+
+    return redacted_string
+
 def get_connection():
     if args.connection:
         if os.path.exists(args.connection):
@@ -112,18 +170,36 @@ def print_banner():
     if not args.shutup:
         console.print(banner)
 
+connections = get_connection()
+
 def match_strings(content):
     matched_strings = []
+
+    if 'notify' in connections:
+        redacted: bool = connections.get('notify', {}).get('redacted', False)
+
     for pattern_name, pattern_regex in patterns.items():
         print_debug(f"Matching pattern: {pattern_name}")
         found = {} 
         ## parse pattern_regex as Regex
         complied_regex = re.compile(pattern_regex, re.IGNORECASE)
         matches = re.findall(complied_regex, content)
+
         if matches:
             found['pattern_name'] = pattern_name
-            found['matches'] = matches
-            found['sample_text'] = content[:50]
+            redacted_matches = []
+            if redacted:
+                for match in matches:
+                    redacted_matches.append(RedactData(match))
+                found['matches'] = redacted_matches
+            else:
+                found['matches'] = matches
+
+            if redacted:
+                found['sample_text'] = RedactData(content[:50])
+            else:
+                found['sample_text'] = content[:50]
+
             matched_strings.append(found)
     return matched_strings
 
@@ -164,18 +240,61 @@ def read_match_strings(file_path, source):
     matched_strings = match_strings(content)
     return matched_strings
 
-def SlackNotify(msg):
-    connections = get_connection()
+def getFileData(file_path):
+    try:
+        # Get file metadata
+        file_stat = os.stat(file_path)
+
+        # Get the username of the file's creator (Windows)
+        creator_name = get_file_owner(file_path)
+        # Convert timestamps to human-readable format
+        created_time = datetime.datetime.fromtimestamp(file_stat.st_ctime)
+        modified_time = datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+        # Create a dictionary with the file information
+        ## should be dict
+        file_info = {
+            "creator": creator_name,
+            "created_time": created_time.strftime("%Y-%m-%d %H:%M:%S"),
+            "modified_time": modified_time.strftime("%Y-%m-%d %H:%M:%S"),
+        }
+        return file_info
 
+    except FileNotFoundError:
+        return json.dumps({"error": "File not found"})
+    except Exception as e:
+        return json.dumps({"error": str(e)})
+
+
+def SlackNotify(msg):
     if 'notify' in connections:
-        slack_config = connections['notify'].get('slack', {})
+        notify_config = connections['notify']
+        # Check if suppress_duplicates is set to True
+        suppress_duplicates = notify_config.get('suppress_duplicates', False)
+        
+        if suppress_duplicates:
+            # Calculate the hash of the message
+            msg_hash = calculate_msg_hash(msg)
+            
+            # Check if the message hash already exists in the previous alerts database
+            Alert = Query()
+            if db.contains(Alert.msg_hash == msg_hash):
+                print_debug("Duplicate message detected. Skipping webhook trigger.")
+                return
+        
+        slack_config = notify_config.get('slack', {})
         webhook_url = slack_config.get('webhook_url', '')
+        
         if webhook_url != '':
             try:
                 payload = {
                     'text': msg,
                 }
                 headers = {'Content-Type': 'application/json'}
                 requests.post(webhook_url, data=json.dumps(payload), headers=headers)
+                
+                if suppress_duplicates:
+                    # Store the message hash in the previous alerts database
+                    db.insert({'msg_hash': msg_hash})
             except Exception as e:
                 print_error(f"An error occurred: {str(e)}")

hawk_scanner/main.py

@@ -115,13 +115,14 @@ def main():
                 )
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: S3 Bucket
+                Data Source: S3 Bucket - {vulnerable_profile}
                 Bucket: {bucket}
                 File Path: {file_path}
                 Pattern Name: {pattern_name}
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     bucket=result['bucket'],
                     file_path=result['file_path'],
                     pattern_name=result['pattern_name'],
@@ -145,7 +146,7 @@ def main():
                 # Slack notification for MySQL
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: MySQL
+                Data Source: MySQL - {vulnerable_profile}
                 Host: {host}
                 Database: {database}
                 Table: {table}
@@ -154,6 +155,7 @@ def main():
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     host=result['host'],
                     database=result['database'],
                     table=result['table'],
@@ -179,7 +181,7 @@ def main():
                 # Slack notification for MongoDB
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: MongoDB
+                Data Source: MongoDB - {vulnerable_profile}
                 Host: {host}
                 Database: {database}
                 Collection: {collection}
@@ -188,6 +190,7 @@ def main():
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     host=result['host'],
                     database=result['database'],
                     collection=result['collection'],
@@ -213,7 +216,7 @@ def main():
                 # Slack notification for PostgreSQL
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: PostgreSQL
+                Data Source: PostgreSQL - {vulnerable_profile}
                 Host: {host}
                 Database: {database}
                 Table: {table}
@@ -222,6 +225,7 @@ def main():
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     host=result['host'],
                     database=result['database'],
                     table=result['table'],
@@ -245,13 +249,14 @@ def main():
                 )
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: Redis
+                Data Source: Redis - {vulnerable_profile}
                 Host: {host}
                 Key: {key}
                 Pattern Name: {pattern_name}
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     host=result['host'],
                     key=result['key'],
                     pattern_name=result['pattern_name'],
@@ -274,13 +279,14 @@ def main():
                 # Slack notification for Firebase/GCS
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: Firebase/GCS
+                Data Source: Firebase/GCS - {vulnerable_profile}
                 Bucket: {bucket}
                 File Path: {file_path}
                 Pattern Name: {pattern_name}
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    vulnerable_profile=result['profile'],
                     bucket=result['bucket'],
                     file_path=result['file_path'],
                     pattern_name=result['pattern_name'],
@@ -302,18 +308,24 @@ def main():
                 )
                 AlertMsg = """
                 *** PII Or Secret Found ***
-                Data Source: File System
-                File Path: {file_path}
+                Data Source: File System - {vulnerable_profile}
+                File Path: {file_path},
+                File Creator: {file_creator},
+                File Created at : {file_created},
+                File Last Modified at : {file_last_modified},
                 Pattern Name: {pattern_name}
                 Total Exposed: {total_exposed}
                 Exposed Values: {exposed_values}
                 """.format(
+                    file_creator = result['file_data']['creator'],
+                    file_created = result['file_data']['created_time'],
+                    file_last_modified = result['file_data']['modified_time'],
+                    vulnerable_profile=result['profile'],
                     file_path=result['file_path'],
                     pattern_name=result['pattern_name'],
                     total_exposed=str(len(result['matches'])),
                     exposed_values=str(', '.join(result['matches']))
                 )
-                
                 system.SlackNotify(AlertMsg)
             else:
                 # Handle other cases or do nothing for unsupported groups

previous_alerts.json

@@ -0,0 +1 @@
+{"_default": {"1": {"msg_hash": "d6906dff68674fbdd59ded71ddca02f992d399111a256fbd87ac18e7a9205edb"}, "2": {"msg_hash": "dad68cd597c5c2236a978b928486a34a0ce1b5884e469760f1d2f8fec2793c9b"}}}

readme.md

@@ -39,11 +39,20 @@ See how this works on Youtube - https://youtu.be/LuPXE7UJKOY
       pip3 install hawk-scanner
    ```
 
-## Example working command (To scan all supported services, or use fs/s3/gcs etc...)
+
+## Example working command (Use all/fs/s3/gcs etc...)
    ```bash
       hawk_scanner all --connection connection.yml --fingerprint fingerprint.yml --json output.json --debug
    ```
 
+### Note: Scanning Postgresql?, then you have to install some extra dependencies.
+
+For scanning postgresql source, this tool requires psycopg2-binary dependency, we can't ship this dependency with main package because psycopg2-binary not works with most of the systems espically with Windows, so you have to install it manually.
+   
+   ```bash
+      pip3 install psycopg2-binary
+   ```
+
 ## Building or running from source
 
 HAWK Eye is a Python-based CLI tool that can be installed using the following steps:

requirements.txt

@@ -6,5 +6,5 @@ redis
 firebase-admin
 google-cloud-core
 google-cloud-storage
-# psycopg2-binary
 pymongo==3.13.0
+tinydb==4.8.0