Merge branch 'Shuffle:master' into new

Author: yashsinghcodes

email/1.3.0/requirements.txt

@@ -1,8 +1,6 @@
 requests==2.25.1
 glom==20.11.0
-eml-parser==1.17.0
-msg-parser==1.2.0
-mail-parser==3.15.0
-extract-msg==0.30.9
 jsonpickle==2.0.0
 
+eml-parser==2.0.0
+msg-parser==1.2.0

email/1.3.0/src/app.py

@@ -11,8 +11,6 @@
 import time
 import random
 import eml_parser
-import mailparser
-import extract_msg
 import jsonpickle
 
 from glom import glom
@@ -392,6 +390,18 @@ def merge(d1, d2):
                 "messages": json.dumps(emails, default=default),
             }
 
+    def remove_similar_items(self, items):
+        # Sort items by length in descending order
+        items = sorted(items, key=len, reverse=True)
+        result = []
+
+        for domain in items:
+            # Check if the domain is part of any domain already in the result
+            if not any(domain in main for main in result):
+                result.append(domain)
+        
+        return result
+
     def parse_eml(self, filedata, extract_attachments=False):
         parsedfile = {
             "success": True,
@@ -443,16 +453,16 @@ def parse_email_file(self, file_id, extract_attachments=False):
         # Replace raw newlines \\r\\n with actual newlines
         # The data is a byte string, so we need to decode it to utf-8
         try:
-            print("Pre size: %d" % len(file_path["data"]))
+            #print("Pre size: %d" % len(file_path["data"]))
             file_path["data"] = file_path["data"].decode("utf-8").replace("\\r\\n", "\n").encode("utf-8")
-            print("Post size: %d" % len(file_path["data"]))
+            #print("Post size: %d" % len(file_path["data"]))
         except Exception as e:
             print(f"Failed to decode file: {e}")
             pass
 
         # Makes msg into eml
         if ".msg" in file_path["filename"] or "." not in file_path["filename"]:
-            print(f"[DEBUG] Working with .msg file {file_path['filename']}. Filesize: {len(file_path['data'])}")
+            self.logger.info(f"[DEBUG] Working with .msg file {file_path['filename']}. Filesize: {len(file_path['data'])}")
             try:
                 result = {}
                 msg = MsOxMessage(file_path['data'])
@@ -471,17 +481,17 @@ def parse_email_file(self, file_id, extract_attachments=False):
         )
 
         try:
-            print("Pre email")
+            self.logger.info("Pre email")
             parsed_eml = ep.decode_email_bytes(file_path['data'])
             #if str(parsed_eml["header"]["date"]) == "1970-01-01 00:00:00+00:00" and len(parsed_eml["header"]["subject"]) == 0:
             #    return {"success":False,"reason":"Not a valid EML/MSG file, or the file have a timestamp or subject defined (required).", "date": str(parsed_eml["header"]["date"]), "subject": str(parsed_eml["header"]["subject"])}
 
             # Put attachments in the shuffle file system
-            print("Pre attachment")
+            self.logger.info("Pre attachment")
             if extract_attachments == True and "attachment" in parsed_eml:
                 cnt = -1 
 
-                print("[INFO] Uploading %d attachments" % len(parsed_eml["attachment"]))
+                self.logger.info("[INFO] Uploading %d attachments" % len(parsed_eml["attachment"]))
                 for value in parsed_eml["attachment"]:
                     cnt += 1
                     if value["raw"] == None:
@@ -502,7 +512,25 @@ def parse_email_file(self, file_id, extract_attachments=False):
             if not "attachment" in parsed_eml:
                 parsed_eml["attachment"] = []
 
-            print("Post attachment")
+            self.logger.info("Post attachment. Has body: %s" % ("body" in parsed_eml))
+
+            try:
+                if "body" in parsed_eml and len(parsed_eml["body"]) > 0:
+
+                    for i in range(len(parsed_eml["body"])):
+                        if "uri" in parsed_eml["body"][i] and len(parsed_eml["body"][i]["uri"]) > 0:
+                            parsed_eml["body"][i]["uri"] = self.remove_similar_items(parsed_eml["body"][i]["uri"])
+
+                        if "email" in parsed_eml["body"][i] and len(parsed_eml["body"][i]["email"]) > 0:
+                            parsed_eml["body"][i]["email"] = self.remove_similar_items(parsed_eml["body"][i]["email"])
+
+                        if "domain" in parsed_eml["body"][i] and len(parsed_eml["body"][i]["domain"]) > 0:
+                            parsed_eml["body"][i]["domain"] = self.remove_similar_items(parsed_eml["body"][i]["domain"])
+
+            except Exception as e:
+                self.logger.info(f"[ERROR] Failed to remove similar items: {e}")
+
+            parsed_eml["success"] = True
             return json.dumps(parsed_eml, default=json_serial)   
         except Exception as e:
             return {"success":False, "reason": f"An exception occured during EML parsing: {e}. Please contact support"} 

shuffle-tools/1.2.0/requirements.txt

@@ -1,10 +1,10 @@
-ioc_finder==7.2.1
-py7zr==0.11.3
-rarfile==4.0
-pyminizip==0.2.4
-requests==2.25.1
-xmltodict==0.11.0
+ioc_finder==7.3.0
+py7zr==0.22.0
+rarfile==4.2
+pyminizip==0.2.6
+requests==2.32.3
+xmltodict==0.14.2
 json2xml==5.0.5
 ipaddress==1.0.23
-google.auth==1.23.0
-paramiko==3.1.0
+google.auth==2.37.0
+paramiko==3.5.0

shuffle-tools/1.2.0/src/app.py

@@ -1,9 +1,12 @@
+import os
+import sys
+import builtins
+
 import hmac
 import datetime
 import json
 import time
 import markupsafe
-import os
 import re
 import subprocess
 import tempfile
@@ -41,6 +44,12 @@
 from walkoff_app_sdk.app_base import AppBase
 #from shuffle_sdk import AppBase
 
+# Override exit(), sys.exit, and os._exit 
+# sys.exit() can be caught, meaning we can have a custom handler for it
+builtins.exit = sys.exit
+os.exit = sys.exit
+os._exit = sys.exit
+
 class Tools(AppBase):
     __version__ = "1.2.0"
     app_name = (
@@ -573,7 +582,6 @@ def execute_python(self, code):
                     "message": f"Filename needs to contain .py",
                 }
 
-
         # Write the code to a file
         # 1. Take the data into a file
         # 2. Subprocess execute file?
@@ -594,7 +602,27 @@ def custom_print(*args, **kwargs):
 
             globals_copy["self"] = self
 
-            exec(code, globals_copy)
+            try:
+                exec(code, globals_copy)
+            except SystemExit as e:
+                # Same as a return
+                pass
+            except SyntaxError as e:
+                # Special handler for return usage. Makes return act as 
+                # an exit()
+                if "'return' outside function" in str(e):
+                    return {
+                        "success": False,
+                        "message": f"Instead of using 'return' without a function, use 'exit()' to return when not inside a function. Raw Syntax error: {e}",
+                    }
+                else:
+                    return {
+                        "success": False,
+                        "message": f"Syntax Error: {e}",
+                    }
+
+            # this doesn't work to capture top-level returns
+            # Reason: SyntaxError makes it crash BEFORE it reaches the return
 
             s = f.getvalue()
             f.close() # why: https://www.youtube.com/watch?v=6SA6S9Ca5-U
@@ -623,7 +651,7 @@ def custom_print(*args, **kwargs):
         except Exception as e:
             return {
                 "success": False,
-                "message": f"exception: {e}",
+                "message": f"Exception: {e}",
             }
 
     def execute_bash(self, code, shuffle_input):