Fixed email, ai & tools parsing app bugs

Author: frikky

email/1.3.0/src/app.py

@@ -409,6 +409,19 @@ def parse_email_file(self, file_id, extract_attachments=False):
                 "reason": "Couldn't get file with ID %s" % file_id
             }
 
+        #print("PRE: ", file_path)
+
+        # Check if data is in base64 and decode it
+        # If it ends with = then it may be bas64
+
+        if str(file_path["data"]).endswith("="):
+            try:
+                file_path["data"] = base64.b64decode(file_path["data"])
+            except Exception as e:
+                print(f"Failed to decode base64: {e}")
+
+        #print("POST: ", file_path)
+
         #print("File: %s" % file_path)
         print('working with .eml file? %s' % file_path["filename"])
 

shuffle-ai/1.0.0/api.yaml

@@ -98,6 +98,36 @@ actions:
     returns:
       schema:
         type: string
+  - name: run_schemaless 
+    description: Runs an automatically translated action  
+    parameters:
+      - name: category 
+        description: The category the action is in
+        required: true
+        multiline: false 
+        schema:
+          type: string
+      - name: action 
+        description: The action label to run
+        required: true
+        multiline: false 
+        schema:
+          type: string
+      - name: app_name 
+        description: The app to run the action in
+        required: false 
+        multiline: false 
+        schema:
+          type: string
+      - name: fields 
+        description: The additional fields to add 
+        required: false 
+        multiline: false 
+        schema:
+          type: string
+    returns:
+      schema:
+        type: string
   - name: transcribe_audio 
     description: Returns text from audio 
     parameters:

shuffle-ai/1.0.0/requirements.txt

@@ -1,3 +1,4 @@
 pytesseract
 pdf2image
 pypdf2
+requests

shuffle-ai/1.0.0/src/app.py

@@ -1,8 +1,9 @@
-import pytesseract
-from pdf2image import convert_from_path
-import PyPDF2
 import json
+import PyPDF2
 import tempfile
+import requests
+import pytesseract
+from pdf2image import convert_from_path
 
 from walkoff_app_sdk.app_base import AppBase
 
@@ -59,10 +60,6 @@ def generate_report(self, apikey, input_data, report_title, report_name="generat
             report_name = report_name + ".html"
 
         report_name = report_name.replace(" ", "_", -1)
-
-        if not formatting:
-            formatting = "auto"
-    
         output_formatting= "Format the following text into an HTML report with relevant graphs and tables. Title of the report should be {report_title}."
         ret = requests.post(
             "https://shuffler.io/api/v1/conversation", 
@@ -217,5 +214,79 @@ def gpt(self, input_text):
             "reason": "Not implemented yet"
         }
 
+    def run_schemaless(self, category, action, app_name="", fields=""):
+        """
+		action := shuffle.CategoryAction{ 
+			Label: step.Name,
+			Category: step.Category,
+			AppName: step.AppName,
+			Fields: step.Fields,
+
+			Environment: step.Environment,
+
+			SkipWorkflow: true,
+		}
+        """
+
+        data = {
+            "label": action,
+            "category": category,
+
+            "app_name": "",
+            "fields": [],
+
+            "skip_workflow": True,
+        }
+
+        if app_name:
+            data["app_name"] = app_name
+
+        if fields:
+            if isinstance(fields, list):
+                data["fields"] = fields
+
+            elif isinstance(fields, dict):
+                for key, value in fields.items(): 
+                    data["fields"].append({
+                        "key": key,
+                        "value": str(value),
+                    })
+
+            else:
+                try:
+                    loadedfields = json.loads(fields)
+                    for key, value in loadedfields.items(): 
+                        data["fields"].append({
+                            "key": key,
+                            "value": value,
+                        })
+
+                except Exception as e:
+                    print("[ERROR] Failed to load fields as JSON: %s" % e)
+                    return json.dumps({
+                        "success": False,
+                        "reason": "Ensure Fields are valid JSON",
+                        "type": type(fields),
+                        "details": "%s" % e,
+                    })
+
+
+        baseurl = "%s/api/v1/apps/categories/run" % self.base_url
+        baseurl += "?execution_id=%s&authorization=%s" % (self.current_execution_id, self.authorization) 
+
+        print("[DEBUG] Running schemaless action with URL '%s', category %s and action label %s" % (baseurl, category, action))
+
+        headers = {}
+        request = requests.post(
+            baseurl,
+            json=data,
+            headers=headers,
+        )
+
+        try: 
+            return request.json()
+        except:
+            return request.text
+
 if __name__ == "__main__":
     Tools.run()

shuffle-tools/1.2.0/src/app.py

@@ -2394,6 +2394,8 @@ def parse_ioc(self, input_string, input_type="all"):
             input_type = input_type.split(",")
             for item in input_type:
                 item = item.strip()
+                if not item.endswith("s"):
+                    item = "%ss" % item
 
             ioc_types = input_type