Merge branch 'release/1.5.1'

Author: fedelemantuano

README.md

@@ -56,7 +56,7 @@ SpamScope can be downloaded, used, and modified free of charge. It is available
 ## Authors
 
 ### Main Author
- Fedele Mantuano (**Twitter**: [@fedelemantuano](https://twitter.com/fedelemantuano))
+ Fedele Mantuano (**LinkedIn**: [Fedele Mantuano](https://www.linkedin.com/in/fmantuano/))
 
 
 

docs/images/Docker02.png

@@ -1,4 +1,4 @@
-(defproject spamscope "1.5.0-SNAPSHOT"
+(defproject spamscope "1.5.1-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

project.clj

@@ -1,7 +1,7 @@
 PyYAML==3.12
 backports.functools-lru-cache==1.3
 chainmap==1.0.2
-elasticsearch==5.3.0
+elasticsearch==5.4.0
 mail-parser==1.1.10
 patool==1.12
 pyparsing==2.2.0
@@ -11,6 +11,6 @@ shodan==1.6.5
 simplejson==3.10.0
 six==1.10.0
 ssdeep==3.2
-streamparse==3.4.0
+streamparse==3.5.0
 tika-app==1.1.0
 virustotal-api==1.1.2

requirements.txt

@@ -65,21 +65,15 @@ def process(self, tup):
         sha256_random = tup.values[0]
         with_attachments = tup.values[1]
 
-        try:
-            # Remove all values
-            self.attach.removeall()
+        # Remove all values
+        self.attach.removeall()
 
-            # Add the new values
-            self.attach.extend(tup.values[2])
+        # Add the new values
+        self.attach.extend(tup.values[2])
 
-            # Run analysis
-            # self.attach.run() == self.attach()
-            self.attach.run()
+        # Run analysis
+        # self.attach.run() == self.attach()
+        self.attach.run()
 
-        except Exception as e:
-            self.log("Failed process attachments for mail: {}".format(
-                sha256_random), "error")
-            self.raise_exception(e, tup)
-
-        else:
-            self.emit([sha256_random, with_attachments, list(self.attach)])
+        # emit
+        self.emit([sha256_random, with_attachments, list(self.attach)])

src/bolts/attachments.py

@@ -57,6 +57,8 @@ def _compose_output(self, greedy_data):
         if attachments:
             mail["with_attachments"] = True
             mail["attachments"] = attachments
+        else:
+            mail["with_attachments"] = False
 
         # Urls in attachments:
         # Add urls attachments because you can have more differents attachments

src/bolts/json_maker.py

@@ -31,21 +31,14 @@ def process(self, tup):
         ipaddress = tup.values[1]
         is_filtered = tup.values[2]
 
-        try:
-            results = {}
-
-            if not is_filtered and ipaddress:
-                for p in processors:
-                    try:
-                        p(self.conf[p.__name__], ipaddress, results)
-                    except KeyError:
-                        self.log("KeyError: {!r} doesn't exist in conf".format(
-                            p.__name__), "error")
-
-        except Exception as e:
-            self.log("Failed process network for mail: {}".format(
-                sha256_random), "error")
-            self.raise_exception(e, tup)
-
-        else:
-            self.emit([sha256_random, results])
+        results = {}
+
+        if not is_filtered and ipaddress:
+            for p in processors:
+                try:
+                    p(self.conf[p.__name__], ipaddress, results)
+                except KeyError:
+                    self.log("KeyError: {!r} doesn't exist in conf".format(
+                        p.__name__), "error")
+
+        self.emit([sha256_random, results, is_filtered])

src/bolts/network.py

@@ -25,6 +25,7 @@
     from collections import UserList
 
 import copy
+import datetime
 import logging
 import os
 import shutil
@@ -232,6 +233,8 @@ def _addmetadata(self):
         file size, content type, if is a archive and archived files.
         """
         for i in self:
+            i["analisys_date"] = datetime.datetime.utcnow().isoformat()
+
             if not i.get("is_filtered", False):
                 payload, size, ext = Attachments._metadata(i)
                 content_type = contenttype(payload)
@@ -265,6 +268,8 @@ def _addmetadata(self):
                                     content_type = contenttype(payload)
                                     filename = os.path.basename(j)
 
+                                    t["analisys_date"] = \
+                                        datetime.datetime.utcnow().isoformat()
                                     t["filename"] = filename
                                     t["extension"] = extension(filename)
                                     t["size"] = len(payload)

src/modules/attachments/attachments.py

@@ -20,6 +20,11 @@
 from __future__ import absolute_import, print_function, unicode_literals
 import logging
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
 try:
     from modules import register
 except ImportError:
@@ -55,7 +60,8 @@ def processor(conf, ipaddress, results):
 """
 
 
-@register(processors, active=True)
+@register(processors, active=False)
+# TODO: to solve issue https://github.com/Parsely/streamparse/issues/368
 def shodan(conf, ipaddress, results):
     """This method updates the attachments results
     with the Tika reports.
@@ -75,8 +81,12 @@ def shodan(conf, ipaddress, results):
 
         try:
             report = api.host(ipaddress)
+            json.dumps(report, ensure_ascii=False)
         except shodan.APIError:
-            log.exception("Shodan API error")
+            return
+        except TypeError:
+            log.error("JSON TypeError in Shodan report for ip {!r}".format(
+                ipaddress))
         else:
             if report:
                 results["shodan"] = report
@@ -99,7 +109,17 @@ def virustotal(conf, ipaddress, results):
     if conf["enabled"]:
         from virus_total_apis import PublicApi as VirusTotalPublicApi
         vt = VirusTotalPublicApi(conf["api_key"])
-        report = vt.get_ip_report(ipaddress)
 
-        if report:
-            results["virustotal"] = report
+        # Error: {u'virustotal': {'error': SSLError(SSLEOFError(8, u'EOF
+        # occurred in violation of protocol (_ssl.c:590)'),)}}')
+        # TypeError: SSLError(SSLEOFError(8, u'EOF occurred in violation of
+        # protocol (_ssl.c:590)'),) is not JSON serializable')
+        try:
+            report = vt.get_ip_report(ipaddress)
+            json.dumps(report, ensure_ascii=False)
+        except TypeError:
+            log.error("TypeError in VirusTotal report for ip {!r}".format(
+                ipaddress))
+        else:
+            if report:
+                results["virustotal"] = report

src/modules/networks/post_processing.py

@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "1.5.0"
+__version__ = "1.5.1"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {