Merge branch 'release/1.5.0'

Author: fedelemantuano

README.md

@@ -98,7 +98,10 @@ Thug is a Python low-interaction honeyclient aimed at mimicing the behavior of a
 You can see a complete SpamScope report with Thug analysis [here](https://goo.gl/Y4kWCv).
 
 ### VirusTotal (optional)
-It's possible add to results (for mail attachments) VirusTotal report. You need a private API key.
+It's possible add to results (for mail attachments and sender ip address) the VirusTotal report. You need a private API key.
+
+### Shodan (optional)
+It's possible add to results the Shodan report for sender ip address. You need a private API key.
 
 ### Elasticsearch (optional)
 It's possible to store the results in Elasticsearch. In this case you should install `elasticsearch` package.
@@ -205,6 +208,8 @@ $ export ZEMANA_ENABLED=True
 $ export ZEMANA_APIKEY="your key"
 $ export ZEMANA_PARTNERID="your partner id"
 $ export ZEMANA_USERID="your userid" 
+$ export SHODAN_ENABLED=True
+$ export SHODAN_APIKEY="your key"
 ```
 
 

conf/spamscope.example.yml

@@ -70,6 +70,23 @@ tokenizer:
     # Max number of hashes saved for filter function
     maxlen_attachments: 1000000
 
+    # If True the same ip address is filtered and not analyzed.
+    filter_network: True
+
+    # Max number of hashes saved for filter function
+    maxlen_network: 1000000
+
+
+# Network bolt configuration
+network:
+    shodan:
+        enabled: False
+        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+    virustotal:
+        enabled: False
+        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
 
 # Attachments bolt configuration
 attachments:

docs/images/schema_topology.jpg

@@ -1,10 +1,10 @@
-(defproject spamscope "1.4.10-SNAPSHOT"
+(defproject spamscope "1.5.0-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"
   :jvm-opts ["-client"]
-  :dependencies  [[org.apache.storm/storm-core "1.0.3"]
-                  [org.apache.storm/flux-core "1.0.3"]]
+  :dependencies  [[org.apache.storm/storm-core "1.1.0"]
+                  [org.apache.storm/flux-core "1.1.0"]]
   :jar-exclusions     [#"log4j\.properties" #"org\.apache\.storm\.(?!flux)" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
   :uberjar-exclusions [#"log4j\.properties" #"org\.apache\.storm\.(?!flux)" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
   )

docs/images/schema_topology.png

@@ -1,12 +1,13 @@
 PyYAML==3.12
 backports.functools-lru-cache==1.3
 chainmap==1.0.2
-elasticsearch==5.2.0
+elasticsearch==5.3.0
 mail-parser==1.1.10
 patool==1.12
 pyparsing==2.2.0
 python-magic==0.4.12
 redis==2.10.5
+shodan==1.6.5
 simplejson==3.10.0
 six==1.10.0
 ssdeep==3.2

project.clj

@@ -24,3 +24,4 @@
 from .tokenizer import Tokenizer
 from .urls_handler_attachments import UrlsHandlerAttachments
 from .urls_handler_body import UrlsHandlerBody
+from .network import Network

requirements.txt

@@ -49,10 +49,14 @@ def _compose_output(self, greedy_data):
         mail["is_filtered"] = greedy_data["tokenizer"][2]
 
         # Attachments
-        mail["with_attachments"] = greedy_data["attachments"][1]
+        # with_raw_attachments: the mail has attachments
+        # with_attachments: the mail has not filtered attachments
+        mail["with_raw_attachments"] = greedy_data["attachments"][1]
+        attachments = greedy_data["attachments"][2]
 
-        if mail["with_attachments"]:
-            mail["attachments"] = greedy_data["attachments"][2]
+        if attachments:
+            mail["with_attachments"] = True
+            mail["attachments"] = attachments
 
         # Urls in attachments:
         # Add urls attachments because you can have more differents attachments
@@ -64,6 +68,11 @@ def _compose_output(self, greedy_data):
             urls = greedy_data["urls-handler-attachments"][2]
             mail["urls_attachments"] = reformat_urls(urls)
 
+        # Network
+        network = greedy_data["network"][1]
+        if network:
+            mail["network"] = network
+
         # Add intelligence output only if mail is not filtered
         if not mail["is_filtered"]:
 

src/bolts/__init__.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+from modules import AbstractBolt
+from modules.networks import processors
+
+
+class Network(AbstractBolt):
+    outputs = ['sha256_random', 'network', 'is_filtered']
+
+    def process(self, tup):
+        sha256_random = tup.values[0]
+        ipaddress = tup.values[1]
+        is_filtered = tup.values[2]
+
+        try:
+            results = {}
+
+            if not is_filtered and ipaddress:
+                for p in processors:
+                    try:
+                        p(self.conf[p.__name__], ipaddress, results)
+                    except KeyError:
+                        self.log("KeyError: {!r} doesn't exist in conf".format(
+                            p.__name__), "error")
+
+        except Exception as e:
+            self.log("Failed process network for mail: {}".format(
+                sha256_random), "error")
+            self.raise_exception(e, tup)
+
+        else:
+            self.emit([sha256_random, results])

src/bolts/json_maker.py

@@ -62,12 +62,13 @@ def _load_lists(self):
         self.log("Phishing targets keywords reloaded")
 
     def _search_phishing(self, greedy_data):
+        with_urls = False
 
         # If mail is filtered don't check for phishing
         is_filtered = greedy_data["tokenizer"][2]
 
         if is_filtered:
-            return False, False
+            return False, False, False
 
         # Reset phishing bitmap
         self._pb.reset_score()
@@ -83,8 +84,9 @@ def _search_phishing(self, greedy_data):
         urls_body = greedy_data["urls-handler-body"][2]
         urls_attachments = greedy_data["urls-handler-attachments"][2]
 
-        # TODO: if an attachment is filtered the score is not complete
-        # more different mails can have same attachment
+        # TODO: if an attachment is filtered, the score is not complete
+        # more different mails can have the same attachment
+        # more different attachments can have the same mail
         attachments = MailAttachments(greedy_data["attachments"][2])
 
         urls = (
@@ -110,14 +112,15 @@ def _search_phishing(self, greedy_data):
         # Target not added because urls come already analyzed text
         for k, v in urls:
             if k:
+                with_urls = True
                 if any(check_urls(k, i) for i in self._t_keys.values()):
                     self._pb.set_property_score(v)
 
         # Check subject
         if swt(subject, self._s_keys):
             self._pb.set_property_score("mail_subject")
 
-        return self._pb.score, list(targets)
+        return self._pb.score, list(targets), with_urls
 
     def process_tick(self, freq):
         """Every freq seconds you reload the keywords. """
@@ -135,10 +138,15 @@ def process(self, tup):
         if not diff:
             with_phishing = False
 
-            score, targets = self._search_phishing(
+            score, targets, with_urls = self._search_phishing(
                 self._mails.pop(sha256_random))
 
-            if score:
+            # There is phishing only if there is also urls
+            # Mail can have target without phishing
+            if score and with_urls:
                 with_phishing = True
+            else:
+                with_phishing = False
+                score = 0
 
             self.emit([sha256_random, with_phishing, score, targets])