Merge branch 'release/2.2.1'

fedelemantuano · fedelemantuano · commit feeaeabc72ca · 2018-02-04T19:18:51.000+01:00
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,15 @@
+[run]
+source = src/modules/
+
+[report]
+omit = src/modules/abstracts.py
+    src/modules/mails/spamassassin_analysis.py
+    src/modules/rabbitmq_client.py
+    src/modules/attachments/thug_analysis.py
+
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise AssertionError
+    raise NotImplementedError
+    if __name__ == .__main__.:
diff --git a/.travis.yml b/.travis.yml
@@ -14,7 +14,6 @@ env:
       TIKA_APP_JAR=/tmp/tika-app-${TIKA_VER}.jar
       FAUP_PATH=/tmp/faup
       ZEMANA_PATH=/tmp/zemana
-      DOCKER_ROOT_PATH=/tmp/docker-root
       DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch
 
 before_install:
@@ -25,26 +24,22 @@ before_install:
         cmake
         libfuzzy-dev
         unrar
-    #- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+    - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+
+    - git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH
 
     # Build latest images spamscope-root, spamscope-elasticsearch
     # make images
     - if [ "$TRAVIS_BRANCH" == "master" ]; then
-        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-root.git $DOCKER_ROOT_PATH &&
-        cd $DOCKER_ROOT_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-root . && cd - &&
-        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'python -m unittest discover -s tests -f -v' &&
-        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v' &&
-        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH &&
-        cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=latest -t $DOCKER_USERNAME/spamscope-elasticsearch . && cd -;
+        cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=master -t $DOCKER_USERNAME/spamscope-elasticsearch . && cd -;
+        docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'python -m unittest discover -s tests -f -v';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';
       fi
 
     - if [ "$TRAVIS_BRANCH" == "develop" ]; then
-        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-root.git $DOCKER_ROOT_PATH &&
-        cd $DOCKER_ROOT_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH . && cd - &&
-        docker run --rm -it $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH /bin/sh -c 'python -m unittest discover -s tests -f -v' &&
-        docker run --rm -it $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v' &&
-        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH &&
-        cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-elasticsearch:$TRAVIS_BRANCH . && cd -;
+        cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=develop -t $DOCKER_USERNAME/spamscope-elasticsearch:develop . && cd -;
+        docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'python -m unittest discover -s tests -f -v';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';
       fi
 
 # command to install dependencies
@@ -62,7 +57,7 @@ before_script:
 # command to run tests
 script:
     # Unittests and coverage
-    - coverage run --include=src/modules/* --omit=src/modules/abstracts.py,src/modules/mails/spamassassin_analysis.py -m unittest discover -s tests -f -v
+    - coverage run -m unittest discover -s tests -f -v
 
     # timing
     - python tests/timing_test_search_keywords.py
@@ -73,19 +68,27 @@ script:
     - spamscope-elasticsearch -v
     - spamscope-topology -v
 
+deploy:
+    provider: pypi
+    user: fmantuano
+    password:
+        secure: "lPuvh9if9jZrtgk/i9++D7v1BGfyEj9kalJy52yfjOARZ3xskrvHLjJMIxu9wFndYYdUAz8J1Y2WsKLDyYyfMwiAFadU2xmsobNunGO95CNUadbfJojaH7LartWQN+gnw98svJTms9GLHLmO8eH2j5BsH2mubOaWm8T0lF3Pjs0NncXdi6HzSQ63qGj6ctjFzAmTqoK14EzbGVAu1KRBzO6hdu41/nhWx1ufRZYBNeEjtNCZfIl74ALdCpEEQfqTl5hgMA9ybuXvcAmZMOWVjBQiXjSeAUoz+u2FYBkzSxx0Sst1MkKoC3iG6p+Z2E4WHya63ew1GQyyug7XSmq8JpaKRFtktU2b5g+AcqKmLsf0nw+2x/pjGxJIyy69OnX7oGr9fQTGGMHg36OU8vdyhOX+40nHEXC+M45rS/ma+tYSrL+j2Mn1kKreOjoC6kV1g+bgX2eSVFicFonB++ySFSt8D5b7zQibIx0BCxzN/WpNu99/0DTFpA0n9ox7+4F4Jzrn3a8IM56I/RyqBqh8ce7TjhOwsF6n8Egjk9ywVuZW8OuohBTxdosU+TAntlNCmGgECcdXzSGqTXLPWjfe2U7C4qtBVwHMlptYxZNs8XvlJksysM8Cyp+dUjXJCLqPH2n55aK9+x6Pzq/cSM1ldzXg+iZcO9XXoEDJ1bLXVDE="
+    on:
+        tags: true
+        repo: SpamScope/spamscope
+        branch: master
+
 after_success:
     - coveralls
 
     - if [ "$TRAVIS_BRANCH" == "master" ]; then
         docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
-        docker push $DOCKER_USERNAME/spamscope-root;
         docker push $DOCKER_USERNAME/spamscope-elasticsearch;
       fi
 
     - if [ "$TRAVIS_BRANCH" == "develop" ]; then
         docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
-        docker push $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH;
-        docker push $DOCKER_USERNAME/spamscope-elasticsearch:$TRAVIS_BRANCH;
+        docker push $DOCKER_USERNAME/spamscope-elasticsearch:develop;
       fi
 
 notifications:
diff --git a/README.md b/README.md
@@ -2,6 +2,7 @@
 [![Build Status](https://travis-ci.org/SpamScope/spamscope.svg?branch=master)](https://travis-ci.org/SpamScope/spamscope)
 [![Coverage Status](https://coveralls.io/repos/github/SpamScope/spamscope/badge.svg?branch=develop)](https://coveralls.io/github/SpamScope/spamscope?branch=develop)
 [![BCH compliance](https://bettercodehub.com/edge/badge/SpamScope/spamscope?branch=develop)](https://bettercodehub.com/)
+[![](https://images.microbadger.com/badges/image/fmantuano/spamscope-elasticsearch.svg)](https://microbadger.com/images/fmantuano/spamscope-elasticsearch "Get your own image badge on microbadger.com")
 
 ![SpamScope](https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png)
 
@@ -225,11 +226,8 @@ $ export SPAMASSASSIN_ENABLED=True
 ## Docker images
 It's possible to use complete Docker images with Apache Storm and SpamScope. Take the following images:
 
- - [Root](https://hub.docker.com/r/fmantuano/spamscope-root/)
- - [Elasticsearch](https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/)
-
-For each image there are two tags: **develop** and **latest**.
-
+ - [Deps](https://hub.docker.com/r/fmantuano/spamscope-deps/): to use as base image
+ - [Elasticsearch](https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/): integrated with Elasticsearch
 
 
 ## Screenshots
diff --git a/README.rst b/README.rst
@@ -1,4 +1,5 @@
 |PyPI version| |Build Status| |Coverage Status| |BCH compliance|
+|image4|
 
 .. figure:: https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png
    :alt: SpamScope
@@ -314,10 +315,10 @@ Docker images
 It's possible to use complete Docker images with Apache Storm and
 SpamScope. Take the following images:
 
--  `Root <https://hub.docker.com/r/fmantuano/spamscope-root/>`__
--  `Elasticsearch <https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/>`__
-
-For each image there are two tags: **develop** and **latest**.
+-  `Deps <https://hub.docker.com/r/fmantuano/spamscope-deps/>`__: to use
+   as base image
+-  `Elasticsearch <https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/>`__:
+   integrated with Elasticsearch
 
 Screenshots
 -----------
@@ -350,5 +351,7 @@ Screenshots
    :target: https://coveralls.io/github/SpamScope/spamscope?branch=develop
 .. |BCH compliance| image:: https://bettercodehub.com/edge/badge/SpamScope/spamscope?branch=develop
    :target: https://bettercodehub.com/
+.. |image4| image:: https://images.microbadger.com/badges/image/fmantuano/spamscope-elasticsearch.svg
+   :target: https://microbadger.com/images/fmantuano/spamscope-elasticsearch
 .. |Donate| image:: https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif
    :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=VEPXYP745KJF2
diff --git a/conf/templates/spamscope.json b/conf/templates/spamscope.json
@@ -1,6 +1,6 @@
 {
   "order": 0,
-  "version": 4,
+  "version": 5,
   "index_patterns": "spamscope_*-*",
   "settings": {
     "analysis": {
@@ -31,7 +31,8 @@
     "index.codec": "best_compression",
     "index.number_of_shards": 2,
     "index.number_of_replicas": 0,
-    "index.refresh_interval": "5s"
+    "index.refresh_interval": "5s",
+    "index.mapping.ignore_malformed": true
   },
   "mappings": {
     "analysis": {
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -0,0 +1,27 @@
+FROM fmantuano/spamscope-deps
+
+MAINTAINER Fedele Mantuano "mantuano.fedele@gmail.com"
+
+# environment variables
+ARG SPAMSCOPE_VER="develop"
+
+ENV SPAMASSASSIN_ENABLED="True" \
+    SPAMSCOPE_CONF_FILE="/etc/spamscope/spamscope.yml" \
+    SPAMSCOPE_PATH="/opt/spamscope" \
+    THUG_ENABLED="True"
+
+# labels
+LABEL description="Spamscope: Advanced Spam Analysis" \
+    spamscope_version=${SPAMSCOPE_VER}
+
+# install SpamScope
+RUN set -ex; \
+    mkdir -p "/var/log/spamscope" "/etc/spamscope"; \
+    git clone -b ${SPAMSCOPE_VER} --single-branch https://github.com/SpamScope/spamscope.git ${SPAMSCOPE_PATH}; \
+    cd $SPAMSCOPE_PATH; \
+    pip install -r requirements_optional.txt; \
+    python setup.py install; \
+    sparse jar -s; \
+    pip install -U thug;
+    
+WORKDIR ${SPAMSCOPE_PATH}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -4,28 +4,22 @@ version: '2.1'
 services:
 
   # SpamScope
-  spamscope:
-    image: ${SPAMSCOPE_IMAGE_NAME}
+  spamscope-debug:
+    image: spamscope-debug
+    build: .
     container_name: spamscope
-    mem_limit: ${SPAMSCOPE_MEM_LIMIT}
-    dns:
-      - 8.8.8.8
-      - 8.8.4.4
+    mem_limit: 4g
     ports:
-      - "${SPAMSCOPE_BIND_IP}:8080:8080"
-      - "${SPAMSCOPE_BIND_IP}:8000:8000"
+      - "127.0.0.1:8080:8080"
+      - "127.0.0.1:8000:8000"
     volumes:
       - ${HOST_SPAMSCOPE_CONF}:/etc/spamscope
       - ${HOST_MAILS_FOLDER}:${DOCKER_MAILS_FOLDER}
     healthcheck:
       test: curl -fs http://localhost:8080/
     networks:
-      - ${NET_NAME}
-
-volumes:
-  esdata:
-    driver: local
+      - spamscope
 
 networks:
-  esnet:
+  spamscope:
     driver: bridge
diff --git a/project.clj b/project.clj
@@ -1,4 +1,4 @@
-(defproject spamscope "2.2.0-SNAPSHOT"
+(defproject spamscope "2.2.1-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"
diff --git a/requirements.txt b/requirements.txt
@@ -3,11 +3,11 @@ astropy>=1.3.3
 backports.functools-lru-cache>=1.3
 chainmap
 lxml
-mail-parser>=3
+mail-parser>=3.2.6
 patool
 pyparsing
 python-magic
 simplejson
 six
 ssdeep
-streamparse==3.13.0
+streamparse==3.13.1
diff --git a/src/bolts/tokenizer.py b/src/bolts/tokenizer.py
@@ -84,6 +84,7 @@ def _make_mail(self, tup):
         mail["mailbox"] = tup.values[2]
         mail["priority"] = tup.values[3]
         mail["sender_ip"] = self.parser.get_server_ipaddress(tup.values[4])
+        mail["to_domains"] = self.parser.to_domains
 
         # Fingerprints of body mail
         (mail["md5"], mail["sha1"], mail["sha256"], mail["sha512"],
diff --git a/src/modules/utils.py b/src/modules/utils.py
@@ -27,7 +27,7 @@
 import re
 import signal
 import tempfile
-from functools import wraps
+import functools
 
 import six
 import yaml
@@ -87,6 +87,7 @@ def decorator(func):
         def _handle_timeout(signum, frame):
             raise TimeoutError(error_message)
 
+        @functools.wraps(func)
         def wrapper(*args, **kwargs):
             signal.signal(signal.SIGALRM, _handle_timeout)
             signal.alarm(seconds)
@@ -96,7 +97,7 @@ def wrapper(*args, **kwargs):
                 signal.alarm(0)
             return result
 
-        return wraps(func)(wrapper)
+        return wrapper
 
     return decorator
 
@@ -163,7 +164,8 @@ def urls_extractor(text, faup):
     for i in set(match.group().strip() for match in RE_URL.finditer(text)):
         faup.decode(i)
         tokens = faup.get()
-        results.setdefault(tokens["domain"], []).append(tokens)
+        if tokens["domain"]:
+            results.setdefault(tokens["domain"], []).append(tokens)
     else:
         return results
 
diff --git a/src/options.py b/src/options.py
@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "2.2.0"
+__version__ = "2.2.1"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {
diff --git a/tests/samples/mail_test_7 b/tests/samples/mail_test_7
diff --git a/tests/test_attachments.py b/tests/test_attachments.py
@@ -17,6 +17,7 @@
 limitations under the License.
 """
 
+import logging
 import os
 import sys
 import unittest
@@ -51,6 +52,8 @@
 
 OPTIONS = ChainMap(os.environ, DEFAULTS)
 
+logging.getLogger().addHandler(logging.NullHandler())
+
 
 class TestAttachments(unittest.TestCase):
 
diff --git a/tests/test_attachments_post_processing.py b/tests/test_attachments_post_processing.py
@@ -17,6 +17,7 @@
 limitations under the License.
 """
 
+import logging
 import os
 import sys
 import unittest
@@ -45,6 +46,8 @@
 
 OPTIONS = ChainMap(os.environ, DEFAULTS)
 
+logging.getLogger().addHandler(logging.NullHandler())
+
 
 class TestPostProcessing(unittest.TestCase):
 
diff --git a/tests/test_attachments_utils.py b/tests/test_attachments_utils.py
@@ -17,6 +17,7 @@
 limitations under the License.
 """
 
+import logging
 import os
 import sys
 import unittest
@@ -34,6 +35,8 @@
 sample_txt = os.path.join(base_path, 'samples', 'test.txt')
 vt_report = os.path.join(base_path, 'samples', 'vt_report.json')
 
+logging.getLogger().addHandler(logging.NullHandler())
+
 
 class TestAttachmentsUtils(unittest.TestCase):
 
diff --git a/tests/test_bitmap.py b/tests/test_bitmap.py
@@ -17,6 +17,7 @@
 limitations under the License.
 """
 
+import logging
 import os
 import sys
 import unittest
@@ -28,6 +29,8 @@
 import src.modules.bitmap as bitmap
 from src.modules.bitmap import PhishingBitMap
 
+logging.getLogger().addHandler(logging.NullHandler())
+
 
 class ValidBitMap(bitmap.BitMap):
 
diff --git a/tests/test_network_post_processing.py b/tests/test_network_post_processing.py
@@ -17,6 +17,7 @@
 limitations under the License.
 """
 
+import logging
 import os
 import six
 import sys
@@ -40,6 +41,8 @@
 
 OPTIONS = ChainMap(os.environ, DEFAULTS)
 
+logging.getLogger().addHandler(logging.NullHandler())
+
 
 class TestPostProcessing(unittest.TestCase):
 
diff --git a/tests/test_phishing.py b/tests/test_phishing.py
diff --git a/tests/test_raw_mail_post_processing.py b/tests/test_raw_mail_post_processing.py
diff --git a/tests/test_redis.py b/tests/test_redis.py
diff --git a/tests/test_spamassassin.py b/tests/test_spamassassin.py
diff --git a/tests/test_utils.py b/tests/test_utils.py
diff --git a/topologies/spamscope_elasticsearch.py b/topologies/spamscope_elasticsearch.py
diff --git a/topologies/spamscope_redis.py b/topologies/spamscope_redis.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-(defproject spamscope "2.2.0-SNAPSHOT"`
	`1`	`+(defproject spamscope "2.2.1-SNAPSHOT"`
`2`	`2`	`:resource-paths ["_resources"]`
`3`	`3`	`:target-path "_build"`
`4`	`4`	`:min-lein-version "2.0.0"`