Merge branch 'master' into certat

sebix · sebix · commit 1070e494ae8a · 2015-10-27T15:23:34.000+01:00
Conflicts:
	docs/Harmonization-fields.md
diff --git a/docs/Data-Harmonization.md b/docs/Data-Harmonization.md
@@ -1,11 +1,12 @@
 ## Table of Contents
 
 1. [Overview](#overview)
-2. [Sections](#sections)
-2. [Data types](#basicdatatypes)
-3. [List of known fields](#fields)
-4. [Type/Taxonomy Mapping](#mapping)
-5. [Minimum required fields](#requirements)
+2. [Rules for keys](#rules)
+3. [Sections](#sections)
+4. [Data types](#basicdatatypes)
+5. [List of known fields](#fields)
+6. [Type/Taxonomy Mapping](#mapping)
+7. [Minimum required fields](#requirements)
 
 
 <a name="overview"></a>
@@ -19,6 +20,7 @@ Every event **MUST** contain a timestamp field.
 
 [IOC](https://en.wikipedia.org/wiki/Indicator_of_compromise) (Indicator of compromise) is a single observation like a log line.
 
+<a name="rules"></a>
 
 ## Rules for keys
 
diff --git a/docs/Developers-Guide.md b/docs/Developers-Guide.md
@@ -324,7 +324,6 @@ Most existing bots are only tested with one message. For newly written test it i
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import json
 import unittest
 
 import intelmq.lib.test as test
diff --git a/docs/Harmonization-fields.md b/docs/Harmonization-fields.md
@@ -108,6 +108,11 @@ Sanitation accepts string 'true' and 'false' and integers 0 and 1.
 
 ### FQDN
 
+Fully qualified domain name type.
+
+All valid domains are accepted, no IP addresses or URLs. Trailing dot is
+not allowed.
+
 
 
 ### Float
diff --git a/docs/User-Guide.md b/docs/User-Guide.md
@@ -115,6 +115,7 @@ useradd -d /opt/intelmq -U -s /bin/bash intelmq
 echo 'export PATH="$PATH:$HOME/bin"' > /opt/intelmq/.profile
 chmod -R 0770 /opt/intelmq
 chown -R intelmq.intelmq /opt/intelmq
+echo 'export INTELMQ_PYTHON=/usr/bin/python3.4' >> /opt/intelmq/.profile
 ```
 
 <a name="install-python27"></a>
@@ -212,6 +213,13 @@ logged to /opt/intelmq/var/log/intelmqctl
 The botnet represents all currently configured bots. To get an overview which
 bots are running, use `intelmqctl -n status`.
 
+### Start bots with non-default Python
+
+The python version/path can be specified by the `INTELMQ_PYTHON` environment variable. By default it's the default python binary. This can be used to start the bots with current Python (version 3), while the default Python version for the operating system is still Legacy Python (version 2).
+
+    $ export INTELMQ_PYTHON=/usr/bin/python3.4
+	$ intelmqctl -n start
+
 ## Utilities
 
 ### Inspecting dumped messages
diff --git a/intelmq/bin/intelmqctl b/intelmq/bin/intelmqctl
@@ -46,7 +46,7 @@ ERROR_MESSAGES = {
     'running': '{} is still running.',
     'stopped': '{} was NOT RUNNING.',
     'stopping': '{} failed to STOP.',
-    'noid': 'No bot ID was given, use --bot-id',
+    'noid': 'No or unconfigured ID was given, use --bot-id',
     'notfound': '{} not found.'
 }
 
@@ -305,7 +305,14 @@ class IntelMQContoller():
         return self.bot_status(bot_id)
 
     def __bot_start(self, bot_id, module):
-        cmd = "python -m {} {}".format(module, bot_id)
+        """
+        Start a bot by calling it as module.
+
+        The python version/path can be specified by the INTELMQ_PYTHON
+        environment variable. By default it's the default python binary.
+        """
+        cmd = "{} -m {} {}".format(os.getenv('INTELMQ_PYTHON', 'python'),
+                                   module, bot_id)
         pid = start_process(bot_id, cmd)
         write_pidfile(bot_id, pid)
 
diff --git a/intelmq/bin/intelmqdump b/intelmq/bin/intelmqdump
@@ -17,7 +17,7 @@ import intelmq.lib.message as message
 import intelmq.lib.pipeline as pipeline
 import intelmq.lib.utils as utils
 from intelmq import DEFAULT_LOGGING_PATH, DEFAULTS_CONF_FILE, RUNTIME_CONF_FILE
-from termstyle import bold, inverted, red
+from termstyle import bold, green, inverted, red
 
 if sys.version_info[0] == 2:
     input = raw_input
@@ -72,25 +72,28 @@ AVAILABLE_IDS = [key for key, value in ACTIONS.items() if value[1]]
 
 def dump_info(fname):
     info = red('unknwon error')
-    try:
-        handle = io.open(fname, 'rt')
-    except OSError as exc:
-        info = red('unable to open file: {!s}'.format(exc))
+    if not os.path.getsize(fname):
+        info = 'empty file'
     else:
         try:
-            content = json.load(handle)
-        except ValueError as exc:
-            info = red('unable to load JSON: {!s}'.format(exc))
+            handle = io.open(fname, 'rt')
+        except OSError as exc:
+            info = red('unable to open file: {!s}'.format(exc))
         else:
             try:
-                info = "{!s} dumps".format(len(content.keys()))
-            except AttributeError as exc:
-                info = red("unable to count dumps: {!s}".format(exc))
-    finally:
-        try:
-            handle.close()
-        except NameError:
-            pass
+                content = json.load(handle)
+            except ValueError as exc:
+                info = red('unable to load JSON: {!s}'.format(exc))
+            else:
+                try:
+                    info = "{!s} dumps".format(len(content.keys()))
+                except AttributeError as exc:
+                    info = red("unable to count dumps: {!s}".format(exc))
+        finally:
+            try:
+                handle.close()
+            except NameError:
+                pass
     return info
 
 
@@ -124,6 +127,9 @@ if __name__ == '__main__':
 
     if args.botid is None:
         filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
+        if not len(filenames):
+            print(green('Nothing to recover from, no dump files found!'))
+            exit(0)
         filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
                      for fname in sorted(filenames)]
 
diff --git a/intelmq/bots/experts/deduplicator/expert.py b/intelmq/bots/experts/deduplicator/expert.py
@@ -7,7 +7,7 @@
 from intelmq.lib.cache import Cache
 
 
-class DeduplicatorBot(Bot):
+class DeduplicatorExpertBot(Bot):
 
     def init(self):
         self.cache = Cache(self.parameters.redis_cache_host,
@@ -42,5 +42,5 @@ def process(self):
 
 
 if __name__ == "__main__":
-    bot = DeduplicatorBot(sys.argv[1])
+    bot = DeduplicatorExpertBot(sys.argv[1])
     bot.start()
diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py
@@ -7,6 +7,7 @@
 import datetime
 import json
 import re
+import sys
 import time
 import traceback
 
@@ -33,10 +34,11 @@ def __init__(self, bot_id):
         self.logger = None
 
         try:
-            self.log_buffer.append(('debug',
-                                    '{} initialized with id {}.'
+            version_info = sys.version.splitlines()[0].strip()
+            self.log_buffer.append(('info',
+                                    '{} initialized with id {} and version {}.'
                                     ''.format(self.__class__.__name__,
-                                              bot_id)))
+                                              bot_id, version_info)))
             self.check_bot_id(bot_id)
             self.bot_id = bot_id
 
diff --git a/intelmq/lib/harmonization.py b/intelmq/lib/harmonization.py
@@ -266,6 +266,12 @@ def sanitize(value):
 
 
 class FQDN(GenericType):
+    """
+    Fully qualified domain name type.
+
+    All valid domains are accepted, no IP addresses or URLs. Trailing dot is
+    not allowed.
+    """
 
     @staticmethod
     def is_valid(value, sanitize=False):
@@ -285,8 +291,15 @@ def is_valid(value, sanitize=False):
         if not len(value.split('.')) > 1:
             return False
 
+        if value[-1] == '.':
+            return False
+
         return True
 
+    @staticmethod
+    def sanitize(value):
+        return value.rstrip('.')
+
     @staticmethod
     def to_ip(value):
         try:
diff --git a/intelmq/lib/message.py b/intelmq/lib/message.py
@@ -25,6 +25,23 @@ class MessageFactory(object):
     serialize: object to JSON encoded object
     """
 
+    @staticmethod
+    def from_dict(message):
+        """
+        Takes dictionary Message object, returns instance of correct class.
+
+        The class is determined by __type attribute.
+        """
+        try:
+            class_reference = getattr(intelmq.lib.message, message["__type"])
+        except AttributeError:
+            raise exceptions.InvalidArgument('__type',
+                                             got=message["__type"],
+                                             expected=list(harm_config.keys()),
+                                             docs=HARMONIZATION_CONF_FILE)
+        del message["__type"]
+        return class_reference(message)
+
     @staticmethod
     def unserialize(raw_message):
         """
diff --git a/intelmq/lib/test.py b/intelmq/lib/test.py
@@ -155,6 +155,8 @@ class Parameters(object):
             for msg in self.input_message:
                 if type(msg) is dict:
                     self.input_queue.append(json.dumps(msg))
+                elif issubclass(type(msg), message.Message):
+                    self.input_queue.append(msg.serialize())
                 else:
                     self.input_queue.append(msg)
             self.input_message = None
@@ -207,9 +209,10 @@ def test_bot_start(self):
     def test_log_init(self):
         """ Test if bot logs initialized message. """
         self.run_bot()
-        self.assertLoglineEqual(0, "{} initialized with id {}."
-                                   "".format(self.bot_name,
-                                             self.bot_id), "DEBUG")
+        self.assertLoglineMatches(0, "{} initialized with id {} and version"
+                                     " [0-9.]{{5}} \([a-zA-Z, 0-9:]+\)."
+                                     "".format(self.bot_name,
+                                               self.bot_id), "INFO")
 
     def test_log_starting(self):
         """ Test if bot logs starting message. """
@@ -327,6 +330,24 @@ def assertLoglineEqual(self, line_no, message, levelname="ERROR"):
         self.assertEqual(levelname, fields["levelname"])
         self.assertEqual(message, fields["message"])
 
+    def assertLoglineMatches(self, line_no, pattern, levelname="ERROR"):
+        """Asserts if a logline matches a specific requirement.
+           Args:
+                line_no: Number of the logline which is asserted
+                pattern: Message text which is compared
+                type: Type of logline which is asserted"""
+
+        self.assertIsNotNone(self.loglines)
+        logline = self.loglines[line_no]
+        fields = utils.parse_logline(logline)
+
+        self.assertEqual(self.bot_id, fields["name"],
+                         "bot_id %s didn't match %s"
+                         "".format(self.bot_id, fields["name"]))
+
+        self.assertEqual(levelname, fields["levelname"])
+        self.assertRegexpMatches(fields["message"], pattern)
+
     def assertRegexpMatchesLog(self, pattern):
         """Asserts that pattern matches against log. """
 
@@ -351,7 +372,7 @@ def assertOutputQueueLen(self, queue_len=0):
         """
         self.assertEqual(len(self.get_output_queue()), queue_len)
 
-    def assertMessageEqual(self, queue_pos, expected_message):
+    def assertMessageEqual(self, queue_pos, expected_msg):
         """
         Asserts that the given expected_message is
         contained in the generated event with
@@ -360,9 +381,13 @@ def assertMessageEqual(self, queue_pos, expected_message):
         event = self.get_output_queue()[queue_pos]
 
         self.assertIsInstance(event, six.text_type)
-        event_dict = json.loads(event)
+        event_dict = message.MessageFactory.from_dict(json.loads(event))
 
-        expected = expected_message.copy()
+        if type(expected_msg) is dict:
+            expected = message.MessageFactory.from_dict(expected_msg.copy())
+        else:
+            expected = expected_msg.copy()
+            del event_dict['__type']
         del event_dict['time.observation']
         del expected['time.observation']
 
diff --git a/intelmq/tests/bots/experts/deduplicator/test_expert.py b/intelmq/tests/bots/experts/deduplicator/test_expert.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import redis
+import unittest
+
+import intelmq.lib.harmonization as harmonization
+import intelmq.lib.message as message
+import intelmq.lib.test as test
+from intelmq.bots.experts.deduplicator.expert import DeduplicatorExpertBot
+
+
+INPUT1 = {"__type": "Event",
+          "classification.identifier": "zeus",
+          "source.ip": "192.0.2.1",
+          "time.observation": harmonization.DateTime.generate_datetime_now(),
+          }
+INPUT2 = INPUT1.copy()
+INPUT2['source.ip'] = '192.168.0.4'
+
+
+class TestDeduplicatorExpertBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for DeduplicatorExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = DeduplicatorExpertBot
+        cls.default_input_message = INPUT1
+        cls.sysconfig = {"redis_cache_ttl": "86400",
+                         "ignore_keys": "raw ,time.observation "}
+        cls.redis = redis.Redis(host=test.BOT_CONFIG['redis_cache_host'],
+                                port=test.BOT_CONFIG['redis_cache_port'],
+                                db=test.BOT_CONFIG['redis_cache_db'],
+                                socket_timeout=5)
+        cls.redis.flushdb()
+
+    def test_suppress(self):
+        msg = message.MessageFactory.from_dict(INPUT1)
+        msg_hash = hash(msg)
+        self.redis.set(msg_hash, 'hash')
+        self.redis.expire(msg_hash, 3600)
+        self.input_message = msg
+        self.run_bot()
+        self.assertOutputQueueLen()
+
+    def test_pass(self):
+        msg = message.MessageFactory.from_dict(INPUT2)
+        self.input_message = msg
+        self.run_bot()
+        self.assertMessageEqual(0, msg)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.redis.flushdb()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/intelmq/tests/bots/experts/reverse_dns/test_expert.py b/intelmq/tests/bots/experts/reverse_dns/test_expert.py
diff --git a/intelmq/tests/bots/parsers/malwaredomainlist/test_parser.py b/intelmq/tests/bots/parsers/malwaredomainlist/test_parser.py
diff --git a/intelmq/tests/lib/test_harmonization.py b/intelmq/tests/lib/test_harmonization.py
diff --git a/intelmq/tests/lib/test_message.py b/intelmq/tests/lib/test_message.py
