Merge remote-tracking branch 'upstream/pr/1751' into develop

Author: Sebastian Wagner

CHANGELOG.md

@@ -11,8 +11,11 @@ CHANGELOG
   - New class `ClassificationTaxonomy` with fixed list of taxonomies and sanitiation
 - `intelmq.lib.bot`:
   - Handle `InvalidValue` exceptions upon message retrieval by dumping the message instead of repeating endlessly (#1765, PR#1766 by Filip Pokorný).
+  - Rewrite of the parameter loading and handling, getting rid of the `parameters` member (PR#1729 by Birger Schacht).
 - `intelmq.lib.exceptions`:
   - `InvalidValue`: Add optional parameter `object` (PR#1766 by Filip Pokorný).
+- `intelmq.lib.utils`:
+  - New function `list_all_bots` to list all available/installed bots as replacement for the BOTS file (#368, #552, #644, #757, #1069, #1750, PR#1751 by Sebastian Waldbauer).
 
 ### Development
 
@@ -36,6 +39,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
   - The type `vulnerable service` has been renamed to `vulnerable-system`.
 
 ### Bots
+- The parameters handling of numerous bots has been refactored (PR#1751, PR#1729, by Birger Schacht, Sebastian Wagner, Sebastian Waldbauer).
+
 #### Collectors
 - Remove `intelmq.bots.collectors.xmpp`: one of the dependencies of the bot was deprecated and according to a short survey on the IntelMQ
   users mailinglist, the bot is not used by anyone. (https://lists.cert.at/pipermail/intelmq-users/2020-October/000177.html, PR#1761, closes #1614)

intelmq/__init__.py

@@ -28,6 +28,5 @@
 HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
 PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf")
 RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf")
-BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS")
 STATE_FILE_PATH = path = os.path.abspath(os.path.join(VAR_STATE_PATH,
                                                       '../state.json'))

intelmq/bin/intelmqctl.py

@@ -22,7 +22,7 @@
 import pkg_resources
 from termstyle import green
 
-from intelmq import (BOTS_FILE, DEFAULT_LOGGING_LEVEL, DEFAULTS_CONF_FILE,  # noqa: F401
+from intelmq import (DEFAULT_LOGGING_LEVEL, DEFAULTS_CONF_FILE,  # noqa: F401
                      HARMONIZATION_CONF_FILE, PIPELINE_CONF_FILE,
                      RUNTIME_CONF_FILE, VAR_RUN_PATH, STATE_FILE_PATH,
                      DEFAULT_LOGGING_PATH, __version_info__,
@@ -865,6 +865,8 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
                                      help='Only show the total '
                                           'number of messages in queues. '
                                           'Only valid for listing queues.')
+            parser_list.add_argument('--configured', '-c', action='store_true',
+                                     help='Only show configured bots')
             parser_list.set_defaults(func=self.list)
 
             parser_clear = subparsers.add_parser('clear', help='Clear a queue')
@@ -1220,11 +1222,11 @@ def botnet_status(self, group=None):
                 retval = 1
         return retval, botnet_status
 
-    def list(self, kind=None, non_zero=False, count=False):
+    def list(self, kind=None, non_zero=False, count=False, configured=False):
         if kind == 'queues':
             return self.list_queues(non_zero=non_zero, count=count)
         elif kind == 'bots':
-            return self.list_bots(non_zero=non_zero)
+            return self.list_bots(non_zero=non_zero, configured=configured)
         elif kind == 'queues-and-status':
             q = self.list_queues()
             b = self.botnet_status()
@@ -1243,25 +1245,29 @@ def write_updated_runtime_config(self, filename=RUNTIME_CONF_FILE):
             self.abort('Can\'t update runtime configuration: Permission denied.')
         return True
 
-    def list_bots(self, non_zero=False):
+    def list_bots(self, non_zero=False, configured=False):
         """
-        Lists all configured bots from runtime.conf with bot id and
-        description.
+        Lists all (configured) bots from runtime.conf or generated on demand
+        with bot id/module and description and parameters.
 
         If description is not set, None is used instead.
         """
-        if RETURN_TYPE == 'text':
-            for bot_id in sorted(self.runtime_configuration.keys(), key=str.lower):
-                if non_zero and not self.runtime_configuration[bot_id].get('enabled'):
-                    continue
-                if QUIET:
-                    print(bot_id)
-                else:
-                    print("Bot ID: {}\nDescription: {}"
-                          "".format(bot_id, self.runtime_configuration[bot_id].get('description')))
-        return 0, [{'id': bot_id,
-                    'description': self.runtime_configuration[bot_id].get('description')}
-                   for bot_id in sorted(self.runtime_configuration.keys())]
+        if configured:
+            if RETURN_TYPE == 'text':
+                for bot_id in sorted(self.runtime_configuration.keys(), key=str.lower):
+                    if non_zero and not self.runtime_configuration[bot_id].get('enabled'):
+                        continue
+                    if QUIET:
+                        print(bot_id)
+                    else:
+                        print("Bot ID: {}\nDescription: {}"
+                              "".format(bot_id, self.runtime_configuration[bot_id].get('description')))
+            return 0, [{'id': bot_id,
+                        'description': self.runtime_configuration[bot_id].get('description')}
+                       for bot_id in sorted(self.runtime_configuration.keys())]
+        else:
+            val = utils.list_all_bots()
+            return 0, val
 
     def get_queues(self, with_internal_queues=False):
         """
@@ -1421,8 +1427,7 @@ def check(self, no_connections=False):
 
         # loading files and syntax check
         files = {DEFAULTS_CONF_FILE: None, PIPELINE_CONF_FILE: None,
-                 RUNTIME_CONF_FILE: None, BOTS_FILE: None,
-                 HARMONIZATION_CONF_FILE: None}
+                 RUNTIME_CONF_FILE: None, HARMONIZATION_CONF_FILE: None}
         check_logger.info('Reading configuration files.')
         for filename in files:
             try:
@@ -1555,7 +1560,7 @@ def check(self, no_connections=False):
             if bot_check:
                 for log_line in bot_check:
                     getattr(check_logger, log_line[0])("Bot %r: %s" % (bot_id, log_line[1]))
-        for group in files[BOTS_FILE].values():
+        for group in utils.list_all_bots():
             for bot_id, bot in group.items():
                 if subprocess.call(['which', bot['module']], stdout=subprocess.DEVNULL,
                                    stderr=subprocess.DEVNULL):
@@ -1874,7 +1879,7 @@ def debug(self, sections=None):
             variables = globals()
             if RETURN_TYPE == 'text':
                 print('Paths:')
-            for path in ('BOTS_FILE', 'DEFAULTS_CONF_FILE',
+            for path in ('DEFAULTS_CONF_FILE',
                          'HARMONIZATION_CONF_FILE', 'PIPELINE_CONF_FILE',
                          'RUNTIME_CONF_FILE', 'VAR_RUN_PATH', 'STATE_FILE_PATH',
                          'DEFAULT_LOGGING_PATH', '__file__',

intelmq/bin/intelmqsetup.py

@@ -44,7 +44,7 @@
 
 from termstyle import red
 from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
-                     VAR_STATE_PATH, BOTS_FILE, STATE_FILE_PATH)
+                     VAR_STATE_PATH, STATE_FILE_PATH)
 from intelmq.bin.intelmqctl import IntelMQController
 
 
@@ -161,13 +161,6 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
         if ownership:
             change_owner(destination_file, owner='intelmq', group='intelmq', log=log_ownership_change)
 
-    if Path(BOTS_FILE).is_symlink():
-        print('Skip writing BOTS file as it is a link.')
-    else:
-        print('Writing BOTS file.')
-        shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
-                    BOTS_FILE)
-
     if ownership:
         print('Setting intelmq as owner for it\'s directories.')
         for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,

intelmq/bots/collectors/amqp/collector_amqp.py

@@ -20,7 +20,6 @@ class AMQPCollectorBot(AMQPTopicOutputBot, CollectorBot):
     Collect data from an AMQP Server and fetch either intelmq or any other messages. Requires the pika python library.
     Inheriting from AMQPTopicOutputBot for connect_server method
     """
-    exchange: bool = False
     connection_attempts: int = 3
     connection_heartbeat: int = 3600
     connection_host: str = "127.0.0.1"  # TODO should be ipaddress

intelmq/bots/collectors/api/collector_api.py

@@ -28,9 +28,9 @@ class APICollectorBot(CollectorBot):
     """Collect data by exposing a HTTP API interface"""
     name: str = "API"
     port: int = 5000
+    __collector_empty_process: bool = True
     provider: str = "APICollector"
-    collector_empty_process: bool = True
-    is_multithreadable: bool = False
+    __is_multithreadable: bool = False
 
     def init(self):
         if IOLoop is None:

intelmq/bots/collectors/github_api/collector_github_api.py renamed to intelmq/bots/collectors/github_api/_collector_github_api.py

@@ -12,7 +12,7 @@
 import re
 
 from intelmq.lib.exceptions import InvalidArgument
-from intelmq.bots.collectors.github_api.collector_github_api import GithubAPICollectorBot
+from intelmq.bots.collectors.github_api._collector_github_api import GithubAPICollectorBot
 
 try:
     import requests

intelmq/bots/collectors/github_api/collector_github_contents_api.py

@@ -29,7 +29,7 @@
 
 class HTTPStreamCollectorBot(CollectorBot):
     "Open a streaming connection to the URL and process data per line"
-    sighup_delay = False
+    _sighup_delay: bool = False
     http_password: str = None
     http_url: str = "<insert url of feed>"
     http_username: str = None