ENH: upgrade: replace url2fqn by url expert

the url2fqn expert is deprecated and will be removed in version 4
remove the bot from the default runtime config
add an upgrade function that replaces the bot in installations by the
url expert and parameters so that the behavior will stay the same

fixes #2430

Author: sebix

CHANGELOG.md

@@ -32,6 +32,7 @@
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
+- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).

intelmq/etc/runtime.yaml

@@ -156,16 +156,16 @@ taxonomy-expert:
   parameters:
     destination_queues:
       _default:
-      - url2fqdn-expert-queue
+      - url-expert-queue
   run_mode: continuous
-url2fqdn-expert:
-  bot_id: url2fqdn-expert
-  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+url-expert:
+  bot_id: url-expert
+  description: Extract additional information for the URL
   enabled: true
   group: Expert
   groupname: experts
-  module: intelmq.bots.experts.url2fqdn.expert
-  name: URL2FQDN
+  module: intelmq.bots.experts.url.expert
+  name: url
   parameters:
     destination_queues:
       _default:

intelmq/lib/upgrades.py

@@ -39,6 +39,7 @@
            'v310_feed_changes',
            'v310_shadowserver_feednames',
            'v320_update_turris_greylist_url',
+           'v322_url_replacement',
            ]
 
 
@@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     return ' '.join(messages) if messages else None, configuration, harmonization
 
 
+def v322_url_replacement(configuration, harmonization, dry_run, **kwargs):
+    """
+    Replace deprecated url2fqdn expert with url expert.
+    """
+    changed = False
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.experts.url2fqdn.expert":
+            configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert"
+            if "parameters" not in configuration[bot_id]:
+                configuration[bot_id]["parameters"] = {}
+            # skip all fields except the fqdn field for backwards compatibility
+            configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     ((3, 0, 2), ()),
     ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)),
     ((3, 2, 0), (v320_update_turris_greylist_url,)),
+    ((3, 2, 2), (v322_url_replacement, )),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/lib/test_upgrades.py

@@ -550,6 +550,29 @@
         "parameters": {}
     }
 }
+V322_URL2FQN_IN = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+        "parameters": {
+        }
+    },
+}
+V322_URL2FQN_IN_1 = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+    },
+}
+V322_URL2FQN_OUT = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url.expert",
+        "parameters": {
+            "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+        },
+    },
+}
 
 
 def generate_function(function):
@@ -762,6 +785,16 @@ def test_v310_feed_changes(self):
                          result[0])
         self.assertEqual(V310_FEED_CHANGES, result[1])
 
+    def test_v322_url_replacement(self):
+        """ Test v322_url_replacement """
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,