Merge pull request #2432 from sebix/fix-2430

ENH: upgrade: replace url2fqn by url expert

Author: sebix

CHANGELOG.md

@@ -32,6 +32,7 @@
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
+- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).

intelmq/etc/runtime.yaml

@@ -156,16 +156,16 @@ taxonomy-expert:
   parameters:
     destination_queues:
       _default:
-      - url2fqdn-expert-queue
+      - url-expert-queue
   run_mode: continuous
-url2fqdn-expert:
-  bot_id: url2fqdn-expert
-  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+url-expert:
+  bot_id: url-expert
+  description: Extract additional information for the URL
   enabled: true
   group: Expert
   groupname: experts
-  module: intelmq.bots.experts.url2fqdn.expert
-  name: URL2FQDN
+  module: intelmq.bots.experts.url.expert
+  name: url
   parameters:
     destination_queues:
       _default:

intelmq/lib/upgrades.py

@@ -39,6 +39,7 @@
            'v310_feed_changes',
            'v310_shadowserver_feednames',
            'v320_update_turris_greylist_url',
+           'v322_url_replacement',
            ]
 
 
@@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     return ' '.join(messages) if messages else None, configuration, harmonization
 
 
+def v322_url_replacement(configuration, harmonization, dry_run, **kwargs):
+    """
+    Replace deprecated url2fqdn expert with url expert.
+    """
+    changed = False
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.experts.url2fqdn.expert":
+            configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert"
+            if "parameters" not in configuration[bot_id]:
+                configuration[bot_id]["parameters"] = {}
+            # skip all fields except the fqdn field for backwards compatibility
+            configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     ((3, 0, 2), ()),
     ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)),
     ((3, 2, 0), (v320_update_turris_greylist_url,)),
+    ((3, 2, 2), (v322_url_replacement, )),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/lib/test_upgrades.py

@@ -550,6 +550,29 @@
         "parameters": {}
     }
 }
+V322_URL2FQN_IN = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+        "parameters": {
+        }
+    },
+}
+V322_URL2FQN_IN_1 = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+    },
+}
+V322_URL2FQN_OUT = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url.expert",
+        "parameters": {
+            "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+        },
+    },
+}
 
 
 def generate_function(function):
@@ -762,6 +785,16 @@ def test_v310_feed_changes(self):
                          result[0])
         self.assertEqual(V310_FEED_CHANGES, result[1])
 
+    def test_v322_url_replacement(self):
+        """ Test v322_url_replacement """
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,