Merge pull request #2658 from sebix/revert-viriback

Revert "feeds: remove viriback feed, is offline"

Author: sebix

CHANGELOG.md

@@ -95,7 +95,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 ### Documentation
 - Fix and refresh links to mailing lists (PR#2609 by Kamil Mańkowski)
 - `Aggregate Bot`: Add illustration graphics (PR#2612 by Sebastian Wagner).
-- Feeds: Remove discontinued feed Viriback (PR#2567 by Sebastian Wagner).
 
 ### Packaging
 - Replace `/opt/intelmq` example paths in bots with variable `VAR_STATE_PATH` for correct paths in LSB-path setups like with packages (PR#2587 by Sebastian Wagner).

NEWS.md

@@ -18,13 +18,6 @@ Python `>=3.9` is now required, which is available on all platforms supported by
 
 ### Tools
 
-### Bots
-#### Blueliv
-The bots `intelmq.bots.collectors.blueliv` and `intelmq.bots.collectors.blueliv` are removed as they used an unmaintained library and do not work any more.
-
-#### Viriback
-The Feed *Viriback C2 Tracker* is removed as the feed and website are no longer reachable and seem to be discontinued.
-
 ### Data Format
 To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
 update is necessary:

intelmq/etc/feeds.yaml

@@ -4,6 +4,31 @@
 
 ---
 providers:
+  ViriBack:
+    C2 Tracker:
+      description: Latest detected C2 servers.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: https://tracker.viriback.com/dump.php
+            rate_limit: 86400
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.generic.csv_parser
+          parameters:
+            skip_header: true
+            defaults_fields:
+              classification.type: malware-distribution
+            columns:
+              - malware.name
+              - source.url
+              - source.ip
+              - time.source
+      revision: 2022-11-15
+      documentation: https://viriback.com/
+      public: true
   Surbl:
     Malicious Domains:
       description: Detected malicious domains. Note that you have to opened up Sponsored Datafeed Service (SDS) access to the SURBL data via rsync for your IP address.

intelmq/lib/upgrades.py

@@ -42,7 +42,7 @@
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
            'v340_deprecations',
-           'v350_feed_removals',
+           'v350_blueliv_removal',
            'v350_new_fields',
            ]
 
@@ -976,37 +976,27 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
-def v350_feed_removals(configuration, harmonization, dry_run, **kwargs):
+def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
     """
     Remove blueliv collector and parser
     """
-    messages = []
+    message = None
     discontinued_bots = []
     discontinued_bots_modules = (
         "intelmq.bots.collectors.blueliv.collector_crimeserver",
         "intelmq.bots.parsers.blueliv.parser_crimeserver",
     )
-    discontinued_feeds = []
 
     for bot_id, bot in configuration.items():
         if bot_id == 'global':
             continue
         if bot["module"] in discontinued_bots_modules:
             discontinued_bots.append(bot_id)
-        elif bot["module"] == "intelmq.bots.collectors.http.collector_http":
-            if bot["parameters"].get("http_url", "") == 'https://tracker.viriback.com/dump.php':
-                discontinued_feeds.append(bot_id)
-
-    if discontinued_feeds:
-        messages.append(f"Found discontinued feeds collected by bots: {', '.join(discontinued_feeds)}")
 
     if discontinued_bots:
-        messages.append(f"Found discontinued bots: {', '.join(discontinued_bots)}.")
-
-    if messages:
-        messages.append("Remove the affected bots from the configuration.")
+        message = f"Found discontinued bots: {', '.join(discontinued_bots)}. Remove the affected bots from the configuration."
 
-    return '\n'.join(messages) if messages else None, configuration, harmonization
+    return message, configuration, harmonization
 
 
 def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
@@ -1068,7 +1058,7 @@ def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 5, 0), (v350_feed_removals, v350_new_fields)),
+    ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/lib/test_upgrades.py

@@ -616,23 +616,18 @@
         "module": "intelmq.bots.collectors.twitter.collector",
     },
 }
-V350_FEED_REMOVAL = {
+V350_BLUELIV_REMOVAL = {
     "global": {},
     "blueliv-collector": {
         "module": "intelmq.bots.collectors.blueliv.collector_crimeserver"
     },
     "blueliv-parser": {
         "module": "intelmq.bots.parsers.blueliv.parser_crimeserver"
-    },
-    "viriback-collector": {
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "parameters": {
-            "http_url": "https://tracker.viriback.com/dump.php"
-        }
     }
 }
 
 
+
 def generate_function(function):
     def test_function(self):
         """ Test if no errors happen for upgrade function %s. """ % function.__name__
@@ -870,13 +865,12 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
-    def test_v350_feed(self):
-        """ Test v350_feed_removals deprecation warning """
-        result = upgrades.v350_feed_removals(V350_FEED_REMOVAL, {}, False)
+    def test_v350_blueliv_removal(self):
+        """ Test v350_blueliv_removal deprecation warning """
+        result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False)
         self.assertIn('blueliv-collector', result[0])
         self.assertIn('blueliv-parser', result[0])
-        self.assertIn('viriback-collector', result[0])
-        self.assertEqual(V350_FEED_REMOVAL, result[1])
+        self.assertEqual(V350_BLUELIV_REMOVAL, result[1])
 
     def test_v350_new_fields(self):
         """ Test adding new harmonisation fields """
@@ -892,6 +886,7 @@ def test_v350_new_fields(self):
         self.assertIn("severity", result[2]["event"])
 
 
+
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,
             generate_function(getattr(upgrades, name)))