ENH: add http status code expert

This adds the module
intelmq.bots.experts.http.expert_status.HttpStatusExpertBot and some
unit tests for the module.
It also adds a description in the documentation and a changelog entry
introducing the new bot.

Author: Birger Schacht

CHANGELOG.md

@@ -44,6 +44,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
 
 #### Experts
 - `intelmq.bots.experts.domain_suffix.expert`: Added `--update-database` option to update domain suffix database.
+- Added `intelmq.bots.experts.http.expert_status`: A bot that fetches the HTTP Status for a given URI and adds it to the message.
 
 #### Outputs
 - Remove `intelmq.bots.outputs.xmpp`: one of the dependencies of the bot was deprecated and according to a short survey on the IntelMQ

docs/user/bots.rst

@@ -2041,6 +2041,21 @@ Other errors result in an exception if not ignored by the parameter `gaierrors_t
 All gaierrors can be found here: http://www.castaglia.org/proftpd/doc/devel-guide/src/lib/glibc-gai_strerror.c.html
 
 
+HTTP Status
+^^^^^^^^^^^
+
+Fetches the HTTP Status for a given URI
+
+**Information**
+
+* `name:` intelmq.bots.experts.http.expert_status
+* `description:` The bot fetches the HTTP status for a given URL and saves it in the event.
+
+**Configuration Parameters**
+
+* `field:` The name of the field containing the URL to be checked (required).
+* `success_status_codes:` A list of success status codes. If this parameter is omitted or the list is empty, successful status codes are the ones between 200 and 400.
+* `overwrite:` Specifies if an existing 'status' value should be overwritten.
 
 IDEA Converter
 ^^^^^^^^^^^^^^

intelmq/bots/BOTS

@@ -844,6 +844,15 @@
                 "overwrite": false
             }
         },
+        "HTTP Status": {
+            "description": "Get the HTTP Status for an URL.",
+            "module": "intelmq.bots.experts.http.expert_status",
+            "parameters": {
+                "field": "source.url",
+                "success_status_codes": [],
+                "overwrite": false
+            }
+        },
         "IDEA Converter": {
             "description": "Convert events into the IDEA format.",
             "module": "intelmq.bots.experts.idea.expert",

intelmq/bots/experts/http/__init__.py

@@ -0,0 +1,58 @@
+"""
+HTTP Status Expert Bot
+
+SPDX-FileCopyrightText: 2021 Birger Schacht <[email protected]>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+from typing import List
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.utils import create_request_session
+
+
+class HttpStatusExpertBot(Bot):
+    """
+    Fetch the HTTP Status for a given URL
+
+    Parameters
+    ----------
+    field: str
+        The name of the field containing the URL to be checked (defaults to 'source.url').
+    success_status_codes: List
+        A list of success status codes. If this parameter is omitted or the list is empty,
+        successful status codes are the ones between 200 and 400.
+    overwrite:
+        Specifies if an existing 'status' value should be overwritten.
+    """
+    field: str = "source.url"  # The field containing the URL
+    success_status_codes: List[int] = []  # A list of status codes for success
+    overwrite: bool = True
+
+    def process(self):
+        event = self.receive_message()
+
+        if self.field in event:
+            self.set_request_parameters()
+            session = create_request_session(self)
+
+            try:
+                response = session.get(event[self.field])
+                # If success_status_codes are configured, we use those
+                # to check the success of the request, otherwise we
+                # rely on the boolean value of the response
+                if (self.success_status_codes and response.status_code in self.success_status_codes) or (response):
+                    event.add('status', "online", overwrite=self.overwrite)
+                else:
+                    event.add('status', 'offline', overwrite=self.overwrite)
+                    event.add('extra.reason', response.reason)
+            except Exception as exc:
+                event.add('status', 'offline', overwrite=self.overwrite)
+                event.add('extra.reason', str(exc))
+        else:
+            self.logger.debug('Field %s was not part of the message.', self.field)
+
+        self.send_message(event)
+        self.acknowledge_message()
+
+
+BOT = HttpStatusExpertBot

intelmq/bots/experts/http/expert_status.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Testing HTTP Status expert
+"""
+import unittest
+import requests_mock
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.experts.http.expert_status import HttpStatusExpertBot
+
+EXAMPLE_INPUT1 = {"__type": "Event",
+                 "source.url": "http://localhost/foo",
+                 "destination.url": "http://localhost/bar",
+                 }
+EXAMPLE_OUTPUT1 = {"__type": "Event",
+                 "source.url": "http://localhost/foo",
+                 "destination.url": "http://localhost/bar",
+                 "status": "online",
+                 }
+EXAMPLE_OUTPUT2 = {"__type": "Event",
+                 "source.url": "http://localhost/foo",
+                 "destination.url": "http://localhost/bar",
+                 "status": "offline",
+                 }
+
+def prepare_mocker(mocker):
+    mocker.get('http://localhost/foo', status_code=200)
+    mocker.get('http://localhost/bar', status_code=404)
+
+
+@requests_mock.Mocker()
+class TestHTTPStatusExpertBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for HTTPStatusExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = HttpStatusExpertBot
+
+    def test_status_200(self, mocker):
+        """ Test with HTTP Status code 200. """
+        prepare_mocker(mocker)
+        self.input_message = EXAMPLE_INPUT1
+        self.run_bot()
+        self.assertMessageEqual(0, EXAMPLE_OUTPUT1)
+
+    def test_status_404(self, mocker):
+        """ Test with HTTP Status code 404 and a custom field. """
+        prepare_mocker(mocker)
+        self.input_message = EXAMPLE_INPUT1
+        self.prepare_bot(parameters={'field': 'destination.url'})
+        self.run_bot(prepare=False)
+        self.assertMessageEqual(0, EXAMPLE_OUTPUT2)
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()