Merge branch 'develop' into gen_v_events

kamil-certat · web-flow · commit 4fbe7ca40823 · 2023-09-12T15:31:06.000+02:00
diff --git a/.github/workflows/ansible-tests.yml b/.github/workflows/ansible-tests.yml
@@ -33,4 +33,4 @@ jobs:
         path: ./src/intelmq-vagrant
 
     - name: Run ansible tests
-      run: podman run -e CI=${CI} -v ${GITHUB_WORKSPACE}/src:/src docker.io/debian:buster-slim /src/intelmq/.github/workflows/scripts/ansible.sh
+      run: podman run -e CI=${CI} -v ${GITHUB_WORKSPACE}/src:/src docker.io/debian:bullseye-slim /src/intelmq/.github/workflows/scripts/ansible.sh
diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml
@@ -23,7 +23,7 @@ jobs:
     name: Build Debian packages
     strategy:
       matrix:
-        codename: ['buster', 'bullseye']
+        codename: ['bullseye', 'bookworm']
 
     steps:
     - name: Checkout repository
diff --git a/.github/workflows/scripts/debian-package.sh b/.github/workflows/scripts/debian-package.sh
@@ -10,7 +10,7 @@ set -x
 set -e
 
 # A list of known Debian releases
-knowncodenames=("stretch" "buster" "bullseye" "stable" "testing")
+knowncodenames=("bullseye" "bookworm" "stable" "testing")
 
 # We want exactly one argument: the name of the release
 if (( $# != 1 ))
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,22 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+   install:
+   - requirements: docs/requirements.txt
diff --git a/.readthedocs.yaml.license b/.readthedocs.yaml.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2023 Institute for Common Good Technology <sebix@commongoodtechnology.org>
+SPDX-License-Identifier: AGPL-3.0-or-later
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,8 @@ CHANGELOG
 ### Configuration
 
 ### Core
+- `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner).
+- `intelmq.lib.exceptions.InvalidKey`: Add optional parameter `additional_text` (PR#2398 by Sebastian Wagner).
 
 ### Development
 
@@ -28,6 +30,7 @@ CHANGELOG
 #### Outputs
 
 ### Documentation
+- Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner).
 
 ### Packaging
 
@@ -143,12 +146,6 @@ This is short list of the most important known issues. The full list can be retr
 - Corrupt dump files when interrupted during writing (#870).
 
 
-3.0.2 (2021-09-10)
-------------------
-
-### Core
-- `intelmq.lib.bot.CollectorBot`: Fixed an issue with within the `new_report` function, which re-loads the harmonization file after a new incoming dataset, which leads to CPU drain and decreased performance (PR#2106 by Sebastian Waldbauer, fixes #2098).
-
 3.1.0 (2023-02-10)
 ------------------
 
diff --git a/debian/changelog b/debian/changelog
@@ -1,8 +1,14 @@
 intelmq (3.2.2-1) UNRELEASED; urgency=medium
 
-  *
+  * 
 
- -- Sebastian Wagner <sebix@sebix.at>  Mon, 28 Aug 2023 17:41:52 +0200
+ -- Sebastian Wagner <sebix@sebix.at>  Tue, 05 Sep 2023 15:05:07 +0200
+
+intelmq (3.2.1-2) stable; urgency=medium
+
+  * Fix build on Debian 12
+
+ -- Sebastian Wagner <sebix@sebix.at>  Tue, 05 Sep 2023 15:04:36 +0200
 
 intelmq (3.2.1-1) stable; urgency=medium
 
diff --git a/debian/control b/debian/control
@@ -1,23 +1,23 @@
 Source: intelmq
-Maintainer: IntelMQ Team <intelmq-team@cert.at>
+Maintainer: Institute for Common Good Technology <packaging@commongoodtechnology.org>
 Section: python
 Priority: optional
 Build-Depends: debhelper (>= 4.1.16),
                dh-python,
                lsb-release,
                python3-all,
                python3-cerberus,
-               python3-dateutil,
-               python3-dnspython,
-               python3-psutil,
-               python3-redis,
-               python3-requests,
+               python3-dateutil (>= 2.5),
+               python3-dnspython (>= 2.0.0),
+               python3-psutil (>= 1.2.1),
+               python3-redis (>= 2.10),
+               python3-requests (>= 2.2.0),
                python3-requests-mock,
                python3-ruamel.yaml,
                python3-setuptools,
                python3-sphinx,
                python3-sphinx-rtd-theme,
-               python3-termstyle,
+               python3-termstyle (>= 0.1.10),
                python3-tz,
                quilt,
                rsync,
@@ -37,7 +37,7 @@ Depends: bash-completion,
          python3-openssl,
          python3-psutil (>= 1.2.1),
          python3-redis (>= 2.10),
-         python3-requests (>= 2.2.1),
+         python3-requests (>= 2.2.0),
          python3-ruamel.yaml,
          python3-termstyle (>= 0.1.10),
          python3-tz,
@@ -48,9 +48,9 @@ Depends: bash-completion,
          ${python3:Depends}
 Suggests: python3-geoip2 (>= 2.2.0),
           python3-imbox (>= 0.8),
-          python3-psycopg2,
+          python3-psycopg2 (>= 2.5.5),
           python3-pyasn (>= 1.5.0),
-          python3-pymongo,
+          python3-pymongo (>= 2.7.1),
           python3-sleekxmpp (>= 1.3.1),
           python3-stomp.py (>= 4.1.9)
 Description: Solution for IT security teams for collecting and processing security feeds
@@ -61,3 +61,11 @@ Description: Solution for IT security teams for collecting and processing securi
  designed by European CERTs/CSIRTs during several InfoSec events. Its main goal
  is to give to incident responders an easy way to collect & process threat
  intelligence thus improving the incident handling processes of CERTs.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
diff --git a/debian/rules b/debian/rules
@@ -36,7 +36,7 @@ override_dh_auto_install: $(BOTDOCS)
 	rm intelmq/bin/rewrite_config_files.py
 	sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py
 	sed -i -f debian/sedfile intelmq/etc/* docs/user/intelmqctl.rst docs/user/bots.rst setup.py contrib/logrotate/intelmq contrib/logcheck/logcheck.logfiles
-	python3 setup.py install --root=debian/intelmq --prefix=/usr
+	dh_auto_install
 	# create directories
 	mkdir -p debian/intelmq/var/log/intelmq
 	mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output
diff --git a/docs/user/installation.rst b/docs/user/installation.rst
@@ -30,7 +30,7 @@ The following instructions assume the following requirements. Python versions >=
 
 Supported and recommended operating systems are:
 
-* Debian 10 Buster, Debian 11 Bullseye, Debian 12 Bookworm
+* Debian Debian 11 Bullseye, Debian 12 Bookworm
 * openSUSE Tumbleweed, Leap 15.5
 * Ubuntu: 20.04 Focal, 22.04 Jammy
 * For the Docker-installation: Docker Engine: 18.x and higher
diff --git a/intelmq/bots/collectors/github_api/REQUIREMENTS.txt b/intelmq/bots/collectors/github_api/REQUIREMENTS.txt
diff --git a/intelmq/lib/exceptions.py b/intelmq/lib/exceptions.py
@@ -88,8 +88,8 @@ def __init__(self, key: str, value: str, reason: Any = None, object: bytes = Non
 
 class InvalidKey(IntelMQHarmonizationException, KeyError):
 
-    def __init__(self, key: str):
-        message = "invalid key %s" % repr(key)
+    def __init__(self, key: str, additional_text: Optional[str] = None):
+        message = f"invalid key {key!r} {additional_text or ''}".strip()  # remove trailing whitespace if additional_text is not given
         super().__init__(message)
 
 
diff --git a/intelmq/lib/message.py b/intelmq/lib/message.py
@@ -13,7 +13,7 @@
 import re
 import warnings
 from collections import defaultdict
-from typing import Any, Dict, Iterable, Optional, Sequence, Union
+from typing import Any, Dict, Iterable, Optional, Sequence, Union, Tuple
 from pkg_resources import resource_filename
 
 import intelmq.lib.exceptions as exceptions
@@ -186,8 +186,9 @@ def is_valid(self, key: str, value: str, sanitize: bool = True) -> bool:
             intelmq.lib.exceptions.InvalidKey: if given key is invalid.
 
         """
-        if not self.__is_valid_key(key):
-            raise exceptions.InvalidKey(key)
+        key_validation = self.__is_valid_key(key)
+        if not key_validation[0]:
+            raise exceptions.InvalidKey(key, additional_text=key_validation[1])
 
         if value is None or value in ["", "-", "N/A"]:
             return False
@@ -243,8 +244,9 @@ def add(self, key: str, value: str, sanitize: bool = True,
                 del self[key]
             return
 
-        if not self.__is_valid_key(key):
-            raise exceptions.InvalidKey(key)
+        key_validation = self.__is_valid_key(key)
+        if not key_validation[0]:
+            raise exceptions.InvalidKey(key, additional_text=key_validation[1])
 
         try:
             if value in ignore:
@@ -330,16 +332,16 @@ def unserialize(message_string: str):
         message = json.loads(message_string)
         return message
 
-    def __is_valid_key(self, key: str):
+    def __is_valid_key(self, key: str) -> Tuple[bool, str]:
         try:
             class_name, subitem = self.__get_type_config(key)
         except KeyError:
-            return False
+            return False, 'This key is not allowed by the harmonization configuration'
         if key in self.harmonization_config or key == '__type':
-            return True
+            return True, None
         if subitem:
-            return HARMONIZATION_KEY_FORMAT.match(key)
-        return False
+            return HARMONIZATION_KEY_FORMAT.match(key), f'Does not match regular expression {HARMONIZATION_KEY_FORMAT.pattern}'
+        return False, 'This key is not allowed by the harmonization configuration'
 
     def __is_valid_value(self, key: str, value: str):
         if key == '__type':
@@ -569,7 +571,7 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
         if isinstance(message, Event):
             super().__init__({}, auto, harmonization)
             for key, value in message.items():
-                if self._Message__is_valid_key(key):
+                if self._Message__is_valid_key(key)[0]:
                     self.add(key, value, sanitize=False)
         else:
             super().__init__(message, auto, harmonization)
diff --git a/intelmq/tests/lib/test_exceptions.py b/intelmq/tests/lib/test_exceptions.py
@@ -64,6 +64,13 @@ def test_MissingDependencyError(self):
         self.assertIn(repr(depname), exc)
         self.assertTrue(exc.endswith(" %s" % additional))
 
+    def test_invalid_key(self):
+        """
+        Check intelmq.lib.exceptions.InvalidKey
+        """
+        exc = excs.InvalidKey('test_key', additional_text='TEST').args[0]
+        self.assertTrue(exc.endswith(' TEST'))
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()
diff --git a/intelmq/tests/lib/test_message.py b/intelmq/tests/lib/test_message.py
@@ -168,8 +168,10 @@ def test_invalid_type2(self):
     def test_report_invalid_key(self):
         """ Test if report raises InvalidKey for invalid key in add(). """
         report = self.new_report()
-        with self.assertRaises(exceptions.InvalidKey):
+        with self.assertRaises(exceptions.InvalidKey) as cm:
             report.add('invalid', 0)
+        self.assertIn('not allowed', cm.exception.args[0])
+
 
     def test_report_add_raw(self):
         """ Test if report can add raw value. """
@@ -764,10 +766,11 @@ def test_invalid_harm_key(self):
             message.Event(harmonization={'event': {'foo.bar.': {}}})
 
     def test_invalid_extra_key_name(self):
-        """ Test if error is raised if an extra field name is invalid. """
+        """ Test if error is raised when an extra field name is invalid and error message is included in exception. """
         event = message.Event(harmonization=HARM)
-        with self.assertRaises(exceptions.InvalidKey):
+        with self.assertRaises(exceptions.InvalidKey) as cm:
             event.add('extra.foo-', 'bar')
+        self.assertIn('Does not match regular expression', cm.exception.args[0])
 
 
 class TestReport(unittest.TestCase):

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+SPDX-FileCopyrightText: 2023 Institute for Common Good Technology <[email protected]>`
	`2`	`+SPDX-License-Identifier: AGPL-3.0-or-later`
-Original file line number
+Diff line change
@@ @@ -1,8 +1,14 @@ @@
 intelmq (3.2.2-1) UNRELEASED; urgency=medium
 -  *
 +  *
 - -- Sebastian Wagner <[email protected]>  Mon, 28 Aug 2023 17:41:52 +0200
 + -- Sebastian Wagner <[email protected]>  Tue, 05 Sep 2023 15:05:07 +0200
++
 +intelmq (3.2.1-2) stable; urgency=medium
++
 +  * Fix build on Debian 12
++
 + -- Sebastian Wagner <[email protected]>  Tue, 05 Sep 2023 15:04:36 +0200
 intelmq (3.2.1-1) stable; urgency=medium