Merge branch 'develop' into monoidic-develop

Author: sebix

.github/workflows/codespell.yml

@@ -18,7 +18,7 @@ on:
 jobs:
   codespell:
     name: Find and notify about common misspellings
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     # This should not fail the whole workflow run
     continue-on-error: true
 

.github/workflows/debian-package.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       # Fixes https://github.com/actions/virtual-environments/issues/3080
       STORAGE_OPTS: overlay.mount_program=/usr/bin/fuse-overlayfs
@@ -34,7 +34,7 @@ jobs:
 
     - name: Upload artifact
       if: ${{ github.event_name == 'push' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: debian-package-${{ matrix.codename }}-${{ github.sha }}
         path: '~/artifacts'

.github/workflows/pycodestyle.yml

@@ -18,7 +18,7 @@ on:
 jobs:
   pycodestyle:
     name: Run pycodestyle
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
     - name: Checkout repository

.github/workflows/python/github.py

@@ -31,20 +31,20 @@ def __init__(self):
 
     def get_reviews(self):
         """ Get a list of reviews on a Github pull request as json object """
-        reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id))
+        reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews')
         reviews.raise_for_status()
         return reviews.json()
 
     def update_review(self, review_id, body):
         """ Update a review given by `review_id` and set its body to `body` """
         payload = {'body': body}
-        resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload)
+        resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload)
         resp.raise_for_status()
         return resp.json()
 
     def post_review(self, body):
         """ Post a pull request review containing `body` and requesting changes """
         payload = {'body': body, 'event': "REQUEST_CHANGES"}
-        resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload)
+        resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload)
         resp.raise_for_status()
         return resp.json()

.github/workflows/python/pycodestyle_comment.py

@@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str:
     """ Format the list of pycodestyle errors and return them a one string. """
     ret = ''
     for error in style_error_list:
-        ret += '* {}\n'.format(error)
+        ret += f'* {error}\n'
     return ret
 
 
@@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str:
         style_errors = list_style_errors()
 
         if style_errors:
-            print("Found {} errors.".format(len(style_errors)))
+            print(f"Found {len(style_errors)} errors.")
 
         gh = github.Github()
 

.github/workflows/regexploit.yml

@@ -19,14 +19,16 @@ on:
 jobs:
   regexploit:
     name: Find regular expressions which are vulnerable to ReDoS
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     # This should not fail the whole workflow run
     continue-on-error: true
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
     - name: Install regexploit
-      run: pip install regexploit
+      #run: pip install regexploit
+      # See https://github.com/doyensec/regexploit/pull/16
+      run: pip install git+https://github.com/sebix/regexploit.git@unsupported-ops-yaml
     - name: Run regexploit
       run: /home/runner/work/intelmq/intelmq/.github/workflows/scripts/regexploit.sh

.github/workflows/scripts/setup-full.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# SPDX-FileCopyrightText: 2020 Birger Schacht
+# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 set -x
@@ -14,6 +14,10 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo
 sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
 sudo systemctl start elasticsearch
 
+sudo apt update
+# for psql (used below)
+DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client
+
 # Install the dependencies of all the bots
 pip install wheel
 for file in intelmq/bots/*/*/REQUIREMENTS.txt; do
@@ -30,7 +34,16 @@ done
 # Setup sudo and install intelmq
 sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers
 sudo pip install .
-sudo intelmqsetup --skip-ownership
+
+intelmq_user_exists=$(getent passwd intelmq ||:)
+if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then
+	# create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work
+	groupadd -r intelmq
+	useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
+	sudo intelmqsetup
+else
+	sudo intelmqsetup --skip-ownership
+fi
 
 # Initialize the postgres database
 intelmq_psql_initdb

.github/workflows/unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         type: ['full', 'basic']
 
     services:
@@ -59,6 +59,7 @@ jobs:
         PGPORT: 5432
         PGUSER: intelmq
         PGPASSWORD: intelmq
+        python_version: ${{ matrix.python-version }}
       run: bash .github/workflows/scripts/setup-full.sh
 
     - name: Install test dependencies

CHANGELOG.md

@@ -1,29 +1,150 @@
 <!-- comment
-   SPDX-FileCopyrightText: 2015-2023 Sebastian Wagner
+   SPDX-FileCopyrightText: 2015-2025 Sebastian Wagner
    SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
 # CHANGELOG
 
 
-3.3.1 (unreleased)
-------------------
+This file lists all changes between IntelMQ releases.
+Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect on the administration of IntelMQ and contains steps that you need to be aware off for the upgrade.
+
+
+3.4.1 Patch release (unreleased)
+--------------------------------
 
 ### Configuration
 
 ### Core
-- `intelmq.lib.utils.drop_privileges`: When IntelMQ is called as `root` and dropping the privileges to user `intelmq`, also set the non-primary groups associated with the `intelmq` user. Makes the behaviour of running intelmqctl as `root` closer to the behaviour of `sudo -u intelmq ...` (PR#2507 by Mikk Margus Möll).
+- Drop support for Python 3.8 (fixes #2616, PR#2617 by Sebastian Wagner).
 
 ### Development
 
 ### Data Format
 
+### Bots
+#### Collectors
+
+#### Parsers
+
+#### Experts
+- `intelmq.bots.experts.asn_lookup.expert`: Print URLs to stdout only in verbose mode (PR#2591 by Sebastian Wagner).
+
+#### Outputs
+
+### Documentation
+- Fix and refresh links to mailing lists (PR#2609 by Kamil Mańkowski)
+- `Aggregate Bot`: Add illustration graphics (PR#2612 by Sebastian Wagner).
+
+### Packaging
+- Replace `/opt/intelmq` example paths in bots with variable `VAR_STATE_PATH` for correct paths in LSB-path setups like with packages (PR#2587 by Sebastian Wagner).
+
+### Tests
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Skip on all Python versions when running on CI (PR#2602 by Sebastian Wagner).
+- `.github/workflows/codespell.yml`, `debian-package.yml`, `regexploit.yml`: Upgrade to `ubuntu-latest` runners (PR#2602 by Sebastian Wagner).
+- `intelmq.test.test_conf`: With changed behaviour in ruamel.yaml on line wrapping since version 0.18.13, only test the parsabilty of `runtime.yaml` (PR#2619 by Sebastian Wagner).
+
+### Tools
+- `intelmq.bin.intelmq_psql_initdb`: Use `JSONB` type by default, Postgres supports it since version 9 (PR#2597 by Sebastian Wagner).
+- `intelmq.bin.rewrite_config_files`: Removed obsolete JSON configuration file rewriter (PR#2613 by Sebastian Wagner).
+
+### Contrib
+
+### Known issues
+
+
+3.4.0 Feature release (2025-03-14)
+----------------------------------
+
+### Configuration
+
+### Core
+- AMQP: Fix maintaining pipeline connection when during interrupted connections (PR#2533 by Kamil Mankowski).
+- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner).
+- `intelmq.lib.utils.list_all_bots`/`intelmqctl check`: Fix check for bot executable in $PATH by using the bot name instead of the import path (fixes #2559, PR#2564 by Sebastian Wagner).
+
+### Bots
+#### Collectors
+- `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
+  - Fixed behaviour if parameter `types` value is empty string, behave the same way as not set, not like no type.
+- `intelmq.bots.collectors.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal)
+- `intelmq.bots.collectors.http.collector_http`: Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+- `intelmq.bots.collectors.mail.collector_mail_url`:
+  - Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+  - Fix import for Timeout exception preventing another exception (fixes #2555, PR#2556 by Sebastian Wagner).
+- Remove `intelmq.bots.collectors.twitter` as it uses an unmaintained library and does not work any more (fixes #2346, #2441, PR#2568 by Sebastian Wagner).
+
+#### Parsers
+- `intelmq.bots.parsers.shadowserver._config`:
+  - fix error message formatting if schema file is absent (PR#2528 by Sebastian Wagner).
+- `intelmq.bots.parsers.shadowserver.parser`:
+  - Fix to avoid schema download if not configured #2530.
+- `intelmq.bots.parsers.misp.parser`: Replace deprecated datetime function `utcfromtimestamp` for Ubuntu 24.04 compatibility (PR#2577 by Sebastian Wagner, fixes #2576, #2571).
+- `intelmq.bots.parsers.cleanmx.parser`: Replace deprecated datetime function `utcfromtimestamp` for Ubuntu 24.04 compatibility (PR#2577 by Sebastian Wagner, fixes #2576, #2571).
+- Renamed `intelmq.bots.parsers.twitter` to `intelmq.bots.parser.ioc_extractor` (PR#2568 by Sebastian Wagner).
+  - Added `intelmq.bots.parsers.twitter` as a stub to load the IoC Extractor parser.
+
+#### Experts
+- `intelmq.bots.experts.securitytxt`:
+  - Added new bot (PR#2538 by Frank Westers and Sebastian Wagner).
+- `intelmq.bots.experts.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal).
+- `intelmq.bots.experts.fake.expert`: New expert to fake data (PR#2567 by Sebastian Wagner).
+
+#### Outputs
+- `intelmq.bots.outputs.cif3.output`:
+  - The requirement can only be installed on Python version < 3.12.
+  - Add a check on the Python version and exit if incompatible.
+  - Add a deprecation warning (PR#2544 by Sebastian Wagner).
+- `intelmq.bots.outputs.sql.output`:
+  - Treat an empty string `fields` parameter as unset parameter, fixing a crash in default configuration (PR#2548 by Sebastian Wagner, fixes #2548).
+
+### Documentation
+- `docs/admin/installation/linux-packages`: Add `[signed-by=]` options, add wget command as alternative to curl (PR#2547 by Sebastian Wagner).
+- Add documentation on the Redis pipeline (databases, configuration), fix generic pipeline documentation and add missing information on parameters, add unlinked intelmqctl docs to the index and TOC (PR#2560 by Sebastian Wagner).
+- Remove empty page tutorials/intelmq-manager (PR#2562 by Sebastian Wagner).
+
+### Packaging
+- Packages for Ubuntu 24.04 (by Sebastian Wagner, fixes #2571).
+
+### Tests
+- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner).
+- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner).
+- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner).
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 and 3.12 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner).
+- Full pytest workflow: Version-independent install of postgres client, for Ubuntu 24.04 (default on GitHub now) test environment compatibility (PR#2557 by Sebastian Wagner).
+- Debian package build workflow: Use artifact upload v4 instead of v3 (PR#2565 by Sebastian Wagner).
+
+### Known issues
+This is short list of the most important known issues. The full list can be retrieved from [GitHub](https://github.com/certtools/intelmq/labels/bug?page=2&q=is%3Aopen+label%3Abug).
+- intelmqctl: interactive run ignores custom log level (#2563).
+- `intelmq.parsers.html_table` may not process invalid URLs in patched Python version due to changes in `urllib` (#2382).
+- Breaking changes in 'rt' 3.0 library (#2367).
+- Type error with SQL output bot's `prepare_values` returning list instead of tuple (#2255).
+- `intelmq_psql_initdb` does not work for SQLite (#2202).
+- intelmqsetup: should install a default state file (#2175).
+- Misp Expert - Crash if misp event already exist (#2170).
+- Spamhaus CERT parser uses wrong field (#2165).
+- Custom headers ignored in HTTPCollectorBot (#2150).
+- intelmqctl log: parsing syslog does not work (#2097).
+- Bash completion scripts depend on old JSON-based configuration files (#2094).
+- Bots started with IntelMQ-API/Manager stop when the webserver is restarted (#952).
+- Corrupt dump files when interrupted during writing (#870).
+
+
+3.3.1 (2024-09-03)
+------------------
+
+### Core
+- `intelmq.lib.utils.drop_privileges`: When IntelMQ is called as `root` and dropping the privileges to user `intelmq`, also set the non-primary groups associated with the `intelmq` user. Makes the behaviour of running intelmqctl as `root` closer to the behaviour of `sudo -u intelmq ...` (PR#2507 by Mikk Margus Möll).
+- `intelmq.lib.utils.unzip`: Ignore directories themselves when extracting data to prevent the extraction of empty data for a directory entries (PR#2512 by Kamil Mankowski).
+
 ### Bots
 #### Collectors
 - `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
-  - Added support for the types parameter to be either a string or a list.
+  - Added support for the types parameter to be either a string or a list (PR#2495 by elsif2).
   - Refactored to utilize the type field returned by the API to match the requested types instead of a sub-string match on the filename.
-  - Fixed timezone issue for collecting reports.
+  - Fixed timezone issue for collecting reports (PR#2506 by elsif2).
+  - Fixed behaviour if parameter `reports` value is empty string, behave the same way as not set, not like no report (PR#2523 by Sebastian Wagner).
 - `intelmq.bots.collectors.shodan.collector_stream` (PR#2492 by Mikk Margus Möll):
   - Add `alert` parameter to Shodan stream collector to allow fetching streams by configured alert ID
 - `intelmq.bots.collectors.mail._lib`: Remove deprecated parameter `attach_unzip` from default parameters (PR#2511 by Sebastian Wagner).
@@ -32,10 +153,13 @@
 - `intelmq.bots.parsers.shadowserver._config`:
   - Fetch schema before first run (PR#2482 by elsif2, fixes #2480).
 - `intelmq.bots.parsers.dataplane.parser`: Use `  |  ` as field delimiter, fix parsing of AS names including `|` (PR#2488 by DigitalTrustCenter).
+- all parsers: add `copy_collector_provided_fields` parameter allowing copying additional fields from the report, e.g. `extra.file_name`.
+  (PR#2513 by Kamil Mankowski).
 
 #### Experts
 - `intelmq.bots.experts.sieve.expert`:
   - For `:contains`, `=~` and `!~`, convert the value to string before matching avoiding an exception. If the value is a dict, convert the value to JSON (PR#2500 by Sebastian Wagner).
+  - Add support for variables in Sieve scripts (PR#2514 by Mikk Margus Möll, fixes #2486).
 - `intelmq.bots.experts.filter.expert`:
   - Treat value `false` for parameter `filter_regex` as false (PR#2499 by Sebastian Wagner).
 
@@ -46,15 +170,20 @@
 ### Documentation
 - Bots: Clarify some section of Mail collectors and the Generic CSV Parser (PR#2510 by Sebastian Wagner).
 
-### Packaging
-
-### Tests
-
-### Tools
-
-### Contrib
-
-### Known issues
+### Known Issues
+This is short list of the most important known issues. The full list can be retrieved from [GitHub](https://github.com/certtools/intelmq/labels/bug?page=2&q=is%3Aopen+label%3Abug).
+- `intelmq.parsers.html_table` may not process invalid URLs in patched Python version due to changes in `urllib` (#2382).
+- Breaking changes in 'rt' 3.0 library (#2367).
+- Type error with SQL output bot's `prepare_values` returning list instead of tuple (#2255).
+- `intelmq_psql_initdb` does not work for SQLite (#2202).
+- intelmqsetup: should install a default state file (#2175).
+- Misp Expert - Crash if misp event already exist (#2170).
+- Spamhaus CERT parser uses wrong field (#2165).
+- Custom headers ignored in HTTPCollectorBot (#2150).
+- intelmqctl log: parsing syslog does not work (#2097).
+- Bash completion scripts depend on old JSON-based configuration files (#2094).
+- Bots started with IntelMQ-API/Manager stop when the webserver is restarted (#952).
+- Corrupt dump files when interrupted during writing (#870).
 
 
 3.3.0 (2024-03-01)
@@ -180,10 +309,6 @@
    -  got support for providing custom harmonization file, generating view for storing `raw` fields separately, and adding `IF NOT EXISTS`/`OR REPLACE` clauses ([PR#2404](https://github.com/certtools/intelmq/pull/2404) by Kamil Mankowski).
    -  got support for generating JSONB fields for PostgreSQL schema (PR#2436 by Kamil Mankowski).
 
-### Contrib
-
-### Known issues
-
 
 3.2.1 (2023-08-28)
 ------------------
@@ -304,7 +429,7 @@ This is short list of the most important known issues. The full list can be retr
 - Added an ExpertBot class - it should be used by all expert bots as a parent class
 - Introduced a module for IntelMQ related datatypes `intelmq.lib.datatypes` which for now only contains an Enum listing the four bot types
 - Added a `bottype` attribute to CollectorBot, ParserBot, ExpertBot, OutputBot
-- Introduces a module for IntelMQ processmanagers. The processmanagers were up until now part of the intelmqct script.
+- Introduces a module for IntelMQ processmanagers. The processmanagers were up until now part of the intelmqctl script.
   They now reside in `intelmq.lib.processmanager` which also contains an interface definition the processmanager implementations must adhere to.
   Both the processmanagers and the `intelmqctl` script were cleaned up a bit.
   The `LogLevel` and `ReturnType` Enums were added to `intelmq.lib.datatypes`.

MANIFEST.in

@@ -10,7 +10,7 @@ graft intelmq/tests
 include COPYRIGHT
 include LICENSE
 include CHANGELOG.md
-recursive-exclude intelmq/bin intelmq_gen_feeds_docs.py intelmq_gen_harm_docs.py rewrite_config_files.py
+recursive-exclude intelmq/bin intelmq_gen_feeds_docs.py intelmq_gen_harm_docs.py
 exclude .*
 global-exclude *~
 global-exclude *.py[co]