ENH: Using msgpack instead of json

.github/workflows/scripts/ansible-runtime.yaml

@@ -0,0 +1,225 @@
+# SPDX-FileCopyrightText: 2021 Birger Schacht
+#
+# SPDX-License-Identifier: CC0-1.0
+cymru-whois-expert:
+  bot_id: cymru-whois-expert
+  description: Cymru Whois (IP to ASN) is the bot responsible to add network information
+    to the events (BGP, ASN, AS Name, Country, etc..).
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.cymru_whois.expert
+  name: Cymru Whois
+  parameters:
+    destination_queues:
+      _default: [file-output-queue]
+    overwrite: true
+    redis_cache_db: 5
+    redis_cache_host: 127.0.0.1
+    redis_cache_password: null
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+deduplicator-expert:
+  bot_id: deduplicator-expert
+  description: Deduplicator is the bot responsible for detection and removal of duplicate
+    messages. Messages get cached for <redis_cache_ttl> seconds. If found in the cache,
+    it is assumed to be a duplicate.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.deduplicator.expert
+  name: Deduplicator
+  parameters:
+    destination_queues:
+      _default: [taxonomy-expert-queue]
+    filter_keys: raw,time.observation
+    filter_type: blacklist
+    redis_cache_db: 6
+    redis_cache_host: 127.0.0.1
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+feodo-tracker-browse-collector:
+  description: Generic URL Fetcher is the bot responsible to get the report from an
+    URL.
+  enabled: true
+  group: Collector
+  module: intelmq.bots.collectors.http.collector_http
+  name: URL Fetcher
+  parameters:
+    destination_queues:
+      _default: [feodo-tracker-browse-parser-queue]
+    extract_files: false
+    http_password: null
+    http_url: https://feodotracker.abuse.ch/browse
+    http_url_formatting: false
+    http_username: null
+    name: Feodo Tracker Browse
+    provider: Abuse.ch
+    rate_limit: 86400
+    ssl_client_certificate: null
+  run_mode: continuous
+feodo-tracker-browse-parser:
+  description: HTML Table Parser is a bot configurable to parse different html table
+    data.
+  enabled: true
+  group: Parser
+  module: intelmq.bots.parsers.html_table.parser
+  name: HTML Table
+  parameters:
+    attribute_name: ''
+    attribute_value: ''
+    columns: time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc
+    default_url_protocol: http://
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+    ignore_values: ',,,,Not listed,,'
+    skip_table_head: true
+    split_column: ''
+    split_index: 0
+    split_separator: ''
+    table_index: 0
+    time_format: null
+    type: c2server
+  run_mode: continuous
+file-input:
+  bod_id: foobar
+  description: foobar
+  enabled: true
+  group: Collectors
+  module: intelmq.bots.collectors.file.collector_file
+  name: File Input
+  parameters:
+    delete_file: false
+    destination_queues:
+      _default: [file-output-queue]
+    path: /assets
+    postfix: .txt
+  run_mode: scheduled
+file-output:
+  bot_id: file-output
+  description: File is the bot responsible to send events to a file.
+  enabled: true
+  group: Output
+  groupname: outputs
+  module: intelmq.bots.outputs.file.output
+  name: File
+  parameters: {file: /var/lib/intelmq/bots/file-output/events.txt, hierarchical_output: false,
+    single_key: null}
+  run_mode: continuous
+gethostbyname-1-expert:
+  bot_id: gethostbyname-1-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+gethostbyname-2-expert:
+  bot_id: gethostbyname-2-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+malc0de-parser:
+  bot_id: malc0de-parser
+  description: Malc0de Parser is the bot responsible to parse the IP Blacklist and
+    either Windows Format or Bind Format reports and sanitize the information.
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.malc0de.parser
+  name: Malc0de
+  parameters:
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+  run_mode: continuous
+malc0de-windows-format-collector:
+  bot_id: malc0de-windows-format-collector
+  description: ''
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: Malc0de Windows Format
+  parameters:
+    destination_queues:
+      _default: [malc0de-parser-queue]
+    http_password: null
+    http_url: https://malc0de.com/bl/BOOT
+    http_username: null
+    name: Windows Format
+    provider: Malc0de
+    rate_limit: 10800
+    ssl_client_certificate: null
+  run_mode: continuous
+spamhaus-drop-collector:
+  bot_id: spamhaus-drop-collector
+  description: ''
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [spamhaus-drop-parser-queue]
+    http_password: null
+    http_url: https://www.spamhaus.org/drop/drop.txt
+    http_username: null
+    name: Drop
+    provider: Spamhaus
+    rate_limit: 3600
+    ssl_client_certificate: null
+  run_mode: continuous
+spamhaus-drop-parser:
+  bot_id: spamhaus-drop-parser
+  description: Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP,
+    DROPv6, and ASN-DROP reports and sanitize the information.
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.spamhaus.parser_drop
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+  run_mode: continuous
+taxonomy-expert:
+  bot_id: taxonomy-expert
+  description: Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all
+    events.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.taxonomy.expert
+  name: Taxonomy
+  parameters:
+    destination_queues:
+      _default: [url2fqdn-expert-queue]
+  run_mode: continuous
+url2fqdn-expert:
+  bot_id: url2fqdn-expert
+  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.url2fqdn.expert
+  name: URL2FQDN
+  parameters:
+    destination_queues:
+      _default: [gethostbyname-1-expert-queue, gethostbyname-2-expert-queue]
+    load_balance: true
+    overwrite: false
+  run_mode: continuous

CHANGELOG.md

@@ -390,7 +390,19 @@ Update allowed classification fields to version 1.3 (2021-05-18) (by Sebastian W
   - Add support for new field `SourceIpInfo.SourceIpv4Int` (PR#1940 by Sebastian Wagner).
   - Fix mapping of "ConnectionType" fields, this is not `protocol.application`. Now mapped to `extra.*.connection_type` (PR#1940 by Sebastian Wagner).
 - `intelmq.bots.parsers.shadowserver._config`:
+<<<<<<< HEAD
+<<<<<<< HEAD
+<<<<<<< HEAD
   - Add support for the new feeds *Honeypot-Amplification-DDoS-Events*, *Honeypot-Brute-Force-Events*, *Honeypot-Darknet*, *IP-Spoofer-Events*, *Sinkhole-Events*, *Sinkhole-HTTP-Events*, *Vulnerable-Exchange-Server*, *Sinkhole-Events-HTTP-Referer* (PR#1950, PR#1952, PR#1953, PR#1954, PR#1970 by Birger Schacht and Sebastian Wagner, PR#1971 by Mikk Margus Möll).
+=======
+  - Add support for the new feeds *Honeypot-Amplification-DDoS-Events*, *Honeypot-Brute-Force-Events*, *Honeypot-Darknet*, *IP-Spoofer-Events*, *Sinkhole-Events*, *Sinkhole-HTTP-Events* (PR#1950, PR#1952, PR#1953 and PR#1954 by Birger Schacht and Sebastian Wagner).
+>>>>>>> 366505cc6 (ENH: add event46_sinkhole shadowserver config & tests)
+=======
+  - Add support for the new feeds *Honeypot-Amplification-DDoS-Events*, *Honeypot-Brute-Force-Events*, *Honeypot-Darknet*, *IP-Spoofer-Events*, *Sinkhole-Events*, *Sinkhole-HTTP-Events*, *Vulnerable-Exchange-Server* (PR#1950, PR#1952, PR#1953, PR#1954, PR#1970 by Birger Schacht and Sebastian Wagner).
+>>>>>>> 4d3f4d647 (ENH+DOC: shadowserver exchange feed)
+=======
+  - Add support for the new feeds *Honeypot-Amplification-DDoS-Events*, *Honeypot-Brute-Force-Events*, *Honeypot-Darknet*, *IP-Spoofer-Events*, *Sinkhole-Events*, *Sinkhole-HTTP-Events*, *Vulnerable-Exchange-Server*, *Sinkhole-Events-HTTP-Referer* (PR#1950, PR#1952, PR#1953, PR#1954, PR#1970 by Birger Schacht and Sebastian Wagner, PR#1971 by Mikk Margus Möll).
+>>>>>>> f056ff7d4 (DOC for PR#1971)
 
 #### Experts
 - `intelmq.bots.experts.splunk_saved_search.expert`:

NEWS.md

@@ -190,6 +190,9 @@ UPDATE events
 UPDATE events
    SET "classification.taxonomy" = 'information-content-security', "classification.type" = 'unauthorised-information-modification'
    WHERE "classification.taxonomy" = 'intrusions', "classification.type" = 'defacement'
+UPDATE events
+   SET "classification.taxonomy" = 'information-content-security', "classification.type" = 'unauthorised-information-modification'
+   WHERE "classification.taxonomy" = 'intrusions', "classification.type" = 'defacement'
 UPDATE events
    SET "classification.taxonomy" = 'malicious-code'
    WHERE "classification.taxonomy" = 'malicious code';
@@ -284,7 +287,7 @@ CentOS 7 (with EPEL) provides both Python 3.4 and Python 3.6. If IntelMQ was ins
   type and reloads them afterwards. Removes any external dependencies (such as curl or wget).
   This is a replacement for shell scripts such as `update-tor-nodes`, `update-asn-data`,
   `update-geoip-data`, `update-rfiprisk-data`.
-  
+
   Usage:
   ```
   intelmq.bots.experts.asn_lookup.expert --update-database

debian/control

@@ -20,6 +20,7 @@ Build-Depends: debhelper (>= 4.1.16),
                python3-sphinx-rtd-theme,
                python3-termstyle,
                python3-tz,
+               python3-msgpack,
                quilt,
                rsync,
                safe-rm
@@ -41,6 +42,7 @@ Depends: bash-completion,
          python3-ruamel.yaml,
          python3-termstyle (>= 0.1.10),
          python3-tz,
+         python3-msgpack,
          redis-server,
          systemd,
          ${misc:Depends},

docs/user/bots.rst

@@ -502,6 +502,9 @@ Requires the `kafka python library <https://pypi.org/project/kafka/>`_.
 * `ssl_ca_certificate`: Optional string of path to trusted CA certificate. Only used by some bots.
 
 
+.. _intelmq.bots.collectors.misp.collector:
+
+
 .. _intelmq.bots.collectors.misp.collector:
 
 MISP Generic
@@ -641,6 +644,9 @@ Requires the rsync executable
 * `temp_directory`: The temporary directory for rsync to use for rsync'd files. Optional. Default: `$VAR_STATE_PATH/rsync_collector`. `$VAR_STATE_PATH` is `/var/run/intelmq/` or `/opt/intelmq/var/run/`.
 
 
+.. _intelmq.bots.collectors.shadowserver.collector_reports_api:
+
+
 .. _intelmq.bots.collectors.shadowserver.collector_reports_api:
 
 Shadowserver Reports API
@@ -1700,6 +1706,7 @@ It is required to look up the correct configuration.
 
 Look at the documentation in the bot's ``_config.py`` file for more information.
 
+.. _intelmq.bots.parsers.shodan.parser:
 
 .. _intelmq.bots.parsers.shodan.parser:
 
@@ -1788,7 +1795,6 @@ Aggregate
 **Configuration Parameters**
 
 * **Cache parameters** (see in section :ref:`common-parameters`)
-
   * TTL is not used, using it would result in data loss.
 * **fields** Given fields which are used to aggregate like `classification.type, classification.identifier`
 * **threshold** If the aggregated event is lower than the given threshold after the timespan, the event will get dropped.
@@ -1856,6 +1862,8 @@ Use this command to create/update the database and reload the bot:
 The database is fetched from `routeviews.org <http://www.routeviews.org/routeviews/>`_ and licensed under the Creative Commons Attribution 4.0 International license (see the `routeviews FAQ <http://www.routeviews.org/routeviews/index.php/faq/#faq-6666>`_).
 
 
+.. _intelmq.bots.experts.csv_converter.expert:
+
 .. _intelmq.bots.experts.csv_converter.expert:
 
 CSV Converter
@@ -1924,6 +1932,8 @@ RemoveAffix
 Remove part of string from string, example: `www.` from domains.
 
 
+.. _intelmq.bots.experts.domain_suffix.expert:
+
 .. _intelmq.bots.experts.domain_suffix.expert:
 
 Domain Suffix
@@ -2696,6 +2706,8 @@ RDAP
    }
 
 
+.. _intelmq.bots.experts.recordedfuture_iprisk.expert:
+
 .. _intelmq.bots.experts.recordedfuture_iprisk.expert:
 
 RecordedFuture IP risk
@@ -2986,6 +2998,23 @@ The following operators may be used to match events:
  * Boolean values can be matched with `==` or `!=` followed by `true` or `false`. Example:
    ``if extra.has_known_vulns == true { ... }``
 
+ * `:equals` tests for equality between lists, including order. Example for checking a hostname-port pair:
+   ``if extra.host_tuple :equals ['dns.google', 53] { ... }``
+ * `:setequals` tests for set-based equality (ignoring duplicates and value order) between a list of given values. Example for checking for the first nameserver of two domains, regardless of the order they are given in the list:
+   ``if extra.hostnames :setequals ['ns1.example.com', 'ns1.example.mx'] { ... }``
+
+ * `:overlaps` tests if there is at least one element in common between the list specified by a key and a list of values. Example for checking if at least one of the ICS, database or vulnerable tags is given:
+   ``if extra.tags :overlaps ['ics', 'database', 'vulnerable'] { ... } ``
+
+ * `:subsetof` tests if the list of values from the given key only contains values from a set of values specified as the argument. Example for checking for a host that has only ns1.example.com and/or ns2.[...] as its apparent hostname:
+   ``if extra.hostnames :subsetof ['ns1.example.com', 'ns2.example.com'] { ... }``
+
+ * `:supersetof` tests if the list of values from the given key is a superset of the values specified as the argument. Example for matching hosts with at least the IoT and vulnerable tags:
+   ``if extra.tags :supersetof ['iot', 'vulnerable'] { ... }``
+
+ * Boolean values can be matched with `==` or `!=` followed by `true` or `false`. Example:
+   ``if extra.has_known_vulns == true { ... }``
+
  * The combination of multiple expressions can be done using parenthesis and boolean operators:
 
   ``if (source.ip == '127.0.0.1') && (comment == 'add field' || classification.taxonomy == 'vulnerable') { ... }``
@@ -3396,7 +3425,6 @@ Events without `source.url`, `source.fqdn`, `source.ip`, or `source.asn`, are ig
 only contains the domain. uWhoisd will automatically strip the subdomain part if it is present in the request.
 
 Example: `https://www.theguardian.co.uk`
-
 * TLD: `co.uk` (uWhoisd uses the `Mozilla public suffix list <https://publicsuffix.org/list/>`_ as a reference)
 * Domain: `theguardian.co.uk`
 * Subdomain: `www`
@@ -4099,6 +4127,8 @@ Then, set the `database` parameter to the `your-db.db` file path.
 
 .. _intelmq.bots.outputs.stomp.output:
 
+.. _intelmq.bots.outputs.stomp.output:
+
 STOMP
 ^^^^^