ENH: new parameter autoupdate_cached_database

New parameter `autoupdate_cached_database` to disable automatic updates (downloads) of cached databases

Author: sebix

CHANGELOG.md

@@ -80,6 +80,7 @@ CHANGELOG
 - `intelmq.bots.experts.asn_lookup.expert`: Fixes update-database script on the last few days of a month (PR#2121 by Filip Pokorný, fixes #2088).
 - `intelmq.bots.experts.threshold.expert`: Correctly use the standard parameter `redis_cache_ttl` instead of the previously used parameter `timeout` (PR#2155 by Karl-Johan Karlsson).
 - `intelmq.bots.experts.jinja2.expert`: Lift restriction on requirement jinja2 < 3 (PR#2158 by Sebastian Wagner).
+- `intelmq.bots.experts.asn_lookup.expert`, `intelmq.bots.experts.domain_suffix.expert`, `intelmq.bots.experts.maxmind_geoip.expert`, `intelmq.bots.experts.recordedfuture_iprisk.expert`, `intelmq.bots.experts.tor_nodes.expert`: New parameter `autoupdate_cached_database` to disable automatic updates (downloads) of cached databases (PR#2180 by Sebastian Wagner).
 
 #### Outputs
 - Removed `intelmq.bots.outputs.postgresql`: this bot was marked as deprecated in 2019 announced to be removed in version 3 of IntelMQ (PR#2045 by Birger Schacht).

intelmq/bots/experts/asn_lookup/expert.py

@@ -26,6 +26,7 @@
 class ASNLookupExpertBot(ExpertBot):
     """Add ASN and netmask information from a local BGP dump"""
     database = None  # TODO: should be pathlib.Path
+    autoupdate_cached_database: bool = True  # Activate/deactivate update-database functionality
 
     def init(self):
         if pyasn is None:
@@ -96,15 +97,15 @@ def update_database(cls, verbose=False):
         runtime_conf = get_bots_settings()
         try:
             for bot in runtime_conf:
-                if runtime_conf[bot]["module"] == __name__:
+                if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('autoupdate_cached_database', True):
                     bots[bot] = runtime_conf[bot]["parameters"]["database"]
 
         except KeyError as e:
             sys.exit(f"Database update failed. Your configuration of {bot} is missing key {e}.")
 
         if not bots:
             if verbose:
-                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf.")
+                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf or database update disabled with parameter 'autoupdate_cached_database'.")
             sys.exit(0)
 
         # we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed

intelmq/bots/experts/domain_suffix/expert.py

@@ -32,6 +32,7 @@ class DomainSuffixExpertBot(ExpertBot):
     """Extract the domain suffix from a domain and save it in the the domain_suffix field. Requires a local file with valid domain suffixes"""
     field: str = None
     suffix_file: str = None  # TODO: should be pathlib.Path
+    autoupdate_cached_database: bool = True  # Activate/deactivate update-database functionality
 
     def init(self):
         if self.field not in ALLOWED_FIELDS:
@@ -84,15 +85,15 @@ def update_database(cls, verbose=False):
         runtime_conf = get_bots_settings()
         try:
             for bot in runtime_conf:
-                if runtime_conf[bot]["module"] == __name__:
+                if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('autoupdate_cached_database', True):
                     bots[bot] = runtime_conf[bot]["parameters"]["suffix_file"]
 
         except KeyError as e:
             sys.exit(f"Database update failed. Your configuration of {bot} is missing key {e}.")
 
         if not bots:
             if verbose:
-                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf.")
+                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf or database update disabled with parameter 'autoupdate_cached_database'.")
             sys.exit(0)
 
         # we only need to import now. If there are no asn_lookup bots, this dependency does not need to be installed

intelmq/bots/experts/maxmind_geoip/expert.py

@@ -31,6 +31,7 @@ class GeoIPExpertBot(ExpertBot):
     license_key: str = "<insert Maxmind license key>"
     overwrite: bool = False
     use_registered: bool = False
+    autoupdate_cached_database: bool = True  # Activate/deactivate update-database functionality
 
     def init(self):
         if geoip2 is None:
@@ -113,7 +114,7 @@ def update_database(cls, verbose=False):
         runtime_conf = get_bots_settings()
         try:
             for bot in runtime_conf:
-                if runtime_conf[bot]["module"] == __name__:
+                if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('autoupdate_cached_database', True):
                     license_key = runtime_conf[bot]["parameters"]["license_key"]
                     bots[bot] = runtime_conf[bot]["parameters"]["database"]
 
@@ -129,7 +130,7 @@ def update_database(cls, verbose=False):
 
         if not bots:
             if verbose:
-                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf.")
+                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf or database update disabled with parameter 'autoupdate_cached_database'.")
             sys.exit(0)
 
         # we only need to import now, if there are no maxmind_geoip bots, this dependency does not need to be installed

intelmq/bots/experts/recordedfuture_iprisk/expert.py

@@ -24,6 +24,7 @@ class RecordedFutureIPRiskExpertBot(ExpertBot):
     api_token: str = "<insert Recorded Future IPRisk API token>"
     database: str = "/opt/intelmq/var/lib/bots/recordedfuture_iprisk/rfiprisk.dat"  # TODO: should be pathlib.Path
     overwrite: bool = False
+    autoupdate_cached_database: bool = True  # Activate/deactivate update-database functionality
 
     _database = dict()
 
@@ -77,7 +78,7 @@ def update_database(cls, verbose=False):
         runtime_conf = get_bots_settings()
         try:
             for bot in runtime_conf:
-                if runtime_conf[bot]["module"] == __name__:
+                if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('autoupdate_cached_database', True):
                     api_token = runtime_conf[bot]["parameters"]["api_token"]
                     bots[bot] = runtime_conf[bot]["parameters"]["database"]
 
@@ -86,7 +87,7 @@ def update_database(cls, verbose=False):
 
         if not bots:
             if verbose:
-                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf.")
+                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf or database update disabled with parameter 'autoupdate_cached_database'.")
             sys.exit(0)
 
         try:

intelmq/bots/experts/tor_nodes/expert.py

@@ -20,6 +20,7 @@ class TorExpertBot(ExpertBot):
     """Check if the IP address is a Tor Exit Node based on a local database of TOR nodes"""
     database: str = "/opt/intelmq/var/lib/bots/tor_nodes/tor_nodes.dat"  # TODO: pathlib.Path
     overwrite: bool = False
+    autoupdate_cached_database: bool = True  # Activate/deactivate update-database functionality
 
     _database = set()
 
@@ -78,15 +79,15 @@ def update_database(cls, verbose=False):
         runtime_conf = get_bots_settings()
         try:
             for bot in runtime_conf:
-                if runtime_conf[bot]["module"] == __name__:
+                if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('autoupdate_cached_database', True):
                     bots[bot] = runtime_conf[bot]["parameters"]["database"]
 
         except KeyError as e:
             sys.exit(f"Database update failed. Your configuration of {bot} is missing key {e}.")
 
         if not bots:
             if verbose:
-                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf.")
+                print(f"Database update skipped. No bots of type {__name__} present in runtime.conf or database update disabled with parameter 'autoupdate_cached_database'.")
             sys.exit(0)
 
         try: