Speedup whitelists

config/whitelist.conf

@@ -162,3 +162,4 @@ organization,google,both,alerts
 organization,apple,both,alerts
 organization,twitter,both,alerts
 domain,markmonitor.com,both,alerts
+domain,whois.nic.co,both,alerts

docs/usage.md

@@ -468,28 +468,6 @@ The values for each column are the following:
             - Ignore alerts: slips reads all the flows, but it just ignores alerting if there is a match.
             - Ignore flows: the flow will be completely discarded.
 
-### Removing values from the Whitelist
-
-Whitelisted IoCs can be updated:
-1. When you re-start Slips
-2. On the fly while running Slips
-
-If you're updating the whitelist while Slips is running, be careful to use ; to comment out the lines you want to remove from the db
-for example, if you have the following line in `whitelist.conf`:
-
-```
-organization,google,both,alerts
-```
-
-To be able to remove this whitelist entry while Slips is running, simply change it to
-
-```
-# organization,google,both,alerts
-```
-
-Comments starting with `;` are not removed from the database and are treated as user comments.
-Comments starting with `#` will cause Slips to attempt to remove that entry from the database.
-
 ## Popup notifications
 
 Slips Support displaying popup notifications whenever there's an alert.
@@ -622,7 +600,7 @@ Check [rotation section](https://stratospherelinuxips.readthedocs.io/en/develop/
 
 But you can also enable storing a copy of zeek log files in the output
 directory after the analysis is done by setting ```store_a_copy_of_zeek_files``` to yes,
-or while zeek is stil generating log files by setting ```store_zeek_files_in_the_output_dir``` to yes.
+or while zeek is still generating log files by setting ```store_zeek_files_in_the_output_dir``` to yes.
 this option stores a copy of the zeek files present in ```zeek_files/``` the moment slips stops.
 so this doesn't include deleted zeek logs.
 

install/requirements.txt

@@ -40,6 +40,7 @@ pre-commit==4.3.0
 coverage==7.11.0
 netifaces==0.11.0
 scapy==2.6.1
+pybloom_live
 pyyaml
 pytest-asyncio
 vulture

managers/process_manager.py

@@ -39,6 +39,7 @@
 
 from slips_files.common.style import green
 from slips_files.core.evidence_handler import EvidenceHandler
+from slips_files.core.helpers.bloom_filters_manager import BFManager
 from slips_files.core.input import Input
 from slips_files.core.output import Output
 from slips_files.core.profiler import Profiler
@@ -111,6 +112,7 @@ def start_profiler_process(self):
             self.main.args,
             self.main.conf,
             self.main.pid,
+            self.main.bloom_filters_man,
             is_profiler_done=self.is_profiler_done,
             profiler_queue=self.profiler_queue,
             is_profiler_done_event=self.is_profiler_done_event,
@@ -134,6 +136,7 @@ def start_evidence_process(self):
             self.main.args,
             self.main.conf,
             self.main.pid,
+            self.main.bloom_filters_man,
         )
         evidence_process.start()
         self.main.print(
@@ -154,6 +157,7 @@ def start_input_process(self):
             self.main.args,
             self.main.conf,
             self.main.pid,
+            self.main.bloom_filters_man,
             is_input_done=self.is_input_done,
             profiler_queue=self.profiler_queue,
             input_type=self.main.input_type,
@@ -399,6 +403,7 @@ def load_modules(self):
                 self.main.args,
                 self.main.conf,
                 self.main.pid,
+                self.main.bloom_filters_man,
             )
             module.start()
             self.main.db.store_pid(module_name, int(module.pid))
@@ -430,17 +435,30 @@ def print_stopped_module(self, module):
             f"\t{green(module)} \tStopped. " f"" f"{green(modules_left)} left."
         )
 
+    def init_bloom_filters_manager(self):
+        """this instance is shared accross all slips IModule instances,
+        because we dont wanna re-create the filters once for each process,
+        this way is more memory efficient"""
+        return BFManager(
+            self.main.logger,
+            self.main.args.output,
+            self.main.redis_port,
+            self.main.conf,
+            self.main.pid,
+        )
+
     def start_update_manager(self, local_files=False, ti_feeds=False):
         """
         starts the update manager process
         PS; this function is blocking, slips.py will not start the rest of the
-         module unless this functionis done
+         module unless this function's done
         :kwarg local_files: if true, updates the local ports and
                 org files from disk
         :kwarg ti_feeds: if true, updates the remote TI feeds.
             PS: this takes time.
         """
         try:
+            bloom_filters_man = getattr(self.main, "bloom_filters_man", None)
             # only one instance of slips should be able to update ports
             # and orgs at a time
             # so this function will only be allowed to run from 1 slips
@@ -456,6 +474,7 @@ def start_update_manager(self, local_files=False, ti_feeds=False):
                     self.main.args,
                     self.main.conf,
                     self.main.pid,
+                    bloom_filters_man,
                 )
 
                 if local_files:

modules/flowalerts/flowalerts.py

@@ -29,7 +29,7 @@ class FlowAlerts(AsyncModule):
 
     def init(self):
         self.subscribe_to_channels()
-        self.whitelist = Whitelist(self.logger, self.db)
+        self.whitelist = Whitelist(self.logger, self.db, self.bloom_filters)
         self.dns = DNS(self.db, flowalerts=self)
         self.software = Software(self.db, flowalerts=self)
         self.notice = Notice(self.db, flowalerts=self)

modules/ip_info/ip_info.py

@@ -64,7 +64,7 @@ def init(self):
             "new_dns": self.c3,
             "check_jarm_hash": self.c4,
         }
-        self.whitelist = Whitelist(self.logger, self.db)
+        self.whitelist = Whitelist(self.logger, self.db, self.bloom_filters)
         self.is_running_non_stop: bool = self.db.is_running_non_stop()
         self.valid_tlds = whois.validTlds()
         self.is_running_in_ap_mode: bool = (

modules/threat_intelligence/threat_intelligence.py

@@ -609,6 +609,10 @@ def is_valid_threat_level(self, threat_level):
         return threat_level in utils.threat_levels
 
     def parse_known_fp_hashes(self, fullpath: str):
+        """
+        That file contains known FalsePositives of hashes to reduce the
+        amount of FP from TI files
+        """
         fp_hashes = {}
         with open(fullpath) as fps:
             # skip comments

modules/update_manager/update_manager.py

@@ -56,7 +56,7 @@ def init(self):
         self.loaded_ti_files = 0
         # don't store iocs older than 1 week
         self.interval = 7
-        self.whitelist = Whitelist(self.logger, self.db)
+        self.whitelist = Whitelist(self.logger, self.db, self.bloom_filters)
         self.slips_logfile = self.db.get_stdfile("stdout")
         self.org_info_path = "slips_files/organizations_info/"
         self.path_to_mac_db = "databases/macaddress-db.json"
@@ -1484,10 +1484,16 @@ def update_local_whitelist(self):
             self.whitelist.update()
 
     def update_org_files(self):
+        """
+        This func handles organizations whitelist files.
+        It updates the local IoCs of every supported organization in the db
+        and initializes the bloom filters
+        """
         for org in utils.supported_orgs:
             org_ips = os.path.join(self.org_info_path, org)
             org_asn = os.path.join(self.org_info_path, f"{org}_asn")
             org_domains = os.path.join(self.org_info_path, f"{org}_domains")
+
             if self.check_if_update_org(org_ips):
                 self.whitelist.parser.load_org_ips(org)
 
@@ -1576,10 +1582,11 @@ def update_online_whitelist(self):
         # delete the old ones
         self.db.delete_tranco_whitelist()
         response = self.responses["tranco_whitelist"]
+        domains = []
         for line in response.text.splitlines():
-            domain = line.split(",")[1]
-            domain.strip()
-            self.db.store_tranco_whitelisted_domain(domain)
+            domain = line.split(",")[1].strip()
+            domains.append(domain)
+        self.db.store_tranco_whitelisted_domains(domains)
 
         self.mark_feed_as_updated("tranco_whitelist")
 

slips/main.py

@@ -27,6 +27,7 @@
 from slips_files.common.slips_utils import utils
 from slips_files.common.style import green, yellow
 from slips_files.core.database.database_manager import DBManager
+from slips_files.core.helpers.bloom_filters_manager import BFManager
 from slips_files.core.helpers.checker import Checker
 
 
@@ -586,6 +587,9 @@ def start(self):
             # if slips is given a .rdb file, don't load the
             # modules as we don't need them
             if not self.args.db:
+                self.bloom_filters_man: BFManager = (
+                    self.proc_man.init_bloom_filters_manager()
+                )
                 # update local files before starting modules
                 # if wait_for_TI_to_finish is set to true in the config file,
                 # slips will wait untill all TI files are updated before
@@ -595,6 +599,10 @@ def start(self):
                     ti_feeds=self.conf.wait_for_TI_to_finish(),
                 )
                 self.print("Starting modules", 1, 0)
+                # initialize_filter must be called after the update manager
+                # is started, and before the modules start. why? because
+                # update manager updates the iocs that the bloom filters need
+                self.bloom_filters_man.initialize_filter()
                 self.proc_man.load_modules()
                 # give outputprocess time to print all the started modules
                 time.sleep(0.5)

slips_files/common/abstracts/imodule.py

@@ -11,6 +11,7 @@
     Optional,
 )
 from slips_files.common.printer import Printer
+from slips_files.core.helpers.bloom_filters_manager import BFManager
 from slips_files.core.output import Output
 from slips_files.common.slips_utils import utils
 from slips_files.core.database.database_manager import DBManager
@@ -38,6 +39,7 @@ def __init__(
         slips_args,
         conf,
         ppid: int,
+        bloom_filters_manager: BFManager,
         **kwargs,
     ):
         Process.__init__(self)
@@ -53,6 +55,7 @@ def __init__(
         # used to tell all slips.py children to stop
         self.termination_event: Event = termination_event
         self.logger = logger
+        self.bloom_filters: BFManager = bloom_filters_manager
         self.printer = Printer(self.logger, self.name)
         self.db = DBManager(
             self.logger, self.output_dir, self.redis_port, self.conf, self.ppid

slips_files/core/database/database_manager.py

@@ -367,8 +367,8 @@ def get_modified_tw(self, *args, **kwargs):
     def get_field_separator(self, *args, **kwargs):
         return self.rdb.get_field_separator(*args, **kwargs)
 
-    def store_tranco_whitelisted_domain(self, *args, **kwargs):
-        return self.rdb.store_tranco_whitelisted_domain(*args, **kwargs)
+    def store_tranco_whitelisted_domains(self, *args, **kwargs):
+        return self.rdb.store_tranco_whitelisted_domains(*args, **kwargs)
 
     def is_whitelisted_tranco_domain(self, *args, **kwargs):
         return self.rdb.is_whitelisted_tranco_domain(*args, **kwargs)
@@ -478,6 +478,9 @@ def get_pids(self, *args, **kwargs):
     def set_org_info(self, *args, **kwargs):
         return self.rdb.set_org_info(*args, **kwargs)
 
+    def set_org_cidrs(self, *args, **kwargs):
+        return self.rdb.set_org_cidrs(*args, **kwargs)
+
     def get_org_info(self, *args, **kwargs):
         return self.rdb.get_org_info(*args, **kwargs)
 
@@ -487,12 +490,21 @@ def get_org_ips(self, *args, **kwargs):
     def set_whitelist(self, *args, **kwargs):
         return self.rdb.set_whitelist(*args, **kwargs)
 
-    def get_all_whitelist(self, *args, **kwargs):
-        return self.rdb.get_all_whitelist(*args, **kwargs)
-
     def get_whitelist(self, *args, **kwargs):
         return self.rdb.get_whitelist(*args, **kwargs)
 
+    def is_whitelisted(self, *args, **kwargs):
+        return self.rdb.is_whitelisted(*args, **kwargs)
+
+    def is_domain_in_org_domains(self, *args, **kwargs):
+        return self.rdb.is_domain_in_org_domains(*args, **kwargs)
+
+    def is_asn_in_org_asn(self, *args, **kwargs):
+        return self.rdb.is_asn_in_org_asn(*args, **kwargs)
+
+    def is_ip_in_org_ips(self, *args, **kwargs):
+        return self.rdb.is_ip_in_org_cidrs(*args, **kwargs)
+
     def has_cached_whitelist(self, *args, **kwargs):
         return self.rdb.has_cached_whitelist(*args, **kwargs)
 

slips_files/core/database/redis_db/alert_handler.py

@@ -53,7 +53,7 @@ def mark_profile_as_malicious(self, profileid: ProfileID):
 
     def get_malicious_profiles(self):
         """returns profiles that generated an alert"""
-        self.r.smembers(self.constants.MALICIOUS_PROFILES)
+        return self.r.smembers(self.constants.MALICIOUS_PROFILES)
 
     def set_evidence_causing_alert(self, alert: Alert):
         """

slips_files/core/database/redis_db/constants.py

@@ -34,7 +34,6 @@ class Constants:
     PIDS = "PIDs"
     MAC = "MAC"
     MODIFIED_TIMEWINDOWS = "ModifiedTW"
-    ORG_INFO = "OrgInfo"
     ACCUMULATED_THREAT_LEVELS = "accumulated_threat_levels"
     TRANCO_WHITELISTED_DOMAINS = "tranco_whitelisted_domains"
     WHITELIST = "whitelist"
@@ -66,7 +65,7 @@ class Constants:
     BLOCKED_PROFILES_AND_TWS = "BlockedProfTW"
     PROFILES = "profiles"
     NUMBER_OF_ALERTS = "number_of_alerts"
-    KNOWN_FPS = "known_fps"
+    KNOWN_FP_MD5_HASHES = "known_fps"
     WILL_SLIPS_HAVE_MORE_FLOWS = "will_slips_have_more_flows"
     SUBS_WHO_PROCESSED_MSG = "number_of_subscribers_who_processed_this_msg"
     FLOWS_ANALYZED_BY_ALL_MODULES_PER_MIN = "flows_analyzed_per_minute"