Merge pull request #1079 from stratosphereips/alya/fix_db_issues_in_the_web_interface

Fix db issues in the web interface

Author: AlyaGomaa

managers/metadata_manager.py

@@ -128,7 +128,7 @@ def update_slips_stats_in_the_db(self) -> Tuple[int, Set[str]]:
         updates the number of processed ips, slips internal time,
          and modified tws so far in the db
         """
-        slips_internal_time = float(self.main.db.getSlipsInternalTime()) + 1
+        slips_internal_time = float(self.main.db.get_slips_internal_time()) + 1
 
         # Get the amount of modified profiles since we last checked
         # this is the modification time of the last timewindow

managers/redis_manager.py

@@ -13,7 +13,7 @@
 
 
 class RedisManager:
-    open_servers_pids: Dict[int, int]
+    open_servers_pids: Dict[int, dict]
 
     def __init__(self, main):
         self.main = main
@@ -240,19 +240,19 @@ def get_pid_of_redis_server(self, port: int) -> int:
         return False
 
     @staticmethod
-    def is_comment(line: str) -> True:
+    def is_comment(line: str) -> bool:
         """returns true if the given line is a comment"""
         return (line.startswith("#") or line.startswith("Date")) or len(
             line
         ) < 3
 
-    def get_open_redis_servers(self) -> Dict[int, int]:
+    def get_open_redis_servers(self) -> Dict[int, dict]:
         """
         fills and returns self.open_servers_PIDs
         with PIDs and ports of the redis servers started by slips
         read from running_slips.info.txt
         """
-        self.open_servers_pids = {}
+        self.open_servers_pids: Dict[int, dict] = {}
         try:
             with open(self.running_logfile, "r") as f:
                 for line in f.read().splitlines():
@@ -263,8 +263,29 @@ def get_open_redis_servers(self) -> Dict[int, int]:
                     line = line.split(",")
 
                     try:
-                        pid, port = int(line[3]), int(line[2])
-                        self.open_servers_pids[pid] = port
+                        (
+                            timestamp,
+                            file_or_interface,
+                            port,
+                            pid,
+                            zeek_dir,
+                            output_dir,
+                            slips_pid,
+                            is_daemon,
+                            save_the_db,
+                        ) = line
+
+                        self.open_servers_pids[pid] = {
+                            "timestamp": timestamp,
+                            "file_or_interface": file_or_interface,
+                            "port": port,
+                            "pid": pid,
+                            "zeek_dir": zeek_dir,
+                            "output_dir": output_dir,
+                            "slips_pid": slips_pid,
+                            "is_daemon": is_daemon,
+                            "save_the_db": save_the_db,
+                        }
                     except ValueError:
                         # sometimes slips can't get the server pid and logs "False"
                         # in the logfile instead of the PID
@@ -379,7 +400,8 @@ def flush_redis_server(self, pid: int = None, port: int = None):
             if not hasattr(self, "open_servers_PIDs"):
                 self.get_open_redis_servers()
 
-            port: int = self.open_servers_pids.get(pid, False)
+            pid_info: Dict[str, str] = self.open_servers_pids.get(pid, {})
+            port: int = pid_info.get("port", False)
             if not port:
                 # try to get the port using a cmd
                 port: int = self.get_port_of_redis_server(pid)

modules/cesnet/cesnet.py

@@ -251,7 +251,7 @@ def import_alerts(self):
 
                 src_ips.update({srcip: json.dumps(event_info)})
 
-        self.db.add_ips_to_IoC(src_ips)
+        self.db.add_ips_to_ioc(src_ips)
 
     def pre_main(self):
         utils.drop_root_privs()

modules/flowalerts/conn.py

@@ -227,7 +227,7 @@ def check_multiple_reconnection_attempts(self, profileid, twid, flow):
         # reset the reconnection attempts of this src->dst
         current_reconnections[key] = (0, [])
 
-        self.db.setReconnections(profileid, twid, current_reconnections)
+        self.db.set_reconnections(profileid, twid, current_reconnections)
 
     def is_ignored_ip_data_upload(self, ip):
         """

modules/threat_intelligence/threat_intelligence.py

@@ -693,11 +693,11 @@ def parse_local_ti_file(self, ti_file_path: str) -> bool:
                     )
 
         # Add all loaded malicious ips to the database
-        self.db.add_ips_to_IoC(malicious_ips)
+        self.db.add_ips_to_ioc(malicious_ips)
         # Add all loaded malicious domains to the database
-        self.db.add_domains_to_IoC(malicious_domains)
-        self.db.add_ip_range_to_IoC(malicious_ip_ranges)
-        self.db.add_asn_to_IoC(malicious_asns)
+        self.db.add_domains_to_ioc(malicious_domains)
+        self.db.add_ip_range_to_ioc(malicious_ip_ranges)
+        self.db.add_asn_to_ioc(malicious_asns)
         return True
 
     def __delete_old_source_ips(self, file):
@@ -724,7 +724,7 @@ def __delete_old_source_ips(self, file):
             if data["source"] == file:
                 old_data.append(ip)
         if old_data:
-            self.db.delete_ips_from_IoC_ips(old_data)
+            self.db.delete_ips_from_ioc_ips(old_data)
 
     def __delete_old_source_domains(self, file):
         """Deletes all domain indicators of compromise (IoCs) associated with a specific
@@ -748,7 +748,7 @@ def __delete_old_source_domains(self, file):
             if data["source"] == file:
                 old_data.append(domain)
         if old_data:
-            self.db.delete_domains_from_IoC_domains(old_data)
+            self.db.delete_domains_from_ioc_domains(old_data)
 
     def __delete_old_source_data_from_database(self, data_file):
         """Deletes old indicators of compromise (IoCs) associated with a specific source
@@ -837,7 +837,7 @@ def parse_ja3_file(self, path):
                     }
                 )
         # Add all loaded JA3 to the database
-        self.db.add_ja3_to_IoC(ja3_dict)
+        self.db.add_ja3_to_ioc(ja3_dict)
         return True
 
     def parse_jarm_file(self, path):
@@ -901,7 +901,7 @@ def parse_jarm_file(self, path):
                         "threat_level": threat_level,
                     }
                 )
-        self.db.add_jarm_to_IoC(jarm_dict)
+        self.db.add_jarm_to_ioc(jarm_dict)
         return True
 
     def should_update_local_ti_file(self, path_to_local_ti_file: str) -> bool:
@@ -1206,7 +1206,7 @@ def ip_has_blacklisted_asn(
         if not asn:
             return
 
-        if asn_info := self.db.is_blacklisted_ASN(asn):
+        if asn_info := self.db.is_blacklisted_asn(asn):
             asn_info = json.loads(asn_info)
             self.set_evidence_malicious_asn(
                 ip,
@@ -1359,7 +1359,7 @@ def is_malicious_ip(
             # not malicious
             return False
 
-        self.db.add_ips_to_IoC({ip: json.dumps(ip_info)})
+        self.db.add_ips_to_ioc({ip: json.dumps(ip_info)})
         if is_dns_response:
             self.set_evidence_malicious_ip_in_dns_response(
                 ip,

modules/update_manager/update_manager.py

@@ -569,7 +569,7 @@ def parse_ssl_feed(self, url, full_path):
                     )
                     continue
         # Add all loaded malicious sha1 to the database
-        self.db.add_ssl_sha1_to_IoC(malicious_ssl_certs)
+        self.db.add_ssl_sha1_to_ioc(malicious_ssl_certs)
         return True
 
     async def update_TI_file(self, link_to_download: str) -> bool:
@@ -693,7 +693,7 @@ def update_riskiq_feed(self):
                                 "source": url,
                             }
                         )
-                        self.db.add_domains_to_IoC(malicious_domains_dict)
+                        self.db.add_domains_to_ioc(malicious_domains_dict)
             except KeyError:
                 self.print(
                     f'RiskIQ returned: {response["message"]}. Update Cancelled.',
@@ -852,7 +852,7 @@ def parse_ja3_feed(self, url, ja3_feed_path: str) -> bool:
                         continue
 
             # Add all loaded malicious ja3 to the database
-            self.db.add_ja3_to_IoC(malicious_ja3_dict)
+            self.db.add_ja3_to_ioc(malicious_ja3_dict)
             return True
 
         except Exception:
@@ -895,7 +895,7 @@ def parse_json_ti_feed(self, link_to_download, ti_file_path: str) -> bool:
                         }
                     )
 
-            self.db.add_ips_to_IoC(malicious_ips_dict)
+            self.db.add_ips_to_ioc(malicious_ips_dict)
             return True
 
         if "hole.cert.pl" in link_to_download:
@@ -932,7 +932,7 @@ def parse_json_ti_feed(self, link_to_download, ti_file_path: str) -> bool:
                             "tags": tags,
                         }
                     )
-            self.db.add_domains_to_IoC(malicious_domains_dict)
+            self.db.add_domains_to_ioc(malicious_domains_dict)
             return True
 
     def get_description_column_index(self, header):
@@ -1386,9 +1386,9 @@ def parse_ti_feed(self, feed_link: str, ti_file_path: str) -> bool:
             ti_file_name: str = ti_file_path.split("/")[-1]
             handlers[data_type](ioc, ti_file_name, feed_link, description)
 
-        self.db.add_ips_to_IoC(self.malicious_ips_dict)
-        self.db.add_domains_to_IoC(self.malicious_domains_dict)
-        self.db.add_ip_range_to_IoC(self.malicious_ip_ranges)
+        self.db.add_ips_to_ioc(self.malicious_ips_dict)
+        self.db.add_domains_to_ioc(self.malicious_domains_dict)
+        self.db.add_ip_range_to_ioc(self.malicious_ip_ranges)
         feed.close()
         return True