Merge pull request #1207 from stratosphereips/alya/fix-analyzing_flows_when_-w_is_given

Fix clearing zeek files from the database when the web interface is started using -w

Author: AlyaGomaa

managers/process_manager.py

@@ -86,7 +86,7 @@ def start_output_process(self, stderr, slips_logfile, stdout=""):
             verbose=self.main.args.verbose or 0,
             debug=self.main.args.debug,
             input_type=self.main.input_type,
-            stop_daemon=self.main.args.stopdaemon,
+            create_logfiles=False if self.main.args.stopdaemon else True,
         )
         self.slips_logfile = output_process.slips_logfile
         return output_process

managers/ui_manager.py

@@ -24,14 +24,16 @@ def check_if_webinterface_started(self):
             # to make sure this function is only executed once
             delattr(self, "webinterface_return_value")
             return
+
         if not self.webinterface_return_value.get():
             # to make sure this function is only executed once
             delattr(self, "webinterface_return_value")
             return
 
         self.main.print(
             f"Slips {green('web interface')} running on "
-            f"http://localhost:{self.web_interface_port}/\n"
+            f"http://localhost:{self.web_interface_port}/ "
+            f"[PID {green(self.web_interface_pid)}]\n"
             f"The port will stay open after slips is done with the "
             f"analysis unless you manually kill it.\n"
             f"You need to kill it to be able to start the web interface "
@@ -70,6 +72,7 @@ def run_webinterface():
             )
 
             self.main.db.store_pid("Web Interface", webinterface.pid)
+            self.web_interface_pid = webinterface.pid
             # we'll assume that it started, and if not, the return value will
             # immediately change and this thread will
             # print an error

slips_files/core/database/database_manager.py

@@ -45,14 +45,11 @@ def __init__(
         # the existing one
         self.sqlite = None
         if start_sqlite:
-            self.sqlite = self.create_sqlite_db(output_dir)
+            self.sqlite = SQLiteDB(self.logger, output_dir)
 
     def print(self, *args, **kwargs):
         return self.printer.print(*args, **kwargs)
 
-    def create_sqlite_db(self, output_dir):
-        return SQLiteDB(self.logger, output_dir)
-
     @classmethod
     def read_configuration(cls):
         conf = ConfigParser()

slips_files/core/database/redis_db/database.py

@@ -154,13 +154,17 @@ def _set_redis_options(cls):
         Updates the default slips options based on the -s param,
         writes the new configs to cls._conf_file
         """
+        # to fix redis.exceptions.ResponseError MISCONF Redis is
+        # configured to save RDB snapshots
+        # configure redis to stop writing to dump.rdb when an error
+        # occurs without throwing errors in slips
         cls._options = {
             "daemonize": "yes",
             "stop-writes-on-bgsave-error": "no",
             "save": '""',
             "appendonly": "no",
         }
-
+        # -s for saving the db
         if "-s" not in sys.argv:
             return
 
@@ -188,6 +192,8 @@ def _set_redis_options(cls):
     @classmethod
     def _read_configuration(cls):
         conf = ConfigParser()
+        # Should we delete the previously stored data in the DB when we start?
+        # By default False. Meaning we don't DELETE the DB by default.
         cls.deletePrevdb: bool = conf.delete_prev_db()
         cls.disabled_detections: List[str] = conf.disabled_detections()
         cls.width = conf.get_tw_width_as_float()
@@ -227,6 +233,11 @@ def init_redis_server(cls) -> Tuple[bool, str]:
             if not connected:
                 return False, err
 
+            # these are the cases that we DO NOT flush the db when we
+            # connect to it, because we need to use it
+            # -d means Read an analysed file (rdb) from disk.
+            # -S stop daemon
+            # -cb clears the blocking chain
             if (
                 cls.deletePrevdb
                 and not (
@@ -237,6 +248,7 @@ def init_redis_server(cls) -> Tuple[bool, str]:
                 # when stopping the daemon, don't flush bc we need to get
                 # the PIDS to close slips files
                 cls.r.flushdb()
+                cls.r.delete(cls.constants.ZEEK_FILES)
 
             # Set the memory limits of the output buffer,
             # For normal clients: no limits
@@ -245,12 +257,6 @@ def init_redis_server(cls) -> Tuple[bool, str]:
             cls.change_redis_limits(cls.r)
             cls.change_redis_limits(cls.rcache)
 
-            # to fix redis.exceptions.ResponseError MISCONF Redis is
-            # configured to save RDB snapshots
-            # configure redis to stop writing to dump.rdb when an error
-            # occurs without throwing errors in slips
-            # Even if the DB is not deleted. We need to delete some temp data
-            cls.r.delete(cls.constants.ZEEK_FILES)
             return True, ""
         except RuntimeError as err:
             return False, str(err)

slips_files/core/database/sqlite_db/database.py

@@ -34,7 +34,8 @@ def __init__(self, logger: Output, output_dir: str):
 
     def connect(self):
         """
-        Creates the db if it doesn't exist and connects to it
+        Creates the db if it doesn't exist and connects to it.
+        OR connects to the existing db if it's there.
         """
         db_newly_created = False
         if not os.path.exists(self._flows_db):

slips_files/core/input.py

@@ -333,6 +333,7 @@ def get_earliest_line(self):
         """
         # Now read lines in order. The line with the earliest timestamp first
         files_sorted_by_ts = sorted(self.file_time, key=self.file_time.get)
+
         try:
             # get the file that has the earliest flow
             file_with_earliest_flow = files_sorted_by_ts[0]
@@ -344,7 +345,8 @@ def get_earliest_line(self):
             self.zeek_files = self.db.get_all_zeek_files()
             return False, False
 
-        # comes here if we're done with all conn.log flows and it's time to process other files
+        # comes here if we're done with all conn.log flows and it's time to
+        # process other files
         earliest_line = self.cache_lines[file_with_earliest_flow]
         return earliest_line, file_with_earliest_flow
 
@@ -386,6 +388,7 @@ def read_zeek_files(self) -> int:
             # Delete this line from the cache and the time list
             del self.cache_lines[file_with_earliest_flow]
             del self.file_time[file_with_earliest_flow]
+
             # Get the new list of files. Since new files may have been created by
             # Zeek while we were processing them.
             self.zeek_files = self.db.get_all_zeek_files()
@@ -432,7 +435,8 @@ def read_zeek_folder(self):
         self.bro_timeout = 10
         growing_zeek_dir: bool = self.db.is_growing_zeek_dir()
         if growing_zeek_dir:
-            # slips is given a dir that is growing i.e zeek dir running on an interface
+            # slips is given a dir that is growing i.e zeek dir running on an
+            # interface
             # don't stop zeek or slips
             self.bro_timeout = float("inf")
 

slips_files/core/output.py

@@ -46,7 +46,7 @@ def __init__(
         stderr="output/errors.log",
         slips_logfile="output/slips.log",
         input_type=False,
-        stop_daemon: bool = None,
+        create_logfiles: bool = True,
         stdout="",
     ):
         super().__init__()
@@ -56,15 +56,17 @@ def __init__(
         self.debug = debug
         self.stdout = stdout
         self.input_type = input_type
-        self.stop_daemon = stop_daemon
         self.errors_logfile = stderr
         self.slips_logfile = slips_logfile
-        # if we're using -S, no need to init all the logfiles
+
+        if self.verbose > 2:
+            print(f"Verbosity: {self.verbose}. Debugging: {self.debug}")
+
+        # when we're using -S, no need to init all the logfiles
         # we just need an instance of this class to be able
         # to start the db from the daemon class
-        if not stop_daemon:
+        if create_logfiles:
             self._read_configuration()
-
             self.create_logfile(self.errors_logfile)
             self.log_branch_info(self.errors_logfile)
             self.create_logfile(self.slips_logfile)
@@ -76,8 +78,6 @@ def __init__(
             utils.change_logfiles_ownership(
                 self.slips_logfile, self.UID, self.GID
             )
-            if self.verbose > 2:
-                print(f"Verbosity: {self.verbose}. Debugging: {self.debug}")
 
     def _read_configuration(self):
         conf = ConfigParser()

webinterface/database/database.py

@@ -47,6 +47,7 @@ def get_db_manager_obj(self, port: int = False) -> Optional[DBManager]:
             stdout=os.path.join(output_dir, "slips.log"),
             stderr=os.path.join(output_dir, "errors.log"),
             slips_logfile=os.path.join(output_dir, "slips.log"),
+            create_logfiles=False,
         )
         try:
             return DBManager(