feat(logfiles): read log files in chunks for efficiency

Refactor log file reading to process in chunks, improving efficiency.
Updated docstrings for better clarity and added detailed argument
descriptions. This change ensures that large log files are handled
more efficiently by reading them in manageable chunks.

Author: mkoura

cardano_node_tests/utils/logfiles.py

@@ -294,21 +294,40 @@ def find_msgs_in_logs(
     timestamp: float,
     only_first: bool = False,
 ) -> list[str]:
-    """Find messages in log."""
+    """Find messages in log.
+
+    Args:
+        regex (str): The regular expression to search for.
+        logfile (Path): The path to the log file.
+        seek_offset (int): The seek offset in the log file.
+        timestamp (float): The timestamp to filter log entries.
+        only_first (bool): Whether to return only the first match.
+
+    Returns:
+        list[str]: A list of matching log lines.
+    """
     regex_comp = re.compile(regex)
     lines_found = []
-    for logfile_rec in _get_rotated_logs(
-        logfile=pl.Path(logfile), seek=seek_offset, timestamp=timestamp
-    ):
+    for logfile_rec in _get_rotated_logs(logfile=logfile, seek=seek_offset, timestamp=timestamp):
         with open(logfile_rec.logfile, encoding="utf-8") as infile:
             infile.seek(logfile_rec.seek)
-            for line in infile:
-                if regex_comp.search(line):
-                    lines_found.append(line)
-                    if only_first:
-                        break
-        if lines_found and only_first:
-            break
+
+            # Read the file in chunks
+            incomplete_line = ""
+            while chunk := infile.read(BUFFER_SIZE):
+                # Prepend any leftover from the last chunk
+                if incomplete_line:
+                    chunk = incomplete_line + chunk
+                lines = chunk.splitlines(keepends=True)  # Preserve line endings
+
+                # Check if the last line is incomplete
+                incomplete_line = lines.pop() if not lines[-1].endswith(("\n", "\r")) else ""
+
+                for line in lines:
+                    if regex_comp.search(line):
+                        lines_found.append(line)
+                        if only_first:
+                            return lines_found
     return lines_found
 
 
@@ -318,7 +337,17 @@ def check_msgs_presence_in_logs(
     state_dir: pl.Path,
     timestamp: float,
 ) -> list[str]:
-    """Check if the expected messages are present in logs."""
+    """Check if the expected messages are present in logs.
+
+    Args:
+        regex_pairs (list[tuple[str, str]]): List of tuples with file globs and regex patterns.
+        seek_offsets (dict[str, int]): Dictionary of file seek offsets.
+        state_dir (Path): Path to the state directory.
+        timestamp (float): Timestamp to filter log entries.
+
+    Returns:
+        list[str]: List of error messages for missing log entries.
+    """
     errors = []
     for files_glob, regex in regex_pairs:
         regex_comp = re.compile(regex)
@@ -337,7 +366,19 @@ def check_msgs_presence_in_logs(
             ):
                 with open(logfile_rec.logfile, encoding="utf-8") as infile:
                     infile.seek(logfile_rec.seek)
-                    for line in infile:
+
+                # Read the file in chunks
+                incomplete_line = ""
+                while chunk := infile.read(BUFFER_SIZE):
+                    # Prepend any leftover from the last chunk
+                    if incomplete_line:
+                        chunk = incomplete_line + chunk
+                    lines = chunk.splitlines(keepends=True)  # Preserve line endings
+
+                    # Check if the last line is incomplete
+                    incomplete_line = lines.pop() if not lines[-1].endswith(("\n", "\r")) else ""
+
+                    for line in lines:
                         if regex_comp.search(line):
                             line_found = True
                             break