Merge pull request #35 from Textualize/headers

Headers

Author: willmcgugan

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "toolong"
-version = "1.2.0"
+version = "1.2.1"
 description = "A terminal log file viewer / tailer / analyzer"
 authors = ["Will McGugan <will@textualize.io>"]
 license = "MIT"

src/toolong/log_file.py

@@ -214,16 +214,15 @@ def scan_timestamps(
         scan_time = monotonic()
         scan = self.timestamp_scanner.scan
         line_no = 0
-        timestamp = self.get_create_time() or datetime.now()
         position = 0
         results: list[tuple[int, int, float]] = []
         append = results.append
         get_length = results.__len__
         while line_bytes := log_mmap.readline():
             line = line_bytes.decode("utf-8", errors="replace")
-            timestamp = scan(line) or timestamp
-            append((line_no, position, timestamp.timestamp()))
+            timestamp = scan(line)
             position += len(line_bytes)
+            append((line_no, position, timestamp.timestamp() if timestamp else 0.0))
             line_no += 1
             if (
                 results

src/toolong/log_lines.py

@@ -342,8 +342,7 @@ def run_scan(self, save_merge: str | None = None) -> None:
     def merge_log_files(self) -> None:
         worker = get_current_worker()
         self._merge_lines = []
-        merge_lines: list[tuple[float, int, LogFile]] = self._merge_lines
-        append_meta = merge_lines.append
+        merge_lines = self._merge_lines
 
         for log_file in self.log_files:
             try:
@@ -363,13 +362,17 @@ def merge_log_files(self) -> None:
         for log_file in self.log_files:
             if not log_file.is_open:
                 continue
-            append = self._line_breaks[log_file].append
+            line_breaks = self._line_breaks[log_file]
+            append = line_breaks.append
+            meta: list[tuple[float, int, LogFile]] = []
+            append_meta = meta.append
             for timestamps in log_file.scan_timestamps():
                 break_position = 0
+
                 for line_no, break_position, timestamp in timestamps:
-                    if break_position:
-                        append_meta((timestamp, line_no, log_file))
-                        append(break_position)
+                    append_meta((timestamp, line_no, log_file))
+                    append(break_position)
+                append(log_file.size)
 
                 self.post_message(
                     ScanProgress(
@@ -383,6 +386,20 @@ def merge_log_files(self) -> None:
                     )
                     return
 
+            # Header may be missing timestamp, so we will attempt to back fill timestamps
+            seconds = 0.0
+            for offset, (seconds, line_no, log_file) in enumerate(meta):
+                if seconds:
+                    for index, (_seconds, line_no, log_file) in zip(
+                        range(offset), meta
+                    ):
+                        meta[index] = (seconds, line_no, log_file)
+                    break
+                if offset > 10:
+                    # May be pointless to scan the entire thing
+                    break
+            self._merge_lines.extend(meta)
+
             position += log_file.size
 
         merge_lines.sort(key=itemgetter(0, 1))
@@ -447,11 +464,12 @@ def get_log_file_from_index(self, index: int) -> tuple[LogFile, int]:
     def index_to_span(self, index: int) -> tuple[LogFile, int, int]:
         log_file, index = self.get_log_file_from_index(index)
         line_breaks = self._line_breaks.setdefault(log_file, [])
+        scan_start = 0 if self._merge_lines else self._scan_start
         if not line_breaks:
-            return (log_file, self._scan_start, self._scan_start)
+            return (log_file, scan_start, self._scan_start)
         index = clamp(index, 0, len(line_breaks))
         if index == 0:
-            return (log_file, self._scan_start, line_breaks[0])
+            return (log_file, scan_start, line_breaks[0])
         start = line_breaks[index - 1]
         end = (
             line_breaks[index]