[tailer] add a python version of the tailer

Related to #1601

Author: tstack

NEWS.md

@@ -21,6 +21,12 @@ Features:
   the default for a particular file.  Files with
   content that lie completely outside of the cutoff
   will be closed to reduce resource usage.
+* Broadened support for viewing files on remote hosts.
+  The "tailer" program that is transferred to the
+  remote host to monitor files and perform other tasks
+  has been translated to Python3.  If the remote host
+  does not have Python, lnav will fall back to the APE
+  binary.
 * The "Files" panel now shows a progress bar for each
   file as it is being indexed and finishes with a
   check-mark if indexing was successful, a warning

docs/source/usage.rst

@@ -108,15 +108,22 @@ file.
 
 .. note::
 
-  Remote file access is implemented by transferring an
-  `αcτµαlly pδrταblε εxεcµταblε <https://justine.lol/ape.html>`_ to the
-  destination and invoking it.  An APE binary can run on most any x86_64
-  machine and OS (i.e. MacOS, Linux, FreeBSD, Windows).  The binary is
-  baked into the lnav executable itself, so there is no extra setup that
+  Remote file access is implemented through a "tailer" executable that is
+  transferred to the host.  The tailer handles requests from lnav for
+  synchronizing file blocks, completing path names, previewing files,
+  and so on.
+
+  There are two implementations of the tailer: Python3 and an
+  `αcτµαlly pδrταblε εxεcµταblε <https://justine.lol/ape.html>`_.
+  If the target host does not have Python3 installed, the APE binary will
+  be used.  An APE binary can run on most any x86_64 machine and OS
+  (i.e. MacOS, Linux, FreeBSD, Windows).  The implementations are baked
+  into the lnav executable itself.  So, there is no extra setup that
   needs to be done on the remote machine.
 
-  The binary file is named ``tailer.bin.XXXXXX`` where *XXXXXX* is 6 random digits.
-  The file is, under normal circumstancies, deleted immediately.
+  The tailer is copied to the user's home directory and is named
+  ``tailer.bin.XXXXXX`` where *XXXXXX* is 6 random digits.
+  Under normal circumstances, the file should be deleted immediately.
 
 Command Output
 ^^^^^^^^^^^^^^

src/CMakeLists.txt

@@ -29,10 +29,12 @@ add_subdirectory(yajl)
 add_subdirectory(yajlpp)
 
 add_executable(bin2c bin2c.hh ../tools/bin2c.c)
+target_include_directories(bin2c PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(bin2c ZLIB::ZLIB)
 
 add_executable(ptimec ptimec.hh ptimec.cc)
 
+
 set(TIME_FORMATS
         "@%@"
         "%Y-%m-%d %H:%M:%S.%f%z"
@@ -146,7 +148,10 @@ add_custom_command(
         COMMAND ptimec ${TIME_FORMATS} > time_fmts.cc)
 
 add_library(lnavdt STATIC config.h.in ptimec.hh ptimec_spec.hh ptimec_rt.cc time_fmts.cc)
-target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR} third-party/date/include)
+target_include_directories(lnavdt PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}
+        ${CMAKE_CURRENT_BINARY_DIR}
+        third-party/date/include)
 
 function(bin2c)
     cmake_parse_arguments(BIN2C_ "" "VARNAME" "" ${ARGN})
@@ -221,7 +226,6 @@ set(FORMAT_FILES
         formats/snaplogic_log.json
         formats/sssd_log.json
         formats/strace_log.json
-        formats/sudo_log.json
         formats/syslog_log.json
         formats/s3_log.json
         formats/tcf_log.json

src/base/date_time_scanner.hh

@@ -38,6 +38,7 @@
 #include <sys/types.h>
 
 #include "date/tz.h"
+#include "intern_string.hh"
 #include "time_util.hh"
 
 /**

src/base/time_util.hh

@@ -43,7 +43,6 @@
 #include "config.h"
 #include "date/date.h"
 #include "date/tz.h"
-#include "humanize.time.hh"
 
 inline std::chrono::microseconds
 to_us(const timeval& tv)

src/cmds.io.cc

@@ -35,6 +35,7 @@
 #include "base/fs_util.hh"
 #include "base/humanize.hh"
 #include "base/humanize.network.hh"
+#include "base/humanize.time.hh"
 #include "base/itertools.enumerate.hh"
 #include "base/itertools.hh"
 #include "base/paths.hh"

src/formats/esx_syslog_log.json

@@ -5,10 +5,10 @@
         "description": "Format specific to the ESXi syslog",
         "regex": {
             "std": {
-                "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))\\s+(?<level>\\w+\\((?<syslog_pri>\\d+)\\))(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[:]+|[^:]+))(?:\\[(?<log_pid>\\d+)\\])?):\\s*(?:(?:(?:debug|info|warning|error) )?\\[(?<logger>[^ ]+)(?: sub=(?<sub>[@\\.\\-\\w\\(\\)\\[\\]]+))?(?: op[iI][dD]=(?<opid>[^ \\]]+))?(?: update=(?<update_num>\\d+))?(?: sid=(?<sid>\\w+))?(?: user=(?<user>(?:[^\\]]|<[^>]+>)+))?(?: reason=(?<reason>[^\\]]+))?\\]\\s*)?(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
+                "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?)Z)\\s+(?<level>\\w+\\((?<syslog_pri>\\d+)\\))(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[:]+|[^:]+))(?:\\[(?<log_pid>\\d+)\\])?):\\s*(?:(?:(?:debug|info|warning|error) )?\\[(?<logger>[^ ]+)(?: sub=(?<sub>[@\\.\\-\\w\\(\\)\\[\\]]+))?(?: op[iI][dD]=(?<opid>[^ \\]]+))?(?: update=(?<update_num>\\d+))?(?: sid=(?<sid>\\w+))?(?: user=(?<user>(?:[^\\]]|<[^>]+>)+))?(?: reason=(?<reason>[^\\]]+))?\\]\\s*)?(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
             },
             "host": {
-                "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))\\s+(?<level>\\w+\\((?<syslog_pri>\\d+)\\))(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?:host-(?<log_pid>\\d+))?)\\s+(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
+                "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?)Z)\\s+(?<level>\\w+\\((?<syslog_pri>\\d+)\\))(?:\\[\\+\\]|\\+)?(?:(?: (?<log_syslog_tag>(?:host-(?<log_pid>\\d+))?)\\s+(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
             },
             "notime": {
                 "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2})\\s+(?<level>\\w+\\((?<syslog_pri>\\d+)\\))\\s+(?<log_procname>[^\\[]+)\\[(?<log_pid>\\d+)\\]:\\s(?<new_time>\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(?<body>.*)"

src/line_buffer.hh

@@ -44,6 +44,7 @@
 
 #include "base/auto_fd.hh"
 #include "base/auto_mem.hh"
+#include "base/enum_util.hh"
 #include "base/file_range.hh"
 #include "base/is_utf8.hh"
 #include "base/lnav.gzip.hh"

src/lnav.prompt.cc

@@ -1067,7 +1067,8 @@ prompt::get_cmd_parameter_completion(textview_curses& tc,
                     = poss_paths | lnav::itertools::similar_to(str, 10)
                     | lnav::itertools::map([&str](const std::string& path_str) {
                           auto escaped_path = shlex::escape(path_str);
-                          if (path_str.find_last_of("/\\") == std::string::npos
+                          if ((path_str.find_last_of("/\\") == std::string::npos
+                               && path_str.back() != ':')
                               || path_str == str)
                           {
                               escaped_path.push_back(' ');

src/tailer/CMakeLists.txt

@@ -8,17 +8,25 @@ add_library(tailerpp tailerpp.hh tailerpp.cc)
 target_link_libraries(tailerpp base)
 
 add_custom_command(
-  OUTPUT tailerbin.h tailerbin.cc
-  COMMAND bin2c -n tailer_bin tailerbin tailer
-  DEPENDS bin2c tailer)
+        OUTPUT tailerbin.h tailerbin.cc
+        COMMAND bin2c -n tailer_bin tailerbin ${CMAKE_CURRENT_SOURCE_DIR}/tailer.main.py tailer
+        DEPENDS bin2c tailer)
 
-add_library(tailerservice tailer.looper.hh tailer.looper.cc
-                          tailer.looper.cfg.hh tailerbin.h tailerbin.cc)
-target_include_directories(tailerservice PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+add_library(tailerservice
+        tailer.looper.hh
+        tailer.looper.cc
+        tailer.looper.cfg.hh
+        tailerbin.h
+        tailerbin.cc)
+target_include_directories(tailerservice
+        PUBLIC
+        ../third-party/digestible/include
+        ${CMAKE_CURRENT_BINARY_DIR}
+)
 target_link_libraries(tailerservice base)
 
 add_executable(drive_tailer drive_tailer.cc)
 
 target_include_directories(drive_tailer PUBLIC . .. ../fmtlib
-                                               ${CMAKE_CURRENT_BINARY_DIR}/..)
+        ${CMAKE_CURRENT_BINARY_DIR}/..)
 target_link_libraries(drive_tailer base tailercommon tailerpp ZLIB::ZLIB)