[ptimec] trying some optimizations

Author: tstack

configure.ac

@@ -8,6 +8,7 @@ AC_PREFIX_DEFAULT(/usr)
 
 AC_CANONICAL_HOST
 AX_PROG_CC_FOR_BUILD
+AX_PROG_CXX_FOR_BUILD
 
 AX_PTHREAD()
 LIBS="$PTHREAD_LIBS $LIBS"

m4/ax_prog_cxx_for_build.m4

@@ -0,0 +1,114 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_prog_cxx_for_build.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_PROG_CXX_FOR_BUILD
+#
+# DESCRIPTION
+#
+#   This macro searches for a C++ compiler that generates native
+#   executables, that is a C++ compiler that surely is not a cross-compiler.
+#   This can be useful if you have to generate source code at compile-time
+#   like for example GCC does.
+#
+#   The macro sets the CXX_FOR_BUILD and CXXCPP_FOR_BUILD macros to anything
+#   needed to compile or link (CXX_FOR_BUILD) and preprocess
+#   (CXXCPP_FOR_BUILD). The value of these variables can be overridden by
+#   the user by specifying a compiler with an environment variable (like you
+#   do for standard CXX).
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Paolo Bonzini <bonzini@gnu.org>
+#   Copyright (c) 2012 Avionic Design GmbH
+#
+#   Based on the AX_PROG_CC_FOR_BUILD macro by Paolo Bonzini.
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 5
+
+AU_ALIAS([AC_PROG_CXX_FOR_BUILD], [AX_PROG_CXX_FOR_BUILD])
+AC_DEFUN([AX_PROG_CXX_FOR_BUILD], [dnl
+AC_REQUIRE([AX_PROG_CC_FOR_BUILD])dnl
+AC_REQUIRE([AC_PROG_CXX])dnl
+AC_REQUIRE([AC_PROG_CXXCPP])dnl
+AC_REQUIRE([AC_CANONICAL_HOST])dnl
+
+dnl Use the standard macros, but make them use other variable names
+dnl
+pushdef([ac_cv_prog_CXXCPP], ac_cv_build_prog_CXXCPP)dnl
+pushdef([ac_cv_prog_gxx], ac_cv_build_prog_gxx)dnl
+pushdef([ac_cv_prog_cxx_works], ac_cv_build_prog_cxx_works)dnl
+pushdef([ac_cv_prog_cxx_cross], ac_cv_build_prog_cxx_cross)dnl
+pushdef([ac_cv_prog_cxx_g], ac_cv_build_prog_cxx_g)dnl
+pushdef([CXX], CXX_FOR_BUILD)dnl
+pushdef([CXXCPP], CXXCPP_FOR_BUILD)dnl
+pushdef([GXX], GXX_FOR_BUILD)dnl
+pushdef([CXXFLAGS], CXXFLAGS_FOR_BUILD)dnl
+pushdef([CPPFLAGS], CPPFLAGS_FOR_BUILD)dnl
+pushdef([CXXCPPFLAGS], CXXCPPFLAGS_FOR_BUILD)dnl
+pushdef([host], build)dnl
+pushdef([host_alias], build_alias)dnl
+pushdef([host_cpu], build_cpu)dnl
+pushdef([host_vendor], build_vendor)dnl
+pushdef([host_os], build_os)dnl
+pushdef([ac_cv_host], ac_cv_build)dnl
+pushdef([ac_cv_host_alias], ac_cv_build_alias)dnl
+pushdef([ac_cv_host_cpu], ac_cv_build_cpu)dnl
+pushdef([ac_cv_host_vendor], ac_cv_build_vendor)dnl
+pushdef([ac_cv_host_os], ac_cv_build_os)dnl
+pushdef([ac_tool_prefix], ac_build_tool_prefix)dnl
+pushdef([am_cv_CXX_dependencies_compiler_type], am_cv_build_CXX_dependencies_compiler_type)dnl
+pushdef([cross_compiling], cross_compiling_build)dnl
+
+cross_compiling_build=no
+
+ac_build_tool_prefix=
+AS_IF([test -n "$build"],      [ac_build_tool_prefix="$build-"],
+      [test -n "$build_alias"],[ac_build_tool_prefix="$build_alias-"])
+
+AC_LANG_PUSH([C++])
+AC_PROG_CXX
+AC_PROG_CXXCPP
+
+dnl Restore the old definitions
+dnl
+popdef([cross_compiling])dnl
+popdef([am_cv_CXX_dependencies_compiler_type])dnl
+popdef([ac_tool_prefix])dnl
+popdef([ac_cv_host_os])dnl
+popdef([ac_cv_host_vendor])dnl
+popdef([ac_cv_host_cpu])dnl
+popdef([ac_cv_host_alias])dnl
+popdef([ac_cv_host])dnl
+popdef([host_os])dnl
+popdef([host_vendor])dnl
+popdef([host_cpu])dnl
+popdef([host_alias])dnl
+popdef([host])dnl
+popdef([CXXCPPFLAGS])dnl
+popdef([CPPFLAGS])dnl
+popdef([CXXFLAGS])dnl
+popdef([CXXCPP])dnl
+popdef([CXX])dnl
+popdef([ac_cv_prog_cxx_g])dnl
+popdef([ac_cv_prog_cxx_cross])dnl
+popdef([ac_cv_prog_cxx_works])dnl
+popdef([ac_cv_prog_gxx])dnl
+popdef([ac_cv_prog_CXXCPP])dnl
+
+dnl restore global variables (dependent on the current
+dnl language after popping):
+AC_LANG_POP([C++])
+
+dnl Finally, set Makefile variables
+dnl
+AC_SUBST([CXXFLAGS_FOR_BUILD])dnl
+AC_SUBST([CXXCPPFLAGS_FOR_BUILD])dnl
+])

src/CMakeLists.txt

@@ -26,7 +26,7 @@ add_subdirectory(yajlpp)
 add_executable(bin2c bin2c.hh ../tools/bin2c.c)
 target_link_libraries(bin2c ZLIB::ZLIB)
 
-add_executable(ptimec ptimec.hh ptimec.c)
+add_executable(ptimec ptimec.hh ptimec.cc)
 
 set(TIME_FORMATS
         "@%@"
@@ -133,8 +133,10 @@ set(TIME_FORMATS
 
 set(GEN_SRCS "")
 
-add_custom_command(OUTPUT time_fmts.cc COMMAND ptimec ${TIME_FORMATS} >
-        time_fmts.cc)
+add_custom_command(
+        OUTPUT time_fmts.cc
+        DEPENDS ptimec
+        COMMAND ptimec ${TIME_FORMATS} > time_fmts.cc)
 
 add_library(lnavdt STATIC config.h.in ptimec.hh ptimec_rt.cc time_fmts.cc)
 target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR} third-party/date/include)
@@ -334,7 +336,10 @@ add_library(
         third-party/scnlib/include/scn/istream.h
         third-party/scnlib/include/scn/scan.h
 )
-target_include_directories(cppscnlib PRIVATE third-party/scnlib/src/deps/fast_float/single_include)
+target_include_directories(
+        cppscnlib PRIVATE
+        third-party/scnlib/src
+        third-party/scnlib/src/deps/fast_float/single_include)
 target_include_directories(cppscnlib PUBLIC third-party/scnlib/include)
 
 add_library(

src/Makefile.am

@@ -90,7 +90,7 @@ builtin-sh-scripts.cc: $(BIN2C_PATH) $(BUILTIN_SHSCRIPTS)
 
 include time_formats.am
 
-time_fmts.cc: ptimec$(BUILD_EXEEXT)
+time_fmts.cc: ptimec$(BUILD_EXEEXT) time_formats.am
 	$(PTIME_V)./ptimec$(BUILD_EXEEXT) $(TIME_FORMATS) > $@
 
 if HAVE_RE2C
@@ -556,11 +556,11 @@ lnav_test_SOURCES = \
     $(PLUGIN_SRCS)
 
 if CROSS_COMPILING
-ptimec$(BUILD_EXEEXT): ptimec.c
-	$(AM_V_CC) $(CC_FOR_BUILD) $(CPPFLAGS_FOR_BUILD) $(LDFLAGS_FOR_BUILD) -g3 -o $@ $? -fPIE
+ptimec$(BUILD_EXEEXT): ptimec.cc
+	$(AM_V_CXX) $(CXX_FOR_BUILD) $(CPPFLAGS_FOR_BUILD) $(LDFLAGS_FOR_BUILD) -g3 -o $@ $? -fPIE
 else
-ptimec$(BUILD_EXEEXT): ptimec.c
-	$(AM_V_CC) $(CC) $(CPPFLAGS) $(LDFLAGS) -g3 -o $@ $? -fPIE
+ptimec$(BUILD_EXEEXT): ptimec.cc
+	$(AM_V_CXX) $(CXX) $(CPPFLAGS) $(LDFLAGS) -g3 -o $@ $? -fPIE
 endif
 
 if HAVE_RE2C
@@ -572,7 +572,7 @@ EXTRA_DIST = \
     third-party/prqlc-c/Cargo.toml \
     third-party/prqlc-c/cbindgen.toml \
     third-party/prqlc-c/src/lib.rs \
-	ptimec.c
+	ptimec.cc
 
 CLEANFILES = \
 	ptimec$(BUILD_EXEEXT)

src/base/is_utf8.cc

@@ -67,10 +67,6 @@ is_utf8(string_fragment str, std::optional<unsigned char> terminator)
     ssize_t i = 0, valid_end = 0;
 
     while (i < str.length()) {
-        if (ustr[i] == '\x1b') {
-            retval.usr_has_ansi = true;
-        }
-
         if (terminator && ustr[i] == terminator.value()) {
             retval.usr_remaining = str.substr(i + 1);
             break;
@@ -86,6 +82,8 @@ is_utf8(string_fragment str, std::optional<unsigned char> terminator)
         if (ustr[i] <= 0x7F) /* 00..7F */ {
             if (ustr[i] == '\t') {
                 retval.usr_column_width_guess += 7;
+            } else if (ustr[i] == '\x1b') {
+                retval.usr_has_ansi = true;
             }
             i += 1;
         } else if (ustr[i] >= 0xC2 && ustr[i] <= 0xDF) /* C2..DF 80..BF */ {

src/line_buffer.cc

@@ -135,8 +135,7 @@ class lock_hack {
 
 #define Z_BUFSIZE      65536U
 #define SYNCPOINT_SIZE (1024 * 1024)
-line_buffer::gz_indexed::
-gz_indexed()
+line_buffer::gz_indexed::gz_indexed()
 {
     if ((this->inbuf = auto_mem<Bytef>::malloc(Z_BUFSIZE)) == nullptr) {
         throw std::bad_alloc();
@@ -368,14 +367,12 @@ line_buffer::gz_indexed::read(void* buf, size_t offset, size_t size)
     return bytes;
 }
 
-line_buffer::
-line_buffer()
+line_buffer::line_buffer()
 {
     ensure(this->invariant());
 }
 
-line_buffer::~
-line_buffer()
+line_buffer::~line_buffer()
 {
     if (this->lb_loader_future.valid()) {
         this->lb_loader_future.wait();
@@ -804,6 +801,8 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
         this->lb_line_has_ansi = std::move(this->lb_alt_line_has_ansi);
         this->lb_alt_line_has_ansi.clear();
         this->lb_stats.s_used_preloads += 1;
+        this->lb_next_line_start_index = 0;
+        this->lb_next_buffer_offset = 0;
     }
     if (this->in_range(start)
         && (max_length == 0 || this->in_range(start + max_length - 1)))
@@ -1114,22 +1113,43 @@ line_buffer::load_next_line(file_range prev_line)
         if (!this->lb_line_starts.empty()) {
             auto buffer_offset = offset - this->lb_file_offset;
 
-            auto start_iter = std::lower_bound(this->lb_line_starts.begin(),
-                                               this->lb_line_starts.end(),
-                                               buffer_offset);
-            if (start_iter != this->lb_line_starts.end()) {
+            if (this->lb_next_buffer_offset == buffer_offset) {
+                auto start_iter = this->lb_line_starts.begin()
+                    + this->lb_next_line_start_index;
                 auto next_line_iter = start_iter + 1;
-
-                // log_debug("found offset %d %d", buffer_offset, *start_iter);
                 if (next_line_iter != this->lb_line_starts.end()) {
                     utf8_end = *next_line_iter - 1 - *start_iter;
                     found_in_cache = true;
                     lf = line_start + utf8_end;
+
+                    this->lb_next_buffer_offset = *next_line_iter;
+                    this->lb_next_line_start_index += 1;
                 } else {
                     // log_debug("no next iter");
                 }
             } else {
-                // log_debug("no buffer_offset found");
+                auto start_iter = std::lower_bound(this->lb_line_starts.begin(),
+                                                   this->lb_line_starts.end(),
+                                                   buffer_offset);
+                if (start_iter != this->lb_line_starts.end()) {
+                    auto next_line_iter = start_iter + 1;
+
+                    // log_debug("found offset %d %d", buffer_offset,
+                    // *start_iter);
+                    if (next_line_iter != this->lb_line_starts.end()) {
+                        utf8_end = *next_line_iter - 1 - *start_iter;
+                        found_in_cache = true;
+                        lf = line_start + utf8_end;
+
+                        this->lb_next_line_start_index = std::distance(
+                            this->lb_alt_line_starts.begin(), next_line_iter);
+                        this->lb_next_buffer_offset = *next_line_iter;
+                    } else {
+                        // log_debug("no next iter");
+                    }
+                } else {
+                    // log_debug("no buffer_offset found");
+                }
             }
         }
 
@@ -1241,8 +1261,7 @@ line_buffer::load_next_line(file_range prev_line)
             (size_t) retval.li_file_range.fr_size,
         };
 
-        auto scan_res = scn::scan<int64_t, int64_t, char>(sv,
-                                  "{}.{}:{};");
+        auto scan_res = scn::scan<int64_t, int64_t, char>(sv, "{}.{}:{};");
         if (scan_res) {
             auto& [tv_sec, tv_usec, level] = scan_res->values();
             retval.li_timestamp.tv_sec = tv_sec;
@@ -1317,8 +1336,8 @@ line_buffer::get_available()
             static_cast<file_ssize_t>(this->lb_buffer.size())};
 }
 
-line_buffer::gz_indexed::indexDict::
-indexDict(const z_stream& s, const file_size_t size)
+line_buffer::gz_indexed::indexDict::indexDict(const z_stream& s,
+                                              const file_size_t size)
 {
     assert((s.data_type & GZ_END_OF_BLOCK_MASK));
     assert(!(s.data_type & GZ_END_OF_FILE_MASK));

src/line_buffer.hh

@@ -371,6 +371,8 @@ private:
     file_off_t lb_last_line_offset{-1}; /*< */
 
     std::vector<uint32_t> lb_line_starts;
+    file_off_t lb_next_buffer_offset{0};
+    size_t lb_next_line_start_index{0};
     std::vector<bool> lb_line_is_utf;
     std::vector<bool> lb_line_has_ansi;
     stats lb_stats;

src/log_format.cc

@@ -228,8 +228,7 @@ std::optional<std::string>
 log_format::opid_descriptor::matches(const string_fragment& sf) const
 {
     if (this->od_extractor.pp_value) {
-        static thread_local auto desc_md
-            = lnav::pcre2pp::match_data::unitialized();
+        thread_local auto desc_md = lnav::pcre2pp::match_data::unitialized();
 
         auto desc_match_res = this->od_extractor.pp_value->capture_from(sf)
                                   .into(desc_md)
@@ -684,7 +683,7 @@ log_format::log_scanf(uint32_t line_number,
     int pat_index = this->last_pattern_index();
 
     while (!done && next_format(fmt, curr_fmt, pat_index)) {
-        static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+        thread_local auto md = lnav::pcre2pp::match_data::unitialized();
 
         auto match_res = fmt[curr_fmt]
                              .pcre->capture_from(line)
@@ -1387,10 +1386,9 @@ external_log_format::scan(logfile& lf,
     int curr_fmt = -1, orig_lock = this->last_pattern_index();
     int pat_index = orig_lock;
     auto line_sf = sbr.to_string_fragment();
+    thread_local auto md = lnav::pcre2pp::match_data::unitialized();
 
     while (::next_format(this->elf_pattern_order, curr_fmt, pat_index)) {
-        thread_local auto md = lnav::pcre2pp::match_data::unitialized();
-
         auto* fpat = this->elf_pattern_order[curr_fmt].get();
         auto* pat = fpat->p_pcre.pp_value.get();
 
@@ -1631,8 +1629,8 @@ external_log_format::scan(logfile& lf,
                 std::optional<double> dvalue_opt;
                 switch (vd.vd_meta.lvm_kind) {
                     case value_kind_t::VALUE_INTEGER: {
-                        auto scan_res = scn::scan_int<int64_t>(
-                            num_cap->to_string_view());
+                        auto scan_res
+                            = scn::scan_int<int64_t>(num_cap->to_string_view());
                         if (scan_res) {
                             dvalue_opt = scan_res->value();
                         }