[perf] improve stuff related to archive extraction

tstack · tstack · commit 7b5350e85c3a · 2025-11-20T22:05:24.000-08:00
diff --git a/src/archive_manager.cc b/src/archive_manager.cc
@@ -216,7 +216,7 @@ copy_data(const std::string& filename,
                     entry_path.filename().string(),
                     entry_path.parent_path().string()));
             }
-            next_space_check += 1024 * 1024;
+            next_space_check += 10 * 1024 * 1024;
         }
 
         r = archive_read_data_block(ar, &buff, &size, &offset);
diff --git a/src/cmds.io.cc b/src/cmds.io.cc
@@ -1232,13 +1232,13 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 retval = "info: watching -- " + fn;
             } else if (lnav::filesystem::is_glob(fn.c_str())) {
                 loo.with_init_location(file_loc);
-                fc.fc_file_names.emplace(fn, loo);
+                fc.fc_file_names.insert2(fn, loo);
                 files_to_front.emplace_back(
                     loo.loo_filename.empty() ? fn : loo.loo_filename);
                 retval = "info: watching -- " + fn;
             } else if (stat(fn.c_str(), &st) == -1) {
                 if (fn.find(':') != std::string::npos) {
-                    fc.fc_file_names.emplace(fn, loo);
+                    fc.fc_file_names.insert2(fn, loo);
                     retval = "info: watching -- " + fn;
                 } else {
                     auto um = lnav::console::user_message::error(
@@ -1304,7 +1304,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 if (dir_wild[dir_wild.size() - 1] == '/') {
                     dir_wild.resize(dir_wild.size() - 1);
                 }
-                fc.fc_file_names.emplace(dir_wild + "/*", loo);
+                fc.fc_file_names.insert2(dir_wild + "/*", loo);
                 retval = "info: watching -- " + dir_wild;
             } else if (!S_ISREG(st.st_mode)) {
                 auto um = lnav::console::user_message::error(
@@ -1351,7 +1351,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 }
                 if (file_iter == lnav_data.ld_active_files.fc_files.end()) {
                     loo.with_init_location(file_loc);
-                    fc.fc_file_names.emplace(fn, loo);
+                    fc.fc_file_names.insert2(fn, loo);
                     retval = "info: opened -- " + fn;
                     files_to_front.emplace_back(fn);
 
diff --git a/src/file_collection.cc b/src/file_collection.cc
@@ -240,8 +240,12 @@ file_collection::merge(file_collection& other)
     if (!other.fc_file_names.empty()) {
         this->fc_files_generation += 1;
     }
-    for (const auto& fn_pair : other.fc_file_names) {
-        this->fc_file_names[fn_pair.first] = fn_pair.second;
+    if (this->fc_file_names.empty()) {
+        this->fc_file_names = other.fc_file_names;
+    } else {
+        for (const auto& fn_pair : other.fc_file_names) {
+            this->fc_file_names[fn_pair.first] = fn_pair.second;
+        }
     }
     if (!other.fc_files.empty()) {
         for (const auto& lf : other.fc_files) {
@@ -353,7 +357,7 @@ file_collection::watch_logfile(const std::string& filename,
             {
                 file_collection retval;
 
-                retval.fc_file_names.emplace(
+                retval.fc_file_names.insert2(
                     wilddir,
                     logfile_open_options()
                         .with_non_utf_visibility(false)
diff --git a/src/file_collection.hh b/src/file_collection.hh
@@ -36,6 +36,7 @@
 #include <future>
 #include <list>
 #include <map>
+#include <memory>
 #include <set>
 #include <string>
 #include <vector>
@@ -51,6 +52,7 @@
 #include "file_format.hh"
 #include "logfile_fwd.hh"
 #include "safe/safe.h"
+#include "tlx/container/btree_map.hpp"
 
 struct tailer_progress {
     std::string tp_message;
@@ -156,7 +158,7 @@ struct file_collection {
 
     std::shared_ptr<safe_name_to_stubs> fc_name_to_stubs{
         std::make_shared<safe_name_to_stubs>()};
-    std::map<std::string, logfile_open_options, strnatless> fc_file_names;
+    tlx::btree_map<std::string, logfile_open_options, strnatless> fc_file_names;
     std::vector<std::shared_ptr<logfile>> fc_files;
     int fc_files_generation{0};
     std::optional<size_t> fc_files_high_mark;
diff --git a/src/files_sub_source.cc b/src/files_sub_source.cc
@@ -192,7 +192,8 @@ files_sub_source::list_input_handle_key(listview_curses& lv, const ncinput& ch)
                     auto name_iter = fc.fc_file_names.begin();
                     while (name_iter != fc.fc_file_names.end()) {
                         if (name_iter->first == es.sb_iter.first) {
-                            name_iter = fc.fc_file_names.erase(name_iter);
+                            fc.fc_file_names.erase(name_iter);
+                            name_iter = fc.fc_file_names.begin();
                             continue;
                         }
 
@@ -204,7 +205,8 @@ files_sub_source::list_input_handle_key(listview_curses& lv, const ncinput& ch)
 
                             if (fmt::to_string(rp.home()) == es.sb_iter.first) {
                                 fc.fc_other_files.erase(name_iter->first);
-                                name_iter = fc.fc_file_names.erase(name_iter);
+                                fc.fc_file_names.erase(name_iter);
+                                name_iter = fc.fc_file_names.begin();
                                 continue;
                             }
                         }
@@ -439,7 +441,7 @@ files_overlay_source::list_static_overlay(const listview_curses& lv,
         value_out.with_ansi_string(fmt::format(
             "{} Extracting " ANSI_COLOR(COLOR_CYAN) "{}" ANSI_NORM
                                                     "... {:>8}/{}",
-            PROG[spinner_index() % PROG_SIZE],
+            humanize::sparkline(prog.ep_out_size, prog.ep_total_size),
             prog.ep_path.filename().string(),
             humanize::file_size(prog.ep_out_size, humanize::alignment::none),
             humanize::file_size(prog.ep_total_size,
diff --git a/src/formats/syslog_log.json b/src/formats/syslog_log.json
@@ -74,7 +74,7 @@
         },
         "search-table": {
             "sudo_log": {
-                "pattern": "sudo: (?<login>\\S+)\\s*: (?:(?<error_msg>[^;]+);)?\\s*TTY=(?<tty>[^;]+)\\s+;\\s*PWD=(?<pwd>[^;]+)\\s+;\\s*USER=(?<user>[^;]+)\\s+;\\s*COMMAND=(?<command>.*)$"
+                "pattern": "sudo:\\s+(?<login>\\S+)\\s*: (?:(?<error_msg>[^;]+);)?(?:\\s*TTY=(?<tty>[^;]+)\\s+;)?\\s*PWD=(?<pwd>[^;]+)\\s+;\\s*USER=(?<user>[^;]+)\\s+;\\s*COMMAND=(?<command>.*)$"
             }
         },
         "sample": [
diff --git a/src/lnav.cc b/src/lnav.cc
@@ -1931,10 +1931,12 @@ VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'),
             if (exec_phase.scanning() && new_files.empty()
                 && indexing_pipers == 0)
             {
-                if (!lnav_data.ld_active_files.fc_other_files.empty()
-                    || lnav_data.ld_active_files.fc_files.size() > 1
-                    || !lnav_data.ld_active_files.fc_name_to_stubs->readAccess()
-                            ->empty())
+                if (!opened_files
+                    && (!lnav_data.ld_active_files.fc_other_files.empty()
+                        || lnav_data.ld_active_files.fc_files.size() > 1
+                        || !lnav_data.ld_active_files.fc_name_to_stubs
+                                ->readAccess()
+                                ->empty()))
                 {
                     opened_files = true;
                     set_view_mode(ln_mode_t::FILES);
@@ -1985,6 +1987,12 @@ VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'),
                 = ui_now + (std::exchange(rescan_needed, false) ? 0ms : 333ms);
         }
 
+        if (!opened_files && exec_phase.scanning()
+            && ui_now - ui_start_time >= 500ms)
+        {
+            set_view_mode(ln_mode_t::FILES);
+        }
+
         if (!rescan_future.valid()
             && (exec_phase.spinning_up()
                 || (lnav_data.ld_active_files.is_below_open_file_limit()
@@ -2109,6 +2117,9 @@ VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'),
             updated_views.emplace_back(&lnav_data.ld_user_message_view);
         }
         if (ui_now >= next_status_update_time) {
+            if (exec_phase.scanning()) {
+                lnav_data.ld_files_view.set_needs_update();
+            }
             if (lnav_data.ld_status[LNS_BOTTOM].get_needs_update()
                 || lnav_data.ld_status[LNS_FILTER].get_needs_update())
             {
@@ -3896,10 +3907,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             }
             auto loo = logfile_open_options().with_time_range(
                 lnav_data.ld_default_time_range);
-            lnav_data.ld_active_files.fc_file_names.emplace(dir_wild + "/*",
+            lnav_data.ld_active_files.fc_file_names.insert2(dir_wild + "/*",
                                                             loo);
         } else {
-            lnav_data.ld_active_files.fc_file_names.emplace(
+            lnav_data.ld_active_files.fc_file_names.insert2(
                 abspath.in(),
                 logfile_open_options()
                     .with_init_location(file_loc)

Original file line number	Diff line number	Diff line change
`@@ -216,7 +216,7 @@ copy_data(const std::string& filename,`
`216`	`216`	`entry_path.filename().string(),`
`217`	`217`	`entry_path.parent_path().string()));`
`218`	`218`	`}`
`219`		`- next_space_check += 1024 * 1024;`
	`219`	`+ next_space_check += 10 * 1024 * 1024;`
`220`	`220`	`}`
`221`	`221`
`222`	`222`	`r = archive_read_data_block(ar, &buff, &size, &offset);`
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,7 @@`
`74`	`74`	`},`
`75`	`75`	`"search-table": {`
`76`	`76`	`"sudo_log": {`
`77`		`- "pattern": "sudo: (?<login>\\S+)\\s: (?:(?<error_msg>[^;]+);)?\\sTTY=(?<tty>[^;]+)\\s+;\\sPWD=(?<pwd>[^;]+)\\s+;\\sUSER=(?<user>[^;]+)\\s+;\\sCOMMAND=(?<command>.)$"`
	`77`	`+ "pattern": "sudo:\\s+(?<login>\\S+)\\s: (?:(?<error_msg>[^;]+);)?(?:\\sTTY=(?<tty>[^;]+)\\s+;)?\\sPWD=(?<pwd>[^;]+)\\s+;\\sUSER=(?<user>[^;]+)\\s+;\\sCOMMAND=(?<command>.)$"`
`78`	`78`	`}`
`79`	`79`	`},`
`80`	`80`	`"sample": [`