Skip to content

Commit f785e4d

Browse files
tstacktstack
committed
[fields] allow log_time to be hidden
1 parent 793c612 commit f785e4d

16 files changed

+207
-133
lines changed

‎NEWS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33
Features:
44
* Log message timestamps are now represented with microsecond
55
precision internally instead of just millisecond.
6+
* The `log_time` and `log_level` fields can now be hidden.
67

78
Bug Fixes:
89
* Improved startup time.
910
* Reduced memory footprint.
1011
* Improved search performance.
12+
* Reduce time to open help text.
1113

1214
Maintenance:
1315
* Replaced ncurses with notcurses.

‎src/base/attr_line.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ attr_line_t::apply_hide()
482482
auto& sa = this->al_attrs;
483483

484484
for (auto& sattr : sa) {
485-
if (sattr.sa_type == &SA_HIDDEN && sattr.sa_range.length() > 3) {
485+
if (sattr.sa_type == &SA_HIDDEN && sattr.sa_range.length() > 1) {
486486
auto& lr = sattr.sa_range;
487487

488488
std::for_each(sa.begin(), sa.end(), [&](string_attr& attr) {

‎src/column_namer.cc‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,16 @@
3838
#include "config.h"
3939
#include "sql_util.hh"
4040

41-
const char column_namer::BUILTIN_COL[] = "col";
41+
const auto column_namer::BUILTIN_COL = string_fragment::from_const("col");
4242

43-
column_namer::column_namer(language lang) : cn_language(lang) {}
43+
column_namer::column_namer(language lang) : cn_language(lang)
44+
{
45+
this->cn_builtin_names.emplace_back(BUILTIN_COL);
46+
this->cn_builtin_names.emplace_back(
47+
string_fragment::from_const("log_time"));
48+
this->cn_builtin_names.emplace_back(
49+
string_fragment::from_const("log_level"));
50+
}
4451

4552
bool
4653
column_namer::existing_name(const string_fragment& in_name) const
@@ -80,7 +87,7 @@ column_namer::add_column(const string_fragment& in_name)
8087
int num = 0;
8188

8289
if (in_name.empty()) {
83-
base_name = string_fragment::from_const(BUILTIN_COL);
90+
base_name = BUILTIN_COL;
8491
} else {
8592
base_name = in_name;
8693
}

‎src/column_namer.hh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,17 @@ public:
4545
JSON,
4646
};
4747

48-
column_namer(language lang);
48+
static const string_fragment BUILTIN_COL;
49+
50+
explicit column_namer(language lang);
4951

5052
bool existing_name(const string_fragment& in_name) const;
5153

5254
string_fragment add_column(const string_fragment& in_name);
5355

54-
static const char BUILTIN_COL[];
55-
5656
ArenaAlloc::Alloc<char> cn_alloc;
5757
language cn_language;
58-
std::vector<string_fragment> cn_builtin_names{string_fragment(BUILTIN_COL)};
58+
std::vector<string_fragment> cn_builtin_names;
5959
std::vector<string_fragment> cn_names;
6060
std::unordered_map<
6161
string_fragment,

‎src/formats/syslog_log.json‎

Lines changed: 99 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,102 @@
11
{
2-
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
3-
"syslog_log": {
4-
"title": "Syslog",
5-
"description": "The system logger format found on most posix systems.",
6-
"url": "http://en.wikipedia.org/wiki/Syslog",
7-
"regex": {
8-
"std": {
9-
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\](?: \\([^\\)]+\\))?)?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
10-
},
11-
"rfc5424": {
12-
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
13-
}
14-
},
15-
"level-field": "body",
16-
"level": {
17-
"error": "(?:(?:(?<![a-zA-Z]))(?:(?i)error(?:s)?)(?:(?![a-zA-Z]))|failed|failure)",
18-
"warning": "(?:(?:(?i)warn)|not responding|init: cannot execute)"
19-
},
20-
"opid-field": "log_syslog_tag",
21-
"multiline": true,
22-
"module-field": "log_procname",
23-
"value": {
24-
"log_pri": {
25-
"kind": "integer",
26-
"foreign-key": true,
27-
"description": "The priority level of the message"
28-
},
29-
"syslog_version": {
30-
"kind": "integer",
31-
"foreign-key": true,
32-
"description": "The version of the syslog format used for this message"
33-
},
34-
"log_hostname": {
35-
"kind": "string",
36-
"collate": "ipaddress",
37-
"identifier": true,
38-
"description": "The name of the host that generated the message"
39-
},
40-
"log_procname": {
41-
"kind": "string",
42-
"identifier": true,
43-
"description": "The name of the process that generated the message"
44-
},
45-
"log_pid": {
46-
"kind": "string",
47-
"identifier": true,
48-
"action-list": [
49-
"dump_pid"
50-
],
51-
"description": "The ID of the process that generated the message"
52-
},
53-
"log_syslog_tag": {
54-
"kind": "string",
55-
"identifier": true,
56-
"description": "The combination of the procname and pid"
57-
},
58-
"log_msgid": {
59-
"kind": "string",
60-
"identifier": true
61-
},
62-
"log_struct": {
63-
"kind": "struct"
64-
}
65-
},
66-
"action": {
67-
"dump_pid": {
68-
"label": "Show Process Info",
69-
"capture-output": true,
70-
"cmd": [
71-
"dump-pid.sh"
2+
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
3+
"syslog_log": {
4+
"title": "Syslog",
5+
"description": "The system logger format found on most posix systems.",
6+
"url": "http://en.wikipedia.org/wiki/Syslog",
7+
"regex": {
8+
"std": {
9+
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[:]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\](?: \\([^\\)]+\\))?)?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
10+
},
11+
"rfc5424": {
12+
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
13+
}
14+
},
15+
"level-field": "body",
16+
"level": {
17+
"error": "(?:(?:(?<![a-zA-Z]))(?:(?i)error(?:s)?)(?:(?![a-zA-Z]))|failed|failure)",
18+
"warning": "(?:(?:(?i)warn)|not responding|init: cannot execute)"
19+
},
20+
"opid-field": "log_syslog_tag",
21+
"multiline": true,
22+
"module-field": "log_procname",
23+
"value": {
24+
"log_pri": {
25+
"kind": "integer",
26+
"foreign-key": true,
27+
"description": "The priority level of the message"
28+
},
29+
"syslog_version": {
30+
"kind": "integer",
31+
"foreign-key": true,
32+
"description": "The version of the syslog format used for this message"
33+
},
34+
"log_hostname": {
35+
"kind": "string",
36+
"collate": "ipaddress",
37+
"identifier": true,
38+
"description": "The name of the host that generated the message"
39+
},
40+
"log_procname": {
41+
"kind": "string",
42+
"identifier": true,
43+
"description": "The name of the process that generated the message"
44+
},
45+
"log_pid": {
46+
"kind": "string",
47+
"identifier": true,
48+
"action-list": [
49+
"dump_pid"
50+
],
51+
"description": "The ID of the process that generated the message"
52+
},
53+
"log_syslog_tag": {
54+
"kind": "string",
55+
"identifier": true,
56+
"description": "The combination of the procname and pid"
57+
},
58+
"log_msgid": {
59+
"kind": "string",
60+
"identifier": true
61+
},
62+
"log_struct": {
63+
"kind": "struct"
64+
}
65+
},
66+
"action": {
67+
"dump_pid": {
68+
"label": "Show Process Info",
69+
"capture-output": true,
70+
"cmd": [
71+
"dump-pid.sh"
72+
]
73+
}
74+
},
75+
"sample": [
76+
{
77+
"line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart."
78+
},
79+
{
80+
"line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB"
81+
},
82+
{
83+
"line": "Jun 20 17:26:13 ip-10-188-149-5 [CLOUDINIT] util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud (recursive=False)"
84+
},
85+
{
86+
"line": "<46>1 2017-04-27T07:50:47.381967+02:00 logserver rsyslogd - - [origin software=\"rsyslogd\" swVersion=\"8.4.2\" x-pid=\"900\" x-info=\"http://www.rsyslog.com\"] start"
87+
},
88+
{
89+
"line": "<30>1 2017-04-27T07:59:12+02:00 nextcloud dhclient - - - DHCPREQUEST on eth0 to 192.168.1.1 port 67"
90+
},
91+
{
92+
"line": "<78>1 2017-04-27T08:09:01+02:00 nextcloud CRON 1472 - - (root) CMD ( [ -x /usr/lib/php5/sessionclean ] && /usr/lib/php5/sessionclean)"
93+
},
94+
{
95+
"line": "Aug 1 00:00:03 Tim-Stacks-iMac com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0C000000-0700-0000-0000-000000000000[50989]): Service exited due to SIGKILL | sent by mds[198]"
96+
},
97+
{
98+
"line": "Jan 4 10:23:26 Tims-MacBook-Air Setup Assistant[1173]: Creating connection"
99+
}
72100
]
73-
}
74-
},
75-
"sample": [
76-
{
77-
"line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart."
78-
},
79-
{
80-
"line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB"
81-
},
82-
{
83-
"line": "Jun 20 17:26:13 ip-10-188-149-5 [CLOUDINIT] util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud (recursive=False)"
84-
},
85-
{
86-
"line": "<46>1 2017-04-27T07:50:47.381967+02:00 logserver rsyslogd - - [origin software=\"rsyslogd\" swVersion=\"8.4.2\" x-pid=\"900\" x-info=\"http://www.rsyslog.com\"] start"
87-
},
88-
{
89-
"line": "<30>1 2017-04-27T07:59:12+02:00 nextcloud dhclient - - - DHCPREQUEST on eth0 to 192.168.1.1 port 67"
90-
},
91-
{
92-
"line": "<78>1 2017-04-27T08:09:01+02:00 nextcloud CRON 1472 - - (root) CMD ( [ -x /usr/lib/php5/sessionclean ] && /usr/lib/php5/sessionclean)"
93-
},
94-
{
95-
"line": "Aug 1 00:00:03 Tim-Stacks-iMac com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0C000000-0700-0000-0000-000000000000[50989]): Service exited due to SIGKILL | sent by mds[198]"
96-
}
97-
]
98-
}
101+
}
99102
}

‎src/lnav.indexing.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ class loading_observer : public logfile_observer {
5656
return indexing_result::CONTINUE;
5757
}
5858

59+
if (lnav_data.ld_sigint_count.load() > 0) {
60+
return indexing_result::BREAK;
61+
}
62+
5963
/* XXX require(off <= total); */
6064
if (off > (off_t) total) {
6165
off = total;

‎src/log_format.cc‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ external_log_format::mod_map_t external_log_format::MODULE_FORMATS;
7272
std::vector<std::shared_ptr<external_log_format>>
7373
external_log_format::GRAPH_ORDERED_FORMATS;
7474

75+
const intern_string_t log_format::LOG_TIME_STR
76+
= intern_string::lookup("log_time");
77+
const intern_string_t log_format::LOG_LEVEL_STR
78+
= intern_string::lookup("log_level");
79+
7580
static const uint32_t DATE_TIME_SET_FLAGS = ETF_YEAR_SET | ETF_MONTH_SET
7681
| ETF_DAY_SET | ETF_HOUR_SET | ETF_MINUTE_SET | ETF_SECOND_SET;
7782

@@ -2793,27 +2798,35 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
27932798
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
27942799
vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
27952800
vd->vd_internal = true;
2801+
2802+
this->elf_value_defs[LOG_TIME_STR] = vd;
27962803
}
27972804
if (startswith(this->elf_level_field.get(), "/")) {
27982805
this->elf_level_field
27992806
= intern_string::lookup(this->elf_level_field.get() + 1);
28002807
}
2801-
if (!this->elf_level_field.empty()
2802-
&& this->elf_value_defs.find(this->elf_level_field)
2803-
== this->elf_value_defs.end())
2804-
{
2805-
auto& vd = this->elf_value_defs[this->elf_level_field];
2806-
if (vd.get() == nullptr) {
2807-
vd = std::make_shared<value_def>(
2808-
this->elf_level_field,
2809-
value_kind_t::VALUE_TEXT,
2810-
logline_value_meta::internal_column{},
2811-
this);
2808+
if (!this->elf_level_field.empty()) {
2809+
auto level_iter = this->elf_value_defs.find(this->elf_level_field);
2810+
if (level_iter == this->elf_value_defs.end()) {
2811+
auto& vd = this->elf_value_defs[this->elf_level_field];
2812+
if (vd.get() == nullptr) {
2813+
vd = std::make_shared<value_def>(
2814+
this->elf_level_field,
2815+
value_kind_t::VALUE_TEXT,
2816+
logline_value_meta::internal_column{},
2817+
this);
2818+
}
2819+
vd->vd_meta.lvm_name = this->elf_level_field;
2820+
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
2821+
vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
2822+
vd->vd_internal = true;
2823+
2824+
if (this->elf_level_field != this->elf_body_field) {
2825+
this->elf_value_defs[LOG_LEVEL_STR] = vd;
2826+
}
2827+
} else {
2828+
this->elf_value_defs[LOG_LEVEL_STR] = level_iter->second;
28122829
}
2813-
vd->vd_meta.lvm_name = this->elf_level_field;
2814-
vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
2815-
vd->vd_meta.lvm_column = logline_value_meta::internal_column{};
2816-
vd->vd_internal = true;
28172830
}
28182831
if (!this->elf_body_field.empty()) {
28192832
auto& vd = this->elf_value_defs[this->elf_body_field];

‎src/log_format.hh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,10 @@ public:
655655
std::equal_to<intern_string_t>>;
656656
desc_cap_map lf_desc_captures;
657657

658+
659+
static const intern_string_t LOG_TIME_STR;
660+
static const intern_string_t LOG_LEVEL_STR;
661+
658662
protected:
659663
static std::vector<std::shared_ptr<log_format>> lf_root_formats;
660664

0 commit comments

Comments
 (0)