[reports] add report-access-log.lnav

Author: tstack

NEWS.md

@@ -4,6 +4,22 @@ Features:
 * Log message timestamps are now represented with microsecond
   precision internally instead of just millisecond.
 * The `log_time` and `log_level` fields can now be hidden.
+* Added a `report-access-log` script that generates a report that
+  is similar to the output of the [goaccess](https://goaccess.io)
+  utility.
+
+Interface changes:
+* DB query results that start with a number are right justified
+  instead of only full numbers.
+
+Breaking changes:
+* The `parse_url()` SQL function no longer raises an error for an
+  invalid URL.
+  Instead, it will return a JSON object with an object with the
+  following properties:
+  - `error` - An identifier for the error.
+  - `url` - The invalid URL itself.
+  - `reason` - A description of the error.
 
 Bug Fixes:
 * Improved startup time.

src/CMakeLists.txt

@@ -270,6 +270,7 @@ set(BUILTIN_LNAV_SCRIPTS
         scripts/partition-by-boot.lnav
         scripts/piper-url-handler.lnav
         scripts/rename-stdin.lnav
+        scripts/report-access-log.lnav
         scripts/search-for.lnav
 )
 

src/db_sub_source.cc

@@ -68,13 +68,14 @@ db_label_source::text_value_for_line(textview_curses& tc,
         auto cell_length
             = utf8_string_length(cell_str).unwrapOr(actual_col_size);
         auto padding = actual_col_size - cell_length;
+        auto rjust = cell_length > 0 && isdigit(cell_str[0]);
         this->dls_cell_width[lpc] = cell_str.length() + padding;
-        if (this->dls_headers[lpc].hm_column_type != SQLITE3_TEXT) {
+        if (rjust) {
             label_out.append(padding, ' ');
         }
         shift_string_attrs(cell_attrs, 0, label_out.size());
         label_out.append(cell_str);
-        if (this->dls_headers[lpc].hm_column_type == SQLITE3_TEXT) {
+        if (!rjust) {
             label_out.append(padding, ' ');
         }
         label_out.append(1, ' ');

src/internals/sql-ref.rst

@@ -334,10 +334,10 @@ SELECT *result-column* FROM *table* WHERE *\[cond\]* GROUP BY *grouping-expr* OR
 
       ;SELECT * FROM lnav_example_log
        log_line  log_part    log_time     log_actual_time log_idle_msecs log_level  log_mark  log_comment log_tags log_filters ex_procname ex_duration log_time_msecs log_path    log_text        log_body     
-               0 <NULL>   2017-02⋯:06.100 2017-02⋯:06.100              0 info               0      <NULL>   <NULL>      <NULL> hw                    2  1486094706000 /tmp/log 2017-02⋯ World! Hello, World!   
-               1 <NULL>   2017-02⋯:06.200 2017-02⋯:06.200            100 error              0      <NULL>   <NULL>      <NULL> gw                    4  1486094706000 /tmp/log 2017-02⋯ World! Goodbye, World! 
-               2 new      2017-02⋯:06.200 2017-02⋯:06.200        1200000 warn               0      <NULL>   <NULL>      <NULL> gw                    1  1486095906000 /tmp/log 2017-02⋯ World! Goodbye, World! 
-               3 new      2017-02⋯:06.200 2017-02⋯:06.200        1800000 debug              0      <NULL>   <NULL>      <NULL> gw                   10  1486097706000 /tmp/log 2017-02⋯ World! Goodbye, World! 
+               0 <NULL>   2017-02⋯:06.100 2017-02⋯:06.100              0 info               0 <NULL>      <NULL>   <NULL>      hw                    2  1486094706000 /tmp/log 2017-02⋯ World! Hello, World!   
+               1 <NULL>   2017-02⋯:06.200 2017-02⋯:06.200            100 error              0 <NULL>      <NULL>   <NULL>      gw                    4  1486094706000 /tmp/log 2017-02⋯ World! Goodbye, World! 
+               2 new      2017-02⋯:06.200 2017-02⋯:06.200        1200000 warn               0 <NULL>      <NULL>   <NULL>      gw                    1  1486095906000 /tmp/log 2017-02⋯ World! Goodbye, World! 
+               3 new      2017-02⋯:06.200 2017-02⋯:06.200        1800000 debug              0 <NULL>      <NULL>   <NULL>      gw                   10  1486097706000 /tmp/log 2017-02⋯ World! Goodbye, World! 
 
 
 ----
@@ -1960,7 +1960,7 @@ json_each(*X*, *\[P\]*)
       ;SELECT * FROM json_each('[null,1,"two",{"three":4.5}]')
       key     value      type    atom  id parent fullkey path 
         0 <NULL>        null    <NULL>  2 <NULL> $[0]    $    
-        1 1             integer 1       3 <NULL> $[1]    $    
+        1             1 integer      1  3 <NULL> $[1]    $    
         2 two           text    two     5 <NULL> $[2]    $    
         3 {"three":4.5} object  <NULL>  9 <NULL> $[3]    $    
 
@@ -2303,11 +2303,11 @@ json_tree(*X*, *\[P\]*)
       ;SELECT key,value,type,atom,fullkey,path FROM json_tree('[null,1,"two",{"three":4.5}]')
        key        value       type    atom   fullkey   path 
       <NULL> [null,1⋯":4.5}] array   <NULL> $          $    
-      0      <NULL>          null    <NULL> $[0]       $    
-      1      1               integer 1      $[1]       $    
-      2      two             text    two    $[2]       $    
-      3      {"three":4.5}   object  <NULL> $[3]       $    
-      three  4.5             real    4.5    $[3].three $[3] 
+           0 <NULL>          null    <NULL> $[0]       $    
+           1               1 integer      1 $[1]       $    
+           2 two             text    two    $[2]       $    
+           3 {"three":4.5}   object  <NULL> $[3]       $    
+      three              4.5 real       4.5 $[3].three $[3] 
 
   **See Also**
     :ref:`jget`, :ref:`json_array_length`, :ref:`json_array`, :ref:`json_concat`, :ref:`json_contains`, :ref:`json_each`, :ref:`json_extract`, :ref:`json_group_array`, :ref:`json_group_object`, :ref:`json_insert`, :ref:`json_object`, :ref:`json_quote`, :ref:`json_remove`, :ref:`json_replace`, :ref:`json_set`, :ref:`json_type`, :ref:`json_valid`, :ref:`json`, :ref:`yaml_to_json`
@@ -3401,12 +3401,12 @@ regexp_capture(*string*, *pattern*)
 
       ;SELECT * FROM regexp_capture('a=1; b=2', '(\w+)=(\d+)')
       match_index capture_index capture_name capture_count range_start range_stop content 
-                0             0       <NULL>             3           1          4 a=1     
-                0             1       <NULL>             3           1          2 a       
-                0             2       <NULL>             3           3          4 1       
-                1             0       <NULL>             3           6          9 b=2     
-                1             1       <NULL>             3           6          7 b       
-                1             2       <NULL>             3           8          9 2       
+                0             0 <NULL>                   3           1          4 a=1     
+                0             1 <NULL>                   3           1          2 a       
+                0             2 <NULL>                   3           3          4       1 
+                1             0 <NULL>                   3           6          9 b=2     
+                1             1 <NULL>                   3           6          7 b       
+                1             2 <NULL>                   3           8          9       2 
 
   **See Also**
     :ref:`anonymize`, :ref:`char`, :ref:`charindex`, :ref:`decode`, :ref:`encode`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_duration`, :ref:`humanize_file_size`, :ref:`humanize_id`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`parse_url`, :ref:`pretty_print`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`timezone`, :ref:`trim`, :ref:`unicode`, :ref:`unparse_url`, :ref:`upper`, :ref:`xpath`
@@ -3840,28 +3840,28 @@ spooky_hash(*str*)
     .. code-block::  custsqlite
 
       ;SELECT spooky_hash('Hello, World!')
-      0b1d52cc5427db4c6a9eed9d3e5700f4
+      0b1d52cc-5427-db4c-6a9e-ed9d3e5700f4
 
     To produce a hash for the parameters where one is NULL:
 
     .. code-block::  custsqlite
 
       ;SELECT spooky_hash('Hello, World!', NULL)
-      c96ee75d48e6ea444fee8af948f6da25
+      c96ee75d-48e6-ea44-4fee-8af948f6da25
 
     To produce a hash for the parameters where one is an empty string:
 
     .. code-block::  custsqlite
 
       ;SELECT spooky_hash('Hello, World!', '')
-      c96ee75d48e6ea444fee8af948f6da25
+      c96ee75d-48e6-ea44-4fee-8af948f6da25
 
     To produce a hash for the parameters where one is a number:
 
     .. code-block::  custsqlite
 
       ;SELECT spooky_hash('Hello, World!', 123)
-      f96b3d9c1a19f4394c97a1b79b1880df
+      f96b3d9c-1a19-f439-4c97-a1b79b1880df
 
   **See Also**
     :ref:`anonymize`, :ref:`char`, :ref:`charindex`, :ref:`decode`, :ref:`encode`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_duration`, :ref:`humanize_file_size`, :ref:`humanize_id`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`parse_url`, :ref:`pretty_print`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`timezone`, :ref:`trim`, :ref:`unicode`, :ref:`unparse_url`, :ref:`upper`, :ref:`xpath`

src/lnav_commands.cc

@@ -60,6 +60,7 @@
 #include "db_sub_source.hh"
 #include "external_opener.hh"
 #include "field_overlay_source.hh"
+#include "fmt/color.h"
 #include "fmt/printf.h"
 #include "hasher.hh"
 #include "itertools.similar.hh"
@@ -1667,9 +1668,11 @@ com_save_to(exec_context& ec,
 
         for (const auto& hdr : dls.dls_headers) {
             auto centered_hdr = center_str(hdr.hm_name, hdr.hm_column_size);
+            auto style = fmt::text_style{};
 
             fprintf(outfile, "\u2503");
-            fprintf(outfile, "%s", centered_hdr.c_str());
+            style |= fmt::emphasis::bold;
+            fmt::print(outfile, style, FMT_STRING("{}"), centered_hdr);
         }
         fprintf(outfile, "\u2503\n");
 
@@ -1703,12 +1706,13 @@ com_save_to(exec_context& ec,
                 auto cell_length
                     = utf8_string_length(cell).unwrapOr(cell.size());
                 auto padding = anonymize ? 1 : hdr.hm_column_size - cell_length;
+                auto rjust = cell_length > 0 && isdigit(cell[0]);
 
-                if (hdr.hm_column_type != SQLITE3_TEXT) {
+                if (rjust) {
                     fprintf(outfile, "%s", std::string(padding, ' ').c_str());
                 }
                 fprintf(outfile, "%s", cell.c_str());
-                if (hdr.hm_column_type == SQLITE3_TEXT) {
+                if (!rjust) {
                     fprintf(outfile, "%s", std::string(padding, ' ').c_str());
                 }
             }

src/scripts/report-access-log.lnav

@@ -0,0 +1,115 @@
+#
+# @synopsis: report-access-log
+# @description: Generate a report of the access_log table
+#
+
+;DROP TABLE IF EXISTS report_access_log
+;CREATE TABLE report_access_log AS
+    SELECT
+        log_time_msecs,
+        c_ip,
+        cs_method,
+        cs_uri_stem,
+        cs_version,
+        sc_status,
+        sc_bytes,
+        CASE cs_referer
+          WHEN '-' THEN NULL
+          ELSE jget(parse_url(cs_referer), '/host')
+        END AS cs_referer,
+        cs_user_agent
+      FROM access_log
+
+;SELECT count(*) AS total_requests,
+        count(DISTINCT spooky_hash(c_ip, cs_user_agent)) AS unique_visitors,
+        count(DISTINCT cs_uri_stem) AS requested_files,
+        humanize_file_size(sum(sc_bytes)) AS tx_amount
+   FROM report_access_log
+
+;SELECT count(*) AS not_found
+   FROM (SELECT 1 FROM report_access_log
+           WHERE sc_status = 404
+           GROUP BY cs_uri_stem)
+
+:echo Total Requests  ${total_requests}  Unique Visitors ${unique_visitors}  Requested Files ${requested_files}
+:echo Not Found ${not_found} Tx. Amount ${tx_amount}
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     strftime('%F', min(log_time_msecs) / 1000, 'unixepoch') AS Date
+   FROM report_access_log
+   GROUP BY timeslice(log_time_msecs, '1d')
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     cs_method AS Mtd,
+     cs_version AS Proto,
+     cs_uri_stem AS URIs
+   FROM report_access_log
+   GROUP BY cs_uri_stem, cs_method
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     cs_method AS Mtd,
+     cs_version AS Proto,
+     cs_uri_stem AS URIs
+   FROM report_access_log
+   WHERE sc_status = 404
+   GROUP BY cs_uri_stem, cs_method
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     c_ip AS Host
+   FROM report_access_log
+   GROUP BY c_ip
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     cs_referer AS Referer
+   FROM report_access_log
+   WHERE cs_referer IS NOT NULL
+   GROUP BY cs_referer
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;SELECT
+     count(*) AS Hits,
+     printf('%.2f%%', 100.0 * count(*) / $total_requests) AS "h%",
+     count(DISTINCT c_ip) AS Vis,
+     humanize_file_size(sum(sc_bytes)) AS "Tx. Amount",
+     printf('%sXX', sc_status / 100) AS Status
+   FROM report_access_log
+   GROUP BY sc_status / 100
+   ORDER BY Hits DESC
+   LIMIT 10
+:write-cols-to -
+
+;DROP TABLE IF EXISTS report_access_log

src/scripts/scripts.am

@@ -8,6 +8,7 @@ BUILTIN_LNAVSCRIPTS = \
     $(srcdir)/scripts/partition-by-boot.lnav \
     $(srcdir)/scripts/piper-url-handler.lnav \
     $(srcdir)/scripts/rename-stdin.lnav \
+    $(srcdir)/scripts/report-access-log.lnav \
     $(srcdir)/scripts/search-for.lnav \
     $(srcdir)/scripts/zk-set-ops.lnav \
     $()

src/string-extension-functions.cc

@@ -18,6 +18,7 @@
 #include <string.h>
 
 #include "base/humanize.hh"
+#include "base/is_utf8.hh"
 #include "base/lnav.gzip.hh"
 #include "base/string_util.hh"
 #include "column_namer.hh"
@@ -631,9 +632,21 @@ sql_parse_url(std::string url)
     auto rc = curl_url_set(
         cu, CURLUPART_URL, url.c_str(), CURLU_NON_SUPPORT_SCHEME);
     if (rc != CURLUE_OK) {
-        throw lnav::console::user_message::error(
-            attr_line_t("invalid URL: ").append(lnav::roles::file(url)))
-            .with_reason(curl_url_strerror(rc));
+        auto_mem<char> url_part(curl_free);
+        yajlpp_gen gen;
+        yajl_gen_config(gen, yajl_gen_beautify, false);
+
+        {
+            yajlpp_map root(gen);
+            root.gen("error");
+            root.gen("invalid-url");
+            root.gen("url");
+            root.gen(url);
+            root.gen("reason");
+            root.gen(curl_url_strerror(rc));
+        }
+
+        return json_string(gen);
     }
 
     auto_mem<char> url_part(curl_free);
@@ -682,7 +695,18 @@ sql_parse_url(std::string url)
         root.gen("path");
         rc = curl_url_get(cu, CURLUPART_PATH, url_part.out(), CURLU_URLDECODE);
         if (rc == CURLUE_OK) {
-            root.gen(string_fragment::from_c_str(url_part.in()));
+            auto path_frag = string_fragment::from_c_str(url_part.in());
+            auto path_utf_res = is_utf8(path_frag);
+            if (path_utf_res.is_valid()) {
+                root.gen(path_frag);
+            } else {
+                rc = curl_url_get(cu, CURLUPART_PATH, url_part.out(), 0);
+                if (rc == CURLUE_OK) {
+                    root.gen(string_fragment::from_c_str(url_part.in()));
+                } else {
+                    root.gen();
+                }
+            }
         } else {
             root.gen();
         }
@@ -721,12 +745,24 @@ sql_parse_url(std::string url)
                 if (eq_index_opt) {
                     auto key = kv_pair_frag.sub_range(0, eq_index_opt.value());
                     auto val = kv_pair_frag.substr(eq_index_opt.value() + 1);
-                    auto key_str = key.to_string();
 
-                    if (seen_keys.count(key_str) == 0) {
-                        seen_keys.emplace(key_str);
-                        query_map.gen(key);
-                        query_map.gen(val);
+                    auto key_utf_res = is_utf8(key);
+                    auto val_utf_res = is_utf8(val);
+                    if (key_utf_res.is_valid()) {
+                        auto key_str = key.to_string();
+
+                        if (seen_keys.count(key_str) == 0) {
+                            seen_keys.emplace(key_str);
+                            query_map.gen(key);
+                            if (val_utf_res.is_valid()) {
+                                query_map.gen(val);
+                            } else {
+                                auto eq = strchr(kv_pair_encoded.data(), '=');
+                                query_map.gen(eq + 1);
+                            }
+                        }
+                    } else {
+
                     }
                 } else {
                     auto val_str = split_res.first.to_string();

src/vtab_module.cc

@@ -65,6 +65,7 @@ sqlite3_error_to_user_message(sqlite3* db)
             return from_res.unwrap();
         }
 
+        log_error("unable to parse error message: %s", errmsg);
         return lnav::console::user_message::error("internal error")
             .with_reason(from_res.unwrapErr()[0].um_message.get_string());
     }