build: Update to the latest version of log-surgeon.

components/core/config/schemas.txt

@@ -1,19 +1,102 @@
-// Delimiters
-delimiters: \t\r\n!"#$%&'\(\)\*,:;<>?@\[\]\^_`\{\|\}~
+// ===
+// Timestamps
+// ===
 
-// Timestamps (using the `timestamp` keyword)
-// E.g. 2015-01-31 15:50:45,392
+// E.g. 2015-01-31T15:50:45.392
+// E.g. 2015-01-31T15:50:45,392
 // E.g. 2015-01-31 15:50:45.392
+// E.g. 2015-01-31 15:50:45,392
+// E.g. 2015/01/31T15:50:45.123
+// E.g. 2015/01/31T15:50:45,123
+// E.g. 2015/01/31 15:50:45.123
+// E.g. 2015/01/31 15:50:45,123
+timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[,\.]\d{3}
+
+// E.g. 2015-01-31T15:50:45
+// E.g. 2015-01-31T15:50:45
 // E.g. 2015-01-31 15:50:45
-timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}
-// E.g. [20150131-15:50:45]
-timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\]
+// E.g. 2015-01-31 15:50:45
+// E.g. 2015/01/31T15:50:45
+// E.g. 2015/01/31T15:50:45
+// E.g. 2015/01/31 15:50:45
+// E.g. 2015/01/31 15:50:45
+timestamp:\d{4}[\-/]\d{2}[\-/]\d{2}[T ]\d{2}:\d{2}:\d{2}
+
+// E.g. [2015-01-31T15:50:45
+timestamp:\[\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}
+
+// E.g. [20170106-16:56:41]
+timestamp:\[\d{4}\d{2}\d{2}\-\d{2}:\d{2}:\d{2}\]
+
+// E.g. [2015-01-31 15:50:45,085]
+timestamp:\[\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2},\d{3}\]
+
+// E.g. 2015-01-31 15:50:45
+// E.g. Started POST /api/v3/internal/allowed for 127.0.0.1 at 2017-06-18 00:20:44
+// E.g. update-alternatives 2015-01-31 15:50:45
+timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}
+
+// E.g. Start-Date: 2015-01-31  15:50:45
+timestamp:\d{4}\-\d{2}\-\d{2}  \d{2}:\d{2}:\d{2}
+
+// E.g. 15/01/31 15:50:45
+timestamp:\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}
+
+// E.g. 150131  9:50:45
+timestamp:\d{2}\d{2}\d{2} [ \d]\d:\d{2}:\d{2}
+
+// E.g. 01 Jan 2016 15:50:17,085
+timestamp:[ \d]\d [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2},\d{3}
+
+// E.g. Jan 01, 2016 3:50:17 PM
+timestamp:[A-Z][a-z]{2} [ \d]\d, \d{4} [ \d]\d:\d{2}:\d{2} [AP]M
 
+// E.g. January 31, 2015 15:50
+timestamp:[A-Z][a-z]+ \d{2}, \d{4} \d{2}:\d{2}
+
+// E.g. E [31/Jan/2015:15:50:45
+// E.g. localhost - - [01/Jan/2016:15:50:17
+// E.g. 192.168.4.5 - - [01/Jan/2016:15:50:17
+timestamp:\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}
+
+// E.g. 192.168.4.5 - - [01/01/2016:15:50:17
+timestamp:\[\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}
+
+// E.g. ERROR: apport (pid 4557) Sun Jan  1 15:50:45 2015
+timestamp:[A-Z][a-z]{2} [A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2} \d{4}
+
+// E.g. <<<2016-11-10 03:02:29:936
+timestamp:\<\<\<\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}:\d{3}
+
+// E.g. Jan 21 11:56:42
+timestamp:[A-Z][a-z]{2} [ \d]\d \d{2}:\d{2}:\d{2}
+
+// E.g. 01-21 11:56:42.392
+timestamp:\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}
+
+// E.g. 2016-05-08 11:34:04.083464
+timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}
+
+// ===
+// Delimiters
+// ===
+
+delimiters: \t\r\n!"#$%&'\(\)\*,:;\<=\>\?@\[\]\^_`\{\|\}~
+
+// ===
 // Specially-encoded variables (using the `int` and `float` keywords)
+// ===
+
 int:\-{0,1}[0-9]+
 float:\-{0,1}[0-9]+\.[0-9]+
 
+// ===
 // Dictionary variables
+// ===
+
 hex:[a-fA-F]+
+
+// NOTE: `equals` and `hasNumber` only work if `=` is a delimiter, due to
+// https://github.com/y-scope/log-surgeon/issues/131
+equals:.*=(?<val>.*[a-zA-Z0-9].*)
 hasNumber:.*\d.*
-equals:.*=.*[a-zA-Z0-9].*

components/core/src/clp/Grep.cpp

@@ -502,8 +502,7 @@ std::optional<Query> Grep::process_raw_query(
         epochtime_t search_begin_ts,
         epochtime_t search_end_ts,
         bool ignore_case,
-        log_surgeon::lexers::ByteLexer& forward_lexer,
-        log_surgeon::lexers::ByteLexer& reverse_lexer,
+        log_surgeon::lexers::ByteLexer& lexer,
         bool use_heuristic
 ) {
     // Add prefix and suffix '*' to make the search a sub-string match
@@ -546,8 +545,7 @@ std::optional<Query> Grep::process_raw_query(
                 begin_pos,
                 end_pos,
                 is_var,
-                forward_lexer,
-                reverse_lexer
+                lexer
         ))
         {
             query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var);
@@ -752,8 +750,7 @@ bool Grep::get_bounds_of_next_potential_var(
         size_t& begin_pos,
         size_t& end_pos,
         bool& is_var,
-        log_surgeon::lexers::ByteLexer& forward_lexer,
-        log_surgeon::lexers::ByteLexer& reverse_lexer
+        log_surgeon::lexers::ByteLexer& lexer
 ) {
     size_t const value_length = value.length();
     if (end_pos >= value_length) {
@@ -774,7 +771,7 @@ bool Grep::get_bounds_of_next_potential_var(
             if (is_escaped) {
                 is_escaped = false;
 
-                if (false == forward_lexer.is_delimiter(c)) {
+                if (false == lexer.is_delimiter(c)) {
                     // Found escaped non-delimiter, so reverse the index to retain the escape
                     // character
                     --begin_pos;
@@ -788,7 +785,7 @@ bool Grep::get_bounds_of_next_potential_var(
                     contains_wildcard = true;
                     break;
                 }
-                if (false == forward_lexer.is_delimiter(c)) {
+                if (false == lexer.is_delimiter(c)) {
                     break;
                 }
             }
@@ -803,7 +800,7 @@ bool Grep::get_bounds_of_next_potential_var(
             if (is_escaped) {
                 is_escaped = false;
 
-                if (forward_lexer.is_delimiter(c)) {
+                if (lexer.is_delimiter(c)) {
                     // Found escaped delimiter, so reverse the index to retain the escape character
                     --end_pos;
                     break;
@@ -814,7 +811,7 @@ bool Grep::get_bounds_of_next_potential_var(
             } else {
                 if (is_wildcard(c)) {
                     contains_wildcard = true;
-                } else if (forward_lexer.is_delimiter(c)) {
+                } else if (lexer.is_delimiter(c)) {
                     // Found delimiter that's not also a wildcard
                     break;
                 }
@@ -832,7 +829,7 @@ bool Grep::get_bounds_of_next_potential_var(
                 }
             }
             SearchToken search_token;
-            if (has_wildcard_in_middle || (has_prefix_wildcard && has_suffix_wildcard)) {
+            if (has_wildcard_in_middle || has_prefix_wildcard) {
                 // DO NOTHING
             } else {
                 StringReader string_reader;
@@ -844,43 +841,22 @@ bool Grep::get_bounds_of_next_potential_var(
                     // string, should be improved when adding a SearchParser to log_surgeon
                     string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1));
                     parser_input_buffer.read_if_safe(reader_wrapper);
-                    forward_lexer.reset();
-                    forward_lexer.scan_with_wildcard(
-                            parser_input_buffer,
-                            value[end_pos - 1],
-                            search_token
-                    );
-                } else if (has_prefix_wildcard) {  // *text
-                    std::string value_reverse
-                            = value.substr(begin_pos + 1, end_pos - begin_pos - 1);
-                    std::reverse(value_reverse.begin(), value_reverse.end());
-                    string_reader.open(value_reverse);
-                    parser_input_buffer.read_if_safe(reader_wrapper);
-                    reverse_lexer.reset();
-                    reverse_lexer.scan_with_wildcard(
-                            parser_input_buffer,
-                            value[begin_pos],
-                            search_token
-                    );
+                    lexer.reset();
+                    lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token);
                 } else {  // no wildcards
                     string_reader.open(value.substr(begin_pos, end_pos - begin_pos));
                     parser_input_buffer.read_if_safe(reader_wrapper);
-                    forward_lexer.reset();
-                    forward_lexer.scan(parser_input_buffer, search_token);
+                    lexer.reset();
+                    auto [err, token] = lexer.scan(parser_input_buffer);
+                    if (log_surgeon::ErrorCode::Success != err) {
+                        return false;
+                    }
+                    search_token = SearchToken{token.value()};
                     search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0));
                 }
-                // TODO: use a set so its faster
-                // auto const& set = search_token.m_type_ids_set;
-                // if (set.find(static_cast<int>(log_surgeon::SymbolID::TokenUncaughtStringID))
-                //            == set.end()
-                //     && set.find(static_cast<int>(log_surgeon::SymbolID::TokenEndID))
-                //            == set.end())
-                // {
-                //     is_var = true;
-                // }
                 auto const& type = search_token.m_type_ids_ptr->at(0);
-                if (type != static_cast<int>(log_surgeon::SymbolID::TokenUncaughtStringID)
-                    && type != static_cast<int>(log_surgeon::SymbolID::TokenEndID))
+                if (type != static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString)
+                    && type != static_cast<int>(log_surgeon::SymbolId::TokenEnd))
                 {
                     is_var = true;
                 }

components/core/src/clp/Grep.hpp

@@ -37,8 +37,7 @@ class Grep {
      * @param search_begin_ts
      * @param search_end_ts
      * @param ignore_case
-     * @param forward_lexer DFA for determining if input is in the schema
-     * @param reverse_lexer DFA for determining if reverse of input is in the schema
+     * @param lexer DFA for determining if input is in the schema
      * @param use_heuristic
      * @return Query if it may match a message, std::nullopt otherwise
      */
@@ -48,8 +47,7 @@ class Grep {
             epochtime_t search_begin_ts,
             epochtime_t search_end_ts,
             bool ignore_case,
-            log_surgeon::lexers::ByteLexer& forward_lexer,
-            log_surgeon::lexers::ByteLexer& reverse_lexer,
+            log_surgeon::lexers::ByteLexer& lexer,
             bool use_heuristic
     );
 
@@ -76,17 +74,15 @@ class Grep {
      * @param begin_pos Begin position of last token, changes to begin position of next token
      * @param end_pos End position of last token, changes to end position of next token
      * @param is_var Whether the token is definitely a variable
-     * @param forward_lexer DFA for determining if input is in the schema
-     * @param reverse_lexer DFA for determining if reverse of input is in the schema
+     * @param lexer DFA for determining if input is in the schema
      * @return true if another potential variable was found, false otherwise
      */
     static bool get_bounds_of_next_potential_var(
             std::string const& value,
             size_t& begin_pos,
             size_t& end_pos,
             bool& is_var,
-            log_surgeon::lexers::ByteLexer& forward_lexer,
-            log_surgeon::lexers::ByteLexer& reverse_lexer
+            log_surgeon::lexers::ByteLexer& lexer
     );
     /**
      * Marks which sub-queries in each query are relevant to the given file

components/core/src/clp/Utils.cpp

@@ -11,6 +11,7 @@
 
 #include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
+#include <log_surgeon/Constants.hpp>
 #include <log_surgeon/SchemaParser.hpp>
 #include <spdlog/spdlog.h>
 #include <string_utils/string_utils.hpp>
@@ -120,12 +121,8 @@ ErrorCode read_list_of_paths(string const& list_path, vector<string>& paths) {
 // TODO: duplicates code in log_surgeon/parser.tpp, should implement a
 // SearchParser in log_surgeon instead and use it here. Specifically, initialization of
 // lexer.m_symbol_id, contains_delimiter error, and add_rule logic.
-void load_lexer_from_file(
-        std::string const& schema_file_path,
-        bool reverse,
-        log_surgeon::lexers::ByteLexer& lexer
-) {
-    log_surgeon::SchemaParser sp;
+void
+load_lexer_from_file(std::string const& schema_file_path, log_surgeon::lexers::ByteLexer& lexer) {
     std::unique_ptr<log_surgeon::SchemaAST> schema_ast
             = log_surgeon::SchemaParser::try_schema_file(schema_file_path);
     if (!lexer.m_symbol_id.empty()) {
@@ -134,52 +131,52 @@ void load_lexer_from_file(
 
     // cTokenEnd and cTokenUncaughtString never need to be added as a rule to the lexer as they are
     // not parsed
-    lexer.m_symbol_id[log_surgeon::cTokenEnd] = static_cast<int>(log_surgeon::SymbolID::TokenEndID);
+    lexer.m_symbol_id[log_surgeon::cTokenEnd] = static_cast<int>(log_surgeon::SymbolId::TokenEnd);
     lexer.m_symbol_id[log_surgeon::cTokenUncaughtString]
-            = static_cast<int>(log_surgeon::SymbolID::TokenUncaughtStringID);
+            = static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString);
     // cTokenInt, cTokenFloat, cTokenFirstTimestamp, and cTokenNewlineTimestamp each have unknown
     // rule(s) until specified by the user so can't be explicitly added and are done by looping over
     // schema_vars (user schema)
-    lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast<int>(log_surgeon::SymbolID::TokenIntId);
+    lexer.m_symbol_id[log_surgeon::cTokenInt] = static_cast<int>(log_surgeon::SymbolId::TokenInt);
     lexer.m_symbol_id[log_surgeon::cTokenFloat]
-            = static_cast<int>(log_surgeon::SymbolID::TokenFloatId);
+            = static_cast<int>(log_surgeon::SymbolId::TokenFloat);
     lexer.m_symbol_id[log_surgeon::cTokenFirstTimestamp]
-            = static_cast<int>(log_surgeon::SymbolID::TokenFirstTimestampId);
+            = static_cast<int>(log_surgeon::SymbolId::TokenFirstTimestamp);
     lexer.m_symbol_id[log_surgeon::cTokenNewlineTimestamp]
-            = static_cast<int>(log_surgeon::SymbolID::TokenNewlineTimestampId);
+            = static_cast<int>(log_surgeon::SymbolId::TokenNewlineTimestamp);
     // cTokenNewline is not added in schema_vars and can be explicitly added as '\n' to catch the
     // end of non-timestamped log messages
     lexer.m_symbol_id[log_surgeon::cTokenNewline]
-            = static_cast<int>(log_surgeon::SymbolID::TokenNewlineId);
+            = static_cast<int>(log_surgeon::SymbolId::TokenNewline);
 
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenEndID)] = log_surgeon::cTokenEnd;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenUncaughtStringID)]
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenEnd)] = log_surgeon::cTokenEnd;
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString)]
             = log_surgeon::cTokenUncaughtString;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenIntId)] = log_surgeon::cTokenInt;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenFloatId)]
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenInt)] = log_surgeon::cTokenInt;
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenFloat)]
             = log_surgeon::cTokenFloat;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenFirstTimestampId)]
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenFirstTimestamp)]
             = log_surgeon::cTokenFirstTimestamp;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenNewlineTimestampId)]
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenNewlineTimestamp)]
             = log_surgeon::cTokenNewlineTimestamp;
-    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolID::TokenNewlineId)]
+    lexer.m_id_symbol[static_cast<int>(log_surgeon::SymbolId::TokenNewline)]
             = log_surgeon::cTokenNewline;
 
     lexer.add_rule(
             lexer.m_symbol_id["newLine"],
             std::move(
                     std::make_unique<log_surgeon::finite_automata::RegexASTLiteral<
-                            log_surgeon::finite_automata::RegexNFAByteState>>(
+                            log_surgeon::finite_automata::ByteNfaState>>(
                             log_surgeon::finite_automata::RegexASTLiteral<
-                                    log_surgeon::finite_automata::RegexNFAByteState>('\n')
+                                    log_surgeon::finite_automata::ByteNfaState>('\n')
                     )
             )
     );
 
     for (auto const& delimiters_ast : schema_ast->m_delimiters) {
         auto* delimiters_ptr = dynamic_cast<log_surgeon::DelimiterStringAST*>(delimiters_ast.get());
         if (delimiters_ptr != nullptr) {
-            lexer.add_delimiters(delimiters_ptr->m_delimiters);
+            lexer.set_delimiters(delimiters_ptr->m_delimiters);
         }
     }
     vector<uint32_t> delimiters;
@@ -203,7 +200,7 @@ void load_lexer_from_file(
         // transform '.' from any-character into any non-delimiter character
         rule->m_regex_ptr->remove_delimiters_from_wildcard(delimiters);
 
-        bool is_possible_input[log_surgeon::cUnicodeMax] = {false};
+        std::array<bool, log_surgeon::cSizeOfUnicode> is_possible_input{};
         rule->m_regex_ptr->set_possible_inputs_to_true(is_possible_input);
         bool contains_delimiter = false;
         uint32_t delimiter_name;
@@ -242,10 +239,6 @@ void load_lexer_from_file(
         }
         lexer.add_rule(lexer.m_symbol_id[rule->m_name], std::move(rule->m_regex_ptr));
     }
-    if (reverse) {
-        lexer.generate_reverse();
-    } else {
-        lexer.generate();
-    }
+    lexer.generate();
 }
 }  // namespace clp