Intermediate changes

commit_hash:3130610c0735089d0be0e14d07555480bbed5a19

Author: robot-piglet

yql/essentials/tools/sql2yql/sql2yql.cpp

@@ -10,6 +10,7 @@
 #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h>
+#include <yql/essentials/sql/v1/lexer/regex/lexer.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
@@ -171,6 +172,7 @@ bool TestLexers(
     lexers.Antlr4PureAnsi = NSQLTranslationV1::MakeAntlr4PureAnsiLexerFactory();
     auto lexerMain = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Default);
     auto lexerPure = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Pure);
+    auto lexerRegex = NSQLTranslationV1::MakeRegexLexerFactory(settings.AnsiLexer)->MakeLexer();
     TVector<NSQLTranslation::TParsedToken> mainTokens;
     if (!lexerMain->Tokenize(query, "", [&](auto token) { mainTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
         Cerr << issues.ToString();
@@ -183,21 +185,40 @@ bool TestLexers(
         return false;
     }
 
-    bool hasErrors = false;
-    if (mainTokens.size() != pureTokens.size()) {
-        hasErrors = true;
-        Cerr << "Mismatch token count, main: " << mainTokens.size() << ", pure: " << pureTokens.size() << "\n";
+    TVector<NSQLTranslation::TParsedToken> regexTokens;
+    if (!lexerRegex->Tokenize(query, "", [&](auto token) { regexTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+        Cerr << issues.ToString();
+        return false;
     }
 
-    for (size_t i = 0; i < Min(mainTokens.size(), pureTokens.size()); ++i) {
-        if (mainTokens[i].Name != pureTokens[i].Name || mainTokens[i].Content != pureTokens[i].Content) {
+    bool hasErrors = false;
+    auto check = [&](const char* name, const TVector<NSQLTranslation::TParsedToken>& otherTokens) {
+        if (mainTokens.size() != otherTokens.size()) {
             hasErrors = true;
-            Cerr << "Mismatch token #" << i << ", main: " << mainTokens[i].Name << ":" << mainTokens[i].Content
-                << ", pure: " << pureTokens[i].Name << ":" << pureTokens[i].Content << "\n";
-            break;
+            Cerr << "Mismatch token count, main: " << mainTokens.size() << ", " << name << ": " << otherTokens.size() << "\n";
         }
-    }
 
+        TStringBuilder textBuilder;
+
+        for (size_t i = 0; i < Min(mainTokens.size(), otherTokens.size()); ++i) {
+            if (mainTokens[i].Name != otherTokens[i].Name || mainTokens[i].Content != otherTokens[i].Content) {
+                hasErrors = true;
+                Cerr << "Mismatch token #" << i << ", main: " << mainTokens[i].Name << ":" << mainTokens[i].Content
+                    << ", " << name << ": " << otherTokens[i].Name << ":" << otherTokens[i].Content << "\n";
+                Cerr << "Text sample: [";
+                TString text = textBuilder;
+                constexpr size_t LexerContextSample = 50;
+                Cerr << text.substr(text.size() >= LexerContextSample ? text.size() - LexerContextSample : 0u, LexerContextSample);
+                Cerr << "]\n";
+                break;
+            }
+
+            textBuilder << mainTokens[i].Content;
+        }
+    };
+
+    check("pure", pureTokens);
+    check("regex", regexTokens);
     return !hasErrors;
 }
 

yql/essentials/tools/sql2yql/ya.make

@@ -17,6 +17,7 @@ PEERDIR(
     yql/essentials/sql/v1/lexer/antlr4_ansi
     yql/essentials/sql/v1/lexer/antlr4_pure
     yql/essentials/sql/v1/lexer/antlr4_pure_ansi
+    yql/essentials/sql/v1/lexer/regex
     yql/essentials/sql/v1/proto_parser/antlr4
     yql/essentials/sql/v1/proto_parser/antlr4_ansi
 )