YQL-20496: Fix highlighting again

Fixed [issues](https://nda.ya.ru/t/e98IQkef7Mhm96):
1. Types had higher precedence than UDF and so `Yson::Parse`
   was badly highlighted. So function unit was split to let
   types to highlight `Decimal(...)` correctly, as well as
   UDFs.
2. There was a hack in a Monaco generator (probably I forgot
   to fix it) with hardcoded rules for embedded language, so
   JSON rule was forgotted to be deleted.
commit_hash:b24527891525dad65bfbafbc209fd3cb7f087f32

Author: vityaman

yql/essentials/sql/v1/highlight/sql_highlight.cpp

@@ -140,12 +140,19 @@ TUnit MakeUnit<EUnitKind::TypeIdentifier>(Syntax& s) {
     };
 }
 
-template <>
-TUnit MakeUnit<EUnitKind::FunctionIdentifier>(Syntax& s) {
+TUnit MakeUnitUDF(Syntax& s) {
     return {
         .Kind = EUnitKind::FunctionIdentifier,
         .Patterns = {
             {s.Concat({"ID_PLAIN", "NAMESPACE", "ID_PLAIN"})},
+        },
+    };
+}
+
+TUnit MakeUnitBuiltin(Syntax& s) {
+    return {
+        .Kind = EUnitKind::FunctionIdentifier,
+        .Patterns = {
             {s.Get("ID_PLAIN"), s.Get("LPAREN")},
         },
     };
@@ -242,8 +249,9 @@ THighlighting MakeHighlighting(const NSQLReflect::TLexerGrammar& grammar) {
     h.Units.emplace_back(MakeUnit<EUnitKind::Comment>(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::Punctuation>(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::OptionIdentifier>(s));
+    h.Units.emplace_back(MakeUnitUDF(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::TypeIdentifier>(s));
-    h.Units.emplace_back(MakeUnit<EUnitKind::FunctionIdentifier>(s));
+    h.Units.emplace_back(MakeUnitBuiltin(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::Literal>(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::Keyword>(s));
     h.Units.emplace_back(MakeUnit<EUnitKind::QuotedIdentifier>(s));

yql/essentials/sql/v1/highlight/ut/suite.json

@@ -53,7 +53,8 @@
     ["Math::Sin", "FFFFFFFFF"],
     ["Math", "IIII"],
     ["Math::", "IIII#PP"],
-    ["::Sin", "PP#III"]
+    ["::Sin", "PP#III"],
+    ["Yson::Parse", "FFFFFFFFFFF"]
   ],
   "SQL": [
     ["SELECT id, alias from users", "KKKKKK#_#II#P#_#IIIII#_#KKKK#_#IIIII"],

yql/essentials/tools/yql_highlight/generator_monarch.cpp

@@ -2,6 +2,8 @@
 
 #include "highlighting.h"
 
+#include <yql/essentials/utils/yql_panic.h>
+
 #include <contrib/libs/re2/re2/re2.h>
 
 #include <library/cpp/json/json_writer.h>
@@ -139,37 +141,12 @@ NJson::TJsonValue ToMonarchMultiLineState(const TUnit& unit, const TRangePattern
 
     NJson::TJsonValue json;
     if (unit.Kind == EUnitKind::StringLiteral) {
-        json.AppendValue(NJson::TJsonArray{
-            "#py",
-            NJson::TJsonMap{
-                {"token", "string.python"},
-                {"nextEmbedded", "python"},
-                {"next", "@embedded"},
-                {"goBack", 3},
-            },
-        });
-        json.AppendValue(NJson::TJsonArray{
-            "\\/\\/js",
-            NJson::TJsonMap{
-                {"token", "string.js"},
-                {"nextEmbedded", "javascript"},
-                {"next", "@embedded"},
-                {"goBack", 4},
-            },
-        });
-        json.AppendValue(NJson::TJsonArray{
-            "{",
-            NJson::TJsonMap{
-                {"token", "string.js"},
-                {"nextEmbedded", "javascript"},
-                {"next", "@embedded"},
-                {"goBack", 1},
-            },
-        });
-        if (auto embedded = EmbeddedLanguage(pattern)) {
-            json.AppendValue(NJson::TJsonArray{
-                "[^" + begin + "]",
-                *embedded});
+        for (const auto& range : unit.RangePatterns) {
+            if (auto embedded = EmbeddedLanguage(range)) {
+                YQL_ENSURE(range.BeginPlain.StartsWith("@@"));
+                TString tag = RE2::QuoteMeta(range.BeginPlain.substr(2));
+                json.AppendValue(NJson::TJsonArray{std::move(tag), *embedded});
+            }
         }
         if (escape) {
             json.AppendValue(NJson::TJsonArray{*escape, group + ".escape"});

yql/essentials/tools/yql_highlight/generator_textmate.cpp

@@ -4,6 +4,7 @@
 
 #include <contrib/libs/re2/re2/re2.h>
 
+#include <library/cpp/iterator/enumerate.h>
 #include <library/cpp/json/json_value.h>
 #include <library/cpp/json/json_writer.h>
 #include <library/cpp/on_disk/tar_archive/archive_writer.h>
@@ -100,13 +101,13 @@ TString ToTextMateGroup(EUnitKind kind) {
     }
 }
 
-TString ToTextMateName(EUnitKind kind) {
-    return ToString(kind);
+TString ToTextMateName(EUnitKind kind, size_t index) {
+    return ToString(kind) + ToString(index);
 }
 
-NTextMate::TMatcher TextMateMultilinePattern(const TUnit& unit, const TRangePattern& range) {
+NTextMate::TMatcher TextMateMultilinePattern(const TUnit& unit, size_t index, const TRangePattern& range) {
     return NTextMate::TMatcher{
-        .Name = ToTextMateName(unit.Kind),
+        .Name = ToTextMateName(unit.Kind, index),
         .Group = ToTextMateGroup(unit.Kind),
         .Pattern = NTextMate::TRange{
             .Begin = RE2::QuoteMeta(range.BeginPlain),
@@ -116,9 +117,9 @@ NTextMate::TMatcher TextMateMultilinePattern(const TUnit& unit, const TRangePatt
     };
 }
 
-NTextMate::TMatcher ToTextMatePattern(const TUnit& unit, const NSQLTranslationV1::TRegexPattern& pattern) {
+NTextMate::TMatcher ToTextMatePattern(const TUnit& unit, size_t index, const NSQLTranslationV1::TRegexPattern& pattern) {
     return NTextMate::TMatcher{
-        .Name = ToTextMateName(unit.Kind),
+        .Name = ToTextMateName(unit.Kind, index),
         .Group = ToTextMateGroup(unit.Kind),
         .Pattern = ToTextMateRegex(unit, pattern),
     };
@@ -133,16 +134,16 @@ NTextMate::TLanguage ToTextMateLanguage(const THighlighting& highlighting) {
         .FileType = highlighting.Extension,
     };
 
-    for (const TUnit& unit : highlighting.Units) {
+    for (const auto& [index, unit] : Enumerate(highlighting.Units)) {
         if (unit.IsCodeGenExcluded) {
             continue;
         }
 
         for (const NSQLTranslationV1::TRegexPattern& pattern : unit.Patterns) {
-            language.Matchers.emplace_back(ToTextMatePattern(unit, pattern));
+            language.Matchers.emplace_back(ToTextMatePattern(unit, index, pattern));
         }
         for (const TRangePattern& range : unit.RangePatterns) {
-            language.Matchers.emplace_back(TextMateMultilinePattern(unit, range));
+            language.Matchers.emplace_back(TextMateMultilinePattern(unit, index, range));
         }
     }
 

yql/essentials/tools/yql_highlight/ut/query.yql

@@ -87,6 +87,8 @@ FROM my_table AS t;
 SELECT ListCreate(OptionalType(DataType("String")));
 $match = Re2::Match("[ax]+\\d");
 
+SELECT Yson::Parse("");
+
 $py = Python3::foo(@@#py
 from yql.typing import *
 
@@ -102,3 +104,18 @@ def foo(
 @@);
 
 SELECT 1;
+
+$r = (
+    PROCESS $r USING Streaming::ProcessInline(
+        TableRows(),
+        @@#!/usr/bin/env bash
+        # Feel free to use any other common interpreter in she-bang or bring
+        # your own in attached file
+
+        awk $1 '{ printf "%05d %s\n", NF-1, $0; }' | sed -e 's/n/N/g'
+        # Prepends zero-padded count of "n" letters to each row
+        # then replaces all "n" with "N"
+        @@,
+        ["-Fn"]
+    )
+);

yql/essentials/tools/yql_highlight/ut/query.yqls

@@ -28,5 +28,6 @@
         return x + y.a + z
 @@))
     (let func (Udf 'SimpleUdf.Repeat))
+    (let func (Udf 'Yson.Parse))
     (return world)
 )