Sync Repos: Release 170.82.0 (#154)

* Merged PR 1756293: [MSJSON] json_arrayagg support for scriptdom

## Description
This pull request introduces full support for the JSON_ARRAYAGG function in ScriptDom by extending the T-SQL grammar and code generation, along with updated test cases to validate the new functionality. Json_ArrayAgg with options NULL ON NULL/ABSENT ON NULL and RETURNING JSON are supported.


* Merged PR 1777076: Adding missing event CREATE_JSON_INDEX

# Pull Request Template for ScriptDom
## Description
Adding missing JSON index event to the event type enum.
Adding missing event CREATE_JSON_INDEX
* Merged PR 1781082: Adding regexp_like syntax
Regexp_like function works as a boolean expression similar to LIKE, MATCH starting from compat level 170.

Adding regexp_like syntax
Related work items: #4005966

* Merged PR 1762947: Nested CTE Implementation - Fabric DW

In this PR we modify the Fabric DW Grammar to support Nested Common Table Expressions with the following syntax:

```
WITH <NESTED_CTE_NAME_LEVEL1> [ (column_name , ...) ] AS
    (WITH <NESTED_CTE_NAME_LEVEL2> [ (column_name , ...) ] AS
        (
            ...
                WITH <NESTED_CTE_NAME_LEVELn-1> [ ( column_name , ...) ] AS
                (
                    WITH <NESTED_CTE_NAME_LEVELn> [ ( column_name , ...) ] AS
                    (
                        Standard_CTE_query_definition
                    )
                    <SELECT statement> -- Data source must include NESTED_CTE_NAME_LEVELn
                )
                <SELECT statement> -- Data source must include NESTED_CTE_NAME_LEVELn-1
            ...
        )
    <SELECT statement> -- Data source must include NESTED_CTE_NAME_LEVEL2
    )
```

This is derived from public [Nested CTE Documentation](https://learn.microsoft.com/en-us/sql/t-sql/queries/nested-common-table-expression?view=sql-server-ver16).

Work Item: https://dev.azure.com/powerbi/AI/_workitems/edit/1672555/
* Merged PR 1785976: [AI_GENERATE_CHUNKS] ENABLE_CHUNK_SET_ID to Integer

# Pull Request Template for ScriptDom
## Description
This PR changes the optional parameter ENABLE_CHUNK_SET_ID of AI_GENERATE_CHUNKS from expression to integer/null only.
* Merged PR 1780003: Fixed a bug when parsing json keyvalue parameter

Some of json functions accept json key value and the key value can be with or without single quote. The problem comes when the parser assumes any name and ':' should be parsed as a label. and label are not supported as function parameters. since it's not easy to differentiate between a label and json value, had to change the parser to accept label as function parameter and then parse it as column reference.
Fixed a bug when parsing json keyvalue parameter


* Merged PR 1787157: Adding release notes for 170.100
Adding release notes for 170.0.96

Author: llali

SqlScriptDom/Parser/TSql/Ast.xml

@@ -236,6 +236,7 @@
     <Member Name="ExpressionName" Type="Identifier" Summary="The expression name."/>
     <Member Name="Columns" Type="Identifier" Collection="true" Summary="The columns. Optional may have zero elements."/>
     <Member Name="QueryExpression" Type="QueryExpression" Summary="The query definition."/>
+    <Member Name="WithCtesAndXmlNamespaces" Type="WithCtesAndXmlNamespaces" Nullable="true" Summary="Nested CTE." />
   </Class>
   <Class Name="WithCtesAndXmlNamespaces" Summary="This class represents a common construct that can have common table expressions and xml namespaces in it.">
     <Member Name="XmlNamespaces" Type="XmlNamespaces" Summary="The xml namespaces. May be null."/>
@@ -349,6 +350,11 @@
     <Member Name="OdbcEscape" Type="bool" Summary="True if escape is defined using odbc delimiters."/>
     <Member Name="EscapeExpression" Type="ScalarExpression" Summary="The escape expression. Optional may be null."/>
   </Class>
+  <Class Name="RegexpLikePredicate" Base="BooleanExpression" Summary="Represents the REGEXP_LIKE boolean predicate.">
+    <Member Name="Text" Type="ScalarExpression" Summary="The text to match against the pattern."/>
+    <Member Name="Pattern" Type="ScalarExpression" Summary="The regular expression pattern."/>
+    <Member Name="Flags" Type="ScalarExpression" Optional="true" Summary="Optional flags for the regular expression match."/>
+  </Class>
   <Class Name="DistinctPredicate" Base="BooleanExpression" Summary="Represents the distinct predicate.">
     <Member Name="FirstExpression" Type="ScalarExpression" Summary="The first expression."/>
     <Member Name="SecondExpression" Type="ScalarExpression" Summary="The second expression."/>

SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs

@@ -523,6 +523,7 @@ internal static class CodeGenerationSupporter
         internal const string JsonArray = "JSON_ARRAY";
         internal const string JsonObject = "JSON_OBJECT";
         internal const string JsonObjectAgg = "JSON_OBJECTAGG";
+        internal const string JsonArrayAgg = "JSON_ARRAYAGG";
         internal const string Keep = "KEEP";
         internal const string KeepDefaults = "KEEPDEFAULTS";
         internal const string KeepFixed = "KEEPFIXED";
@@ -819,6 +820,7 @@ internal static class CodeGenerationSupporter
         internal const string RecursiveTriggers = "RECURSIVE_TRIGGERS";
         internal const string Recovery = "RECOVERY";
         internal const string Regenerate = "REGENERATE";
+        internal const string RegexpLike = "REGEXP_LIKE";
         internal const string RegexpMatches = "REGEXP_MATCHES";
         internal const string RegexpSplitToTable = "REGEXP_SPLIT_TO_TABLE";
         internal const string RejectType = "REJECT_TYPE";

SqlScriptDom/Parser/TSql/EventNotificationEventType.cs

@@ -1091,6 +1091,11 @@ public enum EventNotificationEventType
         /// </summary>
         DropExternalLanguage = 331,
 
+        /// <summary>
+        /// CREATE_JSON_INDEX
+        /// </summary>
+        CreateJsonIndex = 343,
+
         /// <summary>
         /// CREATE_VECTOR_INDEX
         /// </summary>

SqlScriptDom/Parser/TSql/TSql160.g

@@ -31846,15 +31846,31 @@ jsonKeyValueExpression returns [JsonKeyValue vResult = FragmentFactory.CreateFra
 	: 
         (
             vKey=expression
-		    {           
-		        vResult.JsonKeyName=vKey;
-		    }
+            {           
+                vResult.JsonKeyName=vKey;
+           }
             Colon vValue=expression 
-			{            
-			    vResult.JsonValue=vValue;
+            {            
+                vResult.JsonValue=vValue;
             }
-		)
-	;
+               
+        |   
+        
+            label:Label
+            {
+                var identifier = this.FragmentFactory.CreateFragment<Identifier>();
+                var multiPartIdentifier = this.FragmentFactory.CreateFragment<MultiPartIdentifier>();
+                var columnRef = this.FragmentFactory.CreateFragment<ColumnReferenceExpression>();
+                CreateIdentifierFromLabel(label, identifier, multiPartIdentifier);
+                columnRef.MultiPartIdentifier = multiPartIdentifier;
+                vResult.JsonKeyName=columnRef;
+            }
+            vValue=expression 
+            {            
+                vResult.JsonValue=vValue;
+            }
+        )
+    ;
 
 windowClause returns [WindowClause vResult = FragmentFactory.CreateFragment<WindowClause>()]
 {

SqlScriptDom/Parser/TSql/TSql170.g

@@ -19238,7 +19238,10 @@ aiGenerateFixedChunksTableReference [ScalarExpression vSource, Identifier vChunk
             Match(vEnableChunkSetIdParam, CodeGenerationSupporter.EnableChunkSetId);
         }
         EqualsSign
-        vEnableChunkSetId = expression
+        (
+            vEnableChunkSetId = integer       // constant integer
+          | vEnableChunkSetId = nullLiteral   // NULL literal
+        )
         {
             vResult.EnableChunkSetId = vEnableChunkSetId;
         }
@@ -31094,6 +31097,9 @@ booleanExpressionPrimary [ExpressionFlags expressionFlags] returns [BooleanExpre
             vResult = vMatchPredicate;
             UpdateTokenInfo(vResult,tRParen);
         }
+    |
+        {NextTokenMatches(CodeGenerationSupporter.RegexpLike)}?
+            vResult=regexpLikePredicate
     |
         vExpressionFirst=expressionWithFlags[expressionFlags]
         (
@@ -31475,6 +31481,29 @@ tsEqualCall returns [TSEqualCall vResult = this.FragmentFactory.CreateFragment<T
         }
     ;
 
+regexpLikePredicate returns [RegexpLikePredicate vResult = this.FragmentFactory.CreateFragment<RegexpLikePredicate>()]
+{
+    ScalarExpression vText;
+    ScalarExpression vPattern;
+    ScalarExpression vFlags = null;
+}
+    :   tRegexp:Identifier LeftParenthesis vText=expression Comma vPattern=expression
+        {
+            UpdateTokenInfo(vResult,tRegexp);
+            vResult.Text = vText;
+            vResult.Pattern = vPattern;
+        }
+        (Comma vFlags=expression
+        )?
+        {
+            vResult.Flags = vFlags;
+        }
+        tRParen:RightParenthesis
+        {
+            UpdateTokenInfo(vResult,tRParen);
+        }
+    ;
+
 updateCall returns [UpdateCall vResult = this.FragmentFactory.CreateFragment<UpdateCall>()]
 {
     Identifier vIdentifier;
@@ -32502,6 +32531,22 @@ jsonKeyValueExpression returns [JsonKeyValue vResult = FragmentFactory.CreateFra
             {            
                 vResult.JsonValue=vValue;
             }
+       
+        |   
+        
+            label:Label
+            {
+                var identifier = this.FragmentFactory.CreateFragment<Identifier>();
+                var multiPartIdentifier = this.FragmentFactory.CreateFragment<MultiPartIdentifier>();
+                var columnRef = this.FragmentFactory.CreateFragment<ColumnReferenceExpression>();
+                CreateIdentifierFromLabel(label, identifier, multiPartIdentifier);
+                columnRef.MultiPartIdentifier = multiPartIdentifier;
+                vResult.JsonKeyName=columnRef;
+            }
+            vValue=expression 
+            {            
+                vResult.JsonValue=vValue;
+            }
         )
     ;
 
@@ -32785,6 +32830,9 @@ builtInFunctionCall returns [FunctionCall vResult = FragmentFactory.CreateFragme
         |
          {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonObjectAgg)}?
             jsonObjectAggBuiltInFunctionCall[vResult]
+        |
+        {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArrayAgg)}?
+            jsonArrayAggBuiltInFunctionCall[vResult]
         |
          {(vResult.FunctionName != null && vResult.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.Trim) && 
           (NextTokenMatches(CodeGenerationSupporter.Leading) | NextTokenMatches(CodeGenerationSupporter.Trailing) | NextTokenMatches(CodeGenerationSupporter.Both))}?
@@ -32829,6 +32877,32 @@ jsonArrayBuiltInFunctionCall [FunctionCall vParent]
         }
     ;
 
+jsonArrayAggBuiltInFunctionCall [FunctionCall vParent]
+{
+    ScalarExpression vExpression;
+}
+    :   (
+           vExpression=expression
+           {
+               AddAndUpdateTokenInfo(vParent, vParent.Parameters, vExpression);
+           }
+        )
+        (
+           jsonNullClauseFunction[vParent]
+        |
+            /* empty */
+        )
+        (
+            jsonReturningClause[vParent]
+        |
+            /* empty */
+        )
+        tRParen:RightParenthesis
+        {
+            UpdateTokenInfo(vParent, tRParen);
+        }
+    ;
+
 jsonObjectBuiltInFunctionCall [FunctionCall vParent]
 {
 }

SqlScriptDom/Parser/TSql/TSql80ParserBaseInternal.cs

@@ -263,6 +263,25 @@ internal static void UpdateTokenInfo(TSqlFragment fragment, antlr.IToken token)
                 fragment.UpdateTokenInfo(tokenIndex, tokenIndex);
         }
 
+        /// <summary>
+        /// Creates an identifier from a label token and adds it to the multipart identifier.
+        /// </summary>
+        /// <param name="token"></param>
+        /// <param name="identifier"></param>
+        /// <param name="multiPartIdentifier"></param>
+        internal static void CreateIdentifierFromLabel(antlr.IToken token, Identifier identifier, MultiPartIdentifier multiPartIdentifier)
+        {
+            var tokenText = token?.getText();
+            if (string.IsNullOrEmpty(tokenText))
+            {
+                throw GetUnexpectedTokenErrorException(token);
+            }
+            var identifierName = tokenText?.EndsWith(":") == true ? tokenText.Substring(0, tokenText.Length - 1) : tokenText;
+            identifier.SetIdentifier(identifierName);
+            UpdateTokenInfo(identifier, token);
+            AddAndUpdateTokenInfo(multiPartIdentifier, multiPartIdentifier.Identifiers, identifier);
+        }
+
         protected static void AddAndUpdateTokenInfo<TFragmentType>(TSqlFragment node, IList<TFragmentType> collection, TFragmentType item)
             where TFragmentType : TSqlFragment
         {

SqlScriptDom/Parser/TSql/TSqlFabricDW.g

@@ -17746,15 +17746,29 @@ commonTableExpression returns [CommonTableExpression vResult = this.FragmentFact
 {
     Identifier vIdentifier;
     QueryExpression vQueryExpression;
+    WithCtesAndXmlNamespaces vWithCommonTableExpressionsAndXmlNamespaces;
 }
     :   vIdentifier=identifier
         {
             vResult.ExpressionName = vIdentifier;
         }
         (columnNameList[vResult, vResult.Columns])?
-        As tLParen:LeftParenthesis vQueryExpression=subqueryExpression[SubDmlFlags.SelectNotForInsert] tRParen:RightParenthesis
+        As tLParen:LeftParenthesis 
+        (
+            vWithCommonTableExpressionsAndXmlNamespaces=withCommonTableExpressionsAndXmlNamespaces
+            vQueryExpression=subqueryExpression[SubDmlFlags.SelectNotForInsert]
+            {
+                vResult.QueryExpression = vQueryExpression;
+                vResult.WithCtesAndXmlNamespaces = vWithCommonTableExpressionsAndXmlNamespaces;
+            }
+        |
+            vQueryExpression=subqueryExpression[SubDmlFlags.SelectNotForInsert]
+            {
+                vResult.QueryExpression = vQueryExpression;
+            }
+        )
+        tRParen:RightParenthesis
         {
-            vResult.QueryExpression = vQueryExpression;
             UpdateTokenInfo(vResult,tLParen);
             UpdateTokenInfo(vResult,tRParen);
         }

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.CommonTableExpression.cs

@@ -31,7 +31,28 @@ public override void ExplicitVisit(CommonTableExpression node)
             AlignmentPoint subquery = new AlignmentPoint();
             MarkAndPushAlignmentPoint(subquery);
 
-            GenerateQueryExpressionInParentheses(node.QueryExpression);
+            GenerateSymbol(TSqlTokenType.LeftParenthesis);
+
+            AlignmentPoint queryBody = new AlignmentPoint();
+            MarkAndPushAlignmentPoint(queryBody);
+
+            // Generate nested WITH clause if present
+            if (node.WithCtesAndXmlNamespaces != null)
+            {
+                AlignmentPoint clauseBodyNested = new AlignmentPoint(ClauseBody);
+                GenerateFragmentWithAlignmentPointIfNotNull(node.WithCtesAndXmlNamespaces, clauseBodyNested);
+                NewLine();
+            }
+
+            if (node.QueryExpression != null)
+            {
+                AlignmentPoint clauseBodyQuery = new AlignmentPoint(ClauseBody);
+                GenerateFragmentWithAlignmentPointIfNotNull(node.QueryExpression, clauseBodyQuery);
+            }
+
+            PopAlignmentPoint();
+
+            GenerateSymbol(TSqlTokenType.RightParenthesis);
 
             PopAlignmentPoint();
         }

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.FunctionCall.cs

@@ -72,6 +72,17 @@ public override void ExplicitVisit(FunctionCall node)
                 GenerateSymbol(TSqlTokenType.RightParenthesis);
             }
             else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArray)
+            {
+                GenerateCommaSeparatedList(node.Parameters);
+                if (node.Parameters?.Count > 0 && node?.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them
+                    GenerateSpace();
+                GenerateNullOnNullOrAbsentOnNull(node?.AbsentOrNullOnNull);
+				if (node.ReturnType?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them
+                    GenerateSpace();
+                GenerateReturnType(node?.ReturnType);
+                GenerateSymbol(TSqlTokenType.RightParenthesis);
+            }
+			else if (node.FunctionName.Value.ToUpper(CultureInfo.InvariantCulture) == CodeGenerationSupporter.JsonArrayAgg)
             {
                 GenerateCommaSeparatedList(node.Parameters);
                 if (node.Parameters?.Count > 0 && node?.AbsentOrNullOnNull?.Count > 0) //If there are values and null on null or absent on null present then generate space in between them

SqlScriptDom/ScriptDom/SqlServer/ScriptGenerator/SqlScriptGeneratorVisitor.RegexpLikePredicate.cs

@@ -0,0 +1,35 @@
+//------------------------------------------------------------------------------
+// <copyright file="SqlScriptGeneratorVisitor.RegexpLikePredicate.cs" company="Microsoft">
+//         Copyright (c) Microsoft Corporation.  All rights reserved.
+// </copyright>
+//------------------------------------------------------------------------------
+using System;
+using Microsoft.SqlServer.TransactSql.ScriptDom;
+
+namespace Microsoft.SqlServer.TransactSql.ScriptDom.ScriptGenerator
+{
+    internal partial class SqlScriptGeneratorVisitor
+    {
+        public override void ExplicitVisit(RegexpLikePredicate node)
+        {
+            GenerateIdentifier(CodeGenerationSupporter.RegexpLike);
+
+            GenerateSpace();
+            GenerateSymbol(TSqlTokenType.LeftParenthesis);
+
+            GenerateFragmentIfNotNull(node.Text);
+            GenerateSymbol(TSqlTokenType.Comma);
+            GenerateSpace();
+            GenerateFragmentIfNotNull(node.Pattern);
+
+            if (node.Flags != null)
+            {
+                GenerateSymbol(TSqlTokenType.Comma);
+                GenerateSpace();
+                GenerateFragmentIfNotNull(node.Flags);
+            }
+
+            GenerateSymbol(TSqlTokenType.RightParenthesis);
+        }
+    }
+}