Skip to content

Commit 4476447

Browse files
Copilotllalizijchen
authored
Add support for Vector index syntax in T-SQL parser with separate index option classes (#150)
* Initial plan * Implement vector index support - add grammar, AST classes, and script generators Co-authored-by: llali <[email protected]> * Consolidate vector index options and remove duplicate Vector constant Co-authored-by: llali <[email protected]> * Revert to original design with separate VectorMetricIndexOption and VectorTypeIndexOption classes Co-authored-by: llali <[email protected]> * fixing the parser * Update vector index option values to uppercase in baseline test file Co-authored-by: llali <[email protected]> * Add comprehensive vector index error test cases to ParserErrorsTests Co-authored-by: zijchen <[email protected]> * Remove invalid negative test case for vector index - CREATE INDEX syntax is valid for regular indexes Co-authored-by: zijchen <[email protected]> * fixing the test --------- Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: llali <[email protected]> Co-authored-by: Leila Lali <[email protected]> Co-authored-by: zijchen <[email protected]>
1 parent 0a26b6d commit 4476447

17 files changed

+446
-1
lines changed

‎SqlScriptDom/Parser/TSql/Ast.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4607,6 +4607,13 @@
46074607
<Member Name="ForJsonPaths" Type="StringLiteral" Collection="true" Summary="The JSON paths specified in the FOR clause. Optional may have zero elements."/>
46084608
<InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
46094609
</Class>
4610+
<Class Name="CreateVectorIndexStatement" Base="IndexStatement" Summary="Represents the create vector index statement.">
4611+
<InheritedMember Name="Name" ContainerClass="IndexStatement" />
4612+
<InheritedMember Name="OnName" ContainerClass="IndexStatement" />
4613+
<Member Name="VectorColumn" Type="Identifier" Summary="The vector column for the index."/>
4614+
<InheritedMember Name="IndexOptions" ContainerClass="IndexStatement" />
4615+
<Member Name="OnFileGroupOrPartitionScheme" Type="FileGroupOrPartitionScheme" Summary="The filegroup or partition scheme. Might be null."/>
4616+
</Class>
46104617
<Class Name="WindowFrameClause" Summary="Represents the specification of window bounds for windowing aggregates.">
46114618
<Member Name="Top" Type="WindowDelimiter" Summary="Top boundary of the window."/>
46124619
<Member Name="Bottom" Type="WindowDelimiter" Summary="Bottom boundary of the window. Optional may be null."/>
@@ -4639,6 +4646,14 @@
46394646
<Member Name="Expression" Type="ScalarExpression" Summary="Can be an integer in the range 0 - 10080."/>
46404647
<Member Name="TimeUnit" Type="CompressionDelayTimeUnit" GenerateUpdatePositionInfoCall="false" Summary="The optional unit of measure for the compression delay (it can be MINUTE or MINUTES)."/>
46414648
</Class>
4649+
<Class Name="VectorMetricIndexOption" Base="IndexOption" Summary="Represents the vector metric index option.">
4650+
<InheritedClass Name="IndexOption" />
4651+
<Member Name="MetricType" Type="VectorMetricType" GenerateUpdatePositionInfoCall="false" Summary="The vector metric type (cosine, dot, euclidean)."/>
4652+
</Class>
4653+
<Class Name="VectorTypeIndexOption" Base="IndexOption" Summary="Represents the vector type index option.">
4654+
<InheritedClass Name="IndexOption" />
4655+
<Member Name="VectorType" Type="VectorIndexType" GenerateUpdatePositionInfoCall="false" Summary="The vector index type (DiskANN)."/>
4656+
</Class>
46424657
<Class Name="ExternalLibraryStatement" Abstract="true" Base="TSqlStatement" Summary="Represents common parts in create and alter external library statements.">
46434658
<Member Name="Name" Type="Identifier" Summary="The name of the external library."/>
46444659
<Member Name="Language" Type="StringLiteral" Summary="The language of this external library."/>

‎SqlScriptDom/Parser/TSql/CodeGenerationSupporter.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ internal static class CodeGenerationSupporter
225225
internal const string CompressionDelay = "COMPRESSION_DELAY";
226226
internal const string CompressAllRowGroups = "COMPRESS_ALL_ROW_GROUPS";
227227
internal const string Concat = "CONCAT";
228+
internal const string Cosine = "COSINE";
228229
internal const string Configuration = "CONFIGURATION";
229230
internal const string ConnectionOptions = "CONNECTION_OPTIONS";
230231
internal const string Contained = "CONTAINED";
@@ -299,6 +300,8 @@ internal static class CodeGenerationSupporter
299300
internal const string Dependents = "DEPENDENTS";
300301
internal const string Description = "DESCRIPTION";
301302
internal const string DesiredState = "DESIRED_STATE";
303+
internal const string DiskANN = "DISKANN";
304+
internal const string Dot = "DOT";
302305
internal const string Delay = "DELAY";
303306
internal const string DelayedDurability = "DELAYED_DURABILITY";
304307
internal const string DelimitedText = "DELIMITEDTEXT";
@@ -347,6 +350,7 @@ internal static class CodeGenerationSupporter
347350
internal const string EnvironmentVariables = "ENVIRONMENT_VARIABLES";
348351
internal const string Equal = "=";
349352
internal const string Error = "ERROR";
353+
internal const string Euclidean = "EUCLIDEAN";
350354
internal const string ErrorBrokerConversations = "ERROR_BROKER_CONVERSATIONS";
351355
internal const string ErrorDataSource = "ERRORFILE_DATA_SOURCE";
352356
internal const string ErrorFile = "ERRORFILE";
@@ -612,6 +616,7 @@ internal static class CodeGenerationSupporter
612616
internal const string MessageForwarding = "MESSAGE_FORWARDING";
613617
internal const string MessageForwardSize = "MESSAGE_FORWARD_SIZE";
614618
internal const string MigrationState = "MIGRATION_STATE";
619+
internal const string Metric = "METRIC";
615620
internal const string Min = "MIN";
616621
internal const string MinGrantPercent = "MIN_GRANT_PERCENT";
617622
internal const string MinCpuPercent = "MIN_CPU_PERCENT";

‎SqlScriptDom/Parser/TSql/IndexOptionHelper.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ private IndexOptionHelper()
4343

4444
AddOptionMapping(IndexOptionKind.XmlCompression, CodeGenerationSupporter.XmlCompression, SqlVersionFlags.TSql160AndAbove);
4545

46+
AddOptionMapping(IndexOptionKind.VectorMetric, CodeGenerationSupporter.Metric, SqlVersionFlags.TSql170AndAbove);
47+
AddOptionMapping(IndexOptionKind.VectorType, CodeGenerationSupporter.Type, SqlVersionFlags.TSql170AndAbove);
48+
4649
}
4750

4851
internal static readonly IndexOptionHelper Instance = new IndexOptionHelper();

‎SqlScriptDom/Parser/TSql/IndexOptionKind.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public enum IndexOptionKind
3838
WaitAtLowPriority = 21,
3939
OptimizeForSequentialKey = 22,
4040
XmlCompression = 23,
41+
VectorMetric = 24,
42+
VectorType = 25,
4143
}
4244

4345
#pragma warning restore 1591

‎SqlScriptDom/Parser/TSql/OptionsHelper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ internal SqlVersionFlags MapSqlVersionToSqlVersionFlags(SqlVersion sqlVersion)
135135
}
136136
}
137137

138-
internal OptionType ParseOption(antlr.IToken token, SqlVersionFlags version)
138+
internal virtual OptionType ParseOption(antlr.IToken token, SqlVersionFlags version)
139139
{
140140
OptionInfo optionInfo;
141141
if (_stringToOptionInfo.TryGetValue(token.getText(), out optionInfo) &&

‎SqlScriptDom/Parser/TSql/TSql170.g‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -886,6 +886,9 @@ create2005Statements returns [TSqlStatement vResult = null]
886886
|
887887
{NextTokenMatches(CodeGenerationSupporter.Json)}?
888888
vResult=createJsonIndexStatement[null, null]
889+
|
890+
{NextTokenMatches(CodeGenerationSupporter.Vector)}?
891+
vResult=createVectorIndexStatement[null, null]
889892
|
890893
{NextTokenMatches(CodeGenerationSupporter.Contract)}?
891894
vResult=createContractStatement
@@ -16848,6 +16851,7 @@ createIndexStatement returns [TSqlStatement vResult = null]
1684816851
vResult=createRelationalIndexStatement[tUnique, isClustered]
1684916852
| vResult=createColumnStoreIndexStatement[tUnique, isClustered]
1685016853
| vResult=createJsonIndexStatement[tUnique, isClustered]
16854+
| vResult=createVectorIndexStatement[tUnique, isClustered]
1685116855
)
1685216856
)
1685316857
|
@@ -17036,6 +17040,50 @@ createJsonIndexStatement [IToken tUnique, bool? isClustered] returns [CreateJson
1703617040
)?
1703717041
;
1703817042

17043+
createVectorIndexStatement [IToken tUnique, bool? isClustered] returns [CreateVectorIndexStatement vResult = FragmentFactory.CreateFragment<CreateVectorIndexStatement>()]
17044+
{
17045+
Identifier vIdentifier;
17046+
SchemaObjectName vSchemaObjectName;
17047+
Identifier vVectorColumn;
17048+
FileGroupOrPartitionScheme vFileGroupOrPartitionScheme;
17049+
17050+
if (tUnique != null)
17051+
{
17052+
ThrowIncorrectSyntaxErrorException(tUnique);
17053+
}
17054+
if (isClustered.HasValue)
17055+
{
17056+
ThrowIncorrectSyntaxErrorException(LT(1));
17057+
}
17058+
}
17059+
: tVector:Identifier tIndex:Index vIdentifier=identifier
17060+
{
17061+
Match(tVector, CodeGenerationSupporter.Vector);
17062+
vResult.Name = vIdentifier;
17063+
}
17064+
tOn:On vSchemaObjectName=schemaObjectThreePartName
17065+
{
17066+
vResult.OnName = vSchemaObjectName;
17067+
}
17068+
LeftParenthesis vVectorColumn=identifier tRParen:RightParenthesis
17069+
{
17070+
vResult.VectorColumn = vVectorColumn;
17071+
UpdateTokenInfo(vResult, tRParen);
17072+
}
17073+
(
17074+
// Greedy due to conflict with withCommonTableExpressionsAndXmlNamespaces
17075+
options {greedy = true; } :
17076+
With
17077+
indexOptionList[IndexAffectingStatement.CreateIndex, vResult.IndexOptions, vResult]
17078+
)?
17079+
(
17080+
On vFileGroupOrPartitionScheme=filegroupOrPartitionScheme
17081+
{
17082+
vResult.OnFileGroupOrPartitionScheme = vFileGroupOrPartitionScheme;
17083+
}
17084+
)?
17085+
;
17086+
1703917087
indexKeyColumnList[CreateIndexStatement vParent]
1704017088
{
1704117089
ColumnWithSortOrder vColumnWithSortOrder;
@@ -17680,6 +17728,12 @@ indexOption returns [IndexOption vResult = null]
1768017728
|
1768117729
{NextTokenMatches(CodeGenerationSupporter.WaitAtLowPriority)}?
1768217730
vResult=waitAtLowPriorityOption
17731+
|
17732+
{NextTokenMatches(CodeGenerationSupporter.Metric)}?
17733+
vResult=vectorMetricOption
17734+
|
17735+
{NextTokenMatches(CodeGenerationSupporter.Type)}?
17736+
vResult=vectorTypeOption
1768317737
|
1768417738
vResult=indexStateOption
1768517739
;
@@ -27669,6 +27723,27 @@ xmlCompressionOption returns [XmlCompressionOption vResult = FragmentFactory.Cre
2766927723
)?
2767027724
;
2767127725

27726+
vectorMetricOption returns [VectorMetricIndexOption vResult = FragmentFactory.CreateFragment<VectorMetricIndexOption>()]
27727+
: tMetric:Identifier EqualsSign tMetricValue:AsciiStringLiteral
27728+
{
27729+
Match(tMetric, CodeGenerationSupporter.Metric);
27730+
vResult.OptionKind = IndexOptionKind.VectorMetric;
27731+
vResult.MetricType = VectorMetricTypeHelper.Instance.ParseOption(tMetricValue);
27732+
27733+
UpdateTokenInfo(vResult, tMetric);
27734+
}
27735+
;
27736+
27737+
vectorTypeOption returns [VectorTypeIndexOption vResult = FragmentFactory.CreateFragment<VectorTypeIndexOption>()]
27738+
: tType:Identifier EqualsSign tTypeValue:AsciiStringLiteral
27739+
{
27740+
Match(tType, CodeGenerationSupporter.Type);
27741+
vResult.OptionKind = IndexOptionKind.VectorType;
27742+
vResult.VectorType = VectorIndexTypeHelper.Instance.ParseOption(tTypeValue);
27743+
UpdateTokenInfo(vResult, tType);
27744+
}
27745+
;
27746+
2767227747
compressionPartitionRange returns [CompressionPartitionRange vResult = FragmentFactory.CreateFragment<CompressionPartitionRange>()]
2767327748
{
2767427749
ScalarExpression vExpression;

‎SqlScriptDom/Parser/TSql/VectorIndexType.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorIndexType.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
#pragma warning disable 1591
12+
13+
/// <summary>
14+
/// The possible values for vector index type
15+
/// </summary>
16+
public enum VectorIndexType
17+
{
18+
DiskANN = 0
19+
}
20+
21+
#pragma warning restore 1591
22+
}
23+

‎SqlScriptDom/Parser/TSql/VectorIndexTypeHelper.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorIndexTypeHelper.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
12+
internal class VectorIndexTypeHelper : OptionsHelper<VectorIndexType>
13+
{
14+
private VectorIndexTypeHelper()
15+
{
16+
AddOptionMapping(VectorIndexType.DiskANN, "'" + CodeGenerationSupporter.DiskANN + "'");
17+
}
18+
19+
public static readonly VectorIndexTypeHelper Instance = new VectorIndexTypeHelper();
20+
}
21+
}

‎SqlScriptDom/Parser/TSql/VectorMetricType.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorMetricType.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
10+
{
11+
#pragma warning disable 1591
12+
13+
/// <summary>
14+
/// The possible values for vector index metric
15+
/// </summary>
16+
public enum VectorMetricType
17+
{
18+
Cosine = 0,
19+
Dot = 1,
20+
Euclidean = 2
21+
}
22+
23+
#pragma warning restore 1591
24+
}
25+

‎SqlScriptDom/Parser/TSql/VectorMetricTypeHelper.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
//------------------------------------------------------------------------------
2+
// <copyright file="VectorMetricTypeHelper.cs" company="Microsoft">
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
// </copyright>
5+
//------------------------------------------------------------------------------
6+
7+
8+
9+
using static Microsoft.SqlServer.TransactSql.ScriptDom.SensitivityClassification;
10+
11+
namespace Microsoft.SqlServer.TransactSql.ScriptDom
12+
{
13+
14+
internal class VectorMetricTypeHelper : OptionsHelper<VectorMetricType>
15+
{
16+
private VectorMetricTypeHelper()
17+
{
18+
AddOptionMapping(VectorMetricType.Cosine, "'" + CodeGenerationSupporter.Cosine + "'");
19+
AddOptionMapping(VectorMetricType.Dot, "'" + CodeGenerationSupporter.Dot + "'");
20+
AddOptionMapping(VectorMetricType.Euclidean, "'" + CodeGenerationSupporter.Euclidean + "'");
21+
}
22+
23+
public static readonly VectorMetricTypeHelper Instance = new VectorMetricTypeHelper();
24+
}
25+
}

0 commit comments

Comments
 (0)