Skip to content

Commit 1122769

Browse files
authored
Merge branch 'main' into python-sql-escape
2 parents 5513479 + 1b2f318 commit 1122769

62 files changed

Lines changed: 1676 additions & 700 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/devflow-pr-review.yml

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ on:
88
- ready_for_review
99
workflow_dispatch:
1010
inputs:
11-
pr_url:
12-
description: Pull request URL to review
11+
pr_number:
12+
description: Pull request number to review
1313
required: true
1414
type: string
1515

@@ -40,39 +40,27 @@ jobs:
4040
shell: bash
4141
env:
4242
PR_HTML_URL: ${{ github.event.pull_request.html_url }}
43-
PR_NUMBER: ${{ github.event.pull_request.number }}
44-
PR_URL_INPUT: ${{ inputs.pr_url }}
43+
PR_NUMBER_EVENT: ${{ github.event.pull_request.number }}
44+
PR_NUMBER_INPUT: ${{ inputs.pr_number }}
4545
run: |
4646
set -euo pipefail
4747
4848
if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
49-
echo "pr_url=${PR_HTML_URL}" >> "$GITHUB_OUTPUT"
50-
echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT"
51-
echo "repo=${GITHUB_REPOSITORY}" >> "$GITHUB_OUTPUT"
52-
exit 0
49+
pr_number="${PR_NUMBER_EVENT}"
50+
pr_url="${PR_HTML_URL}"
51+
else
52+
pr_number="${PR_NUMBER_INPUT}"
53+
pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${pr_number}"
5354
fi
5455
55-
if [[ -z "$PR_URL_INPUT" ]]; then
56-
echo "workflow_dispatch requires pr_url" >&2
56+
if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then
57+
echo "Could not determine PR number; for workflow_dispatch runs, the 'pr_number' input is required when not running on pull_request_target." >&2
5758
exit 1
5859
fi
5960
60-
if [[ ! "$PR_URL_INPUT" =~ ^https://github\.com/([^/]+/[^/]+)/pull/([0-9]+)([/?].*)?$ ]]; then
61-
echo "Could not parse pull request URL (expected https://github.com/<owner>/<repo>/pull/<number>): $PR_URL_INPUT" >&2
62-
exit 1
63-
fi
64-
65-
pr_repo="${BASH_REMATCH[1]}"
66-
pr_number="${BASH_REMATCH[2]}"
67-
68-
if [[ "$pr_repo" != "$GITHUB_REPOSITORY" ]]; then
69-
echo "PR URL repository ($pr_repo) does not match current repository ($GITHUB_REPOSITORY)" >&2
70-
exit 1
71-
fi
72-
73-
echo "pr_url=${PR_URL_INPUT}" >> "$GITHUB_OUTPUT"
61+
echo "pr_url=${pr_url}" >> "$GITHUB_OUTPUT"
7462
echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT"
75-
echo "repo=${pr_repo}" >> "$GITHUB_OUTPUT"
63+
echo "repo=${GITHUB_REPOSITORY}" >> "$GITHUB_OUTPUT"
7664
7765
# Safe checkout: base repo only, not the untrusted PR head.
7866
- name: Checkout target repo base

.github/workflows/python-integration-tests.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ env:
6868
GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
6969
GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }}
7070
GOOGLE_AI_CLOUD_PROJECT_ID: ${{ vars.GOOGLE_AI_CLOUD_PROJECT_ID }}
71+
GOOGLE_AI_CLOUD_REGION: ${{ vars.GOOGLE_AI_CLOUD_REGION }}
7172
VERTEX_AI_PROJECT_ID: ${{ vars.VERTEX_AI_PROJECT_ID }}
7273
VERTEX_AI_GEMINI_MODEL_ID: ${{ vars.VERTEX_AI_GEMINI_MODEL_ID }}
7374
VERTEX_AI_EMBEDDING_MODEL_ID: ${{ vars.VERTEX_AI_EMBEDDING_MODEL_ID }}

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,3 +502,5 @@ swa-cli.config.json
502502

503503
# dapr extension files
504504
**/dapr.yaml
505+
506+
*.lscache

dotnet/Directory.Packages.props

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@
8989
<PackageVersion Include="ModelContextProtocol" Version="0.4.0-preview.3" />
9090
<PackageVersion Include="MSTest.TestFramework" Version="3.8.0" />
9191
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
92-
<PackageVersion Include="Npgsql" Version="8.0.7" />
92+
<PackageVersion Include="Npgsql" Version="10.0.2" />
9393
<PackageVersion Include="OData2Linq" Version="2.2.0" />
9494
<PackageVersion Include="OllamaSharp" Version="5.4.12" />
9595
<PackageVersion Include="OpenAI" Version="2.9.1" />
@@ -102,7 +102,7 @@
102102
<PackageVersion Include="PdfPig" Version="0.1.13" />
103103
<PackageVersion Include="Pinecone.Client" Version="3.1.0" />
104104
<PackageVersion Include="Prompty.Core" Version="0.2.3-beta" />
105-
<PackageVersion Include="Scriban" Version="6.6.0" />
105+
<PackageVersion Include="Scriban" Version="7.0.3" />
106106
<PackageVersion Include="PuppeteerSharp" Version="20.2.5" />
107107
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.2" />
108108
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.15.0" />

dotnet/SK-release.slnf

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,6 @@
3232
"src\\Connectors\\Connectors.Onnx\\Connectors.Onnx.csproj",
3333
"src\\Connectors\\Connectors.OpenAI\\Connectors.OpenAI.csproj",
3434

35-
"src\\VectorData\\AzureAISearch\\AzureAISearch.csproj",
36-
"src\\VectorData\\Chroma\\Chroma.csproj",
37-
"src\\VectorData\\CosmosMongoDB\\CosmosMongoDB.csproj",
38-
"src\\VectorData\\CosmosNoSql\\CosmosNoSql.csproj",
39-
"src\\VectorData\\InMemory\\InMemory.csproj",
40-
"src\\VectorData\\Milvus\\Milvus.csproj",
41-
"src\\VectorData\\MongoDB\\MongoDB.csproj",
42-
"src\\VectorData\\PgVector\\PgVector.csproj",
43-
"src\\VectorData\\Pinecone\\Pinecone.csproj",
44-
"src\\VectorData\\Qdrant\\Qdrant.csproj",
45-
"src\\VectorData\\Redis\\Redis.csproj",
46-
"src\\VectorData\\SqliteVec\\SqliteVec.csproj",
47-
"src\\VectorData\\SqlServer\\SqlServer.csproj",
48-
"src\\VectorData\\VectorData.Abstractions\\VectorData.Abstractions.csproj",
49-
"src\\VectorData\\Weaviate\\Weaviate.csproj",
50-
5135
"src\\Experimental\\Orchestration.Flow\\Experimental.Orchestration.Flow.csproj",
5236

5337
"src\\Experimental\\Process.Abstractions\\Process.Abstractions.csproj",

dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,14 @@ internal class MongoModelBuilder() : CollectionModelBuilder(s_validationOptions)
2727
UsesExternalSerializer = true,
2828
};
2929

30-
[RequiresUnreferencedCode("Traverses the CLR type's properties with reflection, so not compatible with trimming")]
31-
protected override void ProcessTypeProperties(Type type, VectorStoreCollectionDefinition? definition)
30+
protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type)
3231
{
33-
base.ProcessTypeProperties(type, definition);
32+
base.ProcessProperty(clrProperty, definitionProperty, type);
3433

35-
foreach (var property in this.Properties)
34+
if (clrProperty?.GetCustomAttribute<BsonElementAttribute>() is { } bsonElementAttribute
35+
&& this.PropertyMap.TryGetValue(clrProperty.Name, out var property))
3636
{
37-
if (property.PropertyInfo?.GetCustomAttribute<BsonElementAttribute>() is { } bsonElementAttribute)
38-
{
39-
property.StorageName = bsonElementAttribute.ElementName;
40-
}
37+
property.StorageName = bsonElementAttribute.ElementName;
4138
}
4239
}
4340

dotnet/src/SemanticKernel.Core/Functions/KernelFunctionFromMethod.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -720,7 +720,7 @@ private static (Func<KernelFunction, Kernel, KernelArguments, CancellationToken,
720720
return value;
721721
}
722722

723-
if (converter is not null && value is not JsonElement or JsonDocument or JsonNode)
723+
if (converter is not null && value is not (JsonElement or JsonDocument or JsonNode))
724724
{
725725
try
726726
{

dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,
4141

4242
// The key property in Azure CosmosDB NoSQL is always named 'id'.
4343
// But the external JSON serializer used just above isn't aware of that, and will produce a JSON object with another name, taking into
44-
// account e.g. naming policies. TemporaryStorageName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set
45-
RenameJsonProperty(jsonObject, this._keyProperty.TemporaryStorageName!, CosmosNoSqlConstants.ReservedKeyPropertyName);
44+
// account e.g. naming policies. SerializedKeyName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set
45+
RenameJsonProperty(jsonObject, this._keyProperty.SerializedKeyName!, CosmosNoSqlConstants.ReservedKeyPropertyName);
4646

4747
// Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above.
4848
// Also, for Embedding<T> properties we also need to overwrite with a simple array (since Embedding<T> gets serialized as a complex object).
@@ -116,7 +116,7 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,
116116
public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
117117
{
118118
// See above comment.
119-
RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.TemporaryStorageName!);
119+
RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.SerializedKeyName!);
120120

121121
foreach (var vectorProperty in this._model.VectorProperties)
122122
{

dotnet/src/VectorData/PgVector/PgVector.csproj

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@
3232
<ItemGroup>
3333
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
3434
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
35-
<PackageReference Include="Npgsql" />
35+
<PackageReference Include="Npgsql">
36+
<VersionOverride Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">8.0.7</VersionOverride>
37+
</PackageReference>
3638
<PackageReference Include="Pgvector" />
3739
</ItemGroup>
3840

dotnet/src/VectorData/SqlServer/SqlServerCollection.cs

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ public class SqlServerCollection<TKey, TRecord>
4242
/// <summary>The database schema.</summary>
4343
private readonly string? _schema;
4444

45+
/// <summary>Whether the model contains any DiskAnn vector properties, requiring Azure SQL.</summary>
46+
private readonly bool _requiresAzureSql;
47+
48+
/// <summary>Cached result of the Azure SQL engine edition check (null = not yet checked).</summary>
49+
private bool? _isAzureSql;
50+
4551
/// <summary>
4652
/// Initializes a new instance of the <see cref="SqlServerCollection{TKey, TRecord}"/> class.
4753
/// </summary>
@@ -78,6 +84,16 @@ internal SqlServerCollection(string connectionString, string name, Func<SqlServe
7884

7985
this._mapper = new SqlServerMapper<TRecord>(this._model);
8086

87+
// Check if any vector property uses DiskAnn, which requires Azure SQL.
88+
foreach (var vp in this._model.VectorProperties)
89+
{
90+
if (vp.IndexKind == IndexKind.DiskAnn)
91+
{
92+
this._requiresAzureSql = true;
93+
break;
94+
}
95+
}
96+
8197
var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString);
8298

8399
this._collectionMetadata = new()
@@ -116,6 +132,12 @@ public override Task EnsureCollectionExistsAsync(CancellationToken cancellationT
116132
private async Task CreateCollectionAsync(bool ifNotExists, CancellationToken cancellationToken)
117133
{
118134
using SqlConnection connection = new(this._connectionString);
135+
136+
if (this._requiresAzureSql)
137+
{
138+
await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
139+
}
140+
119141
List<SqlCommand> commands = SqlServerCommandBuilder.CreateTable(
120142
connection,
121143
this._schema,
@@ -604,6 +626,12 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
604626
// Connection and command are going to be disposed by the ReadVectorSearchResultsAsync,
605627
// when the user is done with the results.
606628
SqlConnection connection = new(this._connectionString);
629+
630+
if (vectorProperty.IndexKind == IndexKind.DiskAnn)
631+
{
632+
await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
633+
}
634+
607635
SqlCommand command = SqlServerCommandBuilder.SelectVector(
608636
connection,
609637
this._schema,
@@ -664,6 +692,12 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
664692
// Connection and command are going to be disposed by the ReadVectorSearchResultsAsync,
665693
// when the user is done with the results.
666694
SqlConnection connection = new(this._connectionString);
695+
696+
if (vectorProperty.IndexKind == IndexKind.DiskAnn)
697+
{
698+
await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
699+
}
700+
667701
SqlCommand command = SqlServerCommandBuilder.SelectHybrid(
668702
connection,
669703
this._schema,
@@ -807,4 +841,48 @@ public override async IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord
807841
yield return this._mapper.MapFromStorageToDataModel(reader, options.IncludeVectors);
808842
}
809843
}
844+
845+
/// <summary>
846+
/// Validates that the connection is to Azure SQL Database or SQL database in Microsoft Fabric,
847+
/// which is required for DiskAnn vector indexes and the VECTOR_SEARCH function.
848+
/// </summary>
849+
private async Task EnsureAzureSqlForDiskAnnAsync(SqlConnection connection, CancellationToken cancellationToken)
850+
{
851+
if (this._isAzureSql is true)
852+
{
853+
return;
854+
}
855+
856+
if (this._isAzureSql is false)
857+
{
858+
connection.Dispose();
859+
throw new NotSupportedException(
860+
"DiskAnn vector indexes and the VECTOR_SEARCH function require Azure SQL Database or SQL database in Microsoft Fabric. " +
861+
"They are not supported on SQL Server. Use a Flat index kind with VECTOR_DISTANCE instead.");
862+
}
863+
864+
if (connection.State != System.Data.ConnectionState.Open)
865+
{
866+
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
867+
}
868+
869+
using var command = connection.CreateCommand();
870+
command.CommandText = "SELECT SERVERPROPERTY('EngineEdition')";
871+
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
872+
var engineEdition = Convert.ToInt32(result);
873+
874+
// 5 = Azure SQL Database, 11 = SQL database in Microsoft Fabric
875+
this._isAzureSql = engineEdition is 5 or 11;
876+
877+
if (!this._isAzureSql.Value)
878+
{
879+
// Dispose the connection before throwing; in SearchAsync/HybridSearchAsync the connection
880+
// is not in a using block (it's normally disposed by ReadVectorSearchResultsAsync).
881+
connection.Dispose();
882+
883+
throw new NotSupportedException(
884+
"DiskAnn vector indexes and the VECTOR_SEARCH function require Azure SQL Database or SQL database in Microsoft Fabric. " +
885+
"They are not supported on SQL Server. Use a Flat index kind with VECTOR_DISTANCE instead.");
886+
}
887+
}
810888
}

0 commit comments

Comments
 (0)