Merge branch 'main' into fix/optimize-function-copy-deepcopy

Author: nimanikoo

.gitignore

@@ -502,3 +502,5 @@ swa-cli.config.json
 
 # dapr extension files
 **/dapr.yaml
+
+*.lscache

dotnet/src/VectorData/SqlServer/SqlServerCollection.cs

@@ -42,6 +42,12 @@ public class SqlServerCollection<TKey, TRecord>
     /// <summary>The database schema.</summary>
     private readonly string? _schema;
 
+    /// <summary>Whether the model contains any DiskAnn vector properties, requiring Azure SQL.</summary>
+    private readonly bool _requiresAzureSql;
+
+    /// <summary>Cached result of the Azure SQL engine edition check (null = not yet checked).</summary>
+    private bool? _isAzureSql;
+
     /// <summary>
     /// Initializes a new instance of the <see cref="SqlServerCollection{TKey, TRecord}"/> class.
     /// </summary>
@@ -78,6 +84,16 @@ internal SqlServerCollection(string connectionString, string name, Func<SqlServe
 
         this._mapper = new SqlServerMapper<TRecord>(this._model);
 
+        // Check if any vector property uses DiskAnn, which requires Azure SQL.
+        foreach (var vp in this._model.VectorProperties)
+        {
+            if (vp.IndexKind == IndexKind.DiskAnn)
+            {
+                this._requiresAzureSql = true;
+                break;
+            }
+        }
+
         var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString);
 
         this._collectionMetadata = new()
@@ -116,6 +132,12 @@ public override Task EnsureCollectionExistsAsync(CancellationToken cancellationT
     private async Task CreateCollectionAsync(bool ifNotExists, CancellationToken cancellationToken)
     {
         using SqlConnection connection = new(this._connectionString);
+
+        if (this._requiresAzureSql)
+        {
+            await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
+        }
+
         List<SqlCommand> commands = SqlServerCommandBuilder.CreateTable(
             connection,
             this._schema,
@@ -604,6 +626,12 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
         // Connection and command are going to be disposed by the ReadVectorSearchResultsAsync,
         // when the user is done with the results.
         SqlConnection connection = new(this._connectionString);
+
+        if (vectorProperty.IndexKind == IndexKind.DiskAnn)
+        {
+            await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
+        }
+
         SqlCommand command = SqlServerCommandBuilder.SelectVector(
             connection,
             this._schema,
@@ -664,6 +692,12 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
         // Connection and command are going to be disposed by the ReadVectorSearchResultsAsync,
         // when the user is done with the results.
         SqlConnection connection = new(this._connectionString);
+
+        if (vectorProperty.IndexKind == IndexKind.DiskAnn)
+        {
+            await this.EnsureAzureSqlForDiskAnnAsync(connection, cancellationToken).ConfigureAwait(false);
+        }
+
         SqlCommand command = SqlServerCommandBuilder.SelectHybrid(
             connection,
             this._schema,
@@ -807,4 +841,48 @@ public override async IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord
             yield return this._mapper.MapFromStorageToDataModel(reader, options.IncludeVectors);
         }
     }
+
+    /// <summary>
+    /// Validates that the connection is to Azure SQL Database or SQL database in Microsoft Fabric,
+    /// which is required for DiskAnn vector indexes and the VECTOR_SEARCH function.
+    /// </summary>
+    private async Task EnsureAzureSqlForDiskAnnAsync(SqlConnection connection, CancellationToken cancellationToken)
+    {
+        if (this._isAzureSql is true)
+        {
+            return;
+        }
+
+        if (this._isAzureSql is false)
+        {
+            connection.Dispose();
+            throw new NotSupportedException(
+                "DiskAnn vector indexes and the VECTOR_SEARCH function require Azure SQL Database or SQL database in Microsoft Fabric. " +
+                "They are not supported on SQL Server. Use a Flat index kind with VECTOR_DISTANCE instead.");
+        }
+
+        if (connection.State != System.Data.ConnectionState.Open)
+        {
+            await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
+        }
+
+        using var command = connection.CreateCommand();
+        command.CommandText = "SELECT SERVERPROPERTY('EngineEdition')";
+        var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+        var engineEdition = Convert.ToInt32(result);
+
+        // 5 = Azure SQL Database, 11 = SQL database in Microsoft Fabric
+        this._isAzureSql = engineEdition is 5 or 11;
+
+        if (!this._isAzureSql.Value)
+        {
+            // Dispose the connection before throwing; in SearchAsync/HybridSearchAsync the connection
+            // is not in a using block (it's normally disposed by ReadVectorSearchResultsAsync).
+            connection.Dispose();
+
+            throw new NotSupportedException(
+                "DiskAnn vector indexes and the VECTOR_SEARCH function require Azure SQL Database or SQL database in Microsoft Fabric. " +
+                "They are not supported on SQL Server. Use a Flat index kind with VECTOR_DISTANCE instead.");
+        }
+    }
 }

dotnet/src/VectorData/SqlServer/SqlServerCommandBuilder.cs

@@ -535,41 +535,67 @@ private static SqlCommand SelectVectorWithVectorSearch<TRecord>(
         string distanceMetric,
         string sorting)
     {
-        // VECTOR_SEARCH() currently only supports post-filtering (TOP_N candidates are returned first,
-        // then predicates are applied). Pre-filtering is not supported.
-        if (options.Filter is not null)
-        {
-            throw new NotSupportedException(
-                "Filtering is not supported with approximate vector search (VECTOR_SEARCH). " +
-                "Remove the filter or use IndexKind.Flat for exact search with VECTOR_DISTANCE.");
-        }
-
         SqlCommand command = connection.CreateCommand();
         command.Parameters.AddWithValue("@vector", vector);
 
         StringBuilder sb = new(300);
 
+        // When skip > 0, we need a subquery since TOP and OFFSET/FETCH can't coexist in the same SELECT.
+        bool needsSubquery = options.Skip > 0;
+
+        if (needsSubquery)
+        {
+            sb.Append("SELECT * FROM (");
+        }
+
         // VECTOR_SEARCH returns all columns from the table plus a 'distance' column.
         // We select the needed columns from the table alias and alias 'distance' as 'score'.
-        sb.Append("SELECT ");
+        // The latest version vector indexes require SELECT TOP(N) WITH APPROXIMATE instead of the deprecated TOP_N parameter.
+        sb.Append("SELECT TOP(").Append(top + options.Skip).Append(") WITH APPROXIMATE ");
         sb.AppendIdentifiers(model.Properties, prefix: "t.", includeVectors: options.IncludeVectors);
         sb.AppendLine(",");
         sb.AppendLine("s.[distance] AS [score]");
         sb.Append("FROM VECTOR_SEARCH(TABLE = ");
         sb.AppendTableName(schema, tableName);
         sb.Append(" AS t, COLUMN = ").AppendIdentifier(vectorProperty.StorageName);
-        sb.Append(", SIMILAR_TO = @vector, METRIC = '").Append(distanceMetric).Append('\'');
-        sb.Append(", TOP_N = ").Append(top + options.Skip).AppendLine(") AS s");
+        sb.Append(", SIMILAR_TO = @vector, METRIC = '").Append(distanceMetric).AppendLine("') AS s");
+
+        // With latest version vector indexes, WHERE predicates are applied during the vector search process
+        // (iterative filtering), not after retrieval.
+        if (options.Filter is not null)
+        {
+            int startParamIndex = command.Parameters.Count;
+
+            SqlServerFilterTranslator translator = new(model, options.Filter, sb, startParamIndex: startParamIndex, tableAlias: "t");
+            translator.Translate(appendWhere: true);
+            List<object> parameters = translator.ParameterValues;
+
+            foreach (object parameter in parameters)
+            {
+                command.AddParameter(property: null, $"@_{startParamIndex++}", parameter);
+            }
+
+            sb.AppendLine();
+        }
 
         if (options.ScoreThreshold is not null)
         {
             command.Parameters.AddWithValue("@scoreThreshold", options.ScoreThreshold!.Value);
-            sb.AppendLine("WHERE s.[distance] <= @scoreThreshold");
+            sb.Append(options.Filter is not null ? "AND " : "WHERE ");
+            sb.AppendLine("s.[distance] <= @scoreThreshold");
         }
 
         sb.AppendFormat("ORDER BY [score] {0}", sorting);
-        sb.AppendLine();
-        sb.AppendFormat("OFFSET {0} ROWS FETCH NEXT {1} ROWS ONLY;", options.Skip, top);
+
+        if (needsSubquery)
+        {
+            sb.AppendLine();
+            sb.Append(") AS [inner]");
+            sb.AppendLine();
+            sb.AppendFormat("ORDER BY [score] {0}", sorting);
+            sb.AppendLine();
+            sb.AppendFormat("OFFSET {0} ROWS FETCH NEXT {1} ROWS ONLY;", options.Skip, top);
+        }
 
         command.CommandText = sb.ToString();
         return command;
@@ -587,15 +613,6 @@ internal static SqlCommand SelectHybrid<TRecord>(
     {
         bool useVectorSearch = UseVectorSearch(vectorProperty);
 
-        // VECTOR_SEARCH() currently only supports post-filtering (TOP_N candidates are returned first,
-        // then predicates are applied). Pre-filtering is not supported.
-        if (useVectorSearch && options.Filter is not null)
-        {
-            throw new NotSupportedException(
-                "Filtering is not supported with approximate vector search (VECTOR_SEARCH). " +
-                "Remove the filter or use IndexKind.Flat for exact search with VECTOR_DISTANCE.");
-        }
-
         string distanceFunction = vectorProperty.DistanceFunction ?? DistanceFunction.CosineDistance;
         (string distanceMetric, _) = MapDistanceFunction(distanceFunction);
 
@@ -652,16 +669,32 @@ internal static SqlCommand SelectHybrid<TRecord>(
         // CTE 2: Semantic/vector search
         if (useVectorSearch)
         {
-            // Use VECTOR_SEARCH() for approximate nearest neighbor search with a vector index
+            // Use VECTOR_SEARCH() for approximate nearest neighbor search with a vector index.
+            // The latest version vector indexes require SELECT TOP(N) WITH APPROXIMATE instead of the deprecated TOP_N parameter.
             sb.AppendLine("semantic_search AS (");
-            sb.AppendLine("    SELECT TOP(@candidateCount)");
+            sb.AppendLine("    SELECT TOP(@candidateCount) WITH APPROXIMATE");
             sb.Append("        t.").AppendIdentifier(model.KeyProperty.StorageName).AppendLine(",");
             sb.AppendLine("        RANK() OVER (ORDER BY s.[distance]) AS [rank]");
             sb.AppendLine("    FROM VECTOR_SEARCH(TABLE = ");
             sb.Append("        ").AppendTableName(schema, tableName);
             sb.Append(" AS t, COLUMN = ").AppendIdentifier(vectorProperty.StorageName);
-            sb.Append(", SIMILAR_TO = @vector, METRIC = '").Append(distanceMetric).Append('\'');
-            sb.Append(", TOP_N = @candidateCount").AppendLine(") AS s");
+            sb.Append(", SIMILAR_TO = @vector, METRIC = '").Append(distanceMetric).AppendLine("') AS s");
+
+            // With latest version vector indexes, WHERE predicates are applied during the vector search process
+            // (iterative filtering), not after retrieval.
+            if (options.Filter is not null)
+            {
+                int filterParamStart = command.Parameters.Count;
+                SqlServerFilterTranslator translator = new(model, options.Filter, sb, startParamIndex: filterParamStart, tableAlias: "t");
+                translator.Translate(appendWhere: true);
+                foreach (object parameter in translator.ParameterValues)
+                {
+                    command.AddParameter(property: null, $"@_{filterParamStart++}", parameter);
+                }
+                sb.AppendLine();
+            }
+
+            sb.AppendLine("    ORDER BY s.[distance]");
             sb.AppendLine("),");
         }
         else