Skip to content

Commit 9f33c90

Browse files
rojiroji
authored
.Net: MEVD data type tests (#12567)
Closes #12505
1 parent 26ca944 commit 9f33c90

File tree

46 files changed

+1493
-189
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1493
-189
lines changed

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@ private void TranslateConstant(ConstantExpression constant)
9898

9999
private void GenerateLiteral(object? value)
100100
{
101-
// TODO: Nullable
102101
switch (value)
103102
{
104103
case byte b:
@@ -114,6 +113,13 @@ private void GenerateLiteral(object? value)
114113
this._filter.Append(l);
115114
return;
116115

116+
case float f:
117+
this._filter.Append(f);
118+
return;
119+
case double d:
120+
this._filter.Append(d);
121+
return;
122+
117123
case string untrustedInput:
118124
// This is the only place where we allow untrusted input to be passed in, so we need to quote and escape it.
119125
this._filter.Append('\'').Append(untrustedInput.Replace("'", "''")).Append('\'');
@@ -125,9 +131,9 @@ private void GenerateLiteral(object? value)
125131
this._filter.Append('\'').Append(g.ToString()).Append('\'');
126132
return;
127133

128-
case DateTime:
129-
case DateTimeOffset:
130-
throw new NotImplementedException();
134+
case DateTimeOffset d:
135+
this._filter.Append(d.ToString("o"));
136+
return;
131137

132138
case Array:
133139
throw new NotImplementedException();
@@ -364,11 +370,12 @@ private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out Prop
364370
}
365371

366372
// Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type
373+
var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type;
367374
unwrappedExpression = expression;
368375
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
369376
{
370377
var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type;
371-
if (convertType != property.Type && convertType != typeof(object))
378+
if (convertType != unwrappedPropertyType && convertType != typeof(object))
372379
{
373380
throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'.");
374381
}

‎dotnet/src/VectorData/Common/SqlFilterTranslator.cs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,6 @@ static bool IsNull(Expression expression)
138138

139139
protected virtual void TranslateConstant(object? value)
140140
{
141-
// TODO: Nullable
142141
switch (value)
143142
{
144143
case byte b:
@@ -154,6 +153,16 @@ protected virtual void TranslateConstant(object? value)
154153
this._sql.Append(l);
155154
return;
156155

156+
case float f:
157+
this._sql.Append(f);
158+
return;
159+
case double d:
160+
this._sql.Append(d);
161+
return;
162+
case decimal d:
163+
this._sql.Append(d);
164+
return;
165+
157166
case string untrustedInput:
158167
// This is the only place where we allow untrusted input to be passed in, so we need to quote and escape it.
159168
// Luckily for us, values are escaped in the same way for every provider that we support so far.
@@ -169,7 +178,11 @@ protected virtual void TranslateConstant(object? value)
169178
case DateTime dateTime:
170179
case DateTimeOffset dateTimeOffset:
171180
case Array:
172-
throw new NotImplementedException();
181+
#if NET8_0_OR_GREATER
182+
case DateOnly dateOnly:
183+
case TimeOnly timeOnly:
184+
#endif
185+
throw new UnreachableException("Database-specific format, needs to be implemented in the provider's derived translator.");
173186

174187
case null:
175188
this._sql.Append("NULL");
@@ -350,11 +363,12 @@ private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out Prop
350363
}
351364

352365
// Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type
366+
var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type;
353367
unwrappedExpression = expression;
354368
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
355369
{
356370
var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type;
357-
if (convertType != property.Type && convertType != typeof(object))
371+
if (convertType != unwrappedPropertyType && convertType != typeof(object))
358372
{
359373
throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'.");
360374
}

‎dotnet/src/VectorData/CosmosMongoDB/CosmosMongoFilterTranslator.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,11 +262,12 @@ private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out Prop
262262
}
263263

264264
// Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type
265+
var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type;
265266
unwrappedExpression = expression;
266267
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
267268
{
268269
var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type;
269-
if (convertType != property.Type && convertType != typeof(object))
270+
if (convertType != unwrappedPropertyType && convertType != typeof(object))
270271
{
271272
throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'.");
272273
}

‎dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -299,10 +299,22 @@ public override Task DeleteAsync(TKey key, CancellationToken cancellationToken =
299299
Verify.NotNullOrWhiteSpace(compositeKey.RecordKey);
300300
Verify.NotNullOrWhiteSpace(compositeKey.PartitionKey);
301301

302-
return this.RunOperationAsync("DeleteItem", () =>
303-
this._database
304-
.GetContainer(this.Name)
305-
.DeleteItemAsync<JsonObject>(compositeKey.RecordKey, new PartitionKey(compositeKey.PartitionKey), cancellationToken: cancellationToken));
302+
return this.RunOperationAsync("DeleteItem", async () =>
303+
{
304+
try
305+
{
306+
await this._database
307+
.GetContainer(this.Name)
308+
.DeleteItemAsync<JsonObject>(compositeKey.RecordKey, new PartitionKey(compositeKey.PartitionKey), cancellationToken: cancellationToken)
309+
.ConfigureAwait(false);
310+
return 0;
311+
}
312+
catch (CosmosException e) when (e.StatusCode == System.Net.HttpStatusCode.NotFound)
313+
{
314+
// Ignore not found errors
315+
return 0;
316+
}
317+
});
306318
}
307319

308320
// TODO: Implement bulk delete, #11350

‎dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

33
using System;
4+
using System.Collections;
45
using System.Collections.Generic;
56
using System.Diagnostics;
67
using System.Diagnostics.CodeAnalysis;
8+
using System.Globalization;
79
using System.Linq;
810
using System.Linq.Expressions;
911
using System.Text;
@@ -100,46 +102,74 @@ private void TranslateBinary(BinaryExpression binary)
100102
}
101103

102104
private void TranslateConstant(ConstantExpression constant)
105+
=> this.TranslateConstant(constant.Value);
106+
107+
private void TranslateConstant(object? value)
103108
{
104-
// TODO: Nullable
105-
switch (constant.Value)
109+
switch (value)
106110
{
107-
case byte b:
108-
this._sql.Append(b);
111+
case byte v:
112+
this._sql.Append(v);
113+
return;
114+
case short v:
115+
this._sql.Append(v);
116+
return;
117+
case int v:
118+
this._sql.Append(v);
109119
return;
110-
case short s:
111-
this._sql.Append(s);
120+
case long v:
121+
this._sql.Append(v);
112122
return;
113-
case int i:
114-
this._sql.Append(i);
123+
124+
case float v:
125+
this._sql.Append(v);
115126
return;
116-
case long l:
117-
this._sql.Append(l);
127+
case double v:
128+
this._sql.Append(v);
118129
return;
119130

120-
case string s:
121-
this._sql.Append('"').Append(s.Replace(@"\", @"\\").Replace("\"", "\\\"")).Append('"');
131+
case string v:
132+
this._sql.Append('"').Append(v.Replace(@"\", @"\\").Replace("\"", "\\\"")).Append('"');
133+
return;
134+
case bool v:
135+
this._sql.Append(v ? "true" : "false");
122136
return;
123-
case bool b:
124-
this._sql.Append(b ? "true" : "false");
137+
case Guid v:
138+
this._sql.Append('"').Append(v.ToString()).Append('"');
125139
return;
126-
case Guid g:
127-
this._sql.Append('"').Append(g.ToString()).Append('"');
140+
141+
case DateTimeOffset v:
142+
// Cosmos doesn't support DateTimeOffset with non-zero offset, so we convert it to UTC.
143+
// See https://github.com/dotnet/efcore/issues/35310
144+
this._sql
145+
.Append('"')
146+
.Append(v.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture))
147+
.Append("Z\"");
128148
return;
129149

130-
case DateTime:
131-
case DateTimeOffset:
132-
throw new NotImplementedException();
150+
case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)):
151+
this._sql.Append('[');
133152

134-
case Array:
135-
throw new NotImplementedException();
153+
var i = 0;
154+
foreach (var element in v)
155+
{
156+
if (i++ > 0)
157+
{
158+
this._sql.Append(',');
159+
}
160+
161+
this.TranslateConstant(element);
162+
}
163+
164+
this._sql.Append(']');
165+
return;
136166

137167
case null:
138168
this._sql.Append("null");
139169
return;
140170

141171
default:
142-
throw new NotSupportedException("Unsupported constant type: " + constant.Value.GetType().Name);
172+
throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name);
143173
}
144174
}
145175

‎dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,13 @@ private BsonDocument GenerateEqualityComparison(PropertyModel property, object?
7272
{
7373
if (value is null)
7474
{
75-
throw new NotSupportedException("MongogDB does not support null checks in vector search pre-filters");
75+
throw new NotSupportedException("MongoDB does not support null checks in vector search pre-filters");
76+
}
77+
78+
if (value is DateTime or decimal or IList)
79+
{
80+
// Operand type is not supported for $vectorSearch: date/decimal
81+
throw new NotSupportedException($"MongoDB does not support type {value.GetType().Name} in vector search pre-filters.");
7682
}
7783

7884
// Short form of equality (instead of $eq)
@@ -261,11 +267,12 @@ private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out Prop
261267
}
262268

263269
// Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type
270+
var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type;
264271
unwrappedExpression = expression;
265272
while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert)
266273
{
267274
var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type;
268-
if (convertType != property.Type && convertType != typeof(object))
275+
if (convertType != unwrappedPropertyType && convertType != typeof(object))
269276
{
270277
throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'.");
271278
}

‎dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System;
4+
using System.Collections;
35
using System.Collections.Generic;
6+
using System.Globalization;
47
using System.Linq.Expressions;
58
using System.Text;
69
using Microsoft.Extensions.VectorData.ProviderServices;
@@ -9,7 +12,6 @@ namespace Microsoft.SemanticKernel.Connectors.PgVector;
912

1013
internal sealed class PostgresFilterTranslator : SqlFilterTranslator
1114
{
12-
private readonly List<object> _parameterValues = new();
1315
private int _parameterIndex;
1416

1517
internal PostgresFilterTranslator(
@@ -21,7 +23,49 @@ internal PostgresFilterTranslator(
2123
this._parameterIndex = startParamIndex;
2224
}
2325

24-
internal List<object> ParameterValues => this._parameterValues;
26+
internal List<object> ParameterValues { get; } = new();
27+
28+
protected override void TranslateConstant(object? value)
29+
{
30+
switch (value)
31+
{
32+
// TODO: This aligns with our mapping of DateTime to PG's timestamp (as opposed to timestamptz) - we probably want to
33+
// change that to timestamptz (aligning with Npgsql and EF). See #10641.
34+
case DateTime dateTime:
35+
this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\'');
36+
return;
37+
case DateTimeOffset dateTimeOffset:
38+
if (dateTimeOffset.Offset != TimeSpan.Zero)
39+
{
40+
throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL");
41+
}
42+
43+
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append("Z'");
44+
return;
45+
46+
// Array constants (ARRAY[1, 2, 3])
47+
case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)):
48+
this._sql.Append("ARRAY[");
49+
50+
var i = 0;
51+
foreach (var element in v)
52+
{
53+
if (i++ > 0)
54+
{
55+
this._sql.Append(',');
56+
}
57+
58+
this.TranslateConstant(element);
59+
}
60+
61+
this._sql.Append(']');
62+
return;
63+
64+
default:
65+
base.TranslateConstant(value);
66+
break;
67+
}
68+
}
2569

2670
protected override void TranslateContainsOverArrayColumn(Expression source, Expression item)
2771
{
@@ -49,7 +93,7 @@ protected override void TranslateQueryParameter(object? value)
4993
}
5094
else
5195
{
52-
this._parameterValues.Add(value);
96+
this.ParameterValues.Add(value);
5397
// The param name is just the index, so there is no need for escaping or quoting.
5498
this._sql.Append('$').Append(this._parameterIndex++);
5599
}

‎dotnet/src/VectorData/PgVector/PostgresSqlBuilder.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,22 +60,22 @@ internal static string BuildCreateTableSql(string schema, string tableName, Coll
6060

6161
// Add the key column
6262
var keyPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty.Type);
63-
createTableCommand.AppendLine($" \"{keyName}\" {keyPgTypeInfo.PgType} {(keyPgTypeInfo.IsNullable ? "" : "NOT NULL")},");
63+
createTableCommand.AppendLine($" \"{keyName}\" {keyPgTypeInfo.PgType}{(keyPgTypeInfo.IsNullable ? "" : " NOT NULL")},");
6464

6565
// Add the data columns
6666
foreach (var dataProperty in model.DataProperties)
6767
{
6868
string columnName = dataProperty.StorageName;
6969
var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty.Type);
70-
createTableCommand.AppendLine($" \"{columnName}\" {dataPgTypeInfo.PgType} {(dataPgTypeInfo.IsNullable ? "" : "NOT NULL")},");
70+
createTableCommand.AppendLine($" \"{columnName}\" {dataPgTypeInfo.PgType}{(dataPgTypeInfo.IsNullable ? "" : " NOT NULL")},");
7171
}
7272

7373
// Add the vector columns
7474
foreach (var vectorProperty in model.VectorProperties)
7575
{
7676
string columnName = vectorProperty.StorageName;
7777
var vectorPgTypeInfo = PostgresPropertyMapping.GetPgVectorTypeName(vectorProperty);
78-
createTableCommand.AppendLine($" \"{columnName}\" {vectorPgTypeInfo.PgType} {(vectorPgTypeInfo.IsNullable ? "" : "NOT NULL")},");
78+
createTableCommand.AppendLine($" \"{columnName}\" {vectorPgTypeInfo.PgType}{(vectorPgTypeInfo.IsNullable ? "" : " NOT NULL")},");
7979
}
8080

8181
createTableCommand.AppendLine($" PRIMARY KEY (\"{keyName}\")");

0 commit comments

Comments
 (0)