Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/EFCore.MySql/Infrastructure/MariaDbServerVersion.cs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ internal MariaDbServerVersionSupport([NotNull] ServerVersion serverVersion)
public override bool CommonTableExpressions => ServerVersion.Version >= new Version(10, 2, 1);
public override bool LimitWithinInAllAnySomeSubquery => false;
public override bool LimitWithNonConstantValue => false;
public override bool Vector => ServerVersion.Version >= new Version(11, 7, 0);
public override bool VectorIndex => ServerVersion.Version >= new Version(11, 7, 0);
public override bool JsonTable => ServerVersion.Version >= new Version(10, 6, 0); // Since there seems to be no implicit LATERAL support for JSON_TABLE, this is pretty useless except for cases where the JSON is provided by a parameter instead of a column of an outer table.
public override bool JsonValue => true;
public override bool JsonOverlaps => ServerVersion.Version >= new Version(10, 9, 0);
Expand Down
1 change: 1 addition & 0 deletions src/EFCore.MySql/Infrastructure/MySqlServerVersion.cs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ internal MySqlServerVersionSupport([NotNull] ServerVersion serverVersion)
public override bool JsonOverlaps => ServerVersion.Version >= new Version(8, 0, 0);
public override bool Values => false;
public override bool ValuesWithRows => ServerVersion.Version >= new Version(8, 0, 19);
public override bool Vector => ServerVersion.Version >= new Version(9, 0, 0);
public override bool WhereSubqueryReferencesOuterQuery => false;
public override bool FieldReferenceInTableValueConstructor => true;
public override bool CollationCharacterSetApplicabilityWithFullCollationNameColumn => false;
Expand Down
2 changes: 2 additions & 0 deletions src/EFCore.MySql/Infrastructure/ServerVersionSupport.cs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ public virtual bool PropertyOrVersion(string propertyNameOrServerVersion)
public virtual bool JsonValue => false;
public virtual bool Values => false;
public virtual bool ValuesWithRows => false;
public virtual bool Vector => false;
public virtual bool VectorIndex => false;
public virtual bool WhereSubqueryReferencesOuterQuery => false;
public virtual bool FieldReferenceInTableValueConstructor => false;
public virtual bool CollationCharacterSetApplicabilityWithFullCollationNameColumn => false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public class MySqlQuerySqlGenerator : QuerySqlGenerator
{ "json", new []{ "json" } },
{ "char", new []{ "char", "varchar", "text", "tinytext", "mediumtext", "longtext" } },
{ "nchar", new []{ "nchar", "nvarchar" } },
{ "vector", new []{ "vector" } },
};

private const ulong LimitUpperBound = 18446744073709551610;
Expand Down
3 changes: 2 additions & 1 deletion src/EFCore.MySql/Storage/Internal/MySqlTypeMapping.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public MySqlTypeMapping(
DbType? dbType = null,
bool unicode = false,
int? size = null,
StoreTypePostfix storeTypePostfix = StoreTypePostfix.None,
ValueConverter valueConverter = null,
ValueComparer valueComparer = null,
JsonValueReaderWriter jsonValueReaderWriter = null)
Expand All @@ -44,7 +45,7 @@ public MySqlTypeMapping(
valueComparer,
jsonValueReaderWriter: jsonValueReaderWriter),
storeType,
StoreTypePostfix.None,
storeTypePostfix,
dbType,
unicode,
size))
Expand Down
55 changes: 54 additions & 1 deletion src/EFCore.MySql/Storage/Internal/MySqlTypeMappingSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,15 @@ public class MySqlTypeMappingSource : RelationalTypeMappingSource
private readonly MySqlDateTimeOffsetTypeMapping _dateTimeOffset = MySqlDateTimeOffsetTypeMapping.Default;
private readonly MySqlDateTimeOffsetTypeMapping _timeStampOffset = new MySqlDateTimeOffsetTypeMapping("timestamp");

// vector
private readonly MySqlVectorTypeMapping _floatVector = new MySqlVectorTypeMapping("vector", typeof(float[]));
private readonly MySqlVectorTypeMapping _memoryVector = new MySqlVectorTypeMapping("vector", typeof(Memory<float>));
private readonly MySqlVectorTypeMapping _readOnlyMemoryVector = new MySqlVectorTypeMapping("vector", typeof(ReadOnlyMemory<float>));
private readonly MySqlVectorByteTypeMapping _floatByteVector = new MySqlVectorByteTypeMapping("vector", typeof(float[]));
private readonly MySqlVectorByteTypeMapping _memoryByteVector = new MySqlVectorByteTypeMapping("vector", typeof(Memory<float>));
private readonly MySqlVectorByteTypeMapping _readOnlyByteMemoryVector = new MySqlVectorByteTypeMapping("vector", typeof(ReadOnlyMemory<float>));


private readonly RelationalTypeMapping _binaryRowVersion
= new MySqlDateTimeTypeMapping(
"timestamp",
Expand Down Expand Up @@ -251,6 +260,31 @@ private void Initialize()
{ typeof(MySqlJsonString), _jsonDefaultString }
};

// vector
if (_options.ServerVersion.Supports.Vector)
{
_storeTypeMappings[_floatVector.StoreType] = _options.ServerVersion.Type switch
{
ServerType.MariaDb => new RelationalTypeMapping[] { _floatByteVector, _memoryByteVector, _readOnlyByteMemoryVector },
_ => new RelationalTypeMapping[] { _floatVector, _readOnlyMemoryVector, _memoryVector, }
};
_clrTypeMappings[typeof(float[])] = _options.ServerVersion.Type switch
{
ServerType.MariaDb => _floatByteVector,
_ => _floatVector,
};
_clrTypeMappings[typeof(ReadOnlyMemory<float>)] = _options.ServerVersion.Type switch
{
ServerType.MariaDb => _readOnlyByteMemoryVector,
_ => _readOnlyMemoryVector,
};
_clrTypeMappings[typeof(Memory<float>)] = _options.ServerVersion.Type switch
{
ServerType.MariaDb => _memoryByteVector,
_ => _memoryVector,
};
}

// Boolean
if (_options.DefaultDataTypeMappings.ClrBoolean != MySqlBooleanType.None)
{
Expand All @@ -263,7 +297,7 @@ private void Initialize()
// Guid
if (_guid != null)
{
_storeTypeMappings[_guid.StoreType] = new RelationalTypeMapping[]{ _guid };
_storeTypeMappings[_guid.StoreType] = new RelationalTypeMapping[] { _guid };
_clrTypeMappings[typeof(Guid)] = _guid;
}

Expand All @@ -284,6 +318,25 @@ protected override RelationalTypeMapping FindMapping(in RelationalTypeMappingInf
base.FindMapping(mappingInfo) ??
FindRawMapping(mappingInfo)?.Clone(mappingInfo);

protected override RelationalTypeMapping FindCollectionMapping(
RelationalTypeMappingInfo mappingInfo,
Type modelType,
Type providerType,
CoreTypeMapping elementMapping)
{
// Workaround: prevent EF from trying to treat float[] → byte[] mappings as collections
if (modelType == typeof(byte[]) && (mappingInfo.StoreTypeName?.Equals("vector", StringComparison.OrdinalIgnoreCase) ?? false))
{
if (_storeTypeMappings.TryGetValue(mappingInfo.StoreTypeName, out var mappings))
{
return mappings.First(x => x.ClrType == typeof(ReadOnlyMemory<float>) || x.ClrType == typeof(Memory<float>) || x.ClrType == typeof(float[]));
}
}

// Fallback to base behavior for actual collections
return base.FindCollectionMapping(mappingInfo, modelType, providerType, elementMapping);
}

private RelationalTypeMapping FindRawMapping(RelationalTypeMappingInfo mappingInfo)
{
// Use deferred initialization to support connection (string) based type mapping in
Expand Down
112 changes: 112 additions & 0 deletions src/EFCore.MySql/Storage/Internal/MySqlVectorByteTypeMapping.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Copyright (c) Pomelo Foundation. All rights reserved.
// Licensed under the MIT. See LICENSE in the project root for license information.

using System;
using System.Linq;
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using MySqlConnector;

namespace Pomelo.EntityFrameworkCore.MySql.Storage.Internal
{
/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
public class MySqlVectorByteTypeMapping : MySqlTypeMapping
{
/// <summary>
/// This API supports the Entity Framework Core infrastructure and is not intended to be used
/// directly from your code. This API may change or be removed in future releases.
/// </summary>
public MySqlVectorByteTypeMapping(
[NotNull] string storeType,
[NotNull] Type clrType,
int? size = null)
: base(
storeType,
clrType,
MySqlDbType.Vector,
size: size,
unicode: false,
storeTypePostfix: StoreTypePostfix.Size,
valueConverter: GetValueConverter(clrType),
valueComparer: GetValueComparer(clrType))
{
}

protected MySqlVectorByteTypeMapping(RelationalTypeMappingParameters parameters, MySqlDbType mySqlDbType)
: base(parameters, mySqlDbType)
{
}

protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters)
=> new MySqlVectorByteTypeMapping(parameters, MySqlDbType);
private static ValueConverter GetValueConverter(Type clrType)
{
return clrType switch
{
Type t when t == typeof(float[]) =>
new ValueConverter<float[], byte[]>(
v => FloatArrayToBytes(v),
v => BytesToFloatArray(v)),

Type t when t == typeof(ReadOnlyMemory<float>) =>
new ValueConverter<ReadOnlyMemory<float>, byte[]>(
v => FloatArrayToBytes(v.ToArray()),
v => new ReadOnlyMemory<float>(BytesToFloatArray(v))),

Type t when t == typeof(Memory<float>) =>
new ValueConverter<Memory<float>, byte[]>(
v => FloatArrayToBytes(v.ToArray()),
v => new Memory<float>(BytesToFloatArray(v))),

_ => throw new InvalidOperationException($"Unsupported CLR type for vector: {clrType}"),
};
}

private static ValueComparer GetValueComparer(Type clrType)
{
return clrType switch
{
Type t when t == typeof(float[]) =>
new ValueComparer<float[]>(
(a, b) => a.SequenceEqual(b),
v => v.Aggregate(0, (hash, x) => HashCode.Combine(hash, x.GetHashCode())),
v => v.ToArray()),

Type t when t == typeof(ReadOnlyMemory<float>) =>
new ValueComparer<ReadOnlyMemory<float>>(
(a, b) => a.ToArray().SequenceEqual(b.ToArray()),
v => v.ToArray().Aggregate(0, (hash, x) => HashCode.Combine(hash, x.GetHashCode())),
v => new ReadOnlyMemory<float>(v.ToArray())),

Type t when t == typeof(Memory<float>) =>
new ValueComparer<Memory<float>>(
(a, b) => a.ToArray().SequenceEqual(b.ToArray()),
v => v.ToArray().Aggregate(0, (hash, x) => HashCode.Combine(hash, x.GetHashCode())),
v => new Memory<float>(v.ToArray())),

_ => throw new InvalidOperationException($"Unsupported CLR type for vector: {clrType}"),
};
}

private static byte[] FloatArrayToBytes(float[] values)
{
var result = new byte[values.Length * sizeof(float)];
Buffer.BlockCopy(values, 0, result, 0, result.Length);
return result;
}

private static float[] BytesToFloatArray(byte[] bytes)
{
var result = new float[bytes.Length / sizeof(float)];
Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length);
return result;
}
}
}
58 changes: 58 additions & 0 deletions src/EFCore.MySql/Storage/Internal/MySqlVectorTypeMapping.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright (c) Pomelo Foundation. All rights reserved.
// Licensed under the MIT. See LICENSE in the project root for license information.

using System;
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using MySqlConnector;

namespace Pomelo.EntityFrameworkCore.MySql.Storage.Internal
{
/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
public class MySqlVectorTypeMapping : MySqlTypeMapping
{
/// <summary>
/// This API supports the Entity Framework Core infrastructure and is not intended to be used
/// directly from your code. This API may change or be removed in future releases.
/// </summary>
public MySqlVectorTypeMapping(
[NotNull] string storeType,
[NotNull] Type clrType,
int? size = null)
: base(
storeType,
clrType,
MySqlDbType.Vector,
size: size,
unicode: false,
storeTypePostfix: StoreTypePostfix.Size,
valueConverter: GetValueConverter(clrType))
{
}

protected MySqlVectorTypeMapping(RelationalTypeMappingParameters parameters, MySqlDbType mySqlDbType)
: base(parameters, mySqlDbType)
{
}

protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters)
=> new MySqlVectorTypeMapping(parameters, MySqlDbType);

private static ValueConverter GetValueConverter(Type clrType)
{
return clrType switch
{
Type t when t == typeof(float[]) => new ValueConverter<float[], ReadOnlyMemory<float>>(v => v.AsMemory(), v => v.ToArray()),
Type t when t == typeof(ReadOnlyMemory<float>) => null,
Type t when t == typeof(Memory<float>) => new ValueConverter<Memory<float>, ReadOnlyMemory<float>>(v => v, v => v.ToArray()),
_ => throw new InvalidOperationException($"Unsupported CLR type for vector: {clrType}"),
};
}
}
}
24 changes: 22 additions & 2 deletions test/EFCore.MySql.IntegrationTests/Models/DataTypes.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Linq;

namespace Pomelo.EntityFrameworkCore.MySql.IntegrationTests.Models
{
Expand Down Expand Up @@ -101,9 +103,20 @@ public class DataTypesVariable

public byte[] TypeByteArrayN { get; set; }

[MaxLength(384)]
[Column(TypeName = "vector")]
public float[] TypeVectorFloatArray { get; set; }

// json not null
[Required]
[MaxLength(384)]
[Column(TypeName = "vector")]
public ReadOnlyMemory<float> TypeVectorReadonlyMemory { get; set; }

[MaxLength(384)]
[Column(TypeName = "vector")]
public Memory<float> TypeVectorMemory { get; set; }

// json not null
[Required]
public List<string> TypeJsonArray { get; set; }

[Required]
Expand All @@ -117,6 +130,10 @@ public class DataTypesVariable

// static method to create a new empty object
public static readonly byte[] EmptyByteArray = Array.Empty<byte>();

// MariaDb requires a fully filled vector to store in the database. Using [0.0, 0.0, ..., 0.0]
// as an empty vector
public static readonly float[] EmptyFloatArray = Enumerable.Repeat(0.0f, 384).ToArray();
public static readonly List<string> EmptyJsonArray = new List<string>();
public static readonly Dictionary<string, string> EmptyJsonObject = new Dictionary<string, string>();

Expand All @@ -128,6 +145,9 @@ public static DataTypesVariable CreateEmpty()
TypeString255 = "",
TypeByteArray = EmptyByteArray,
TypeByteArray255 = EmptyByteArray,
TypeVectorFloatArray = EmptyFloatArray,
TypeVectorMemory = EmptyFloatArray,
TypeVectorReadonlyMemory = EmptyFloatArray,
TypeJsonArray = EmptyJsonArray,
TypeJsonObject = EmptyJsonObject,
};
Expand Down
Loading
Loading