From 77bc63c7580a47571f65ab37407b3be4f827a618 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:00:20 -0800 Subject: [PATCH 01/16] Support VECTOR data type Related to #1549 --- For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/mysql-net/MySqlConnector/issues/1549?shareId=XXXX-XXXX-XXXX-XXXX). --- src/MySqlConnector/Core/ColumnTypeMetadata.cs | 5 +++ src/MySqlConnector/Core/TypeMapper.cs | 8 ++++ src/MySqlConnector/MySqlDataReader.cs | 18 ++++++++- src/MySqlConnector/MySqlDbType.cs | 7 ++++ src/MySqlConnector/MySqlParameter.cs | 19 +++++++++ .../MySqlDataReaderTests.cs | 39 +++++++++++++++++++ .../MySqlParameterTests.cs | 33 ++++++++++++++++ 7 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 tests/MySqlConnector.Tests/MySqlDataReaderTests.cs diff --git a/src/MySqlConnector/Core/ColumnTypeMetadata.cs b/src/MySqlConnector/Core/ColumnTypeMetadata.cs index eadb27de4..cee8c6590 100644 --- a/src/MySqlConnector/Core/ColumnTypeMetadata.cs +++ b/src/MySqlConnector/Core/ColumnTypeMetadata.cs @@ -16,3 +16,8 @@ internal sealed class ColumnTypeMetadata(string dataTypeName, DbTypeMapping dbTy public string CreateLookupKey() => CreateLookupKey(DataTypeName, IsUnsigned, Length); } + +internal static class ColumnTypeMetadataExtensions +{ + public static ColumnTypeMetadata Vector { get; } = new("VECTOR", new DbTypeMapping(typeof(float[]), new[] { DbType.Object }, convert: o => (float[])o), MySqlDbType.Vector, isUnsigned: false, binary: true, length: 0, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})"); +} diff --git a/src/MySqlConnector/Core/TypeMapper.cs b/src/MySqlConnector/Core/TypeMapper.cs index 6397f9182..f07922e5b 100644 --- a/src/MySqlConnector/Core/TypeMapper.cs +++ b/src/MySqlConnector/Core/TypeMapper.cs @@ -55,6 +55,10 @@ private TypeMapper() AddColumnTypeMetadata(new("DOUBLE", typeDouble, MySqlDbType.Double)); AddColumnTypeMetadata(new("FLOAT", typeFloat, MySqlDbType.Float)); + // vector + var typeFloatArray = AddDbTypeMapping(new(typeof(float[]), [DbType.Object], convert: static o => (float[])o)); + AddColumnTypeMetadata(new("VECTOR", typeFloatArray, MySqlDbType.Vector, binary: true, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})")); + // string var typeFixedString = AddDbTypeMapping(new(typeof(string), [DbType.StringFixedLength, DbType.AnsiStringFixedLength], convert: Convert.ToString!)); var typeString = AddDbTypeMapping(new(typeof(string), [DbType.String, DbType.AnsiString, DbType.Xml], convert: Convert.ToString!)); @@ -303,6 +307,9 @@ public static MySqlDbType ConvertToMySqlDbType(ColumnDefinitionPayload columnDef case ColumnType.Set: return MySqlDbType.Set; + case ColumnType.Vector: + return MySqlDbType.Vector; + default: throw new NotImplementedException($"ConvertToMySqlDbType for {columnDefinition.ColumnType} is not implemented"); } @@ -339,6 +346,7 @@ public static ushort ConvertToColumnTypeAndFlags(MySqlDbType dbType, MySqlGuidFo MySqlDbType.NewDecimal => ColumnType.NewDecimal, MySqlDbType.Geometry => ColumnType.Geometry, MySqlDbType.Null => ColumnType.Null, + MySqlDbType.Vector => ColumnType.Vector, _ => throw new NotImplementedException($"ConvertToColumnTypeAndFlags for {dbType} is not implemented"), }; return (ushort) ((byte) columnType | (isUnsigned ? 0x8000 : 0)); diff --git a/src/MySqlConnector/MySqlDataReader.cs b/src/MySqlConnector/MySqlDataReader.cs index 800fafe47..639bf839f 100644 --- a/src/MySqlConnector/MySqlDataReader.cs +++ b/src/MySqlConnector/MySqlDataReader.cs @@ -252,7 +252,15 @@ public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int #endif public override Type GetFieldType(int ordinal) => GetResultSet().GetFieldType(ordinal); - public override object GetValue(int ordinal) => GetResultSet().GetCurrentRow().GetValue(ordinal); + public override object GetValue(int ordinal) + { + var resultSet = GetResultSet(); + if (resultSet.GetDataTypeName(ordinal) == "VECTOR") + { + return resultSet.GetCurrentRow().GetValue(ordinal); + } + return resultSet.GetCurrentRow().GetValue(ordinal); + } public override IEnumerator GetEnumerator() => new DbEnumerator(this, closeReader: false); @@ -428,6 +436,14 @@ public override T GetFieldValue(int ordinal) return (T) (object) GetDateOnly(ordinal); if (typeof(T) == typeof(TimeOnly)) return (T) (object) GetTimeOnly(ordinal); + if (typeof(T) == typeof(ReadOnlySpan)) + { + var value = GetValue(ordinal); + if (value is float[] floatArray) + { + return (T) (object) new ReadOnlySpan(floatArray); + } + } #endif return base.GetFieldValue(ordinal); diff --git a/src/MySqlConnector/MySqlDbType.cs b/src/MySqlConnector/MySqlDbType.cs index 77f990383..438473644 100644 --- a/src/MySqlConnector/MySqlDbType.cs +++ b/src/MySqlConnector/MySqlDbType.cs @@ -2,6 +2,9 @@ namespace MySqlConnector; #pragma warning disable CA1720 // Identifier contains type name +/// +/// Specifies the MySQL data type of a field, property, for use in a . +/// public enum MySqlDbType { Bool = -1, @@ -37,6 +40,10 @@ public enum MySqlDbType VarChar, String, Geometry, + /// + /// A MySQL VECTOR data type. + /// + Vector = 242, UByte = 501, UInt16, UInt32, diff --git a/src/MySqlConnector/MySqlParameter.cs b/src/MySqlConnector/MySqlParameter.cs index 6d493cba8..e24e4d766 100644 --- a/src/MySqlConnector/MySqlParameter.cs +++ b/src/MySqlConnector/MySqlParameter.cs @@ -554,6 +554,17 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions { writer.WriteString((ulong) Value); } + else if (Value is float[] floatArrayValue) + { + writer.Write((byte) '['); + for (int i = 0; i < floatArrayValue.Length; i++) + { + if (i > 0) + writer.Write((byte) ','); + writer.WriteString(floatArrayValue[i]); + } + writer.Write((byte) ']'); + } else { throw new NotSupportedException($"Parameter type {Value.GetType().Name} is not supported; see https://mysqlconnector.net/param-type. Value: {Value}"); @@ -871,6 +882,14 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar { writer.Write((ulong) value); } + else if (value is float[] floatArrayValue) + { + writer.WriteLengthEncodedInteger((ulong) (floatArrayValue.Length * 4)); + foreach (var floatValue in floatArrayValue) + { + writer.Write(BitConverter.GetBytes(floatValue)); + } + } else { throw new NotSupportedException($"Parameter type {value.GetType().Name} is not supported; see https://mysqlconnector.net/param-type. Value: {value}"); diff --git a/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs b/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs new file mode 100644 index 000000000..e5e5cd6df --- /dev/null +++ b/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs @@ -0,0 +1,39 @@ +using System; +using System.Data; +using MySqlConnector; +using Xunit; + +namespace MySqlConnector.Tests +{ + public class MySqlDataReaderTests + { + [Fact] + public void GetVectorDataType() + { + using var connection = new MySqlConnection("your_connection_string"); + connection.Open(); + + using var command = new MySqlCommand("SELECT CAST('[1.0, 2.0, 3.0]' AS VECTOR)", connection); + using var reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + var vector = reader.GetValue(0) as float[]; + Assert.NotNull(vector); + Assert.Equal(new float[] { 1.0f, 2.0f, 3.0f }, vector); + } + + [Fact] + public void GetReadOnlySpanFloat() + { + using var connection = new MySqlConnection("your_connection_string"); + connection.Open(); + + using var command = new MySqlCommand("SELECT CAST('[1.0, 2.0, 3.0]' AS VECTOR)", connection); + using var reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + var span = reader.GetFieldValue>(0); + Assert.Equal(new float[] { 1.0f, 2.0f, 3.0f }, span.ToArray()); + } + } +} diff --git a/tests/MySqlConnector.Tests/MySqlParameterTests.cs b/tests/MySqlConnector.Tests/MySqlParameterTests.cs index 710c699cf..ee60ec2ae 100644 --- a/tests/MySqlConnector.Tests/MySqlParameterTests.cs +++ b/tests/MySqlConnector.Tests/MySqlParameterTests.cs @@ -423,4 +423,37 @@ public void ScaleMixed() ((IDbDataParameter) parameter).Scale = 12; Assert.Equal((byte) 12, ((MySqlParameter) parameter).Scale); } + + [Fact] + public void SetValueToFloatArrayInfersType() + { + var parameter = new MySqlParameter { Value = new float[] { 1.0f, 2.0f, 3.0f } }; + Assert.Equal(DbType.Object, parameter.DbType); + Assert.Equal(MySqlDbType.Vector, parameter.MySqlDbType); + } + + [Fact] + public void ConstructorNameTypeVector() + { + var parameter = new MySqlParameter("@vector", MySqlDbType.Vector); + Assert.Equal("@vector", parameter.ParameterName); + Assert.Equal(MySqlDbType.Vector, parameter.MySqlDbType); + Assert.Equal(DbType.Object, parameter.DbType); + Assert.False(parameter.IsNullable); + Assert.Null(parameter.Value); + Assert.Equal(ParameterDirection.Input, parameter.Direction); + Assert.Equal(0, parameter.Precision); + Assert.Equal(0, parameter.Scale); + Assert.Equal(0, parameter.Size); +#if MYSQL_DATA + Assert.Equal(DataRowVersion.Default, parameter.SourceVersion); +#else + Assert.Equal(DataRowVersion.Current, parameter.SourceVersion); +#endif +#if MYSQL_DATA + Assert.Null(parameter.SourceColumn); +#else + Assert.Equal("", parameter.SourceColumn); +#endif + } } From 2ec0975779bc8d6dc28a383c4fb544a534003be8 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:02:20 -0800 Subject: [PATCH 02/16] Delete unnecessary class. Signed-off-by: Bradley Grainger --- src/MySqlConnector/Core/ColumnTypeMetadata.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/MySqlConnector/Core/ColumnTypeMetadata.cs b/src/MySqlConnector/Core/ColumnTypeMetadata.cs index cee8c6590..eadb27de4 100644 --- a/src/MySqlConnector/Core/ColumnTypeMetadata.cs +++ b/src/MySqlConnector/Core/ColumnTypeMetadata.cs @@ -16,8 +16,3 @@ internal sealed class ColumnTypeMetadata(string dataTypeName, DbTypeMapping dbTy public string CreateLookupKey() => CreateLookupKey(DataTypeName, IsUnsigned, Length); } - -internal static class ColumnTypeMetadataExtensions -{ - public static ColumnTypeMetadata Vector { get; } = new("VECTOR", new DbTypeMapping(typeof(float[]), new[] { DbType.Object }, convert: o => (float[])o), MySqlDbType.Vector, isUnsigned: false, binary: true, length: 0, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})"); -} From 6c3e8051bd7eb547ba2c92a7706fdc11e40e8d49 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:02:56 -0800 Subject: [PATCH 03/16] Revert bad change. Signed-off-by: Bradley Grainger --- src/MySqlConnector/MySqlDataReader.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/MySqlConnector/MySqlDataReader.cs b/src/MySqlConnector/MySqlDataReader.cs index 639bf839f..46946150a 100644 --- a/src/MySqlConnector/MySqlDataReader.cs +++ b/src/MySqlConnector/MySqlDataReader.cs @@ -252,15 +252,7 @@ public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int #endif public override Type GetFieldType(int ordinal) => GetResultSet().GetFieldType(ordinal); - public override object GetValue(int ordinal) - { - var resultSet = GetResultSet(); - if (resultSet.GetDataTypeName(ordinal) == "VECTOR") - { - return resultSet.GetCurrentRow().GetValue(ordinal); - } - return resultSet.GetCurrentRow().GetValue(ordinal); - } + public override object GetValue(int ordinal) => GetResultSet().GetCurrentRow().GetValue(ordinal); public override IEnumerator GetEnumerator() => new DbEnumerator(this, closeReader: false); From c0f22bf680b5be00317801a06f782ee0f4f4e4d8 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:03:55 -0800 Subject: [PATCH 04/16] Delete test in wrong project. Signed-off-by: Bradley Grainger --- .../MySqlDataReaderTests.cs | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 tests/MySqlConnector.Tests/MySqlDataReaderTests.cs diff --git a/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs b/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs deleted file mode 100644 index e5e5cd6df..000000000 --- a/tests/MySqlConnector.Tests/MySqlDataReaderTests.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Data; -using MySqlConnector; -using Xunit; - -namespace MySqlConnector.Tests -{ - public class MySqlDataReaderTests - { - [Fact] - public void GetVectorDataType() - { - using var connection = new MySqlConnection("your_connection_string"); - connection.Open(); - - using var command = new MySqlCommand("SELECT CAST('[1.0, 2.0, 3.0]' AS VECTOR)", connection); - using var reader = command.ExecuteReader(); - - Assert.True(reader.Read()); - var vector = reader.GetValue(0) as float[]; - Assert.NotNull(vector); - Assert.Equal(new float[] { 1.0f, 2.0f, 3.0f }, vector); - } - - [Fact] - public void GetReadOnlySpanFloat() - { - using var connection = new MySqlConnection("your_connection_string"); - connection.Open(); - - using var command = new MySqlCommand("SELECT CAST('[1.0, 2.0, 3.0]' AS VECTOR)", connection); - using var reader = command.ExecuteReader(); - - Assert.True(reader.Read()); - var span = reader.GetFieldValue>(0); - Assert.Equal(new float[] { 1.0f, 2.0f, 3.0f }, span.ToArray()); - } - } -} From 52b2736c8c8f5a8db9ff7a597e139795cdf645cc Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:09:13 -0800 Subject: [PATCH 05/16] Delete comments for only one enum value. Signed-off-by: Bradley Grainger --- src/MySqlConnector/MySqlDbType.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/MySqlConnector/MySqlDbType.cs b/src/MySqlConnector/MySqlDbType.cs index 438473644..f1d0708a0 100644 --- a/src/MySqlConnector/MySqlDbType.cs +++ b/src/MySqlConnector/MySqlDbType.cs @@ -2,9 +2,6 @@ namespace MySqlConnector; #pragma warning disable CA1720 // Identifier contains type name -/// -/// Specifies the MySQL data type of a field, property, for use in a . -/// public enum MySqlDbType { Bool = -1, @@ -40,9 +37,6 @@ public enum MySqlDbType VarChar, String, Geometry, - /// - /// A MySQL VECTOR data type. - /// Vector = 242, UByte = 501, UInt16, From c3b65a50498c23cd2241580be37dd463b032ff13 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 13:22:53 -0800 Subject: [PATCH 06/16] Clean up broken AI code. Signed-off-by: Bradley Grainger --- src/MySqlConnector/Core/TypeMapper.cs | 2 +- src/MySqlConnector/MySqlDataReader.cs | 8 ----- src/MySqlConnector/MySqlParameter.cs | 28 +++++----------- src/MySqlConnector/Protocol/ColumnType.cs | 1 + .../MySqlParameterTests.cs | 33 ------------------- 5 files changed, 10 insertions(+), 62 deletions(-) diff --git a/src/MySqlConnector/Core/TypeMapper.cs b/src/MySqlConnector/Core/TypeMapper.cs index f07922e5b..1058a99e1 100644 --- a/src/MySqlConnector/Core/TypeMapper.cs +++ b/src/MySqlConnector/Core/TypeMapper.cs @@ -56,7 +56,7 @@ private TypeMapper() AddColumnTypeMetadata(new("FLOAT", typeFloat, MySqlDbType.Float)); // vector - var typeFloatArray = AddDbTypeMapping(new(typeof(float[]), [DbType.Object], convert: static o => (float[])o)); + var typeFloatArray = AddDbTypeMapping(new(typeof(float[]), [DbType.Object])); AddColumnTypeMetadata(new("VECTOR", typeFloatArray, MySqlDbType.Vector, binary: true, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})")); // string diff --git a/src/MySqlConnector/MySqlDataReader.cs b/src/MySqlConnector/MySqlDataReader.cs index 46946150a..800fafe47 100644 --- a/src/MySqlConnector/MySqlDataReader.cs +++ b/src/MySqlConnector/MySqlDataReader.cs @@ -428,14 +428,6 @@ public override T GetFieldValue(int ordinal) return (T) (object) GetDateOnly(ordinal); if (typeof(T) == typeof(TimeOnly)) return (T) (object) GetTimeOnly(ordinal); - if (typeof(T) == typeof(ReadOnlySpan)) - { - var value = GetValue(ordinal); - if (value is float[] floatArray) - { - return (T) (object) new ReadOnlySpan(floatArray); - } - } #endif return base.GetFieldValue(ordinal); diff --git a/src/MySqlConnector/MySqlParameter.cs b/src/MySqlConnector/MySqlParameter.cs index e24e4d766..438ceb49e 100644 --- a/src/MySqlConnector/MySqlParameter.cs +++ b/src/MySqlConnector/MySqlParameter.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Numerics; +using System.Runtime.InteropServices; using System.Text; #if NET8_0_OR_GREATER using System.Text.Unicode; @@ -282,7 +283,7 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions { writer.WriteString(ulongValue); } - else if (Value is byte[] or ReadOnlyMemory or Memory or ArraySegment or MySqlGeometry or MemoryStream) + else if (Value is byte[] or ReadOnlyMemory or Memory or ArraySegment or MySqlGeometry or MemoryStream or float[]) { var inputSpan = Value switch { @@ -291,6 +292,7 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions Memory memory => memory.Span, MySqlGeometry geometry => geometry.ValueSpan, MemoryStream memoryStream => memoryStream.TryGetBuffer(out var streamBuffer) ? streamBuffer.AsSpan() : memoryStream.ToArray().AsSpan(), + float[] floatArray => MemoryMarshal.AsBytes(floatArray.AsSpan()), _ => ((ReadOnlyMemory) Value).Span, }; @@ -554,17 +556,6 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions { writer.WriteString((ulong) Value); } - else if (Value is float[] floatArrayValue) - { - writer.Write((byte) '['); - for (int i = 0; i < floatArrayValue.Length; i++) - { - if (i > 0) - writer.Write((byte) ','); - writer.WriteString(floatArrayValue[i]); - } - writer.Write((byte) ']'); - } else { throw new NotSupportedException($"Parameter type {Value.GetType().Name} is not supported; see https://mysqlconnector.net/param-type. Value: {Value}"); @@ -740,6 +731,11 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar { writer.Write(unchecked((ulong) BitConverter.DoubleToInt64Bits(doubleValue))); } + else if (value is float[] floatArrayValue) + { + writer.WriteLengthEncodedInteger(unchecked((ulong) floatArrayValue.Length * 4)); + writer.Write(MemoryMarshal.AsBytes(floatArrayValue.AsSpan())); + } else if (value is decimal decimalValue) { writer.WriteLengthEncodedAsciiString(decimalValue.ToString(CultureInfo.InvariantCulture)); @@ -882,14 +878,6 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar { writer.Write((ulong) value); } - else if (value is float[] floatArrayValue) - { - writer.WriteLengthEncodedInteger((ulong) (floatArrayValue.Length * 4)); - foreach (var floatValue in floatArrayValue) - { - writer.Write(BitConverter.GetBytes(floatValue)); - } - } else { throw new NotSupportedException($"Parameter type {value.GetType().Name} is not supported; see https://mysqlconnector.net/param-type. Value: {value}"); diff --git a/src/MySqlConnector/Protocol/ColumnType.cs b/src/MySqlConnector/Protocol/ColumnType.cs index 4b3701106..b031a41a5 100644 --- a/src/MySqlConnector/Protocol/ColumnType.cs +++ b/src/MySqlConnector/Protocol/ColumnType.cs @@ -24,6 +24,7 @@ internal enum ColumnType Bit = 16, Timestamp2 = 17, DateTime2 = 18, + Vector = 242, Json = 0xF5, NewDecimal = 0xF6, Enum = 0xF7, diff --git a/tests/MySqlConnector.Tests/MySqlParameterTests.cs b/tests/MySqlConnector.Tests/MySqlParameterTests.cs index ee60ec2ae..710c699cf 100644 --- a/tests/MySqlConnector.Tests/MySqlParameterTests.cs +++ b/tests/MySqlConnector.Tests/MySqlParameterTests.cs @@ -423,37 +423,4 @@ public void ScaleMixed() ((IDbDataParameter) parameter).Scale = 12; Assert.Equal((byte) 12, ((MySqlParameter) parameter).Scale); } - - [Fact] - public void SetValueToFloatArrayInfersType() - { - var parameter = new MySqlParameter { Value = new float[] { 1.0f, 2.0f, 3.0f } }; - Assert.Equal(DbType.Object, parameter.DbType); - Assert.Equal(MySqlDbType.Vector, parameter.MySqlDbType); - } - - [Fact] - public void ConstructorNameTypeVector() - { - var parameter = new MySqlParameter("@vector", MySqlDbType.Vector); - Assert.Equal("@vector", parameter.ParameterName); - Assert.Equal(MySqlDbType.Vector, parameter.MySqlDbType); - Assert.Equal(DbType.Object, parameter.DbType); - Assert.False(parameter.IsNullable); - Assert.Null(parameter.Value); - Assert.Equal(ParameterDirection.Input, parameter.Direction); - Assert.Equal(0, parameter.Precision); - Assert.Equal(0, parameter.Scale); - Assert.Equal(0, parameter.Size); -#if MYSQL_DATA - Assert.Equal(DataRowVersion.Default, parameter.SourceVersion); -#else - Assert.Equal(DataRowVersion.Current, parameter.SourceVersion); -#endif -#if MYSQL_DATA - Assert.Null(parameter.SourceColumn); -#else - Assert.Equal("", parameter.SourceColumn); -#endif - } } From 6d73d95bf7842e9d7204f3ebacb4ac1720254ca0 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 15:03:37 -0800 Subject: [PATCH 07/16] Add ServerFeatures.Vector and simple test. Signed-off-by: Bradley Grainger --- azure-pipelines.yml | 18 +++++++++--------- .../ColumnReaders/ColumnReader.cs | 3 +++ .../ColumnReaders/VectorColumnReader.cs | 12 ++++++++++++ src/MySqlConnector/Core/Row.cs | 2 +- src/MySqlConnector/packages.lock.json | 14 -------------- tests/IntegrationTests/DataTypes.cs | 12 ++++++++++++ tests/IntegrationTests/DataTypesFixture.cs | 18 ++++++++++++++++++ tests/IntegrationTests/ServerFeatures.cs | 5 +++++ 8 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 src/MySqlConnector/ColumnReaders/VectorColumnReader.cs diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fa4d3d285..811c77f61 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -51,7 +51,7 @@ jobs: arguments: 'tests\IntegrationTests\IntegrationTests.csproj -c MySqlData' testRunTitle: 'MySql.Data integration tests' env: - DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,StreamingResults,TlsFingerprintValidation,UnixDomainSocket' + DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,StreamingResults,TlsFingerprintValidation,UnixDomainSocket,Vector' DATA__CONNECTIONSTRING: 'server=localhost;port=3306;user id=root;password=test;database=mysqltest;ssl mode=none;DefaultCommandTimeout=3600' DATA__CERTIFICATESPATH: '$(Build.Repository.LocalPath)\.ci\server\certs\' DATA__MYSQLBULKLOADERLOCALCSVFILE: '$(Build.Repository.LocalPath)\tests\TestData\LoadData_UTF8_BOM_Unix.CSV' @@ -120,7 +120,7 @@ jobs: arguments: '-c Release --no-restore -p:TestTfmsInParallel=false' testRunTitle: ${{ format('{0}, $(Agent.OS), {1}, {2}', 'mysql:8.0', 'net481/net9.0', 'No SSL') }} env: - DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket' + DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,Vector' DATA__CONNECTIONSTRING: 'server=localhost;port=3306;user id=mysqltest;password=test;database=mysqltest;ssl mode=none;DefaultCommandTimeout=3600;AllowPublicKeyRetrieval=True;UseCompression=True' - job: windows_integration_tests_2 @@ -158,7 +158,7 @@ jobs: arguments: '-c Release --no-restore -p:TestTfmsInParallel=false' testRunTitle: ${{ format('{0}, $(Agent.OS), {1}, {2}', 'mysql:8.0', 'net8.0', 'No SSL') }} env: - DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket' + DATA__UNSUPPORTEDFEATURES: 'Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,Vector' DATA__CONNECTIONSTRING: 'server=localhost;port=3306;user id=mysqltest;password=test;database=mysqltest;ssl mode=none;DefaultCommandTimeout=3600;AllowPublicKeyRetrieval=True' - job: linux_integration_tests @@ -171,11 +171,11 @@ jobs: 'MySQL 8.0': image: 'mysql:8.0' connectionStringExtra: 'AllowPublicKeyRetrieval=True' - unsupportedFeatures: 'Ed25519,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,ZeroDateTime' + unsupportedFeatures: 'Ed25519,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,Vector,ZeroDateTime' 'MySQL 8.4': image: 'mysql:8.4' connectionStringExtra: 'AllowPublicKeyRetrieval=True' - unsupportedFeatures: 'Ed25519,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,ZeroDateTime' + unsupportedFeatures: 'Ed25519,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,Vector,ZeroDateTime' 'MySQL 9.2': image: 'mysql:9.2' connectionStringExtra: 'AllowPublicKeyRetrieval=True' @@ -183,19 +183,19 @@ jobs: 'MariaDB 10.6': image: 'mariadb:10.6' connectionStringExtra: '' - unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,TlsFingerprintValidation,UuidToBin' + unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,TlsFingerprintValidation,UuidToBin,Vector' 'MariaDB 10.11': image: 'mariadb:10.11' connectionStringExtra: '' - unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,TlsFingerprintValidation,UuidToBin' + unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,TlsFingerprintValidation,UuidToBin,Vector' 'MariaDB 11.4': image: 'mariadb:11.4' connectionStringExtra: '' - unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Sha256Password,Tls11,UuidToBin,Redirection' + unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin,Vector' 'MariaDB 11.6': image: 'mariadb:11.6' connectionStringExtra: '' - unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin' + unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin,Vector' steps: - template: '.ci/integration-tests-steps.yml' parameters: diff --git a/src/MySqlConnector/ColumnReaders/ColumnReader.cs b/src/MySqlConnector/ColumnReaders/ColumnReader.cs index 86ba9145f..24baba810 100644 --- a/src/MySqlConnector/ColumnReaders/ColumnReader.cs +++ b/src/MySqlConnector/ColumnReaders/ColumnReader.cs @@ -113,6 +113,9 @@ public static ColumnReader Create(bool isBinary, ColumnDefinitionPayload columnD case ColumnType.Null: return NullColumnReader.Instance; + case ColumnType.Vector: + return VectorColumnReader.Instance; + default: throw new NotImplementedException($"Reading {columnDefinition.ColumnType} not implemented"); } diff --git a/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs new file mode 100644 index 000000000..637b0f491 --- /dev/null +++ b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs @@ -0,0 +1,12 @@ +using System.Runtime.InteropServices; +using MySqlConnector.Protocol.Payloads; + +namespace MySqlConnector.ColumnReaders; + +internal sealed class VectorColumnReader : ColumnReader +{ + public static VectorColumnReader Instance { get; } = new(); + + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) => + MemoryMarshal.Cast(data).ToArray(); +} diff --git a/src/MySqlConnector/Core/Row.cs b/src/MySqlConnector/Core/Row.cs index bdbc81658..a9f424c9d 100644 --- a/src/MySqlConnector/Core/Row.cs +++ b/src/MySqlConnector/Core/Row.cs @@ -455,7 +455,7 @@ private void CheckBinaryColumn(int ordinal) if ((column.ColumnFlags & ColumnFlags.Binary) == 0 || (columnType != ColumnType.String && columnType != ColumnType.VarString && columnType != ColumnType.TinyBlob && columnType != ColumnType.Blob && columnType != ColumnType.MediumBlob && columnType != ColumnType.LongBlob && - columnType != ColumnType.Geometry)) + columnType != ColumnType.Geometry && columnType != ColumnType.Vector)) { throw new InvalidCastException($"Can't convert {columnType} to bytes."); } diff --git a/src/MySqlConnector/packages.lock.json b/src/MySqlConnector/packages.lock.json index 3fc5d7c53..3b22aa928 100644 --- a/src/MySqlConnector/packages.lock.json +++ b/src/MySqlConnector/packages.lock.json @@ -300,15 +300,6 @@ "System.Memory": "4.5.5" } }, - "Microsoft.NETFramework.ReferenceAssemblies": { - "type": "Direct", - "requested": "[1.0.3, )", - "resolved": "1.0.3", - "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==", - "dependencies": { - "Microsoft.NETFramework.ReferenceAssemblies.net48": "1.0.3" - } - }, "Microsoft.SourceLink.GitHub": { "type": "Direct", "requested": "[8.0.0, )", @@ -366,11 +357,6 @@ "resolved": "8.0.0", "contentHash": "bZKfSIKJRXLTuSzLudMFte/8CempWjVamNUR5eHJizsy+iuOuO/k2gnh7W0dHJmYY0tBf+gUErfluCv5mySAOQ==" }, - "Microsoft.NETFramework.ReferenceAssemblies.net48": { - "type": "Transitive", - "resolved": "1.0.3", - "contentHash": "zMk4D+9zyiEWByyQ7oPImPN/Jhpj166Ky0Nlla4eXlNL8hI/BtSJsgR8Inldd4NNpIAH3oh8yym0W2DrhXdSLQ==" - }, "Microsoft.SourceLink.Common": { "type": "Transitive", "resolved": "8.0.0", diff --git a/tests/IntegrationTests/DataTypes.cs b/tests/IntegrationTests/DataTypes.cs index b03650cbd..3f1ee3a51 100644 --- a/tests/IntegrationTests/DataTypes.cs +++ b/tests/IntegrationTests/DataTypes.cs @@ -1599,6 +1599,18 @@ public void QueryJson(string column, string[] expected) DoQuery("json_core", column, dataTypeName, expected, reader => reader.GetString(0), omitWhereTest: true); } +#if !MYSQL_DATA + [SkippableTheory(ServerFeatures.Vector)] + [InlineData("value", new[] { null, "0,0,0", "1,1,1", "1,2,3", "3.40282347E+38,3.40282347E+38,3.40282347E+38" })] + public void QueryVector(string column, string[] expected) + { + string dataTypeName = "VECTOR"; + DoQuery("vector", column, dataTypeName, + expected.Select(x => x?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()).ToArray(), + static x => (float[]) x.GetValue(0), omitWhereTest: true); + } +#endif + [SkippableTheory(MySqlData = "https://bugs.mysql.com/bug.php?id=97067")] [InlineData(false, "MIN", 0)] [InlineData(false, "MAX", uint.MaxValue)] diff --git a/tests/IntegrationTests/DataTypesFixture.cs b/tests/IntegrationTests/DataTypesFixture.cs index f9ad28a38..1dbba6675 100644 --- a/tests/IntegrationTests/DataTypesFixture.cs +++ b/tests/IntegrationTests/DataTypesFixture.cs @@ -242,6 +242,24 @@ insert into datatypes_json_core (value) ('{""a"": ""b""}'); "); } + + if (AppConfig.SupportedFeatures.HasFlag(ServerFeatures.Vector)) + { + Connection.Execute(""" + drop table if exists datatypes_vector; + create table datatypes_vector ( + rowid integer not null primary key auto_increment, + value vector(3) null + ); + insert into datatypes_vector (value) + values + (null), + (STRING_TO_VECTOR('[0, 0, 0]')), + (STRING_TO_VECTOR('[1, 1, 1]')), + (STRING_TO_VECTOR('[1, 2, 3]')), + (STRING_TO_VECTOR('[3.40282347E+38, 3.40282347E+38, 3.40282347E+38]')); + """); + } Connection.Close(); } } diff --git a/tests/IntegrationTests/ServerFeatures.cs b/tests/IntegrationTests/ServerFeatures.cs index 120b541bf..c1f12591b 100644 --- a/tests/IntegrationTests/ServerFeatures.cs +++ b/tests/IntegrationTests/ServerFeatures.cs @@ -45,4 +45,9 @@ public enum ServerFeatures /// Server provides hash of TLS certificate in first OK packet. /// TlsFingerprintValidation = 0x100_0000, + + /// + /// Server supports the VECTOR data type. + /// + Vector = 0x200_0000, } From 6af96e49bbf4ca186ef98012f28525ac81fbffaf Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 15:35:43 -0800 Subject: [PATCH 08/16] Fix Appveyor build. Signed-off-by: Bradley Grainger --- .ci/config/config.compression+ssl.json | 2 +- .ci/config/config.compression.json | 2 +- .ci/config/config.json | 2 +- .ci/config/config.ssl.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/config/config.compression+ssl.json b/.ci/config/config.compression+ssl.json index f7d4263c8..e0bf3504d 100644 --- a/.ci/config/config.compression+ssl.json +++ b/.ci/config/config.compression+ssl.json @@ -4,7 +4,7 @@ "SocketPath": "./../../../../.ci/run/mysql/mysqld.sock", "PasswordlessUser": "no_password", "SecondaryDatabase": "testdb2", - "UnsupportedFeatures": "CachingSha2Password,Redirection,RsaEncryption,Tls12,Tls13,TlsFingerprintValidation,UuidToBin", + "UnsupportedFeatures": "CachingSha2Password,Redirection,RsaEncryption,Tls12,Tls13,TlsFingerprintValidation,UuidToBin,Vector", "MySqlBulkLoaderLocalCsvFile": "../../../TestData/LoadData_UTF8_BOM_Unix.CSV", "MySqlBulkLoaderLocalTsvFile": "../../../TestData/LoadData_UTF8_BOM_Unix.TSV", "CertificatesPath": "../../../../.ci/server/certs" diff --git a/.ci/config/config.compression.json b/.ci/config/config.compression.json index 09326f1f6..d6206ed1d 100644 --- a/.ci/config/config.compression.json +++ b/.ci/config/config.compression.json @@ -4,7 +4,7 @@ "SocketPath": "./../../../../.ci/run/mysql/mysqld.sock", "PasswordlessUser": "no_password", "SecondaryDatabase": "testdb2", - "UnsupportedFeatures": "Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,ZeroDateTime", + "UnsupportedFeatures": "Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,Vector,ZeroDateTime", "MySqlBulkLoaderLocalCsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.CSV", "MySqlBulkLoaderLocalTsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.TSV" } diff --git a/.ci/config/config.json b/.ci/config/config.json index bc38f605a..b539060a9 100644 --- a/.ci/config/config.json +++ b/.ci/config/config.json @@ -4,7 +4,7 @@ "SocketPath": "./../../../../.ci/run/mysql/mysqld.sock", "PasswordlessUser": "no_password", "SecondaryDatabase": "testdb2", - "UnsupportedFeatures": "Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,ZeroDateTime", + "UnsupportedFeatures": "Ed25519,QueryAttributes,Redirection,StreamingResults,Tls11,TlsFingerprintValidation,UnixDomainSocket,Vector,ZeroDateTime", "MySqlBulkLoaderLocalCsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.CSV", "MySqlBulkLoaderLocalTsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.TSV" } diff --git a/.ci/config/config.ssl.json b/.ci/config/config.ssl.json index a7511faa4..e136a243d 100644 --- a/.ci/config/config.ssl.json +++ b/.ci/config/config.ssl.json @@ -4,7 +4,7 @@ "SocketPath": "./../../../../.ci/run/mysql/mysqld.sock", "PasswordlessUser": "no_password", "SecondaryDatabase": "testdb2", - "UnsupportedFeatures": "CachingSha2Password,Redirection,RsaEncryption,Tls12,Tls13,TlsFingerprintValidation,UuidToBin", + "UnsupportedFeatures": "CachingSha2Password,Redirection,RsaEncryption,Tls12,Tls13,TlsFingerprintValidation,UuidToBin,Vector", "MySqlBulkLoaderLocalCsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.CSV", "MySqlBulkLoaderLocalTsvFile": "../../../../tests/TestData/LoadData_UTF8_BOM_Unix.TSV", "CertificatesPath": "../../../../.ci/server/certs" From 6f6210efc2d5cd51ecb8d6fadf8de1ed414a5ec0 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 15:44:48 -0800 Subject: [PATCH 09/16] Test column schema for VECTOR. Signed-off-by: Bradley Grainger --- src/MySqlConnector/MySqlDbColumn.cs | 1 + tests/IntegrationTests/DataTypes.cs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/MySqlConnector/MySqlDbColumn.cs b/src/MySqlConnector/MySqlDbColumn.cs index b55e9a02a..9696951ed 100644 --- a/src/MySqlConnector/MySqlDbColumn.cs +++ b/src/MySqlConnector/MySqlDbColumn.cs @@ -15,6 +15,7 @@ internal MySqlDbColumn(int ordinal, ColumnDefinitionPayload column, bool allowZe var type = columnTypeMetadata.DbTypeMapping.ClrType; var columnSize = type == typeof(string) || type == typeof(Guid) ? column.ColumnLength / ProtocolUtility.GetBytesPerCharacter(column.CharacterSet) : + column.ColumnType == ColumnType.Vector ? column.ColumnLength / 4 : column.ColumnLength; AllowDBNull = (column.ColumnFlags & ColumnFlags.NotNull) == 0; diff --git a/tests/IntegrationTests/DataTypes.cs b/tests/IntegrationTests/DataTypes.cs index 3f1ee3a51..bf4bd0e66 100644 --- a/tests/IntegrationTests/DataTypes.cs +++ b/tests/IntegrationTests/DataTypes.cs @@ -1143,6 +1143,9 @@ private static object CreateGeometry(byte[] data) [InlineData("Int64", "datatypes_integers", MySqlDbType.Int64, 20, typeof(long), "N", 0, 0)] [InlineData("UInt64", "datatypes_integers", MySqlDbType.UInt64, 20, typeof(ulong), "N", 0, 0)] [InlineData("value", "datatypes_json_core", MySqlDbType.JSON, int.MaxValue, typeof(string), "LN", 0, 0)] +#if !MYSQL_DATA + [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 3, typeof(float[]), "N", 0, 31)] +#endif [InlineData("Single", "datatypes_reals", MySqlDbType.Float, 12, typeof(float), "N", 0, 31)] [InlineData("Double", "datatypes_reals", MySqlDbType.Double, 22, typeof(double), "N", 0, 31)] [InlineData("SmallDecimal", "datatypes_reals", MySqlDbType.NewDecimal, 7, typeof(decimal), "N", 5, 2)] @@ -1195,6 +1198,8 @@ private void DoGetSchemaTable(string column, string table, MySqlDbType mySqlDbTy { if (table == "datatypes_json_core" && !AppConfig.SupportsJson) return; + if (table == "datatypes_vector" && !AppConfig.SupportedFeatures.HasFlag(ServerFeatures.Vector)) + return; var isAutoIncrement = flags.IndexOf('A') != -1; var isKey = flags.IndexOf('K') != -1; From 4ff262085f04e11116dc6b9c5dd6066a55dbfc0f Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 16:32:11 -0800 Subject: [PATCH 10/16] Add more tests and test with MySql.Data. MySql.Data 8.4.0 already supports the VECTOR data type. Signed-off-by: Bradley Grainger --- tests/IntegrationTests/DataTypes.cs | 25 +++++++-- tests/IntegrationTests/QueryTests.cs | 52 +++++++++++++++++++ .../IntegrationTests/StoredProcedureTests.cs | 38 ++++++++++++++ 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/tests/IntegrationTests/DataTypes.cs b/tests/IntegrationTests/DataTypes.cs index bf4bd0e66..5fb6b599d 100644 --- a/tests/IntegrationTests/DataTypes.cs +++ b/tests/IntegrationTests/DataTypes.cs @@ -1,4 +1,6 @@ using System.Globalization; +using System.Runtime.InteropServices; + #if MYSQL_DATA using MySql.Data.Types; #endif @@ -1143,7 +1145,9 @@ private static object CreateGeometry(byte[] data) [InlineData("Int64", "datatypes_integers", MySqlDbType.Int64, 20, typeof(long), "N", 0, 0)] [InlineData("UInt64", "datatypes_integers", MySqlDbType.UInt64, 20, typeof(ulong), "N", 0, 0)] [InlineData("value", "datatypes_json_core", MySqlDbType.JSON, int.MaxValue, typeof(string), "LN", 0, 0)] -#if !MYSQL_DATA +#if MYSQL_DATA + [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 12, typeof(byte[]), "N", 0, 31)] +#else [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 3, typeof(float[]), "N", 0, 31)] #endif [InlineData("Single", "datatypes_reals", MySqlDbType.Float, 12, typeof(float), "N", 0, 31)] @@ -1604,17 +1608,28 @@ public void QueryJson(string column, string[] expected) DoQuery("json_core", column, dataTypeName, expected, reader => reader.GetString(0), omitWhereTest: true); } -#if !MYSQL_DATA [SkippableTheory(ServerFeatures.Vector)] [InlineData("value", new[] { null, "0,0,0", "1,1,1", "1,2,3", "3.40282347E+38,3.40282347E+38,3.40282347E+38" })] public void QueryVector(string column, string[] expected) { string dataTypeName = "VECTOR"; DoQuery("vector", column, dataTypeName, - expected.Select(x => x?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()).ToArray(), - static x => (float[]) x.GetValue(0), omitWhereTest: true); - } + expected.Select(x => +#if MYSQL_DATA + // Connector/NET returns the float array as a byte[] + x is null ? null : MemoryMarshal.AsBytes(x.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()).ToArray()) +#else + x?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()) +#endif + .ToArray(), +#if !MYSQL_DATA + static x => (float[]) x.GetValue(0), +#else + // NOTE: Connector/NET returns 'null' for NULL so simulate an exception for the tests + x => x.IsDBNull(0) ? throw new GetValueWhenNullException() : x.GetValue(0), #endif + omitWhereTest: true); + } [SkippableTheory(MySqlData = "https://bugs.mysql.com/bug.php?id=97067")] [InlineData(false, "MIN", 0)] diff --git a/tests/IntegrationTests/QueryTests.cs b/tests/IntegrationTests/QueryTests.cs index a6077f5fe..fecaa3141 100644 --- a/tests/IntegrationTests/QueryTests.cs +++ b/tests/IntegrationTests/QueryTests.cs @@ -1,3 +1,5 @@ +using System.Runtime.InteropServices; + namespace IntegrationTests; public class QueryTests : IClassFixture, IDisposable @@ -1688,6 +1690,56 @@ public void GetBytesByName() } #endif + [SkippableTheory(ServerFeatures.Vector)] + [InlineData(false)] + [InlineData(true)] + public void QueryVector(bool prepare) + { + using var connection = new MySqlConnection(AppConfig.ConnectionString); + connection.Open(); + + connection.Execute(""" + drop table if exists test_vector; + create table test_vector(id int auto_increment not null primary key, vec vector not null); + """); + + using var cmd = m_database.Connection.CreateCommand(); + cmd.CommandText = "INSERT INTO test_vector(vec) VALUES(@vec)"; + cmd.Parameters.Add(new MySqlParameter + { + ParameterName = "@vec", + MySqlDbType = MySqlDbType.Vector, + }); + + var floatArray = new[] { 1.2f, 3.4f, 5.6f }; +#if MYSQL_DATA + // Connector/NET requires the float vector to be passed as a byte array + cmd.Parameters[0].Value = MemoryMarshal.AsBytes(floatArray).ToArray(); +#else + cmd.Parameters[0].Value = floatArray; +#endif + + if (prepare) + cmd.Prepare(); + cmd.ExecuteNonQuery(); + + // Select and verify the value + cmd.CommandText = "SELECT vec FROM test_vector"; + if (prepare) + cmd.Prepare(); + + using var reader = cmd.ExecuteReader(); + Assert.True(reader.Read()); + var value = reader.GetValue(0); + +#if MYSQL_DATA + var result = MemoryMarshal.Cast((byte[]) value).ToArray(); +#else + var result = (float[]) value; +#endif + Assert.Equal(floatArray, result); + } + private class BoolTest { public int Id { get; set; } diff --git a/tests/IntegrationTests/StoredProcedureTests.cs b/tests/IntegrationTests/StoredProcedureTests.cs index cd0e79ede..18e5efde0 100644 --- a/tests/IntegrationTests/StoredProcedureTests.cs +++ b/tests/IntegrationTests/StoredProcedureTests.cs @@ -761,6 +761,44 @@ public void PassJsonParameter() Assert.False(reader.Read()); } +#if false + [SkippableTheory(ServerFeatures.Vector)] + [InlineData(false)] + [InlineData(true)] + public void VectorOutputParameter(bool prepare) + { + using var cmd = m_database.Connection.CreateCommand(); + cmd.CommandText = """ + DROP PROCEDURE IF EXISTS sp_vector_out; + CREATE PROCEDURE sp_vector_out (OUT vec VECTOR) + BEGIN + SELECT STRING_TO_VECTOR('[1.2, 3.4, 5.6]') INTO vec; + END; + """; + cmd.ExecuteNonQuery(); + + cmd.CommandText = "sp_vector_out"; + cmd.CommandType = CommandType.StoredProcedure; + cmd.Parameters.Add(new MySqlParameter + { + ParameterName = "@vec", + MySqlDbType = MySqlDbType.Vector, + Direction = ParameterDirection.Output, + }); + + if (prepare) + cmd.Prepare(); + cmd.ExecuteNonQuery(); + + var value = cmd.Parameters[0].Value; + Assert.IsType(value); + + var result = (float[]) value; + + Assert.Equal(new float[] { 1.2f, 3.4f, 5.6f }, result); + } +#endif + private static Action AssertParameter(string name, ParameterDirection direction, MySqlDbType mySqlDbType) { return x => From f99c6923965762f0c8997cba71ec078baebe3438 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 20:10:27 -0800 Subject: [PATCH 11/16] Support VECTOR for MariaDB 11.7. Signed-off-by: Bradley Grainger --- azure-pipelines.yml | 6 ++-- .../Core/SingleCommandPayloadCreator.cs | 4 +++ tests/IntegrationTests/CharacterSetTests.cs | 2 +- tests/IntegrationTests/DataTypes.cs | 29 ++++++++++++++----- tests/IntegrationTests/DataTypesFixture.cs | 6 +++- tests/IntegrationTests/QueryTests.cs | 5 ++-- tests/IntegrationTests/ServerFeatures.cs | 7 ++++- 7 files changed, 44 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 811c77f61..cc793ad80 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -192,10 +192,10 @@ jobs: image: 'mariadb:11.4' connectionStringExtra: '' unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin,Vector' - 'MariaDB 11.6': - image: 'mariadb:11.6' + 'MariaDB 11.7': + image: 'mariadb:11.7' connectionStringExtra: '' - unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin,Vector' + unsupportedFeatures: 'CachingSha2Password,CancelSleepSuccessfully,Json,RoundDateTime,QueryAttributes,Redirection,Sha256Password,Tls11,UuidToBin,VectorType' steps: - template: '.ci/integration-tests-steps.yml' parameters: diff --git a/src/MySqlConnector/Core/SingleCommandPayloadCreator.cs b/src/MySqlConnector/Core/SingleCommandPayloadCreator.cs index e6c2c641d..a9abe0b97 100644 --- a/src/MySqlConnector/Core/SingleCommandPayloadCreator.cs +++ b/src/MySqlConnector/Core/SingleCommandPayloadCreator.cs @@ -165,6 +165,10 @@ private static void WriteBinaryParameters(ByteBufferWriter writer, MySqlParamete mySqlDbType = TypeMapper.Instance.GetMySqlDbTypeForDbType(dbType); } + // HACK: MariaDB doesn't have a dedicated Vector type so mark it as binary data + if (mySqlDbType == MySqlDbType.Vector && command.Connection!.Session.ServerVersion.IsMariaDb) + mySqlDbType = MySqlDbType.LongBlob; + writer.Write(TypeMapper.ConvertToColumnTypeAndFlags(mySqlDbType, command.Connection!.GuidFormat)); if (supportsQueryAttributes) diff --git a/tests/IntegrationTests/CharacterSetTests.cs b/tests/IntegrationTests/CharacterSetTests.cs index 2dcb685a8..317d49ddc 100644 --- a/tests/IntegrationTests/CharacterSetTests.cs +++ b/tests/IntegrationTests/CharacterSetTests.cs @@ -79,7 +79,7 @@ public void CollationConnection(bool reopenConnection) var collation = connection.Query(@"select @@collation_connection;").Single(); var expected = connection.ServerVersion.Substring(0, 2) is "8." or "9." ? "utf8mb4_0900_ai_ci" : - connection.ServerVersion.StartsWith("11.4.", StringComparison.Ordinal) || connection.ServerVersion.StartsWith("11.6.", StringComparison.Ordinal) ? "utf8mb4_uca1400_ai_ci" : + connection.ServerVersion.StartsWith("11.4.", StringComparison.Ordinal) || connection.ServerVersion.StartsWith("11.7.", StringComparison.Ordinal) ? "utf8mb4_uca1400_ai_ci" : "utf8mb4_general_ci"; Assert.Equal(expected, collation); } diff --git a/tests/IntegrationTests/DataTypes.cs b/tests/IntegrationTests/DataTypes.cs index 5fb6b599d..8a95e8360 100644 --- a/tests/IntegrationTests/DataTypes.cs +++ b/tests/IntegrationTests/DataTypes.cs @@ -1205,6 +1205,15 @@ private void DoGetSchemaTable(string column, string table, MySqlDbType mySqlDbTy if (table == "datatypes_vector" && !AppConfig.SupportedFeatures.HasFlag(ServerFeatures.Vector)) return; + // adjust for databases that don't have a dedicated on-the-wire type for VECTOR(n) + if (mySqlDbType == MySqlDbType.Vector && !AppConfig.SupportedFeatures.HasFlag(ServerFeatures.VectorType)) + { + mySqlDbType = MySqlDbType.VarBinary; + columnSize *= 4; + dataType = typeof(byte[]); + scale = 0; + } + var isAutoIncrement = flags.IndexOf('A') != -1; var isKey = flags.IndexOf('K') != -1; var isLong = flags.IndexOf('L') != -1; @@ -1609,26 +1618,32 @@ public void QueryJson(string column, string[] expected) } [SkippableTheory(ServerFeatures.Vector)] - [InlineData("value", new[] { null, "0,0,0", "1,1,1", "1,2,3", "3.40282347E+38,3.40282347E+38,3.40282347E+38" })] + [InlineData("value", new[] { null, "0,0,0", "1,1,1", "1,2,3", "-1,-1,-1" })] public void QueryVector(string column, string[] expected) { - string dataTypeName = "VECTOR"; + var hasVectorType = AppConfig.SupportedFeatures.HasFlag(ServerFeatures.VectorType); + string dataTypeName = hasVectorType ? "VECTOR" : "BLOB"; DoQuery("vector", column, dataTypeName, expected.Select(x => -#if MYSQL_DATA - // Connector/NET returns the float array as a byte[] - x is null ? null : MemoryMarshal.AsBytes(x.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()).ToArray()) +#if !MYSQL_DATA + hasVectorType ? (object) GetFloatArray(x) : GetByteArray(x)) #else - x?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray()) + // Connector/NET returns the float array as a byte[] + GetByteArray(x)) #endif .ToArray(), #if !MYSQL_DATA - static x => (float[]) x.GetValue(0), + x => hasVectorType ? (float[]) x.GetValue(0) : (byte[]) x.GetValue(0), #else // NOTE: Connector/NET returns 'null' for NULL so simulate an exception for the tests x => x.IsDBNull(0) ? throw new GetValueWhenNullException() : x.GetValue(0), #endif omitWhereTest: true); + + static float[] GetFloatArray(string value) => value?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray(); + + static byte[] GetByteArray(string value) => + GetFloatArray(value) is { } floats ? MemoryMarshal.AsBytes(floats).ToArray() : null; } [SkippableTheory(MySqlData = "https://bugs.mysql.com/bug.php?id=97067")] diff --git a/tests/IntegrationTests/DataTypesFixture.cs b/tests/IntegrationTests/DataTypesFixture.cs index 1dbba6675..9f3fbac72 100644 --- a/tests/IntegrationTests/DataTypesFixture.cs +++ b/tests/IntegrationTests/DataTypesFixture.cs @@ -245,6 +245,10 @@ insert into datatypes_json_core (value) if (AppConfig.SupportedFeatures.HasFlag(ServerFeatures.Vector)) { + // create a helper function for MariaDB 11.7+ + if (Connection.ServerVersion.StartsWith("11.7.", StringComparison.Ordinal)) + Connection.Execute("create function if not exists STRING_TO_VECTOR(s text) returns vector(3) deterministic return Vec_FromText(s);"); + Connection.Execute(""" drop table if exists datatypes_vector; create table datatypes_vector ( @@ -257,7 +261,7 @@ insert into datatypes_vector (value) (STRING_TO_VECTOR('[0, 0, 0]')), (STRING_TO_VECTOR('[1, 1, 1]')), (STRING_TO_VECTOR('[1, 2, 3]')), - (STRING_TO_VECTOR('[3.40282347E+38, 3.40282347E+38, 3.40282347E+38]')); + (STRING_TO_VECTOR('[-1, -1, -1]')); """); } Connection.Close(); diff --git a/tests/IntegrationTests/QueryTests.cs b/tests/IntegrationTests/QueryTests.cs index fecaa3141..3c0aa6e4a 100644 --- a/tests/IntegrationTests/QueryTests.cs +++ b/tests/IntegrationTests/QueryTests.cs @@ -1700,7 +1700,7 @@ public void QueryVector(bool prepare) connection.Execute(""" drop table if exists test_vector; - create table test_vector(id int auto_increment not null primary key, vec vector not null); + create table test_vector(id int auto_increment not null primary key, vec vector(3) not null); """); using var cmd = m_database.Connection.CreateCommand(); @@ -1735,7 +1735,8 @@ public void QueryVector(bool prepare) #if MYSQL_DATA var result = MemoryMarshal.Cast((byte[]) value).ToArray(); #else - var result = (float[]) value; + var result = AppConfig.SupportedFeatures.HasFlag(ServerFeatures.VectorType) ? (float[]) value : + MemoryMarshal.Cast((byte[]) value).ToArray(); #endif Assert.Equal(floatArray, result); } diff --git a/tests/IntegrationTests/ServerFeatures.cs b/tests/IntegrationTests/ServerFeatures.cs index c1f12591b..8e37bc0b2 100644 --- a/tests/IntegrationTests/ServerFeatures.cs +++ b/tests/IntegrationTests/ServerFeatures.cs @@ -47,7 +47,12 @@ public enum ServerFeatures TlsFingerprintValidation = 0x100_0000, /// - /// Server supports the VECTOR data type. + /// Server supports the VECTOR SQL type. /// Vector = 0x200_0000, + + /// + /// Server has a dedicated type on the wire for VECTOR. + /// + VectorType = 0x400_0000, } From c6f228f0c0cdb37ad8a5da6f8db713955402ba38 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 23 Feb 2025 20:33:49 -0800 Subject: [PATCH 12/16] Fix references to 11.6. Signed-off-by: Bradley Grainger --- .ci/docker-run.sh | 2 +- docs/content/home.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker-run.sh b/.ci/docker-run.sh index 7a791e21d..4f2999c2d 100755 --- a/.ci/docker-run.sh +++ b/.ci/docker-run.sh @@ -28,7 +28,7 @@ MYSQL=mysql if [[ "$IMAGE" == mariadb* ]]; then MYSQL_EXTRA='--in-predicate-conversion-threshold=100000 --plugin-maturity=beta' fi -if [ "$IMAGE" == "mariadb:11.4" ] || [ "$IMAGE" == "mariadb:11.6" ]; then +if [ "$IMAGE" == "mariadb:11.4" ] || [ "$IMAGE" == "mariadb:11.7" ]; then MYSQL='mariadb' fi diff --git a/docs/content/home.md b/docs/content/home.md index 0db29f09c..a687d7c49 100644 --- a/docs/content/home.md +++ b/docs/content/home.md @@ -64,7 +64,7 @@ Server | Versions | Notes Amazon Aurora RDS | 2.x, 3.x | Use `Pipelining=False` [for Aurora 2.x](https://mysqlconnector.net/troubleshooting/aurora-freeze/) Azure Database for MySQL | 5.7, 8.0 | Single Server and Flexible Server Google Cloud SQL for MySQL | 5.6, 5.7, 8.0 | -MariaDB | 10.x (**10.6**, **10.11**), 11.x (**11.4**, **11.6**) | +MariaDB | 10.x (**10.6**, **10.11**), 11.x (**11.4**, **11.7**) | MySQL | 5.5, 5.6, 5.7, 8.x (**8.0**, **8.4**), 9.x (**9.2**) | 5.5 is EOL and has some [compatibility issues](https://github.com/mysql-net/MySqlConnector/issues/1192); 5.6 and 5.7 are EOL Percona Server | 5.6, 5.7, 8.0 | PlanetScale | | See PlanetScale [MySQL compatibility notes](https://planetscale.com/docs/reference/mysql-compatibility) From c27edde83d2a34fb53c88d2b4a571effb74055f6 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sat, 8 Mar 2025 12:34:56 -0800 Subject: [PATCH 13/16] Retrieve stored procedure out parameters as byte[]. Signed-off-by: Bradley Grainger --- src/MySqlConnector/MySqlDataReader.cs | 2 +- tests/IntegrationTests/StoredProcedureTests.cs | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/MySqlConnector/MySqlDataReader.cs b/src/MySqlConnector/MySqlDataReader.cs index 800fafe47..98e07fdd0 100644 --- a/src/MySqlConnector/MySqlDataReader.cs +++ b/src/MySqlConnector/MySqlDataReader.cs @@ -671,7 +671,7 @@ private static async Task ReadOutParametersAsync(IMySqlCommand command, ResultSe if (param.HasSetDbType && !row.IsDBNull(columnIndex)) { var dbTypeMapping = TypeMapper.Instance.GetDbTypeMapping(param.DbType); - if (dbTypeMapping is not null) + if (dbTypeMapping is not null && param.DbType is not DbType.Object) { param.Value = dbTypeMapping.DoConversion(row.GetValue(columnIndex)); continue; diff --git a/tests/IntegrationTests/StoredProcedureTests.cs b/tests/IntegrationTests/StoredProcedureTests.cs index 18e5efde0..dc327f863 100644 --- a/tests/IntegrationTests/StoredProcedureTests.cs +++ b/tests/IntegrationTests/StoredProcedureTests.cs @@ -1,3 +1,5 @@ +using System.Runtime.InteropServices; + namespace IntegrationTests; public class StoredProcedureTests : IClassFixture @@ -761,8 +763,7 @@ public void PassJsonParameter() Assert.False(reader.Read()); } -#if false - [SkippableTheory(ServerFeatures.Vector)] + [SkippableTheory(ServerFeatures.Vector | ServerFeatures.VectorType)] [InlineData(false)] [InlineData(true)] public void VectorOutputParameter(bool prepare) @@ -781,9 +782,9 @@ CREATE PROCEDURE sp_vector_out (OUT vec VECTOR) cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add(new MySqlParameter { - ParameterName = "@vec", - MySqlDbType = MySqlDbType.Vector, Direction = ParameterDirection.Output, + MySqlDbType = MySqlDbType.Vector, + ParameterName = "@vec", }); if (prepare) @@ -791,13 +792,9 @@ CREATE PROCEDURE sp_vector_out (OUT vec VECTOR) cmd.ExecuteNonQuery(); var value = cmd.Parameters[0].Value; - Assert.IsType(value); - - var result = (float[]) value; - - Assert.Equal(new float[] { 1.2f, 3.4f, 5.6f }, result); + var result = Assert.IsType(value); + Assert.Equal(new float[] { 1.2f, 3.4f, 5.6f }, MemoryMarshal.Cast(result).ToArray()); } -#endif private static Action AssertParameter(string name, ParameterDirection direction, MySqlDbType mySqlDbType) { From c106b0b6cb0c94cbb84e5c747ebb231cdab0c497 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sat, 17 May 2025 13:12:36 -0700 Subject: [PATCH 14/16] Use ReadOnlyMemory as the type for VECTOR. Based on these comments: https://github.com/dotnet/runtime/issues/115148#issuecomment-2844209601. Signed-off-by: Bradley Grainger --- .../ColumnReaders/VectorColumnReader.cs | 2 +- src/MySqlConnector/Core/TypeMapper.cs | 4 ++-- tests/IntegrationTests/DataTypes.cs | 14 +++++++++++--- tests/IntegrationTests/QueryTests.cs | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs index 637b0f491..a1603f6f0 100644 --- a/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs +++ b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs @@ -8,5 +8,5 @@ internal sealed class VectorColumnReader : ColumnReader public static VectorColumnReader Instance { get; } = new(); public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) => - MemoryMarshal.Cast(data).ToArray(); + new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); } diff --git a/src/MySqlConnector/Core/TypeMapper.cs b/src/MySqlConnector/Core/TypeMapper.cs index 1058a99e1..e40ab707d 100644 --- a/src/MySqlConnector/Core/TypeMapper.cs +++ b/src/MySqlConnector/Core/TypeMapper.cs @@ -56,8 +56,8 @@ private TypeMapper() AddColumnTypeMetadata(new("FLOAT", typeFloat, MySqlDbType.Float)); // vector - var typeFloatArray = AddDbTypeMapping(new(typeof(float[]), [DbType.Object])); - AddColumnTypeMetadata(new("VECTOR", typeFloatArray, MySqlDbType.Vector, binary: true, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})")); + var typeFloatReadOnlyMemory = AddDbTypeMapping(new(typeof(ReadOnlyMemory), [DbType.Object])); + AddColumnTypeMetadata(new("VECTOR", typeFloatReadOnlyMemory, MySqlDbType.Vector, binary: true, simpleDataTypeName: "VECTOR", createFormat: "VECTOR({0})")); // string var typeFixedString = AddDbTypeMapping(new(typeof(string), [DbType.StringFixedLength, DbType.AnsiStringFixedLength], convert: Convert.ToString!)); diff --git a/tests/IntegrationTests/DataTypes.cs b/tests/IntegrationTests/DataTypes.cs index 8a95e8360..080fb1055 100644 --- a/tests/IntegrationTests/DataTypes.cs +++ b/tests/IntegrationTests/DataTypes.cs @@ -1148,7 +1148,7 @@ private static object CreateGeometry(byte[] data) #if MYSQL_DATA [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 12, typeof(byte[]), "N", 0, 31)] #else - [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 3, typeof(float[]), "N", 0, 31)] + [InlineData("value", "datatypes_vector", MySqlDbType.Vector, 3, typeof(ReadOnlyMemory), "N", 0, 31)] #endif [InlineData("Single", "datatypes_reals", MySqlDbType.Float, 12, typeof(float), "N", 0, 31)] [InlineData("Double", "datatypes_reals", MySqlDbType.Double, 22, typeof(double), "N", 0, 31)] @@ -1626,18 +1626,26 @@ public void QueryVector(string column, string[] expected) DoQuery("vector", column, dataTypeName, expected.Select(x => #if !MYSQL_DATA - hasVectorType ? (object) GetFloatArray(x) : GetByteArray(x)) + hasVectorType ? (GetFloatArray(x) is float[] a ? (object) new ReadOnlyMemory(a) : null) : GetByteArray(x)) #else // Connector/NET returns the float array as a byte[] GetByteArray(x)) #endif .ToArray(), #if !MYSQL_DATA - x => hasVectorType ? (float[]) x.GetValue(0) : (byte[]) x.GetValue(0), + x => hasVectorType ? (ReadOnlyMemory) x.GetValue(0) : (byte[]) x.GetValue(0), #else // NOTE: Connector/NET returns 'null' for NULL so simulate an exception for the tests x => x.IsDBNull(0) ? throw new GetValueWhenNullException() : x.GetValue(0), #endif + assertEqual: (l, r) => + { + if (l is ReadOnlyMemory roml) + l = roml.ToArray(); + if (r is ReadOnlyMemory romr) + r = romr.ToArray(); + Assert.Equal(l, r); + }, omitWhereTest: true); static float[] GetFloatArray(string value) => value?.Split(',').Select(x => float.Parse(x, CultureInfo.InvariantCulture)).ToArray(); diff --git a/tests/IntegrationTests/QueryTests.cs b/tests/IntegrationTests/QueryTests.cs index 3c0aa6e4a..1d19747c3 100644 --- a/tests/IntegrationTests/QueryTests.cs +++ b/tests/IntegrationTests/QueryTests.cs @@ -1735,7 +1735,7 @@ public void QueryVector(bool prepare) #if MYSQL_DATA var result = MemoryMarshal.Cast((byte[]) value).ToArray(); #else - var result = AppConfig.SupportedFeatures.HasFlag(ServerFeatures.VectorType) ? (float[]) value : + var result = AppConfig.SupportedFeatures.HasFlag(ServerFeatures.VectorType) ? (ReadOnlyMemory) value : MemoryMarshal.Cast((byte[]) value).ToArray(); #endif Assert.Equal(floatArray, result); From 54ced4fb94274f4179f94e8f429ca749cf3fd873 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sun, 18 May 2025 14:26:25 -0700 Subject: [PATCH 15/16] Support Memory as a parameter type. Signed-off-by: Bradley Grainger --- .../troubleshooting/parameter-types.md | 1 + src/MySqlConnector/MySqlParameter.cs | 14 +++++++++++++- tests/IntegrationTests/QueryTests.cs | 19 +++++++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/content/troubleshooting/parameter-types.md b/docs/content/troubleshooting/parameter-types.md index 8085b2d9c..82cd238d0 100644 --- a/docs/content/troubleshooting/parameter-types.md +++ b/docs/content/troubleshooting/parameter-types.md @@ -39,5 +39,6 @@ In some cases, this may be as simple as calling `.ToString()` or `.ToString(Cult * .NET primitives: `bool`, `byte`, `char`, `double`, `float`, `int`, `long`, `sbyte`, `short`, `uint`, `ulong`, `ushort` * Common types: `BigInteger`, `DateOnly`, `DateTime`, `DateTimeOffset`, `decimal`, `enum`, `Guid`, `string`, `TimeOnly`, `TimeSpan` * BLOB types: `ArraySegment`, `byte[]`, `Memory`, `ReadOnlyMemory` +* Vector types: `float[]`, `Memory`, `ReadOnlyMemory` * String types: `Memory`, `ReadOnlyMemory`, `StringBuilder` * Custom MySQL types: `MySqlDateTime`, `MySqlDecimal`, `MySqlGeometry` diff --git a/src/MySqlConnector/MySqlParameter.cs b/src/MySqlConnector/MySqlParameter.cs index 85d45c9e4..f0aebdaa9 100644 --- a/src/MySqlConnector/MySqlParameter.cs +++ b/src/MySqlConnector/MySqlParameter.cs @@ -285,7 +285,7 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions { writer.WriteString(ulongValue); } - else if (Value is byte[] or ReadOnlyMemory or Memory or ArraySegment or MySqlGeometry or MemoryStream or float[]) + else if (Value is byte[] or ReadOnlyMemory or Memory or ArraySegment or MySqlGeometry or MemoryStream or float[] or ReadOnlyMemory or Memory) { var inputSpan = Value switch { @@ -295,6 +295,8 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions MySqlGeometry geometry => geometry.ValueSpan, MemoryStream memoryStream => memoryStream.TryGetBuffer(out var streamBuffer) ? streamBuffer.AsSpan() : memoryStream.ToArray().AsSpan(), float[] floatArray => MemoryMarshal.AsBytes(floatArray.AsSpan()), + Memory memory => MemoryMarshal.AsBytes(memory.Span), + ReadOnlyMemory memory => MemoryMarshal.AsBytes(memory.Span), _ => ((ReadOnlyMemory) Value).Span, }; @@ -738,6 +740,16 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar writer.WriteLengthEncodedInteger(unchecked((ulong) floatArrayValue.Length * 4)); writer.Write(MemoryMarshal.AsBytes(floatArrayValue.AsSpan())); } + else if (value is Memory floatMemory) + { + writer.WriteLengthEncodedInteger(unchecked((ulong) floatMemory.Length * 4)); + writer.Write(MemoryMarshal.AsBytes(floatMemory.Span)); + } + else if (value is ReadOnlyMemory floatReadOnlyMemory) + { + writer.WriteLengthEncodedInteger(unchecked((ulong) floatReadOnlyMemory.Length * 4)); + writer.Write(MemoryMarshal.AsBytes(floatReadOnlyMemory.Span)); + } else if (value is decimal decimalValue) { writer.WriteLengthEncodedAsciiString(decimalValue.ToString(CultureInfo.InvariantCulture)); diff --git a/tests/IntegrationTests/QueryTests.cs b/tests/IntegrationTests/QueryTests.cs index 1d19747c3..eae6175da 100644 --- a/tests/IntegrationTests/QueryTests.cs +++ b/tests/IntegrationTests/QueryTests.cs @@ -1691,9 +1691,13 @@ public void GetBytesByName() #endif [SkippableTheory(ServerFeatures.Vector)] - [InlineData(false)] - [InlineData(true)] - public void QueryVector(bool prepare) + [InlineData(false, 0)] + [InlineData(false, 1)] + [InlineData(false, 2)] + [InlineData(true, 0)] + [InlineData(true, 1)] + [InlineData(true, 2)] + public void QueryVector(bool prepare, int dataFormat) { using var connection = new MySqlConnection(AppConfig.ConnectionString); connection.Open(); @@ -1715,8 +1719,15 @@ public void QueryVector(bool prepare) #if MYSQL_DATA // Connector/NET requires the float vector to be passed as a byte array cmd.Parameters[0].Value = MemoryMarshal.AsBytes(floatArray).ToArray(); + Assert.InRange(dataFormat, 0, 2); #else - cmd.Parameters[0].Value = floatArray; + cmd.Parameters[0].Value = dataFormat switch + { + 0 => floatArray, + 1 => new Memory(floatArray), + 2 => new ReadOnlyMemory(floatArray), + _ => throw new NotSupportedException(), + }; #endif if (prepare) From c7bb9b3e34cac4a44a9de7368154e4bf584bda98 Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sat, 24 May 2025 09:19:36 -0700 Subject: [PATCH 16/16] Handle big-endian floating-point values. Signed-off-by: Bradley Grainger --- .../ColumnReaders/VectorColumnReader.cs | 29 +++++++- src/MySqlConnector/MySqlParameter.cs | 69 ++++++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs index a1603f6f0..23f574e72 100644 --- a/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs +++ b/src/MySqlConnector/ColumnReaders/VectorColumnReader.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Runtime.InteropServices; using MySqlConnector.Protocol.Payloads; @@ -7,6 +8,30 @@ internal sealed class VectorColumnReader : ColumnReader { public static VectorColumnReader Instance { get; } = new(); - public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) => - new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + public override object ReadValue(ReadOnlySpan data, ColumnDefinitionPayload columnDefinition) + { + if (BitConverter.IsLittleEndian) + { + return new ReadOnlyMemory(MemoryMarshal.Cast(data).ToArray()); + } + else + { + var floats = new float[data.Length / 4]; + +#if !NET5_0_OR_GREATER + var bytes = data.ToArray(); +#endif + for (var i = 0; i < floats.Length; i++) + { +#if NET5_0_OR_GREATER + floats[i] = BinaryPrimitives.ReadSingleLittleEndian(data.Slice(i * 4)); +#else + Array.Reverse(bytes, i * 4, 4); + floats[i] = BitConverter.ToSingle(bytes, i * 4); +#endif + } + + return new ReadOnlyMemory(floats); + } + } } diff --git a/src/MySqlConnector/MySqlParameter.cs b/src/MySqlConnector/MySqlParameter.cs index f0aebdaa9..3913c48ce 100644 --- a/src/MySqlConnector/MySqlParameter.cs +++ b/src/MySqlConnector/MySqlParameter.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Buffers.Text; using System.Data.Common; using System.Diagnostics; @@ -294,9 +295,9 @@ internal void AppendSqlString(ByteBufferWriter writer, StatementPreparerOptions Memory memory => memory.Span, MySqlGeometry geometry => geometry.ValueSpan, MemoryStream memoryStream => memoryStream.TryGetBuffer(out var streamBuffer) ? streamBuffer.AsSpan() : memoryStream.ToArray().AsSpan(), - float[] floatArray => MemoryMarshal.AsBytes(floatArray.AsSpan()), - Memory memory => MemoryMarshal.AsBytes(memory.Span), - ReadOnlyMemory memory => MemoryMarshal.AsBytes(memory.Span), + float[] floatArray => ConvertFloatsToBytes(floatArray.AsSpan()), + Memory memory => ConvertFloatsToBytes(memory.Span), + ReadOnlyMemory memory => ConvertFloatsToBytes(memory.Span), _ => ((ReadOnlyMemory) Value).Span, }; @@ -729,26 +730,52 @@ private void AppendBinary(ByteBufferWriter writer, object value, StatementPrepar } else if (value is float floatValue) { - writer.Write(BitConverter.GetBytes(floatValue)); +#if NET5_0_OR_GREATER + Span bytes = stackalloc byte[4]; + BinaryPrimitives.WriteSingleLittleEndian(bytes, floatValue); + writer.Write(bytes); +#else + // convert float to bytes with correct endianness (MySQL uses little-endian) + var bytes = BitConverter.GetBytes(floatValue); + if (!BitConverter.IsLittleEndian) + Array.Reverse(bytes); + writer.Write(bytes); +#endif } else if (value is double doubleValue) { - writer.Write(unchecked((ulong) BitConverter.DoubleToInt64Bits(doubleValue))); +#if NET5_0_OR_GREATER + Span bytes = stackalloc byte[8]; + BinaryPrimitives.WriteDoubleLittleEndian(bytes, doubleValue); + writer.Write(bytes); +#else + if (BitConverter.IsLittleEndian) + { + writer.Write(unchecked((ulong) BitConverter.DoubleToInt64Bits(doubleValue))); + } + else + { + // convert double to bytes with correct endianness (MySQL uses little-endian) + var bytes = BitConverter.GetBytes(doubleValue); + Array.Reverse(bytes); + writer.Write(bytes); + } +#endif } else if (value is float[] floatArrayValue) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatArrayValue.Length * 4)); - writer.Write(MemoryMarshal.AsBytes(floatArrayValue.AsSpan())); + writer.Write(ConvertFloatsToBytes(floatArrayValue.AsSpan())); } else if (value is Memory floatMemory) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatMemory.Length * 4)); - writer.Write(MemoryMarshal.AsBytes(floatMemory.Span)); + writer.Write(ConvertFloatsToBytes(floatMemory.Span)); } else if (value is ReadOnlyMemory floatReadOnlyMemory) { writer.WriteLengthEncodedInteger(unchecked((ulong) floatReadOnlyMemory.Length * 4)); - writer.Write(MemoryMarshal.AsBytes(floatReadOnlyMemory.Span)); + writer.Write(ConvertFloatsToBytes(floatReadOnlyMemory.Span)); } else if (value is decimal decimalValue) { @@ -967,6 +994,32 @@ private static void WriteTime(ByteBufferWriter writer, TimeSpan timeSpan) } } + private static ReadOnlySpan ConvertFloatsToBytes(ReadOnlySpan floats) + { + if (BitConverter.IsLittleEndian) + { + return MemoryMarshal.AsBytes(floats); + } + else + { + // for big-endian platforms, we need to convert each float individually + var bytes = new byte[floats.Length * 4]; + + for (var i = 0; i < floats.Length; i++) + { +#if NET5_0_OR_GREATER + BinaryPrimitives.WriteSingleLittleEndian(bytes.AsSpan(i * 4), floats[i]); +#else + var floatBytes = BitConverter.GetBytes(floats[i]); + Array.Reverse(floatBytes); + floatBytes.CopyTo(bytes, i * 4); +#endif + } + + return bytes; + } + } + private static ReadOnlySpan BinaryBytes => "_binary'"u8; private DbType m_dbType;