Automatically UNHEX binary data. Fixes #816

Author: bgrainger

docs/content/api/mysql-bulk-copy.md

@@ -121,7 +121,9 @@ Source columns that don't have an entry in `MySqlBulkCopy.ColumnMappings` will b
 (unless the `ColumnMappings` collection is empty, in which case all columns will be mapped
 one-to-one).
 
-Columns containing binary data must be mapped using an expression that uses the `UNHEX` function.
+MySqlConnector will transmit all binary data as hex, so any expression that operates on it
+must decode it with the `UNHEX` function first. (This will be performed automatically if no
+`Expression` is specified, but will be necessary to specify manually for more complex expressions.)
 
 ### Examples
 
@@ -137,10 +139,4 @@ new MySqlBulkCopyColumnMapping
     DestinationColumn = "@tmp",
     Expression = "SET column_value = @tmp * 2",
 },
-new MySqlBulkCopyColumnMapping
-{
-    SourceOrdinal = 1,
-    DestinationColumn = "@tmp2",
-    Expression = "SET binary_column = UNHEX(@tmp2)",
-},
 ```

src/MySqlConnector/MySql.Data.MySqlClient/MySqlBulkCopy.cs

@@ -9,6 +9,7 @@
 using System.Threading.Tasks;
 using MySql.Data.Types;
 using MySqlConnector.Core;
+using MySqlConnector.Logging;
 using MySqlConnector.Protocol;
 using MySqlConnector.Protocol.Serialization;
 using MySqlConnector.Utilities;
@@ -150,17 +151,18 @@ private async ValueTask WriteToServerAsync(IOBehavior ioBehavior, CancellationTo
 				closeConnection = true;
 			}
 
-			// if no user-supplied column mappings, compute them from the destination schema
-			if (ColumnMappings.Count == 0)
+			// merge column mappings with the destination schema
+			var columnMappings = new List<MySqlBulkCopyColumnMapping>(ColumnMappings);
+			var addDefaultMappings = columnMappings.Count == 0;
+			using (var cmd = new MySqlCommand("select * from " + tableName + ";", m_connection, m_transaction))
+			using (var reader = (MySqlDataReader) await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, ioBehavior, cancellationToken).ConfigureAwait(false))
 			{
-				using var cmd = new MySqlCommand("select * from " + tableName + ";", m_connection, m_transaction);
-				using var reader = (MySqlDataReader) await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, ioBehavior, cancellationToken).ConfigureAwait(false);
 				var schema = reader.GetColumnSchema();
 				for (var i = 0; i < schema.Count; i++)
 				{
 					if (schema[i].DataTypeName == "BIT")
 					{
-						ColumnMappings.Add(new MySqlBulkCopyColumnMapping(i, $"@col{i}", $"`{reader.GetName(i)}` = CAST(@col{i} AS UNSIGNED)"));
+						AddColumnMapping(columnMappings, addDefaultMappings, i, reader.GetName(i), $"@`\uE002\bcol{i}`", $"%COL% = CAST(%VAR% AS UNSIGNED)");
 					}
 					else if (schema[i].DataTypeName == "YEAR")
 					{
@@ -172,11 +174,11 @@ private async ValueTask WriteToServerAsync(IOBehavior ioBehavior, CancellationTo
 						var type = schema[i].DataType;
 						if (type == typeof(byte[]) || (type == typeof(Guid) && (m_connection.GuidFormat == MySqlGuidFormat.Binary16 || m_connection.GuidFormat == MySqlGuidFormat.LittleEndianBinary16 || m_connection.GuidFormat == MySqlGuidFormat.TimeSwapBinary16)))
 						{
-							ColumnMappings.Add(new MySqlBulkCopyColumnMapping(i, $"@col{i}", $"`{reader.GetName(i)}` = UNHEX(@col{i})"));
+							AddColumnMapping(columnMappings, addDefaultMappings, i, reader.GetName(i), $"@`\uE002\bcol{i}`", $"%COL% = UNHEX(%VAR%)");
 						}
-						else
+						else if (addDefaultMappings)
 						{
-							ColumnMappings.Add(new MySqlBulkCopyColumnMapping(i, reader.GetName(i)));
+							columnMappings.Add(new MySqlBulkCopyColumnMapping(i, reader.GetName(i)));
 						}
 					}
 				}
@@ -185,9 +187,10 @@ private async ValueTask WriteToServerAsync(IOBehavior ioBehavior, CancellationTo
 			// set columns and expressions from the column mappings
 			for (var i = 0; i < m_valuesEnumerator!.FieldCount; i++)
 			{
-				var columnMapping = ColumnMappings.FirstOrDefault(x => x.SourceOrdinal == i);
+				var columnMapping = columnMappings.FirstOrDefault(x => x.SourceOrdinal == i);
 				if (columnMapping is null)
 				{
+					Log.Info("Ignoring column with SourceOrdinal {0}", i);
 					bulkLoader.Columns.Add("@`\uE002\bignore`");
 				}
 				else
@@ -213,6 +216,28 @@ private async ValueTask WriteToServerAsync(IOBehavior ioBehavior, CancellationTo
 #endif
 
 			static string QuoteIdentifier(string identifier) => "`" + identifier.Replace("`", "``") + "`";
+
+			static void AddColumnMapping(List<MySqlBulkCopyColumnMapping> columnMappings, bool addDefaultMappings, int destinationOrdinal, string destinationColumn, string variableName, string expression)
+			{
+				expression = expression.Replace("%COL%", "`" + destinationColumn + "`").Replace("%VAR%", variableName);
+				var columnMapping = columnMappings.FirstOrDefault(x => destinationColumn.Equals(x.DestinationColumn, StringComparison.OrdinalIgnoreCase));
+				if (columnMapping is object)
+				{
+					if (columnMapping.Expression is object)
+					{
+						Log.Warn("Column mapping for SourceOrdinal {0}, DestinationColumn {1} already has Expression {2}", columnMapping.SourceOrdinal, columnMapping.DestinationColumn, columnMapping.Expression);
+					}
+					else
+					{
+						columnMappings.Remove(columnMapping);
+						columnMappings.Add(new MySqlBulkCopyColumnMapping(columnMapping.SourceOrdinal, variableName, expression));
+					}
+				}
+				else if (addDefaultMappings)
+				{
+					columnMappings.Add(new MySqlBulkCopyColumnMapping(destinationOrdinal, variableName, expression));
+				}
+			}
 		}
 
 		internal async Task SendDataReaderAsync(IOBehavior ioBehavior, CancellationToken cancellationToken)
@@ -515,6 +540,7 @@ static bool WriteBytes(ReadOnlySpan<byte> value, Span<byte> output, out int byte
 
 		private static ReadOnlySpan<byte> EscapedNull => new byte[] { 0x5C, 0x4E };
 		private static readonly char[] s_specialCharacters = new char[] { '\t', '\\', '\n' };
+		private static readonly IMySqlConnectorLogger Log = MySqlConnectorLogManager.CreateLogger(nameof(MySqlBulkCopy));
 
 		readonly MySqlConnection m_connection;
 		readonly MySqlTransaction? m_transaction;

tests/SideBySide/BulkLoaderSync.cs

@@ -793,7 +793,7 @@ public void BulkCopyColumnMappings()
 			using var connection = new MySqlConnection(GetLocalConnectionString());
 			connection.Open();
 			using (var cmd = new MySqlCommand(@"drop table if exists bulk_copy_column_mapping;
-				create table bulk_copy_column_mapping(intvalue int, `text` text);", connection))
+				create table bulk_copy_column_mapping(intvalue int, `text` text, data blob);", connection))
 			{
 				cmd.ExecuteNonQuery();
 			}
@@ -805,6 +805,7 @@ public void BulkCopyColumnMappings()
 				{
 					new MySqlBulkCopyColumnMapping(1, "@val", "intvalue = @val + 1"),
 					new MySqlBulkCopyColumnMapping(3, "text"),
+					new MySqlBulkCopyColumnMapping(4, "data"),
 				},
 			};
 
@@ -816,12 +817,13 @@ public void BulkCopyColumnMappings()
 					new DataColumn("c2", typeof(int)),
 					new DataColumn("c3", typeof(string)),
 					new DataColumn("c4", typeof(string)),
+					new DataColumn("c5", typeof(byte[])),
 				},
 				Rows =
 				{
-					new object[] { 1, 100, "a", "A" },
-					new object[] { 2, 200, "bb", "BB" },
-					new object[] { 3, 300, "ccc", "CCC" },
+					new object[] { 1, 100, "a", "A", new byte[] { 0x33, 0x30 } },
+					new object[] { 2, 200, "bb", "BB", new byte[] { 0x33, 0x31 }  },
+					new object[] { 3, 300, "ccc", "CCC", new byte[] { 0x33, 0x32 }  },
 				}
 			};
 
@@ -831,12 +833,15 @@ public void BulkCopyColumnMappings()
 			Assert.True(reader.Read());
 			Assert.Equal(101, reader.GetValue(0));
 			Assert.Equal("A", reader.GetValue(1));
+			Assert.Equal(new byte[] { 0x33, 0x30 }, reader.GetValue(2));
 			Assert.True(reader.Read());
 			Assert.Equal(201, reader.GetValue(0));
 			Assert.Equal("BB", reader.GetValue(1));
+			Assert.Equal(new byte[] { 0x33, 0x31 }, reader.GetValue(2));
 			Assert.True(reader.Read());
 			Assert.Equal(301, reader.GetValue(0));
 			Assert.Equal("CCC", reader.GetValue(1));
+			Assert.Equal(new byte[] { 0x33, 0x32 }, reader.GetValue(2));
 			Assert.False(reader.Read());
 		}
 #endif