From a2cb6a43464a4d1d6d71f1e43e711e65ebccb60f Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 17:11:48 -0400 Subject: [PATCH 01/15] Allow running all TFM tests in parallel by sanitizing table names to include TFM --- .../ADO/ConnectionTests.cs | 4 +-- .../AbstractConnectionTestFixture.cs | 24 ++++++++++++- .../BulkCopy/BulkCopyTests.cs | 34 +++++++++---------- .../BulkCopy/BulkCopyWithDefaultsTests.cs | 2 +- .../Types/AggregateHelperTests.cs | 2 +- 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs index 2c45f7d6..fc66b0b0 100644 --- a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs +++ b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs @@ -106,7 +106,7 @@ public async Task ClientShouldSetQueryId() } [Test] - public async Task ClientShouldSetUserAgent() + public void ClientShouldSetUserAgent() { var headers = new HttpRequestMessage().Headers; connection.AddDefaultHttpHeaders(headers); @@ -218,7 +218,7 @@ public void ShouldSaveProtocolAtConnectionString(string protocol) [Test] public async Task ShouldPostDynamicallyGeneratedRawStream() { - var targetTable = "test.raw_stream"; + var targetTable = $"test.{SanitizeTableName("raw_stream")}"; await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value Int32) ENGINE Null"); diff --git a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs index eca6f6c6..e64e186c 100644 --- a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs +++ b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs @@ -6,7 +6,7 @@ namespace ClickHouse.Driver.Tests; [TestFixture] -public class AbstractConnectionTestFixture : IDisposable +public abstract class AbstractConnectionTestFixture : IDisposable { protected readonly ClickHouseConnection connection; @@ -27,9 +27,31 @@ protected static string SanitizeTableName(string input) builder.Append(c); } + // When running in parallel, we need to avoid false failures due to running against the same tables + var frameworkSuffix = GetFrameworkSuffix(); + if (!string.IsNullOrEmpty(frameworkSuffix)) + builder.Append('_').Append(frameworkSuffix); + return builder.ToString(); } + private static string GetFrameworkSuffix() + { +#if NET462 + return "net462"; +#elif NET48 + return "net48"; +#elif NET6_0 + return "net6"; +#elif NET8_0 + return "net8"; +#elif NET9_0 + return "net9"; +#else + return ""; +#endif + } + [OneTimeTearDown] public void Dispose() => connection?.Dispose(); } diff --git a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs index c1ce20ba..f19c9806 100644 --- a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs +++ b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs @@ -35,7 +35,7 @@ public static IEnumerable GetInsertSingleValueTestCases() [TestCaseSource(typeof(BulkCopyTests), nameof(GetInsertSingleValueTestCases))] public async Task ShouldExecuteSingleValueInsertViaBulkCopy(string clickHouseType, object insertedValue) { - var targetTable = "test." + SanitizeTableName($"bulk_single_{clickHouseType}"); + var targetTable = $"test.{SanitizeTableName($"bulk_single_{clickHouseType}")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value {clickHouseType}) ENGINE Memory"); @@ -70,7 +70,7 @@ public async Task ShouldExecuteSingleValueInsertViaBulkCopy(string clickHouseTyp [RequiredFeature(Feature.Date32)] public async Task ShouldInsertDateOnly() { - var targetTable = "test.bulk_dateonly"; + var targetTable = "test." + SanitizeTableName("bulk_dateonly"); await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value Date32) ENGINE Memory"); @@ -102,7 +102,7 @@ public async Task ShouldExecuteMultipleBulkInsertions() var sw = new Stopwatch(); var duration = TimeSpan.FromMinutes(5); - var targetTable = "test." + SanitizeTableName($"bulk_load_test"); + var targetTable = $"test.{SanitizeTableName($"bulk_load_test")}"; await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (int Int32, str String, dt DateTime) ENGINE Null"); @@ -138,7 +138,7 @@ public async Task ShouldExecuteMultipleBulkInsertions() [Test] public async Task ShouldExecuteInsertWithLessColumns() { - var targetTable = $"test.multiple_columns"; + var targetTable = $"test.{SanitizeTableName("multiple_columns")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value1 Nullable(UInt8), value2 Nullable(Float32), value3 Nullable(Int8)) ENGINE Memory"); @@ -157,7 +157,7 @@ public async Task ShouldExecuteInsertWithLessColumns() [Test] public async Task ShouldExecuteInsertWithBacktickedColumns() { - var targetTable = $"test.backticked_columns"; + var targetTable = $"test.{SanitizeTableName("backticked_columns")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (`field.id` Nullable(UInt8), `@value` Nullable(UInt8)) ENGINE Memory"); @@ -176,7 +176,7 @@ public async Task ShouldExecuteInsertWithBacktickedColumns() [Test] public async Task ShouldDetectColumnsAutomaticallyOnInit() { - var targetTable = $"test.auto_detect_columns"; + var targetTable = $"test.{SanitizeTableName("auto_detect_columns")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (field1 UInt8, field2 Int8, field3 String) ENGINE Memory"); @@ -208,7 +208,7 @@ public async Task ShouldDetectColumnsAutomaticallyOnInit() [TestCase("with!exclamation")] public async Task ShouldExecuteBulkInsertWithComplexColumnName(string columnName) { - var targetTable = "test." + SanitizeTableName($"bulk_complex_{columnName}"); + var targetTable = $"test.{SanitizeTableName($"bulk_complex_{columnName}")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (`{columnName.Replace("`", "\\`")}` Int32) ENGINE Memory"); @@ -229,7 +229,7 @@ public async Task ShouldExecuteBulkInsertWithComplexColumnName(string columnName [Test] public async Task ShouldInsertIntoTableWithLotsOfColumns() { - var tableName = "test.bulk_long_columns"; + var tableName = $"test.{SanitizeTableName("bulk_long_columns")}"; var columnCount = 3900; //Generating create tbl statement with a lot of columns @@ -252,7 +252,7 @@ public async Task ShouldInsertIntoTableWithLotsOfColumns() [Test] public async Task ShouldThrowSpecialExceptionOnSerializationFailure() { - var targetTable = "test." + SanitizeTableName($"bulk_exception_uint8"); + var targetTable = $"test.{SanitizeTableName($"bulk_exception_uint8")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value UInt8) ENGINE Memory"); @@ -276,7 +276,7 @@ public async Task ShouldThrowSpecialExceptionOnSerializationFailure() [Test] public async Task ShouldExecuteBulkInsertIntoSimpleAggregatedFunctionColumn() { - var targetTable = "test." + SanitizeTableName($"bulk_simple_aggregated_function"); + var targetTable = $"test.{SanitizeTableName($"bulk_simple_aggregated_function")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value SimpleAggregateFunction(anyLast,Nullable(Float64))) ENGINE Memory"); @@ -303,7 +303,7 @@ public async Task ShouldExecuteBulkInsertIntoSimpleAggregatedFunctionColumn() [Test] public async Task ShouldNotLoseRowsOnMultipleBatches() { - var targetTable = "test.bulk_multiple_batches"; ; + var targetTable = $"test.{SanitizeTableName("bulk_multiple_batches")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value Int32) ENGINE Memory"); @@ -331,7 +331,7 @@ public async Task ShouldNotLoseRowsOnMultipleBatches() [Test] public async Task ShouldExecuteWithDBNullArrays() { - var targetTable = $"test.bulk_dbnull_array"; + var targetTable = $"test.{SanitizeTableName("bulk_dbnull_array")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (stringValue Array(String), intValue Array(Int32)) ENGINE Memory"); @@ -355,7 +355,7 @@ await bulkCopy.WriteToServerAsync(new List [Test] public async Task ShouldInsertNestedTable() { - var targetTable = "test.bulk_nested"; + var targetTable = $"test.{SanitizeTableName("bulk_nested")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (`_id` UUID, `Comments` Nested(Id Nullable(String), Comment Nullable(String))) ENGINE Memory"); @@ -381,7 +381,7 @@ public async Task ShouldInsertNestedTable() [Test] public async Task ShouldInsertDoubleNestedTable() { - var targetTable = "test.bulk_double_nested"; + var targetTable = $"test.{SanitizeTableName("bulk_double_nested")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (Id Int64, Threads Nested(Id Int64, Comments Nested(Id Int64, Text String))) ENGINE Memory"); @@ -427,7 +427,7 @@ public async Task ShouldThrowExceptionOnInnerException(double fraction) const int setSize = 3000000; int dbNullIndex = (int)(setSize * fraction); - var targetTable = "test." + SanitizeTableName($"bulk_million_inserts"); + var targetTable = $"test.{SanitizeTableName($"bulk_million_inserts")}"; var data = Enumerable.Repeat(new object[] { 1 }, setSize).ToArray(); @@ -451,7 +451,7 @@ public async Task ShouldThrowExceptionOnInnerException(double fraction) [Test] public async Task ShouldNotAffectSharedArrayPool() { - var targetTable = "test." + SanitizeTableName($"array_pool"); + var targetTable = $"test.{SanitizeTableName($"array_pool")}"; await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (int Int32, str String, dt DateTime) ENGINE Null"); @@ -475,7 +475,7 @@ public async Task ShouldNotAffectSharedArrayPool() [RequiredFeature(Feature.Json)] public async Task ShouldInsertJson() { - var targetTable = "test." + SanitizeTableName($"bulk_json"); + var targetTable = $"test.{SanitizeTableName($"bulk_json")}"; await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory"); diff --git a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs index b4d3f89a..5b2ee3cf 100644 --- a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs +++ b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs @@ -30,7 +30,7 @@ private static IEnumerable Get() [TestCaseSource(typeof(BulkCopyWithDefaultsTests), nameof(Get))] public async Task ShouldExecuteSingleValueInsertViaBulkCopyWithDefaults(string clickhouseType, object insertValue, object expectedValue, string tableName) { - var targetTable = "test." + SanitizeTableName($"bulk_single_default_{tableName}"); + var targetTable = $"test.{SanitizeTableName($"bulk_single_default_{tableName}")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync( diff --git a/ClickHouse.Driver.Tests/Types/AggregateHelperTests.cs b/ClickHouse.Driver.Tests/Types/AggregateHelperTests.cs index eea410ff..7665e4b1 100644 --- a/ClickHouse.Driver.Tests/Types/AggregateHelperTests.cs +++ b/ClickHouse.Driver.Tests/Types/AggregateHelperTests.cs @@ -9,7 +9,7 @@ public class AggregateHelperTests : AbstractConnectionTestFixture [Test] public async Task ShouldThrowCorrectExceptionWhenSelectingAggregateFunction() { - var targetTable = "test.aggregate_test"; + var targetTable = $"test.{SanitizeTableName("aggregate_test")}"; await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value AggregateFunction(uniq, UInt8)) ENGINE Memory"); From 094b9bd3c7b0cd10ff48d2face8363b42f060511 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 17:49:56 -0400 Subject: [PATCH 02/15] Resolve multiple transitive TFM warnings due to mismatched dependencies Later versions of System.Text.Json and its transitive assemblies don't support .NET 6, forcing a suppression of a warning by Microsoft that the configuration is unsupported and used at our own risk. To mitigate this, we can choose appropriate dependencies for each TFM to avoid taking on this risk. Corrected Warnings: System.Text.Json 9.0.8 doesn't support net6.0 and has not been tested with it. Consider upgrading your TargetFramework to net8.0 or later. You may also set true in the project file to ignore this warning and attempt to run in this unsupported configuration at your own risk. System.Text.Encodings.Web 9.0.8 doesn't support net6.0 and has not been tested with it. Consider upgrading your TargetFramework to net8.0 or later. You may also set true in the project file to ignore this warning and attempt to run in this unsupported configuration at your own risk. System.IO.Pipelines 9.0.8 doesn't support net6.0 and has not been tested with it. Consider upgrading your TargetFramework to net8.0 or later. You may also set true in the project file to ignore this warning and attempt to run in this unsupported configuration at your own risk. Microsoft.Bcl.AsyncInterfaces 9.0.8 doesn't support net6.0 and has not been tested with it. Consider upgrading your TargetFramework to net8.0 or later. You may also set true in the project file to ignore this warning and attempt to run in this unsupported configuration at your own risk. --- .../ClickHouse.Driver.Tests.csproj | 18 +++++- .../JsonNodeEqualityComparer.cs | 57 +++++++++++++++++-- ClickHouse.Driver/ClickHouse.Driver.csproj | 22 ++++++- ClickHouse.Driver/Types/JsonType.cs | 47 +++++++++++++-- 4 files changed, 129 insertions(+), 15 deletions(-) diff --git a/ClickHouse.Driver.Tests/ClickHouse.Driver.Tests.csproj b/ClickHouse.Driver.Tests/ClickHouse.Driver.Tests.csproj index 766e0ecc..f4e6c400 100644 --- a/ClickHouse.Driver.Tests/ClickHouse.Driver.Tests.csproj +++ b/ClickHouse.Driver.Tests/ClickHouse.Driver.Tests.csproj @@ -1,4 +1,4 @@ - + net462;net48;net6.0;net8.0;net9.0 @@ -17,8 +17,6 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -38,6 +36,20 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + diff --git a/ClickHouse.Driver.Tests/JsonNodeEqualityComparer.cs b/ClickHouse.Driver.Tests/JsonNodeEqualityComparer.cs index 74df6531..881cb040 100644 --- a/ClickHouse.Driver.Tests/JsonNodeEqualityComparer.cs +++ b/ClickHouse.Driver.Tests/JsonNodeEqualityComparer.cs @@ -1,11 +1,60 @@ -using System.Collections; -using System.Collections.Generic; +using System.Collections.Generic; using System.Text.Json.Nodes; -using NUnit.Framework.Constraints; namespace ClickHouse.Driver.Tests; internal class JsonNodeEqualityComparer : IComparer { - public int Compare(JsonObject x, JsonObject y) => JsonNode.DeepEquals(x, y) ? 0 : 1; + public int Compare(JsonObject x, JsonObject y) + { +#if NET6_0 + return DeepCompareJsonNodes(x, y) ? 0 : 1; +#else + return JsonNode.DeepEquals(x, y) ? 0 : 1; +#endif + } + +#if NET6_0 + private static bool DeepCompareJsonNodes(JsonNode x, JsonNode y) + { + if (x == null && y == null) return true; + if (x == null || y == null) return false; + + if (x is JsonObject xObject && y is JsonObject yObject) + { + if (xObject.Count != yObject.Count) + return false; + + foreach (var property in xObject) + { + if (!yObject.TryGetPropertyValue(property.Key, out var yValue)) + return false; + + if (!DeepCompareJsonNodes(property.Value, yValue)) + return false; + } + + return true; + } + + if (x is JsonArray xArray && y is JsonArray yArray) + { + if (xArray.Count != yArray.Count) + return false; + + for (var i = 0; i < xArray.Count; i++) + { + if (!DeepCompareJsonNodes(xArray[i], yArray[i])) + return false; + } + + return true; + } + + if (x is JsonValue xVal && y is JsonValue yVal) + return xVal.ToJsonString() == yVal.ToJsonString(); + + return false; + } +#endif } diff --git a/ClickHouse.Driver/ClickHouse.Driver.csproj b/ClickHouse.Driver/ClickHouse.Driver.csproj index b70d7f76..4b141f5d 100644 --- a/ClickHouse.Driver/ClickHouse.Driver.csproj +++ b/ClickHouse.Driver/ClickHouse.Driver.csproj @@ -31,7 +31,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - @@ -40,7 +39,26 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + + + + + + + + + + + + + + + + + + + + diff --git a/ClickHouse.Driver/Types/JsonType.cs b/ClickHouse.Driver/Types/JsonType.cs index 59cc11f4..3ba9e9eb 100644 --- a/ClickHouse.Driver/Types/JsonType.cs +++ b/ClickHouse.Driver/Types/JsonType.cs @@ -17,6 +17,41 @@ internal class JsonType : ClickHouseType public override string ToString() => "Json"; +#if NET6_0 + private static bool IsJsonNull(JsonNode node) => node == null || node.ToJsonString() == "null"; + + private static bool IsJsonObject(JsonNode node) => node is JsonObject; + + private static JsonValueKind GetJsonValueKind(JsonNode node) + { + if (node == null) return JsonValueKind.Null; + if (node is JsonObject) return JsonValueKind.Object; + if (node is JsonArray) return JsonValueKind.Array; + if (node is JsonValue val) + { + var str = val.ToJsonString(); + if (str == "null") return JsonValueKind.Null; + if (str == "true") return JsonValueKind.True; + if (str == "false") return JsonValueKind.False; + if (str.StartsWith("\"")) return JsonValueKind.String; + return JsonValueKind.Number; + } + return JsonValueKind.Undefined; + } + + private static JsonValueKind GetJsonValueKind(JsonValue val) => GetJsonValueKind((JsonNode)val); +#else + + private static bool IsJsonNull(JsonNode node) => node == null || node.GetValueKind() == JsonValueKind.Null; + + private static bool IsJsonObject(JsonNode node) => node.GetValueKind() == JsonValueKind.Object; + + private static JsonValueKind GetJsonValueKind(JsonNode node) => node?.GetValueKind() ?? JsonValueKind.Null; + + private static JsonValueKind GetJsonValueKind(JsonValue val) => val.GetValueKind(); + +#endif + public override object Read(ExtendedBinaryReader reader) { JsonObject root = new(); @@ -94,7 +129,7 @@ internal static void FlattenJson(JsonObject parent, ref StringBuilder currentPat { FlattenJson(jObject, ref currentPath, ref fields); } - else if (property.Value is null || property.Value.GetValueKind() == JsonValueKind.Null) + else if (property.Value is null || IsJsonNull(property.Value)) { fields[currentPath.ToString()] = null; } @@ -177,7 +212,7 @@ internal static void WriteJsonArray(ExtendedBinaryWriter writer, JsonArray array { writer.Write((byte)0x1E); - var kind = array.Count > 0 ? array[0].GetValueKind() : JsonValueKind.Null; + var kind = array.Count > 0 ? GetJsonValueKind(array[0]) : JsonValueKind.Null; // Step 1: Write binary tag for array element type switch (kind) @@ -212,7 +247,7 @@ internal static void WriteJsonArray(ExtendedBinaryWriter writer, JsonArray array // Step 3: Write array elements foreach (var value in array) { - if (value.GetValueKind() != kind) + if (GetJsonValueKind(value) != kind) { throw new SerializationException("Array contains mixed value types"); } @@ -237,14 +272,14 @@ internal static void WriteJsonArray(ExtendedBinaryWriter writer, JsonArray array WriteJsonObject(writer, (JsonObject)value); break; default: - throw new SerializationException($"Unsupported JSON value kind: {value.GetValueKind()}"); + throw new SerializationException($"Unsupported JSON value kind: {GetJsonValueKind(value)}"); } } } internal static void WriteJsonValue(ExtendedBinaryWriter writer, JsonValue value) { - switch (value.GetValueKind()) + switch (GetJsonValueKind(value)) { case JsonValueKind.Undefined: case JsonValueKind.String: @@ -264,7 +299,7 @@ internal static void WriteJsonValue(ExtendedBinaryWriter writer, JsonValue value writer.Write((byte)0x00); break; default: - throw new SerializationException($"Unsupported JSON value kind: {value.GetValueKind()}"); + throw new SerializationException($"Unsupported JSON value kind: {GetJsonValueKind(value)}"); } } } From c23c1324c862f1093df3eabb123d5b63bc98d18c Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 18:08:24 -0400 Subject: [PATCH 03/15] In NET_FRAMEWORK TFMS, floating point string formats break unit tests due to scientific notation and epsilon differences --- .../Extensions/AssertionExtensions.cs | 38 +++++++++++++++++++ ClickHouse.Driver.Tests/ORM/DapperTests.cs | 3 +- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 ClickHouse.Driver.Tests/Extensions/AssertionExtensions.cs diff --git a/ClickHouse.Driver.Tests/Extensions/AssertionExtensions.cs b/ClickHouse.Driver.Tests/Extensions/AssertionExtensions.cs new file mode 100644 index 00000000..d1e6aec2 --- /dev/null +++ b/ClickHouse.Driver.Tests/Extensions/AssertionExtensions.cs @@ -0,0 +1,38 @@ +using System; +using System.Globalization; + +namespace ClickHouse.Driver.Tests.Extensions; + +internal static class AssertionExtensions +{ + private const double DefaultEpsilon = 1e-7; + + public static void AssertFloatingPointEquals(this string actualResult, object expectedValue, double epsilon = DefaultEpsilon) + { + switch (expectedValue) + { + case float @float: + float.Parse(actualResult, CultureInfo.InvariantCulture).AssertFloatingPointEquals(@float, (float)epsilon); + break; + case double @double: + double.Parse(actualResult, CultureInfo.InvariantCulture).AssertFloatingPointEquals(@double, epsilon); + break; + default: + var expected = Convert.ToString(expectedValue, CultureInfo.InvariantCulture); + Assert.That(actualResult, Is.EqualTo(expected)); + break; + } + } + + public static void AssertFloatingPointEquals(this double actual, double expected, double epsilon = DefaultEpsilon) + { + Assert.That(Math.Abs(actual - expected), Is.LessThan(epsilon), + $"Expected: {expected}, Actual: {actual}"); + } + + public static void AssertFloatingPointEquals(this float actual, float expected, float epsilon = (float)DefaultEpsilon) + { + Assert.That(Math.Abs(actual - expected), Is.LessThan(epsilon), + $"Expected: {expected}, Actual: {actual}"); + } +} diff --git a/ClickHouse.Driver.Tests/ORM/DapperTests.cs b/ClickHouse.Driver.Tests/ORM/DapperTests.cs index 5429d872..ad22573d 100644 --- a/ClickHouse.Driver.Tests/ORM/DapperTests.cs +++ b/ClickHouse.Driver.Tests/ORM/DapperTests.cs @@ -8,6 +8,7 @@ using System.Text.Json.Serialization; using System.Threading.Tasks; using ClickHouse.Driver.Numerics; +using ClickHouse.Driver.Tests.Extensions; using ClickHouse.Driver.Utility; using Dapper; using NUnit.Framework; @@ -130,7 +131,7 @@ public async Task ShouldExecuteSelectStringWithSingleParameterValue(string sql, } var parameters = new Dictionary { { "value", value } }; var results = await connection.QueryAsync(sql, parameters); - Assert.That(results.Single(), Is.EqualTo(Convert.ToString(value, CultureInfo.InvariantCulture))); + results.Single().AssertFloatingPointEquals(value); } [Test] From 7a623cad7c01a33a396edc4e8a2440c16741c56a Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 19:01:25 -0400 Subject: [PATCH 04/15] Dapper type handlers are not thread-safe This causes test failures when running multiple TFMs in parallel. We need to build a sync primitive around adding type handlers to avoid race conditions. --- ClickHouse.Driver.Tests/ORM/DapperTests.cs | 50 ++++++++++++++-------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/ClickHouse.Driver.Tests/ORM/DapperTests.cs b/ClickHouse.Driver.Tests/ORM/DapperTests.cs index ad22573d..4328692f 100644 --- a/ClickHouse.Driver.Tests/ORM/DapperTests.cs +++ b/ClickHouse.Driver.Tests/ORM/DapperTests.cs @@ -5,13 +5,11 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Text.Json.Nodes; -using System.Text.Json.Serialization; using System.Threading.Tasks; using ClickHouse.Driver.Numerics; using ClickHouse.Driver.Tests.Extensions; using ClickHouse.Driver.Utility; using Dapper; -using NUnit.Framework; namespace ClickHouse.Driver.Tests.ORM; @@ -23,15 +21,31 @@ public class DapperTests : AbstractConnectionTestFixture .Where(s => !s.ClickHouseType.StartsWith("Array")) // Dapper issue, see ShouldExecuteSelectWithParameters test .Select(sample => new TestCaseData($"SELECT {{value:{sample.ClickHouseType}}}", sample.ExampleValue)); - static DapperTests() + private static readonly object TypeHandlerLock = new(); + private static volatile bool _typeHandlersRegistered; + + public DapperTests() { - SqlMapper.AddTypeHandler(new ClickHouseDecimalHandler()); - SqlMapper.AddTypeHandler(new DateTimeOffsetHandler()); + lock (TypeHandlerLock) + { + if (_typeHandlersRegistered) + return; + + try + { + SqlMapper.AddTypeHandler(new ClickHouseDecimalHandler()); + SqlMapper.AddTypeHandler(new DateTimeOffsetHandler()); #if NET48 || NET5_0_OR_GREATER - SqlMapper.AddTypeHandler(new ITupleHandler()); + SqlMapper.AddTypeHandler(new ITupleHandler()); #endif - SqlMapper.AddTypeMap(typeof(DateTime), DbType.DateTime2); - SqlMapper.AddTypeMap(typeof(DateTimeOffset), DbType.DateTime2); + SqlMapper.AddTypeMap(typeof(DateTime), DbType.DateTime2); + SqlMapper.AddTypeMap(typeof(DateTimeOffset), DbType.DateTime2); + } + finally + { + _typeHandlersRegistered = true; + } + } } // "The member value of type cannot be used as a parameter value" @@ -110,7 +124,7 @@ private class ClickHouseDecimalHandler : SqlMapper.TypeHandler throw new ArgumentException(nameof(value)) }; } - + [Test] public async Task ShouldExecuteSimpleSelect() { @@ -194,14 +208,15 @@ public async Task ShouldExecuteSelectReturningDecimal() [TestCase(0.0001)] public async Task ShouldWriteDecimalWithTypeInference(decimal expected) { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.dapper_decimal"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.dapper_decimal (balance Decimal64(4)) ENGINE Memory"); + var tableName = SanitizeTableName("test.dapper_decimal"); + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {tableName}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {tableName} (balance Decimal64(4)) ENGINE Memory"); - var sql = @"INSERT INTO test.dapper_decimal (balance) VALUES (@balance)"; + var sql = $"INSERT INTO {tableName} (balance) VALUES (@balance)"; await connection.ExecuteAsync(sql, new { balance = expected }); - var actual = (ClickHouseDecimal) await connection.ExecuteScalarAsync("SELECT * FROM test.dapper_decimal"); + var actual = (ClickHouseDecimal) await connection.ExecuteScalarAsync($"SELECT * FROM {tableName}"); Assert.That(actual.ToDecimal(CultureInfo.InvariantCulture), Is.EqualTo(expected)); } @@ -220,13 +235,14 @@ public async Task ShouldWriteTwoFieldsWithTheSamePrefix() [TestCase(null)] public async Task ShouldWriteNullableDoubleWithTypeInference(double? expected) { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.dapper_nullable_double"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.dapper_nullable_double (balance Nullable(Float64)) ENGINE Memory"); + var tableName = SanitizeTableName("test.dapper_nullable_double"); + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {tableName}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {tableName} (balance Nullable(Float64)) ENGINE Memory"); - var sql = @"INSERT INTO test.dapper_nullable_double (balance) VALUES (@balance)"; + var sql = $"INSERT INTO {tableName} (balance) VALUES (@balance)"; await connection.ExecuteAsync(sql, new { balance = expected }); - var actual = await connection.ExecuteScalarAsync("SELECT * FROM test.dapper_nullable_double"); + var actual = await connection.ExecuteScalarAsync($"SELECT * FROM {tableName}"); if (expected is null) Assert.That(actual, Is.InstanceOf()); else From 3d63021be8f16cc6da0a4833b99fe15da3a1a331 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 19:16:13 -0400 Subject: [PATCH 05/15] Fix parallel test execution isolation for DapperContribTests --- .../ORM/DapperContribTests.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ClickHouse.Driver.Tests/ORM/DapperContribTests.cs b/ClickHouse.Driver.Tests/ORM/DapperContribTests.cs index 63ac84f2..49165d6a 100644 --- a/ClickHouse.Driver.Tests/ORM/DapperContribTests.cs +++ b/ClickHouse.Driver.Tests/ORM/DapperContribTests.cs @@ -2,7 +2,6 @@ using System.Threading.Tasks; using ClickHouse.Driver.Utility; using Dapper.Contrib.Extensions; -using NUnit.Framework; namespace ClickHouse.Driver.Tests.ORM; @@ -12,16 +11,23 @@ public class DapperContribTests : AbstractConnectionTestFixture // TODO: Non-UTC timezones // TODO: DateTimeTimeOffset private readonly static TestRecord referenceRecord = new(1, "value", new DateTime(2023, 4, 15, 1, 2, 3, DateTimeKind.Utc)); + private string tableName; - [Table("test.dapper_contrib")] public record class TestRecord(int Id, string Value, DateTime Timestamp); + [OneTimeSetUp] + public void ConfigureDapperContrib() + { + tableName = SanitizeTableName("test.dapper_contrib"); + SqlMapperExtensions.TableNameMapper = x => x == typeof(TestRecord) ? tableName : null; + } + [SetUp] public async Task SetUp() { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.dapper_contrib"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.dapper_contrib (Id Int32, Value String, Timestamp DateTime('UTC')) ENGINE Memory"); - await connection.ExecuteStatementAsync("INSERT INTO test.dapper_contrib VALUES (1, 'value', toDateTime('2023/04/15 01:02:03', 'UTC'))"); + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {tableName}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {tableName} (Id Int32, Value String, Timestamp DateTime('UTC')) ENGINE Memory"); + await connection.ExecuteStatementAsync($"INSERT INTO {tableName} VALUES (1, 'value', toDateTime('2023/04/15 01:02:03', 'UTC'))"); } [Test] From 90d14aecc61355b18f6dcd5af2b9ee4c576f4c1d Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 19:34:21 -0400 Subject: [PATCH 06/15] Fix test isolation for specific Query IDs --- ClickHouse.Driver.Tests/ADO/ConnectionTests.cs | 4 ++-- ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs index fc66b0b0..c8d7724a 100644 --- a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs +++ b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs @@ -97,7 +97,7 @@ public async Task ServerShouldSetQueryId() [Test] public async Task ClientShouldSetQueryId() { - string queryId = "MyQueryId123456"; + string queryId = GetUniqueQueryId("MyQueryId123456"); var command = connection.CreateCommand(); command.CommandText = "SELECT 1"; command.QueryId = queryId; @@ -119,7 +119,7 @@ public void ClientShouldSetUserAgent() public async Task ReplaceRunningQuerySettingShouldReplace() { connection.CustomSettings.Add("replace_running_query", 1); - string queryId = "MyQueryId123456"; + string queryId = GetUniqueQueryId("MyQueryId123456"); var command1 = connection.CreateCommand(); var command2 = connection.CreateCommand(); diff --git a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs index e64e186c..6a089855 100644 --- a/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs +++ b/ClickHouse.Driver.Tests/AbstractConnectionTestFixture.cs @@ -52,6 +52,12 @@ private static string GetFrameworkSuffix() #endif } + protected static string GetUniqueQueryId(string baseId) + { + var suffix = GetFrameworkSuffix(); + return !string.IsNullOrEmpty(suffix) ? $"{baseId}_{suffix}" : baseId; + } + [OneTimeTearDown] public void Dispose() => connection?.Dispose(); } From 2bc74f5baa7445fca21973b1e9c45626b3becb21 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 20:26:35 -0400 Subject: [PATCH 07/15] Fix .NET Framework TFMs blocking test execution due to different in HTTP handling between frameworks Try to match the .NET Core sockets parameters --- ClickHouse.Driver.Tests/TestSetup.cs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ClickHouse.Driver.Tests/TestSetup.cs diff --git a/ClickHouse.Driver.Tests/TestSetup.cs b/ClickHouse.Driver.Tests/TestSetup.cs new file mode 100644 index 00000000..f71bd725 --- /dev/null +++ b/ClickHouse.Driver.Tests/TestSetup.cs @@ -0,0 +1,20 @@ +using System.Net; + +namespace ClickHouse.Driver.Tests; + +[SetUpFixture] +public class TestSetup +{ + [OneTimeSetUp] + public void RunBeforeAnyTests() + { +#if NETFRAMEWORK + // In .NET Framework TFMs, we need to account for connection limits not present in other implementations + ServicePointManager.DefaultConnectionLimit = 1000; + ServicePointManager.Expect100Continue = false; + ServicePointManager.UseNagleAlgorithm = false; + ServicePointManager.MaxServicePointIdleTime = 10000; + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; +#endif + } +} From 4c82b9ce8590c3496d420b79b9c6961d51f562be Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 20:53:59 -0400 Subject: [PATCH 08/15] Fix port exhaustion in parallel test execution using a custom pool In .NET Framework TFMs, when running thousands of tests in parallel over HTTP, we can exhaust ports due to too many created connections. So we have to pool a reasonable number of them to unblock execution. The default pool implementation is still suspect, as it creates an HttpClient on every single request, obviating the benefit of a factory. --- .../TestPoolHttpClientFactory.cs | 50 +++++++++++++++++++ ClickHouse.Driver.Tests/TestUtilities.cs | 4 +- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 ClickHouse.Driver.Tests/Infrastructure/TestPoolHttpClientFactory.cs diff --git a/ClickHouse.Driver.Tests/Infrastructure/TestPoolHttpClientFactory.cs b/ClickHouse.Driver.Tests/Infrastructure/TestPoolHttpClientFactory.cs new file mode 100644 index 00000000..4899c72e --- /dev/null +++ b/ClickHouse.Driver.Tests/Infrastructure/TestPoolHttpClientFactory.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http; +using System.Threading; + +namespace ClickHouse.Driver.Tests.Infrastructure; + +/// +/// HttpClientFactory that uses connection pooling to prevent port exhaustion during heavy parallel load in .NET Framework TFMs. +/// +internal sealed class TestPoolHttpClientFactory : IHttpClientFactory +{ +#if NETFRAMEWORK + private const int PoolSize = 16; + private static readonly ConcurrentBag HandlerPool = new ConcurrentBag(); + private static readonly int[] Slots = new int[1]; + private static readonly HttpClientHandler[] Handlers; + + static TestPoolHttpClientFactory() + { + Handlers = new HttpClientHandler[PoolSize]; + for (int i = 0; i < PoolSize; i++) + { + var handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + MaxConnectionsPerServer = 100, + UseProxy = false + }; + Handlers[i] = handler; + HandlerPool.Add(handler); + } + } + + public TimeSpan Timeout { get; init; } = TimeSpan.FromMinutes(2); + + public HttpClient CreateClient(string name) + { + var index = (uint)Interlocked.Increment(ref Slots[0]) % PoolSize; + return new HttpClient(Handlers[index], false) { Timeout = Timeout }; + } +#else + private static readonly HttpClientHandler DefaultHttpClientHandler = new() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; + + public TimeSpan Timeout { get; init; } = TimeSpan.FromMinutes(2); + + public HttpClient CreateClient(string name) => new(DefaultHttpClientHandler, false) { Timeout = Timeout }; +#endif +} diff --git a/ClickHouse.Driver.Tests/TestUtilities.cs b/ClickHouse.Driver.Tests/TestUtilities.cs index 5419ba5a..3ea052c8 100644 --- a/ClickHouse.Driver.Tests/TestUtilities.cs +++ b/ClickHouse.Driver.Tests/TestUtilities.cs @@ -9,6 +9,7 @@ using ClickHouse.Driver.Numerics; using ClickHouse.Driver.Utility; using System.Text.Json.Nodes; +using ClickHouse.Driver.Tests.Infrastructure; namespace ClickHouse.Driver.Tests; @@ -70,7 +71,8 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression { builder["set_allow_experimental_dynamic_type"] = 1; } - var connection = new ClickHouseConnection(builder.ConnectionString); + + var connection = new ClickHouseConnection(builder.ConnectionString, new TestPoolHttpClientFactory()); connection.Open(); return connection; } From 51b3f4cc305c73581bb688eb46c37ff2aa5bc669 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 23:00:59 -0400 Subject: [PATCH 09/15] Implement fallback support for Tuples in net462 Tests fail in net462 TFM because the Tuple polyfill uses object arrays due to missing ITuple, which will not conform to the way these types are consumed in library code at runtime, causing multiple test failures, and likely issues for consumers using them. --- .../Formats/ExtendedBinaryWriterExtensions.cs | 42 ++++ .../Formats/HttpParameterFormatter.cs | 3 + ClickHouse.Driver/Types/ArrayType.cs | 7 + ClickHouse.Driver/Types/NestedType.cs | 6 + ClickHouse.Driver/Types/TupleType.cs | 17 +- ClickHouse.Driver/Utility/TupleHelper.cs | 222 ++++++++++++++++++ 6 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 ClickHouse.Driver/Formats/ExtendedBinaryWriterExtensions.cs create mode 100644 ClickHouse.Driver/Utility/TupleHelper.cs diff --git a/ClickHouse.Driver/Formats/ExtendedBinaryWriterExtensions.cs b/ClickHouse.Driver/Formats/ExtendedBinaryWriterExtensions.cs new file mode 100644 index 00000000..799c0f84 --- /dev/null +++ b/ClickHouse.Driver/Formats/ExtendedBinaryWriterExtensions.cs @@ -0,0 +1,42 @@ +#if NET462 +using System; +using ClickHouse.Driver.Types; +using ClickHouse.Driver.Utility; + +namespace ClickHouse.Driver.Formats; + +internal static class ExtendedBinaryWriterExtensions +{ + public static void WriteTuple(this ExtendedBinaryWriter writer, object value, ClickHouseType[] underlyingTypes) + { + if (value == null) + throw new ArgumentNullException(nameof(value)); + + var type = value.GetType(); + var length = TupleHelper.GetTupleLength(type); + if (length != underlyingTypes.Length) + throw new ArgumentException("Wrong number of elements in Tuple", nameof(value)); + + var properties = TupleHelper.GetTuplePropertiesWithRest(type); + + for (var i = 0; i < underlyingTypes.Length; i++) + { + var property = properties[i]; + if (property == null) + throw new ArgumentException($"Property for index {i} not found on tuple type {type}", nameof(value)); + + var itemValue = property.GetValue(value); + + // Rest returns Tuple + if (i == 7 && property.Name == "Rest" && itemValue != null && TupleHelper.IsTupleType(itemValue.GetType())) + { + var restProperties = TupleHelper.GetTupleProperties(itemValue.GetType()); + if (restProperties.Length > 0) + itemValue = restProperties[0].GetValue(itemValue); + } + + underlyingTypes[i].Write(writer, itemValue); + } + } +} +#endif diff --git a/ClickHouse.Driver/Formats/HttpParameterFormatter.cs b/ClickHouse.Driver/Formats/HttpParameterFormatter.cs index 1ed99769..5743f596 100644 --- a/ClickHouse.Driver/Formats/HttpParameterFormatter.cs +++ b/ClickHouse.Driver/Formats/HttpParameterFormatter.cs @@ -85,6 +85,9 @@ internal static string Format(ClickHouseType type, object value, bool quote) #if !NET462 case TupleType tupleType when value is ITuple tuple: return $"({string.Join(",", tupleType.UnderlyingTypes.Select((x, i) => Format(x, tuple[i], true)))})"; +#else + case TupleType tupleType when TupleHelper.IsTupleType(value?.GetType()): + return TupleHelper.FormatTuple(value, tupleType.UnderlyingTypes, Format, NullValueString); #endif case TupleType tupleType when value is IList list: diff --git a/ClickHouse.Driver/Types/ArrayType.cs b/ClickHouse.Driver/Types/ArrayType.cs index 0615bd27..39284866 100644 --- a/ClickHouse.Driver/Types/ArrayType.cs +++ b/ClickHouse.Driver/Types/ArrayType.cs @@ -2,6 +2,7 @@ using System.Collections; using ClickHouse.Driver.Formats; using ClickHouse.Driver.Types.Grammar; +using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Types; @@ -26,6 +27,12 @@ public override ParameterizedType Parse(SyntaxTreeNode node, Func + typeArgs[7] = typeof(Tuple<>).MakeGenericType(typeArgs[7]); +#endif + var genericType = Type.GetType("System.Tuple`" + typeArgs.Length); return genericType.MakeGenericType(typeArgs); } @@ -90,7 +99,7 @@ public override object Read(ExtendedBinaryReader reader) #if !NET462 return MakeTuple(contents); #else - return contents; + return TupleHelper.CreateTuple(contents, frameworkType, UnderlyingTypes); #endif } @@ -107,6 +116,12 @@ public override void Write(ExtendedBinaryWriter writer, object value) } return; } +#else + if (value != null && TupleHelper.IsTupleType(value.GetType())) + { + writer.WriteTuple(value, UnderlyingTypes); + return; + } #endif if (value is IList list) { diff --git a/ClickHouse.Driver/Utility/TupleHelper.cs b/ClickHouse.Driver/Utility/TupleHelper.cs new file mode 100644 index 00000000..e33ce8fa --- /dev/null +++ b/ClickHouse.Driver/Utility/TupleHelper.cs @@ -0,0 +1,222 @@ +#if NET462 +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Reflection; +using ClickHouse.Driver.Formats; +using ClickHouse.Driver.Types; + +namespace ClickHouse.Driver.Utility; + +/// +/// Compatibility shim for Tuple in .NET 4.6.2 +/// For now, only eight elements are supported, with the eighth element sourced from `Rest`. +/// +/// +internal static class TupleHelper +{ + private static readonly ConcurrentDictionary TypeCache = new(); + private static readonly ConcurrentDictionary PropertyCache = new(); + + public static bool IsTupleType(Type type) + { + if (type == null) return false; + + return TypeCache.GetOrAdd(type, t => + { + if (!t.IsGenericType) + return false; + + var definition = t.GetGenericTypeDefinition(); + return definition == typeof(Tuple<>) || + definition == typeof(Tuple<,>) || + definition == typeof(Tuple<,,>) || + definition == typeof(Tuple<,,,>) || + definition == typeof(Tuple<,,,,>) || + definition == typeof(Tuple<,,,,,>) || + definition == typeof(Tuple<,,,,,,>) || + definition == typeof(Tuple<,,,,,,,>); + }); + } + + public static PropertyInfo[] GetTupleProperties(Type type) + { + return PropertyCache.GetOrAdd(type, t => + { + var properties = new List(); + + for (var i = 1; i <= 8; i++) + { + var property = t.GetProperty($"Item{i}"); + if (property == null) + break; + + properties.Add(property); + } + return properties.ToArray(); + }); + } + + public static PropertyInfo[] GetTuplePropertiesWithRest(Type type) + { + return PropertyCache.GetOrAdd(type, t => + { + var properties = new PropertyInfo[8]; + var length = t.GetGenericArguments().Length; + + for (var i = 0; i < Math.Min(length, 8); i++) + { + var propertyName = i == 7 && length == 8 ? "Rest" : $"Item{i + 1}"; + properties[i] = t.GetProperty(propertyName); + } + + return properties; + }); + } + + public static int GetTupleLength(Type tupleType) + { + if (!IsTupleType(tupleType)) + return 0; + + var arguments = tupleType.GetGenericArguments(); + if (arguments.Length == 8 && IsTupleType(arguments[7])) + return 8; + + return arguments.Length; + } + + public static string FormatTuple(object value, ClickHouseType[] underlyingTypes, Func formatter, string nullValue) + { + if (value == null) + return nullValue; + + var type = value.GetType(); + var properties = GetTupleProperties(type); + + var items = new List(); + var count = Math.Min(properties.Length, underlyingTypes.Length); + + for (var i = 0; i < count; i++) + { + var itemValue = properties[i].GetValue(value); + items.Add(formatter(underlyingTypes[i], itemValue, true)); + } + + return $"({string.Join(",", items)})"; + } + + public static object CreateTuple(object[] values, Type frameworkType, ClickHouseType[] underlyingTypes) + { + var count = values.Length; + if (count > 8) + return values; + + var arguments = frameworkType.GetGenericArguments(); + var typedValues = new object[count]; + for (var i = 0; i < count; i++) + { + var expectedType = arguments[i]; + var value = values[i]; + + if (i == 7 && count == 8) + expectedType = expectedType.GetGenericArguments()[0]; + + if (value is null or DBNull) + { + typedValues[i] = null; + } + else if (expectedType.IsInstanceOfType(value)) + { + typedValues[i] = value; + } + else if (expectedType.IsGenericType && + expectedType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + var underlyingType = Nullable.GetUnderlyingType(expectedType); + if (underlyingType != null) + typedValues[i] = Convert.ChangeType(value, underlyingType, CultureInfo.InvariantCulture); + else + typedValues[i] = null; + } + else if (IsTupleType(expectedType) && value is object[] nested) + { + if (underlyingTypes[i] is TupleType nestedTupleType) + { + typedValues[i] = CreateTuple(nested, nestedTupleType.FrameworkType, nestedTupleType.UnderlyingTypes); + } + else + { + typedValues[i] = value; + } + } + else + { + try + { + typedValues[i] = Convert.ChangeType(value, expectedType, CultureInfo.InvariantCulture); + } + catch + { + typedValues[i] = value; + } + } + } + + if (count == 8) // Tuple + { + var wrapped = new object[8]; + Array.Copy(typedValues, 0, wrapped, 0, 7); + wrapped[7] = Activator.CreateInstance(arguments[7], typedValues[7]); + return Activator.CreateInstance(frameworkType, wrapped); + } + + return Activator.CreateInstance(frameworkType, typedValues); + } + + public static Array ReadArrayWithRuntimeType(ExtendedBinaryReader reader, int length, ClickHouseType elementType, Type fallbackFrameworkType) + { + var values = new object[length]; + for (var i = 0; i < length; i++) + { + var value = elementType.Read(reader); + values[i] = value is DBNull ? null : value; + } + + if (length > 0 && values[0] != null) + { + var typedArray = Array.CreateInstance(values[0].GetType(), length); + for (var i = 0; i < length; i++) + { + typedArray.SetValue(values[i], i); + } + return typedArray; + } + + return values; + } + + public static Array ReadNestedArrayWithRuntimeType(ExtendedBinaryReader reader, int length, TupleType tupleType) + { + var values = new object[length]; + for (var i = 0; i < length; i++) + { + var value = tupleType.Read(reader); + values[i] = value is DBNull ? null : value; + } + + if (length > 0 && values[0] != null) + { + var typedArray = Array.CreateInstance(values[0].GetType(), length); + for (var i = 0; i < length; i++) + { + typedArray.SetValue(values[i], i); + } + return typedArray; + } + + return values; + } +} +#endif From 69abc16a9d4980f2d2a637c0caaf7848ff034521 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 23:01:47 -0400 Subject: [PATCH 10/15] Fix an issue where tests can fail if you run them close to midnight, due to differences in CH vs. client wall time --- ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs index 5b2ee3cf..64e3a80c 100644 --- a/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs +++ b/ClickHouse.Driver.Tests/BulkCopy/BulkCopyWithDefaultsTests.cs @@ -36,6 +36,10 @@ public async Task ShouldExecuteSingleValueInsertViaBulkCopyWithDefaults(string c await connection.ExecuteStatementAsync( $"CREATE TABLE IF NOT EXISTS {targetTable} (`value` {clickhouseType}) ENGINE Memory"); + // Use server time, otherwise a mismatch between the backing instance and the running client can cause false test failures + if (clickhouseType.Contains("toDate(now())") && insertValue == DBDefault.Value && expectedValue is DateTime time && time == DateTime.Today) + expectedValue = await connection.ExecuteScalarAsync("SELECT toDate(now())"); + using var bulkCopyWithDefaults = new ClickHouseBulkCopy(connection, RowBinaryFormat.RowBinaryWithDefaults) { DestinationTableName = targetTable, From ad0d5c59cb1fd53b827de459b9353099a234f7f1 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 23:39:05 -0400 Subject: [PATCH 11/15] Fix recursion bug in TupleHelper --- ClickHouse.Driver/Utility/TupleHelper.cs | 58 ++++++++++++------------ 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/ClickHouse.Driver/Utility/TupleHelper.cs b/ClickHouse.Driver/Utility/TupleHelper.cs index e33ce8fa..35c0ad08 100644 --- a/ClickHouse.Driver/Utility/TupleHelper.cs +++ b/ClickHouse.Driver/Utility/TupleHelper.cs @@ -164,13 +164,14 @@ public static object CreateTuple(object[] values, Type frameworkType, ClickHouse } } - if (count == 8) // Tuple - { - var wrapped = new object[8]; - Array.Copy(typedValues, 0, wrapped, 0, 7); - wrapped[7] = Activator.CreateInstance(arguments[7], typedValues[7]); - return Activator.CreateInstance(frameworkType, wrapped); - } + if (count != 8) + return Activator.CreateInstance(frameworkType, typedValues); + + // Tuple + var wrapped = new object[8]; + Array.Copy(typedValues, 0, wrapped, 0, 7); + wrapped[7] = Activator.CreateInstance(arguments[7], typedValues[7]); + typedValues = wrapped; return Activator.CreateInstance(frameworkType, typedValues); } @@ -184,17 +185,14 @@ public static Array ReadArrayWithRuntimeType(ExtendedBinaryReader reader, int le values[i] = value is DBNull ? null : value; } - if (length > 0 && values[0] != null) - { - var typedArray = Array.CreateInstance(values[0].GetType(), length); - for (var i = 0; i < length; i++) - { - typedArray.SetValue(values[i], i); - } - return typedArray; - } + if (length == 0 || values[0] == null) + return values; + + var typedArray = Array.CreateInstance(values[0].GetType(), length); + for (var i = 0; i < length; i++) + typedArray.SetValue(values[i], i); - return values; + return typedArray; } public static Array ReadNestedArrayWithRuntimeType(ExtendedBinaryReader reader, int length, TupleType tupleType) @@ -202,21 +200,25 @@ public static Array ReadNestedArrayWithRuntimeType(ExtendedBinaryReader reader, var values = new object[length]; for (var i = 0; i < length; i++) { - var value = tupleType.Read(reader); - values[i] = value is DBNull ? null : value; - } - - if (length > 0 && values[0] != null) - { - var typedArray = Array.CreateInstance(values[0].GetType(), length); - for (var i = 0; i < length; i++) + var count = tupleType.UnderlyingTypes.Length; + var contents = new object[count]; + for (var j = 0; j < count; j++) { - typedArray.SetValue(values[i], i); + var value = tupleType.UnderlyingTypes[j].Read(reader); + contents[j] = value is DBNull ? null : value; } - return typedArray; + var type = tupleType.FrameworkType.IsArray ? tupleType.FrameworkType.GetElementType() : tupleType.FrameworkType; + values[i] = CreateTuple(contents, type, tupleType.UnderlyingTypes); } - return values; + if (length == 0 || values[0] == null) + return values; + + var typedArray = Array.CreateInstance(values[0].GetType(), length); + for (var i = 0; i < length; i++) + typedArray.SetValue(values[i], i); + + return typedArray; } } #endif From aa401adcfd9f9d3202cb5baa73ed5a027ac43448 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 23:56:07 -0400 Subject: [PATCH 12/15] Fix race condition on timeout exception criteria On .NET Framework TFMs, due to tight timeout windows, the internal WebException on request timeout might throw before TaskCanceledException does, causing a false failure. --- ClickHouse.Driver.Tests/ADO/ConnectionTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs index c8d7724a..becac2ca 100644 --- a/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs +++ b/ClickHouse.Driver.Tests/ADO/ConnectionTests.cs @@ -79,6 +79,12 @@ public async Task TimeoutShouldCancelConnection() _ = await task; Assert.Fail("The task should have been cancelled before completion"); } +#if NETFRAMEWORK + catch (WebException ex) when (ex.Status == WebExceptionStatus.RequestCanceled) + { + /* Expected: request cancelled */ + } +#endif catch (TaskCanceledException) { /* Expected: task cancelled */ From fc9351567621e566b312dc833188b986e20abced Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Wed, 6 Aug 2025 09:54:09 -0400 Subject: [PATCH 13/15] Fixed another case of parallel table name conflict --- .../SQL/ParameterizedInsertTests.cs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs index a26b09ed..47c8a6c9 100644 --- a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs +++ b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs @@ -12,30 +12,32 @@ public class ParameterizedInsertTests : AbstractConnectionTestFixture [Test] public async Task ShouldInsertParameterizedFloat64Array() { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.float_array"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.float_array (arr Array(Float64)) ENGINE Memory"); + var targetTable = $"test.{SanitizeTableName("float_array")}"; + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (arr Array(Float64)) ENGINE Memory"); var command = connection.CreateCommand(); command.AddParameter("values", new[] { 1.0, 2.0, 3.0 }); - command.CommandText = "INSERT INTO test.float_array VALUES ({values:Array(Float32)})"; + command.CommandText = $"INSERT INTO {targetTable} VALUES ({{values:Array(Float32)}})"; await command.ExecuteNonQueryAsync(); - var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM test.float_array"); + var count = await connection.ExecuteScalarAsync($"SELECT COUNT(*) FROM {targetTable}"); Assert.That(count, Is.EqualTo(1)); } [Test] public async Task ShouldInsertEnum8() { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.insert_enum8"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.insert_enum8 (enum Enum8('a' = -1, 'b' = 127)) ENGINE Memory"); + var targetTable = $"test.{SanitizeTableName("insert_enum8")}"; + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (enum Enum8('a' = -1, 'b' = 127)) ENGINE Memory"); var command = connection.CreateCommand(); command.AddParameter("value", "a"); - command.CommandText = "INSERT INTO test.insert_enum8 VALUES ({value:Enum8('a' = -1, 'b' = 127)})"; + command.CommandText = $"INSERT INTO {targetTable} VALUES ({{value:Enum8('a' = -1, 'b' = 127)}})"; await command.ExecuteNonQueryAsync(); - var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM test.insert_enum8"); + var count = await connection.ExecuteScalarAsync($"SELECT COUNT(*) FROM {targetTable}"); Assert.That(count, Is.EqualTo(1)); } @@ -43,35 +45,37 @@ public async Task ShouldInsertEnum8() [RequiredFeature(Feature.UUIDParameters)] public async Task ShouldInsertParameterizedUUIDArray() { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.uuid_array"); + var targetTable = $"test.{SanitizeTableName("uuid_array")}"; + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync( - "CREATE TABLE IF NOT EXISTS test.uuid_array (arr Array(UUID)) ENGINE Memory"); + $"CREATE TABLE IF NOT EXISTS {targetTable} (arr Array(UUID)) ENGINE Memory"); var command = connection.CreateCommand(); command.AddParameter("values", new[] { Guid.NewGuid(), Guid.NewGuid(), }); - command.CommandText = "INSERT INTO test.uuid_array VALUES ({values:Array(UUID)})"; + command.CommandText = $"INSERT INTO {targetTable} VALUES ({{values:Array(UUID)}})"; await command.ExecuteNonQueryAsync(); - var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM test.uuid_array"); + var count = await connection.ExecuteScalarAsync($"SELECT COUNT(*) FROM {targetTable}"); Assert.That(count, Is.EqualTo(1)); } [Test] public async Task ShouldInsertStringWithNewline() { - await connection.ExecuteStatementAsync("TRUNCATE TABLE IF EXISTS test.string_with_newline"); + var targetTable = $"test.{SanitizeTableName("string_with_newline")}"; + await connection.ExecuteStatementAsync($"TRUNCATE TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync( - "CREATE TABLE IF NOT EXISTS test.string_with_newline (str_value String) ENGINE Memory"); + $"CREATE TABLE IF NOT EXISTS {targetTable} (str_value String) ENGINE Memory"); var command = connection.CreateCommand(); var strValue = "Hello \n ClickHouse"; command.AddParameter("str_value", strValue); - command.CommandText = "INSERT INTO test.string_with_newline VALUES ({str_value:String})"; + command.CommandText = $"INSERT INTO {targetTable} VALUES ({{str_value:String}})"; await command.ExecuteNonQueryAsync(); - var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM test.string_with_newline"); + var count = await connection.ExecuteScalarAsync($"SELECT COUNT(*) FROM {targetTable}"); Assert.That(count, Is.EqualTo(1)); } } From 9cb1203069929f4caa4f6d56b936c54cbc3475ee Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Fri, 7 Nov 2025 15:07:56 +0100 Subject: [PATCH 14/15] fix table name sanitization --- .../SQL/ParameterizedInsertTests.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs index 75b6ebf5..168967de 100644 --- a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs +++ b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs @@ -13,8 +13,8 @@ public class ParameterizedInsertTests : AbstractConnectionTestFixture public async Task ShouldInsertParameterizedFloat64Array() { var targetTable = $"test.{SanitizeTableName("float_array")}"; - await connection.ExecuteStatementAsync("DROP TABLE IF EXISTS test.float_array"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.float_array (arr Array(Float64)) ENGINE Memory"); + await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (arr Array(Float64)) ENGINE Memory"); var command = connection.CreateCommand(); command.AddParameter("values", new[] { 1.0, 2.0, 3.0 }); @@ -29,8 +29,8 @@ public async Task ShouldInsertParameterizedFloat64Array() public async Task ShouldInsertEnum8() { var targetTable = $"test.{SanitizeTableName("insert_enum8")}"; - await connection.ExecuteStatementAsync("DROP TABLE IF EXISTS test.insert_enum8"); - await connection.ExecuteStatementAsync("CREATE TABLE IF NOT EXISTS test.insert_enum8 (enum Enum8('a' = -1, 'b' = 127)) ENGINE Memory"); + await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); + await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (enum Enum8('a' = -1, 'b' = 127)) ENGINE Memory"); var command = connection.CreateCommand(); command.AddParameter("value", "a"); @@ -63,7 +63,7 @@ await connection.ExecuteStatementAsync( public async Task ShouldInsertStringWithNewline() { var targetTable = $"test.{SanitizeTableName("string_with_newline")}"; - await connection.ExecuteStatementAsync("DROP TABLE IF EXISTS test.string_with_newline"); + await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); await connection.ExecuteStatementAsync( $"CREATE TABLE IF NOT EXISTS {targetTable} (str_value String) ENGINE Memory"); @@ -82,9 +82,10 @@ await connection.ExecuteStatementAsync( [Test] public async Task ShouldInsertWithExceptSyntax() { - await connection.ExecuteStatementAsync("DROP TABLE IF EXISTS test.insert_except"); - await connection.ExecuteStatementAsync(@" - CREATE TABLE IF NOT EXISTS test.insert_except ( + var targetTable = $"test.{SanitizeTableName("insert_except")}"; + await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}"); + await connection.ExecuteStatementAsync($@" + CREATE TABLE IF NOT EXISTS {targetTable} ( id Int32, name String, value Float64, From 8e127772a9de272fe279e35243d7a9e2956a47ca Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Fri, 7 Nov 2025 15:19:38 +0100 Subject: [PATCH 15/15] table name sanitization fix --- ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs index 168967de..14e5ddd4 100644 --- a/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs +++ b/ClickHouse.Driver.Tests/SQL/ParameterizedInsertTests.cs @@ -99,14 +99,14 @@ updated DateTime DEFAULT now() command.AddParameter("id", 42); command.AddParameter("name", "test-except"); command.AddParameter("value", 99.99); - command.CommandText = "INSERT INTO test.insert_except (* EXCEPT (created, updated)) VALUES ({id:Int32}, {name:String}, {value:Float64})"; + command.CommandText = $"INSERT INTO {targetTable} (* EXCEPT (created, updated)) VALUES " + "({id:Int32}, {name:String}, {value:Float64})"; await command.ExecuteNonQueryAsync(); - var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM test.insert_except"); + var count = await connection.ExecuteScalarAsync($"SELECT COUNT(*) FROM {targetTable}"); Assert.That(count, Is.EqualTo(1)); // Verify all columns including defaults using SELECT * - using var reader = await connection.ExecuteReaderAsync("SELECT * FROM test.insert_except"); + using var reader = await connection.ExecuteReaderAsync($"SELECT * FROM {targetTable}"); Assert.That(reader.Read(), Is.True); Assert.That(reader.FieldCount, Is.EqualTo(5)); Assert.That(reader.GetInt32(0), Is.EqualTo(42));