Json, Dynamic, and type parsing improvements (#95)

* Add BinaryTypeDecoder for binary type parsing

* Improve Json support

* Expand test cases with new types and composite type generation

* Expand and improve Json type tests

* Expand and improve Dynamic tests

* re-enable connection test

* Improve json binary writing of various numeric types, and add more json tests

* Pass type settings when reading dynamic type

* Update release notes

Author: alex-clickhouse

ClickHouse.Driver.Tests/ADO/ConnectionTests.cs

@@ -177,7 +177,6 @@ public void ShouldFetchSchemaTables()
     }
 
     [Test]
-    [Ignore("Needs support for named tuple parameters")]
     public void ShouldFetchSchemaDatabaseColumns()
     {
         var schema = connection.GetSchema("Columns", ["system"]);

ClickHouse.Driver.Tests/BulkCopy/BulkCopyTests.cs

@@ -497,6 +497,114 @@ public async Task ShouldInsertJson()
         }
     }
 
+    [Test]
+    [RequiredFeature(Feature.Json)]
+    public async Task ShouldInsertJsonWithComplexTypes()
+    {
+        var targetTable = "test." + SanitizeTableName($"bulk_json_complex");
+        await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}");
+        await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory");
+
+        using var bulkCopy = new ClickHouseBulkCopy(connection)
+        {
+            DestinationTableName = targetTable,
+        };
+
+        var jsonObject = new JsonObject
+        {
+            ["boolValue"] = true,
+            ["numberValue"] = 42.5,
+            ["stringValue"] = "hello",
+            ["nullValue"] = null,
+            ["arrayValueDouble"] = new JsonArray(1.5, 2.5, 3.5),
+            ["arrayValueInt"] = new JsonArray(1, 2, 3),
+            ["nestedObject"] = new JsonObject
+            {
+                ["innerString"] = "nested",
+                ["innerNumber"] = 123,
+                ["innerLong"] = 124L,
+            },
+        };
+
+        await bulkCopy.InitAsync();
+        await bulkCopy.WriteToServerAsync([[jsonObject]]);
+
+        using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}");
+        Assert.That(reader.Read(), Is.True);
+
+        var result = (JsonObject)reader.GetValue(0);
+
+        Assert.That((bool)result["boolValue"], Is.EqualTo(true));
+        Assert.That((double)result["numberValue"], Is.EqualTo(42.5));
+        Assert.That((string)result["stringValue"], Is.EqualTo("hello"));
+        Assert.That(result["nullValue"], Is.Null);
+        Assert.That(JsonNode.DeepEquals(result["arrayValueDouble"], new JsonArray(1.5, 2.5, 3.5)), Is.True);
+        Assert.That(JsonNode.DeepEquals(result["arrayValueInt"], new JsonArray(1, 2, 3)), Is.True);
+        Assert.That(JsonNode.DeepEquals(result["nestedObject"], new JsonObject { ["innerString"] = "nested", ["innerNumber"] = 123, ["innerLong"] = 124L }), Is.True);
+
+        Assert.That(reader.Read(), Is.False);
+    }
+
+    [Test]
+    [RequiredFeature(Feature.Json)]
+    public async Task ShouldInsertJsonFromString()
+    {
+        var targetTable = "test." + SanitizeTableName($"bulk_json_string");
+        await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}");
+        await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory");
+
+        using var bulkCopy = new ClickHouseBulkCopy(connection)
+        {
+            DestinationTableName = targetTable,
+        };
+
+        var jsonString = "{\"name\": \"test\", \"count\": 42, \"active\": true}";
+
+        await bulkCopy.InitAsync();
+        await bulkCopy.WriteToServerAsync([[jsonString]]);
+
+        using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}");
+        Assert.That(reader.Read(), Is.True);
+
+        var result = (JsonObject)reader.GetValue(0);
+
+        Assert.That((string)result["name"], Is.EqualTo("test"));
+        Assert.That((long)result["count"], Is.EqualTo(42));
+        Assert.That((bool)result["active"], Is.EqualTo(true));
+
+        Assert.That(reader.Read(), Is.False);
+    }
+
+    [Test]
+    [RequiredFeature(Feature.Json)]
+    public async Task ShouldInsertJsonFromAnonymousObject()
+    {
+        var targetTable = "test." + SanitizeTableName($"bulk_json_anon");
+        await connection.ExecuteStatementAsync($"DROP TABLE IF EXISTS {targetTable}");
+        await connection.ExecuteStatementAsync($"CREATE TABLE IF NOT EXISTS {targetTable} (value JSON) ENGINE Memory");
+
+        using var bulkCopy = new ClickHouseBulkCopy(connection)
+        {
+            DestinationTableName = targetTable,
+        };
+
+        var obj = new { name = "test", count = 42, active = true, arrayBool = new bool[] { true, false} };
+
+        await bulkCopy.InitAsync();
+        await bulkCopy.WriteToServerAsync([[obj]]);
+
+        using var reader = await connection.ExecuteReaderAsync($"SELECT * from {targetTable}");
+        Assert.That(reader.Read(), Is.True);
+
+        var result = (JsonObject)reader.GetValue(0);
+
+        Assert.That((string)result["name"], Is.EqualTo("test"));
+        Assert.That((long)result["count"], Is.EqualTo(42));
+        Assert.That((bool)result["active"], Is.EqualTo(true));
+        Assert.That(JsonNode.DeepEquals(result["arrayBool"], new JsonArray(true, false)), Is.True);
+
+        Assert.That(reader.Read(), Is.False);
+    }
 
     [Test]
     public async Task ShouldInsertTupleWithEnum()

ClickHouse.Driver.Tests/SQL/SqlParameterizedSelectTests.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using ClickHouse.Driver.ADO;
+using ClickHouse.Driver.Types;
 using ClickHouse.Driver.Utility;
 using NUnit.Framework;
 
@@ -40,7 +41,7 @@ public async Task ShouldExecuteParameterizedCompareWithTypeDetection(string exam
         using var command = connection.CreateCommand();
         command.CommandText = $"SELECT {exampleExpression} as expected, {{var:{clickHouseType}}} as actual, expected = actual as equals";
         command.AddParameter("var", value);
-
+//TODO step through this test and see exactly what happens with HttpParameterFormatter and ClickHouseDbParameter, they seem to be doing duplicate work
         var result = (await command.ExecuteReaderAsync()).GetEnsureSingleRow();
         TestUtilities.AssertEqual(result[0], result[1]);
 
@@ -74,6 +75,7 @@ public async Task ShouldExecuteParameterizedCompareWithExplicitType(string examp
     {
         if (clickHouseType.StartsWith("Enum"))
             clickHouseType = "String";
+
         using var command = connection.CreateCommand();
         command.CommandText = $"SELECT {exampleExpression} as expected, {{var:{clickHouseType}}} as actual, expected = actual as equals";
         command.AddParameter("var", clickHouseType, value);

ClickHouse.Driver.Tests/Types/DynamicTests.cs

@@ -1,25 +1,87 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
+using System.Numerics;
+using System.Text.Json;
+using System.Text.Json.Nodes;
 using System.Threading.Tasks;
 using ClickHouse.Driver.ADO;
 using ClickHouse.Driver.ADO.Readers;
+using ClickHouse.Driver.Numerics;
 using ClickHouse.Driver.Tests.Attributes;
+using ClickHouse.Driver.Types;
 using ClickHouse.Driver.Utility;
 
 namespace ClickHouse.Driver.Tests.Types;
 
 public class DynamicTests : AbstractConnectionTestFixture
 {
+    public static IEnumerable<TestCaseData> DirectDynamicCastQueries
+    {
+        get
+        {
+            foreach (var sample in TestUtilities.GetDataTypeSamples().Where(s => ShouldBeSupportedInDynamic(s.ClickHouseType)))
+            {
+                yield return new TestCaseData(sample.ExampleExpression, sample.ClickHouseType, sample.ExampleValue)
+                    .SetName($"Direct_{sample.ClickHouseType}_{sample.ExampleValue}");
+            }
+
+            // Some additional test cases for dynamic specifically
+            // JSON with complex type hints
+            yield return new TestCaseData(
+                "'{\"a\": 1}'",
+                "Json(max_dynamic_paths=10, max_dynamic_types=3, a Int64, SKIP path.to.skip, SKIP REGEXP 'regex.path.*')",
+                new JsonObject { ["a"] = 1L }
+            ).SetName("Direct_Json_Complex");
+            
+            yield return new TestCaseData(
+                "1::Int32",
+                "Dynamic",
+                1
+            ).SetName("Nested_Dynamic");
+        }
+    }
+
+    [Test]
+    [RequiredFeature(Feature.Dynamic)]
+    [TestCaseSource(typeof(DynamicTests), nameof(DirectDynamicCastQueries))]
+    public async Task ShouldParseDirectDynamicCast(string valueSql, string clickHouseType, object expectedValue)
+    {
+        // Direct cast to Dynamic without going through JSON
+        using var reader =
+            (ClickHouseDataReader)await connection.ExecuteReaderAsync(
+                $"SELECT ({valueSql}::{clickHouseType})::Dynamic");
+
+        ClassicAssert.IsTrue(reader.Read());
+        var result = reader.GetValue(0);
+        TestUtilities.AssertEqual(expectedValue, result);
+        ClassicAssert.IsFalse(reader.Read());
+    }
+
+    private static bool ShouldBeSupportedInDynamic(string clickHouseType)
+    {
+        // Geo types not supported
+        if (clickHouseType is "Point" or "Ring" or "Polygon" or "MultiPolygon" or "Nothing")
+        {
+            return false;
+        }
+
+        return true;
+    }
+
     public static IEnumerable<TestCaseData> SimpleSelectQueries => TestUtilities.GetDataTypeSamples()
         .Where(s => ShouldBeSupportedInJson(s.ClickHouseType))
-        .Select(sample => GetTestCaseData(sample.ExampleExpression, sample.ClickHouseType, sample.ExampleValue));
+        .Select(sample => GetTestCaseData(sample.ExampleExpression, sample.ClickHouseType, sample.ExampleValue))
+        .Where(x => x != null);
 
     [Test]
     [RequiredFeature(Feature.Dynamic)]
     [TestCaseSource(typeof(DynamicTests), nameof(SimpleSelectQueries))]
-    public async Task ShouldMatchFrameworkType(string valueSql, Type frameworkType)
+    public async Task ShouldMatchFrameworkTypeViaJson(string valueSql, Type frameworkType)
     {
+        // This query returns the value as Dynamic type via JSON. The dynamicType may or may not match the actual type provided.
+        // eg IPv4 will be a String.
         using var reader =
             (ClickHouseDataReader) await connection.ExecuteReaderAsync(
                 $"select json.value from (select map('value', {valueSql})::JSON as json)");
@@ -37,6 +99,11 @@ private static TestCaseData GetTestCaseData(string exampleExpression, string cli
             return new TestCaseData(exampleExpression, typeof(DateTime));
         }
 
+        if (clickHouseType.StartsWith("Time"))
+        {
+            return new TestCaseData(exampleExpression, typeof(string));
+        }
+
         if (clickHouseType.StartsWith("Int") || clickHouseType.StartsWith("UInt"))
         {
             return new TestCaseData(exampleExpression, typeof(long));
@@ -46,11 +113,6 @@ private static TestCaseData GetTestCaseData(string exampleExpression, string cli
         {
             return new TestCaseData(exampleExpression, typeof(string));
         }
-
-        if (clickHouseType.StartsWith("Time"))
-        {
-            return new TestCaseData(exampleExpression, typeof(TimeSpan));
-        }
 
         if (clickHouseType.StartsWith("Float"))
         {
@@ -72,16 +134,28 @@ floatRemainder is 0
         {
             case "Array(Int32)" or "Array(Nullable(Int32))":
                 return new TestCaseData(exampleExpression, typeof(long?[]));
+            case "Array(Float32)" or "Array(Nullable(Float32))":
+                return new TestCaseData(exampleExpression, typeof(double?[]));
             case "Array(String)":
                 return new TestCaseData(exampleExpression, typeof(string[]));
-            case "IPv4" or "IPv6" or "String" or "UUID":
+            case "Array(Bool)":
+                return new TestCaseData(exampleExpression, typeof(bool?[]));
+            case "String" or "UUID":
                 return new TestCaseData(exampleExpression, typeof(string));
             case "Nothing":
                 return new TestCaseData(exampleExpression, typeof(DBNull));
             case "Bool":
                 return new TestCaseData(exampleExpression, typeof(bool));
+            case "IPv4" or "IPv6":
+                return new TestCaseData(exampleExpression, typeof(string));
         }
 
+        if (clickHouseType.StartsWith("Array"))
+        {
+            // Array handling is already covered above, we don't need to re-do it for every element type
+            return null;
+        }
+        
         throw new ArgumentException($"{clickHouseType} not supported");
     }
 
@@ -100,12 +174,6 @@ private static bool ShouldBeSupportedInJson(string clickHouseType)
             return false;
         }
 
-        // Time and Time64 are not currently supported
-        if (clickHouseType.StartsWith("Time"))
-        {
-            return false;
-        }
-
         switch (clickHouseType)
         {
             case "Int128":