diff --git a/CHANGELOG.md b/CHANGELOG.md index c1a4de08c..a7461d236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - v5.1.0 - Added `APPLICATION_PATH` to `CLIENT_ENVIRONMENT` sent during authentication to identify the application connecting to Snowflake. - Renew idle sessions in the pool if keep alive is enabled. + - Added support for native arrow structured types. - Added `CRLDOWNLOADMAXSIZE` connection parameter to limit the maximum size of CRL files downloaded during certificate revocation checks. - AWS WIF will now also check the application config and AWS profile credential store when determining the current AWS region - Allow users to configure the maximum amount of connections via `SERVICE_POINT_CONNECTION_LIMIT` property. diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index 0511d4d2f..a413b53e4 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Data; using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -10,19 +12,33 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredArraysIT : StructuredTypesIT { + private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; + + public StructuredArraysIT(ResultFormat resultFormat, bool nativeArrow) + { + _resultFormat = resultFormat; + _nativeArrow = nativeArrow; + } + [Test] public void TestDataTableLoadOnStructuredArray() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var expectedValueA = 'a'; var expectedValueB = 'b'; var expectedValueC = 'c'; @@ -52,7 +68,7 @@ public void TestSelectArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a','b','c')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -64,6 +80,17 @@ public void TestSelectArray() // assert Assert.AreEqual(3, array.Length); CollectionAssert.AreEqual(new[] { "a", "b", "c" }, array); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JArray.Parse(jsonString), JArray.Parse(arrowString))); + } } } } @@ -77,7 +104,7 @@ public void TestSelectArrayOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfObjects = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('name', 'Alex'), OBJECT_CONSTRUCT('name', 'Brian'))::ARRAY(OBJECT(name VARCHAR))"; command.CommandText = $"SELECT {arrayOfObjects}"; @@ -103,7 +130,7 @@ public void TestSelectArrayOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfArrays = "ARRAY_CONSTRUCT(ARRAY_CONSTRUCT('a', 'b'), ARRAY_CONSTRUCT('c', 'd'))::ARRAY(ARRAY(TEXT))"; command.CommandText = $"SELECT {arrayOfArrays}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -128,7 +155,7 @@ public void TestSelectArrayOfMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfMap = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('a', 'b'))::ARRAY(MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {arrayOfMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -159,7 +186,7 @@ public void TestSelectSemiStructuredTypesInArray(string valueSfString, string ex connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -183,7 +210,7 @@ public void TestSelectArrayOfIntegers() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfIntegers = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfIntegers}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -208,7 +235,7 @@ public void TestSelectArrayOfLong() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfLongs = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(BIGINT)"; command.CommandText = $"SELECT {arrayOfLongs}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -233,7 +260,7 @@ public void TestSelectArrayOfFloats() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfFloats = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(FLOAT)"; command.CommandText = $"SELECT {arrayOfFloats}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -258,7 +285,7 @@ public void TestSelectArrayOfDoubles() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfDoubles = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -283,7 +310,7 @@ public void TestSelectArrayOfDoublesWithExponentNotation() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfDoubles = "ARRAY_CONSTRUCT(1.0e100, 1.0e-100)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -299,6 +326,82 @@ public void TestSelectArrayOfDoublesWithExponentNotation() } } + [Test] + public void TestSelectArrayOfBooleans() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfBooleans = "ARRAY_CONSTRUCT(true, false)::ARRAY(BOOLEAN)"; + command.CommandText = $"SELECT {arrayOfBooleans}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + + // assert + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new[] { true, false }, array); + } + } + } + + [Test] + public void TestSelectArrayOfBinaries() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfBinaries = "ARRAY_CONSTRUCT(TO_BINARY('AB', 'UTF-8'), TO_BINARY('BC', 'UTF-8'))::ARRAY(BINARY)"; + command.CommandText = $"SELECT {arrayOfBinaries}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + var strings = array.Select(b => Encoding.UTF8.GetString(b)).ToArray(); + + // assert + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new[] { "AB", "BC" }, strings); + } + } + } + + [Test] + public void TestSelectArrayOfDates() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfDates = "ARRAY_CONSTRUCT('2024-01-01'::DATE)::ARRAY(DATE)"; + command.CommandText = $"SELECT {arrayOfDates}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + + // assert + Assert.AreEqual(1, array.Length); + CollectionAssert.AreEqual(new[] { DateTime.Parse("2024-01-01") }, array); + } + } + } + [Test] public void TestSelectStringArrayWithNulls() { @@ -308,7 +411,7 @@ public void TestSelectStringArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a',NULL,'b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -333,7 +436,7 @@ public void TestSelectIntArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfNumberSFString = "ARRAY_CONSTRUCT(3,NULL,5)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfNumberSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -358,7 +461,7 @@ public void TestSelectNullArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullArraySFString = "NULL::ARRAY(TEXT)"; command.CommandText = $"SELECT {nullArraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -382,7 +485,7 @@ public void TestThrowExceptionForInvalidArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('x', 'y')::ARRAY"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -408,7 +511,7 @@ public void TestThrowExceptionForInvalidArrayElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a76dacad-0e35-497b-bf9b-7cd49262b68b', 'z76dacad-0e35-497b-bf9b-7cd49262b68b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -419,7 +522,11 @@ public void TestThrowExceptionForInvalidArrayElement() // assert SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + else + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an array.")); + } } } @@ -433,7 +540,7 @@ public void TestThrowExceptionForNextedInvalidElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = @"ARRAY_CONSTRUCT( OBJECT_CONSTRUCT('x', 'a', 'y', 'b') )::ARRAY(OBJECT(x VARCHAR, y VARCHAR))"; @@ -445,9 +552,17 @@ public void TestThrowExceptionForNextedInvalidElement() var thrown = Assert.Throws(() => reader.GetArray(0)); // assert - SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[0][1]")); - Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[0][1]")); + Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + } + else + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an array.")); + } } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index 65190f018..67ed38120 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -9,19 +10,33 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredMapsIT : StructuredTypesIT { + private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; + + public StructuredMapsIT(ResultFormat resultFormat, bool nativeArrow) + { + _resultFormat = resultFormat; + _nativeArrow = nativeArrow; + } + [Test] public void TestDataTableLoadOnStructuredMap() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)"; @@ -50,7 +65,7 @@ public void TestSelectMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', '01-234')::MAP(VARCHAR, VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -65,6 +80,17 @@ public void TestSelectMap() Assert.AreEqual("San Mateo", map["city"]); Assert.AreEqual("CA", map["state"]); Assert.AreEqual("01-234", map["zip"]); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JObject.Parse(jsonString), JObject.Parse(arrowString))); + } } } } @@ -78,7 +104,7 @@ public void TestSelectMapWithIntegerKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -106,7 +132,7 @@ public void TestSelectMapWithLongKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -134,7 +160,7 @@ public void TestSelectMapOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWitObjectValueSFString = @"OBJECT_CONSTRUCT( 'Warsaw', OBJECT_CONSTRUCT('prefix', '01', 'postfix', '234'), 'San Mateo', OBJECT_CONSTRUCT('prefix', '02', 'postfix', '567') @@ -164,7 +190,7 @@ public void TestSelectMapOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -190,7 +216,7 @@ public void TestSelectMapOfLists() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -216,7 +242,7 @@ public void TestSelectMapOfMaps() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapAsSFString = "OBJECT_CONSTRUCT('a', OBJECT_CONSTRUCT('b', 'c'))::MAP(TEXT, MAP(TEXT, TEXT))"; command.CommandText = $"SELECT {mapAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -246,7 +272,7 @@ public void TestSelectSemiStructuredTypesInMap(string valueSfString, string expe connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -271,7 +297,7 @@ public void TestSelectNullMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullMapSFString = "NULL::MAP(TEXT,TEXT)"; command.CommandText = $"SELECT {nullMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -295,7 +321,7 @@ public void TestThrowExceptionForInvalidMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var invalidMapSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {invalidMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -321,7 +347,7 @@ public void TestThrowExceptionForInvalidMapElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var invalidMapSFString = @"OBJECT_CONSTRUCT( 'x', 'a76dacad-0e35-497b-bf9b-7cd49262b68b', 'y', 'z76dacad-0e35-497b-bf9b-7cd49262b68b' @@ -335,7 +361,10 @@ public void TestThrowExceptionForInvalidMapElement() // assert SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + else + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting a map.")); } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index 3327f40ef..25ae55d23 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Data; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -8,19 +9,33 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredObjectIT : StructuredTypesIT { + private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; + + public StructuredObjectIT(ResultFormat resultFormat, bool nativeArrow) + { + _resultFormat = resultFormat; + _nativeArrow = nativeArrow; + } + [Test] public void TestDataTableLoadOnStructuredObject() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::OBJECT(city VARCHAR)"; @@ -49,7 +64,7 @@ public void TestSelectStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA')::OBJECT(city VARCHAR, state VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -62,6 +77,17 @@ public void TestSelectStructuredTypeObject() Assert.AreEqual("San Mateo", address.city); Assert.AreEqual("CA", address.state); Assert.IsNull(address.zip); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JObject.Parse(jsonString), JObject.Parse(arrowString))); + } } } } @@ -75,7 +101,7 @@ public void TestSelectNestedStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', OBJECT_CONSTRUCT('prefix', '00', 'postfix', '11'))::OBJECT(city VARCHAR, state VARCHAR, zip OBJECT(prefix VARCHAR, postfix VARCHAR))"; command.CommandText = $"SELECT {addressAsSFString}"; @@ -104,7 +130,7 @@ public void TestSelectObjectWithMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithMap = "OBJECT_CONSTRUCT('names', OBJECT_CONSTRUCT('Excellent', '6', 'Poor', '1'))::OBJECT(names MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {objectWithMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -131,7 +157,7 @@ public void TestSelectObjectWithArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -156,7 +182,7 @@ public void TestSelectObjectWithList() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -184,7 +210,7 @@ public void TestSelectSemiStructuredTypesInObject(string valueSfString, string e connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -208,7 +234,7 @@ public void TestSelectStructuredTypesAsNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', NULL, 'ListValue', NULL, @@ -252,7 +278,7 @@ public void TestSelectNestedStructuredTypesNotNull() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', OBJECT_CONSTRUCT('Name', 'John'), 'ListValue', ARRAY_CONSTRUCT('a', 'b'), @@ -300,7 +326,7 @@ public void TestRenamePropertyForPropertiesNamesConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'IntegerValue', '8', 'x', 'abc' @@ -333,7 +359,7 @@ public void TestIgnorePropertyForPropertiesOrderConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -366,7 +392,7 @@ public void TestConstructorConstructionMethod() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -399,7 +425,7 @@ public void TestSelectNullObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullObjectSFString = "NULL::OBJECT(Name TEXT)"; command.CommandText = $"SELECT {nullObjectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -423,7 +449,7 @@ public void TestThrowExceptionForInvalidObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -449,7 +475,7 @@ public void TestThrowExceptionForInvalidPropertyType() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = "OBJECT_CONSTRUCT('x', 'a', 'y', 'b')::OBJECT(x VARCHAR, y VARCHAR)"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -459,9 +485,17 @@ public void TestThrowExceptionForInvalidPropertyType() var thrown = Assert.Throws(() => reader.GetObject(0)); // assert - SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1].")); - Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1].")); + Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + } + else + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an object.")); + } } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs index a10841005..3721568d4 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs @@ -1,11 +1,12 @@ using System.Linq; using Snowflake.Data.Client; +using Snowflake.Data.Core; namespace Snowflake.Data.Tests.IntegrationTests { public abstract class StructuredTypesIT : SFBaseTest { - protected void EnableStructuredTypes(SnowflakeDbConnection connection) + protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFormat resultFormat = ResultFormat.JSON, bool nativeArrow = false) { using (var command = connection.CreateCommand()) { @@ -13,8 +14,15 @@ protected void EnableStructuredTypes(SnowflakeDbConnection connection) command.ExecuteNonQuery(); command.CommandText = "alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"; command.ExecuteNonQuery(); - command.CommandText = "ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = JSON"; + command.CommandText = $"ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"; command.ExecuteNonQuery(); + if (resultFormat == ResultFormat.ARROW) + { + command.CommandText = $"ALTER SESSION SET ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = {nativeArrow}"; + command.ExecuteNonQuery(); + command.CommandText = $"ALTER SESSION SET FORCE_ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = {nativeArrow}"; + command.ExecuteNonQuery(); + } } } diff --git a/Snowflake.Data/Client/SnowflakeDbDataReader.cs b/Snowflake.Data/Client/SnowflakeDbDataReader.cs index 1b0ae172e..710b2f335 100755 --- a/Snowflake.Data/Client/SnowflakeDbDataReader.cs +++ b/Snowflake.Data/Client/SnowflakeDbDataReader.cs @@ -262,9 +262,19 @@ public T GetObject(int ordinal) { throw new StructuredTypesReadingException($"Method GetObject<{typeof(T)}> can be used only for structured object"); } - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertObject(fields, json); + var val = GetValue(ordinal); + switch (val) + { + case string stringValue: + { + var json = JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertObject(fields, json); + } + case Dictionary structArray: + return ArrowConverter.ConvertObject(structArray); + default: + return null; + } } catch (Exception e) { @@ -286,10 +296,19 @@ public T[] GetArray(int ordinal) { throw new StructuredTypesReadingException($"Method GetArray<{typeof(T)}> can be used only for structured array or vector types"); } - - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JArray.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertArray(fields, json); + var val = GetValue(ordinal); + switch (val) + { + case string stringValue: + { + var json = stringValue == null ? null : JArray.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertArray(fields, json); + } + case List listArray: + return ArrowConverter.ConvertArray(listArray); + default: + return null; + } } catch (Exception e) { @@ -309,10 +328,19 @@ public Dictionary GetMap(int ordinal) { throw new StructuredTypesReadingException($"Method GetMap<{typeof(TKey)}, {typeof(TValue)}> can be used only for structured map"); } - - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertMap(fields, json); + var val = GetValue(ordinal); + switch (val) + { + case string stringValue: + { + var json = stringValue == null ? null : JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertMap(fields, json); + } + case Dictionary mapArray: + return ArrowConverter.ConvertMap(mapArray); + default: + return null; + } } catch (Exception e) { diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index c9213e827..5486316c3 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Apache.Arrow; +using Apache.Arrow.Types; namespace Snowflake.Data.Core { @@ -200,16 +201,26 @@ public object ExtractCell(int columnIndex, SFDataType srcType, long scale) case SFDataType.ARRAY: case SFDataType.VARIANT: case SFDataType.OBJECT: - if (_byte[columnIndex] == null || _int[columnIndex] == null) + case SFDataType.MAP: + switch (column) { - _byte[columnIndex] = ((StringArray)column).Values.ToArray(); - _int[columnIndex] = ((StringArray)column).ValueOffsets.ToArray(); + case StructArray structArray: + return ExtractStructArray(structArray, _currentRecordIndex); + case MapArray mapArray: + return ExtractMapArray(mapArray, _currentRecordIndex); + case ListArray listArray: + return ExtractListArray(listArray, _currentRecordIndex); + default: + if (_byte[columnIndex] == null || _int[columnIndex] == null) + { + _byte[columnIndex] = ((StringArray)column).Values.ToArray(); + _int[columnIndex] = ((StringArray)column).ValueOffsets.ToArray(); + } + return StringArray.DefaultEncoding.GetString( + _byte[columnIndex], + _int[columnIndex][_currentRecordIndex], + _int[columnIndex][_currentRecordIndex + 1] - _int[columnIndex][_currentRecordIndex]); } - return StringArray.DefaultEncoding.GetString( - _byte[columnIndex], - _int[columnIndex][_currentRecordIndex], - _int[columnIndex][_currentRecordIndex + 1] - _int[columnIndex][_currentRecordIndex]); - case SFDataType.VECTOR: var col = (FixedSizeListArray)column; var values = col.Values; @@ -368,5 +379,93 @@ private long ExtractFraction(long value, long scale) { return ((value % s_powersOf10[scale]) * s_powersOf10[9 - scale]); } + + private object ConvertArrowValue(IArrowArray array, int index) + { + switch (array) + { + case StructArray strct: return ExtractStructArray(strct, index); + case MapArray map: return ExtractMapArray(map, index); + case ListArray list: return ExtractListArray(list, index); + case DoubleArray doubles: return doubles.GetValue(index); + case FloatArray floats: return floats.GetValue(index); + case Decimal128Array decimals: return decimals.GetValue(index); + case Date32Array dates: return dates.GetDateTime(index); + case Int8Array bytes: return bytes.GetValue(index); + case Int16Array shorts: return shorts.GetValue(index); + case Int32Array ints: return ints.GetValue(index); + case Int64Array longs: return longs.GetValue(index); + case BooleanArray booleans: return booleans.GetValue(index); + case StringArray strArray: + var str = strArray.GetString(index); + return string.IsNullOrEmpty(str) ? null : str; + case BinaryArray binary: return binary.GetBytes(index).ToArray(); + default: + throw new NotSupportedException($"Unsupported array type: {array.GetType()}"); + } + } + + private Dictionary ExtractStructArray(StructArray structArray, int index) + { + var result = new Dictionary(); + var structTypeFields = ((StructType)structArray.Data.DataType).Fields; + + for (int i = 0; i < structArray.Fields.Count; i++) + { + var field = structArray.Fields[i]; + var fieldName = structTypeFields[i].Name; + var value = ConvertArrowValue(field, index); + + if (value == null && structArray.Fields.Count == 1) + return null; + + result[fieldName] = value; + } + + return result; + } + + private List ExtractListArray(ListArray listArray, int index) + { + int start = listArray.ValueOffsets[index]; + int end = listArray.ValueOffsets[index + 1]; + + if (start == end) + return null; + + var values = listArray.Values; + var result = new List(end - start); + + for (int i = start; i < end; i++) + { + result.Add(ConvertArrowValue(values, i)); + } + + return result; + } + + private Dictionary ExtractMapArray(MapArray mapArray, int index) + { + int start = mapArray.ValueOffsets[index]; + int end = mapArray.ValueOffsets[index + 1]; + + if (start == end) + return null; + + var keyValuesArray = mapArray.KeyValues.Slice(start, end - start) as StructArray; + var keyArray = keyValuesArray.Fields[0]; + var valueArray = keyValuesArray.Fields[1]; + + var result = new Dictionary(); + + for (int i = 0; i < end - start; i++) + { + var key = ConvertArrowValue(keyArray, i); + var value = ConvertArrowValue(valueArray, i); + result[key] = value; + } + + return result; + } } } diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index 3859fd501..11cb31e51 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; @@ -205,6 +206,15 @@ internal override object GetValue(int ordinal) case DateTimeOffset ret: obj = ret; break; + case Dictionary ret: + obj = ret; + break; + case Dictionary ret: + obj = ret; + break; + case List ret: + obj = ret; + break; default: { var dstType = sfResultSetMetaData.GetCSharpTypeByIndex(ordinal); @@ -388,13 +398,17 @@ internal override string GetString(int ordinal) { var value = GetObjectInternal(ordinal); if (value == DBNull.Value) - return (string)value; + return null; var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); switch (value) { case string ret: return ret; + case Dictionary _: + case Dictionary _: + case List _: + return string.IsNullOrEmpty(value.ToString()) ? null : Newtonsoft.Json.JsonConvert.SerializeObject(value); case DateTime ret: if (type == SFDataType.DATE) return SFDataConverter.ToDateString(ret, sfResultSetMetaData.dateOutputFormat); diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs new file mode 100644 index 000000000..149ceb38a --- /dev/null +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -0,0 +1,184 @@ +using Snowflake.Data.Client; +using System.Collections.Generic; +using System.Reflection; +using System; +using System.Linq; + +namespace Snowflake.Data.Core.Converter +{ + internal static class ArrowConverter + { + internal static T ConvertObject(Dictionary dict) where T : new() + { + T obj = new T(); + Type type = typeof(T); + if (type.GetCustomAttributes(false).Any(attribute => attribute.GetType() == typeof(SnowflakeObject))) + { + var constructionMethod = JsonToStructuredTypeConverter.GetConstructionMethod(type); + switch (constructionMethod) + { + case SnowflakeObjectConstructionMethod.PROPERTIES_NAMES: + MapPropertiesByNames(obj, dict, type); + break; + case SnowflakeObjectConstructionMethod.PROPERTIES_ORDER: + MapPropertiesByOrder(obj, dict, type); + break; + case SnowflakeObjectConstructionMethod.CONSTRUCTOR: + return MapUsingConstructor(dict, type); + } + } + else + { + foreach (var kvp in dict) + { + var prop = type.GetProperty(kvp.Key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (prop == null) + continue; + prop.SetValue(obj, ConvertValue(kvp.Value, prop.PropertyType)); + } + } + return obj; + } + + private static void MapPropertiesByNames(object obj, Dictionary dict, Type type) + { + foreach (var kvp in dict) + { + var prop = FindPropertyByName(type, kvp.Key); + if (prop != null) + { + var converted = ConvertValue(kvp.Value, prop.PropertyType); + prop.SetValue(obj, converted); + } + } + } + + private static PropertyInfo FindPropertyByName(Type type, string name) + { + var prop = type.GetProperty(name, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (prop != null) + return prop; + + return type.GetProperties() + .FirstOrDefault(p => p.GetCustomAttributes() + .OfType() + .Any(attr => attr?.Name == name)); + } + + private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) + { + var index = 0; + foreach (var property in type.GetProperties().OrderBy(p => p.MetadataToken)) + { + if (index >= dict.Count) + break; + + var attribute = property.GetCustomAttributes().OfType().FirstOrDefault(); + if (attribute == null || !attribute.IgnoreForPropertyOrder) + { + var converted = ConvertValue(dict.ElementAt(index).Value, property.PropertyType); + property.SetValue(obj, converted); + index++; + } + } + } + + private static T MapUsingConstructor(Dictionary dict, Type type) + { + var matchingConstructor = type.GetConstructors() + .FirstOrDefault(c => c.GetParameters().Length == dict.Count) ?? + throw new StructuredTypesReadingException($"No constructor found for type: {type}"); + var parameters = matchingConstructor.GetParameters() + .Select((param, index) => ConvertValue(dict.ElementAt(index).Value, param.ParameterType)) + .ToArray(); + return (T)matchingConstructor.Invoke(parameters); + } + + internal static object CallMethod(Type type, object obj, string methodName, Type type2 = null) + { + MethodInfo genericMethod = typeof(ArrowConverter) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo constructedMethod = type2 == null + ? genericMethod.MakeGenericMethod(type) + : genericMethod.MakeGenericMethod(type, type2); + return constructedMethod.Invoke(null, new object[] { obj }); + } + + internal static T[] ConvertArray(List list) + { + var targetType = typeof(T); + var result = new T[list.Count]; + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) + targetType = Nullable.GetUnderlyingType(targetType); + for (int i = 0; i < list.Count; i++) + { + result[i] = (T)ConvertValue(list[i], targetType); + } + return result; + } + + private static List ConvertList(List list) + { + var targetType = typeof(T); + var result = new List(list.Count); + foreach (var item in list) + { + result.Add((T)ConvertValue(item, targetType)); + } + return result; + } + + internal static Dictionary ConvertMap(Dictionary dict) + { + var keyType = typeof(TKey); + var valueType = typeof(TValue); + var result = new Dictionary(); + foreach (var kvp in dict) + { + var key = (TKey)ConvertValue(kvp.Key, keyType); + var value = (TValue)ConvertValue(kvp.Value, valueType); + result[key] = value; + } + return result; + } + + private static object ConvertValue(object value, Type targetType) + { + switch (value) + { + case null: + return null; + case var _ when targetType.IsAssignableFrom(value.GetType()): + return value; + case Dictionary objDict: + return CallMethod(targetType, objDict, nameof(ConvertObject)); + case Dictionary mapDict: + if (targetType.IsGenericType) + { + var genericArgs = targetType.GetGenericArguments(); + if (genericArgs.Length == 2) + { + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + } + } + goto default; + case List objList: + if (targetType.IsArray) + { + var elementType = targetType.GetElementType(); + return CallMethod(elementType, objList, nameof(ConvertArray)); + } + else if (targetType.IsGenericType) + { + var elementType = targetType.GetGenericArguments()[0]; + return CallMethod(elementType, objList, nameof(ConvertList)); + } + goto default; + default: + return Convert.ChangeType(value, targetType); + } + } + } +} diff --git a/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs index fae3a83b0..fbaba1e07 100644 --- a/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs @@ -90,7 +90,7 @@ private static object ConvertToObject(Type type, List fields, Str return objectBuilder.Build(); } - private static SnowflakeObjectConstructionMethod GetConstructionMethod(Type type) + internal static SnowflakeObjectConstructionMethod GetConstructionMethod(Type type) { return type.GetCustomAttributes(false) .Where(attribute => attribute.GetType() == typeof(SnowflakeObject))