From f55ce8269d3ef33b79eaa042fa69fe56bd021285 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 21 May 2025 13:20:03 -0700 Subject: [PATCH 01/33] Add arrow support for structured types --- .../IntegrationTests/StructuredArraysIT.cs | 44 +++++++++++-------- .../IntegrationTests/StructuredMapsIT.cs | 34 ++++++++------ .../IntegrationTests/StructuredObjectsIT.cs | 40 ++++++++++------- .../IntegrationTests/StructuredTypesIT.cs | 13 +++--- Snowflake.Data/Core/ArrowResultChunk.cs | 1 + Snowflake.Data/Core/ArrowResultSet.cs | 4 +- 6 files changed, 82 insertions(+), 54 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index 6f99d48a8..96f7c075f 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -10,9 +10,17 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW)] + [TestFixture(ResultFormat.JSON)] public class StructuredArraysIT: StructuredTypesIT { + private readonly ResultFormat _resultFormat; + + public StructuredArraysIT(ResultFormat resultFormat) + { + _resultFormat = resultFormat; + } + [Test] public void TestDataTableLoadOnStructuredArray() { @@ -22,7 +30,7 @@ public void TestDataTableLoadOnStructuredArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var expectedValueA = 'a'; var expectedValueB = 'b'; var expectedValueC = 'c'; @@ -52,7 +60,7 @@ public void TestSelectArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arraySFString = "ARRAY_CONSTRUCT('a','b','c')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -77,7 +85,7 @@ public void TestSelectArrayOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfObjects = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('name', 'Alex'), OBJECT_CONSTRUCT('name', 'Brian'))::ARRAY(OBJECT(name VARCHAR))"; command.CommandText = $"SELECT {arrayOfObjects}"; @@ -103,7 +111,7 @@ public void TestSelectArrayOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfArrays = "ARRAY_CONSTRUCT(ARRAY_CONSTRUCT('a', 'b'), ARRAY_CONSTRUCT('c', 'd'))::ARRAY(ARRAY(TEXT))"; command.CommandText = $"SELECT {arrayOfArrays}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -128,7 +136,7 @@ public void TestSelectArrayOfMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfMap = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('a', 'b'))::ARRAY(MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {arrayOfMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -159,7 +167,7 @@ public void TestSelectSemiStructuredTypesInArray(string valueSfString, string ex connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -183,7 +191,7 @@ public void TestSelectArrayOfIntegers() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfIntegers = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfIntegers}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); @@ -208,7 +216,7 @@ public void TestSelectArrayOfLong() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfLongs = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(BIGINT)"; command.CommandText = $"SELECT {arrayOfLongs}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); @@ -233,7 +241,7 @@ public void TestSelectArrayOfFloats() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfFloats = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(FLOAT)"; command.CommandText = $"SELECT {arrayOfFloats}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); @@ -258,7 +266,7 @@ public void TestSelectArrayOfDoubles() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfDoubles = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); @@ -283,7 +291,7 @@ public void TestSelectArrayOfDoublesWithExponentNotation() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfDoubles = "ARRAY_CONSTRUCT(1.0e100, 1.0e-100)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); @@ -308,7 +316,7 @@ public void TestSelectStringArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arraySFString = "ARRAY_CONSTRUCT('a',NULL,'b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -333,7 +341,7 @@ public void TestSelectIntArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arrayOfNumberSFString = "ARRAY_CONSTRUCT(3,NULL,5)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfNumberSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -358,7 +366,7 @@ public void TestSelectNullArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var nullArraySFString = "NULL::ARRAY(TEXT)"; command.CommandText = $"SELECT {nullArraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -382,7 +390,7 @@ public void TestThrowExceptionForInvalidArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arraySFString = "ARRAY_CONSTRUCT('x', 'y')::ARRAY"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -408,7 +416,7 @@ public void TestThrowExceptionForInvalidArrayElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arraySFString = "ARRAY_CONSTRUCT('a76dacad-0e35-497b-bf9b-7cd49262b68b', 'z76dacad-0e35-497b-bf9b-7cd49262b68b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -433,7 +441,7 @@ public void TestThrowExceptionForNextedInvalidElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var arraySFString = @"ARRAY_CONSTRUCT( OBJECT_CONSTRUCT('x', 'a', 'y', 'b') )::ARRAY(OBJECT(x VARCHAR, y VARCHAR))"; diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index 207dd7f58..dc42e3202 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -9,9 +9,17 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW)] + [TestFixture(ResultFormat.JSON)] public class StructuredMapsIT: StructuredTypesIT { + private readonly ResultFormat _resultFormat; + + public StructuredMapsIT(ResultFormat resultFormat) + { + _resultFormat = resultFormat; + } + [Test] public void TestDataTableLoadOnStructuredMap() { @@ -21,7 +29,7 @@ public void TestDataTableLoadOnStructuredMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)"; @@ -50,7 +58,7 @@ public void TestSelectMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', '01-234')::MAP(VARCHAR, VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -78,7 +86,7 @@ public void TestSelectMapWithIntegerKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -106,7 +114,7 @@ public void TestSelectMapWithLongKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -134,7 +142,7 @@ public void TestSelectMapOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapWitObjectValueSFString = @"OBJECT_CONSTRUCT( 'Warsaw', OBJECT_CONSTRUCT('prefix', '01', 'postfix', '234'), 'San Mateo', OBJECT_CONSTRUCT('prefix', '02', 'postfix', '567') @@ -164,7 +172,7 @@ public void TestSelectMapOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -190,7 +198,7 @@ public void TestSelectMapOfLists() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -216,7 +224,7 @@ public void TestSelectMapOfMaps() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var mapAsSFString = "OBJECT_CONSTRUCT('a', OBJECT_CONSTRUCT('b', 'c'))::MAP(TEXT, MAP(TEXT, TEXT))"; command.CommandText = $"SELECT {mapAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -246,7 +254,7 @@ public void TestSelectSemiStructuredTypesInMap(string valueSfString, string expe connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -271,7 +279,7 @@ public void TestSelectNullMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var nullMapSFString = "NULL::MAP(TEXT,TEXT)"; command.CommandText = $"SELECT {nullMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -295,7 +303,7 @@ public void TestThrowExceptionForInvalidMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var invalidMapSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {invalidMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -321,7 +329,7 @@ public void TestThrowExceptionForInvalidMapElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var invalidMapSFString = @"OBJECT_CONSTRUCT( 'x', 'a76dacad-0e35-497b-bf9b-7cd49262b68b', 'y', 'z76dacad-0e35-497b-bf9b-7cd49262b68b' diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index e46e61ab9..f1a57b072 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -8,9 +8,17 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture] + [TestFixture(ResultFormat.ARROW)] + [TestFixture(ResultFormat.JSON)] public class StructuredObjectIT: StructuredTypesIT { + private readonly ResultFormat _resultFormat; + + public StructuredObjectIT(ResultFormat resultFormat) + { + _resultFormat = resultFormat; + } + [Test] public void TestDataTableLoadOnStructuredObject() { @@ -20,7 +28,7 @@ public void TestDataTableLoadOnStructuredObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::OBJECT(city VARCHAR)"; @@ -49,7 +57,7 @@ public void TestSelectStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA')::OBJECT(city VARCHAR, state VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -75,7 +83,7 @@ public void TestSelectNestedStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', OBJECT_CONSTRUCT('prefix', '00', 'postfix', '11'))::OBJECT(city VARCHAR, state VARCHAR, zip OBJECT(prefix VARCHAR, postfix VARCHAR))"; command.CommandText = $"SELECT {addressAsSFString}"; @@ -104,7 +112,7 @@ public void TestSelectObjectWithMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectWithMap = "OBJECT_CONSTRUCT('names', OBJECT_CONSTRUCT('Excellent', '6', 'Poor', '1'))::OBJECT(names MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {objectWithMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -131,7 +139,7 @@ public void TestSelectObjectWithArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -156,7 +164,7 @@ public void TestSelectObjectWithList() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -184,7 +192,7 @@ public void TestSelectSemiStructuredTypesInObject(string valueSfString, string e connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader) command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -208,7 +216,7 @@ public void TestSelectStructuredTypesAsNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', NULL, 'ListValue', NULL, @@ -252,7 +260,7 @@ public void TestSelectNestedStructuredTypesNotNull() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', OBJECT_CONSTRUCT('Name', 'John'), 'ListValue', ARRAY_CONSTRUCT('a', 'b'), @@ -300,7 +308,7 @@ public void TestRenamePropertyForPropertiesNamesConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = @"OBJECT_CONSTRUCT( 'IntegerValue', '8', 'x', 'abc' @@ -333,7 +341,7 @@ public void TestIgnorePropertyForPropertiesOrderConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -366,7 +374,7 @@ public void TestConstructorConstructionMethod() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -399,7 +407,7 @@ public void TestSelectNullObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var nullObjectSFString = "NULL::OBJECT(Name TEXT)"; command.CommandText = $"SELECT {nullObjectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -423,7 +431,7 @@ public void TestThrowExceptionForInvalidObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -449,7 +457,7 @@ public void TestThrowExceptionForInvalidPropertyType() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection); + EnableStructuredTypes(connection, _resultFormat); var objectSFString = "OBJECT_CONSTRUCT('x', 'a', 'y', 'b')::OBJECT(x VARCHAR, y VARCHAR)"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs index a10841005..7b37562c9 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs @@ -1,19 +1,20 @@ using System.Linq; using Snowflake.Data.Client; +using Snowflake.Data.Core; namespace Snowflake.Data.Tests.IntegrationTests { public abstract class StructuredTypesIT : SFBaseTest { - protected void EnableStructuredTypes(SnowflakeDbConnection connection) + protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFormat resultFormat = ResultFormat.JSON) { using (var command = connection.CreateCommand()) { - command.CommandText = "alter session set ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE = true"; - command.ExecuteNonQuery(); - command.CommandText = "alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"; - command.ExecuteNonQuery(); - command.CommandText = "ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = JSON"; + //command.CommandText = "alter session set ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE = true"; + //command.ExecuteNonQuery(); + //command.CommandText = "alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"; + //command.ExecuteNonQuery(); + command.CommandText = $"ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"; command.ExecuteNonQuery(); } } diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 87c041584..db7ebe59e 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -200,6 +200,7 @@ public object ExtractCell(int columnIndex, SFDataType srcType, long scale) case SFDataType.ARRAY: case SFDataType.VARIANT: case SFDataType.OBJECT: + case SFDataType.MAP: if (_byte[columnIndex] == null || _int[columnIndex] == null) { _byte[columnIndex] = ((StringArray)column).Values.ToArray(); diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index f6b04726f..ad20f8ccb 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -379,7 +379,9 @@ internal override string GetString(int ordinal) { var value = GetObjectInternal(ordinal); if (value == DBNull.Value) - return (string)value; + { + return null; + } var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); switch (value) From 5499f89cec6a08dd0a96496581061a82506f9fe9 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 21 May 2025 13:45:25 -0700 Subject: [PATCH 02/33] Revert commented lines --- .../IntegrationTests/StructuredTypesIT.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs index 7b37562c9..3dda97a90 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs @@ -10,10 +10,10 @@ protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFor { using (var command = connection.CreateCommand()) { - //command.CommandText = "alter session set ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE = true"; - //command.ExecuteNonQuery(); - //command.CommandText = "alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"; - //command.ExecuteNonQuery(); + command.CommandText = "alter session set ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE = true"; + command.ExecuteNonQuery(); + command.CommandText = "alter session set IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE = true"; + command.ExecuteNonQuery(); command.CommandText = $"ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"; command.ExecuteNonQuery(); } From 938d36ac3a018157832b4c0899fc3e72a195a31d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 21 May 2025 13:47:10 -0700 Subject: [PATCH 03/33] Remove bracket --- Snowflake.Data/Core/ArrowResultSet.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index db126b319..622518c80 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -388,9 +388,7 @@ internal override string GetString(int ordinal) { var value = GetObjectInternal(ordinal); if (value == DBNull.Value) - { return null; - } var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); switch (value) From ac0a7db2356421106fa5cf914299d483bfb7c1f8 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 5 Jun 2025 09:42:17 -0700 Subject: [PATCH 04/33] Fix formatting --- Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs | 2 +- Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs | 2 +- Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index a6abcad0d..f5dd748c8 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -12,7 +12,7 @@ namespace Snowflake.Data.Tests.IntegrationTests { [TestFixture(ResultFormat.ARROW)] [TestFixture(ResultFormat.JSON)] - public class StructuredArraysIT: StructuredTypesIT + public class StructuredArraysIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index de13c7bbe..ab3c97178 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -11,7 +11,7 @@ namespace Snowflake.Data.Tests.IntegrationTests { [TestFixture(ResultFormat.ARROW)] [TestFixture(ResultFormat.JSON)] - public class StructuredMapsIT: StructuredTypesIT + public class StructuredMapsIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index 8447f1af5..28d0ccf7e 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -10,7 +10,7 @@ namespace Snowflake.Data.Tests.IntegrationTests { [TestFixture(ResultFormat.ARROW)] [TestFixture(ResultFormat.JSON)] - public class StructuredObjectIT: StructuredTypesIT + public class StructuredObjectIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; From 5add0a330734e60e97071a7d393ad5b4f3bf252c Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 5 Jun 2025 11:06:28 -0700 Subject: [PATCH 05/33] Revert "SNOW-2115047 Log http calls time (#1180)" This reverts commit a03479167184871a607f873edc5f1c3a3afa5949. --- Snowflake.Data/Core/RestRequester.cs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Snowflake.Data/Core/RestRequester.cs b/Snowflake.Data/Core/RestRequester.cs index af131c318..320d75197 100644 --- a/Snowflake.Data/Core/RestRequester.cs +++ b/Snowflake.Data/Core/RestRequester.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using Newtonsoft.Json; using System.Net.Http; using System.Threading; @@ -108,22 +107,20 @@ protected virtual async Task SendAsync(HttpRequestMessage m restRequestTimeout.Token)) { HttpResponseMessage response = null; - logger.Debug($"Executing: {sid} {message.Method} {message.RequestUri} HTTP/{message.Version}"); - var watch = new Stopwatch(); try { - watch.Start(); + logger.Debug($"Executing: {sid} {message.Method} {message.RequestUri} HTTP/{message.Version}"); + response = await _HttpClient .SendAsync(message, HttpCompletionOption.ResponseHeadersRead, linkedCts.Token) .ConfigureAwait(false); - watch.Stop(); if (!response.IsSuccessStatusCode) { - logger.Error($"Failed response after {watch.ElapsedMilliseconds} ms: {sid} {message.Method} {message.RequestUri} StatusCode: {(int)response.StatusCode}, ReasonPhrase: '{response.ReasonPhrase}'"); + logger.Error($"Failed Response: {sid} {message.Method} {message.RequestUri} StatusCode: {(int)response.StatusCode}, ReasonPhrase: '{response.ReasonPhrase}'"); } else { - logger.Debug($"Succeeded response after {watch.ElapsedMilliseconds} ms: {sid} {message.Method} {message.RequestUri}"); + logger.Debug($"Succeeded Response: {sid} {message.Method} {message.RequestUri}"); } response.EnsureSuccessStatusCode(); @@ -131,11 +128,6 @@ protected virtual async Task SendAsync(HttpRequestMessage m } catch (Exception e) { - if (watch.IsRunning) - { - watch.Stop(); - logger.Error($"Response receiving interrupted by exception after {watch.ElapsedMilliseconds} ms. {sid} {message.Method} {message.RequestUri}"); - } // Disposing of the response if not null now that we don't need it anymore response?.Dispose(); if (restRequestTimeout.IsCancellationRequested) From d131fafff57dbdccef7023663539a1da2633bcb8 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 5 Jun 2025 12:35:09 -0700 Subject: [PATCH 06/33] Reapply "SNOW-2115047 Log http calls time (#1180)" This reverts commit 5add0a330734e60e97071a7d393ad5b4f3bf252c. --- Snowflake.Data/Core/RestRequester.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Snowflake.Data/Core/RestRequester.cs b/Snowflake.Data/Core/RestRequester.cs index 320d75197..af131c318 100644 --- a/Snowflake.Data/Core/RestRequester.cs +++ b/Snowflake.Data/Core/RestRequester.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using Newtonsoft.Json; using System.Net.Http; using System.Threading; @@ -107,20 +108,22 @@ protected virtual async Task SendAsync(HttpRequestMessage m restRequestTimeout.Token)) { HttpResponseMessage response = null; + logger.Debug($"Executing: {sid} {message.Method} {message.RequestUri} HTTP/{message.Version}"); + var watch = new Stopwatch(); try { - logger.Debug($"Executing: {sid} {message.Method} {message.RequestUri} HTTP/{message.Version}"); - + watch.Start(); response = await _HttpClient .SendAsync(message, HttpCompletionOption.ResponseHeadersRead, linkedCts.Token) .ConfigureAwait(false); + watch.Stop(); if (!response.IsSuccessStatusCode) { - logger.Error($"Failed Response: {sid} {message.Method} {message.RequestUri} StatusCode: {(int)response.StatusCode}, ReasonPhrase: '{response.ReasonPhrase}'"); + logger.Error($"Failed response after {watch.ElapsedMilliseconds} ms: {sid} {message.Method} {message.RequestUri} StatusCode: {(int)response.StatusCode}, ReasonPhrase: '{response.ReasonPhrase}'"); } else { - logger.Debug($"Succeeded Response: {sid} {message.Method} {message.RequestUri}"); + logger.Debug($"Succeeded response after {watch.ElapsedMilliseconds} ms: {sid} {message.Method} {message.RequestUri}"); } response.EnsureSuccessStatusCode(); @@ -128,6 +131,11 @@ protected virtual async Task SendAsync(HttpRequestMessage m } catch (Exception e) { + if (watch.IsRunning) + { + watch.Stop(); + logger.Error($"Response receiving interrupted by exception after {watch.ElapsedMilliseconds} ms. {sid} {message.Method} {message.RequestUri}"); + } // Disposing of the response if not null now that we don't need it anymore response?.Dispose(); if (restRequestTimeout.IsCancellationRequested) From 493264ccfd3ecff45b212d8772210f3e2155a64a Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 19 Jun 2025 17:02:23 -0700 Subject: [PATCH 07/33] Add native arrow format result --- .../IntegrationTests/StructuredArraysIT.cs | 43 ++++++++++--------- .../IntegrationTests/StructuredMapsIT.cs | 33 +++++++------- .../IntegrationTests/StructuredObjectsIT.cs | 39 +++++++++-------- .../IntegrationTests/StructuredTypesIT.cs | 9 +++- 4 files changed, 70 insertions(+), 54 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index f5dd748c8..9b4235a63 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -10,15 +10,18 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture(ResultFormat.ARROW)] - [TestFixture(ResultFormat.JSON)] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredArraysIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; - public StructuredArraysIT(ResultFormat resultFormat) + public StructuredArraysIT(ResultFormat resultFormat, bool nativeArrow) { _resultFormat = resultFormat; + _nativeArrow = nativeArrow; } [Test] @@ -30,7 +33,7 @@ public void TestDataTableLoadOnStructuredArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var expectedValueA = 'a'; var expectedValueB = 'b'; var expectedValueC = 'c'; @@ -60,7 +63,7 @@ public void TestSelectArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a','b','c')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -85,7 +88,7 @@ public void TestSelectArrayOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfObjects = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('name', 'Alex'), OBJECT_CONSTRUCT('name', 'Brian'))::ARRAY(OBJECT(name VARCHAR))"; command.CommandText = $"SELECT {arrayOfObjects}"; @@ -111,7 +114,7 @@ public void TestSelectArrayOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfArrays = "ARRAY_CONSTRUCT(ARRAY_CONSTRUCT('a', 'b'), ARRAY_CONSTRUCT('c', 'd'))::ARRAY(ARRAY(TEXT))"; command.CommandText = $"SELECT {arrayOfArrays}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -136,7 +139,7 @@ public void TestSelectArrayOfMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfMap = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('a', 'b'))::ARRAY(MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {arrayOfMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -167,7 +170,7 @@ public void TestSelectSemiStructuredTypesInArray(string valueSfString, string ex connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -191,7 +194,7 @@ public void TestSelectArrayOfIntegers() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfIntegers = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfIntegers}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -216,7 +219,7 @@ public void TestSelectArrayOfLong() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfLongs = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(BIGINT)"; command.CommandText = $"SELECT {arrayOfLongs}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -241,7 +244,7 @@ public void TestSelectArrayOfFloats() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfFloats = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(FLOAT)"; command.CommandText = $"SELECT {arrayOfFloats}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -266,7 +269,7 @@ public void TestSelectArrayOfDoubles() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfDoubles = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -291,7 +294,7 @@ public void TestSelectArrayOfDoublesWithExponentNotation() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfDoubles = "ARRAY_CONSTRUCT(1.0e100, 1.0e-100)::ARRAY(DOUBLE)"; command.CommandText = $"SELECT {arrayOfDoubles}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -316,7 +319,7 @@ public void TestSelectStringArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a',NULL,'b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -341,7 +344,7 @@ public void TestSelectIntArrayWithNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arrayOfNumberSFString = "ARRAY_CONSTRUCT(3,NULL,5)::ARRAY(INTEGER)"; command.CommandText = $"SELECT {arrayOfNumberSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -366,7 +369,7 @@ public void TestSelectNullArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullArraySFString = "NULL::ARRAY(TEXT)"; command.CommandText = $"SELECT {nullArraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -390,7 +393,7 @@ public void TestThrowExceptionForInvalidArray() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('x', 'y')::ARRAY"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -416,7 +419,7 @@ public void TestThrowExceptionForInvalidArrayElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = "ARRAY_CONSTRUCT('a76dacad-0e35-497b-bf9b-7cd49262b68b', 'z76dacad-0e35-497b-bf9b-7cd49262b68b')::ARRAY(TEXT)"; command.CommandText = $"SELECT {arraySFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -441,7 +444,7 @@ public void TestThrowExceptionForNextedInvalidElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var arraySFString = @"ARRAY_CONSTRUCT( OBJECT_CONSTRUCT('x', 'a', 'y', 'b') )::ARRAY(OBJECT(x VARCHAR, y VARCHAR))"; diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index ab3c97178..c4f0050fe 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -9,15 +9,18 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture(ResultFormat.ARROW)] - [TestFixture(ResultFormat.JSON)] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredMapsIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; - public StructuredMapsIT(ResultFormat resultFormat) + public StructuredMapsIT(ResultFormat resultFormat, bool nativeArrow) { _resultFormat = resultFormat; + _nativeArrow = nativeArrow; } [Test] @@ -29,7 +32,7 @@ public void TestDataTableLoadOnStructuredMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)"; @@ -58,7 +61,7 @@ public void TestSelectMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', '01-234')::MAP(VARCHAR, VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -86,7 +89,7 @@ public void TestSelectMapWithIntegerKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -114,7 +117,7 @@ public void TestSelectMapWithLongKeys() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)"; command.CommandText = $"SELECT {mapSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -142,7 +145,7 @@ public void TestSelectMapOfObjects() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWitObjectValueSFString = @"OBJECT_CONSTRUCT( 'Warsaw', OBJECT_CONSTRUCT('prefix', '01', 'postfix', '234'), 'San Mateo', OBJECT_CONSTRUCT('prefix', '02', 'postfix', '567') @@ -172,7 +175,7 @@ public void TestSelectMapOfArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -198,7 +201,7 @@ public void TestSelectMapOfLists() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))"; command.CommandText = $"SELECT {mapWithArrayValueSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -224,7 +227,7 @@ public void TestSelectMapOfMaps() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var mapAsSFString = "OBJECT_CONSTRUCT('a', OBJECT_CONSTRUCT('b', 'c'))::MAP(TEXT, MAP(TEXT, TEXT))"; command.CommandText = $"SELECT {mapAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -254,7 +257,7 @@ public void TestSelectSemiStructuredTypesInMap(string valueSfString, string expe connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -279,7 +282,7 @@ public void TestSelectNullMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullMapSFString = "NULL::MAP(TEXT,TEXT)"; command.CommandText = $"SELECT {nullMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -303,7 +306,7 @@ public void TestThrowExceptionForInvalidMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var invalidMapSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {invalidMapSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -329,7 +332,7 @@ public void TestThrowExceptionForInvalidMapElement() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var invalidMapSFString = @"OBJECT_CONSTRUCT( 'x', 'a76dacad-0e35-497b-bf9b-7cd49262b68b', 'y', 'z76dacad-0e35-497b-bf9b-7cd49262b68b' diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index 28d0ccf7e..1494a5aac 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -8,15 +8,18 @@ namespace Snowflake.Data.Tests.IntegrationTests { - [TestFixture(ResultFormat.ARROW)] - [TestFixture(ResultFormat.JSON)] + [TestFixture(ResultFormat.ARROW, false)] + [TestFixture(ResultFormat.ARROW, true)] + [TestFixture(ResultFormat.JSON, false)] public class StructuredObjectIT : StructuredTypesIT { private readonly ResultFormat _resultFormat; + private readonly bool _nativeArrow; - public StructuredObjectIT(ResultFormat resultFormat) + public StructuredObjectIT(ResultFormat resultFormat, bool nativeArrow) { _resultFormat = resultFormat; + _nativeArrow = nativeArrow; } [Test] @@ -28,7 +31,7 @@ public void TestDataTableLoadOnStructuredObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var key = "city"; var value = "San Mateo"; var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::OBJECT(city VARCHAR)"; @@ -57,7 +60,7 @@ public void TestSelectStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA')::OBJECT(city VARCHAR, state VARCHAR)"; command.CommandText = $"SELECT {addressAsSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -83,7 +86,7 @@ public void TestSelectNestedStructuredTypeObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', OBJECT_CONSTRUCT('prefix', '00', 'postfix', '11'))::OBJECT(city VARCHAR, state VARCHAR, zip OBJECT(prefix VARCHAR, postfix VARCHAR))"; command.CommandText = $"SELECT {addressAsSFString}"; @@ -112,7 +115,7 @@ public void TestSelectObjectWithMap() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithMap = "OBJECT_CONSTRUCT('names', OBJECT_CONSTRUCT('Excellent', '6', 'Poor', '1'))::OBJECT(names MAP(VARCHAR,VARCHAR))"; command.CommandText = $"SELECT {objectWithMap}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -139,7 +142,7 @@ public void TestSelectObjectWithArrays() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -164,7 +167,7 @@ public void TestSelectObjectWithList() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; command.CommandText = $"SELECT {objectWithArray}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -192,7 +195,7 @@ public void TestSelectSemiStructuredTypesInObject(string valueSfString, string e connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); command.CommandText = $"SELECT {valueSfString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); Assert.IsTrue(reader.Read()); @@ -216,7 +219,7 @@ public void TestSelectStructuredTypesAsNulls() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', NULL, 'ListValue', NULL, @@ -260,7 +263,7 @@ public void TestSelectNestedStructuredTypesNotNull() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT_KEEP_NULL( 'ObjectValue', OBJECT_CONSTRUCT('Name', 'John'), 'ListValue', ARRAY_CONSTRUCT('a', 'b'), @@ -308,7 +311,7 @@ public void TestRenamePropertyForPropertiesNamesConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'IntegerValue', '8', 'x', 'abc' @@ -341,7 +344,7 @@ public void TestIgnorePropertyForPropertiesOrderConstruction() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -374,7 +377,7 @@ public void TestConstructorConstructionMethod() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = @"OBJECT_CONSTRUCT( 'x', 'abc', 'IntegerValue', '8' @@ -407,7 +410,7 @@ public void TestSelectNullObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var nullObjectSFString = "NULL::OBJECT(Name TEXT)"; command.CommandText = $"SELECT {nullObjectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -431,7 +434,7 @@ public void TestThrowExceptionForInvalidObject() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); @@ -457,7 +460,7 @@ public void TestThrowExceptionForInvalidPropertyType() connection.Open(); using (var command = connection.CreateCommand()) { - EnableStructuredTypes(connection, _resultFormat); + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); var objectSFString = "OBJECT_CONSTRUCT('x', 'a', 'y', 'b')::OBJECT(x VARCHAR, y VARCHAR)"; command.CommandText = $"SELECT {objectSFString}"; var reader = (SnowflakeDbDataReader)command.ExecuteReader(); diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs index 3dda97a90..3721568d4 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs @@ -6,7 +6,7 @@ namespace Snowflake.Data.Tests.IntegrationTests { public abstract class StructuredTypesIT : SFBaseTest { - protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFormat resultFormat = ResultFormat.JSON) + protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFormat resultFormat = ResultFormat.JSON, bool nativeArrow = false) { using (var command = connection.CreateCommand()) { @@ -16,6 +16,13 @@ protected void EnableStructuredTypes(SnowflakeDbConnection connection, ResultFor command.ExecuteNonQuery(); command.CommandText = $"ALTER SESSION SET DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"; command.ExecuteNonQuery(); + if (resultFormat == ResultFormat.ARROW) + { + command.CommandText = $"ALTER SESSION SET ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = {nativeArrow}"; + command.ExecuteNonQuery(); + command.CommandText = $"ALTER SESSION SET FORCE_ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT = {nativeArrow}"; + command.ExecuteNonQuery(); + } } } From 16247c3c0cb939d4578fc2bb465ed7127314557a Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 19 Jun 2025 21:57:01 -0700 Subject: [PATCH 08/33] Add support for native arrow formats --- Snowflake.Data/Core/ArrowResultChunk.cs | 113 ++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 4f5238df8..1995f391c 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Apache.Arrow; +using Apache.Arrow.Types; namespace Snowflake.Data.Core { @@ -201,16 +202,25 @@ public object ExtractCell(int columnIndex, SFDataType srcType, long scale) case SFDataType.VARIANT: case SFDataType.OBJECT: case SFDataType.MAP: - if (_byte[columnIndex] == null || _int[columnIndex] == null) + switch (column) { - _byte[columnIndex] = ((StringArray)column).Values.ToArray(); - _int[columnIndex] = ((StringArray)column).ValueOffsets.ToArray(); + case StructArray structArray: + return FormatStructArray(structArray, _currentBatchIndex); + case MapArray mapArray: + return FormatArrowMapArray(mapArray, _currentRecordIndex); + case ListArray listArray: + return FormatArrowListArray(listArray, _currentRecordIndex); + default: + if (_byte[columnIndex] == null || _int[columnIndex] == null) + { + _byte[columnIndex] = ((StringArray)column).Values.ToArray(); + _int[columnIndex] = ((StringArray)column).ValueOffsets.ToArray(); + } + return StringArray.DefaultEncoding.GetString( + _byte[columnIndex], + _int[columnIndex][_currentRecordIndex], + _int[columnIndex][_currentRecordIndex + 1] - _int[columnIndex][_currentRecordIndex]); } - return StringArray.DefaultEncoding.GetString( - _byte[columnIndex], - _int[columnIndex][_currentRecordIndex], - _int[columnIndex][_currentRecordIndex + 1] - _int[columnIndex][_currentRecordIndex]); - case SFDataType.VECTOR: var col = (FixedSizeListArray)column; var values = col.Values; @@ -369,5 +379,92 @@ private long ExtractFraction(long value, long scale) { return ((value % s_powersOf10[scale]) * s_powersOf10[9 - scale]); } + + private string FormatArrowValue(IArrowArray array, int index) + { + switch (array) + { + case StructArray strct: return FormatStructArray(strct, index); + case MapArray map: return FormatArrowMapArray(map, index); + case ListArray list: return FormatArrowListArray(list, index); + case DoubleArray doubles: return doubles.GetValue(index).ToString(); + case FloatArray floats: return floats.GetValue(index).ToString(); + case Decimal128Array decimals: return decimals.GetValue(index).ToString(); + case Int32Array ints: return ints.GetValue(index).ToString(); + case Int64Array longs: return longs.GetValue(index).ToString(); + default: + { + var str = ((StringArray)array).GetString(index); + return string.IsNullOrEmpty(str) ? "undefined" : + $"\"{str}\"" + .Replace("\"{", "{").Replace("}\"", "}") + .Replace("\"[", "[").Replace("]\"", "]"); + } + }; + } + + private string FormatStructArray(StructArray structArray, int index) + { + var sb = new StringBuilder(); + var structTypeFields = ((StructType)structArray.Data.DataType).Fields; + var end = structArray.Fields.Count; + sb.Append("{"); + for (int i = 0; i < end; i++) + { + var field = structArray.Fields[i]; + var value = FormatArrowValue(field, index); + if (value == "undefined" && end == 1) + return "null"; + sb.Append($"\"{structTypeFields[i].Name}\""); + sb.Append(": "); + sb.Append($"{value}"); + if (i != end - 1) + sb.Append(", "); + } + sb.Append("}"); + return sb.ToString(); + } + + private string FormatArrowListArray(ListArray listArray, int index) + { + var start = listArray.ValueOffsets[index]; + var end = listArray.ValueOffsets[index + 1]; + if (start == end) + return "null"; + var sb = new StringBuilder(); + var values = listArray.Values; + sb.Append("["); + for (int i = start; i < end; i++) + { + sb.Append($"{FormatArrowValue(values, i)}"); + if (i != end - 1) + sb.Append(", "); + } + sb.Append("]"); + return sb.ToString(); + } + + private string FormatArrowMapArray(MapArray mapArray, int index) + { + var start = mapArray.ValueOffsets[index]; + var end = mapArray.ValueOffsets[index + 1]; + if (start == end) + return "null"; + var sb = new StringBuilder(); + var keyValuesArray = mapArray.KeyValues.Slice(start, end - start) as StructArray; + var keyArray = keyValuesArray.Fields[0]; + var valueArray = keyValuesArray.Fields[1]; + sb.Append("{"); + for (int i = start; i < end; i++) + { + sb.Append($"{FormatArrowValue(keyArray, i)}"); + sb.Append(": "); + sb.Append($"{FormatArrowValue(valueArray, i)}"); + if (i != end - 1) + sb.Append(", "); + } + sb.Append("}"); + return sb.ToString(); + } } } From 76c7f04e241cd852a9d42c18c6326fbd3c5f394c Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Jul 2025 18:06:11 -0700 Subject: [PATCH 09/33] Use JsonSerializer to escape special characters --- Snowflake.Data/Core/ArrowResultChunk.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 1995f391c..0e8428d22 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using System.Text.Json; using Apache.Arrow; using Apache.Arrow.Types; @@ -396,9 +397,7 @@ private string FormatArrowValue(IArrowArray array, int index) { var str = ((StringArray)array).GetString(index); return string.IsNullOrEmpty(str) ? "undefined" : - $"\"{str}\"" - .Replace("\"{", "{").Replace("}\"", "}") - .Replace("\"[", "[").Replace("]\"", "]"); + JsonSerializer.Serialize(str); } }; } From 10dcf1b52d56e37f14e2d2b73d358b585962389f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 2 Sep 2025 08:36:51 -0700 Subject: [PATCH 10/33] Add converter for native arrow array types --- .../IntegrationTests/StructuredArraysIT.cs | 23 ++- .../IntegrationTests/StructuredMapsIT.cs | 8 +- .../IntegrationTests/StructuredObjectsIT.cs | 17 +- .../Client/SnowflakeDbDataReader.cs | 38 ++-- Snowflake.Data/Core/ArrowResultChunk.cs | 115 ++++++----- Snowflake.Data/Core/ArrowResultSet.cs | 14 ++ .../ArrowToStructuredTypeConverter.cs | 179 ++++++++++++++++++ .../JsonToStructuredTypeConverter.cs | 2 +- 8 files changed, 317 insertions(+), 79 deletions(-) create mode 100644 Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index 9b4235a63..a19d75fd1 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -27,6 +27,9 @@ public StructuredArraysIT(ResultFormat resultFormat, bool nativeArrow) [Test] public void TestDataTableLoadOnStructuredArray() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { @@ -430,7 +433,11 @@ public void TestThrowExceptionForInvalidArrayElement() // assert SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + else + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an array.")); + } } } @@ -456,9 +463,17 @@ public void TestThrowExceptionForNextedInvalidElement() var thrown = Assert.Throws(() => reader.GetArray(0)); // assert - SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[0][1]")); - Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[0][1]")); + Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + } + else + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an array.")); + } } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index c4f0050fe..cebc8fdd0 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -26,6 +26,9 @@ public StructuredMapsIT(ResultFormat resultFormat, bool nativeArrow) [Test] public void TestDataTableLoadOnStructuredMap() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { @@ -346,7 +349,10 @@ public void TestThrowExceptionForInvalidMapElement() // assert SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]")); + else + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting a map.")); } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index 1494a5aac..fd25684f6 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -25,6 +25,9 @@ public StructuredObjectIT(ResultFormat resultFormat, bool nativeArrow) [Test] public void TestDataTableLoadOnStructuredObject() { + if (_resultFormat != ResultFormat.JSON) + Assert.Ignore("skip test on arrow"); + // arrange using (var connection = new SnowflakeDbConnection(ConnectionString)) { @@ -470,9 +473,17 @@ public void TestThrowExceptionForInvalidPropertyType() var thrown = Assert.Throws(() => reader.GetObject(0)); // assert - SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); - Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1].")); - Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + if (_resultFormat == ResultFormat.JSON || !_nativeArrow) + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_DETAILED_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1].")); + Assert.That(thrown.Message, Does.Contain("Could not read text type into System.Int32")); + } + else + { + SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR); + Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting an object.")); + } } } } diff --git a/Snowflake.Data/Client/SnowflakeDbDataReader.cs b/Snowflake.Data/Client/SnowflakeDbDataReader.cs index 313829054..b8827c9cc 100755 --- a/Snowflake.Data/Client/SnowflakeDbDataReader.cs +++ b/Snowflake.Data/Client/SnowflakeDbDataReader.cs @@ -262,9 +262,15 @@ public T GetObject(int ordinal) { throw new StructuredTypesReadingException($"Method GetObject<{typeof(T)}> can be used only for structured object"); } - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertObject(fields, json); + var val = GetValue(ordinal); + if (val is string stringValue) + { + var json = stringValue == null ? null : JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertObject(fields, json); + } + if (val is Dictionary structArray) + return ArrowConverter.ConvertObject(structArray); + return null; } catch (Exception e) { @@ -286,10 +292,15 @@ public T[] GetArray(int ordinal) { throw new StructuredTypesReadingException($"Method GetArray<{typeof(T)}> can be used only for structured array or vector types"); } - - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JArray.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertArray(fields, json); + var val = GetValue(ordinal); + if (val is string stringValue) + { + var json = stringValue == null ? null : JArray.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertArray(fields, json); + } + if (val is List listArray) + return ArrowConverter.ConvertArray(listArray); + return null; } catch (Exception e) { @@ -309,10 +320,15 @@ public Dictionary GetMap(int ordinal) { throw new StructuredTypesReadingException($"Method GetMap<{typeof(TKey)}, {typeof(TValue)}> can be used only for structured map"); } - - var stringValue = GetString(ordinal); - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertMap(fields, json); + var val = GetValue(ordinal); + if (val is string stringValue) + { + var json = stringValue == null ? null : JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertMap(fields, json); + } + if (val is Dictionary mapArray) + return ArrowConverter.ConvertMap(mapArray); + return null; } catch (Exception e) { diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 0e8428d22..3820290f9 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Text; -using System.Text.Json; using Apache.Arrow; using Apache.Arrow.Types; @@ -206,11 +205,11 @@ public object ExtractCell(int columnIndex, SFDataType srcType, long scale) switch (column) { case StructArray structArray: - return FormatStructArray(structArray, _currentBatchIndex); + return ExtractStructArray(structArray, _currentRecordIndex); case MapArray mapArray: - return FormatArrowMapArray(mapArray, _currentRecordIndex); + return ExtractMapArray(mapArray, _currentRecordIndex); case ListArray listArray: - return FormatArrowListArray(listArray, _currentRecordIndex); + return ExtractListArray(listArray, _currentRecordIndex); default: if (_byte[columnIndex] == null || _int[columnIndex] == null) { @@ -381,89 +380,87 @@ private long ExtractFraction(long value, long scale) return ((value % s_powersOf10[scale]) * s_powersOf10[9 - scale]); } - private string FormatArrowValue(IArrowArray array, int index) + private object ConvertArrowValue(IArrowArray array, int index) { switch (array) { - case StructArray strct: return FormatStructArray(strct, index); - case MapArray map: return FormatArrowMapArray(map, index); - case ListArray list: return FormatArrowListArray(list, index); - case DoubleArray doubles: return doubles.GetValue(index).ToString(); - case FloatArray floats: return floats.GetValue(index).ToString(); - case Decimal128Array decimals: return decimals.GetValue(index).ToString(); - case Int32Array ints: return ints.GetValue(index).ToString(); - case Int64Array longs: return longs.GetValue(index).ToString(); + case StructArray strct: return ExtractStructArray(strct, index); + case MapArray map: return ExtractMapArray(map, index); + case ListArray list: return ExtractListArray(list, index); + case DoubleArray doubles: return doubles.GetValue(index); + case FloatArray floats: return floats.GetValue(index); + case Decimal128Array decimals: return decimals.GetValue(index); + case Int32Array ints: return ints.GetValue(index); + case Int64Array longs: return longs.GetValue(index); + case StringArray strArray: + var str = strArray.GetString(index); + return string.IsNullOrEmpty(str) ? null : str; default: - { - var str = ((StringArray)array).GetString(index); - return string.IsNullOrEmpty(str) ? "undefined" : - JsonSerializer.Serialize(str); - } - }; + throw new NotSupportedException($"Unsupported array type: {array.GetType()}"); + } } - private string FormatStructArray(StructArray structArray, int index) + private Dictionary ExtractStructArray(StructArray structArray, int index) { - var sb = new StringBuilder(); + var result = new Dictionary(); var structTypeFields = ((StructType)structArray.Data.DataType).Fields; - var end = structArray.Fields.Count; - sb.Append("{"); - for (int i = 0; i < end; i++) + + for (int i = 0; i < structArray.Fields.Count; i++) { var field = structArray.Fields[i]; - var value = FormatArrowValue(field, index); - if (value == "undefined" && end == 1) - return "null"; - sb.Append($"\"{structTypeFields[i].Name}\""); - sb.Append(": "); - sb.Append($"{value}"); - if (i != end - 1) - sb.Append(", "); + var fieldName = structTypeFields[i].Name; + var value = ConvertArrowValue(field, index); + + if (value == null && structArray.Fields.Count == 1) + return null; + + result[fieldName] = value; } - sb.Append("}"); - return sb.ToString(); + + return result; } - private string FormatArrowListArray(ListArray listArray, int index) + private List ExtractListArray(ListArray listArray, int index) { - var start = listArray.ValueOffsets[index]; - var end = listArray.ValueOffsets[index + 1]; + int start = listArray.ValueOffsets[index]; + int end = listArray.ValueOffsets[index + 1]; + if (start == end) - return "null"; - var sb = new StringBuilder(); + return null; + var values = listArray.Values; - sb.Append("["); + var result = new List(end - start); + for (int i = start; i < end; i++) { - sb.Append($"{FormatArrowValue(values, i)}"); - if (i != end - 1) - sb.Append(", "); + result.Add(ConvertArrowValue(values, i)); } - sb.Append("]"); - return sb.ToString(); + + return result; } - private string FormatArrowMapArray(MapArray mapArray, int index) + private Dictionary ExtractMapArray(MapArray mapArray, int index) { - var start = mapArray.ValueOffsets[index]; - var end = mapArray.ValueOffsets[index + 1]; + int start = mapArray.ValueOffsets[index]; + int end = mapArray.ValueOffsets[index + 1]; + if (start == end) - return "null"; - var sb = new StringBuilder(); + return null; + var keyValuesArray = mapArray.KeyValues.Slice(start, end - start) as StructArray; var keyArray = keyValuesArray.Fields[0]; var valueArray = keyValuesArray.Fields[1]; - sb.Append("{"); - for (int i = start; i < end; i++) + + var result = new Dictionary(); + + for (int i = 0; i < end - start; i++) { - sb.Append($"{FormatArrowValue(keyArray, i)}"); - sb.Append(": "); - sb.Append($"{FormatArrowValue(valueArray, i)}"); - if (i != end - 1) - sb.Append(", "); + var key = ConvertArrowValue(keyArray, i); + var value = ConvertArrowValue(valueArray, i); + result[key] = value; } - sb.Append("}"); - return sb.ToString(); + + return result; } } } diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index 622518c80..1eafc9171 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; @@ -205,6 +206,15 @@ internal override object GetValue(int ordinal) case DateTimeOffset ret: obj = ret; break; + case Dictionary ret: + obj = ret; + break; + case Dictionary ret: + obj = ret; + break; + case List ret: + obj = ret; + break; default: { var dstType = sfResultSetMetaData.GetCSharpTypeByIndex(ordinal); @@ -395,6 +405,10 @@ internal override string GetString(int ordinal) { case string ret: return ret; + case Dictionary _: + case Dictionary _: + case List _: + return string.IsNullOrEmpty(value.ToString()) ? null :System.Text.Json.JsonSerializer.Serialize(value); case DateTime ret: if (type == SFDataType.DATE) return SFDataConverter.ToDateString(ret, sfResultSetMetaData.dateOutputFormat); diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs new file mode 100644 index 000000000..1a95ce204 --- /dev/null +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -0,0 +1,179 @@ +using Snowflake.Data.Client; +using System.Collections.Generic; +using System.Reflection; +using System; +using System.Linq; + +namespace Snowflake.Data.Core.Converter +{ + internal static class ArrowConverter + { + internal static T ConvertObject(Dictionary dict) where T : new() + { + T obj = new T(); + Type type = typeof(T); + if (type.GetCustomAttributes(false).Any(attribute => attribute.GetType() == typeof(SnowflakeObject))) + { + var constructionMethod = JsonToStructuredTypeConverter.GetConstructionMethod(type); + if (constructionMethod == SnowflakeObjectConstructionMethod.PROPERTIES_NAMES) + { + MapPropertiesByNames(obj, dict, type); + } + else if (constructionMethod == SnowflakeObjectConstructionMethod.PROPERTIES_ORDER) + { + MapPropertiesByOrder(obj, dict, type); + } + else if (constructionMethod == SnowflakeObjectConstructionMethod.CONSTRUCTOR) + { + return MapUsingConstructor(dict, type); + } + } + else + { + foreach (var kvp in dict) + { + var prop = type.GetProperty(kvp.Key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (prop == null) + continue; + prop.SetValue(obj, ConvertValue(kvp.Value, prop.PropertyType)); + } + } + return obj; + } + + private static void MapPropertiesByNames(object obj, Dictionary dict, Type type) + { + foreach (var kvp in dict) + { + var prop = type.GetProperty(kvp.Key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (prop != null) + { + var converted = ConvertValue(kvp.Value, prop.PropertyType); + prop.SetValue(obj, converted); + } + else + { + foreach (var property in type.GetProperties()) + { + foreach (var attr in property.GetCustomAttributes().OfType()) + { + if (attr?.Name == kvp.Key) + { + var converted = ConvertValue(kvp.Value, property.PropertyType); + property.SetValue(obj, converted); + } + } + } + } + } + } + + private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) + { + var index = 0; + foreach (var property in type.GetProperties()) + { + if (index >= dict.Count) + break; + + var attributes = property.GetCustomAttributes().OfType().ToList(); + if (attributes.Count == 0 || attributes.All(attr => !attr.IgnoreForPropertyOrder)) + { + var converted = ConvertValue(dict.ElementAt(index).Value, property.PropertyType); + property.SetValue(obj, converted); + index++; + } + } + } + + private static T MapUsingConstructor(Dictionary dict, Type type) + { + var matchingConstructor = type.GetConstructors() + .FirstOrDefault(c => c.GetParameters().Length == dict.Count) ?? + throw new StructuredTypesReadingException($"No constructor found for type: {type}"); + var parameters = matchingConstructor.GetParameters() + .Select((param, index) => ConvertValue(dict.ElementAt(index).Value, param.ParameterType)) + .ToArray(); + return (T)matchingConstructor.Invoke(parameters); + } + + internal static object CallMethod(Type type, object obj, string methodName, Type type2 = null) + { + MethodInfo genericMethod = typeof(ArrowConverter) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo constructedMethod = type2 == null + ? genericMethod.MakeGenericMethod(type) + : genericMethod.MakeGenericMethod(type, type2); + return constructedMethod.Invoke(null, new object[] { obj }); + } + + internal static T[] ConvertArray(List list) + { + var targetType = typeof(T); + var result = new T[list.Count]; + for (int i = 0; i < list.Count; i++) + { + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) + targetType = Nullable.GetUnderlyingType(targetType); + result[i] = (T)ConvertValue(list[i], targetType); + } + return result; + } + + private static List ConvertList(List list) + { + var targetType = typeof(T); + var result = new List(list.Count); + foreach (var item in list) + { + result.Add((T)ConvertValue(item, targetType)); + } + return result; + } + + internal static Dictionary ConvertMap(Dictionary dict) + { + var keyType = typeof(TKey); + var valueType = typeof(TValue); + var result = new Dictionary(); + foreach (var kvp in dict) + { + var key = (TKey)ConvertValue(kvp.Key, keyType); + var value = (TValue)ConvertValue(kvp.Value, valueType); + result[key] = value; + } + return result; + } + + private static object ConvertValue(object value, Type targetType) + { + if (value == null) + return null; + if (targetType.IsAssignableFrom(value.GetType())) + return value; + if (value is Dictionary objDict) + return CallMethod(targetType, objDict, nameof(ConvertObject)); + if (value is Dictionary mapDict) + { + var genericArgs = targetType.GetGenericArguments(); + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + } + if (value is List objList) + { + if (targetType.IsArray) + { + var elementType = targetType.GetElementType(); + return CallMethod(elementType, objList, nameof(ConvertArray)); + } + else if (targetType.IsGenericType) + { + var elementType = targetType.GetGenericArguments()[0]; + return CallMethod(elementType, objList, nameof(ConvertList)); + } + } + return Convert.ChangeType(value, targetType); + } + } +} diff --git a/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs index fae3a83b0..fbaba1e07 100644 --- a/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs @@ -90,7 +90,7 @@ private static object ConvertToObject(Type type, List fields, Str return objectBuilder.Build(); } - private static SnowflakeObjectConstructionMethod GetConstructionMethod(Type type) + internal static SnowflakeObjectConstructionMethod GetConstructionMethod(Type type) { return type.GetCustomAttributes(false) .Where(attribute => attribute.GetType() == typeof(SnowflakeObject)) From d1da946f636abd94618aa6fd8f37ac4c34d241fc Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 10 Sep 2025 07:28:18 -0700 Subject: [PATCH 11/33] Modify test to compare GetString result from arrow and json --- .../IntegrationTests/StructuredArraysIT.cs | 12 ++++++++++++ .../IntegrationTests/StructuredMapsIT.cs | 12 ++++++++++++ .../IntegrationTests/StructuredObjectsIT.cs | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index a19d75fd1..ffd4ca36e 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -78,6 +79,17 @@ public void TestSelectArray() // assert Assert.AreEqual(3, array.Length); CollectionAssert.AreEqual(new[] { "a", "b", "c" }, array); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JArray.Parse(jsonString), JArray.Parse(arrowString))); + } } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs index cebc8fdd0..67ed38120 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -79,6 +80,17 @@ public void TestSelectMap() Assert.AreEqual("San Mateo", map["city"]); Assert.AreEqual("CA", map["state"]); Assert.AreEqual("01-234", map["zip"]); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JObject.Parse(jsonString), JObject.Parse(arrowString))); + } } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs index fd25684f6..25ae55d23 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Data; +using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Core; @@ -76,6 +77,17 @@ public void TestSelectStructuredTypeObject() Assert.AreEqual("San Mateo", address.city); Assert.AreEqual("CA", address.state); Assert.IsNull(address.zip); + + if (_nativeArrow) + { + var arrowString = reader.GetString(0); + EnableStructuredTypes(connection, ResultFormat.JSON); + reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + var jsonString = reader.GetString(0); + + Assert.IsTrue(JToken.DeepEquals(JObject.Parse(jsonString), JObject.Parse(arrowString))); + } } } } From 4eae30c257d146427c3f8784a3d13e67e42691d6 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 17 Sep 2025 10:52:05 -0700 Subject: [PATCH 12/33] Use converter that supports .NET Framework --- Snowflake.Data/Core/ArrowResultSet.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index d0353d50f..11cb31e51 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -408,7 +408,7 @@ internal override string GetString(int ordinal) case Dictionary _: case Dictionary _: case List _: - return string.IsNullOrEmpty(value.ToString()) ? null :System.Text.Json.JsonSerializer.Serialize(value); + return string.IsNullOrEmpty(value.ToString()) ? null : Newtonsoft.Json.JsonConvert.SerializeObject(value); case DateTime ret: if (type == SFDataType.DATE) return SFDataConverter.ToDateString(ret, sfResultSetMetaData.dateOutputFormat); From 85c7a10a30dc44363c8340ff7268c0d7984fe203 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 17 Oct 2025 10:35:44 -0700 Subject: [PATCH 13/33] Add case for boolean --- Snowflake.Data/Core/ArrowResultChunk.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 3820290f9..850db3beb 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -392,6 +392,7 @@ private object ConvertArrowValue(IArrowArray array, int index) case Decimal128Array decimals: return decimals.GetValue(index); case Int32Array ints: return ints.GetValue(index); case Int64Array longs: return longs.GetValue(index); + case BooleanArray booleans: return booleans.GetValue(index); case StringArray strArray: var str = strArray.GetString(index); return string.IsNullOrEmpty(str) ? null : str; From 43225924ceb5a2e36f09066e13311c2635b59243 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 31 Oct 2025 11:07:53 -0700 Subject: [PATCH 14/33] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c757ff65..3cce67b67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,3 +10,5 @@ - Added the `changelog.yml` GitHub workflow to ensure changelog is updated on release PRs. - Removed internal classes from public API. - Added support for explicitly setting Azure managed identity client ID via `MANAGED_IDENTITY_CLIENT_ID` environment variable. +- v5.0.1 + - Add support for native arrow structured types. From 29bdea5eb0680f0f96046aecc61c471928da4a9e Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 31 Oct 2025 12:19:50 -0700 Subject: [PATCH 15/33] Add other cases --- Snowflake.Data/Core/ArrowResultChunk.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 850db3beb..81d712fcb 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -387,15 +387,20 @@ private object ConvertArrowValue(IArrowArray array, int index) case StructArray strct: return ExtractStructArray(strct, index); case MapArray map: return ExtractMapArray(map, index); case ListArray list: return ExtractListArray(list, index); + case FixedSizeListArray fixedSizeList: return ExtractFixedSizeListArray(fixedSizeList, index); case DoubleArray doubles: return doubles.GetValue(index); case FloatArray floats: return floats.GetValue(index); case Decimal128Array decimals: return decimals.GetValue(index); + case Date32Array dates: return dates.GetValue(index); + case Int8Array bytes: return bytes.GetValue(index); + case Int16Array shorts: return shorts.GetValue(index); case Int32Array ints: return ints.GetValue(index); case Int64Array longs: return longs.GetValue(index); case BooleanArray booleans: return booleans.GetValue(index); case StringArray strArray: var str = strArray.GetString(index); return string.IsNullOrEmpty(str) ? null : str; + case BinaryArray binary: return binary.GetBytes(index).ToArray(); default: throw new NotSupportedException($"Unsupported array type: {array.GetType()}"); } @@ -463,5 +468,21 @@ private Dictionary ExtractMapArray(MapArray mapArray, int index) return result; } + + private List ExtractFixedSizeListArray(FixedSizeListArray fixedSizeListArray, int index) + { + var subArray = fixedSizeListArray.GetSlicedValues(index); + + if (subArray.Length == 0) + return null; + + var result = new List(subArray.Length); + for (int i = 0; i < subArray.Length; i++) + { + result.Add(ConvertArrowValue(subArray, i)); + } + + return result; + } } } From 7093415616885983c62b51360058e69df1e809fa Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 3 Nov 2025 09:40:53 -0800 Subject: [PATCH 16/33] Refactor to switch case --- .../Client/SnowflakeDbDataReader.cs | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/Snowflake.Data/Client/SnowflakeDbDataReader.cs b/Snowflake.Data/Client/SnowflakeDbDataReader.cs index b8827c9cc..b7f0d181d 100755 --- a/Snowflake.Data/Client/SnowflakeDbDataReader.cs +++ b/Snowflake.Data/Client/SnowflakeDbDataReader.cs @@ -263,14 +263,18 @@ public T GetObject(int ordinal) throw new StructuredTypesReadingException($"Method GetObject<{typeof(T)}> can be used only for structured object"); } var val = GetValue(ordinal); - if (val is string stringValue) + switch (val) { - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertObject(fields, json); + case string stringValue: + { + var json = JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertObject(fields, json); + } + case Dictionary structArray: + return ArrowConverter.ConvertObject(structArray); + default: + return null; } - if (val is Dictionary structArray) - return ArrowConverter.ConvertObject(structArray); - return null; } catch (Exception e) { @@ -293,14 +297,18 @@ public T[] GetArray(int ordinal) throw new StructuredTypesReadingException($"Method GetArray<{typeof(T)}> can be used only for structured array or vector types"); } var val = GetValue(ordinal); - if (val is string stringValue) + switch (val) { - var json = stringValue == null ? null : JArray.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertArray(fields, json); + case string stringValue: + { + var json = stringValue == null ? null : JArray.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertArray(fields, json); + } + case List listArray: + return ArrowConverter.ConvertArray(listArray); + default: + return null; } - if (val is List listArray) - return ArrowConverter.ConvertArray(listArray); - return null; } catch (Exception e) { @@ -321,14 +329,18 @@ public Dictionary GetMap(int ordinal) throw new StructuredTypesReadingException($"Method GetMap<{typeof(TKey)}, {typeof(TValue)}> can be used only for structured map"); } var val = GetValue(ordinal); - if (val is string stringValue) + switch (val) { - var json = stringValue == null ? null : JObject.Parse(stringValue); - return JsonToStructuredTypeConverter.ConvertMap(fields, json); + case string stringValue: + { + var json = stringValue == null ? null : JObject.Parse(stringValue); + return JsonToStructuredTypeConverter.ConvertMap(fields, json); + } + case Dictionary mapArray: + return ArrowConverter.ConvertMap(mapArray); + default: + return null; } - if (val is Dictionary mapArray) - return ArrowConverter.ConvertMap(mapArray); - return null; } catch (Exception e) { From 56d041026f942dc7ca4a2e49bc81a2aa9f4b68ff Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 3 Nov 2025 12:14:36 -0800 Subject: [PATCH 17/33] Refactor to switch case --- .../ArrowToStructuredTypeConverter.cs | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 1a95ce204..c223558f7 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -147,33 +147,34 @@ internal static Dictionary ConvertMap(Dictionary objDict) - return CallMethod(targetType, objDict, nameof(ConvertObject)); - if (value is Dictionary mapDict) + switch (value) { - var genericArgs = targetType.GetGenericArguments(); - var keyType = genericArgs[0]; - var valueType = genericArgs[1]; - return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); - } - if (value is List objList) - { - if (targetType.IsArray) - { - var elementType = targetType.GetElementType(); - return CallMethod(elementType, objList, nameof(ConvertArray)); - } - else if (targetType.IsGenericType) - { - var elementType = targetType.GetGenericArguments()[0]; - return CallMethod(elementType, objList, nameof(ConvertList)); - } + case null: + return null; + case var _ when targetType.IsAssignableFrom(value.GetType()): + return value; + case Dictionary objDict: + return CallMethod(targetType, objDict, nameof(ConvertObject)); + case Dictionary mapDict: + var genericArgs = targetType.GetGenericArguments(); + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + case List objList: + if (targetType.IsArray) + { + var elementType = targetType.GetElementType(); + return CallMethod(elementType, objList, nameof(ConvertArray)); + } + else if (targetType.IsGenericType) + { + var elementType = targetType.GetGenericArguments()[0]; + return CallMethod(elementType, objList, nameof(ConvertList)); + } + goto default; + default: + return Convert.ChangeType(value, targetType); } - return Convert.ChangeType(value, targetType); } } } From 673ec13bdb493e20de451cc561694e1a98b1788d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 3 Nov 2025 12:17:35 -0800 Subject: [PATCH 18/33] Refactor function --- .../ArrowToStructuredTypeConverter.cs | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index c223558f7..c3eabfe28 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -45,29 +45,27 @@ private static void MapPropertiesByNames(object obj, Dictionary { foreach (var kvp in dict) { - var prop = type.GetProperty(kvp.Key, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + var prop = FindPropertyByName(type, kvp.Key); if (prop != null) { var converted = ConvertValue(kvp.Value, prop.PropertyType); prop.SetValue(obj, converted); } - else - { - foreach (var property in type.GetProperties()) - { - foreach (var attr in property.GetCustomAttributes().OfType()) - { - if (attr?.Name == kvp.Key) - { - var converted = ConvertValue(kvp.Value, property.PropertyType); - property.SetValue(obj, converted); - } - } - } - } } } + private static PropertyInfo FindPropertyByName(Type type, string name) + { + var prop = type.GetProperty(name, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (prop != null) + return prop; + + return type.GetProperties() + .FirstOrDefault(p => p.GetCustomAttributes() + .OfType() + .Any(attr => attr?.Name == name)); + } + private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) { var index = 0; From 4acdec4bd339235519c115a91edb7dc2b1aa5f67 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 3 Nov 2025 12:20:19 -0800 Subject: [PATCH 19/33] Refactor to switch case --- .../ArrowToStructuredTypeConverter.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index c3eabfe28..80af32642 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -15,17 +15,16 @@ internal static class ArrowConverter if (type.GetCustomAttributes(false).Any(attribute => attribute.GetType() == typeof(SnowflakeObject))) { var constructionMethod = JsonToStructuredTypeConverter.GetConstructionMethod(type); - if (constructionMethod == SnowflakeObjectConstructionMethod.PROPERTIES_NAMES) + switch (constructionMethod) { - MapPropertiesByNames(obj, dict, type); - } - else if (constructionMethod == SnowflakeObjectConstructionMethod.PROPERTIES_ORDER) - { - MapPropertiesByOrder(obj, dict, type); - } - else if (constructionMethod == SnowflakeObjectConstructionMethod.CONSTRUCTOR) - { - return MapUsingConstructor(dict, type); + case SnowflakeObjectConstructionMethod.PROPERTIES_NAMES: + MapPropertiesByNames(obj, dict, type); + break; + case SnowflakeObjectConstructionMethod.PROPERTIES_ORDER: + MapPropertiesByOrder(obj, dict, type); + break; + case SnowflakeObjectConstructionMethod.CONSTRUCTOR: + return MapUsingConstructor(dict, type); } } else From 74c0ac5d9dac177b5702004617d87ac40ab570f4 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 4 Nov 2025 12:39:29 -0800 Subject: [PATCH 20/33] Move if check outside the for loop --- .../Core/Converter/ArrowToStructuredTypeConverter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 80af32642..e4c249d18 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -108,10 +108,10 @@ internal static T[] ConvertArray(List list) { var targetType = typeof(T); var result = new T[list.Count]; + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) + targetType = Nullable.GetUnderlyingType(targetType); for (int i = 0; i < list.Count; i++) { - if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) - targetType = Nullable.GetUnderlyingType(targetType); result[i] = (T)ConvertValue(list[i], targetType); } return result; From ef3179f54f26bc16f09bd7f0bda6de9d7abd8e77 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 4 Nov 2025 12:50:29 -0800 Subject: [PATCH 21/33] Fix indentation --- Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index e4c249d18..623c139fa 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -109,7 +109,7 @@ internal static T[] ConvertArray(List list) var targetType = typeof(T); var result = new T[list.Count]; if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(Nullable<>)) - targetType = Nullable.GetUnderlyingType(targetType); + targetType = Nullable.GetUnderlyingType(targetType); for (int i = 0; i < list.Count; i++) { result[i] = (T)ConvertValue(list[i], targetType); From 22140ec5f3d11c57700a7b3483d0ca6e91e8534a Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 13:18:50 -0800 Subject: [PATCH 22/33] Add type validation --- .../Converter/ArrowToStructuredTypeConverter.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 623c139fa..def1e6a6b 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -153,10 +153,17 @@ private static object ConvertValue(object value, Type targetType) case Dictionary objDict: return CallMethod(targetType, objDict, nameof(ConvertObject)); case Dictionary mapDict: - var genericArgs = targetType.GetGenericArguments(); - var keyType = genericArgs[0]; - var valueType = genericArgs[1]; - return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + if (targetType.GetGenericTypeDefinition() == typeof(Dictionary<,>)) + { + var genericArgs = targetType.GetGenericArguments(); + if (genericArgs.Length == 2) + { + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + } + } + goto default; case List objList: if (targetType.IsArray) { From 98cfe0223f1675cf5637b4ae7d23d18e799a864c Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 13:39:28 -0800 Subject: [PATCH 23/33] Check if type has more than 2 arguments --- Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index def1e6a6b..3d6124a60 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -156,7 +156,7 @@ private static object ConvertValue(object value, Type targetType) if (targetType.GetGenericTypeDefinition() == typeof(Dictionary<,>)) { var genericArgs = targetType.GetGenericArguments(); - if (genericArgs.Length == 2) + if (genericArgs.Length >= 2) { var keyType = genericArgs[0]; var valueType = genericArgs[1]; From b6c9959c931d75caff26972f0453f131281c0822 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 14:03:38 -0800 Subject: [PATCH 24/33] Revert type check --- .../Converter/ArrowToStructuredTypeConverter.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 3d6124a60..c852d96d9 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -153,15 +153,13 @@ private static object ConvertValue(object value, Type targetType) case Dictionary objDict: return CallMethod(targetType, objDict, nameof(ConvertObject)); case Dictionary mapDict: - if (targetType.GetGenericTypeDefinition() == typeof(Dictionary<,>)) + var genericArgs = targetType.GetGenericArguments(); + Console.WriteLine(targetType.FullName + $"\n{genericArgs.Length}"); + if (genericArgs.Length == 2) { - var genericArgs = targetType.GetGenericArguments(); - if (genericArgs.Length >= 2) - { - var keyType = genericArgs[0]; - var valueType = genericArgs[1]; - return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); - } + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); } goto default; case List objList: From 7a90867940e75c556c12de517485b732ad77af84 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 14:39:32 -0800 Subject: [PATCH 25/33] Re-add type check --- .../Converter/ArrowToStructuredTypeConverter.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index c852d96d9..66cb2164d 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -153,13 +153,15 @@ private static object ConvertValue(object value, Type targetType) case Dictionary objDict: return CallMethod(targetType, objDict, nameof(ConvertObject)); case Dictionary mapDict: - var genericArgs = targetType.GetGenericArguments(); - Console.WriteLine(targetType.FullName + $"\n{genericArgs.Length}"); - if (genericArgs.Length == 2) + if (targetType.IsGenericType) { - var keyType = genericArgs[0]; - var valueType = genericArgs[1]; - return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + var genericArgs = targetType.GetGenericArguments(); + if (genericArgs.Length == 2) + { + var keyType = genericArgs[0]; + var valueType = genericArgs[1]; + return CallMethod(keyType, mapDict, nameof(ConvertMap), valueType); + } } goto default; case List objList: From 1123430abf6ad6f5ac605b6d8f5eca392c9048d6 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 16:16:05 -0800 Subject: [PATCH 26/33] Use ObjectBuilder to get properties --- .../ArrowToStructuredTypeConverter.cs | 19 ++++++++----------- .../Builder/ObjectBuilderByPropertyOrder.cs | 9 +++++++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 66cb2164d..02e72f4b8 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -3,6 +3,7 @@ using System.Reflection; using System; using System.Linq; +using Snowflake.Data.Core.Converter.Builder; namespace Snowflake.Data.Core.Converter { @@ -67,19 +68,15 @@ private static PropertyInfo FindPropertyByName(Type type, string name) private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) { - var index = 0; - foreach (var property in type.GetProperties()) + var properties = ObjectBuilderByPropertyOrder.GetProperties(type); + var i = 0; + foreach (var property in properties) { - if (index >= dict.Count) + if (i >= dict.Count) break; - - var attributes = property.GetCustomAttributes().OfType().ToList(); - if (attributes.Count == 0 || attributes.All(attr => !attr.IgnoreForPropertyOrder)) - { - var converted = ConvertValue(dict.ElementAt(index).Value, property.PropertyType); - property.SetValue(obj, converted); - index++; - } + var converted = ConvertValue(dict.ElementAt(i).Value, property.PropertyType); + property.SetValue(obj, converted); + i++; } } diff --git a/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs b/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs index 45c1c0db8..7d0e5c385 100644 --- a/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs +++ b/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs @@ -17,12 +17,17 @@ internal class ObjectBuilderByPropertyOrder : IObjectBuilder public ObjectBuilderByPropertyOrder(Type type) { _type = type; - _properties = type.GetProperties().Where(property => !IsIgnoredForPropertiesOrder(property)).ToArray(); + _properties = GetProperties(type); _index = -1; _result = new List>(); } - private bool IsIgnoredForPropertiesOrder(PropertyInfo property) + internal static PropertyInfo[] GetProperties(Type type) + { + return type.GetProperties().Where(property => !IsIgnoredForPropertiesOrder(property)).ToArray(); + } + + private static bool IsIgnoredForPropertiesOrder(PropertyInfo property) { var sfAnnotation = property.GetCustomAttributes().FirstOrDefault(); return sfAnnotation != null && sfAnnotation.IgnoreForPropertyOrder; From 7433fbec18c03401684a70636c90aa94ce982f87 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 16:23:49 -0800 Subject: [PATCH 27/33] Revert last and use FirstOrDefault to get custom attributes --- .../ArrowToStructuredTypeConverter.cs | 19 +++++++++++-------- .../Builder/ObjectBuilderByPropertyOrder.cs | 9 ++------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 02e72f4b8..4097c0bad 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -3,7 +3,6 @@ using System.Reflection; using System; using System.Linq; -using Snowflake.Data.Core.Converter.Builder; namespace Snowflake.Data.Core.Converter { @@ -68,15 +67,19 @@ private static PropertyInfo FindPropertyByName(Type type, string name) private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) { - var properties = ObjectBuilderByPropertyOrder.GetProperties(type); - var i = 0; - foreach (var property in properties) + var index = 0; + foreach (var property in type.GetProperties()) { - if (i >= dict.Count) + if (index >= dict.Count) break; - var converted = ConvertValue(dict.ElementAt(i).Value, property.PropertyType); - property.SetValue(obj, converted); - i++; + + var attribute = property.GetCustomAttributes().OfType().FirstOrDefault(); + if (attribute != null && !attribute.IgnoreForPropertyOrder) + { + var converted = ConvertValue(dict.ElementAt(index).Value, property.PropertyType); + property.SetValue(obj, converted); + index++; + } } } diff --git a/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs b/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs index 7d0e5c385..45c1c0db8 100644 --- a/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs +++ b/Snowflake.Data/Core/Converter/Builder/ObjectBuilderByPropertyOrder.cs @@ -17,17 +17,12 @@ internal class ObjectBuilderByPropertyOrder : IObjectBuilder public ObjectBuilderByPropertyOrder(Type type) { _type = type; - _properties = GetProperties(type); + _properties = type.GetProperties().Where(property => !IsIgnoredForPropertiesOrder(property)).ToArray(); _index = -1; _result = new List>(); } - internal static PropertyInfo[] GetProperties(Type type) - { - return type.GetProperties().Where(property => !IsIgnoredForPropertiesOrder(property)).ToArray(); - } - - private static bool IsIgnoredForPropertiesOrder(PropertyInfo property) + private bool IsIgnoredForPropertiesOrder(PropertyInfo property) { var sfAnnotation = property.GetCustomAttributes().FirstOrDefault(); return sfAnnotation != null && sfAnnotation.IgnoreForPropertyOrder; From 47acd9a4cc8c54d615c02e61f23ecb40e02f77a6 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 5 Nov 2025 16:54:23 -0800 Subject: [PATCH 28/33] Fix if check --- Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 4097c0bad..3ba1e9762 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -74,7 +74,7 @@ private static void MapPropertiesByOrder(object obj, Dictionary break; var attribute = property.GetCustomAttributes().OfType().FirstOrDefault(); - if (attribute != null && !attribute.IgnoreForPropertyOrder) + if (attribute == null || !attribute.IgnoreForPropertyOrder) { var converted = ConvertValue(dict.ElementAt(index).Value, property.PropertyType); property.SetValue(obj, converted); From 2b5cdb4af991b09243c4df6fa7e186a8bab3199e Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 7 Nov 2025 11:16:36 -0800 Subject: [PATCH 29/33] Get properties in order of metadata --- Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs index 3ba1e9762..149ceb38a 100644 --- a/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs +++ b/Snowflake.Data/Core/Converter/ArrowToStructuredTypeConverter.cs @@ -68,7 +68,7 @@ private static PropertyInfo FindPropertyByName(Type type, string name) private static void MapPropertiesByOrder(object obj, Dictionary dict, Type type) { var index = 0; - foreach (var property in type.GetProperties()) + foreach (var property in type.GetProperties().OrderBy(p => p.MetadataToken)) { if (index >= dict.Count) break; From 461c1f93b2007425561d5dc2142474bd06e9d5b4 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 7 Nov 2025 11:17:32 -0800 Subject: [PATCH 30/33] Fix whitespace --- Snowflake.Data/Core/ArrowResultChunk.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 81d712fcb..90b50855b 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -391,7 +391,7 @@ private object ConvertArrowValue(IArrowArray array, int index) case DoubleArray doubles: return doubles.GetValue(index); case FloatArray floats: return floats.GetValue(index); case Decimal128Array decimals: return decimals.GetValue(index); - case Date32Array dates: return dates.GetValue(index); + case Date32Array dates: return dates.GetValue(index); case Int8Array bytes: return bytes.GetValue(index); case Int16Array shorts: return shorts.GetValue(index); case Int32Array ints: return ints.GetValue(index); From 888c1f3cccf68b96fad91d443c2f20f71a6a0264 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 12 Nov 2025 11:42:25 -0800 Subject: [PATCH 31/33] Remove unsupported case of structured types --- Snowflake.Data/Core/ArrowResultChunk.cs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index 90b50855b..a3faede18 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -387,7 +387,6 @@ private object ConvertArrowValue(IArrowArray array, int index) case StructArray strct: return ExtractStructArray(strct, index); case MapArray map: return ExtractMapArray(map, index); case ListArray list: return ExtractListArray(list, index); - case FixedSizeListArray fixedSizeList: return ExtractFixedSizeListArray(fixedSizeList, index); case DoubleArray doubles: return doubles.GetValue(index); case FloatArray floats: return floats.GetValue(index); case Decimal128Array decimals: return decimals.GetValue(index); @@ -468,21 +467,5 @@ private Dictionary ExtractMapArray(MapArray mapArray, int index) return result; } - - private List ExtractFixedSizeListArray(FixedSizeListArray fixedSizeListArray, int index) - { - var subArray = fixedSizeListArray.GetSlicedValues(index); - - if (subArray.Length == 0) - return null; - - var result = new List(subArray.Length); - for (int i = 0; i < subArray.Length; i++) - { - result.Add(ConvertArrowValue(subArray, i)); - } - - return result; - } } } From b775286bc80511123808071a1f360279f42b444e Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 12 Nov 2025 14:30:17 -0800 Subject: [PATCH 32/33] Fix getting value from Date32Array --- Snowflake.Data/Core/ArrowResultChunk.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index a3faede18..5486316c3 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -390,7 +390,7 @@ private object ConvertArrowValue(IArrowArray array, int index) case DoubleArray doubles: return doubles.GetValue(index); case FloatArray floats: return floats.GetValue(index); case Decimal128Array decimals: return decimals.GetValue(index); - case Date32Array dates: return dates.GetValue(index); + case Date32Array dates: return dates.GetDateTime(index); case Int8Array bytes: return bytes.GetValue(index); case Int16Array shorts: return shorts.GetValue(index); case Int32Array ints: return ints.GetValue(index); From 4909f52d2fa6f4e20f60152cdc8790697527bbba Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Wed, 12 Nov 2025 14:31:03 -0800 Subject: [PATCH 33/33] Add tests for more array types --- .../IntegrationTests/StructuredArraysIT.cs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs index ffd4ca36e..a413b53e4 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; +using System.Text; using Newtonsoft.Json.Linq; using NUnit.Framework; using Snowflake.Data.Client; @@ -325,6 +326,82 @@ public void TestSelectArrayOfDoublesWithExponentNotation() } } + [Test] + public void TestSelectArrayOfBooleans() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfBooleans = "ARRAY_CONSTRUCT(true, false)::ARRAY(BOOLEAN)"; + command.CommandText = $"SELECT {arrayOfBooleans}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + + // assert + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new[] { true, false }, array); + } + } + } + + [Test] + public void TestSelectArrayOfBinaries() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfBinaries = "ARRAY_CONSTRUCT(TO_BINARY('AB', 'UTF-8'), TO_BINARY('BC', 'UTF-8'))::ARRAY(BINARY)"; + command.CommandText = $"SELECT {arrayOfBinaries}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + var strings = array.Select(b => Encoding.UTF8.GetString(b)).ToArray(); + + // assert + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new[] { "AB", "BC" }, strings); + } + } + } + + [Test] + public void TestSelectArrayOfDates() + { + // arrange + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + EnableStructuredTypes(connection, _resultFormat, _nativeArrow); + var arrayOfDates = "ARRAY_CONSTRUCT('2024-01-01'::DATE)::ARRAY(DATE)"; + command.CommandText = $"SELECT {arrayOfDates}"; + var reader = (SnowflakeDbDataReader)command.ExecuteReader(); + Assert.IsTrue(reader.Read()); + + // act + var array = reader.GetArray(0); + + // assert + Assert.AreEqual(1, array.Length); + CollectionAssert.AreEqual(new[] { DateTime.Parse("2024-01-01") }, array); + } + } + } + [Test] public void TestSelectStringArrayWithNulls() {