SNOW-1488703: Add arrow support for structured types

CHANGELOG.md

@@ -7,6 +7,7 @@
 - v5.1.0
     - Added `APPLICATION_PATH` to `CLIENT_ENVIRONMENT` sent during authentication to identify the application connecting to Snowflake.
     - Renew idle sessions in the pool if keep alive is enabled.
+    - Added support for native arrow structured types.
     - Added `CRLDOWNLOADMAXSIZE` connection parameter to limit the maximum size of CRL files downloaded during certificate revocation checks.
     - AWS WIF will now also check the application config and AWS profile credential store when determining the current AWS region
     - Allow users to configure the maximum amount of connections via `SERVICE_POINT_CONNECTION_LIMIT` property.

Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
+using Newtonsoft.Json.Linq;
 using NUnit.Framework;
 using Snowflake.Data.Client;
 using Snowflake.Data.Core;
@@ -9,19 +10,33 @@
 
 namespace Snowflake.Data.Tests.IntegrationTests
 {
-    [TestFixture]
+    [TestFixture(ResultFormat.ARROW, false)]
+    [TestFixture(ResultFormat.ARROW, true)]
+    [TestFixture(ResultFormat.JSON, false)]
     public class StructuredMapsIT : StructuredTypesIT
     {
+        private readonly ResultFormat _resultFormat;
+        private readonly bool _nativeArrow;
+
+        public StructuredMapsIT(ResultFormat resultFormat, bool nativeArrow)
+        {
+            _resultFormat = resultFormat;
+            _nativeArrow = nativeArrow;
+        }
+
         [Test]
         public void TestDataTableLoadOnStructuredMap()
         {
+            if (_resultFormat != ResultFormat.JSON)
+                Assert.Ignore("skip test on arrow");
+
             // arrange
             using (var connection = new SnowflakeDbConnection(ConnectionString))
             {
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var key = "city";
                     var value = "San Mateo";
                     var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)";
@@ -50,7 +65,7 @@ public void TestSelectMap()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', '01-234')::MAP(VARCHAR, VARCHAR)";
                     command.CommandText = $"SELECT {addressAsSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -65,6 +80,17 @@ public void TestSelectMap()
                     Assert.AreEqual("San Mateo", map["city"]);
                     Assert.AreEqual("CA", map["state"]);
                     Assert.AreEqual("01-234", map["zip"]);
+
+                    if (_nativeArrow)
+                    {
+                        var arrowString = reader.GetString(0);
+                        EnableStructuredTypes(connection, ResultFormat.JSON);
+                        reader = (SnowflakeDbDataReader)command.ExecuteReader();
+                        Assert.IsTrue(reader.Read());
+                        var jsonString = reader.GetString(0);
+
+                        Assert.IsTrue(JToken.DeepEquals(JObject.Parse(jsonString), JObject.Parse(arrowString)));
+                    }
                 }
             }
         }
@@ -78,7 +104,7 @@ public void TestSelectMapWithIntegerKeys()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)";
                     command.CommandText = $"SELECT {mapSfString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -106,7 +132,7 @@ public void TestSelectMapWithLongKeys()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)";
                     command.CommandText = $"SELECT {mapSfString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -134,7 +160,7 @@ public void TestSelectMapOfObjects()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapWitObjectValueSFString = @"OBJECT_CONSTRUCT(
                         'Warsaw', OBJECT_CONSTRUCT('prefix', '01', 'postfix', '234'),
                         'San Mateo', OBJECT_CONSTRUCT('prefix', '02', 'postfix', '567')
@@ -164,7 +190,7 @@ public void TestSelectMapOfArrays()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))";
                     command.CommandText = $"SELECT {mapWithArrayValueSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -190,7 +216,7 @@ public void TestSelectMapOfLists()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))";
                     command.CommandText = $"SELECT {mapWithArrayValueSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -216,7 +242,7 @@ public void TestSelectMapOfMaps()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var mapAsSFString = "OBJECT_CONSTRUCT('a', OBJECT_CONSTRUCT('b', 'c'))::MAP(TEXT, MAP(TEXT, TEXT))";
                     command.CommandText = $"SELECT {mapAsSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -246,7 +272,7 @@ public void TestSelectSemiStructuredTypesInMap(string valueSfString, string expe
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     command.CommandText = $"SELECT {valueSfString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
                     Assert.IsTrue(reader.Read());
@@ -271,7 +297,7 @@ public void TestSelectNullMap()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var nullMapSFString = "NULL::MAP(TEXT,TEXT)";
                     command.CommandText = $"SELECT {nullMapSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -295,7 +321,7 @@ public void TestThrowExceptionForInvalidMap()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var invalidMapSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT";
                     command.CommandText = $"SELECT {invalidMapSFString}";
                     var reader = (SnowflakeDbDataReader)command.ExecuteReader();
@@ -321,7 +347,7 @@ public void TestThrowExceptionForInvalidMapElement()
                 connection.Open();
                 using (var command = connection.CreateCommand())
                 {
-                    EnableStructuredTypes(connection);
+                    EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
                     var invalidMapSFString = @"OBJECT_CONSTRUCT(
                         'x', 'a76dacad-0e35-497b-bf9b-7cd49262b68b',
                         'y', 'z76dacad-0e35-497b-bf9b-7cd49262b68b'
@@ -335,7 +361,10 @@ public void TestThrowExceptionForInvalidMapElement()
 
                     // assert
                     SnowflakeDbExceptionAssert.HasErrorCode(thrown, SFError.STRUCTURED_TYPE_READ_ERROR);
-                    Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]"));
+                    if (_resultFormat == ResultFormat.JSON || !_nativeArrow)
+                        Assert.That(thrown.Message, Does.Contain("Failed to read structured type when reading path $[1]"));
+                    else
+                        Assert.That(thrown.Message, Does.Contain("Failed to read structured type when getting a map."));
                 }
             }
         }