Skip to content

SNOW-1488703: Add arrow support for structured types #1178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 48 commits into
base: master
Choose a base branch
from
+590 −78
Open

SNOW-1488703: Add arrow support for structured types #1178

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
f55ce82
Add arrow support for structured types
sfc-gh-ext-simba-lf May 21, 2025
a1750f1
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf May 21, 2025
5499f89
Revert commented lines
sfc-gh-ext-simba-lf May 21, 2025
938d36a
Remove bracket
sfc-gh-ext-simba-lf May 21, 2025
3464096
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Jun 3, 2025
ac0a7db
Fix formatting
sfc-gh-ext-simba-lf Jun 5, 2025
5add0a3
Revert "SNOW-2115047 Log http calls time (#1180)"
sfc-gh-ext-simba-lf Jun 5, 2025
d131faf
Reapply "SNOW-2115047 Log http calls time (#1180)"
sfc-gh-ext-simba-lf Jun 5, 2025
25201de
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Jun 16, 2025
493264c
Add native arrow format result
sfc-gh-ext-simba-lf Jun 20, 2025
16247c3
Add support for native arrow formats
sfc-gh-ext-simba-lf Jun 20, 2025
79b059f
Merge branch 'master' into SNOW-1488703-Arrow-Support-For-Structured-…
sfc-gh-ext-simba-hx Jul 16, 2025
eae55dd
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Jul 16, 2025
ac9fb99
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Jul 22, 2025
76c7f04
Use JsonSerializer to escape special characters
sfc-gh-ext-simba-lf Jul 22, 2025
8a42858
Merge branch 'SNOW-1488703-Arrow-Support-For-Structured-Types' of htt…
sfc-gh-ext-simba-lf Jul 22, 2025
10dcf1b
Add converter for native arrow array types
sfc-gh-ext-simba-lf Sep 2, 2025
0a245d3
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Sep 8, 2025
d1da946
Modify test to compare GetString result from arrow and json
sfc-gh-ext-simba-lf Sep 10, 2025
4eae30c
Use converter that supports .NET Framework
sfc-gh-ext-simba-lf Sep 17, 2025
43c012c
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Oct 7, 2025
85c7a10
Add case for boolean
sfc-gh-ext-simba-lf Oct 17, 2025
8517d9e
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Oct 21, 2025
4322592
Update CHANGELOG.md
sfc-gh-ext-simba-lf Oct 31, 2025
29bdea5
Add other cases
sfc-gh-ext-simba-lf Oct 31, 2025
4563a00
Merge branch 'master' into SNOW-1488703-Arrow-Support-For-Structured-…
sfc-gh-ext-simba-lf Oct 31, 2025
7093415
Refactor to switch case
sfc-gh-ext-simba-lf Nov 3, 2025
56d0410
Refactor to switch case
sfc-gh-ext-simba-lf Nov 3, 2025
673ec13
Refactor function
sfc-gh-ext-simba-lf Nov 3, 2025
4acdec4
Refactor to switch case
sfc-gh-ext-simba-lf Nov 3, 2025
74c0ac5
Move if check outside the for loop
sfc-gh-ext-simba-lf Nov 4, 2025
a843589
Merge branch 'master' into SNOW-1488703-Arrow-Support-For-Structured-…
sfc-gh-ext-simba-lf Nov 4, 2025
ef3179f
Fix indentation
sfc-gh-ext-simba-lf Nov 4, 2025
87c1457
Merge branch 'SNOW-1488703-Arrow-Support-For-Structured-Types' of htt…
sfc-gh-ext-simba-lf Nov 4, 2025
22140ec
Add type validation
sfc-gh-ext-simba-lf Nov 5, 2025
98cfe02
Check if type has more than 2 arguments
sfc-gh-ext-simba-lf Nov 5, 2025
b6c9959
Revert type check
sfc-gh-ext-simba-lf Nov 5, 2025
7a90867
Re-add type check
sfc-gh-ext-simba-lf Nov 5, 2025
1123430
Use ObjectBuilder to get properties
sfc-gh-ext-simba-lf Nov 6, 2025
7433fbe
Revert last and use FirstOrDefault to get custom attributes
sfc-gh-ext-simba-lf Nov 6, 2025
47acd9a
Fix if check
sfc-gh-ext-simba-lf Nov 6, 2025
2b5cdb4
Get properties in order of metadata
sfc-gh-ext-simba-lf Nov 7, 2025
461c1f9
Fix whitespace
sfc-gh-ext-simba-lf Nov 7, 2025
165c28b
Merge branch 'master' of https://github.com/snowflakedb/snowflake-con…
sfc-gh-ext-simba-lf Nov 7, 2025
888c1f3
Remove unsupported case of structured types
sfc-gh-ext-simba-lf Nov 12, 2025
b775286
Fix getting value from Date32Array
sfc-gh-ext-simba-lf Nov 12, 2025
4909f52
Add tests for more array types
sfc-gh-ext-simba-lf Nov 12, 2025
d9521a2
Merge branch 'master' into SNOW-1488703-Arrow-Support-For-Structured-…
sfc-gh-ext-simba-lf Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 29 additions & 18 deletions Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,20 @@

namespace Snowflake.Data.Tests.IntegrationTests
{
[TestFixture]
[TestFixture(ResultFormat.ARROW, false)]
[TestFixture(ResultFormat.ARROW, true)]
[TestFixture(ResultFormat.JSON, false)]
public class StructuredArraysIT : StructuredTypesIT
{
private readonly ResultFormat _resultFormat;
private readonly bool _nativeArrow;

public StructuredArraysIT(ResultFormat resultFormat, bool nativeArrow)
{
_resultFormat = resultFormat;
_nativeArrow = nativeArrow;
}

[Test]
public void TestDataTableLoadOnStructuredArray()
{
Expand All @@ -22,7 +33,7 @@ public void TestDataTableLoadOnStructuredArray()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var expectedValueA = 'a';
var expectedValueB = 'b';
var expectedValueC = 'c';
Expand Down Expand Up @@ -52,7 +63,7 @@ public void TestSelectArray()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arraySFString = "ARRAY_CONSTRUCT('a','b','c')::ARRAY(TEXT)";
command.CommandText = $"SELECT {arraySFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -77,7 +88,7 @@ public void TestSelectArrayOfObjects()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfObjects =
"ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('name', 'Alex'), OBJECT_CONSTRUCT('name', 'Brian'))::ARRAY(OBJECT(name VARCHAR))";
command.CommandText = $"SELECT {arrayOfObjects}";
Expand All @@ -103,7 +114,7 @@ public void TestSelectArrayOfArrays()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfArrays = "ARRAY_CONSTRUCT(ARRAY_CONSTRUCT('a', 'b'), ARRAY_CONSTRUCT('c', 'd'))::ARRAY(ARRAY(TEXT))";
command.CommandText = $"SELECT {arrayOfArrays}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -128,7 +139,7 @@ public void TestSelectArrayOfMap()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfMap = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('a', 'b'))::ARRAY(MAP(VARCHAR,VARCHAR))";
command.CommandText = $"SELECT {arrayOfMap}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand Down Expand Up @@ -159,7 +170,7 @@ public void TestSelectSemiStructuredTypesInArray(string valueSfString, string ex
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
command.CommandText = $"SELECT {valueSfString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Assert.IsTrue(reader.Read());
Expand All @@ -183,7 +194,7 @@ public void TestSelectArrayOfIntegers()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfIntegers = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(INTEGER)";
command.CommandText = $"SELECT {arrayOfIntegers}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -208,7 +219,7 @@ public void TestSelectArrayOfLong()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfLongs = "ARRAY_CONSTRUCT(3, 5, 8)::ARRAY(BIGINT)";
command.CommandText = $"SELECT {arrayOfLongs}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -233,7 +244,7 @@ public void TestSelectArrayOfFloats()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfFloats = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(FLOAT)";
command.CommandText = $"SELECT {arrayOfFloats}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -258,7 +269,7 @@ public void TestSelectArrayOfDoubles()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfDoubles = "ARRAY_CONSTRUCT(3.1, 5.2, 8.11)::ARRAY(DOUBLE)";
command.CommandText = $"SELECT {arrayOfDoubles}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -283,7 +294,7 @@ public void TestSelectArrayOfDoublesWithExponentNotation()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfDoubles = "ARRAY_CONSTRUCT(1.0e100, 1.0e-100)::ARRAY(DOUBLE)";
command.CommandText = $"SELECT {arrayOfDoubles}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -308,7 +319,7 @@ public void TestSelectStringArrayWithNulls()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arraySFString = "ARRAY_CONSTRUCT('a',NULL,'b')::ARRAY(TEXT)";
command.CommandText = $"SELECT {arraySFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -333,7 +344,7 @@ public void TestSelectIntArrayWithNulls()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arrayOfNumberSFString = "ARRAY_CONSTRUCT(3,NULL,5)::ARRAY(INTEGER)";
command.CommandText = $"SELECT {arrayOfNumberSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -358,7 +369,7 @@ public void TestSelectNullArray()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var nullArraySFString = "NULL::ARRAY(TEXT)";
command.CommandText = $"SELECT {nullArraySFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -382,7 +393,7 @@ public void TestThrowExceptionForInvalidArray()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arraySFString = "ARRAY_CONSTRUCT('x', 'y')::ARRAY";
command.CommandText = $"SELECT {arraySFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -408,7 +419,7 @@ public void TestThrowExceptionForInvalidArrayElement()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arraySFString = "ARRAY_CONSTRUCT('a76dacad-0e35-497b-bf9b-7cd49262b68b', 'z76dacad-0e35-497b-bf9b-7cd49262b68b')::ARRAY(TEXT)";
command.CommandText = $"SELECT {arraySFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -433,7 +444,7 @@ public void TestThrowExceptionForNextedInvalidElement()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var arraySFString = @"ARRAY_CONSTRUCT(
OBJECT_CONSTRUCT('x', 'a', 'y', 'b')
)::ARRAY(OBJECT(x VARCHAR, y VARCHAR))";
Expand Down
37 changes: 24 additions & 13 deletions Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,20 @@

namespace Snowflake.Data.Tests.IntegrationTests
{
[TestFixture]
[TestFixture(ResultFormat.ARROW, false)]
[TestFixture(ResultFormat.ARROW, true)]
[TestFixture(ResultFormat.JSON, false)]
public class StructuredMapsIT : StructuredTypesIT
{
private readonly ResultFormat _resultFormat;
private readonly bool _nativeArrow;

public StructuredMapsIT(ResultFormat resultFormat, bool nativeArrow)
{
_resultFormat = resultFormat;
_nativeArrow = nativeArrow;
}

[Test]
public void TestDataTableLoadOnStructuredMap()
{
Expand All @@ -21,7 +32,7 @@ public void TestDataTableLoadOnStructuredMap()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var key = "city";
var value = "San Mateo";
var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)";
Expand Down Expand Up @@ -50,7 +61,7 @@ public void TestSelectMap()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', '01-234')::MAP(VARCHAR, VARCHAR)";
command.CommandText = $"SELECT {addressAsSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand Down Expand Up @@ -78,7 +89,7 @@ public void TestSelectMapWithIntegerKeys()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)";
command.CommandText = $"SELECT {mapSfString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand Down Expand Up @@ -106,7 +117,7 @@ public void TestSelectMapWithLongKeys()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapSfString = "OBJECT_CONSTRUCT('5','San Mateo', '8', 'CA', '13', '01-234')::MAP(INTEGER, VARCHAR)";
command.CommandText = $"SELECT {mapSfString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand Down Expand Up @@ -134,7 +145,7 @@ public void TestSelectMapOfObjects()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapWitObjectValueSFString = @"OBJECT_CONSTRUCT(
'Warsaw', OBJECT_CONSTRUCT('prefix', '01', 'postfix', '234'),
'San Mateo', OBJECT_CONSTRUCT('prefix', '02', 'postfix', '567')
Expand Down Expand Up @@ -164,7 +175,7 @@ public void TestSelectMapOfArrays()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))";
command.CommandText = $"SELECT {mapWithArrayValueSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -190,7 +201,7 @@ public void TestSelectMapOfLists()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapWithArrayValueSFString = "OBJECT_CONSTRUCT('a', ARRAY_CONSTRUCT('b', 'c'))::MAP(VARCHAR, ARRAY(TEXT))";
command.CommandText = $"SELECT {mapWithArrayValueSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -216,7 +227,7 @@ public void TestSelectMapOfMaps()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var mapAsSFString = "OBJECT_CONSTRUCT('a', OBJECT_CONSTRUCT('b', 'c'))::MAP(TEXT, MAP(TEXT, TEXT))";
command.CommandText = $"SELECT {mapAsSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand Down Expand Up @@ -246,7 +257,7 @@ public void TestSelectSemiStructuredTypesInMap(string valueSfString, string expe
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
command.CommandText = $"SELECT {valueSfString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Assert.IsTrue(reader.Read());
Expand All @@ -271,7 +282,7 @@ public void TestSelectNullMap()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var nullMapSFString = "NULL::MAP(TEXT,TEXT)";
command.CommandText = $"SELECT {nullMapSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -295,7 +306,7 @@ public void TestThrowExceptionForInvalidMap()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var invalidMapSFString = "OBJECT_CONSTRUCT('x', 'y')::OBJECT";
command.CommandText = $"SELECT {invalidMapSFString}";
var reader = (SnowflakeDbDataReader)command.ExecuteReader();
Expand All @@ -321,7 +332,7 @@ public void TestThrowExceptionForInvalidMapElement()
connection.Open();
using (var command = connection.CreateCommand())
{
EnableStructuredTypes(connection);
EnableStructuredTypes(connection, _resultFormat, _nativeArrow);
var invalidMapSFString = @"OBJECT_CONSTRUCT(
'x', 'a76dacad-0e35-497b-bf9b-7cd49262b68b',
'y', 'z76dacad-0e35-497b-bf9b-7cd49262b68b'
Expand Down
Loading
Loading