SNOW-1618183: Fix ConstraintException when calling DataTable.Load (#1100)

Author: sfc-gh-ext-simba-lf

Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs

@@ -1636,6 +1636,38 @@ public void TestTimestampNtz(string testValue, int scale)
             }
         }
 
+        [Test]
+        [TestCase("array")]
+        [TestCase("object")]
+        [TestCase("variant")]
+        public void TestDataTableLoadOnSemiStructuredColumn(string type)
+        {
+            using (var conn = CreateAndOpenConnection())
+            {
+                var colName = "c1";
+                var expectedVal = "id:1";
+                CreateOrReplaceTable(conn, TableName, new[] { $"{colName} {type}" });
+
+                using (var cmd = conn.CreateCommand())
+                {
+                    string insertCommand = $"insert into {TableName} select parse_json('{{{expectedVal}}}')";
+                    cmd.CommandText = insertCommand;
+
+                    var count = cmd.ExecuteNonQuery();
+                    Assert.AreEqual(1, count);
+
+                    cmd.CommandText = $"select {colName} from {TableName}";
+                    using (var reader = cmd.ExecuteReader())
+                    {
+                        ValidateResultFormat(reader);
+                        var dt = new DataTable();
+                        dt.Load(reader);
+                        Assert.AreEqual(expectedVal, DataTableParser.GetFirstRowValue(dt, colName));
+                    }
+                }
+            }
+        }
+
         private DbConnection CreateAndOpenConnection()
         {
             var conn = new SnowflakeDbConnection(ConnectionString);

Snowflake.Data.Tests/IntegrationTests/StructuredArraysIT.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Linq;
 using NUnit.Framework;
 using Snowflake.Data.Client;
@@ -12,6 +13,36 @@ namespace Snowflake.Data.Tests.IntegrationTests
     [TestFixture]
     public class StructuredArraysIT: StructuredTypesIT
     {
+        [Test]
+        public void TestDataTableLoadOnStructuredArray()
+        {
+            // arrange
+            using (var connection = new SnowflakeDbConnection(ConnectionString))
+            {
+                connection.Open();
+                using (var command = connection.CreateCommand())
+                {
+                    EnableStructuredTypes(connection);
+                    var expectedValueA = 'a';
+                    var expectedValueB = 'b';
+                    var expectedValueC = 'c';
+                    var arraySFString = $"ARRAY_CONSTRUCT('{expectedValueA}','{expectedValueB}','{expectedValueC}')::ARRAY(TEXT)";
+                    var colName = "colA";
+                    command.CommandText = $"SELECT {arraySFString} AS {colName}";
+
+                    // act
+                    using (var reader = command.ExecuteReader())
+                    {
+                        var dt = new DataTable();
+                        dt.Load(reader);
+
+                        // assert
+                        Assert.AreEqual($"{expectedValueA},{expectedValueB},{expectedValueC}", DataTableParser.GetFirstRowValue(dt, colName));
+                    }
+                }
+            }
+        }
+
         [Test]
         public void TestSelectArray()
         {

Snowflake.Data.Tests/IntegrationTests/StructuredMapsIT.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Data;
 using NUnit.Framework;
 using Snowflake.Data.Client;
 using Snowflake.Data.Core;
@@ -11,6 +12,35 @@ namespace Snowflake.Data.Tests.IntegrationTests
     [TestFixture]
     public class StructuredMapsIT: StructuredTypesIT
     {
+        [Test]
+        public void TestDataTableLoadOnStructuredMap()
+        {
+            // arrange
+            using (var connection = new SnowflakeDbConnection(ConnectionString))
+            {
+                connection.Open();
+                using (var command = connection.CreateCommand())
+                {
+                    EnableStructuredTypes(connection);
+                    var key = "city";
+                    var value = "San Mateo";
+                    var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::MAP(VARCHAR, VARCHAR)";
+                    var colName = "colA";
+                    command.CommandText = $"SELECT {addressAsSFString} AS {colName}";
+
+                    // act
+                    using (var reader = command.ExecuteReader())
+                    {
+                        var dt = new DataTable();
+                        dt.Load(reader);
+
+                        // assert
+                        Assert.AreEqual($"{key}:{value}", DataTableParser.GetFirstRowValue(dt, colName));
+                    }
+                }
+            }
+        }
+
         [Test]
         public void TestSelectMap()
         {

Snowflake.Data.Tests/IntegrationTests/StructuredObjectsIT.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Data;
 using NUnit.Framework;
 using Snowflake.Data.Client;
 using Snowflake.Data.Core;
@@ -10,6 +11,35 @@ namespace Snowflake.Data.Tests.IntegrationTests
     [TestFixture]
     public class StructuredObjectIT: StructuredTypesIT
     {
+        [Test]
+        public void TestDataTableLoadOnStructuredObject()
+        {
+            // arrange
+            using (var connection = new SnowflakeDbConnection(ConnectionString))
+            {
+                connection.Open();
+                using (var command = connection.CreateCommand())
+                {
+                    EnableStructuredTypes(connection);
+                    var key = "city";
+                    var value = "San Mateo";
+                    var addressAsSFString = $"OBJECT_CONSTRUCT('{key}','{value}')::OBJECT(city VARCHAR)";
+                    var colName = "colA";
+                    command.CommandText = $"SELECT {addressAsSFString} AS {colName}";
+
+                    // act
+                    using (var reader = command.ExecuteReader())
+                    {
+                        var dt = new DataTable();
+                        dt.Load(reader);
+
+                        // assert
+                        Assert.AreEqual($"{key}:{value}", DataTableParser.GetFirstRowValue(dt, colName));
+                    }
+                }
+            }
+        }
+
         [Test]
         public void TestSelectStructuredTypeObject()
         {

Snowflake.Data.Tests/Util/DataTableParser.cs

@@ -0,0 +1,44 @@
+using Newtonsoft.Json.Linq;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+
+namespace Snowflake.Data.Tests.Util
+{
+    public static class DataTableParser
+    {
+        internal static string GetFirstRowValue(DataTable dt, string colName)
+        {
+            var jsonString = dt.Rows[0][colName].ToString();
+            var token = JToken.Parse(jsonString);
+
+            if (token.Type == JTokenType.Object)
+            {
+                return ParseKeyValue(token);
+            }
+            else if (token.Type == JTokenType.Array)
+            {
+                var element = token.First();
+                if (element.Type == JTokenType.Object)
+                {
+                    return ParseKeyValue(element);
+                }
+                else
+                {
+                    var list = token.ToObject<List<string>>();
+                    return string.Join(",", list);
+                }
+            }
+
+            return jsonString;
+        }
+
+        private static string ParseKeyValue(JToken token)
+        {
+            var keyValue = token.ToObject<Dictionary<string, string>>();
+            var key = keyValue.Keys.First();
+            var value = keyValue[key];
+            return $"{key}:{value}";
+        }
+    }
+}

Snowflake.Data/Client/SnowflakeDbDataReader.cs

@@ -28,6 +28,8 @@ public class SnowflakeDbDataReader : DbDataReader
 
         internal ResultFormat ResultFormat => resultSet.ResultFormat;
 
+        private const int MaxStringLength = 16777216; // Default maximum allowed length for VARCHAR
+
         internal SnowflakeDbDataReader(SnowflakeDbCommand command, SFBaseResultSet resultSet)
         {
             this.dbCommand = command;
@@ -118,7 +120,7 @@ private DataTable PopulateSchemaTable(SFBaseResultSet resultSet)
 
                 row[SchemaTableColumn.ColumnName] = rowType.name;
                 row[SchemaTableColumn.ColumnOrdinal] = columnOrdinal;
-                row[SchemaTableColumn.ColumnSize] = (int)rowType.length;
+                row[SchemaTableColumn.ColumnSize] = IsStructuredOrSemiStructuredType(rowType.type) && rowType.length == 0 ? MaxStringLength : (int)rowType.length;
                 row[SchemaTableColumn.NumericPrecision] = (int)rowType.precision;
                 row[SchemaTableColumn.NumericScale] = (int)rowType.scale;
                 row[SchemaTableColumn.AllowDBNull] = rowType.nullable;
@@ -365,5 +367,11 @@ public override void Close()
             resultSet.close();
             isClosed = true;
         }
+
+        private bool IsStructuredOrSemiStructuredType(string type)
+        {
+            type = type.ToLower();
+            return type == "array" || type == "object" || type == "variant"|| type == "map";
+        }
     }
 }