SNOW-1271212 Fixed values uploaded to stage for bindings exceeding CLIENT_STAGE_ARRAY_BINDING_THRESHOLD (#897)

sfc-gh-mhofman · web-flow · commit dfba44e48873 · 2024-04-04T13:38:50.000+02:00
### Description
When number of binded values during query execution exceeds the
threshold of a session parameter CLIENT_STAGE_ARRAY_BINDING_THRESHOLD
then values are written as a CSV file to a stage and it get's picked
during query execution.
Improper values (or values truncating nanos) has been uploaded prior to this fix
for date and time related columns of type: DATE, TIME, TIMESTAMP_LTZ,
TIMESTAMP_NTZ, TIMESTAMP_TZ.

### Checklist
- [x] Code compiles correctly
- [x] Code is formatted according to [Coding
Conventions](../CodingConventions.md)
- [x] Created tests which fail without the change (if possible)
- [x] All tests passing (`dotnet test`)
- [x] Extended the README / documentation, if necessary
- [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name
diff --git a/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs
@@ -12,6 +12,7 @@
 using System.Runtime.InteropServices;
 using NUnit.Framework;
 using Snowflake.Data.Client;
+using Snowflake.Data.Log;
 using Snowflake.Data.Tests.Util;
 
 [assembly:LevelOfParallelism(10)]
@@ -56,6 +57,8 @@ public static void TearDownContext()
     #endif
     public class SFBaseTestAsync
     {
+        private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger<SFBaseTestAsync>();
+
         private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" +
                                                               "account={3};role={4};db={5};schema={6};warehouse={7}";
         private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};";
@@ -106,10 +109,16 @@ private void RemoveTables()
         }
 
         protected void CreateOrReplaceTable(IDbConnection conn, string tableName, IEnumerable<string> columns, string additionalQueryStr = null)
+        {
+            CreateOrReplaceTable(conn, tableName, "", columns, additionalQueryStr);
+        }
+
+        protected void CreateOrReplaceTable(IDbConnection conn, string tableName, string tableType, IEnumerable<string> columns, string additionalQueryStr = null)
         {
             var columnsStr = string.Join(", ", columns);
             var cmd = conn.CreateCommand();
-            cmd.CommandText = $"CREATE OR REPLACE TABLE {tableName}({columnsStr}) {additionalQueryStr}";
+            cmd.CommandText = $"CREATE OR REPLACE {tableType} TABLE {tableName}({columnsStr}) {additionalQueryStr}";
+            s_logger.Debug(cmd.CommandText);
             cmd.ExecuteNonQuery();
 
             _tablesToRemove.Add(tableName);
diff --git a/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs b/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+using System;
+using NUnit.Framework;
+using Snowflake.Data.Core;
+
+namespace Snowflake.Data.Tests.UnitTests
+{
+    [TestFixture]
+    class SFBindUploaderTest
+    {
+        private readonly SFBindUploader _bindUploader = new SFBindUploader(null, "test");
+
+        [TestCase(SFDataType.DATE, "0", "1/1/1970")]
+        [TestCase(SFDataType.DATE, "73785600000", "5/4/1972")]
+        [TestCase(SFDataType.DATE, "1709164800000", "2/29/2024")]
+        public void TestCsvDataConversionForDate(SFDataType dbType, string input, string expected)
+        {
+            // Arrange
+            var dateExpected = DateTime.Parse(expected);
+            var check = SFDataConverter.csharpValToSfVal(SFDataType.DATE, dateExpected);
+            Assert.AreEqual(check, input);
+            // Act
+            DateTime dateActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
+            // Assert
+            Assert.AreEqual(dateExpected, dateActual);
+        }
+
+        [TestCase(SFDataType.TIME, "0", "00:00:00.000000")]
+        [TestCase(SFDataType.TIME, "100000000", "00:00:00.100000")]
+        [TestCase(SFDataType.TIME, "1000000000", "00:00:01.000000")]
+        [TestCase(SFDataType.TIME, "60123456000", "00:01:00.123456")]
+        [TestCase(SFDataType.TIME, "46801000000000", "13:00:01.000000")]
+        public void TestCsvDataConversionForTime(SFDataType dbType, string input, string expected)
+        {
+            // Arrange
+            DateTime timeExpected = DateTime.Parse(expected);
+            var check = SFDataConverter.csharpValToSfVal(SFDataType.TIME, timeExpected);
+            Assert.AreEqual(check, input);
+            // Act
+            DateTime timeActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
+            // Assert
+            Assert.AreEqual(timeExpected, timeActual);
+        }
+        
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")] 
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")]
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")]
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")]
+        public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected)
+        {
+            // Arrange
+            var timestampExpected = DateTimeOffset.Parse(expected);
+            var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_LTZ, timestampExpected);
+            Assert.AreEqual(check, input);
+            // Act
+            var timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
+            // Assert
+            Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual);
+        }
+        
+        [TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")]
+        [TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")]
+        public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected)
+        {
+            // Arrange
+            DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected);
+            var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_TZ, timestampExpected);
+            Assert.AreEqual(check, input);
+            // Act
+            DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
+            // Assert
+            Assert.AreEqual(timestampExpected, timestampActual);
+        }
+        
+        [TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")]
+        [TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")]
+        public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected)
+        {
+            // Arrange 
+            DateTime timestampExpected = DateTime.Parse(expected);
+            var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected);
+            Assert.AreEqual(check, input);
+            // Act
+            DateTime timestampActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
+            // Assert
+            Assert.AreEqual(timestampExpected, timestampActual);
+        }
+
+        [TestCase(SFDataType.TEXT, "", "\"\"")]
+        [TestCase(SFDataType.TEXT, "\"", "\"\"\"\"")]
+        [TestCase(SFDataType.TEXT, "\n", "\"\n\"")]
+        [TestCase(SFDataType.TEXT, "\t", "\"\t\"")]
+        [TestCase(SFDataType.TEXT, ",", "\",\"")]
+        [TestCase(SFDataType.TEXT, "Sample text", "Sample text")]
+        public void TestCsvDataConversionForText(SFDataType dbType, string input, string expected)
+        {
+            // Act
+            var actual = _bindUploader.GetCSVData(dbType.ToString(), input);
+            // Assert
+            Assert.AreEqual(expected, actual);
+        }
+
+    }
+}
diff --git a/Snowflake.Data.Tests/Util/DbCommandExtensions.cs b/Snowflake.Data.Tests/Util/DbCommandExtensions.cs
@@ -0,0 +1,18 @@
+using System.Data;
+
+namespace Snowflake.Data.Tests.Util
+{
+    public static class DbCommandExtensions
+    {
+        internal static IDbDataParameter Add(this IDbCommand command, string name, DbType dbType, object value)
+        {
+            var parameter = command.CreateParameter();
+            parameter.ParameterName = name;
+            parameter.DbType = dbType;
+            parameter.Value = value;
+            command.Parameters.Add(parameter);
+            return parameter;
+        }
+
+    }
+}
diff --git a/Snowflake.Data.Tests/Util/DbConnectionExtensions.cs b/Snowflake.Data.Tests/Util/DbConnectionExtensions.cs
@@ -0,0 +1,23 @@
+using System.Data;
+
+namespace Snowflake.Data.Tests.Util
+{
+    public static class DbConnectionExtensions
+    {
+        internal static IDbCommand CreateCommand(this IDbConnection connection, string commandText)
+        {
+            var command = connection.CreateCommand();
+            command.Connection = connection;
+            command.CommandText = commandText;
+            return command;
+        }
+        
+        internal static int ExecuteNonQuery(this IDbConnection connection, string commandText)
+        {
+            var command = connection.CreateCommand();
+            command.Connection = connection;
+            command.CommandText = commandText;
+            return command.ExecuteNonQuery();
+        }
+    }
+}
diff --git a/Snowflake.Data.Tests/Util/TableTypeExtensions.cs b/Snowflake.Data.Tests/Util/TableTypeExtensions.cs
@@ -0,0 +1,30 @@
+using System;
+using NUnit.Framework;
+
+namespace Snowflake.Data.Tests.Util
+{
+    public enum SFTableType 
+    {
+        Standard,
+        Hybrid,
+        Iceberg
+    }
+
+    static class TableTypeExtensions
+    {
+        internal static string TableDDLCreationPrefix(this SFTableType val) => val == SFTableType.Standard ? "" : val.ToString().ToUpper();
+        
+        internal static string TableDDLCreationFlags(this SFTableType val)
+        {
+            if (val != SFTableType.Iceberg)
+                return "";
+            var externalVolume = Environment.GetEnvironmentVariable("ICEBERG_EXTERNAL_VOLUME"); 
+            var catalog = Environment.GetEnvironmentVariable("ICEBERG_CATALOG"); 
+            var baseLocation = Environment.GetEnvironmentVariable("ICEBERG_BASE_LOCATION"); 
+            Assert.IsNotNull(externalVolume, "env ICEBERG_EXTERNAL_VOLUME not set!");
+            Assert.IsNotNull(catalog, "env ICEBERG_CATALOG not set!");
+            Assert.IsNotNull(baseLocation, "env ICEBERG_BASE_LOCATION not set!");
+            return $"EXTERNAL_VOLUME = '{externalVolume}' CATALOG = '{catalog}' BASE_LOCATION = '{baseLocation}'";
+        }
+    }
+}
diff --git a/Snowflake.Data/Client/SnowflakeDbCommand.cs b/Snowflake.Data/Client/SnowflakeDbCommand.cs
@@ -460,5 +460,7 @@ private void CheckIfCommandTextIsSet()
                 throw new Exception(errorMessage);
             }
         }
+
+        internal string GetBindStage() => sfStatement?.GetBindStage();
     }
 }
diff --git a/Snowflake.Data/Core/SFBindUploader.cs b/Snowflake.Data/Core/SFBindUploader.cs
@@ -224,13 +224,13 @@ internal async Task UploadStreamAsync(MemoryStream stream, string destFileName,
             statement.SetUploadStream(stream, destFileName, stagePath);
             await statement.ExecuteTransferAsync(putStmt, cancellationToken).ConfigureAwait(false);
         }
-        private string GetCSVData(string sType, string sValue)
+        
+        internal string GetCSVData(string sType, string sValue)
         {
             if (sValue == null)
                 return sValue;
 
-            DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified);
-            DateTimeOffset dateTimeOffset = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified);
+            DateTime epoch = SFDataConverter.UnixEpoch;
             switch (sType)
             {
                 case "TEXT":
@@ -246,33 +246,29 @@ private string GetCSVData(string sType, string sValue)
                         return '"' + sValue.Replace("\"", "\"\"") + '"';
                     return sValue;
                 case "DATE":
-                    long dateLong = long.Parse(sValue);
-                    DateTime date = dateTime.AddMilliseconds(dateLong).ToUniversalTime();
+                    long msFromEpoch = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ms] from Epoch
+                    DateTime date = epoch.AddMilliseconds(msFromEpoch);
                     return date.ToShortDateString();
                 case "TIME":
-                    long timeLong = long.Parse(sValue);
-                    DateTime time = dateTime.AddMilliseconds(timeLong).ToUniversalTime();
-                    return time.ToLongTimeString();
+                    long nsSinceMidnight = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Midnight
+                    DateTime time = epoch.AddTicks(nsSinceMidnight/100);
+                    return time.ToString("HH:mm:ss.fffffff");
                 case "TIMESTAMP_LTZ":
-                    long ltzLong = long.Parse(sValue);
-                    TimeSpan ltzts = new TimeSpan(ltzLong / 100);
-                    DateTime ltzdt = dateTime + ltzts;
-                    return ltzdt.ToString();
+                    long nsFromEpochLtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch
+                    DateTime ltz = epoch.AddTicks(nsFromEpochLtz/100);
+                    return ltz.ToLocalTime().ToString("O"); // ISO 8601 format
                 case "TIMESTAMP_NTZ":
-                    long ntzLong = long.Parse(sValue);
-                    TimeSpan ts = new TimeSpan(ntzLong/100);
-                    DateTime dt = dateTime + ts;
-                    return dt.ToString("yyyy-MM-dd HH:mm:ss.fffffff");
+                    long nsFromEpochNtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch
+                    DateTime ntz = epoch.AddTicks(nsFromEpochNtz/100);
+                    return ntz.ToString("yyyy-MM-dd HH:mm:ss.fffffff");
                 case "TIMESTAMP_TZ":
                     string[] tstzString = sValue.Split(' ');
-                    long tzLong = long.Parse(tstzString[0]);
-                    int tzInt = (int.Parse(tstzString[1]) - 1440) / 60;
-                    TimeSpan tzts = new TimeSpan(tzLong/100);
-                    DateTime tzdt = dateTime + tzts;
-                    TimeSpan tz = new TimeSpan(tzInt, 0, 0);
-                    DateTimeOffset tzDateTimeOffset = new DateTimeOffset(tzdt, tz);
+                    long nsFromEpochTz = long.Parse(tstzString[0]); // SFDateConverter provides in [ns] from Epoch
+                    int timeZoneOffset = int.Parse(tstzString[1]) - 1440; // SFDateConverter provides in minutes increased by 1440m 
+                    DateTime timestamp = epoch.AddTicks(nsFromEpochTz/100).AddMinutes(timeZoneOffset);
+                    TimeSpan offset = TimeSpan.FromMinutes(timeZoneOffset);
+                    DateTimeOffset tzDateTimeOffset = new DateTimeOffset(timestamp.Ticks, offset);
                     return tzDateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz");
-                    
             }
             return sValue;
         }
diff --git a/Snowflake.Data/Core/SFStatement.cs b/Snowflake.Data/Core/SFStatement.cs
@@ -147,6 +147,8 @@ internal SFStatement(SFSession session)
             _restRequester = session.restRequester;
         }
 
+        internal string GetBindStage() => _bindStage;
+
         private void AssignQueryRequestId()
         {
             lock (_requestIdLock)

Original file line number	Diff line number	Diff line change
`@@ -460,5 +460,7 @@ private void CheckIfCommandTextIsSet()`
`460`	`460`	`throw new Exception(errorMessage);`
`461`	`461`	`}`
`462`	`462`	`}`
	`463`	`+`
	`464`	`+ internal string GetBindStage() => sfStatement?.GetBindStage();`
`463`	`465`	`}`
`464`	`466`	`}`
Original file line number	Diff line number	Diff line change
`@@ -147,6 +147,8 @@ internal SFStatement(SFSession session)`
`147`	`147`	`_restRequester = session.restRequester;`
`148`	`148`	`}`
`149`	`149`
	`150`	`+ internal string GetBindStage() => _bindStage;`
	`151`	`+`
`150`	`152`	`private void AssignQueryRequestId()`
`151`	`153`	`{`
`152`	`154`	`lock (_requestIdLock)`