Fix | Allow SqlBulkCopy to operate on hidden columns (#3590)

Author: edwardneal

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs

@@ -420,9 +420,6 @@ private string CreateInitialQuery()
             {
                 throw SQL.BulkLoadInvalidDestinationTable(DestinationTableName, null);
             }
-            string TDSCommand;
-
-            TDSCommand = "select @@trancount; SET FMTONLY ON select * from " + ADP.BuildMultiPartName(parts) + " SET FMTONLY OFF ";
 
             string TableCollationsStoredProc;
             if (_connection.Is2008OrNewer)
@@ -456,27 +453,41 @@ private string CreateInitialQuery()
             string CatalogName = parts[MultipartIdentifier.CatalogIndex];
             if (isTempTable && string.IsNullOrEmpty(CatalogName))
             {
-                TDSCommand += string.Format("exec tempdb..{0} N'{1}.{2}'",
-                    TableCollationsStoredProc,
-                    SchemaName,
-                    TableName
-                );
+                CatalogName = "tempdb";
             }
-            else
+            else if (!string.IsNullOrEmpty(CatalogName))
             {
-                // Escape the catalog name
-                if (!string.IsNullOrEmpty(CatalogName))
-                {
-                    CatalogName = SqlServerEscapeHelper.EscapeIdentifier(CatalogName);
-                }
-                TDSCommand += string.Format("exec {0}..{1} N'{2}.{3}'",
-                    CatalogName,
-                    TableCollationsStoredProc,
-                    SchemaName,
-                    TableName
-                );
+                CatalogName = SqlServerEscapeHelper.EscapeIdentifier(CatalogName);
             }
-            return TDSCommand;
+
+            string objectName = ADP.BuildMultiPartName(parts);
+            string escapedObjectName = SqlServerEscapeHelper.EscapeStringAsLiteral(objectName);
+            // Specify the column names explicitly. This is to ensure that we can map to hidden columns (e.g. columns in temporal tables.)
+            // If the target table doesn't exist, OBJECT_ID will return NULL and @Column_Names will remain non-null. The subsequent SELECT *
+            // query will then continue to fail with "Invalid object name" rather than with an unusual error because the query being executed
+            // is NULL.
+            // Some hidden columns (e.g. SQL Graph columns) cannot be selected, so we need to exclude them explicitly.
+            return $"""
+SELECT @@TRANCOUNT;
+
+DECLARE @Column_Names NVARCHAR(MAX) = NULL;
+IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sys.all_columns') AND [name] = 'graph_type')
+BEGIN
+    SELECT @Column_Names = COALESCE(@Column_Names + ', ', '') + QUOTENAME([name]) FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = OBJECT_ID('{escapedObjectName}') AND COALESCE([graph_type], 0) NOT IN (1, 3, 4, 6, 7) ORDER BY [column_id] ASC;
+END
+ELSE
+BEGIN
+    SELECT @Column_Names = COALESCE(@Column_Names + ', ', '') + QUOTENAME([name]) FROM {CatalogName}.[sys].[all_columns] WHERE [object_id] = OBJECT_ID('{escapedObjectName}') ORDER BY [column_id] ASC;
+END
+
+SELECT @Column_Names = COALESCE(@Column_Names, '*');
+
+SET FMTONLY ON;
+EXEC(N'SELECT ' + @Column_Names + N' FROM {escapedObjectName}');
+SET FMTONLY OFF;
+
+EXEC {CatalogName}..{TableCollationsStoredProc} N'{SchemaName}.{TableName}';
+""";
         }
 
         // Creates and then executes initial query to get information about the targettable

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj

@@ -143,10 +143,12 @@
     <Compile Include="SQL\SqlBulkCopyTest\CopyWithEventAsync.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\FireTrigger.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Helpers.cs" />
+    <Compile Include="SQL\SqlBulkCopyTest\HiddenTargetColumn.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\MissingTargetColumn.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\MissingTargetColumns.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\MissingTargetTable.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\SqlBulkCopyTest.cs" />
+    <Compile Include="SQL\SqlBulkCopyTest\SqlGraphTables.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\TestBulkCopyWithUTF8.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Transaction.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Transaction1.cs" />

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CopyAllFromReader.cs

@@ -61,9 +61,9 @@ public static void Test(string srcConstr, string dstConstr, string dstTable)
                             DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersReceived"], "Unexpected BuffersReceived value.");
                             DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersSent"], "Unexpected BuffersSent value.");
                             DataTestUtility.AssertEqualsWithDescription((long)0, stats["IduCount"], "Unexpected IduCount value.");
-                            DataTestUtility.AssertEqualsWithDescription((long)3, stats["SelectCount"], "Unexpected SelectCount value.");
+                            DataTestUtility.AssertEqualsWithDescription((long)6, stats["SelectCount"], "Unexpected SelectCount value.");
                             DataTestUtility.AssertEqualsWithDescription((long)3, stats["ServerRoundtrips"], "Unexpected ServerRoundtrips value.");
-                            DataTestUtility.AssertEqualsWithDescription((long)4, stats["SelectRows"], "Unexpected SelectRows value.");
+                            DataTestUtility.AssertEqualsWithDescription((long)9, stats["SelectRows"], "Unexpected SelectRows value.");
                             DataTestUtility.AssertEqualsWithDescription((long)2, stats["SumResultSets"], "Unexpected SumResultSets value.");
                             DataTestUtility.AssertEqualsWithDescription((long)0, stats["Transactions"], "Unexpected Transactions value.");
                         }

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/HiddenTargetColumn.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Data.Common;
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SqlBulkCopyTests
+{
+    public class HiddenTargetColumn
+    {
+        [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))]
+        public void WriteToServer_CopyToHiddenTargetColumn_ThrowsSqlException()
+        {
+            string connectionString = DataTestUtility.TCPConnectionString;
+            string destinationTable = DataTestUtility.GetShortName("HiddenTargetColumn");
+            string destinationHistoryTable = DataTestUtility.GetShortName("HiddenTargetColumn_History");
+
+            using (SqlConnection dstConn = new SqlConnection(connectionString))
+            using (SqlCommand dstCmd = dstConn.CreateCommand())
+            {
+                dstConn.Open();
+
+                try
+                {
+                    DataTestUtility.CreateTable(dstConn, destinationTable, $"""
+(
+    Column1 int primary key not null,
+    Column2 nvarchar(10) not null,
+    [Employee's First Name] varchar(max) null,
+    ValidFrom datetime2 generated always as row start hidden not null,
+    ValidTo datetime2 generated always as row end hidden not null,
+    period for system_time (ValidFrom, ValidTo)
+)
+with (system_versioning = on(history_table = dbo.{destinationHistoryTable}));
+""");
+
+                    using (SqlConnection srcConn = new SqlConnection(connectionString))
+                    using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, FirstName, LastName, HireDate, sysdatetime() as CurrentDate from employees", srcConn))
+                    {
+                        srcConn.Open();
+
+                        using (DbDataReader reader = srcCmd.ExecuteReader())
+                        using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn))
+                        {
+                            bulkcopy.DestinationTableName = destinationTable;
+                            SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings;
+
+                            ColumnMappings.Add("EmployeeID", "Column1");
+                            ColumnMappings.Add("LastName", "Column2");
+                            ColumnMappings.Add("FirstName", "Employee's First Name");
+                            ColumnMappings.Add("HireDate", "ValidFrom");
+                            ColumnMappings.Add("CurrentDate", "ValidTo");
+
+                            SqlException sqlEx = Assert.Throws<SqlException>(() => bulkcopy.WriteToServer(reader));
+
+                            Assert.Equal(13536, sqlEx.Number);
+                            Assert.StartsWith("Cannot insert an explicit value into a GENERATED ALWAYS column in table", sqlEx.Message);
+                        }
+                    }
+                }
+                finally
+                {
+                    DataTestUtility.RunNonQuery(connectionString, $"""
+alter table {destinationTable} set (system_versioning = off);
+alter table {destinationTable} drop period for system_time;
+""");
+                    DataTestUtility.DropTable(dstConn, destinationTable);
+                    DataTestUtility.DropTable(dstConn, destinationHistoryTable);
+                }
+            }
+        }
+    }
+}

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlGraphTables.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Data;
+using System.Data.Common;
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SqlBulkCopyTests
+{
+    public class SqlGraphTables
+    {
+        [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))]
+        public void WriteToServer_CopyToSqlGraphNodeTable_Succeeds()
+        {
+            string connectionString = DataTestUtility.TCPConnectionString;
+            string destinationTable = DataTestUtility.GetShortName("SqlGraphNodeTable");
+
+            using SqlConnection dstConn = new SqlConnection(connectionString);
+            using DataTable nodes = new DataTable()
+            {
+                Columns = { new DataColumn("Name", typeof(string)) }
+            };
+
+            dstConn.Open();
+
+            for (int i = 0; i < 5; i++)
+            {
+                nodes.Rows.Add($"Name {i}");
+            }
+
+            try
+            {
+                DataTestUtility.CreateTable(dstConn, destinationTable, "(Id INT PRIMARY KEY IDENTITY(1,1), [Name] VARCHAR(100)) AS NODE");
+
+                using SqlBulkCopy nodeCopy = new SqlBulkCopy(dstConn);
+
+                nodeCopy.DestinationTableName = destinationTable;
+                nodeCopy.ColumnMappings.Add("Name", "Name");
+                nodeCopy.WriteToServer(nodes);
+            }
+            finally
+            {
+                DataTestUtility.DropTable(dstConn, destinationTable);
+            }
+        }
+    }
+}