- Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables!

cajuncoding · cajuncoding · commit 21916f81ed22 · 2023-10-24T17:13:42.000-05:00
- Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema.
diff --git a/NetStandard.SqlBulkHelpers/Database/SqlBulkHelpersDBSchemaLoader.Async.cs b/NetStandard.SqlBulkHelpers/Database/SqlBulkHelpersDBSchemaLoader.Async.cs
@@ -58,6 +58,17 @@ protected SqlCommand CreateSchemaQuerySqlCommand(
         )
         {
             var tableSchemaQuerySql = GetTableSchemaSqlQuery(detailLevel);
+
+            //Enable Support for Temp Table Schema Loading...
+            if (tableNameTerm.IsTempTableName)
+            {
+                //NOTE: For Temp Table support all references to INFORMATION_SCHEMA must be replaced with tempdb.INFORMATION_SCHEMA
+                //          and DB_NAME() must be changed to 'tempdb', otherwise we dynamically resolve the true Temp Table Name in the Cte...
+                tableSchemaQuerySql = tableSchemaQuerySql
+                    .Replace("INFORMATION_SCHEMA.", "tempdb.INFORMATION_SCHEMA.")
+                    .Replace("DB_NAME()", "'tempdb'");
+            }
+
             var sqlCmd = new SqlCommand(tableSchemaQuerySql, sqlConnection, sqlTransaction);
             
             //Configure the timeout for retrieving the Schema details...
@@ -67,6 +78,7 @@ protected SqlCommand CreateSchemaQuerySqlCommand(
             var sqlParams = sqlCmd.Parameters;
             sqlParams.Add(new SqlParameter("@TableSchema", tableNameTerm.SchemaName));
             sqlParams.Add(new SqlParameter("@TableName", tableNameTerm.TableName));
+            sqlParams.Add(new SqlParameter("@IsTempTable", tableNameTerm.IsTempTableName));
             return sqlCmd;
         }
 
@@ -131,7 +143,8 @@ protected async Task<SqlBulkHelpersTableDefinition> GetTableSchemaDefinitionInte
             var tableNameTerm = tableName.ParseAsTableNameTerm();
             var cacheKey = CreateCacheKeyInternal(tableNameTerm, detailLevel);
 
-            if (forceCacheReload)
+            //NOTE: We also prevent caching of Temp Table schemas that are by definition Transient!
+            if (forceCacheReload || tableNameTerm.IsTempTableName)
                 TableDefinitionsCaseInsensitiveLazyCache.TryRemoveAsyncValue(cacheKey);
 
             var tableDefinition = await TableDefinitionsCaseInsensitiveLazyCache.GetOrAddAsync(
diff --git a/NetStandard.SqlBulkHelpers/Database/SqlBulkHelpersDBSchemaLoader.Sync.cs b/NetStandard.SqlBulkHelpers/Database/SqlBulkHelpersDBSchemaLoader.Sync.cs
@@ -75,7 +75,8 @@ protected SqlBulkHelpersTableDefinition GetTableSchemaDefinitionInternal(
             var tableNameTerm = tableName.ParseAsTableNameTerm();
             var cacheKey = CreateCacheKeyInternal(tableNameTerm, detailLevel);
 
-            if (forceCacheReload)
+            //NOTE: We also prevent caching of Temp Table schemas that are by definition Transient!
+            if (forceCacheReload || tableNameTerm.IsTempTableName)
                 TableDefinitionsCaseInsensitiveLazyCache.TryRemove(cacheKey);
 
             var tableDefinitionResult = TableDefinitionsCaseInsensitiveLazyCache.GetOrAdd(
diff --git a/NetStandard.SqlBulkHelpers/Database/SqlQueries/QueryDBTableSchemaBasicDetailsJson.sql b/NetStandard.SqlBulkHelpers/Database/SqlQueries/QueryDBTableSchemaBasicDetailsJson.sql
@@ -1,14 +1,19 @@
-﻿	WITH TablesCte AS (
+﻿	--NOTE: For Temp Table support all references to INFORMATION_SCHEMA must be replaced with tempdb.INFORMATION_SCHEMA
+	--			and DB_NAME() must be changed to 'tempdb', otherwise we dynamically resolve the true Temp Table Name in the Cte...
+	WITH TablesCte AS (
 		SELECT TOP (1)
 			TableSchema = t.[TABLE_SCHEMA], 
 			TableName = t.[TABLE_NAME],
 			TableCatalog = t.[TABLE_CATALOG],
-			ObjectId = OBJECT_ID('['+t.TABLE_SCHEMA+'].['+t.TABLE_NAME+']')
+			ObjectId = OBJECT_ID(CONCAT('[', t.TABLE_CATALOG, '].[', t.TABLE_SCHEMA, '].[', t.TABLE_NAME, ']'))
 		FROM INFORMATION_SCHEMA.TABLES t
         WHERE 
             t.TABLE_SCHEMA = @TableSchema
-            AND t.TABLE_NAME = @TableName
 			AND t.TABLE_CATALOG = DB_NAME()
+			AND t.TABLE_NAME = CASE
+				WHEN @IsTempTable = 0 THEN @TableName
+				ELSE (SELECT TOP (1) t.[name] FROM tempdb.sys.objects t WHERE t.[object_id] = OBJECT_ID(CONCAT(N'tempdb.[', @TableSchema, '].[', @TableName, ']')))
+			END
 	)
 	SELECT
 		t.TableSchema, 
diff --git a/NetStandard.SqlBulkHelpers/Database/SqlQueries/QueryDBTableSchemaExtendedDetailsJson.sql b/NetStandard.SqlBulkHelpers/Database/SqlQueries/QueryDBTableSchemaExtendedDetailsJson.sql
@@ -1,14 +1,19 @@
-﻿	WITH TablesCte AS (
+﻿	--NOTE: For Temp Table support all references to INFORMATION_SCHEMA must be replaced with tempdb.INFORMATION_SCHEMA
+	--			and DB_NAME() must be changed to 'tempdb', otherwise we dynamically resolve the true Temp Table Name in the Cte...
+	WITH TablesCte AS (
 		SELECT TOP (1)
 			TableSchema = t.[TABLE_SCHEMA], 
 			TableName = t.[TABLE_NAME],
 			TableCatalog = t.[TABLE_CATALOG],
-			ObjectId = OBJECT_ID('['+t.TABLE_SCHEMA+'].['+t.TABLE_NAME+']')
+			ObjectId = OBJECT_ID(CONCAT('[', t.TABLE_CATALOG, '].[', t.TABLE_SCHEMA, '].[', t.TABLE_NAME, ']'))
 		FROM INFORMATION_SCHEMA.TABLES t
         WHERE 
             t.TABLE_SCHEMA = @TableSchema
-            AND t.TABLE_NAME = @TableName
 			AND t.TABLE_CATALOG = DB_NAME()
+			AND t.TABLE_NAME = CASE
+				WHEN @IsTempTable = 0 THEN @TableName
+				ELSE (SELECT TOP (1) t.[name] FROM tempdb.sys.objects t WHERE t.[object_id] = OBJECT_ID(CONCAT(N'tempdb.[', @TableSchema, '].[', @TableName, ']')))
+			END
 	)
 	SELECT
 		t.TableSchema, 
diff --git a/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs b/NetStandard.SqlBulkHelpers/Database/TableNameTerm.cs
@@ -21,6 +21,7 @@ public TableNameTerm(string schemaName, string tableName)
         public string TableName { get; }
         public string TableNameVariable { get; }
         public string FullyQualifiedTableName { get; }
+        public bool IsTempTableName => TableName.StartsWith("#");
 
         public override string ToString() => FullyQualifiedTableName;
 
diff --git a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj
@@ -8,17 +8,18 @@
 	<PackageLicenseExpression>MIT</PackageLicenseExpression>
     <Authors>BBernard / CajunCoding</Authors>
     <Company>CajunCoding</Company>
-	<Version>2.4.0</Version>
+	<Version>2.4.1</Version>
 	<PackageProjectUrl>https://github.com/cajuncoding/SqlBulkHelpers</PackageProjectUrl>
 	<RepositoryUrl>https://github.com/cajuncoding/SqlBulkHelpers</RepositoryUrl>
     <Description>A library for easy, efficient and high performance bulk insert and update of data, into a Sql Database, from .Net applications. By leveraging the power of the SqlBulkCopy classes with added support for Identity primary key table columns this library provides a greatly simplified interface to process Identity based Entities with Bulk Performance with the wide compatibility of .NetStandard 2.0.</Description>
 	<PackageTags>sql server database table bulk insert update identity column sqlbulkcopy orm dapper linq2sql materialization materialized data view materialized-data materialized-view sync replication replica readonly</PackageTags>
     <PackageReleaseNotes>
-		-Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type).
-		-Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning.
-			This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process.
+		- Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables!
+		- Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema.
 
 		Prior Relese Notes:
+		-Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type).
+		-Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process.
 		- Fixed bug with Sql Bulk Insert/Update processing with Model Properties that have mapped database names via mapping attribute (e.g. [SqlBulkColumn("")], [Map("")], [Column("")], etc.).
 		- Changed default behaviour to no longer clone tables/schema inside a Transaction which creates a full Schema Lock -- as this greatly impacts Schema aware ORMs such as SqlBulkHelpers, RepoDb, etc.
 		- New separate methods is now added to handle the CleanupMaterializeDataProcessAsync() but must be explicitly called as it is no longer implicitly called with FinishMaterializeDataProcessAsync().
diff --git a/NetStandard.SqlBulkHelpers/SqlBulkHelper/QueryProcessing/SqlBulkHelpersMergeQueryBuilder.cs b/NetStandard.SqlBulkHelpers/SqlBulkHelper/QueryProcessing/SqlBulkHelpersMergeQueryBuilder.cs
@@ -52,6 +52,12 @@ public virtual SqlMergeScriptResults BuildSqlMergeScripts(
                     }
                 }
 
+                if (sanitizedQualifierFields.IsNullOrEmpty())
+                    throw new ArgumentException($"The merge match qualifier fields specified [{matchQualifierExpression.MatchQualifierFields.Select(f => f.Name).ToCsv()}] " +
+                                                $"are invalid and could not be resolved to valid fields on the target table {tableDefinition.TableFullyQualifiedName}. " +
+                                                $"This could be the result of an invalid argument or potentially due to database schema changes that are not reflected in the schema cache; " +
+                                                $"you may try bypassing the cache via the bool forceCacheReload parameter or the static SqlBulkHelpersSchemaLoaderCache.ClearCache() method.");
+
                 //If we have valid Fields, then we must re-initialize a valid Qualifier Expression parameter with ONLY the valid fields...
                 sanitizedQualifierExpression = new SqlMergeMatchQualifierExpression(sanitizedQualifierFields)
                 {
diff --git a/SqlBulkHelpers.Tests/IntegrationTests/SchemaLoadingTests/SchemaLoaderTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/SchemaLoadingTests/SchemaLoaderTests.cs
@@ -1,6 +1,8 @@
 ﻿using System;
 using System.Diagnostics;
 using Microsoft.Data.SqlClient;
+using RepoDb;
+using SqlBulkHelpers.CustomExtensions;
 using SqlBulkHelpers.SqlBulkHelpers;
 
 namespace SqlBulkHelpers.Tests.IntegrationTests
@@ -22,6 +24,25 @@ public void TestTableDefinitionLoadingBasicDetailsWithTransactionSyncMethod()
             AssertTableDefinitionIsValidForTestElementParentTable(tableDefinition, TableSchemaDetailLevel.BasicDetails);
         }
 
+        [TestMethod]
+        public void TestTableDefinitionLoadingBasicDetailsForTempTableWithTransactionSyncMethod()
+        {
+            using var sqlConn = SqlConnectionHelper.NewConnection();
+            using var sqlTransaction = sqlConn.BeginTransaction();
+
+            var tempTableName = "#TempTableSchemaLoaderTest";
+            sqlConn.ExecuteNonQuery($@"
+                CREATE TABLE [{tempTableName}] ([Id] INT NOT NULL PRIMARY KEY);
+            ", transaction: sqlTransaction);
+
+            var tableDefinition = sqlTransaction.GetTableSchemaDefinition(
+                tempTableName,
+                TableSchemaDetailLevel.BasicDetails
+            );
+
+            AssertTableDefinitionIsValidForTempTable(tempTableName, tableDefinition);
+        }
+
         [TestMethod]
         public void TestTableDefinitionLoadingExtendedDetailsSyncMethods()
         {
@@ -49,6 +70,27 @@ public async Task TestTableDefinitionLoadingBasicDetailsWithTransactionAsync()
             AssertTableDefinitionIsValidForTestElementParentTable(tableDefinition, TableSchemaDetailLevel.BasicDetails);
         }
 
+        [TestMethod]
+        public async Task TestTableDefinitionLoadingBasicDetailsForTempTableWithTransactionAsync()
+        {
+            var sqlConnectionProvider = SqlConnectionHelper.GetConnectionProvider();
+            await using var sqlConn = await sqlConnectionProvider.NewConnectionAsync().ConfigureAwait(false);
+            await using var sqlTransaction = (SqlTransaction)await sqlConn.BeginTransactionAsync();
+
+            var tempTableName = "#TempTableSchemaLoaderTest";
+            await sqlConn.ExecuteNonQueryAsync($@"
+                CREATE TABLE [{tempTableName}] ([Id] INT NOT NULL PRIMARY KEY);
+            ", transaction: sqlTransaction);
+
+            var tableDefinition = await sqlTransaction.GetTableSchemaDefinitionAsync(
+                tempTableName,
+                TableSchemaDetailLevel.BasicDetails
+            ).ConfigureAwait(false);
+
+            AssertTableDefinitionIsValidForTempTable(tempTableName, tableDefinition);
+        }
+
+
         [TestMethod]
         public async Task TestTableDefinitionLoadingExtendedDetailsAsync()
         {
@@ -85,6 +127,22 @@ private void AssertTableDefinitionIsValidForTestElementParentTable(SqlBulkHelper
             }
         }
 
+        private void AssertTableDefinitionIsValidForTempTable(string tempTableName, SqlBulkHelpersTableDefinition tableDefinition)
+        {
+            Assert.IsNotNull(tableDefinition);
+
+            var tableNameTerm = TableNameTerm.From(tempTableName);
+            Assert.IsTrue(tableNameTerm.IsTempTableName);
+            Assert.AreEqual(TableSchemaDetailLevel.BasicDetails, tableDefinition.SchemaDetailLevel);
+            Assert.AreEqual(tableNameTerm.SchemaName, tableDefinition.TableSchema);
+            //NOTE: Table Names will not match exactly due to internal Hashing of the Temp Name for Session isolation, etc...
+            //Assert.AreEqual(tableNameTerm.TableName, tableDefinition.TableName);
+            //Assert.AreEqual(tableNameTerm.FullyQualifiedTableName, tableDefinition.TableFullyQualifiedName);
+            Assert.AreEqual(1, tableDefinition.TableColumns.Count);
+            Assert.IsNotNull(tableDefinition.PrimaryKeyConstraint);
+            Assert.AreEqual(0, tableDefinition.ForeignKeyConstraints.Count);
+        }
+
         [TestMethod]
         public void TestTableDefinitionLoadingAndCachingSyncMethods()
         {

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,12 @@ public virtual SqlMergeScriptResults BuildSqlMergeScripts(`
`52`	`52`	`}`
`53`	`53`	`}`
`54`	`54`
	`55`	`+ if (sanitizedQualifierFields.IsNullOrEmpty())`
	`56`	`+ throw new ArgumentException($"The merge match qualifier fields specified [{matchQualifierExpression.MatchQualifierFields.Select(f => f.Name).ToCsv()}] " +`
	`57`	`+ $"are invalid and could not be resolved to valid fields on the target table {tableDefinition.TableFullyQualifiedName}. " +`
	`58`	`+ $"This could be the result of an invalid argument or potentially due to database schema changes that are not reflected in the schema cache; " +`
	`59`	`+ $"you may try bypassing the cache via the bool forceCacheReload parameter or the static SqlBulkHelpersSchemaLoaderCache.ClearCache() method.");`
	`60`	`+`
`55`	`61`	`//If we have valid Fields, then we must re-initialize a valid Qualifier Expression parameter with ONLY the valid fields...`
`56`	`62`	`sanitizedQualifierExpression = new SqlMergeMatchQualifierExpression(sanitizedQualifierFields)`
`57`	`63`	`{`