- Updated version to 2.4.2 for release to Nuget!

cajuncoding · cajuncoding · commit c4434fecdbb0 · 2023-11-07T15:26:54.000-06:00
- Add Support to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` via `materializeDataContext.DisableMaterializedStagingTableCleanup()`; always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. This provides support for advanced debugging and control flow support.
- Improved SqlBulkHelpers Configuration API to now provide Clone() and Configure() methods to more easily copy/clone existing configuration and change values is specific instances; including copy/clone of the Defaults for unique exeuctions.
diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs b/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs
@@ -26,7 +26,9 @@ public interface IMaterializeDataContext
         TableNameTerm GetLoadingTableName(string tableName);
         TableNameTerm GetLoadingTableName<TModel>();
         TableNameTerm GetLoadingTableName(Type modelType);
-        void CancelMaterializationProcess();
         bool IsCancelled { get; }
+        IMaterializeDataContext CancelMaterializationProcess();
+        bool IsMaterializedLoadingTableCleanupEnabled { get; }
+        IMaterializeDataContext DisableMaterializedLoadingTableCleanup();
     }
 }
diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs
@@ -14,6 +14,7 @@ public class MaterializeDataContext : IMaterializeDataContextCompletionSource, I
         protected ISqlBulkHelpersConfig BulkHelpersConfig { get; }
         protected bool IsDisposed { get; set; } = false;
         public bool IsCancelled { get; protected set; } = false;
+        public bool IsMaterializedLoadingTableCleanupEnabled { get; protected set; } = true;
 
         protected List<SqlBulkHelpersTableDefinition> TablesWithFullTextIndexesRemoved { get; set; } = new List<SqlBulkHelpersTableDefinition>();
 
@@ -25,7 +26,30 @@ public class MaterializeDataContext : IMaterializeDataContextCompletionSource, I
 
         public MaterializationTableInfo this[Type modelType] => FindMaterializationTableInfoCaseInsensitive(modelType);
 
-        public void CancelMaterializationProcess() => IsCancelled = true;
+        /// <summary>
+        /// Provides ability to manually control if materialization process is Cancelled for advanced validation and control flow support.
+        /// </summary>
+        /// <returns></returns>
+        public IMaterializeDataContext CancelMaterializationProcess()
+        {
+            IsCancelled = true;
+            return this;
+        }
+
+        /// <summary>
+        /// Provides ability to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks`
+        ///     for advanced debugging and control flow support; always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used.
+        /// </summary>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException"></exception>
+        public IMaterializeDataContext DisableMaterializedLoadingTableCleanup()
+        {
+            if (BulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.InsideTransactionAllowSchemaLocks)
+                throw new InvalidOperationException("You cannot disable the cleanup of Materialized Loading tables when using SchemaCopyMode.InsideTransactionAllowSchemaLocks.");
+            
+            IsMaterializedLoadingTableCleanupEnabled = false;
+            return this;
+        }
 
         public TableNameTerm GetLoadingTableName(string tableName)
         {
diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs
@@ -72,10 +72,12 @@ public async Task<IMaterializeDataContext> CleanupMaterializeDataProcessAsync(Sq
             //Explicitly clean up all Loading/Discarding Tables (contains old data) to free resources -- this leaves us with only the (new) Live Table in place!
             foreach (var materializationTableInfo in materializationTables)
             {
-                switchScriptBuilder
-                    //Finally cleanup the Loading and Discarding tables...
-                    .DropTableIfExists(materializationTableInfo.LoadingTable)
-                    .DropTableIfExists(materializationTableInfo.DiscardingTable);
+                //ALWAYS cleanup the Discarding tables...
+                switchScriptBuilder.DropTableIfExists(materializationTableInfo.DiscardingTable);
+                
+                //IF enabled (default is always Enabled) then cleanup up the Loading Tables
+                if (materializedDataContext.IsMaterializedLoadingTableCleanupEnabled)
+                    switchScriptBuilder.DropTableIfExists(materializationTableInfo.LoadingTable);
             }
 
             await sqlTransaction.ExecuteMaterializedDataSqlScriptAsync(
diff --git a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj
@@ -8,18 +8,22 @@
 	<PackageLicenseExpression>MIT</PackageLicenseExpression>
     <Authors>BBernard / CajunCoding</Authors>
     <Company>CajunCoding</Company>
-	<Version>2.4.1</Version>
+	<Version>2.4.2</Version>
 	<PackageProjectUrl>https://github.com/cajuncoding/SqlBulkHelpers</PackageProjectUrl>
 	<RepositoryUrl>https://github.com/cajuncoding/SqlBulkHelpers</RepositoryUrl>
     <Description>A library for easy, efficient and high performance bulk insert and update of data, into a Sql Database, from .Net applications. By leveraging the power of the SqlBulkCopy classes with added support for Identity primary key table columns this library provides a greatly simplified interface to process Identity based Entities with Bulk Performance with the wide compatibility of .NetStandard 2.0.</Description>
 	<PackageTags>sql server database table bulk insert update identity column sqlbulkcopy orm dapper linq2sql materialization materialized data view materialized-data materialized-view sync replication replica readonly</PackageTags>
     <PackageReleaseNotes>
+		- Add Support to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` via `materializeDataContext.DisableMaterializedStagingTableCleanup()`;
+		always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. This provides support for advanced debugging and control flow support.
+		- Improved SqlBulkHelpers Configuration API to now provide Clone() and Configure() methods to more easily copy/clone existing configuration and change values is specific instances;
+		including copy/clone of the Defaults for unique exeuctions.
 		- Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables!
 		- Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema.
+		- Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type).
+		- Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process.
 
 		Prior Relese Notes:
-		-Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type).
-		-Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process.
 		- Fixed bug with Sql Bulk Insert/Update processing with Model Properties that have mapped database names via mapping attribute (e.g. [SqlBulkColumn("")], [Map("")], [Column("")], etc.).
 		- Changed default behaviour to no longer clone tables/schema inside a Transaction which creates a full Schema Lock -- as this greatly impacts Schema aware ORMs such as SqlBulkHelpers, RepoDb, etc.
 		- New separate methods is now added to handle the CleanupMaterializeDataProcessAsync() but must be explicitly called as it is no longer implicitly called with FinishMaterializeDataProcessAsync().
diff --git a/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs b/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs
@@ -52,19 +52,54 @@ public interface ISqlBulkHelpersConfig
         int MaxConcurrentConnections { get; }
         bool IsConcurrentConnectionProcessingEnabled { get; }
         bool IsFullTextIndexHandlingEnabled { get; }
+
+        ISqlBulkHelpersConfig Clone();
+        ISqlBulkHelpersConfig Configure(Action<SqlBulkHelpersConfig> configAction);
     }
 
     public class SqlBulkHelpersConfig : ISqlBulkHelpersConfig
     {
         public static ISqlBulkHelpersConfig DefaultConfig { get; private set; } = new SqlBulkHelpersConfig();
 
-        public static SqlBulkHelpersConfig Create(Action<SqlBulkHelpersConfig> configAction)
+        // Constructor for cloning
+        private SqlBulkHelpersConfig(ISqlBulkHelpersConfig otherConfigToClone = null)
         {
-            configAction.AssertArgumentIsNotNull(nameof(configAction));
+            if (otherConfigToClone == null) return;
+
+            // Copy property values from the other instance
+            this.SqlBulkBatchSize = otherConfigToClone.SqlBulkBatchSize;
+            this.SqlBulkPerBatchTimeoutSeconds = otherConfigToClone.SqlBulkPerBatchTimeoutSeconds;
+            this.IsSqlBulkTableLockEnabled = otherConfigToClone.IsSqlBulkTableLockEnabled;
+            this.SqlBulkCopyOptions = otherConfigToClone.SqlBulkCopyOptions;
+            this.DbSchemaLoaderQueryTimeoutSeconds = otherConfigToClone.DbSchemaLoaderQueryTimeoutSeconds;
+            this.MaterializeDataStructureProcessingTimeoutSeconds = otherConfigToClone.MaterializeDataStructureProcessingTimeoutSeconds;
+            this.MaterializedDataSwitchTableWaitTimeoutMinutes = otherConfigToClone.MaterializedDataSwitchTableWaitTimeoutMinutes;
+            this.MaterializedDataSwitchTimeoutAction = otherConfigToClone.MaterializedDataSwitchTimeoutAction;
+            this.MaterializedDataSchemaCopyMode = otherConfigToClone.MaterializedDataSchemaCopyMode;
+            this.MaterializedDataLoadingTableDataCopyMode = otherConfigToClone.MaterializedDataLoadingTableDataCopyMode;
+            this.MaterializedDataMakeSchemaCopyNamesUnique = otherConfigToClone.MaterializedDataMakeSchemaCopyNamesUnique;
+            this.MaterializedDataLoadingSchema = otherConfigToClone.MaterializedDataLoadingSchema;
+            this.MaterializedDataLoadingTablePrefix = otherConfigToClone.MaterializedDataLoadingTablePrefix;
+            this.MaterializedDataLoadingTableSuffix = otherConfigToClone.MaterializedDataLoadingTableSuffix;
+            this.MaterializedDataDiscardingSchema = otherConfigToClone.MaterializedDataDiscardingSchema;
+            this.MaterializedDataDiscardingTablePrefix = otherConfigToClone.MaterializedDataDiscardingTablePrefix;
+            this.MaterializedDataDiscardingTableSuffix = otherConfigToClone.MaterializedDataDiscardingTableSuffix;
+            this.IsCloningIdentitySeedValueEnabled = otherConfigToClone.IsCloningIdentitySeedValueEnabled;
+            this.ConcurrentConnectionFactory = otherConfigToClone.ConcurrentConnectionFactory;
+            this.MaxConcurrentConnections = otherConfigToClone.MaxConcurrentConnections;
+            this.IsFullTextIndexHandlingEnabled = otherConfigToClone.IsFullTextIndexHandlingEnabled;
+        }
+
+        public static SqlBulkHelpersConfig Create(Action<SqlBulkHelpersConfig> configAction, ISqlBulkHelpersConfig otherConfigToClone = null)
+            => (SqlBulkHelpersConfig)new SqlBulkHelpersConfig(otherConfigToClone).Configure(configAction);
+
+        public ISqlBulkHelpersConfig Clone() => new SqlBulkHelpersConfig(this);
 
-            var newConfig = new SqlBulkHelpersConfig();
-            configAction.Invoke(newConfig);
-            return newConfig;
+        public ISqlBulkHelpersConfig Configure(Action<SqlBulkHelpersConfig> configAction)
+        {
+            configAction.AssertArgumentIsNotNull(nameof(configAction));
+            configAction.Invoke(this);
+            return this;
         }
 
         /// <summary>
@@ -125,6 +160,8 @@ public void EnableConcurrentSqlConnectionProcessing(
             this.IsFullTextIndexHandlingEnabled = IsFullTextIndexHandlingEnabled || enableFullTextIndexHandling;
         }
 
+        #region All Public Properties / Config Setttings...
+
         public int SqlBulkBatchSize { get; set; } = 2000; //General guidance is that 2000-5000 is efficient enough.
 
         public int SqlBulkPerBatchTimeoutSeconds { get; set; }
@@ -186,5 +223,7 @@ public int MaxConcurrentConnections
         /// Recommended to use the SqlBulkHelpersConfig.EnableConcurrentSqlConnectionProcessing() convenience method(s) to enable this more easily!
         /// </summary>
         public bool IsFullTextIndexHandlingEnabled { get; set; } = false;
+
+        #endregion
     }
 }
diff --git a/README.md b/README.md
@@ -215,6 +215,21 @@ public class TestDataService
 ## Nuget Package
 To use in your project, add the [SqlBulkHelpers NuGet package](https://www.nuget.org/packages/SqlBulkHelpers/) to your project.
 
+### v2.4.2 Release Notes:
+- Add Support to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` via `materializeDataContext.DisableMaterializedStagingTableCleanup()`;
+ always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. This provides support for advanced debugging and control flow support.
+- Improved SqlBulkHelpers Configuration API to now provide Clone() and Configure() methods to more easily copy/clone existing configuration and change values is specific instances;
+ including copy/clone of the Defaults for unique exeuctions.
+
+### v2.4.1 Release Notes:
+- Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables!
+- Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema.
+
+### v2.4.0 Release Notes:
+- Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type).
+- Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. 
+ This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process.
+
 ## v2.3.1 Release Notes:
 - Fixed bug with Sql Bulk Insert/Update processing with Model Properties that have mapped database names via mapping attribute (e.g. [SqlBulkColumn("")], [Map("")], [Column("")], etc.).
 
diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs
@@ -21,8 +21,8 @@ public async Task TestCloneTableStructureByAnnotationAsync()
             {
                 var cloneInfo = await sqlTrans.CloneTableAsync<TestElementWithMappedNames>().ConfigureAwait(false);
 
-                var sourceTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.SourceTable.FullyQualifiedTableName);
-                var clonedTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.TargetTable.FullyQualifiedTableName);
+                var sourceTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.SourceTable.FullyQualifiedTableName).ConfigureAwait(false);
+                var clonedTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.TargetTable.FullyQualifiedTableName).ConfigureAwait(false);
 
                 await sqlTrans.RollbackAsync().ConfigureAwait(false);
                 //await sqlTransaction.CommitAsync().ConfigureAwait(false);
diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,9 @@ public interface IMaterializeDataContext`
`26`	`26`	`TableNameTerm GetLoadingTableName(string tableName);`
`27`	`27`	`TableNameTerm GetLoadingTableName<TModel>();`
`28`	`28`	`TableNameTerm GetLoadingTableName(Type modelType);`
`29`		`- void CancelMaterializationProcess();`
`30`	`29`	`bool IsCancelled { get; }`
	`30`	`+ IMaterializeDataContext CancelMaterializationProcess();`
	`31`	`+ bool IsMaterializedLoadingTableCleanupEnabled { get; }`
	`32`	`+ IMaterializeDataContext DisableMaterializedLoadingTableCleanup();`
`31`	`33`	`}`
`32`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,8 +21,8 @@ public async Task TestCloneTableStructureByAnnotationAsync()`
`21`	`21`	`{`
`22`	`22`	`var cloneInfo = await sqlTrans.CloneTableAsync<TestElementWithMappedNames>().ConfigureAwait(false);`
`23`	`23`
`24`		`- var sourceTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.SourceTable.FullyQualifiedTableName);`
`25`		`- var clonedTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.TargetTable.FullyQualifiedTableName);`
	`24`	`+ var sourceTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.SourceTable.FullyQualifiedTableName).ConfigureAwait(false);`
	`25`	`+ var clonedTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.TargetTable.FullyQualifiedTableName).ConfigureAwait(false);`
`26`	`26`
`27`	`27`	`await sqlTrans.RollbackAsync().ConfigureAwait(false);`
`28`	`28`	`//await sqlTransaction.CommitAsync().ConfigureAwait(false);`