Merge pull request #11 from cajuncoding/feature/add_support_for_bigint_identity_values

Feature/add support for bigint identity values

Author: cajuncoding

NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs

@@ -147,7 +147,7 @@ public async Task FinishMaterializeDataProcessAsync()
             //      are valid based on newly materialized data populated in the respective loading tables (for each Live table)!
             //1) First Add & Sync all missing FKey constraints (intentionally not added during initial cloning) that will prevent us from being able to switch -- this is still safe within our Transaction!
             //   NOTE: We could not add FKey constraints at initial load because the links will result in Transaction locks on the Live Tables which we must avoid!!!
-            //   NOTE: WE also disable all FKey constraints on the Live Table so taht ALL FKeys across live/loading/temp are all disabled so that Switching can Occur -- this is still safe within our Transaction!
+            //   NOTE: WE also disable all FKey constraints on the Live Table so that ALL FKeys across live/loading/temp are all disabled so that Switching can Occur -- this is still safe within our Transaction!
             foreach (var materializationTableInfo in materializationTables)
             {
                 var fkeyConstraints = materializationTableInfo.LiveTableDefinition.ForeignKeyConstraints.AsArray();
@@ -169,8 +169,6 @@ public async Task FinishMaterializeDataProcessAsync()
                     .DisableAllTableConstraintChecks(materializationTableInfo.DiscardingTable);
             }
 
-            var timeoutConvertedToMinutes = Math.Max(1, (int)Math.Ceiling((decimal)BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds / 60));
-
             //2) Then we can switch all existing Live tables to the Discarding Schema -- this Frees the Live table up to be updated in the next step!
             foreach (var materializationTableInfo in materializationTables)
                 switchScriptBuilder.SwitchTables(
@@ -209,6 +207,8 @@ public async Task FinishMaterializeDataProcessAsync()
                     .DropTable(materializationTableInfo.DiscardingTable);
             }
 
+            //var timeoutConvertedToMinutes = Math.Max(1, (int)Math.Ceiling((decimal)BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds / 60));
+
             await SqlTransaction.ExecuteMaterializedDataSqlScriptAsync(
                 switchScriptBuilder,
                 BulkHelpersConfig.MaterializeDataStructureProcessingTimeoutSeconds

NetStandard.SqlBulkHelpers/SqlBulkHelper/BaseSqlBulkHelper.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Microsoft.Data.SqlClient;
 using FastMember;
 using SqlBulkHelpers.Interfaces;
@@ -9,11 +10,14 @@ namespace SqlBulkHelpers
     internal static class TypeCache
     {
         public static readonly Type SqlBulkHelperIdentitySetter = typeof(ISqlBulkHelperIdentitySetter);
+        public static readonly Type SqlBulkHelperBigIntIdentitySetter = typeof(ISqlBulkHelperBigIntIdentitySetter);
     }
 
     //BBernard - Base Class for future flexibility...
     internal abstract class BaseSqlBulkHelper<T> : BaseHelper<T> where T : class
     {
+        protected static Type CachedEntityType { get; } = typeof(T);
+
         #region Constructors
 
         /// <inheritdoc/>
@@ -60,7 +64,7 @@ SqlMergeMatchQualifierExpression matchQualifierExpression
         protected class MergeResult
         {
             public int RowNumber { get; set; }
-            public int IdentityId { get; set; }
+            public long IdentityId { get; set; }
             //public SqlBulkHelpersMergeAction MergeAction { get; set; }
         }
 
@@ -76,6 +80,11 @@ SqlMergeMatchQualifierExpression sqlMatchQualifierExpression
 
             bool uniqueMatchValidationEnabled = sqlMatchQualifierExpression.AssertArgumentIsNotNull(nameof(sqlMatchQualifierExpression)).ThrowExceptionIfNonUniqueMatchesOccur;
             bool hasIdentityColumn = identityColumnDefinition != null;
+            //Small performance improvement here by pre-determining which, if any, identity setters may be implemented by the client;
+            //  since this isn't the most often use case it's helpful to more efficiently skip the type checks while processing...
+            bool implementsIntIdentitySetter = TypeCache.SqlBulkHelperIdentitySetter.IsAssignableFrom(CachedEntityType);
+            bool implementsBigIntIdentitySetter = TypeCache.SqlBulkHelperBigIntIdentitySetter.IsAssignableFrom(CachedEntityType);
+            bool identitySetterInterfaceSupported = implementsIntIdentitySetter || implementsBigIntIdentitySetter;
 
             //If there was no Identity Column or the validation of Unique Merge actions was disabled then we can
             //  short circuit the post-processing of results as there is nothing to do...
@@ -89,10 +98,7 @@ SqlMergeMatchQualifierExpression sqlMatchQualifierExpression
             //  we attempt to use Reflection to set the value...
             string identityPropertyName = null;
 
-            Type entityType = typeof(T);
-            TypeAccessor fastTypeAccessor = TypeAccessor.Create(entityType);
-
-            if (hasIdentityColumn && !TypeCache.SqlBulkHelperIdentitySetter.IsAssignableFrom(entityType))
+            if (hasIdentityColumn && !identitySetterInterfaceSupported)
             {
                 var processingDefinition = SqlBulkHelpersProcessingDefinition.GetProcessingDefinition<T>(identityColumnDefinition);
                 identityPropertyName = processingDefinition.IdentityPropDefinition?.PropertyName;
@@ -108,9 +114,11 @@ SqlMergeMatchQualifierExpression sqlMatchQualifierExpression
             //      so there's no reason to filter the merge results anymore; this is more performant.
             var uniqueMatchesHashSet = new HashSet<int>();
 
-            var entityResultsList = new List<T>();
+            var identitySetterAction = hasIdentityColumn
+                ? ResolveIdentitySetterAction(entityList, identityPropertyName)
+                : null;
 
-            //foreach (var mergeResult in mergeResultsList.Where(r => r.MergeAction.HasFlag(SqlBulkHelpersMergeAction.Insert)))
+            var entityResultsList = new List<T>();
             foreach (var mergeResult in mergeResultsList)
             {
                 //ONLY Process uniqueness validation if necessary... otherwise skip the logic altogether.
@@ -135,20 +143,7 @@ SqlMergeMatchQualifierExpression sqlMatchQualifierExpression
                 //ONLY Process Identity value updates if appropriate... otherwise skip the logic altogether.
                 //NOTE: List is 0 (zero) based, but our RowNumber is 1 (one) based.
                 var entity = entityList[mergeResult.RowNumber - 1];
-                if (hasIdentityColumn)
-                {
-                    //BBernard
-                    //If the entity supports our interface we can set the value with native performance via the Interface!
-                    if (entity is ISqlBulkHelperIdentitySetter identitySetterEntity)
-                    {
-                        identitySetterEntity.SetIdentityId(mergeResult.IdentityId);
-                    }
-                    else
-                    {
-                        //GENERICALLY Set the Identity Value to the Int value returned, this eliminates any dependency on a Base Class!
-                        fastTypeAccessor[entity, identityPropertyName] = mergeResult.IdentityId;
-                    }
-                }
+                identitySetterAction?.Invoke(entity, mergeResult);
 
                 entityResultsList.Add(entity);
             }
@@ -158,5 +153,63 @@ SqlMergeMatchQualifierExpression sqlMatchQualifierExpression
             return entityResultsList;
         }
 
+        private Action<T, MergeResult> ResolveIdentitySetterAction(List<T> entityList, string identityPropertyName)
+        {
+            var sampleEntity = entityList.FirstOrDefault();
+            switch (sampleEntity)
+            {
+                case null:
+                    //Break out to end for Invalid Operation handling...
+                    break;
+                //BBernard
+                //For Performance if the entity type supports our Interfaces then we use those to set the Identity ID delegating all logic to the class to handle.
+                case ISqlBulkHelperIdentitySetter _:
+                    //Downcast our MergeResult long IdentityId to int...
+                    return (entity, mergeResult) => ((ISqlBulkHelperIdentitySetter)entity).SetIdentityId((int)mergeResult.IdentityId);
+                case ISqlBulkHelperBigIntIdentitySetter _:
+                    return (entity, mergeResult) => ((ISqlBulkHelperBigIntIdentitySetter)entity).SetIdentityId(mergeResult.IdentityId);
+                default:
+                {
+                    //Create our TypeAccessor once here, so it can be captured by scope in our Action but is NOT CREATED on each Action execution!
+                    //NOTE: This also helps encapsulate our use of TypeAccessor in case we choose another approach to setting the property in the future!
+                    var fastTypeAccessor = TypeAccessor.Create(CachedEntityType);
+                    var identityPropType = fastTypeAccessor[sampleEntity, identityPropertyName]?.GetType();
+                    if (identityPropType != null)
+                    {
+                        //BBernard
+                        //For Performance we try to identity the primary Integer types and implement the Setter Action with explicit casting, however
+                        //  as a fallback we will attempt to generically convert the type via Convert.ChangeType() for really strange edge cases where the Model type is
+                        //  something awkward... (Heaven forbid a string), but hey we'll try to make it work.
+                        if (identityPropType == typeof(long)) //BIGINT Sql Type
+                        {
+                            //MergeResult IdentityId is already a Long to support the superset of any other Int property types by down-casting...
+                            return (entity, mergeResult) => fastTypeAccessor[entity, identityPropertyName] = mergeResult.IdentityId;
+                        }
+                        else if (identityPropType == typeof(int)) //INT Sql Type
+                        {
+                            return (entity, mergeResult) => fastTypeAccessor[entity, identityPropertyName] = (int)mergeResult.IdentityId;
+                        }
+                        else if (identityPropType == typeof(short)) //SMALLINT Sql Type
+                        {
+                            return (entity, mergeResult) => fastTypeAccessor[entity, identityPropertyName] = (short)mergeResult.IdentityId;
+                        }
+                        else if (identityPropType == typeof(byte)) //TINYINT Sql Type
+                        {
+                            return (entity, mergeResult) => fastTypeAccessor[entity, identityPropertyName] = (byte)mergeResult.IdentityId;
+                        }
+                        else //For NUMERIC(X, 0) Sql Type or any other, we attempt to generically change the type to match...
+                        {
+                            return (entity, mergeResult) => fastTypeAccessor[entity, identityPropertyName] = Convert.ChangeType(mergeResult.IdentityId, identityPropType);
+                        }
+                    }
+
+                    //Break out to end for Invalid Operation handling...
+                    break;
+                }
+            }
+            
+            return (entity, mergeResult) => throw new InvalidOperationException($"Unable to properly map the Identity value result into the entity Property [{identityPropertyName}] of type [{entity.GetType()}].");
+        }
+
     }
 }

NetStandard.SqlBulkHelpers/SqlBulkHelper/Interfaces/ISqlBulkHelperIdentitySetter.cs

@@ -1,7 +1,14 @@
-namespace SqlBulkHelpers.Interfaces
+using System;
+
+namespace SqlBulkHelpers.Interfaces
 {
     public interface ISqlBulkHelperIdentitySetter
     {
         void SetIdentityId(int id);
     }
+    
+    public interface ISqlBulkHelperBigIntIdentitySetter
+    {
+        void SetIdentityId(long id);
+    }
 }

NetStandard.SqlBulkHelpers/SqlBulkHelper/QueryProcessing/SqlBulkHelpersMergeQueryBuilder.cs

@@ -19,7 +19,6 @@ public virtual SqlMergeScriptResults BuildSqlMergeScripts(
             var tempStagingTableName = $"#SqlBulkHelpers_STAGING_TABLE_{Guid.NewGuid()}";
             var tempOutputIdentityTableName = $"#SqlBulkHelpers_OUTPUT_IDENTITY_TABLE_{Guid.NewGuid()}";
             var hasIdentityColumn = tableDefinition.IdentityColumn != null;
-            var identityColumnName = tableDefinition.IdentityColumn?.ColumnName ?? string.Empty;
 
             //Validate the MatchQualifiers that may be specified, and limit to ONLY valid fields of the Table Definition...
             //NOTE: We use the parameter argument for Match Qualifier if specified, otherwise we fall-back to to use the Identity Column.
@@ -93,30 +92,41 @@ public virtual SqlMergeScriptResults BuildSqlMergeScripts(
             string mergeTempTablesSql, mergeOutputSql;
             if (hasIdentityColumn)
             {
+                var identityColumn = tableDefinition.IdentityColumn;
+                var identityColumnName = identityColumn.ColumnName;
+                var identityColumnDataType = identityColumn.DataType;
+
                 mergeTempTablesSql = $@"
                     SELECT TOP(0)
-                        -1 as [{identityColumnName}],
+                        [{identityColumnName}] = CONVERT({identityColumnDataType}, -1),
                         {columnNamesWithoutIdentityCSV},
-                        -1 as [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] 
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] = CONVERT(INT, -1)
                     INTO [{tempStagingTableName}] 
                     FROM {tableDefinition.TableFullyQualifiedName};
 
                     SELECT TOP(0)
-                        CAST(-1 AS int) [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}],
-                        CAST(-1 AS int) as [IDENTITY_ID],
-                        CAST('' AS nvarchar(10)) as [MERGE_ACTION]
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] = CONVERT(INT, -1),
+                        [IDENTITY_ID] = CONVERT({identityColumnDataType}, -1)
+                        --,[MERGE_ACTION] = CONVERT(VARCHAR(10), '') --Removed as Small Performance Improvement since the Action is not used.
                     INTO [{tempOutputIdentityTableName}];
                 ";
 
                 //All results with Identity Values need to be written to the Output so that we can return them efficiently!
                 mergeOutputSql = $@"
-                    OUTPUT source.[{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], INSERTED.[{identityColumnName}], $action
-                        INTO [{tempOutputIdentityTableName}] ([{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], [IDENTITY_ID], [MERGE_ACTION]);
+                    OUTPUT 
+                        source.[{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], 
+                        INSERTED.[{identityColumnName}]
+                        --, $action --Removed as Small Performance  Improvement since the Action is not used.
+                    INTO [{tempOutputIdentityTableName}] (
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], 
+                        [IDENTITY_ID]
+                        --, [MERGE_ACTION] --Removed as Small Performance Improvement since the Action is not used.
+                    );
 
                     SELECT 
-                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], 
-                        [IDENTITY_ID], 
-                        [MERGE_ACTION]
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}],
+                        [IDENTITY_ID]
+                        --,[MERGE_ACTION] --Removed as Small Performance Improvement since the Action is not used.
                     FROM [{tempOutputIdentityTableName}]
                     ORDER BY [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] ASC, [IDENTITY_ID] ASC;
                 ";
@@ -126,25 +136,30 @@ FROM [{tempOutputIdentityTableName}]
                 mergeTempTablesSql = $@"
                     SELECT TOP(0)
                         {columnNamesWithoutIdentityCSV},
-                        -1 as [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] 
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] = CONVERT(INT, -1)
                     INTO [{tempStagingTableName}] 
                     FROM {tableDefinition.TableFullyQualifiedName};
                     
                     SELECT TOP(0)
-                        CAST(-1 AS int) [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}],
-                        CAST('' AS nvarchar(10)) as [MERGE_ACTION]
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] = CONVERT(INT, -1)
+                        --,[MERGE_ACTION] = CONVERT(VARCHAR(10), '') --Removed as Small Performance  Improvement since the Action is not used.
                     INTO [{tempOutputIdentityTableName}];
                 ";
 
                 //All actions need to be written to the Output so that we can return all Identity values for new inserts, and/or post-process
                 //  only results that have been actually changed...
                 mergeOutputSql = $@"
-                    OUTPUT source.[{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], $action
-                        INTO [{tempOutputIdentityTableName}] ([{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], [MERGE_ACTION]);
+                    OUTPUT 
+                        source.[{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}]
+                        --, $action --Removed as Small Performance  Improvement since the Action is not used.
+                    INTO [{tempOutputIdentityTableName}] (
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}]
+                        --, [MERGE_ACTION] --Removed as Small Performance Improvement since the Action is not used.
+                    );
 
                     SELECT 
-                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}], 
-                        [MERGE_ACTION]
+                        [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}]
+                        --,[MERGE_ACTION] --Removed as Small Performance Improvement since the Action is not used.
                     FROM [{tempOutputIdentityTableName}]
                     ORDER BY [{SqlBulkHelpersConstants.ROWNUMBER_COLUMN_NAME}] ASC;
                 ";

NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs

@@ -111,10 +111,8 @@ public bool IsSqlBulkTableLockEnabled
             get => SqlBulkCopyOptions.HasFlag(SqlBulkCopyOptions.TableLock);
             set
             {
-                if(value)
-                    SqlBulkCopyOptions |= SqlBulkCopyOptions.TableLock;
-                else
-                    SqlBulkCopyOptions &= ~SqlBulkCopyOptions.TableLock;
+                if(value) SqlBulkCopyOptions |= SqlBulkCopyOptions.TableLock;
+                else SqlBulkCopyOptions &= ~SqlBulkCopyOptions.TableLock;
             }
         }