Optional Column Mappings

stevekirks · stevekirks · commit 071906030c1b · 2021-04-03T16:56:20.000+10:00
diff --git a/.gitignore b/.gitignore
@@ -132,3 +132,5 @@ orleans.codegen.cs
 
 # Local History for Visual Studio
 .localhistory/
+
+.vs
diff --git a/README.md b/README.md
@@ -1,10 +1,14 @@
 ﻿# SQL Bulk Copy & Merge
 
+This library aims to make copying table data between SQL databases easier.
+
 SQLBulkCopy is useful to copy between databases, but truncating the destination table each time before copying is not always possible or efficient.
+An alternative is to use SQLBulkCopy to copy to a temporary table and then run SQL MERGE between the temporary table and target.
+Some other solutions that do this require extra work defining the table schemas or are dependant on a stored proc.
 
-To solve this problem, this .NET library has the following methods:
+This .NET library has the following methods:
 
-### CopyAndMerge
+## CopyAndMerge
 Uses SQLBulkCopy to copy data from a table or view in the source database to a temporary table in the target database before running SQL MERGE from the temporary table to the destination table.
 
 The specific steps it performs:
@@ -15,7 +19,7 @@ The specific steps it performs:
 -   Run MERGE statement
 -   Drop the temp table
 
-Usage:
+### Usage:
 ```
 var copyService = new SqlBulkCopyMergeService(sourceDbConnectionString, targetDbConnectionString);
 var result = await copyService.CopyAndMerge(sourceTable, targetTable);
@@ -24,7 +28,17 @@ Console.WriteLine("Rows Updated: " + result.Updated);
 Console.WriteLine("Rows Deleted: " + result.Deleted);
 ```
 
-### CopyLatest
+If the column names between tables are different you can specify them, for example:
+```
+var columnMappings = new List<ColumnMapping>
+{
+    new ColumnMapping("id", "code"),
+    new ColumnMapping("notes", "description")
+};
+var result = await copyService.CopyAndMerge(sourceTable, targetTable, columnMappings);
+```
+
+## CopyLatest
 For source tables that are only ever added to it is more efficient to copy only the new rows into the target table.
 This method copies the latest data determined by the keyColumnName.
 
@@ -35,14 +49,14 @@ If no keyColumnName is specified, the primary key is used. If more than one prim
 
 The result of the source query is directly copied into the target table using SQLBulkCopy.
 
-Usage:
+### Usage:
 ```
 var copyService = new SqlBulkCopyMergeService(sourceDbConnectionString, targetDbConnectionString);
 var result = await copyService.CopyLatest(sourceTable, targetTable, keyColumnName);
 Console.WriteLine("Rows Copied: " + result.RowsCopied);
 ```
 
-### Notes
+## Notes
 For the connections, the Source database requires READER permission, the Target database requires READER + WRITER + CREATE TABLE + EXECUTE permissions.
 
 Tested on SQL Server 2019.
diff --git a/src/SqlBulkCopyMerge/Models/ColumnMapping.cs b/src/SqlBulkCopyMerge/Models/ColumnMapping.cs
@@ -0,0 +1,15 @@
+﻿namespace SqlBulkCopyMerge.Models
+{
+    public class ColumnMapping
+    {
+        public ColumnMapping() { }
+        public ColumnMapping(string source, string target)
+        {
+            Source = source;
+            Target = target;
+        }
+
+        public string Source { get; set; }
+        public string Target { get; set; }
+    }
+}
diff --git a/src/SqlBulkCopyMerge/Models/CopyAndMergeConfig.cs b/src/SqlBulkCopyMerge/Models/CopyAndMergeConfig.cs
@@ -2,6 +2,11 @@
 {
     public class CopyAndMergeConfig
     {
+        /// <summary>
+        /// Set to true if the MERGE is only to Insert and Update rows, not Delete
+        /// </summary>
+        public bool NoRowDeletion { get; set; }
+
         public SqlBulkCopyConfig SqlBulkCopyConfig { get; set; }
         public SqlMergeConfig SqlMergeConfig { get; set; }
     }
diff --git a/src/SqlBulkCopyMerge/SqlBulkCopyMerge.csproj b/src/SqlBulkCopyMerge/SqlBulkCopyMerge.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Authors>Steve Kirkegard</Authors>
     <Product>SqlBulkCopyMerge</Product>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFrameworks>netstandard2.1;net5.0</TargetFrameworks>
     <Description>Copy data from a table in one database to a table in another database. It does not TRUNCATE the target table. Uses SQLBulkCopy behind the scenes.</Description>
     <GeneratePackageOnBuild>false</GeneratePackageOnBuild>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
diff --git a/src/SqlBulkCopyMerge/SqlBulkCopyMergeService.cs b/src/SqlBulkCopyMerge/SqlBulkCopyMergeService.cs
diff --git a/src/SqlBulkCopyMerge/SqlUtilExtensions.cs b/src/SqlBulkCopyMerge/SqlUtilExtensions.cs
@@ -1,6 +1,9 @@
 ﻿using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;
 using Microsoft.SqlServer.Management.Common;
+using SqlBulkCopyMerge.Models;
 
 namespace SqlBulkCopyMerge
 {
@@ -17,13 +20,13 @@ internal static (string TableSchema, string TableName) GetTableSchemaAndNameInPa
             if (r.IsMatch(table))
             {
                 var match = r.Match(table);
-                if (match.Groups.ContainsKey(regexLabelTableName)
+                if (match.Groups.Cast<Group>().ToList().Any(a => a.Name == regexLabelTableName)
                     && match.Groups[regexLabelTableName].Success
                     && !string.IsNullOrWhiteSpace(match.Groups[regexLabelTableName].Value))
                 {
                     var tableSchema = "dbo"; // default
                     var tableName = match.Groups[regexLabelTableName].Value;
-                    if (match.Groups.ContainsKey(regexLabelTableSchema)
+                    if (match.Groups.Cast<Group>().ToList().Any(a => a.Name == regexLabelTableSchema)
                         && match.Groups[regexLabelTableSchema].Success
                         && !string.IsNullOrWhiteSpace(match.Groups[regexLabelTableSchema].Value))
                     {
@@ -60,5 +63,33 @@ internal static string AddSquareBrackets(this string val)
             if (val == null) return null;
             return $"[{RemoveSquareBrackets(val)}]";
         }
+        
+        internal static List<ColumnMapping> ValidateAndFormatColumnMappings(this List<ColumnMapping> columnMappings, List<ColumnSchemaModel> sourceColumns, List<ColumnSchemaModel> targetColumns)
+        {
+            if (columnMappings?.Any() == true)
+            {
+                // Validate column Mappings
+                foreach (var columnMapping in columnMappings)
+                {
+                    if (string.IsNullOrWhiteSpace(columnMapping.Source) ||
+                        string.IsNullOrWhiteSpace(columnMapping.Target))
+                        throw new Exception("One or more column mappings are invalid");
+
+                    columnMapping.Source = columnMapping.Source.RemoveSquareBrackets();
+                    columnMapping.Target = columnMapping.Target.RemoveSquareBrackets();
+
+                    if (!sourceColumns.Any(a => string.Equals(a.Name, columnMapping.Source, StringComparison.InvariantCultureIgnoreCase)))
+                    {
+                        throw new Exception($"Invalid Column Mappings. {columnMapping.Source} does not exist");
+                    }
+                    if (!targetColumns.Any(a => string.Equals(a.Name, columnMapping.Target, StringComparison.InvariantCultureIgnoreCase)))
+                    {
+                        throw new Exception($"Invalid Column Mappings. {columnMapping.Target} does not exist");
+                    }
+                }
+            }
+
+            return columnMappings;
+        }
     }
 }
diff --git a/test/SqlBulkCopyMerge.Tests/CopyAndMergeTests.cs b/test/SqlBulkCopyMerge.Tests/CopyAndMergeTests.cs
@@ -1,4 +1,6 @@
-﻿using System.Threading.Tasks;
+﻿using System.Collections.Generic;
+using System.Threading.Tasks;
+using SqlBulkCopyMerge.Models;
 using SqlBulkCopyMerge.Tests.Docker;
 using Xunit;
 using Xunit.Abstractions;
@@ -43,5 +45,38 @@ public async Task CopiesAndMergesFromViewWithSubsetOfColumnsCorrectly()
             Assert.Equal(1, result.Updated);
             Assert.Equal(3, result.Deleted);
         }
+
+        [Fact]
+        public async Task CopiesAndMergesWithDeletesDisabled()
+        {
+            var sourceTable = "test_copy_and_merge_with_deletes_disabled";
+            var targetTable = sourceTable;
+            var config = new CopyAndMergeConfig
+            {
+                NoRowDeletion = true
+            };
+            var result = await _sqlBulkCopyCautiouslyService.CopyAndMerge(sourceTable, targetTable, copyAndMergeConfig: config);
+            Assert.Equal(1, result.Inserted);
+            Assert.Equal(1, result.Updated);
+            Assert.Equal(0, result.Deleted);
+        }
+
+        [Fact]
+        public async Task CopiesAndMergesWithDifferentColumnNames()
+        {
+            var sourceTable = "test_copy_and_merge_with_different_column_names";
+            var targetTable = "test_copy_and_merge_with_different_column_names_d";
+            var columnMappings = new List<ColumnMapping>
+            {
+                new ColumnMapping("id", "d_id"),
+                new ColumnMapping("notes", "[d_notes]"),
+                new ColumnMapping("[timestamp]", "d_timestamp"),
+                new ColumnMapping("[version_control]", "[d_version_control]")
+            };
+            var result = await _sqlBulkCopyCautiouslyService.CopyAndMerge(sourceTable, targetTable, columnMappings);
+            Assert.Equal(1, result.Inserted);
+            Assert.Equal(1, result.Updated);
+            Assert.Equal(1, result.Deleted);
+        }
     }
 }
diff --git a/test/SqlBulkCopyMerge.Tests/Docker/SeedDatabases.sql b/test/SqlBulkCopyMerge.Tests/Docker/SeedDatabases.sql
@@ -110,6 +110,39 @@ CREATE VIEW [dbo].[vtest_copy_and_merge_subset] AS
 )
 GO
 
+IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = 'test_copy_and_merge_with_deletes_disabled')
+	DROP TABLE [dbo].[test_copy_and_merge_with_deletes_disabled]
+GO
+CREATE TABLE [dbo].[test_copy_and_merge_with_deletes_disabled](
+	[id] INT NOT NULL,
+	[notes] NVARCHAR(MAX),
+	[timestamp] DATETIME2,
+	[geom] GEOMETRY,
+	[version_control] BINARY(8)
+	CONSTRAINT [PK_test_copy_and_merge_with_deletes_disabled] PRIMARY KEY ([id])
+);
+INSERT INTO [dbo].[test_copy_and_merge_with_deletes_disabled] ( [id], [notes], [timestamp], [version_control] ) VALUES
+(2, 'Note 2 update', '2020-01-02', CONVERT(VARBINARY(8), 10000002)), 
+(3, 'Note 3 new', '2020-01-03', CONVERT(VARBINARY(8), 10000003))
+GO
+
+IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = 'test_copy_and_merge_with_different_column_names')
+	DROP TABLE [dbo].[test_copy_and_merge_with_different_column_names]
+GO
+CREATE TABLE [dbo].[test_copy_and_merge_with_different_column_names](
+	[id] INT NOT NULL,
+	[notes] NVARCHAR(MAX),
+	[timestamp] DATETIME2,
+	[geom] GEOMETRY,
+	[notes_2] NVARCHAR(200),
+	[version_control] BINARY(8)
+	CONSTRAINT [PK_test_copy_and_merge_with_different_column_names] PRIMARY KEY ([id])
+);
+INSERT INTO [dbo].[test_copy_and_merge_with_different_column_names] ( [id], [notes], [timestamp], [notes_2], [version_control] ) VALUES
+(1, 'Note Update', '2020-01-01', NULL, CONVERT(VARBINARY(8), 10000001)), 
+(3, 'Note New', '2020-01-03', NULL, CONVERT(VARBINARY(8), 10000003)), 
+(4, 'Note Unchanged', '2020-01-04', 'Note not included', CONVERT(VARBINARY(8), 10000004))
+GO
 
 
 -- *************************************************************************************************************************
@@ -191,5 +224,39 @@ INSERT INTO [dbo].[test_copy_and_merge_subset] ( [id], [notes], [timestamp_no_ma
 (5, NULL, '2020-01-05', CONVERT(VARBINARY(8), 10000050));
 GO
 
+IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = 'test_copy_and_merge_with_deletes_disabled')
+	DROP TABLE [dbo].[test_copy_and_merge_with_deletes_disabled]
+GO
+CREATE TABLE [dbo].[test_copy_and_merge_with_deletes_disabled](
+	[id] INT NOT NULL,
+	[notes] NVARCHAR(MAX),
+	[timestamp] DATETIME2,
+	[geom] GEOMETRY,
+	[version_control] BINARY(8)
+	CONSTRAINT [PK_test_copy_and_merge_with_deletes_disabled] PRIMARY KEY ([id])
+);
+INSERT INTO [dbo].[test_copy_and_merge_with_deletes_disabled] ( [id], [notes], [timestamp], [version_control] ) VALUES
+(1, 'Note wont be deleted even though it doesnt exist in source', '2020-01-01', CONVERT(VARBINARY(8), 10000001)), 
+(2, 'Note 2', '2020-01-02', CONVERT(VARBINARY(8), 10000002))
+GO
+
+IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = 'test_copy_and_merge_with_different_column_names_d')
+	DROP TABLE [dbo].[test_copy_and_merge_with_different_column_names_d]
+GO
+CREATE TABLE [dbo].[test_copy_and_merge_with_different_column_names_d](
+	[d_id] INT NOT NULL,
+	[d_notes] NVARCHAR(200),
+	[d_timestamp] DATETIME2,
+	[d_geom] GEOMETRY,
+	[notes_2] NVARCHAR(200),
+	[d_version_control] BINARY(8)
+	CONSTRAINT [PK_test_copy_and_merge_with_different_column_names_d] PRIMARY KEY ([d_id])
+);
+INSERT INTO [dbo].[test_copy_and_merge_with_different_column_names_d] ( [d_id], [d_notes], [d_timestamp], [notes_2], [d_version_control] ) VALUES
+(1, 'Note 1', '2020-01-01', NULL, CONVERT(VARBINARY(8), 10000001)), 
+(2, 'Note 2', '2020-01-02', NULL, CONVERT(VARBINARY(8), 10000002)), 
+(4, 'Note Unchanged', '2020-01-04', 'Note not updated', CONVERT(VARBINARY(8), 10000004))
+GO
+
 
 USE [master]

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,11 @@`
`2`	`2`	`{`
`3`	`3`	`public class CopyAndMergeConfig`
`4`	`4`	`{`
	`5`	`+ /// <summary>`
	`6`	`+ /// Set to true if the MERGE is only to Insert and Update rows, not Delete`
	`7`	`+ /// </summary>`
	`8`	`+ public bool NoRowDeletion { get; set; }`
	`9`	`+`
`5`	`10`	`public SqlBulkCopyConfig SqlBulkCopyConfig { get; set; }`
`6`	`11`	`public SqlMergeConfig SqlMergeConfig { get; set; }`
`7`	`12`	`}`