Skip to content

Commit 13f3314

Browse files
rdteviotdalerdteviotdale
authored
Merge pull request #2320 from HicServices/task/simple-mssql-merge
RDMP-355 SQL Merge Extraction Destination Component
2 parents be359b2 + 3814aef commit 13f3314

File tree

7 files changed

+465
-4
lines changed

7 files changed

+465
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1818
- Simplify use of Extraction Progress
1919
- Update CHI validation to allow mod10 CHIs
2020
- Hardened Confluence export functionality
21+
- Add new SQL merge extraction component
2122

2223
## [9.1.1] - 2025-12-02
2324
- Allow Atlassian service workers to write to Confluence from RDMP

‎Directory.Packages.props‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
<PackageVersion Include="HIC.SynthEHR" Version="2.0.1" />
1212
<PackageVersion Include="HIC.FAnsiSql" Version="3.2.7" />
1313
<PackageVersion Include="LibArchive.Net" Version="0.3.0" />
14-
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.2" />
14+
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.10" />
15+
<PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="9.0.13" />
1516
<PackageVersion Include="Microsoft.Net.Sdk.Compilers.Toolset" Version="9.0.300" />
1617
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
1718
<PrivateAssets>all</PrivateAssets>

‎Rdmp.Core.Tests/DataExport/DataExtraction/MSSQLMergeDestinationTests.cs‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
using FAnsi;
2+
using NUnit.Framework;
3+
using Rdmp.Core.Curation.Data;
4+
using Rdmp.Core.Curation.Data.Pipelines;
5+
using Rdmp.Core.DataExport.Data;
6+
using Rdmp.Core.DataExport.DataExtraction.Commands;
7+
using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations;
8+
using Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources;
9+
using Rdmp.Core.DataExport.DataExtraction.UserPicks;
10+
using Rdmp.Core.DataFlowPipeline;
11+
using Rdmp.Core.MapsDirectlyToDatabaseTable;
12+
using Rdmp.Core.QueryBuilding;
13+
using Rdmp.Core.ReusableLibraryCode.Progress;
14+
using System;
15+
using System.Collections.Generic;
16+
using System.Data;
17+
using System.Linq;
18+
using System.Text;
19+
using System.Threading.Tasks;
20+
using Tests.Common;
21+
using Tests.Common.Scenarios;
22+
23+
namespace Rdmp.Core.Tests.DataExport.DataExtraction
24+
{
25+
26+
internal class MSSqlMergeDestination_Test: MSSqlMergeDestination
27+
{
28+
public void Execute(DataTable dt) {
29+
Assert.DoesNotThrow(()=>WriteRows(dt, ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken(),new System.Diagnostics.Stopwatch()));
30+
31+
}
32+
}
33+
34+
public class MSSQLMergeDestinationTests: TestsRequiringAnExtractionConfiguration
35+
{
36+
//create table first time
37+
[Test]
38+
public void MSSQLMerge_Creates_Table()
39+
{
40+
var destination = new MSSqlMergeDestination_Test();
41+
42+
var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null)
43+
{
44+
Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name,
45+
Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny,
46+
Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny
47+
};
48+
_extractionServer.SaveToDatabase();
49+
destination.TargetDatabaseServer = _extractionServer;
50+
destination.DatabaseNamingPattern = "MSSQLMerge_Creates_Table";
51+
destination.TableNamingPattern = "MSSQLMerge_Creates_Table";
52+
destination.DeleteMergeTempTable = true;
53+
destination.PreInitialize(new Project(RepositoryLocator.DataExportRepository, "test project"), ThrowImmediatelyDataLoadEventListener.Quiet);
54+
var dt = new DataTable();
55+
dt.Columns.Add("chi");
56+
dt.Columns.Add("description");
57+
dt.PrimaryKey= new DataColumn[] { dt.Columns["chi"] };
58+
dt.Rows.Add("10", "one");
59+
destination.Execute(dt);
60+
var tbl = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(destination.DatabaseNamingPattern).ExpectTable(destination.TableNamingPattern);
61+
Assert.That(tbl.Exists());
62+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(1));
63+
}
64+
//merge in new data
65+
[Test]
66+
public void MSSQLMerge_Merge_Data()
67+
{
68+
var destination = new MSSqlMergeDestination_Test();
69+
70+
var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null)
71+
{
72+
Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name,
73+
Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny,
74+
Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny
75+
};
76+
_extractionServer.SaveToDatabase();
77+
destination.TargetDatabaseServer = _extractionServer;
78+
destination.DatabaseNamingPattern = "MSSQLMerge_Creates_Table";
79+
destination.TableNamingPattern = "MSSQLMerge_Merge_Data";
80+
destination.DeleteMergeTempTable = true;
81+
destination.PreInitialize(new Project(RepositoryLocator.DataExportRepository, "test project"), ThrowImmediatelyDataLoadEventListener.Quiet);
82+
var dt = new DataTable();
83+
dt.Columns.Add("chi");
84+
dt.Columns.Add("description");
85+
dt.PrimaryKey = new DataColumn[] { dt.Columns["chi"] };
86+
dt.Rows.Add("10", "one");
87+
destination.Execute(dt);
88+
var tbl = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(destination.DatabaseNamingPattern).ExpectTable(destination.TableNamingPattern);
89+
Assert.That(tbl.Exists());
90+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(1));
91+
dt.Rows.Remove(dt.Rows[0]);
92+
dt.Rows.Add("2", "two");
93+
destination.Execute(dt);
94+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(2));
95+
tbl.Drop();
96+
}
97+
// merge in data with dupicates
98+
[Test]
99+
public void MSSQLMerge_Merge_Update()
100+
{
101+
var destination = new MSSqlMergeDestination_Test();
102+
103+
var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null)
104+
{
105+
Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name,
106+
Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny,
107+
Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny
108+
};
109+
_extractionServer.SaveToDatabase();
110+
destination.TargetDatabaseServer = _extractionServer;
111+
destination.DatabaseNamingPattern = "MSSQLMerge_Creates_Table";
112+
destination.TableNamingPattern = "MSSQLMerge_Merge_Update";
113+
destination.DeleteMergeTempTable= true;
114+
destination.PreInitialize(new Project(RepositoryLocator.DataExportRepository, "test project"), ThrowImmediatelyDataLoadEventListener.Quiet);
115+
var dt = new DataTable();
116+
dt.Columns.Add("chi");
117+
dt.Columns.Add("description");
118+
dt.PrimaryKey = new DataColumn[] { dt.Columns["chi"] };
119+
dt.Rows.Add("10", "one");
120+
destination.Execute(dt);
121+
var tbl = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(destination.DatabaseNamingPattern).ExpectTable(destination.TableNamingPattern);
122+
Assert.That(tbl.Exists());
123+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(1));
124+
dt.Rows.Remove(dt.Rows[0]);
125+
dt.Rows.Add("2", "two");
126+
destination.Execute(dt);
127+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(2));
128+
dt.Rows.Add("10", "thr");
129+
destination.Execute(dt);
130+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(2));
131+
Assert.That(tbl.GetDataTable().Rows[1].ItemArray, Is.EqualTo(new List<object>() { 10, "thr" }));
132+
Assert.That(tbl.GetDataTable().Rows[0].ItemArray, Is.EqualTo(new List<object>() { 2, "two" }));
133+
tbl.Drop();
134+
}
135+
//megre in with perform delete
136+
[Test]
137+
public void MSSQLMerge_Merge_Delete()
138+
{
139+
var destination = new MSSqlMergeDestination_Test();
140+
141+
var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null)
142+
{
143+
Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name,
144+
Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny,
145+
Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny
146+
};
147+
_extractionServer.SaveToDatabase();
148+
destination.TargetDatabaseServer = _extractionServer;
149+
destination.DatabaseNamingPattern = "MSSQLMerge_Creates_Table";
150+
destination.TableNamingPattern = "MSSQLMerge_Merge_Delete";
151+
destination.DeleteMergeTempTable = true;
152+
destination.AllowMergeToPerformDeletes = true;
153+
destination.PreInitialize(new Project(RepositoryLocator.DataExportRepository, "test project"), ThrowImmediatelyDataLoadEventListener.Quiet);
154+
var dt = new DataTable();
155+
dt.Columns.Add("chi");
156+
dt.Columns.Add("description");
157+
dt.PrimaryKey = new DataColumn[] { dt.Columns["chi"] };
158+
dt.Rows.Add("10", "one");
159+
destination.Execute(dt);
160+
var tbl = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(destination.DatabaseNamingPattern).ExpectTable(destination.TableNamingPattern);
161+
Assert.That(tbl.Exists());
162+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(1));
163+
dt.Rows.Remove(dt.Rows[0]);
164+
dt.Rows.Add("2", "two");
165+
destination.Execute(dt);
166+
Assert.That(tbl.GetDataTable().Rows.Count, Is.EqualTo(1));
167+
Assert.That(tbl.GetDataTable().Rows[0].ItemArray, Is.EqualTo(new List<object>() {2, "two" }));
168+
tbl.Drop();
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)