diff --git a/CHANGELOG.md b/CHANGELOG.md index 721de42e66..ebedb52cda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Internal Note to Catalogue - Fix issue where project associations were not copied when a CIC was cloned - Fix issue with using Internal Catalogues in Cohort Identification Configurations - -## [9.1.2] - Unreleased +- Fix issue with using Project Specific Catalogues in Cohort Identification Configurations when not associated to project - Automatically fetch user settings from previous versions of RDMP when installing the latest version - Allow new columns to be added to archive extractions - Simplify use of Extraction Progress - Update CHI validation to allow mod10 CHIs - Hardened Confluence export functionality +- Fix issue with using Internal Catalogues in Cohort Identification Configurations ## [9.1.1] - 2025-12-02 - Allow Atlassian service workers to write to Confluence from RDMP diff --git a/Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs b/Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs index 8e4730b9b9..59f2a2cf92 100644 --- a/Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs +++ b/Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs @@ -4,11 +4,6 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; using NSubstitute; using NUnit.Framework; using Rdmp.Core.Caching; @@ -16,6 +11,7 @@ using Rdmp.Core.Caching.Pipeline.Destinations; using Rdmp.Core.Caching.Pipeline.Sources; using Rdmp.Core.Caching.Requests; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cache; @@ -25,6 +21,11 @@ using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; using Tests.Common; namespace Rdmp.Core.Tests.Caching.Integration; @@ -182,7 +183,7 @@ public void Check(ICheckNotifier notifier) private ILoadDirectory project; - public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ILoadDirectory value, IDataLoadEventListener listener) { project = value; } diff --git a/Rdmp.Core.Tests/Caching/Unit/TestIFileDataFlowComponents.cs b/Rdmp.Core.Tests/Caching/Unit/TestIFileDataFlowComponents.cs index 776d933c7a..06ce786839 100644 --- a/Rdmp.Core.Tests/Caching/Unit/TestIFileDataFlowComponents.cs +++ b/Rdmp.Core.Tests/Caching/Unit/TestIFileDataFlowComponents.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using Rdmp.Core.Caching.Layouts; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Cache; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Progress; @@ -83,12 +84,12 @@ public void Dispose(IDataLoadEventListener listener) throw new NotImplementedException(); } - public void PreInitialize(CacheProgress cacheProgress, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, CacheProgress cacheProgress, IDataLoadEventListener listener) { CacheProgress = cacheProgress; } - public void PreInitialize(DirectoryInfo cacheDirectory, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, DirectoryInfo cacheDirectory, IDataLoadEventListener listener) { CacheDirectory = cacheDirectory; } diff --git a/Rdmp.Core.Tests/CohortCommitting/CommitCohortExample.cs b/Rdmp.Core.Tests/CohortCommitting/CommitCohortExample.cs index e15da53f2b..4d8102340b 100644 --- a/Rdmp.Core.Tests/CohortCommitting/CommitCohortExample.cs +++ b/Rdmp.Core.Tests/CohortCommitting/CommitCohortExample.cs @@ -4,18 +4,19 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System.Data; -using System.Linq; using FAnsi; using NUnit.Framework; using Rdmp.Core.CohortCommitting; using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.CohortCommitting.Pipeline.Destinations; using Rdmp.Core.CohortCommitting.Pipeline.Destinations.IdentifierAllocation; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System.Data; +using System.Linq; using Tests.Common; namespace Rdmp.Core.Tests.CohortCommitting; @@ -63,7 +64,7 @@ public void CommitCohortExampleTest(DatabaseType dbType, string privateDataType) }; //initialize the destination - pipelineDestination.PreInitialize( + pipelineDestination.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), new CohortCreationRequest(project, definition, DataExportRepository, "A cohort created in an example unit test"), ThrowImmediatelyDataLoadEventListener.Quiet); diff --git a/Rdmp.Core.Tests/CohortCommitting/CreateNewCohortDatabaseWizardTests.cs b/Rdmp.Core.Tests/CohortCommitting/CreateNewCohortDatabaseWizardTests.cs index 078658042d..c6d5705c45 100644 --- a/Rdmp.Core.Tests/CohortCommitting/CreateNewCohortDatabaseWizardTests.cs +++ b/Rdmp.Core.Tests/CohortCommitting/CreateNewCohortDatabaseWizardTests.cs @@ -4,15 +4,13 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Linq; using FAnsi; using NUnit.Framework; using Rdmp.Core.CohortCommitting; using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.CohortCommitting.Pipeline.Destinations; using Rdmp.Core.CohortCommitting.Pipeline.Destinations.IdentifierAllocation; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; @@ -20,6 +18,9 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; using Rdmp.Core.ReusableLibraryCode.Settings; +using System; +using System.Data; +using System.Linq; using Tests.Common; namespace Rdmp.Core.Tests.CohortCommitting; @@ -181,7 +182,7 @@ public void TestActuallyCreatingIt(DatabaseType type) //the destination component that will put it there var dest = new BasicCohortDestination(); - dest.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), request, ThrowImmediatelyDataLoadEventListener.Quiet); //tell it to use the guid allocator dest.ReleaseIdentifierAllocator = typeof(GuidReleaseIdentifierAllocator); diff --git a/Rdmp.Core.Tests/CohortCreation/CohortCompilerRunnerTests.cs b/Rdmp.Core.Tests/CohortCreation/CohortCompilerRunnerTests.cs index 33524376a0..3bf9e3765c 100644 --- a/Rdmp.Core.Tests/CohortCreation/CohortCompilerRunnerTests.cs +++ b/Rdmp.Core.Tests/CohortCreation/CohortCompilerRunnerTests.cs @@ -11,6 +11,7 @@ using FAnsi.Discovery; using NUnit.Framework; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; @@ -28,7 +29,7 @@ public void CacheIdentifierListWithRunner_SimpleCase() { SetupCohort(out var db, out var cic, out var dt); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator,null), cic); var runner = new CohortCompilerRunner(compiler, 5000); runner.Run(new CancellationToken()); @@ -62,7 +63,7 @@ public void CacheIdentifierListWithRunner_WithCaching() cic.QueryCachingServer_ID = serverReference.ID; cic.SaveToDatabase(); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cic); var runner = new CohortCompilerRunner(compiler, 5000); runner.Run(new CancellationToken()); diff --git a/Rdmp.Core.Tests/CohortCreation/CohortCompilerTests.cs b/Rdmp.Core.Tests/CohortCreation/CohortCompilerTests.cs index 001d2a7cd3..f8f5a00070 100644 --- a/Rdmp.Core.Tests/CohortCreation/CohortCompilerTests.cs +++ b/Rdmp.Core.Tests/CohortCreation/CohortCompilerTests.cs @@ -11,9 +11,12 @@ using Rdmp.Core.CohortCreation; using Rdmp.Core.CohortCreation.Execution; using Rdmp.Core.CohortCreation.Execution.Joinables; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort.Joinables; +using Rdmp.Core.DataExport.Data; namespace Rdmp.Core.Tests.CohortCreation; @@ -22,7 +25,7 @@ public class CohortCompilerTests : CohortIdentificationTests [Test] public void AddSameTaskTwice_StaysAtOne() { - var compiler = new CohortCompiler(cohortIdentificationConfiguration); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator,null), cohortIdentificationConfiguration); container1.AddChild(aggregate1, 0); try { @@ -66,7 +69,7 @@ public void AddSameTaskTwice_StaysAtOne() [Test] public void AddContainer_StaysAtOne() { - var compiler = new CohortCompiler(cohortIdentificationConfiguration); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration); rootcontainer.AddChild(aggregate1, 1); compiler.AddTask(rootcontainer, null); //add the root container @@ -144,7 +147,7 @@ public void TestCompilerAddAllTasks(TestCompilerAddAllTasksTestCase testCase, bo //Joinable:aggregate5 (patient index table, the other Aggregates could JOIN to this) - var compiler = new CohortCompiler(cohortIdentificationConfiguration); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration); rootcontainer.AddChild(aggregate1, 1); rootcontainer.AddChild(container1); container1.Order = 2; @@ -201,7 +204,7 @@ public void TestCompilerAddAllTasks(TestCompilerAddAllTasksTestCase testCase, bo [Test] public void TestCompilerWithInternalCatalogues() { - var compiler = new CohortCompiler(cohortIdentificationConfiguration); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration); testData.catalogue.IsInternalDataset = true; testData.catalogue.SaveToDatabase(); @@ -217,4 +220,57 @@ public void TestCompilerWithInternalCatalogues() Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(3)); Assert.That(compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Any(m => m.Contains(" is marked as Internal. Internal Catalogues cannot be used in Cohort Identification Configurations."))); } + + + [Test] + public void TestCompilerWithProjectSpecificCatalogues_Fail() + { + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration); + + var proj = new Project(DataExportRepository, "TestCompilerWithProjectSpecificCatalogues_Fail"); + proj.SaveToDatabase(); + + //having an issue making it project specific + var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(new ThrowImmediatelyActivator(RepositoryLocator), aggregate1.Catalogue, proj, true); + Assert.DoesNotThrow(() => cmd.Execute()); + rootcontainer.AddChild(aggregate1, 1); + rootcontainer.AddChild(container1); + container1.Order = 2; + container1.SaveToDatabase(); + + cohortIdentificationConfiguration.RootCohortAggregateContainer_ID = rootcontainer.ID; + cohortIdentificationConfiguration.SaveToDatabase(); + + compiler.AddAllTasks(true); + Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(2)); + Assert.That(compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Contains("Catalogue BulkData is marked as Project Specific, but this Cohort Identification Configurations is not associated with the same Project.")); + } + [Test] + public void TestCompilerWithProjectSpecificCatalogues_Pass() + { + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration); + + var proj = new Project(DataExportRepository, "TestCompilerWithProjectSpecificCatalogues_Fail"); + proj.SaveToDatabase(); + + //having an issue making it project specific + var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(new ThrowImmediatelyActivator(RepositoryLocator), aggregate1.Catalogue, proj, true); + Assert.DoesNotThrow(() => cmd.Execute()); + rootcontainer.AddChild(aggregate1, 1); + rootcontainer.AddChild(container1); + container1.Order = 2; + container1.SaveToDatabase(); + + cohortIdentificationConfiguration.RootCohortAggregateContainer_ID = rootcontainer.ID; + cohortIdentificationConfiguration.SaveToDatabase(); + + var associateCmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(new ThrowImmediatelyActivator(RepositoryLocator)); + associateCmd.SetTarget(proj); + associateCmd.SetTarget(cohortIdentificationConfiguration); + Assert.DoesNotThrow(() => associateCmd.Execute()); + + compiler.AddAllTasks(true); + Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(2)); + Assert.That(!compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Contains("Catalogue BulkData is marked as Project Specific, but this Cohort Identification Configurations is not associated with the same Project.")); + } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/CohortCreation/PluginCohortCompilerTests.cs b/Rdmp.Core.Tests/CohortCreation/PluginCohortCompilerTests.cs index 514477dfb8..83e67c85af 100644 --- a/Rdmp.Core.Tests/CohortCreation/PluginCohortCompilerTests.cs +++ b/Rdmp.Core.Tests/CohortCreation/PluginCohortCompilerTests.cs @@ -8,6 +8,7 @@ using NUnit.Framework; using Rdmp.Core.CohortCommitting.Pipeline.Sources; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.CommandLine.Interactive; @@ -15,11 +16,11 @@ using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; using Rdmp.Core.Tests.CohortCreation.QueryTests; using System; using System.Data; -using Rdmp.Core.ReusableLibraryCode.Checks; -using Rdmp.Core.ReusableLibraryCode.Progress; namespace Rdmp.Core.Tests.CohortCreation; @@ -53,7 +54,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly() // run the cic var source = new CohortIdentificationConfigurationSource(); - source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); // 5 random chi numbers @@ -65,7 +66,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly() // run the cic again source = new CohortIdentificationConfigurationSource(); - source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet); dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); // because the rules changed to generate 2 chis only there should be a new result @@ -76,7 +77,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly() // run the cic again with no changes, the results should be unchanged since there is no config changed // I.e. no new chis should be generated and the cached values returned source = new CohortIdentificationConfigurationSource(); - source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet); dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.That(dt.Rows, Has.Count.EqualTo(2)); @@ -223,7 +224,7 @@ public void TestIPluginCohortCompiler_AsPatientIndexTable() // run the cic again var source = new CohortIdentificationConfigurationSource(); - source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet); var result = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.That(result.Rows, Has.Count.EqualTo(1)); } diff --git a/Rdmp.Core.Tests/CohortCreation/QueryTests/CohortCompilerCacheJoinableTest.cs b/Rdmp.Core.Tests/CohortCreation/QueryTests/CohortCompilerCacheJoinableTest.cs index be957641bd..5547f2aa84 100644 --- a/Rdmp.Core.Tests/CohortCreation/QueryTests/CohortCompilerCacheJoinableTest.cs +++ b/Rdmp.Core.Tests/CohortCreation/QueryTests/CohortCompilerCacheJoinableTest.cs @@ -6,19 +6,20 @@ using NUnit.Framework; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.Cohort.Joinables; using Rdmp.Core.Databases; +using Rdmp.Core.MapsDirectlyToDatabaseTable.Versioning; +using Rdmp.Core.QueryCaching.Aggregation; +using Rdmp.Core.ReusableLibraryCode.Checks; using System; using System.Data; using System.Linq; -using Rdmp.Core.MapsDirectlyToDatabaseTable.Versioning; using Tests.Common.Scenarios; -using Rdmp.Core.QueryCaching.Aggregation; -using Rdmp.Core.ReusableLibraryCode.Checks; using static Rdmp.Core.CohortCreation.Execution.CohortCompilerRunner; namespace Rdmp.Core.Tests.CohortCreation.QueryTests; @@ -113,7 +114,7 @@ public void CohortIdentificationConfiguration_Join_PatientIndexTable() Assert.That(acPatIndex.IsJoinablePatientIndexTable()); }); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cic); var runner = new CohortCompilerRunner(compiler, 50); diff --git a/Rdmp.Core.Tests/Curation/CrossPlatformParameterTests/BasicParameterUseTests.cs b/Rdmp.Core.Tests/Curation/CrossPlatformParameterTests/BasicParameterUseTests.cs index f95b4612ca..9262d501ad 100644 --- a/Rdmp.Core.Tests/Curation/CrossPlatformParameterTests/BasicParameterUseTests.cs +++ b/Rdmp.Core.Tests/Curation/CrossPlatformParameterTests/BasicParameterUseTests.cs @@ -4,10 +4,9 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System.Data; -using System.Linq; using FAnsi; using NUnit.Framework; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Spontaneous; @@ -17,6 +16,8 @@ using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; +using System.Data; +using System.Linq; using Tests.Common; namespace Rdmp.Core.Tests.Curation.CrossPlatformParameterTests; @@ -48,7 +49,7 @@ public void Test_DatabaseTypeQueryWithParameter_IntParameter(DatabaseType dbType { ///////////////////////UPLOAD THE DataTable TO THE DESTINATION//////////////////////////////////////////// var uploader = new DataTableUploadDestination(); - uploader.PreInitialize(db, new ThrowImmediatelyDataLoadJob()); + uploader.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), db, new ThrowImmediatelyDataLoadJob()); uploader.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken()); uploader.Dispose(new ThrowImmediatelyDataLoadJob(), null); diff --git a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/AggregateDataBasedTests.cs b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/AggregateDataBasedTests.cs index 251b6a0927..c2b6278817 100644 --- a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/AggregateDataBasedTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/AggregateDataBasedTests.cs @@ -4,14 +4,12 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Linq; using FAnsi; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery.QuerySyntax.Aggregation; using NUnit.Framework; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Spontaneous; @@ -21,6 +19,9 @@ using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Linq; using Tests.Common; using TypeGuesser; @@ -75,7 +76,7 @@ private DiscoveredTable UploadTestDataAsTableToServer(DatabaseType type, out ICa var data = GetTestDataTable(); var uploader = new DataTableUploadDestination(); - uploader.PreInitialize(db, listener); + uploader.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), db, listener); uploader.ProcessPipelineData(data, listener, new GracefulCancellationToken()); uploader.Dispose(listener, null); var tbl = db.ExpectTable(uploader.TargetTableName); diff --git a/Rdmp.Core.Tests/Curation/Unit/PreInitializeTests.cs b/Rdmp.Core.Tests/Curation/Unit/PreInitializeTests.cs index 971de5aae0..43c1bf013c 100644 --- a/Rdmp.Core.Tests/Curation/Unit/PreInitializeTests.cs +++ b/Rdmp.Core.Tests/Curation/Unit/PreInitializeTests.cs @@ -4,12 +4,13 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; using NUnit.Framework; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; namespace Rdmp.Core.Tests.Curation.Unit; @@ -85,7 +86,7 @@ private class FishUser : IPipelineRequirement, IDataFlowComponent, IDataFlowComponent< { public IFish IFish; - public void PreInitialize(Fish value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, Fish value, IDataLoadEventListener listener) { IFish = value; } diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteCrossServerDatasetExtractionSourceTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteCrossServerDatasetExtractionSourceTest.cs index 845b472ef9..f90fcbc12a 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteCrossServerDatasetExtractionSourceTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteCrossServerDatasetExtractionSourceTest.cs @@ -117,7 +117,7 @@ SELECT DISTINCT { TemporaryDatabaseName = "tempdb" }; - s.PreInitialize(_request, ThrowImmediatelyDataLoadEventListener.Quiet); + s.PreInitialize(null,_request, ThrowImmediatelyDataLoadEventListener.Quiet); var hacked = s.HackExtractionSQL(_request.QueryBuilder.SQL, ThrowImmediatelyDataLoadEventListener.QuietPicky); diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteDatasetExtractionFlatFileDestinationTests.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteDatasetExtractionFlatFileDestinationTests.cs index 98ec316b49..b879716ca6 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteDatasetExtractionFlatFileDestinationTests.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteDatasetExtractionFlatFileDestinationTests.cs @@ -6,14 +6,15 @@ using FAnsi.Discovery; using NUnit.Framework; +using NUnit.Framework.Legacy; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.Logging; +using Rdmp.Core.ReusableLibraryCode.Progress; using System; using System.Data; using System.IO; -using NUnit.Framework.Legacy; -using Rdmp.Core.ReusableLibraryCode.Progress; using Tests.Common.Scenarios; namespace Rdmp.Core.Tests.DataExport.DataExtraction; @@ -40,9 +41,9 @@ public void ExtractionDestination_FloatRounding(bool lotsOfDecimalPlaces) if (_request.QueryBuilder == null) _request.GenerateQueryBuilder(); dest.RoundFloatsTo = lotsOfDecimalPlaces ? 10 : 2; - dest.PreInitialize(_request, ThrowImmediatelyDataLoadEventListener.Quiet); - dest.PreInitialize(_project, ThrowImmediatelyDataLoadEventListener.Quiet); - dest.PreInitialize((DataLoadInfo)dli, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), _request, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), _project, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), (DataLoadInfo)dli, ThrowImmediatelyDataLoadEventListener.Quiet); dest.ProcessPipelineData(dt, ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); dest.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, null); diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlChecksTests.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlChecksTests.cs index f0537382bb..e6af6db323 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlChecksTests.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlChecksTests.cs @@ -99,8 +99,8 @@ public void ServerDatabaseIsPresentAndCorrect(bool alreadyExists) try { var destination = new ExecuteFullExtractionToDatabaseMSSql(); - destination.PreInitialize(_projectStub, ThrowImmediatelyDataLoadEventListener.Quiet); - destination.PreInitialize(_commandStub, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,_projectStub, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,_commandStub, ThrowImmediatelyDataLoadEventListener.Quiet); destination.TargetDatabaseServer = server; destination.TableNamingPattern = "$d"; @@ -142,8 +142,8 @@ public void ServerDatabaseIsPresentAndCorrectButHasTablesInIt() try { var destination = new ExecuteFullExtractionToDatabaseMSSql(); - destination.PreInitialize(_projectStub, ThrowImmediatelyDataLoadEventListener.Quiet); - destination.PreInitialize(_commandStub, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,_projectStub, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,_commandStub, ThrowImmediatelyDataLoadEventListener.Quiet); destination.TargetDatabaseServer = server; destination.TableNamingPattern = "$d"; destination.DatabaseNamingPattern = "FictionalDatabase"; diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecutePkSynthesizerDatasetExtractionSourceTests.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecutePkSynthesizerDatasetExtractionSourceTests.cs index 9e79e915b7..4ef15daa8f 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecutePkSynthesizerDatasetExtractionSourceTests.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecutePkSynthesizerDatasetExtractionSourceTests.cs @@ -45,7 +45,7 @@ public void Test_CatalogueItems_ExtractionInformationPrimaryKey_IsRespected() SetupExtractDatasetCommand("ExtractionInformationPrimaryKey_IsRespected", new[] { "DateOfBirth" }); var source = new ExecutePkSynthesizerDatasetExtractionSource(); - source.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,request, ThrowImmediatelyDataLoadEventListener.Quiet); var chunk = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.Multiple(() => @@ -65,7 +65,7 @@ public void Test_CatalogueItems_ExtractionInformationMultiPrimaryKey_IsRespected new[] { "PrivateID", "DateOfBirth" }); var source = new ExecutePkSynthesizerDatasetExtractionSource(); - source.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,request, ThrowImmediatelyDataLoadEventListener.Quiet); var chunk = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.Multiple(() => @@ -84,7 +84,7 @@ public void Test_CatalogueItems_NonExtractedPrimaryKey_AreRespected() new[] { "DateOfBirth" }); var source = new ExecutePkSynthesizerDatasetExtractionSource(); - source.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,request, ThrowImmediatelyDataLoadEventListener.Quiet); var chunk = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.Multiple(() => @@ -107,7 +107,7 @@ public void Test_CatalogueItems_NonExtractedPrimaryKey_MultiTable_PksAreMerged() new[] { "DateOfBirth" }, true, true); var source = new ExecutePkSynthesizerDatasetExtractionSource(); - source.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,request, ThrowImmediatelyDataLoadEventListener.Quiet); var chunk = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.Multiple(() => @@ -133,7 +133,7 @@ public void Test_CatalogueItems_NonExtractedPrimaryKey_LookupsOnly_IsRespected() new[] { "DateOfBirth" }, true); var source = new ExecutePkSynthesizerDatasetExtractionSource(); - source.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,request, ThrowImmediatelyDataLoadEventListener.Quiet); var chunk = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); Assert.Multiple(() => diff --git a/Rdmp.Core.Tests/DataExport/ImportFileTests.cs b/Rdmp.Core.Tests/DataExport/ImportFileTests.cs index 0c74766cb0..9812d0d517 100644 --- a/Rdmp.Core.Tests/DataExport/ImportFileTests.cs +++ b/Rdmp.Core.Tests/DataExport/ImportFileTests.cs @@ -4,16 +4,17 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using Microsoft.Data.SqlClient; -using System.IO; using FAnsi.Discovery; +using Microsoft.Data.SqlClient; using NUnit.Framework; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.DataLoad.Modules.DataFlowSources; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.IO; using Tests.Common; namespace Rdmp.Core.Tests.DataExport; @@ -48,7 +49,7 @@ public void ImportFile() StronglyTypeInput = true }; - source.PreInitialize(new FlatFileToLoad(new FileInfo(file)), + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), new FlatFileToLoad(new FileInfo(file)), ThrowImmediatelyDataLoadEventListener.Quiet); //this is the file we want to load source.Check(ThrowImmediatelyCheckNotifier.Quiet); diff --git a/Rdmp.Core.Tests/DataExport/ProjectChecksTestsComplex.cs b/Rdmp.Core.Tests/DataExport/ProjectChecksTestsComplex.cs index 4e336d908a..0f8ceae779 100644 --- a/Rdmp.Core.Tests/DataExport/ProjectChecksTestsComplex.cs +++ b/Rdmp.Core.Tests/DataExport/ProjectChecksTestsComplex.cs @@ -40,7 +40,7 @@ public void DatasetIsDisabled() //but if the user goes ahead and executes the extraction that should fail too var source = new ExecuteDatasetExtractionSource(); - source.PreInitialize(_request, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), _request, ThrowImmediatelyDataLoadEventListener.Quiet); var exception2 = Assert.Throws(() => source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken())); diff --git a/Rdmp.Core.Tests/DataExport/TableValuedFunctionTests/EndToEndTableValuedFunction.cs b/Rdmp.Core.Tests/DataExport/TableValuedFunctionTests/EndToEndTableValuedFunction.cs index a06af0e9f3..d5891a3756 100644 --- a/Rdmp.Core.Tests/DataExport/TableValuedFunctionTests/EndToEndTableValuedFunction.cs +++ b/Rdmp.Core.Tests/DataExport/TableValuedFunctionTests/EndToEndTableValuedFunction.cs @@ -4,16 +4,14 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Linq; using FAnsi; using FAnsi.Discovery; using NUnit.Framework; -using Rdmp.Core.QueryBuilding; using Rdmp.Core.CohortCommitting; using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.CohortCommitting.Pipeline.Destinations; using Rdmp.Core.CohortCommitting.Pipeline.Sources; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; @@ -24,9 +22,12 @@ using Rdmp.Core.DataExport.DataExtraction.Pipeline.Sources; using Rdmp.Core.DataExport.DataExtraction.UserPicks; using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.QueryBuilding; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Linq; using Tests.Common; namespace Rdmp.Core.Tests.DataExport.TableValuedFunctionTests; @@ -450,7 +451,7 @@ private void TestDataExportOfTvf() var source = new ExecuteDatasetExtractionSource(); - source.PreInitialize(extractionCommand, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), extractionCommand, ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs index 95382b5950..f65a81161b 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs @@ -33,7 +33,7 @@ public void DataTableChangesLengths_NoReAlter() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -133,7 +133,7 @@ public void DataTableChangesLengths_RandomColumnOrder(bool createIdentity, int n //the bulk insert is var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); //order is inverted where name comes out at the end column (index 2) using var dt1 = new DataTable(); @@ -192,7 +192,7 @@ ALTER TABLE DroppedColumnsTable add color varchar(1) //the bulk insert is var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); //order is inverted where name comes out at the end column (index 2) using var dt1 = new DataTable(); @@ -227,7 +227,7 @@ public void DataTableEmpty_ThrowHelpfulException() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable { @@ -248,7 +248,7 @@ public void DataTableNoRows_ThrowHelpfulException() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("GoTeamGo"); @@ -269,7 +269,7 @@ public void DataTableChangesLengths_AllowAlter() var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -303,7 +303,7 @@ public void DoubleResizingBetweenIntAndDouble() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -337,7 +337,7 @@ public void BatchResizing(string expectedDatatypeInDatabase, object batch1Value, var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -374,7 +374,7 @@ public void BatchResizing_WithExplicitWriteTypes(string expectedDatatypeInDataba var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -411,7 +411,7 @@ public void VeryLongStringIsVarcharMax() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; @@ -449,7 +449,7 @@ public void DecimalResizing(bool negative) var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -516,7 +516,7 @@ public void DataTypeEstimation(string expectedDatatypeInDatabase, object[] rowVa var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("myCol", typeof(string)); @@ -560,7 +560,7 @@ public void DecimalZeros(bool sendTheZero) var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -721,7 +721,7 @@ public void TestResizing_WithDetection(DatabaseType dbType) { AllowResizingColumnsAtUploadTime = true }; - dest.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); dest.ProcessPipelineData(dt, ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); dest.ProcessPipelineData(dt2, ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); @@ -749,7 +749,7 @@ public void Test_SingleQuote_InText(DatabaseType dbType, string testValue) { AllowResizingColumnsAtUploadTime = true }; - dest.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); dest.ProcessPipelineData(dt, ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); dest.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, null); @@ -769,7 +769,7 @@ public void DodgyTypes() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -815,7 +815,7 @@ public void TypeAlteringlResizing() var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -853,7 +853,7 @@ public void MySqlTest_Simple() var toConsole = ThrowImmediatelyDataLoadEventListener.Quiet; var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt = new DataTable(); @@ -912,7 +912,7 @@ public void MySqlTest_Resize() var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -994,7 +994,7 @@ public void TestDestinationAlreadyExistingIsOk(bool targetTableIsEmpty) var listener = ThrowImmediatelyDataLoadEventListener.Quiet; //pre initialzie with the database (which must be part of any pipeline use case involving a DataTableUploadDestination) - destinationComponent.PreInitialize(db, listener); + destinationComponent.PreInitialize(null,db, listener); //tell the destination component to process the data destinationComponent.ProcessPipelineData(dt, listener, new GracefulCancellationToken()); @@ -1019,7 +1019,7 @@ public void DataTableUploadDestinationTests_PrimaryKeyDataTableWithAlterSizeLate AllowResizingColumnsAtUploadTime = true }; - destination.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + destination.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); var dt1 = new DataTable { @@ -1099,7 +1099,7 @@ public void TestDestinationAlreadyExisting_ColumnSubset() var listener = ThrowImmediatelyDataLoadEventListener.Quiet; //pre initialzie with the database (which must be part of any pipeline use case involving a DataTableUploadDestination) - destinationComponent.PreInitialize(db, listener); + destinationComponent.PreInitialize(null,db, listener); //tell the destination component to process the data destinationComponent.ProcessPipelineData(dt, listener, new GracefulCancellationToken()); @@ -1123,7 +1123,7 @@ public void Test_DataTableUploadDestination_ScientificNotation(DatabaseType dbTy dt.Rows.Add("-4.10235746055587E-05"); //this string is untyped var dest = new DataTableUploadDestination(); - dest.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); try { @@ -1172,7 +1172,7 @@ public void Test_DataTableUploadDestination_ForceBool(DatabaseType dbType) dt.Rows.Add("F", "Def"); var dest = new DataTableUploadDestination(); - dest.PreInitialize(db, ThrowImmediatelyDataLoadEventListener.Quiet); + dest.PreInitialize(null,db, ThrowImmediatelyDataLoadEventListener.Quiet); dest.Adjuster = typeof(AdjustColumnDelegater); AdjustColumnDelegater.AdjusterDelegate = s => @@ -1233,7 +1233,7 @@ public void TwoBatch_BooleanResizingTest(DatabaseType dbType, bool giveNullValue var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -1308,7 +1308,7 @@ public void TwoBatch_MiscellaneousTest(DatabaseType dbType, string v1, string v2 var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; var dt1 = new DataTable(); @@ -1369,7 +1369,7 @@ public void TwoBatch_ExplicitRealDataType() var toMemory = new ToMemoryDataLoadEventListener(true); var destination = new DataTableUploadDestination(); - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); destination.AllowResizingColumnsAtUploadTime = true; destination.AddExplicitWriteType("FloatCol", "real"); @@ -1417,7 +1417,7 @@ public void CreateIndex_OK() IndexTableName = "CreateIndex_OK", UserDefinedIndexes = new List() { "name" } }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1456,7 +1456,7 @@ public void CreateIndex_NO_PK() IndexTables = true, IndexTableName = "CreateIndex_OK" }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1495,7 +1495,7 @@ public void CreateIndex_RecreateExistingFail() IndexTableName = "CreateIndex_OK", UserDefinedIndexes = new List() { "name" } }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1535,7 +1535,7 @@ public void UseTriggerwithoutPrimaryKey() { UseTrigger = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1570,7 +1570,7 @@ public void UseTriggerwithPrimaryKey() { UseTrigger = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1609,7 +1609,7 @@ public void DataTableUploadClashSameRow() UserDefinedIndexes = new List() { "name" }, AppendDataIfTableExists = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1656,7 +1656,7 @@ public void DataTableUploadClashSameRowWithTrigger() AppendDataIfTableExists = true, UseTrigger = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1702,7 +1702,7 @@ public void DataTableUploadClashUpdateNoTrigger() UserDefinedIndexes = new List() { "name" }, AppendDataIfTableExists = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1733,7 +1733,7 @@ public void DataTableUploadClashUpdateNoTrigger() UserDefinedIndexes = new List() { "name" }, AppendDataIfTableExists = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); Assert.Throws(() => destination.ProcessPipelineData(dt1, toConsole, token)); destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, null); var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); @@ -1758,7 +1758,7 @@ public void DataTableUploadClashUpdateWithTrigger() AppendDataIfTableExists = true, UseTrigger = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); @@ -1813,7 +1813,7 @@ public void DataTableUploadClashUpdateDropColumnWithTrigger() AppendDataIfTableExists = true, UseTrigger = true }; - destination.PreInitialize(db, toConsole); + destination.PreInitialize(null,db, toConsole); var dt1 = new DataTable(); dt1.Columns.Add("name", typeof(string)); diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests.cs index 7c631c6b67..d2058dea6f 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests.cs @@ -55,7 +55,7 @@ public void SeparatorNotSet_Throws() { var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); var ex = Assert.Throws(() => source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken())); Assert.That(ex.Message, Does.Contain("Separator has not been set")); @@ -67,7 +67,7 @@ public void LoadCSVWithCorrectDatatypes_ForceHeadersWhitespace() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.ForceHeaders = "chi ,Study ID\t ,Date"; source.ForceHeadersReplacesFirstLineInFile = true; @@ -103,7 +103,7 @@ public void LoadCSVWithCorrectDatatypes_DatatypesAreCorrect() { var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.StronglyTypeInput = true; //makes the source interpret the file types properly @@ -129,7 +129,7 @@ public void OverrideDatatypes_ForcedFreakyTypesCorrect() { var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.StronglyTypeInput = true; //makes the source interpret the file types properly @@ -173,7 +173,7 @@ public void TestIgnoreQuotes() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.IgnoreQuotes = true; source.MaxBatchSize = 10000; @@ -212,7 +212,7 @@ public void BadDataTestExtraColumns(BadDataHandlingStrategy strategy) var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -272,7 +272,7 @@ public void DelimitedFlatFileDataFlowSource_ProperQuoteEscaping() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -318,7 +318,7 @@ public void DelimitedFlatFileDataFlowSource_LoadDataWithQuotesInMiddle_IgnoreBad var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -365,7 +365,7 @@ public void DelimitedFlatFileDataFlowSource_TrashFile_IgnoreBadReads() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -405,7 +405,7 @@ public void DelimitedFlatFileDataFlowSource_LoadDataWithQuotesInMiddle_WithMulti var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; source.AttemptToResolveNewLinesInRecords = true; @@ -447,7 +447,7 @@ public void BadDataTestExtraColumns_ErrorIsOnLastLine(BadDataHandlingStrategy st var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -510,7 +510,7 @@ public void NewLinesInConstantString_EscapedCorrectly() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -552,7 +552,7 @@ public void NewLinesInConstantString_NotEscaped(BadDataHandlingStrategy strategy var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = 10000; @@ -612,7 +612,7 @@ public void OverrideHeadersAndTab() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = "\\t"; //<-- Important this is the string value SLASH T not an actual escaped tab as C# understands it. This reflects the user pressing slash and t on his keyboard for the Separator argument in the UI source.ForceHeaders = "CHI\tStudyID\tDate"; @@ -652,7 +652,7 @@ public void Test_IgnoreColumns() var testFile = new FileInfo(filename); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(testFile), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = "\\t"; //<-- Important this is the string value SLASH T not an actual escaped tab as C# understands it. This reflects the user pressing slash and t on his keyboard for the Separator argument in the UI source.ForceHeaders = "CHI\tStudyID\tDate\tSomeText"; @@ -716,7 +716,7 @@ public void Test_ScientificNotation_StronglyTyped() File.WriteAllText(f, sb.ToString()); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(new FileInfo(f)), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(new FileInfo(f)), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = DelimitedFlatFileDataFlowSource.MinimumStronglyTypeInputBatchSize; source.StronglyTypeInputBatchSize = DelimitedFlatFileDataFlowSource.MinimumStronglyTypeInputBatchSize; @@ -754,7 +754,7 @@ public void Test_IgnoreQuotes() 2,""The Killer""? I've heard of him hes a bad un"); var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(new FileInfo(f)), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(new FileInfo(f)), ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.MaxBatchSize = DelimitedFlatFileDataFlowSource.MinimumStronglyTypeInputBatchSize; source.StronglyTypeInputBatchSize = DelimitedFlatFileDataFlowSource.MinimumStronglyTypeInputBatchSize; diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTestsBase.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTestsBase.cs index c28a5a5594..a690d82a75 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTestsBase.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTestsBase.cs @@ -54,7 +54,7 @@ protected static DataTable RunGetChunk(FlatFileToLoad file, BadDataHandlingStrat protected static DataTable RunGetChunk(FlatFileToLoad file, Action adjust = null) { var source = new DelimitedFlatFileDataFlowSource(); - source.PreInitialize(file, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,file, ThrowImmediatelyDataLoadEventListener.Quiet); source.Separator = ","; source.StronglyTypeInput = true; //makes the source interpret the file types properly source.StronglyTypeInputBatchSize = 100; diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/SourceTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/SourceTests.cs index 73ba5099dc..2e400ae032 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/SourceTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/SourceTests.cs @@ -4,9 +4,8 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; using NUnit.Framework; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.DataFlowPipeline; @@ -16,6 +15,8 @@ using Rdmp.Core.DataLoad.Engine.Pipeline.Sources; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; using Tests.Common; namespace Rdmp.Core.Tests.DataLoad.Engine.Integration.PipelineTests.Sources; @@ -196,7 +197,7 @@ public void Abort(IDataLoadEventListener listener) throw new NotImplementedException(); } - public void PreInitialize(TableInfo value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, TableInfo value, IDataLoadEventListener listener) { PreInitToThis = value; } @@ -220,13 +221,13 @@ public void Abort(IDataLoadEventListener listener) throw new NotImplementedException(); } - public void PreInitialize(TableInfo value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, TableInfo value, IDataLoadEventListener listener) { PreInitToThis = value; } - public void PreInitialize(LoadModuleAssembly value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, LoadModuleAssembly value, IDataLoadEventListener listener) { throw new NotImplementedException(); } @@ -267,7 +268,7 @@ public void Abort(IDataLoadEventListener listener) throw new NotImplementedException(); } - public void PreInitialize(object value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, object value, IDataLoadEventListener listener) { Object = value; } @@ -292,12 +293,12 @@ public void Abort(IDataLoadEventListener listener) throw new NotImplementedException(); } - public void PreInitialize(object value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, object value, IDataLoadEventListener listener) { Object = value; } - public void PreInitialize(string value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, string value, IDataLoadEventListener listener) { Object = value; } diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Unit/CohortSamplerTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Unit/CohortSamplerTests.cs index ab3e46a9fb..23e48f9387 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Unit/CohortSamplerTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Unit/CohortSamplerTests.cs @@ -6,14 +6,15 @@ using NUnit.Framework; using Rdmp.Core.CohortCommitting.Pipeline; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataLoad.Modules.DataFlowOperations; +using Rdmp.Core.ReusableLibraryCode.Progress; using System; using System.Data; using System.Linq; using System.Threading; -using Rdmp.Core.ReusableLibraryCode.Progress; using Tests.Common; namespace Rdmp.Core.Tests.DataLoad.Engine.Unit; @@ -191,7 +192,7 @@ private CohortSampler GetCohortSampler() var request = new CohortCreationRequest(p, definition, Repository, "Cohort read from space!!!"); - sampler.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet); + sampler.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), request, ThrowImmediatelyDataLoadEventListener.Quiet); return sampler; } diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Unit/ExcelTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Unit/ExcelTests.cs index 5a97877254..ca75d7d93a 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Unit/ExcelTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Unit/ExcelTests.cs @@ -68,7 +68,7 @@ public void DontTryToOpenWithDelimited_ThrowsInvalidFileExtension() { Separator = "," }; - invalid.PreInitialize(new FlatFileToLoad(new FileInfo(TestFile)), ThrowImmediatelyDataLoadEventListener.Quiet); + invalid.PreInitialize(null,new FlatFileToLoad(new FileInfo(TestFile)), ThrowImmediatelyDataLoadEventListener.Quiet); var ex = Assert.Throws(() => invalid.Check(ThrowImmediatelyCheckNotifier.Quiet)); Assert.That(ex?.Message, Does.Contain("File Book1.xlsx has a prohibited file extension .xlsx")); } @@ -80,7 +80,7 @@ public void NormalBook_FirstRowCorrect(string versionOfTestFile) { var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(_fileLocations[versionOfTestFile]), + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[versionOfTestFile]), ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); @@ -112,7 +112,7 @@ public void NormalBook_FirstRowCorrect_AddFilenameColumnNamed(string versionOfTe AddFilenameColumnNamed = "Path" }; - source.PreInitialize(new FlatFileToLoad(_fileLocations[versionOfTestFile]), + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[versionOfTestFile]), ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); @@ -152,7 +152,7 @@ public void ExcelDateTimeDeciphering(string versionOfTestFile) var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(_fileLocations[versionOfTestFile]), listener); + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[versionOfTestFile]), listener); var dt = source.GetChunk(listener, new GracefulCancellationToken()); Assert.That(dt.Rows, Has.Count.EqualTo(5)); @@ -191,7 +191,7 @@ public void TestOddFormats() WorkSheetName = "MySheet" }; - source.PreInitialize(new FlatFileToLoad(_fileLocations[OddFormatsFile]), listener); + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[OddFormatsFile]), listener); var dt = source.GetChunk(listener, new GracefulCancellationToken()); Assert.Multiple(() => @@ -233,7 +233,7 @@ public void NormalBook_NoEmptyRowsRead() var listener = new ToMemoryDataLoadEventListener(true); - source.PreInitialize(new FlatFileToLoad(_fileLocations[TestFile]), listener); + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[TestFile]), listener); var dt = source.GetChunk(listener, new GracefulCancellationToken()); Assert.That(dt.Rows, Has.Count.EqualTo(5)); @@ -246,7 +246,7 @@ public void FreakyTestFile_WarningsCorrect() var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(_fileLocations[FreakyTestFile]), + source.PreInitialize(null,new FlatFileToLoad(_fileLocations[FreakyTestFile]), ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(messages, new GracefulCancellationToken()); @@ -268,7 +268,7 @@ public void BlankFirstLineFile() "DataLoad", "Engine", "Resources", "BlankLineBook.xlsx")); Assert.That(fi.Exists); - source.PreInitialize(new FlatFileToLoad(fi), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(fi), ThrowImmediatelyDataLoadEventListener.Quiet); var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken()); @@ -296,7 +296,7 @@ public void BlankWorkbook() "BlankBook.xlsx")); Assert.That(fi.Exists); - source.PreInitialize(new FlatFileToLoad(fi), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(fi), ThrowImmediatelyDataLoadEventListener.Quiet); var ex = Assert.Throws(() => @@ -308,7 +308,7 @@ public void BlankWorkbook() public void Checks_ValidFileExtension_Pass() { var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(new FileInfo("bob.xlsx")), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(new FileInfo("bob.xlsx")), ThrowImmediatelyDataLoadEventListener.Quiet); source.Check(ThrowImmediatelyCheckNotifier.QuietPicky); } @@ -316,7 +316,7 @@ public void Checks_ValidFileExtension_Pass() public void Checks_ValidFileExtension_InvalidExtensionPass() { var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(new FileInfo("bob.csv")), ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,new FlatFileToLoad(new FileInfo("bob.csv")), ThrowImmediatelyDataLoadEventListener.Quiet); var ex = Assert.Throws(() => source.Check(ThrowImmediatelyCheckNotifier.QuietPicky)); Assert.That(ex?.Message, Is.EqualTo("File extension bob.csv has an invalid extension:.csv (this class only accepts:.xlsx,.xls)")); } diff --git a/Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs b/Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs index 955e6c357d..053bb7f704 100644 --- a/Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs +++ b/Rdmp.Core.Tests/QueryCaching/QueryCachingCrossServerTests.cs @@ -23,11 +23,13 @@ using Tests.Common; using Tests.Common.Scenarios; using TypeGuesser; +using Rdmp.Core.CommandExecution; namespace Rdmp.Core.Tests.QueryCaching; internal class QueryCachingCrossServerTests : TestsRequiringA { + [TestCase(DatabaseType.MicrosoftSQLServer, typeof(QueryCachingPatcher))] [TestCase(DatabaseType.MySql, typeof(QueryCachingPatcher))] [TestCase(DatabaseType.Oracle, typeof(QueryCachingPatcher))] @@ -85,7 +87,7 @@ public void Test_SingleServer_WithOneParameter(DatabaseType dbType, bool usePara var root = cic.RootCohortAggregateContainer; root.AddChild(ac1, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator,null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -143,7 +145,7 @@ public void Test_SingleServer_WithTwoParameters(DatabaseType dbType, bool useCac root.AddChild(ac1, 0); root.AddChild(ac2, 1); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -220,7 +222,7 @@ public void Test_SingleServerPatientIndexTable_WithTwoParameters(DatabaseType db var root = cic.RootCohortAggregateContainer; root.AddChild(ac, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -296,7 +298,7 @@ public void Test_EXCEPT_TwoAggregates(DatabaseType dbType) root.AddChild(ac1, 0); root.AddChild(ac2, 1); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -367,7 +369,7 @@ public void Join_PatientIndexTable_OptionalCacheOnSameServer(DatabaseType dbType var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -427,7 +429,7 @@ public void Join_PatientIndexTable_DoNotUseCacheOnDifferentServer(DatabaseType d var hospitalAdmissions = SetupPatientIndexTableUser(db, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -479,7 +481,7 @@ public void Join_PatientIndexTable_NotOnCacheServer() var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -535,7 +537,7 @@ public void Join_PatientIndexTable_ThreeServers() var hospitalAdmissions = SetupPatientIndexTableUser(server2, people, r, cic, joinable); cic.RootCohortAggregateContainer.AddChild(hospitalAdmissions, 0); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); @@ -602,7 +604,7 @@ public void Join_PatientIndexTable_ThenShipToCacheForSets(DatabaseType dbType) root.Operation = SetOperation.EXCEPT; root.SaveToDatabase(); - var compiler = new CohortCompiler(cic); + var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null),cic); var runner = new CohortCompilerRunner(compiler, 50000); runner.Run(new CancellationToken()); diff --git a/Rdmp.Core/Caching/Pipeline/Destinations/CacheFilesystemDestination.cs b/Rdmp.Core/Caching/Pipeline/Destinations/CacheFilesystemDestination.cs index ab68e4811a..bf19717c8a 100644 --- a/Rdmp.Core/Caching/Pipeline/Destinations/CacheFilesystemDestination.cs +++ b/Rdmp.Core/Caching/Pipeline/Destinations/CacheFilesystemDestination.cs @@ -4,15 +4,16 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.IO; using Rdmp.Core.Caching.Layouts; using Rdmp.Core.Caching.Requests; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.IO; namespace Rdmp.Core.Caching.Pipeline.Destinations; @@ -41,7 +42,7 @@ public abstract class CacheFilesystemDestination : ICacheFileSystemDestination, public abstract ICacheChunk ProcessPipelineData(ICacheChunk toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken); - public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ILoadDirectory value, IDataLoadEventListener listener) { // CacheDirectory overrides LoadDirectory, so only set CacheDirectory if it is null (i.e. no alternative cache location has been configured in the destination component) CacheDirectory ??= value.Cache ?? diff --git a/Rdmp.Core/Caching/Pipeline/Sources/CacheSource.cs b/Rdmp.Core/Caching/Pipeline/Sources/CacheSource.cs index 484bec2f27..dfe1b23e23 100644 --- a/Rdmp.Core/Caching/Pipeline/Sources/CacheSource.cs +++ b/Rdmp.Core/Caching/Pipeline/Sources/CacheSource.cs @@ -4,15 +4,16 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; using Rdmp.Core.Caching.Requests; using Rdmp.Core.Caching.Requests.FetchRequestProvider; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; namespace Rdmp.Core.Caching.Pipeline.Sources; @@ -86,12 +87,12 @@ public virtual T GetChunk(IDataLoadEventListener listener, GracefulCancellationT public abstract T DoGetChunk(ICacheFetchRequest request, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken); - public void PreInitialize(ICacheFetchRequestProvider value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ICacheFetchRequestProvider value, IDataLoadEventListener listener) { RequestProvider = value; } - public void PreInitialize(IPermissionWindow value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IPermissionWindow value, IDataLoadEventListener listener) { PermissionWindow = value; } @@ -101,7 +102,7 @@ public void PreInitialize(IPermissionWindow value, IDataLoadEventListener listen public abstract T TryGetPreview(); public abstract void Check(ICheckNotifier notifier); - public void PreInitialize(ICatalogueRepository value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ICatalogueRepository value, IDataLoadEventListener listener) { CatalogueRepository = value; } diff --git a/Rdmp.Core/CohortCommitting/Pipeline/CreateTableFromAggregateUseCase.cs b/Rdmp.Core/CohortCommitting/Pipeline/CreateTableFromAggregateUseCase.cs index dd596cf24f..b7bfe1f4be 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/CreateTableFromAggregateUseCase.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/CreateTableFromAggregateUseCase.cs @@ -38,7 +38,7 @@ public CreateTableFromAggregateUseCase(AggregateConfiguration aggregateConfigura if (constrainByCohort == null) { var src = new AggregateConfigurationTableSource(); - src.PreInitialize(aggregateConfiguration, ThrowImmediatelyDataLoadEventListener.Quiet); + src.PreInitialize(null,aggregateConfiguration, ThrowImmediatelyDataLoadEventListener.Quiet); src.TableName = table.GetRuntimeName(); ExplicitSource = src; } @@ -47,8 +47,8 @@ public CreateTableFromAggregateUseCase(AggregateConfiguration aggregateConfigura AddInitializationObject(constrainByCohort); var src = new PatientIndexTableSource(); - src.PreInitialize(aggregateConfiguration, ThrowImmediatelyDataLoadEventListener.Quiet); - src.PreInitialize(constrainByCohort, ThrowImmediatelyDataLoadEventListener.Quiet); + src.PreInitialize(null,aggregateConfiguration, ThrowImmediatelyDataLoadEventListener.Quiet); + src.PreInitialize(null,constrainByCohort, ThrowImmediatelyDataLoadEventListener.Quiet); src.TableName = table.GetRuntimeName(); ExplicitSource = src; } diff --git a/Rdmp.Core/CohortCommitting/Pipeline/Destinations/BasicCohortDestination.cs b/Rdmp.Core/CohortCommitting/Pipeline/Destinations/BasicCohortDestination.cs index 26d9fae175..305c558a53 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/Destinations/BasicCohortDestination.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/Destinations/BasicCohortDestination.cs @@ -9,6 +9,7 @@ using System.Data; using System.Linq; using Rdmp.Core.CohortCommitting.Pipeline.Destinations.IdentifierAllocation; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.Repositories.Construction; @@ -219,9 +220,10 @@ public virtual void Abort(IDataLoadEventListener listener) /// /// Initialises /// + /// /// /// - public virtual void PreInitialize(ICohortCreationRequest value, IDataLoadEventListener listener) + public virtual void PreInitialize(IBasicActivateItems activator, ICohortCreationRequest value, IDataLoadEventListener listener) { Request = value; diff --git a/Rdmp.Core/CohortCommitting/Pipeline/Sources/AggregateConfigurationTableSource.cs b/Rdmp.Core/CohortCommitting/Pipeline/Sources/AggregateConfigurationTableSource.cs index 48c1191bf6..ae9c25e1ca 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/Sources/AggregateConfigurationTableSource.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/Sources/AggregateConfigurationTableSource.cs @@ -6,6 +6,7 @@ using System; using System.Data; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; @@ -129,7 +130,7 @@ public virtual void Check(ICheckNotifier notifier) } } - public virtual void PreInitialize(AggregateConfiguration value, IDataLoadEventListener listener) + public virtual void PreInitialize(IBasicActivateItems activator, AggregateConfiguration value, IDataLoadEventListener listener) { AggregateConfiguration = value; diff --git a/Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs b/Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs index 7f5485a69a..dfe5bf8895 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/Sources/CohortIdentificationConfigurationSource.cs @@ -10,6 +10,7 @@ using System.Threading; using Rdmp.Core.CohortCreation; using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataFlowPipeline; @@ -41,6 +42,8 @@ public class CohortIdentificationConfigurationSource : IPluginDataFlowSource /// If you are refreshing a cohort or running a cic which was run and cached a long time ago you might want to clear out the cache. This will mean that /// when run you will get a view of the live tables (which might be recached as part of building the cic) rather than the (potentially stale) current cache @@ -86,7 +89,7 @@ private DataTable GetDataTable(IDataLoadEventListener listener) listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, $"CohortIdentificationConfiguration '{_cohortIdentificationConfiguration}' has no RootCohortAggregateContainer_ID, is it empty?")); - var cohortCompiler = new CohortCompiler(_cohortIdentificationConfiguration); + var cohortCompiler = new CohortCompiler(_activator,_cohortIdentificationConfiguration); var rootContainerTask = //no caching set up so no point in running CohortCompilerRunner @@ -234,8 +237,9 @@ public void Check(ICheckNotifier notifier) } - public void PreInitialize(CohortIdentificationConfiguration value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, CohortIdentificationConfiguration value, IDataLoadEventListener listener) { + _activator = activator; _cohortIdentificationConfiguration = value; } } \ No newline at end of file diff --git a/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs b/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs index d8a814d305..61cc1b3e32 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIdentifierColumnSource.cs @@ -4,8 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; @@ -13,6 +12,8 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; namespace Rdmp.Core.CohortCommitting.Pipeline.Sources; @@ -101,7 +102,7 @@ public void Check(ICheckNotifier notifier) } } - public void PreInitialize(ExtractionInformation value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ExtractionInformation value, IDataLoadEventListener listener) { _extractionInformation = value; } diff --git a/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIndexTableSource.cs b/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIndexTableSource.cs index d2332d10f0..1238f6e3ba 100644 --- a/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIndexTableSource.cs +++ b/Rdmp.Core/CohortCommitting/Pipeline/Sources/PatientIndexTableSource.cs @@ -4,13 +4,14 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.QueryBuilding; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Linq; namespace Rdmp.Core.CohortCommitting.Pipeline.Sources; @@ -61,14 +62,14 @@ protected override string GetSQL() $"{sql[..insertionPoint]}{Environment.NewLine}{impromptuSql}{Environment.NewLine}{sql[insertionPoint..]}"; } - public void PreInitialize(ExtractableCohort value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ExtractableCohort value, IDataLoadEventListener listener) { _extractableCohort = value; } - public override void PreInitialize(AggregateConfiguration value, IDataLoadEventListener listener) + public override void PreInitialize(IBasicActivateItems activator, AggregateConfiguration value, IDataLoadEventListener listener) { - base.PreInitialize(value, listener); + base.PreInitialize(activator,value, listener); if (CohortIdentificationConfigurationIfAny == null) listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, diff --git a/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs b/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs index ebcef5d8a7..990dca720d 100644 --- a/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs +++ b/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using MongoDB.Driver; using Rdmp.Core.CohortCreation.Execution; using Rdmp.Core.CohortCreation.Execution.Joinables; using Rdmp.Core.CommandExecution; @@ -33,22 +34,23 @@ public class CohortIdentificationConfigurationUICommon private CancellationTokenSource _cancelGlobalOperations; private ISqlParameter[] _globals; public CohortCompilerRunner Runner; - + private IBasicActivateItems _activator; /// /// User interface layer for modal dialogs, showing Exceptions etc /// - public IBasicActivateItems Activator; + public IBasicActivateItems Activator { get => _activator; set { _activator = value; Compiler = new CohortCompiler(value, null); } } /// /// Duration in seconds to allow tasks to run for before cancelling /// public int Timeout = 3000; - public CohortCompiler Compiler { get; } + public CohortCompiler Compiler { get; private set; } - public CohortIdentificationConfigurationUICommon() + public CohortIdentificationConfigurationUICommon(IBasicActivateItems activator) { - Compiler = new CohortCompiler(null); + Activator = activator; + Compiler = new CohortCompiler(Activator, null); } public object Working_AspectGetter(object rowobject) => GetKey(rowobject)?.State; diff --git a/Rdmp.Core/CohortCreation/Execution/CohortCompiler.cs b/Rdmp.Core/CohortCreation/Execution/CohortCompiler.cs index f78d2b065c..6c95e20e53 100644 --- a/Rdmp.Core/CohortCreation/Execution/CohortCompiler.cs +++ b/Rdmp.Core/CohortCreation/Execution/CohortCompiler.cs @@ -4,18 +4,14 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using FAnsi.Discovery; using Rdmp.Core.CohortCreation.Execution.Joinables; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.Cohort.Joinables; +using Rdmp.Core.DataExport.Data; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.Providers; using Rdmp.Core.QueryBuilding; @@ -23,6 +19,12 @@ using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace Rdmp.Core.CohortCreation.Execution; @@ -74,9 +76,11 @@ public ICoreChildProvider CoreChildProvider public List Threads = new(); private ICoreChildProvider _coreChildProvider; + private IBasicActivateItems _activator; - public CohortCompiler(CohortIdentificationConfiguration cohortIdentificationConfiguration) + public CohortCompiler(IBasicActivateItems activator, CohortIdentificationConfiguration cohortIdentificationConfiguration) { + _activator = activator; CohortIdentificationConfiguration = cohortIdentificationConfiguration; } @@ -320,15 +324,29 @@ public ICompileable AddTask(IMapsDirectlyToDatabaseTable runnable, ISqlParameter task.State = CompilationState.Crashed; } - if(task.Child is CohortAggregateContainer cac) + if (task.Child is CohortAggregateContainer cac) { foreach (var cacac in cac.GetAggregateConfigurations()) { - if (cacac.Catalogue != null && cacac.Catalogue.IsInternalDataset) + if (cacac.Catalogue != null) { - task.CrashMessage = new ArgumentException($"Catalogue {cacac.Catalogue.Name} is marked as Internal. Internal Catalogues cannot be used in Cohort Identification Configurations."); - task.State = CompilationState.Crashed; - break; + if (cacac.Catalogue.IsInternalDataset) + { + task.CrashMessage = new ArgumentException($"Catalogue {cacac.Catalogue.Name} is marked as Internal. Internal Catalogues cannot be used in Cohort Identification Configurations."); + task.State = CompilationState.Crashed; + break; + } + if (_activator is not null && cacac.Catalogue.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)) + { + var cicAssociatedProjects = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("CohortIdentificationConfiguration_ID", CohortIdentificationConfiguration.ID).Select(c => c.Project).ToList(); + var catalogueAssociatedProjects = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", cacac.Catalogue.ID).SelectMany(eds => eds.Projects); + if (!catalogueAssociatedProjects.Intersect(cicAssociatedProjects).Any()) + { + task.CrashMessage = new ArgumentException($"Catalogue {cacac.Catalogue.Name} is marked as Project Specific, but this Cohort Identification Configurations is not associated with the same Project."); + task.State = CompilationState.Crashed; + break; + } + } } } } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteDatasetExtractionFlatFileDestination.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteDatasetExtractionFlatFileDestination.cs index 8c69badece..56f963fd7a 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteDatasetExtractionFlatFileDestination.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteDatasetExtractionFlatFileDestination.cs @@ -4,11 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Diagnostics; -using System.IO; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -20,6 +16,11 @@ using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -49,7 +50,7 @@ public ExecuteDatasetExtractionFlatFileDestination() : base(true) { } - protected override void PreInitializeImpl(IExtractCommand request, IDataLoadEventListener listener) + protected override void PreInitializeImpl(IBasicActivateItems activator, IExtractCommand request, IDataLoadEventListener listener) { if (_request is ExtractGlobalsCommand) { diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index c1a36c561f..9ea6416957 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using FAnsi.Discovery; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -122,6 +123,7 @@ You must have either $a or $d private bool _tableDidNotExistAtStartOfLoad; private bool _isTableAlreadyNamed; private DataTable _toProcess; + private IBasicActivateItems _activator; public ExecuteFullExtractionToDatabaseMSSql() : base(false) { @@ -287,7 +289,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis _destination.IndexTableName = GetIndexName(); if (UserDefinedIndex is not null) _destination.UserDefinedIndexes = UserDefinedIndex.Split(',').Select(i => i.Trim()).ToList(); - _destination.PreInitialize(_destinationDatabase, listener); + _destination.PreInitialize(_activator,_destinationDatabase, listener); return _destination; @@ -528,8 +530,9 @@ public override void Abort(IDataLoadEventListener listener) _destination?.Abort(listener); } - protected override void PreInitializeImpl(IExtractCommand value, IDataLoadEventListener listener) + protected override void PreInitializeImpl(IBasicActivateItems activator, IExtractCommand value, IDataLoadEventListener listener) { + _activator = activator; } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs index 04d848ccc3..90cd8cd37b 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs @@ -4,12 +4,9 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Diagnostics; -using System.IO; using FAnsi.Discovery; using Microsoft.Data.SqlClient; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -25,6 +22,10 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.DataAccess; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Diagnostics; +using System.IO; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -85,7 +86,7 @@ public ExtractionDestination(bool generatesFiles) #region PreInitialize - public void PreInitialize(IExtractCommand request, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IExtractCommand request, IDataLoadEventListener listener) { _request = request; @@ -101,18 +102,18 @@ public void PreInitialize(IExtractCommand request, IDataLoadEventListener listen DirectoryPopulated = request.GetExtractionDirectory(); - PreInitializeImpl(request, listener); + PreInitializeImpl(activator,request, listener); } - protected abstract void PreInitializeImpl(IExtractCommand request, IDataLoadEventListener listener); + protected abstract void PreInitializeImpl(IBasicActivateItems activator, IExtractCommand request, IDataLoadEventListener listener); - public virtual void PreInitialize(DataLoadInfo value, IDataLoadEventListener listener) + public virtual void PreInitialize(IBasicActivateItems activator, DataLoadInfo value, IDataLoadEventListener listener) { _dataLoadInfo = value; } - public virtual void PreInitialize(IProject value, IDataLoadEventListener listener) + public virtual void PreInitialize(IBasicActivateItems activator, IProject value, IDataLoadEventListener listener) { _project = value; } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/FileExtractor.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/FileExtractor.cs index 2fb695d071..a3711f356f 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/FileExtractor.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/FileExtractor.cs @@ -4,13 +4,14 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataExport.DataExtraction.Commands; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline; @@ -35,7 +36,7 @@ public virtual void Dispose(IDataLoadEventListener listener, Exception pipelineF { } - public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IExtractCommand value, IDataLoadEventListener listener) { // We only want to extract the files once so let's do it as part of extracting globals _command = value as ExtractGlobalsCommand; diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteCrossServerDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteCrossServerDatasetExtractionSource.cs index f6afbc17c4..44ba4b892e 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteCrossServerDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteCrossServerDatasetExtractionSource.cs @@ -292,7 +292,7 @@ private void CopyCohortToDataServer(IDataLoadEventListener listener, GracefulCan } var destination = new DataTableUploadDestination(); - destination.PreInitialize(_tempDb, listener); + destination.PreInitialize(null,_tempDb, listener); destination.ProcessPipelineData(cohortDataTable, listener, cancellationToken); destination.Dispose(listener, null); diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index b99b44803d..18247b0c9e 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -6,6 +6,7 @@ using FAnsi; using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -691,7 +692,7 @@ public virtual DataTable TryGetPreview() return toReturn; } - public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IExtractCommand value, IDataLoadEventListener listener) { if (value is ExtractDatasetCommand datasetCommand) Initialize(datasetCommand); diff --git a/Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs b/Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs index 89ceb6c04b..696ba7fb84 100644 --- a/Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs +++ b/Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs @@ -216,12 +216,12 @@ public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureEx } } - public void PreInitialize(Project value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, Project value, IDataLoadEventListener listener) { _project = value; } - public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ReleaseData value, IDataLoadEventListener listener) { _releaseData = value; } diff --git a/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs b/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs index f84fafcedb..1cd6bfecb8 100644 --- a/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs +++ b/Rdmp.Core/DataExport/DataRelease/Pipeline/BasicDataReleaseDestination.cs @@ -4,10 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction; @@ -16,6 +13,10 @@ using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; namespace Rdmp.Core.DataExport.DataRelease.Pipeline; @@ -132,12 +133,12 @@ public void Check(ICheckNotifier notifier) ((ICheckable)ReleaseSettings).Check(notifier); } - public void PreInitialize(Project value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, Project value, IDataLoadEventListener listener) { _project = value; } - public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ReleaseData value, IDataLoadEventListener listener) { _releaseData = value; } diff --git a/Rdmp.Core/DataExport/DataRelease/Pipeline/FixedReleaseSource.cs b/Rdmp.Core/DataExport/DataRelease/Pipeline/FixedReleaseSource.cs index 2f36e451c0..f97abb0932 100644 --- a/Rdmp.Core/DataExport/DataRelease/Pipeline/FixedReleaseSource.cs +++ b/Rdmp.Core/DataExport/DataRelease/Pipeline/FixedReleaseSource.cs @@ -4,12 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction; using Rdmp.Core.DataExport.DataRelease.Potential; @@ -20,6 +15,12 @@ using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; using Rdmp.Core.Ticketing; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; namespace Rdmp.Core.DataExport.DataRelease.Pipeline; @@ -60,7 +61,7 @@ public T GetChunk(IDataLoadEventListener listener, GracefulCancellationToken can public T TryGetPreview() => null; - public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ReleaseData value, IDataLoadEventListener listener) { _releaseData = value; } diff --git a/Rdmp.Core/DataExport/DataRelease/Pipeline/ReleaseFolderProvider.cs b/Rdmp.Core/DataExport/DataRelease/Pipeline/ReleaseFolderProvider.cs index fb7b8bed53..99be34943d 100644 --- a/Rdmp.Core/DataExport/DataRelease/Pipeline/ReleaseFolderProvider.cs +++ b/Rdmp.Core/DataExport/DataRelease/Pipeline/ReleaseFolderProvider.cs @@ -4,15 +4,16 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.IO; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.IO; +using System.Linq; namespace Rdmp.Core.DataExport.DataRelease.Pipeline; @@ -57,12 +58,12 @@ public void Check(ICheckNotifier notifier) PrepareAndCheckReleaseFolder(notifier); } - public void PreInitialize(Project value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, Project value, IDataLoadEventListener listener) { _project = value; } - public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ReleaseData value, IDataLoadEventListener listener) { _releaseData = value; } diff --git a/Rdmp.Core/DataFlowPipeline/Requirements/DataFlowPipelineContext.cs b/Rdmp.Core/DataFlowPipeline/Requirements/DataFlowPipelineContext.cs index 156299bb57..0277e99561 100644 --- a/Rdmp.Core/DataFlowPipeline/Requirements/DataFlowPipelineContext.cs +++ b/Rdmp.Core/DataFlowPipeline/Requirements/DataFlowPipelineContext.cs @@ -300,7 +300,7 @@ private static Type PreInitializeComponentWithSingleObject(IDataLoadEventListene //invoke it try { - preInit.Invoke(component, new[] { value, listener }); + preInit.Invoke(component, new[] { null,value, listener }); } catch (Exception e) { diff --git a/Rdmp.Core/DataFlowPipeline/Requirements/IPipelineRequirement.cs b/Rdmp.Core/DataFlowPipeline/Requirements/IPipelineRequirement.cs index d31114d016..96c30a30eb 100644 --- a/Rdmp.Core/DataFlowPipeline/Requirements/IPipelineRequirement.cs +++ b/Rdmp.Core/DataFlowPipeline/Requirements/IPipelineRequirement.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using Rdmp.Core.CommandExecution; using Rdmp.Core.ReusableLibraryCode.Progress; namespace Rdmp.Core.DataFlowPipeline.Requirements; @@ -28,7 +29,8 @@ public interface IPipelineRequirement /// /// IMPORTANT: You might be being checked and not actually run so when implementing this method you should not make any system changes or advanced auditing stuff. /// + /// /// An object /// - public void PreInitialize(T value, IDataLoadEventListener listener); + public void PreInitialize(IBasicActivateItems activator, T value, IDataLoadEventListener listener); } \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/BasicAnonymisationEngine.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/BasicAnonymisationEngine.cs index 43655cf9ef..704bd5dcf1 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/BasicAnonymisationEngine.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Components/Anonymisation/BasicAnonymisationEngine.cs @@ -4,17 +4,18 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; namespace Rdmp.Core.DataLoad.Engine.Pipeline.Components.Anonymisation; @@ -32,7 +33,7 @@ public class BasicAnonymisationEngine : IPluginDataFlowComponent, IPi public TableInfo TableToLoad { get; set; } - public void PreInitialize(TableInfo target, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, TableInfo target, IDataLoadEventListener listener) { TableToLoad = target; _bInitialized = true; diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs index c1b8fec7e3..c6d369c6ff 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Components/CleanStrings.cs @@ -4,16 +4,17 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Linq; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; namespace Rdmp.Core.DataLoad.Engine.Pipeline.Components; @@ -99,7 +100,7 @@ public void Abort(IDataLoadEventListener listener) { } - public void PreInitialize(TableInfo target, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, TableInfo target, IDataLoadEventListener listener) { if (target == null) throw new Exception("Without TableInfo we cannot figure out what columns to clean"); diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs index 9ecaca4fd1..51b0535682 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs @@ -9,6 +9,7 @@ using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; using FAnsi.Discovery.TableCreation; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; @@ -704,7 +705,7 @@ private void StartAuditIfExists(string tableName) } } - public void PreInitialize(DiscoveredDatabase value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, DiscoveredDatabase value, IDataLoadEventListener listener) { _database = value; _server = value.Server; diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs index 4fed6bc89f..6ebb859514 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/SqlBulkInsertDestination.cs @@ -4,17 +4,18 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Linq; using FAnsi.Discovery; +using Rdmp.Core.CommandExecution; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.Logging; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; namespace Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; @@ -187,7 +188,7 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener return null; } - public void PreInitialize(ITableLoadInfo value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ITableLoadInfo value, IDataLoadEventListener listener) { TableLoadInfo = value; } diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/DelimitedFlatFileAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/DelimitedFlatFileAttacher.cs index aad6f3cf60..9bd8e21a40 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/DelimitedFlatFileAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/DelimitedFlatFileAttacher.cs @@ -156,7 +156,7 @@ protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener lis Source.StronglyTypeInput = false; Source.StronglyTypeInputBatchSize = 0; _listener = listener; - Source.PreInitialize(new FlatFileToLoad(fileToLoad), listener); + Source.PreInitialize(null,new FlatFileToLoad(fileToLoad), listener); _currentFile = fileToLoad; } diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs index f251bc5c23..c980cc987d 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs @@ -78,7 +78,7 @@ protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener lis AddFilenameColumnNamed = AddFilenameColumnNamed }; - _hostedSource.PreInitialize(new FlatFileToLoad(fileToLoad), listener); + _hostedSource.PreInitialize(null,new FlatFileToLoad(fileToLoad), listener); listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to start processing {fileToLoad.FullName}")); diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs index 58ad558d25..e141c48fbe 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/CohortSampler.cs @@ -4,17 +4,18 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; using Rdmp.Core.CohortCommitting.Pipeline; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations; @@ -53,7 +54,7 @@ public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureEx { } - public void PreInitialize(CohortCreationRequest value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, CohortCreationRequest value, IDataLoadEventListener listener) { _ect = value.NewCohortDefinition.LocationOfCohort; _project = value.Project; diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ExtractCatalogueMetadata.cs b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ExtractCatalogueMetadata.cs index dba13162ca..9ef4209664 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ExtractCatalogueMetadata.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ExtractCatalogueMetadata.cs @@ -108,12 +108,12 @@ public void Check(ICheckNotifier notifier) CheckResult.Fail)); } - public void PreInitialize(IExtractCommand value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IExtractCommand value, IDataLoadEventListener listener) { _request = value; } - public void PreInitialize(IBasicActivateItems value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IBasicActivateItems value, IDataLoadEventListener listener) { _activator = value; } diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ReleaseMetadata.cs b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ReleaseMetadata.cs index f44bac0302..1a3dba6cf2 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ReleaseMetadata.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/ReleaseMetadata.cs @@ -70,7 +70,7 @@ public void Abort(IDataLoadEventListener listener) { } - public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, ReleaseData value, IDataLoadEventListener listener) { _releaseData = value; } @@ -80,7 +80,7 @@ public void Check(ICheckNotifier notifier) notifier.OnCheckPerformed(new CheckEventArgs("No checking needed", CheckResult.Success)); } - public void PreInitialize(IBasicActivateItems value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, IBasicActivateItems value, IDataLoadEventListener listener) { _activator = value; } diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/TableVersionNamer.cs b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/TableVersionNamer.cs index dc3e6df59d..b1b9792abf 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowOperations/TableVersionNamer.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowOperations/TableVersionNamer.cs @@ -4,15 +4,16 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; -using System.Data; -using System.Linq; using FAnsi.Discovery; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Linq; namespace Rdmp.Core.DataLoad.Modules.DataFlowOperations; @@ -66,7 +67,7 @@ public void Check(ICheckNotifier notifier) new CheckEventArgs("MaximumNumberOfVersionsAllowed cannot be 0", CheckResult.Fail)); } - public void PreInitialize(DiscoveredDatabase value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, DiscoveredDatabase value, IDataLoadEventListener listener) { _tableNamesAtDestination = value.DiscoverTables(true).Select(t => t.GetRuntimeName()).ToArray(); } diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowSources/DelimitedFlatFileDataFlowSource.cs b/Rdmp.Core/DataLoad/Modules/DataFlowSources/DelimitedFlatFileDataFlowSource.cs index 55502ac647..d38b4862e6 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowSources/DelimitedFlatFileDataFlowSource.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowSources/DelimitedFlatFileDataFlowSource.cs @@ -12,6 +12,7 @@ using CsvHelper; using CsvHelper.Configuration; using FAnsi.Discovery; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; @@ -494,7 +495,7 @@ protected int IterativelyBatchLoadDataIntoDataTable(DataTable dt, int batchSize) } - public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, FlatFileToLoad value, IDataLoadEventListener listener) { //we have been given a new file we no longer know the headers. Headers = null; diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs b/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs index bb30fa2bc8..068fa36e4c 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs @@ -18,6 +18,7 @@ using NPOI.HSSF.UserModel; using NPOI.SS.UserModel; using NPOI.XSSF.UserModel; +using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; @@ -409,7 +410,7 @@ public DataTable TryGetPreview() return dt; } - public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener) + public void PreInitialize(IBasicActivateItems activator, FlatFileToLoad value, IDataLoadEventListener listener) { _fileToLoad = value; } diff --git a/Rdmp.Core/DataLoad/Modules/DataProvider/FlatFileManipulation/ExcelToCSVFilesConverter.cs b/Rdmp.Core/DataLoad/Modules/DataProvider/FlatFileManipulation/ExcelToCSVFilesConverter.cs index 1b8e0a5494..b0d6cd0780 100644 --- a/Rdmp.Core/DataLoad/Modules/DataProvider/FlatFileManipulation/ExcelToCSVFilesConverter.cs +++ b/Rdmp.Core/DataLoad/Modules/DataProvider/FlatFileManipulation/ExcelToCSVFilesConverter.cs @@ -88,7 +88,7 @@ private void ProcessFile(FileInfo fileInfo, IDataLoadJob job) try { var source = new ExcelDataFlowSource(); - source.PreInitialize(new FlatFileToLoad(fileInfo), job); + source.PreInitialize(null,new FlatFileToLoad(fileInfo), job); for (var i = 0; i < wb.NumberOfSheets; i++) { diff --git a/Rdmp.UI/NavigationTrack.cs b/Rdmp.UI/NavigationTrack.cs index 622d0099cc..9437fb6c2a 100644 --- a/Rdmp.UI/NavigationTrack.cs +++ b/Rdmp.UI/NavigationTrack.cs @@ -64,7 +64,7 @@ public NavigationTrack(Func aliveDelegate, Action activate) public void Prune() { _navigationStack = new Stack(_navigationStack.ToArray().Take(MaxHistory + 1).Reverse().Where(_isAlive)); - _forward = new Stack(_forward.AsEnumerable().ToArray().AsEnumerable().Reverse().Where(_isAlive)); + _forward = new Stack(_forward.ToArray().AsEnumerable().Reverse().Where(_isAlive)); } /// diff --git a/Rdmp.UI/SimpleDialogs/SimpleFileImporting/CreateNewCatalogueByImportingFileUI.cs b/Rdmp.UI/SimpleDialogs/SimpleFileImporting/CreateNewCatalogueByImportingFileUI.cs index a924e63db2..7d25ff141c 100644 --- a/Rdmp.UI/SimpleDialogs/SimpleFileImporting/CreateNewCatalogueByImportingFileUI.cs +++ b/Rdmp.UI/SimpleDialogs/SimpleFileImporting/CreateNewCatalogueByImportingFileUI.cs @@ -287,7 +287,7 @@ private void ddPipeline_SelectedIndexChanged(object sender, EventArgs e) try { var source = DataFlowPipelineEngineFactory.CreateSourceIfExists(p); - ((IPipelineRequirement)source).PreInitialize(new FlatFileToLoad(_selectedFile), + ((IPipelineRequirement)source).PreInitialize(null,new FlatFileToLoad(_selectedFile), new FromCheckNotifierToDataLoadEventListener(ragSmileyFile)); ((ICheckable)source).Check(ragSmileyFile); } @@ -331,7 +331,7 @@ private void btnPreview_Click(object sender, EventArgs e) var source = (IDataFlowSource)DataFlowPipelineEngineFactory.CreateSourceIfExists(p); - ((IPipelineRequirement)source).PreInitialize(new FlatFileToLoad(_selectedFile), + ((IPipelineRequirement)source).PreInitialize(null,new FlatFileToLoad(_selectedFile), new FromCheckNotifierToDataLoadEventListener(ragSmileyFile)); Cursor.Current = Cursors.WaitCursor; diff --git a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs index dc12479c10..a602ba4b1a 100644 --- a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs @@ -73,13 +73,13 @@ public partial class CohortIdentificationConfigurationUI : CohortIdentificationC private ExecuteCommandClearQueryCache _clearCacheCommand; - private CohortIdentificationConfigurationUICommon Common = new(); + private CohortIdentificationConfigurationUICommon Common = new(null); public CohortIdentificationConfigurationUI() { InitializeComponent(); - Common = new CohortIdentificationConfigurationUICommon(); + Common = new CohortIdentificationConfigurationUICommon(Activator); olvExecute.IsButton = true; olvExecute.ButtonSizing = OLVColumn.ButtonSizingMode.CellBounds; @@ -169,6 +169,7 @@ public override void SetDatabaseObject(IActivateItems activator, CohortIdentific base.SetDatabaseObject(activator, databaseObject); version.Setup(databaseObject, activator); Common.Configuration = databaseObject; + Common.Activator = activator; Common.Compiler.CohortIdentificationConfiguration = databaseObject; RebuildClearCacheCommand(); diff --git a/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs b/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs index 5d47ac564f..7466fbef8e 100644 --- a/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs +++ b/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs @@ -263,7 +263,7 @@ private void cbxCohort_SelectionChangeCommitted(object sender, EventArgs e) { Timeout = 5 }; - source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet); + source.PreInitialize(null,cic, ThrowImmediatelyDataLoadEventListener.Quiet); source.Check(ragCic); ClearFile(); diff --git a/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs b/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs index 1361e18d0b..a1928fb310 100644 --- a/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs +++ b/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs @@ -23,7 +23,7 @@ namespace Rdmp.Core.CommandLine.Gui; public partial class ConsoleGuiCohortIdentificationConfigurationUI { private readonly IBasicActivateItems _activator; - private CohortIdentificationConfigurationUICommon Common = new (); + private CohortIdentificationConfigurationUICommon Common = new (null); private bool _isDisposed; private List RowObjects = new(); private bool _contextMenuShowing = false;