Task/rdmp 360 catalogue status change behaviour (#2319)

* add drag and drop fix for internal catalogues

* prevent commit of internal catalogues

* add changelog

* fix typo

* add internal cohort test

* logic

* check for project specific catalogues in cohort builder

* fix build

* fix preinit invoker

* fix project specific check

* update test

* fix build

* fix test

Author: JFriel

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add Internal Note to Catalogue
 - Fix issue where project associations were not copied when a CIC was cloned
 - Fix issue with using Internal Catalogues in Cohort Identification Configurations
+- Fix issue with using Project Specific Catalogues in Cohort Identification Configurations when not associated to project
+- Fix bug with copying project specific Catalogues between Projects when committing a Cohort
+- Remove Tree filters from UI
+
 - Simplify Project Cohorts tree
 - Fix bug with copying project specific Catalogues between Projects when committing a Cohort
 - Remove Tree filters from UI
@@ -18,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Simplify use of Extraction Progress
 - Update CHI validation to allow mod10 CHIs
 - Hardened Confluence export functionality
+- Fix issue with using Internal Catalogues in Cohort Identification Configurations
 - Add new SQL merge extraction component
 
 ## [9.1.1] - 2025-12-02

Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs

@@ -4,18 +4,14 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 using NSubstitute;
 using NUnit.Framework;
 using Rdmp.Core.Caching;
 using Rdmp.Core.Caching.Layouts;
 using Rdmp.Core.Caching.Pipeline.Destinations;
 using Rdmp.Core.Caching.Pipeline.Sources;
 using Rdmp.Core.Caching.Requests;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Cache;
@@ -25,6 +21,11 @@
 using Rdmp.Core.Repositories;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
 using Tests.Common;
 
 namespace Rdmp.Core.Tests.Caching.Integration;
@@ -182,7 +183,7 @@ public void Check(ICheckNotifier notifier)
 
     private ILoadDirectory project;
 
-    public void PreInitialize(ILoadDirectory value, IDataLoadEventListener listener)
+    public void PreInitialize(IBasicActivateItems activator, ILoadDirectory value, IDataLoadEventListener listener)
     {
         project = value;
     }

Rdmp.Core.Tests/Caching/Unit/TestIFileDataFlowComponents.cs

@@ -9,6 +9,7 @@
 using System.IO;
 using System.Linq;
 using Rdmp.Core.Caching.Layouts;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation.Data.Cache;
 using Rdmp.Core.DataFlowPipeline.Requirements;
 using Rdmp.Core.ReusableLibraryCode.Progress;
@@ -83,12 +84,12 @@ public void Dispose(IDataLoadEventListener listener)
         throw new NotImplementedException();
     }
 
-    public void PreInitialize(CacheProgress cacheProgress, IDataLoadEventListener listener)
+    public void PreInitialize(IBasicActivateItems activator, CacheProgress cacheProgress, IDataLoadEventListener listener)
     {
         CacheProgress = cacheProgress;
     }
 
-    public void PreInitialize(DirectoryInfo cacheDirectory, IDataLoadEventListener listener)
+    public void PreInitialize(IBasicActivateItems activator, DirectoryInfo cacheDirectory, IDataLoadEventListener listener)
     {
         CacheDirectory = cacheDirectory;
     }

Rdmp.Core.Tests/CohortCommitting/CommitCohortExample.cs

@@ -4,18 +4,19 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
-using System.Data;
-using System.Linq;
 using FAnsi;
 using NUnit.Framework;
 using Rdmp.Core.CohortCommitting;
 using Rdmp.Core.CohortCommitting.Pipeline;
 using Rdmp.Core.CohortCommitting.Pipeline.Destinations;
 using Rdmp.Core.CohortCommitting.Pipeline.Destinations.IdentifierAllocation;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.DataExport.Data;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
+using System.Data;
+using System.Linq;
 using Tests.Common;
 
 namespace Rdmp.Core.Tests.CohortCommitting;
@@ -63,7 +64,7 @@ public void CommitCohortExampleTest(DatabaseType dbType, string privateDataType)
         };
 
         //initialize the destination
-        pipelineDestination.PreInitialize(
+        pipelineDestination.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null),
             new CohortCreationRequest(project, definition, DataExportRepository,
                 "A cohort created in an example unit test"),
             ThrowImmediatelyDataLoadEventListener.Quiet);

Rdmp.Core.Tests/CohortCommitting/CreateNewCohortDatabaseWizardTests.cs

@@ -4,22 +4,23 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
-using System;
-using System.Data;
-using System.Linq;
 using FAnsi;
 using NUnit.Framework;
 using Rdmp.Core.CohortCommitting;
 using Rdmp.Core.CohortCommitting.Pipeline;
 using Rdmp.Core.CohortCommitting.Pipeline.Destinations;
 using Rdmp.Core.CohortCommitting.Pipeline.Destinations.IdentifierAllocation;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.DataExport.Data;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.Providers;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
 using Rdmp.Core.ReusableLibraryCode.Settings;
+using System;
+using System.Data;
+using System.Linq;
 using Tests.Common;
 
 namespace Rdmp.Core.Tests.CohortCommitting;
@@ -181,7 +182,7 @@ public void TestActuallyCreatingIt(DatabaseType type)
         //the destination component that will put it there
         var dest = new BasicCohortDestination();
 
-        dest.PreInitialize(request, ThrowImmediatelyDataLoadEventListener.Quiet);
+        dest.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), request, ThrowImmediatelyDataLoadEventListener.Quiet);
 
         //tell it to use the guid allocator
         dest.ReleaseIdentifierAllocator = typeof(GuidReleaseIdentifierAllocator);

Rdmp.Core.Tests/CohortCreation/CohortCompilerRunnerTests.cs

@@ -11,6 +11,7 @@
 using FAnsi.Discovery;
 using NUnit.Framework;
 using Rdmp.Core.CohortCreation.Execution;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.Curation.Data.Cohort;
@@ -28,7 +29,7 @@ public void CacheIdentifierListWithRunner_SimpleCase()
     {
         SetupCohort(out var db, out var cic, out var dt);
 
-        var compiler = new CohortCompiler(cic);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator,null), cic);
 
         var runner = new CohortCompilerRunner(compiler, 5000);
         runner.Run(new CancellationToken());
@@ -62,7 +63,7 @@ public void CacheIdentifierListWithRunner_WithCaching()
         cic.QueryCachingServer_ID = serverReference.ID;
         cic.SaveToDatabase();
 
-        var compiler = new CohortCompiler(cic);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cic);
 
         var runner = new CohortCompilerRunner(compiler, 5000);
         runner.Run(new CancellationToken());

Rdmp.Core.Tests/CohortCreation/CohortCompilerTests.cs

@@ -11,9 +11,12 @@
 using Rdmp.Core.CohortCreation;
 using Rdmp.Core.CohortCreation.Execution;
 using Rdmp.Core.CohortCreation.Execution.Joinables;
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.CommandExecution.AtomicCommands;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.Curation.Data.Cohort.Joinables;
+using Rdmp.Core.DataExport.Data;
 
 namespace Rdmp.Core.Tests.CohortCreation;
 
@@ -22,7 +25,7 @@ public class CohortCompilerTests : CohortIdentificationTests
     [Test]
     public void AddSameTaskTwice_StaysAtOne()
     {
-        var compiler = new CohortCompiler(cohortIdentificationConfiguration);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator,null), cohortIdentificationConfiguration);
         container1.AddChild(aggregate1, 0);
         try
         {
@@ -66,7 +69,7 @@ public void AddSameTaskTwice_StaysAtOne()
     [Test]
     public void AddContainer_StaysAtOne()
     {
-        var compiler = new CohortCompiler(cohortIdentificationConfiguration);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration);
         rootcontainer.AddChild(aggregate1, 1);
 
         compiler.AddTask(rootcontainer, null); //add the root container
@@ -144,7 +147,7 @@ public void TestCompilerAddAllTasks(TestCompilerAddAllTasksTestCase testCase, bo
 
             //Joinable:aggregate5 (patient index table, the other Aggregates could JOIN to this)
 
-            var compiler = new CohortCompiler(cohortIdentificationConfiguration);
+            var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration);
             rootcontainer.AddChild(aggregate1, 1);
             rootcontainer.AddChild(container1);
             container1.Order = 2;
@@ -201,7 +204,7 @@ public void TestCompilerAddAllTasks(TestCompilerAddAllTasksTestCase testCase, bo
     [Test]
     public void TestCompilerWithInternalCatalogues()
     {
-        var compiler = new CohortCompiler(cohortIdentificationConfiguration);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration);
         testData.catalogue.IsInternalDataset = true;
         testData.catalogue.SaveToDatabase();
 
@@ -217,4 +220,57 @@ public void TestCompilerWithInternalCatalogues()
         Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(3));
         Assert.That(compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Any(m => m.Contains(" is marked as Internal. Internal Catalogues cannot be used in Cohort Identification Configurations.")));
     }
+
+
+    [Test]
+    public void TestCompilerWithProjectSpecificCatalogues_Fail()
+    {
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration);
+        
+        var proj = new Project(DataExportRepository, "TestCompilerWithProjectSpecificCatalogues_Fail");
+        proj.SaveToDatabase();
+
+        //having an issue making it project specific
+        var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(new ThrowImmediatelyActivator(RepositoryLocator), aggregate1.Catalogue, proj, true);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        rootcontainer.AddChild(aggregate1, 1);
+        rootcontainer.AddChild(container1);
+        container1.Order = 2;
+        container1.SaveToDatabase();
+
+        cohortIdentificationConfiguration.RootCohortAggregateContainer_ID = rootcontainer.ID;
+        cohortIdentificationConfiguration.SaveToDatabase();
+
+        compiler.AddAllTasks(true);
+        Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(3));
+        Assert.That(compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Contains("Catalogue BulkData is marked as Project Specific, but this Cohort Identification Configurations is not associated with the same Project."));
+    }
+    [Test]
+    public void TestCompilerWithProjectSpecificCatalogues_Pass()
+    {
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cohortIdentificationConfiguration);
+
+        var proj = new Project(DataExportRepository, "TestCompilerWithProjectSpecificCatalogues_Fail");
+        proj.SaveToDatabase();
+
+        //having an issue making it project specific
+        var cmd = new ExecuteCommandMakeCatalogueProjectSpecific(new ThrowImmediatelyActivator(RepositoryLocator), aggregate1.Catalogue, proj, true);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        rootcontainer.AddChild(aggregate1, 1);
+        rootcontainer.AddChild(container1);
+        container1.Order = 2;
+        container1.SaveToDatabase();
+
+        cohortIdentificationConfiguration.RootCohortAggregateContainer_ID = rootcontainer.ID;
+        cohortIdentificationConfiguration.SaveToDatabase();
+
+        var associateCmd = new ExecuteCommandAssociateCohortIdentificationConfigurationWithProject(new ThrowImmediatelyActivator(RepositoryLocator));
+        associateCmd.SetTarget(proj);
+        associateCmd.SetTarget(cohortIdentificationConfiguration);
+        Assert.DoesNotThrow(() => associateCmd.Execute());
+
+        compiler.AddAllTasks(true);
+        Assert.That(compiler.Tasks.Keys.Select(k => k.State).Where(s => s == CompilationState.Crashed).ToList(), Has.Count.EqualTo(2));
+        Assert.That(!compiler.Tasks.Keys.Where(s => s.State == CompilationState.Crashed).Select(k => k.CrashMessage.Message).ToList().Contains("Catalogue BulkData is marked as Project Specific, but this Cohort Identification Configurations is not associated with the same Project."));
+    }
 }

Rdmp.Core.Tests/CohortCreation/PluginCohortCompilerTests.cs

@@ -8,18 +8,19 @@
 using NUnit.Framework;
 using Rdmp.Core.CohortCommitting.Pipeline.Sources;
 using Rdmp.Core.CohortCreation.Execution;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.CommandExecution.AtomicCommands;
 using Rdmp.Core.CommandExecution.Combining;
 using Rdmp.Core.CommandLine.Interactive;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Cohort;
 using Rdmp.Core.Curation.Data.Cohort.Joinables;
 using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using Rdmp.Core.ReusableLibraryCode.Progress;
 using Rdmp.Core.Tests.CohortCreation.QueryTests;
 using System;
 using System.Data;
-using Rdmp.Core.ReusableLibraryCode.Checks;
-using Rdmp.Core.ReusableLibraryCode.Progress;
 
 namespace Rdmp.Core.Tests.CohortCreation;
 
@@ -53,7 +54,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly()
 
         // run the cic
         var source = new CohortIdentificationConfigurationSource();
-        source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet);
+        source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet);
         var dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken());
 
         // 5 random chi numbers
@@ -65,7 +66,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly()
 
         // run the cic again
         source = new CohortIdentificationConfigurationSource();
-        source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet);
+        source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet);
         dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken());
 
         // because the rules changed to generate 2 chis only there should be a new result
@@ -76,7 +77,7 @@ public void TestIPluginCohortCompiler_PopulatesCacheCorrectly()
         // run the cic again with no changes, the results should be unchanged since there is no config changed
         // I.e. no new chis should be generated and the cached values returned
         source = new CohortIdentificationConfigurationSource();
-        source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet);
+        source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet);
         dt = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken());
 
         Assert.That(dt.Rows, Has.Count.EqualTo(2));
@@ -223,7 +224,7 @@ public void TestIPluginCohortCompiler_AsPatientIndexTable()
 
         // run the cic again
         var source = new CohortIdentificationConfigurationSource();
-        source.PreInitialize(cic, ThrowImmediatelyDataLoadEventListener.Quiet);
+        source.PreInitialize(new ThrowImmediatelyActivator(RepositoryLocator, null), cic, ThrowImmediatelyDataLoadEventListener.Quiet);
         var result = source.GetChunk(ThrowImmediatelyDataLoadEventListener.Quiet, new GracefulCancellationToken());
         Assert.That(result.Rows, Has.Count.EqualTo(1));
     }

Rdmp.Core.Tests/CohortCreation/QueryTests/CohortCompilerCacheJoinableTest.cs

@@ -6,19 +6,20 @@
 
 using NUnit.Framework;
 using Rdmp.Core.CohortCreation.Execution;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.Curation.Data.Cohort;
 using Rdmp.Core.Curation.Data.Cohort.Joinables;
 using Rdmp.Core.Databases;
+using Rdmp.Core.MapsDirectlyToDatabaseTable.Versioning;
+using Rdmp.Core.QueryCaching.Aggregation;
+using Rdmp.Core.ReusableLibraryCode.Checks;
 using System;
 using System.Data;
 using System.Linq;
-using Rdmp.Core.MapsDirectlyToDatabaseTable.Versioning;
 using Tests.Common.Scenarios;
-using Rdmp.Core.QueryCaching.Aggregation;
-using Rdmp.Core.ReusableLibraryCode.Checks;
 using static Rdmp.Core.CohortCreation.Execution.CohortCompilerRunner;
 
 namespace Rdmp.Core.Tests.CohortCreation.QueryTests;
@@ -113,7 +114,7 @@ public void CohortIdentificationConfiguration_Join_PatientIndexTable()
             Assert.That(acPatIndex.IsJoinablePatientIndexTable());
         });
 
-        var compiler = new CohortCompiler(cic);
+        var compiler = new CohortCompiler(new ThrowImmediatelyActivator(RepositoryLocator, null), cic);
 
         var runner = new CohortCompilerRunner(compiler, 50);