[Datasets] Task/rdmp 33 dataset integration interface

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs

@@ -26,7 +26,7 @@ public void TestDatasetCreationOKExtendedParameters()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset2","somedoi","some source");
         Assert.DoesNotThrow(cmd.Execute);
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source");
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First(static ds => ds.Name == "dataset2" && ds.DigitalObjectIdentifier == "somedoi" && ds.Source == "some source");
         Assert.That(founddataset,Is.Not.Null);
     }
 }

Rdmp.Core.Tests/CommandExecution/ExecuteCommandDeleteDatasetTest.cs

@@ -13,20 +13,20 @@ public void TestDeleteExistingDataset()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
-        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Dataset>(), Has.Length.EqualTo(1));
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First(static ds => ds.Name == "dataset");
+        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>(), Has.Length.EqualTo(1));
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First(static ds => ds.Name == "dataset");
         var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset);
         Assert.DoesNotThrow(() => delCmd.Execute());
-        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Dataset>(), Is.Empty);
+        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>(), Is.Empty);
     }
 
     [Test]
     public void TestDeleteNonExistantDataset()
     {
-        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Dataset>(), Is.Empty);
-        var founddataset = new Core.Curation.Data.Dataset();
+        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>(), Is.Empty);
+        var founddataset = new Core.Curation.Data.Datasets.Dataset();
         var delCmd = new ExecuteCommandDeleteDataset(GetMockActivator(), founddataset);
         Assert.Throws<NullReferenceException>(() => delCmd.Execute());
-        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Rdmp.Core.Curation.Data.Dataset>(), Is.Empty);
+        Assert.That(GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>(), Is.Empty);
     }
 }

Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs

@@ -34,15 +34,10 @@ public void TestLinkCatalogueToDataset()
 
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(cmd.Execute);
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().First(static c => c.Name == "Dataset1");
         var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset);
         Assert.DoesNotThrow(linkCmd.Execute);
-        var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<ColumnInfo>();
-        foreach (var ci in columInfo)
-        {
-            Assert.That(ci.Dataset_ID, Is.EqualTo(founddataset.ID));
-        }
         founddataset.DeleteInDatabase();
         foundCatalogue.DeleteInDatabase();
 
@@ -72,29 +67,17 @@ public void TestLinkCatalogueToDatasetNotAll()
 
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(cmd.Execute);
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().First(c => c.Name == "Dataset1");
         var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), foundCatalogue, founddataset, false);
         Assert.DoesNotThrow(linkCmd.Execute);
-        var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<CatalogueItem>().Where(ci => _cata1.CatalogueItems.Contains(ci));
-        foreach (var ci in columInfo)
-        {
-            Assert.That(ci.ColumnInfo.Dataset_ID, Is.EqualTo(founddataset.ID));
-        }
-
-        var columInfo2 = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<CatalogueItem>().Where(ci => _cata2.CatalogueItems.Contains(ci));
-        foreach (var ci in columInfo2)
-        {
-            Assert.That(ci.ColumnInfo.Dataset_ID, Is.Null);
-        }
-
     }
     [Test]
     public void TestLinkCatalogueToDatasetBadCatalogue()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(cmd.Execute);
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, founddataset, false);
         Assert.Throws<ImpossibleCommandException>(linkCmd.Execute);
     }
@@ -104,12 +87,12 @@ public void TestLinkCatalogueToDatasetBadDataset()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(cmd.Execute);
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
-        var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository,"catalogue"), null, false);
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
+        var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), new Catalogue(GetMockActivator().RepositoryLocator.CatalogueRepository, "catalogue"), null, false);
         Assert.Throws<ImpossibleCommandException>(linkCmd.Execute);
     }
 
-    [Test] 
+    [Test]
     public void TestLinkCatalogueToDatasetBadEverything()
     {
         var linkCmd = new ExecuteCommandLinkCatalogueToDataset(GetMockActivator(), null, null, false);

Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs

@@ -33,7 +33,7 @@ public void TestLinkColumnInfoToDataset()
 
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset);
         Assert.DoesNotThrow(() => linkCmd.Execute());
         var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<ColumnInfo>();
@@ -67,7 +67,7 @@ public void TestLinkColumnInfoToDatasetNotAll()
 
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset, false);
         Assert.DoesNotThrow(() => linkCmd.Execute());
         var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<CatalogueItem>().Where(ci => _cata1.CatalogueItems.Contains(ci));
@@ -87,7 +87,7 @@ public void TestLinkCatalogueToDatasetBadColumnInfo()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
+        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Datasets.Dataset>().First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), null, founddataset, false);
         Assert.Throws<NullReferenceException>(() => linkCmd.Execute());
     }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateDataset.cs

@@ -1,5 +1,4 @@
 using Rdmp.Core.Curation.Data;
-
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
 public class ExecuteCommandCreateDataset : BasicCommandExecution
@@ -24,7 +23,7 @@ public ExecuteCommandCreateDataset(IBasicActivateItems activator, [DemandsInitia
     public override void Execute()
     {
         base.Execute();
-        var dataset = new Curation.Data.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source };
+        var dataset = new Rdmp.Core.Curation.Data.Datasets.Dataset(BasicActivator.RepositoryLocator.CatalogueRepository, _name) { DigitalObjectIdentifier = _doi, Source = _source };
         dataset.SaveToDatabase();
         _activator.Publish(dataset);
     }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeleteDataset.cs

@@ -4,9 +4,9 @@
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 public sealed class ExecuteCommandDeleteDataset: BasicCommandExecution
 {
-    private readonly Curation.Data.Dataset _dataset;
+    private readonly Curation.Data.Datasets.Dataset _dataset;
     private readonly IBasicActivateItems _activator;
-public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Dataset dataset)
+public ExecuteCommandDeleteDataset(IBasicActivateItems activator, [DemandsInitialization("The Dataset to delete")]Curation.Data.Datasets.Dataset dataset)
     {
         _dataset = dataset;
         _activator = activator;

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider.cs

@@ -0,0 +1,62 @@
+using Rdmp.Core.Curation.Data.Datasets;
+using Rdmp.Core.Curation.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NPOI.OpenXmlFormats.Vml.Office;
+
+namespace Rdmp.Core.CommandExecution.AtomicCommands
+{
+    public class ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider : BasicCommandExecution, IAtomicCommand
+    {
+
+        private readonly IBasicActivateItems _activator;
+        private readonly IDatasetProvider _provider;
+        private readonly bool _includeExtractable;
+        private readonly bool _includeInternal;
+        private readonly bool _includeProjectSpecific;
+        private readonly bool _includeDeprecated;
+
+        public ExecuteCommandImportExistingCataloguesIntoExternalDatasetProvider(IBasicActivateItems activator, IDatasetProvider provider, bool includeExtractable, bool includeInternal, bool includeProjectSpecific, bool includeDeprecated)
+        {
+            _activator = activator;
+            _provider = provider;
+            _includeExtractable = includeExtractable;
+            _includeInternal = includeInternal;
+            _includeProjectSpecific = includeProjectSpecific;
+            _includeDeprecated = includeDeprecated;
+        }
+
+
+        public override void Execute()
+        {
+            var catalogues = _activator.RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().ToList();
+            if (!_includeInternal)
+            {
+                catalogues = catalogues.Where(c => !c.IsInternalDataset).ToList();
+            }
+            if (!_includeProjectSpecific)
+            {
+                catalogues = catalogues.Where(c => !c.IsProjectSpecific(_activator.RepositoryLocator.DataExportRepository)).ToList();
+            }
+            if (!_includeDeprecated)
+            {
+                catalogues = catalogues.Where(c => !c.IsDeprecated).ToList();
+            }
+            if (!_includeExtractable)
+            {
+                catalogues = catalogues.Where(c => !c.GetExtractabilityStatus(_activator.RepositoryLocator.DataExportRepository).IsExtractable).ToList();
+            }
+            //todo check this catalogue filtering works
+            foreach (var catalogue in catalogues)
+            {
+                var dataset = _provider.Create(catalogue);
+                var ds = _provider.AddExistingDatasetWithReturn(null, dataset.GetID());
+                var cmd = new ExecuteCommandLinkCatalogueToDataset(_activator, catalogue, ds);
+                cmd.Execute();
+            }
+        }
+    }
+}

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkCatalogueToDataset.cs

@@ -7,15 +7,16 @@
 using System;
 using Rdmp.Core.Curation.Data;
 using System.Linq;
+using Rdmp.Core.Curation.Data.Datasets;
 
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
 public sealed class ExecuteCommandLinkCatalogueToDataset : BasicCommandExecution
 {
     private readonly Catalogue _catalogue;
-    private readonly Curation.Data.Dataset _dataset;
+    private readonly Curation.Data.Datasets.Dataset _dataset;
     private readonly bool _linkAll;
-    public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator)
+    public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [DemandsInitialization("The catalogue To link")]Catalogue catalogue, [DemandsInitialization("The dataset to link to")]Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator)
     {
         _catalogue = catalogue;
         _dataset = dataset;
@@ -29,21 +30,9 @@ public ExecuteCommandLinkCatalogueToDataset(IBasicActivateItems activator, [Dema
     public override void Execute()
     {
         base.Execute();
-        var items = _catalogue.CatalogueItems.ToList();
-        foreach (var ci in items.Select(static item => item.ColumnInfo).Where(ci => ci?.Dataset_ID != _dataset.ID))
-        {
-            ci.Dataset_ID = _dataset.ID;
-            ci.SaveToDatabase();
-            if (!_linkAll) continue;
-
-            var databaseName = ci.Name[..ci.Name.LastIndexOf('.')];
-            var catalogueItems = ci.CatalogueRepository.GetAllObjects<ColumnInfo>().Where(ci => ci.Name[..ci.Name.LastIndexOf(".", StringComparison.Ordinal)] == databaseName).ToList();
-            foreach (var aci in catalogueItems)
-            {
-                aci.Dataset_ID = _dataset.ID;
-                aci.SaveToDatabase();
-            }
-        }
+        var linkage = new CatalogueDatasetLinkage(_catalogue.CatalogueRepository, _catalogue, _dataset);
+        linkage.SaveToDatabase();
+        Publish(linkage);
 
     }
 }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandLinkColumnInfoToDataset.cs

@@ -13,9 +13,9 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands;
 public sealed class ExecuteCommandLinkColumnInfoToDataset : BasicCommandExecution
 {
     private readonly ColumnInfo _columnInfo;
-    private readonly Curation.Data.Dataset _dataset;
+    private readonly Curation.Data.Datasets.Dataset _dataset;
     private readonly bool _linkAll;
-    public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Dataset dataset, bool linkAllOtherColumns = true) : base(activator)
+    public ExecuteCommandLinkColumnInfoToDataset(IBasicActivateItems activator, [DemandsInitialization("The column to link")] ColumnInfo columnInfo, [DemandsInitialization("The dataset to link to")] Curation.Data.Datasets.Dataset dataset, bool linkAllOtherColumns = true) : base(activator)
     {
         _columnInfo = columnInfo;
         _dataset = dataset;

Rdmp.Core/Curation/Data/Catalogue.cs

@@ -14,8 +14,10 @@
 using FAnsi.Discovery;
 using FAnsi.Discovery.QuerySyntax;
 using Rdmp.Core.CohortCreation.Execution;
+using Rdmp.Core.CommandExecution;
 using Rdmp.Core.Curation.Data.Aggregation;
 using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.Curation.Data.Datasets;
 using Rdmp.Core.Curation.Data.Defaults;
 using Rdmp.Core.Curation.Data.ImportExport;
 using Rdmp.Core.Curation.Data.Serialization;
@@ -30,6 +32,7 @@
 using Rdmp.Core.ReusableLibraryCode.Annotations;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.DataAccess;
+using Rdmp.Core.Startup;
 using Rdmp.Core.Ticketing;
 
 namespace Rdmp.Core.Curation.Data;
@@ -1147,6 +1150,25 @@ internal Catalogue(ShareManager shareManager, ShareDefinition shareDefinition)
     /// <inheritdoc/>
     public override string ToString() => Name;
 
+    public List<Datasets.Dataset> GetLinkedDatasets()
+    {
+        return CatalogueRepository.GetAllObjectsWhere<CatalogueDatasetLinkage>("Catalogue_ID", this.ID).Select(l => l.Dataset).Distinct().ToList();
+    }
+
+    public override void SaveToDatabase()
+    {
+        base.SaveToDatabase();
+        foreach (var dataset in CatalogueRepository.GetAllObjectsWhere<CatalogueDatasetLinkage>("Catalogue_ID", this.ID).Where(cdl => cdl.Autoupdate).Select(cld => cld.Dataset))
+        {
+            var provider = CatalogueRepository.GetObjectByID<DatasetProviderConfiguration>((int)dataset.Provider_ID);
+            var providerConfiguration = CatalogueRepository.GetObjectByID<DatasetProviderConfiguration>((int)dataset.Provider_ID);
+            var repositoryProvider = new UserSettingsRepositoryFinder();
+            var activator = new ThrowImmediatelyActivator(repositoryProvider, ThrowImmediatelyCheckNotifier.Quiet);
+            var providerInstance = providerConfiguration.GetProviderInstance(activator);
+            providerInstance.UpdateUsingCatalogue(dataset, this);
+        }
+    }
+
     /// <summary>
     /// Sorts alphabetically based on <see cref="Name"/>
     /// </summary>