Task/rdmp 353 delta extractions (#2303)

* allow the addition of new columns to extraction archives

* update changelog

* allow extraction progess with no dqe set

* add option to no clone extraction progress

* update changelog

* fix typo

Author: JFriel

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.1.2] - Unreleased
+- Allow new columns to be added to archive extractions
+- Simplify use of Extraction Progress
+
 ## [9.1.1] - 2025-12-02
 - Allow Atlassian service workers to write to Confluence from RDMP
 

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAddExtractionProgress.cs

@@ -28,12 +28,6 @@ public ExecuteCommandAddExtractionProgress(IBasicActivateItems activator, ISelec
             SetImpossible("Catalogue already has an ExtractionProgress");
             return;
         }
-
-        if (_sds.GetCatalogue()?.TimeCoverage_ExtractionInformation_ID == null)
-        {
-            SetImpossible("Catalogue does not have a time coverage field configured");
-            return;
-        }
     }
 
     public override Image<Rgba32> GetImage(IIconProvider iconProvider) =>

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneExtractionConfiguration.cs

@@ -80,6 +80,15 @@ public override void Execute()
         CheckForDeprecatedCatalogues();
 
         var clone = _extractionConfiguration.DeepCloneWithNewIDs();
+        if (_activeItems.IsInteractive && clone.SelectedDataSets.Any(sds => sds.ExtractionProgressIfAny != null))
+        {
+            if(!_activeItems.YesNo("Do you want to include this extraction's extraction progress in this clone?", "Include Extraction Progress?")) {
+                foreach (var ep in clone.SelectedDataSets.Where(sds => sds.ExtractionProgressIfAny != null).Select(sds => sds.ExtractionProgressIfAny))
+                {
+                    ep.DeleteInDatabase();
+                }
+            }
+        }
         foreach (var ds in _toRemove.Cast<ExtractableDataSet>())
         {
             clone.RemoveDatasetFromConfiguration(ds);

Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs

@@ -216,6 +216,38 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis
                     {
                         present = false;
                     }
+                    //check the columns are correct, we might have added some
+                    var existingColumns = existing.DiscoverColumns();
+                    var existingColumnNames = existingColumns.Select(ec => ec.GetRuntimeName());
+                    var toProcessColumnNames = toProcess.Columns.Cast<DataColumn>().Select(col => col.ColumnName);
+                    var newColumns = toProcessColumnNames.Where(c => !existingColumnNames.Contains(c));
+                    if (newColumns.Any())
+                    {
+                        var archiveTable = _destinationDatabase.ExpectTable(tblName+"_Archive");
+                        if (archiveTable.Exists())
+                        {
+                            foreach (var column in newColumns)
+                            {
+                                existing.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000);
+                                archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true,30000);
+                            }
+                            if (present)
+                            {
+                                string triggerProblems = "";
+                                string triggerOK = "";
+                                implementor.DropTrigger(out triggerProblems, out triggerOK);
+                                if(triggerProblems != "")
+                                {
+                                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, triggerProblems));   
+                                }
+
+                                existing = _destinationDatabase.ExpectTable(tblName);
+                                implementor = triggerFactory.Create(existing);
+                                present = false;
+                            }
+                        }
+                    }
+
                     if (!present)
                     {
                         implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet);

SharedAssemblyInfo.cs

@@ -10,6 +10,6 @@
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
-[assembly: AssemblyVersion("9.1.1")]
-[assembly: AssemblyFileVersion("9.1.1")]
-[assembly: AssemblyInformationalVersion("9.1.1")]
+[assembly: AssemblyVersion("9.1.2")]
+[assembly: AssemblyFileVersion("9.1.2")]
+[assembly: AssemblyInformationalVersion("9.1.2")]