Experiment: Create Nodes (Area\Iteration) at validation and not up front. (#1733)

* Node creation on demand!
* Add flag for this configuration +semver: minor

Author: MrHinsh

README.md

@@ -79,6 +79,7 @@ We use these tools with our customers, and for fun, to do real world migrations
 
 ## Change Log
 
+- 14.3 - Created a flag for `ShouldCreateNodesUpFront`, while the default is `true` this is a private preview of a new feature. Instead of creating the area and iteration paths up front, this new feature instead creates the Area & Iteration paths at validation time. For each missing path it will create it, and for those that exist it will simply pass over after a `GetNodeFromPath` call. For tose wishing to participate in the preview please set `ShouldCreateNodesUpFront` to `false`, and to create the nodes rather than just list them set `ShouldCreateMissingRevisionPaths` to `true` as well. It will still list nodes that are not able to be created and require a mapping. NOTE: You can also run with `"ShouldCreateMissingRevisionPaths": false` to list all the nodes that will be created so that you can create more elaborate mappings. 
 - 14.2 - Removed the `StopMigrationOnMissingAreaIterationNodes` flag. All missing nodes MUST be present or mapped using `AreaMaps` and `IterationMaps`. System will always stop on missing nodes.
 - 14.1 - Enabled auto update on client devices, server still used choco
 - 14.0 - Move from Chocolaty to Winget as the base installer. We will continue to publish to Chocolaty, but we recommend using `winget install nkdAgility.AzureDevOpsMigrationTools` for future installs. Main executable renamed to "devopsmigration.exe" to prevent conflict with other applications with symbolic links.

docs/Reference/Generated/MigrationTools.Host.xml

@@ -17,7 +17,7 @@ configurationSamples:
       "AttachmentMigration": true,
       "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\",
       "FixHtmlAttachmentLinks": false,
-      "SkipToFinalRevisedWorkItemType": true,
+      "SkipToFinalRevisedWorkItemType": false,
       "WorkItemCreateRetryLimit": 5,
       "FilterWorkItemsThatAlreadyExistInTarget": true,
       "PauseAfterEachWorkItem": false,
@@ -38,7 +38,8 @@ configurationSamples:
       "MaxGracefulFailures": 0,
       "SkipRevisionWithInvalidIterationPath": false,
       "SkipRevisionWithInvalidAreaPath": false,
-      "ShouldCreateMissingRevisionPaths": true
+      "ShouldCreateMissingRevisionPaths": true,
+      "ShouldCreateNodesUpFront": true
     }
   sampleFor: MigrationTools._EngineV1.Configuration.Processing.WorkItemMigrationConfig
 description: WorkItemMigrationConfig is the main processor used to Migrate Work Items, Links, and Attachments. Use `WorkItemMigrationConfig` to configure.
@@ -122,6 +123,10 @@ options:
   type: Boolean
   description: When set to True the susyem will try to create any missing missing area or iteration paths from the revisions.
   defaultValue: missng XML code comments
+- parameterName: ShouldCreateNodesUpFront
+  type: Boolean
+  description: missng XML code comments
+  defaultValue: missng XML code comments
 - parameterName: SkipRevisionWithInvalidAreaPath
   type: Boolean
   description: When set to true, this setting will skip a revision if the source area has not been migrated, has been deleted or is somehow invalid, etc.

docs/_data/reference.v1.processors.workitemmigrationcontext.yaml

@@ -15,7 +15,8 @@ configurationSamples:
       "IterationMaps": {
         "$type": "Dictionary`2"
       },
-      "ShouldCreateMissingRevisionPaths": true
+      "ShouldCreateMissingRevisionPaths": true,
+      "ShouldCreateNodesUpFront": true
     }
   sampleFor: MigrationTools.Enrichers.TfsNodeStructureOptions
 description: missng XML code comments
@@ -51,6 +52,10 @@ options:
   type: Boolean
   description: missng XML code comments
   defaultValue: missng XML code comments
+- parameterName: ShouldCreateNodesUpFront
+  type: Boolean
+  description: missng XML code comments
+  defaultValue: missng XML code comments
 status: missng XML code comments
 processingTarget: missng XML code comments
 classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs

docs/_data/reference.v2.processorenrichers.tfsnodestructure.yaml

@@ -18,7 +18,7 @@ configurationSamples:
       "AttachmentMigration": true,
       "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\",
       "FixHtmlAttachmentLinks": false,
-      "SkipToFinalRevisedWorkItemType": true,
+      "SkipToFinalRevisedWorkItemType": false,
       "WorkItemCreateRetryLimit": 5,
       "FilterWorkItemsThatAlreadyExistInTarget": true,
       "PauseAfterEachWorkItem": false,
@@ -39,7 +39,8 @@ configurationSamples:
       "MaxGracefulFailures": 0,
       "SkipRevisionWithInvalidIterationPath": false,
       "SkipRevisionWithInvalidAreaPath": false,
-      "ShouldCreateMissingRevisionPaths": true
+      "ShouldCreateMissingRevisionPaths": true,
+      "ShouldCreateNodesUpFront": true
     }
   sampleFor: MigrationTools._EngineV1.Configuration.Processing.WorkItemMigrationConfig
 description: WorkItemMigrationConfig is the main processor used to Migrate Work Items, Links, and Attachments. Use `WorkItemMigrationConfig` to configure.
@@ -123,6 +124,10 @@ options:
   type: Boolean
   description: When set to True the susyem will try to create any missing missing area or iteration paths from the revisions.
   defaultValue: missng XML code comments
+- parameterName: ShouldCreateNodesUpFront
+  type: Boolean
+  description: missng XML code comments
+  defaultValue: missng XML code comments
 - parameterName: SkipRevisionWithInvalidAreaPath
   type: Boolean
   description: When set to true, this setting will skip a revision if the source area has not been migrated, has been deleted or is somehow invalid, etc.
@@ -440,6 +445,76 @@ topics:
       `TargetProject\NewArea\ValidArea\` but `OriginalProject\DescopeThis` would not
       be modified by this rule.
 
+    ### More Complex Regex
+
+
+    Before your migration starts it will validate that all of the Areas and Iterations from the **Source** work items revisions exist on the **Target**. Any that do not exist will be flagged in the logs and if and the migration will stop just after it outputs a list of the missing nodes.
+
+
+    Our algorithm that converts the Source nodes to Target nodes processes the [mappings](https://nkdagility.com/learn/azure-devops-migration-tools/Reference/v1/Processors/WorkItemMigrationContext/#iteration-maps-and-area-maps) at that time. This means that any valid mapped nodes will never be caught by the `This path is not anchored in the source project` message as they are already altered to be valid.
+
+
+    > We recently updated the logging for this part of the system to more easily debug both your mappings and to see what they system is doing with the nodes and their current state. You can set `"LogLevel": "Debug"` to see the details.
+
+
+    To add a mapping, you can follow [the documentation](https://nkdagility.com/learn/azure-devops-migration-tools/Reference/v1/Processors/WorkItemMigrationContext/#iteration-maps-and-area-maps) with this being the simplest way:
+
+
+    ```
+
+    "IterationMaps": {
+      "WorkItemMovedFromProjectName\\\\Iteration 1": "TargetProject\\Sprint 1",
+    },
+
+    "AreaMaps": {
+       "WorkItemMovedFromProjectName\\\\Team 2": "TargetProject\\ProductA\\Team 2",
+    }
+
+    ```
+
+    Or you can use regular expressions to match the missing area or iteration paths:
+
+
+    ```
+
+    "IterationMaps": {
+      "^OriginalProject\\\\Path1(?=\\\\Sprint 2022)": "TargetProject\\AnotherPath\\NewTeam",
+      "^OriginalProject\\\\Path1(?=\\\\Sprint 2020)": "TargetProject\\AnotherPath\\Archives\\Sprints 2020",
+      "^OriginalProject\\\\Path2": "TargetProject\\YetAnotherPath\\Path2",
+    },
+
+    "AreaMaps": {
+      "^OriginalProject\\\\(DescopeThis|DescopeThat)": "TargetProject\\Archive\\Descoped\\",
+      "^OriginalProject\\\\(?!DescopeThis|DescopeThat)": "TargetProject\\NewArea\\",
+    }
+
+    ```
+
+
+    If you want to use the matches in the replacement you can use the following:
+
+
+    ```
+
+    "IterationMaps": {
+      "^\\\\oldproject1(?:\\\\([^\\\\]+))?\\\\([^\\\\]+)$": "TargetProject\\Q1\$2",
+    }
+
+    ```
+
+    If the olf iteration path was `\oldproject1\Custom Reporting\Sprint 13`, then this would result in a match for each Iteration node after the project node. You would then be able to reference any of the nodes using "$" and then the number of the match.
+
+
+
+    Regular expressions are much more difficult to build and debug so it is a good idea to use a [regular expression tester](https://regex101.com/) to check that you are matching the right things and to build them in ChatGTP.
+
+
+    _NOTE: You need `\\` to escape a `\` the pattern, and `\\` to escape a `\` in JSON. Therefor on the left of the match you need 4 `\` to represent the `\\` for the pattern and only 2 `\` in the match_ 
+
+
+    ![image](https://github.com/nkdAgility/azure-devops-migration-tools/assets/5205575/2cf50929-7ea9-4a71-beab-dd8ff3b5b2a8)
+
+
     ## More Complex Team Migrations
 
     The above options allow you to bring over a sub-set of the WIs (using the `WIQLQueryBit`) and move their area or iteration path to a default location. However you may wish to do something more complex e.g. re-map the team structure. This can be done with addition of a `FieldMaps` block to configuration in addition to the `NodeBasePaths`.

docs/collections/_reference/reference.v1.processors.workitemmigrationcontext.md

@@ -16,7 +16,8 @@ configurationSamples:
       "IterationMaps": {
         "$type": "Dictionary`2"
       },
-      "ShouldCreateMissingRevisionPaths": true
+      "ShouldCreateMissingRevisionPaths": true,
+      "ShouldCreateNodesUpFront": true
     }
   sampleFor: MigrationTools.Enrichers.TfsNodeStructureOptions
 description: missng XML code comments
@@ -52,6 +53,10 @@ options:
   type: Boolean
   description: missng XML code comments
   defaultValue: missng XML code comments
+- parameterName: ShouldCreateNodesUpFront
+  type: Boolean
+  description: missng XML code comments
+  defaultValue: missng XML code comments
 status: missng XML code comments
 processingTarget: missng XML code comments
 classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs

docs/collections/_reference/reference.v2.processorenrichers.tfsnodestructure.md

@@ -170,7 +170,15 @@ private NodeInfo GetOrCreateNode(string nodePath, DateTime? startDate, DateTime?
                     _pathToKnownNodeMap.TryGetValue(currentAncestorPath, out parentNode);
                     if (parentNode == null)
                     {
-                        parentNode= _targetCommonStructureService.GetNodeFromPath(currentAncestorPath);
+                        try
+                        {
+                            parentNode = _targetCommonStructureService.GetNodeFromPath(currentAncestorPath);
+                        } catch (Exception ex)
+                        {
+                            Log.LogDebug("  Not Found:", currentAncestorPath);
+                            parentNode = null;
+                        }
+                        
                     }
                 }
                 else
@@ -251,7 +259,10 @@ public override void ProcessorExecutionBegin(IProcessor processor)
             {
                 Log.LogInformation("Migrating all Nodes before the Processor run.");
                 EntryForProcessorType(processor);
-                MigrateAllNodeStructures();
+                if (Options.ShouldCreateNodesUpFront) // FLAG for Expermiment to Do on Demand.
+                {
+                    MigrateAllNodeStructures();
+                }
                 RefreshForProcessorType(processor);
             }
         }

src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs

@@ -12,13 +12,15 @@ public class TfsNodeStructureOptions : ProcessorEnricherOptions, ITfsNodeStructu
         public Dictionary<string, string> AreaMaps { get; set; }
         public Dictionary<string, string> IterationMaps { get; set; }
         public bool ShouldCreateMissingRevisionPaths { get; set; }
+        public bool ShouldCreateNodesUpFront { get; set; }   // FLAG for Expermiment to Do on Demand.
 
         public override void SetDefaults()
         {
             Enabled = true;
             AreaMaps = new Dictionary<string, string>();
             IterationMaps = new Dictionary<string, string>();
             ShouldCreateMissingRevisionPaths = true;
+            ShouldCreateNodesUpFront = true;  // FLAG for Expermiment to Do on Demand.
         }
     }
 

src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructureOptions.cs

@@ -151,7 +151,7 @@ public static IHostBuilder CreateDefaultBuilder(string[] args)
         private static string GetVersionTextForLog()
         {
             Version runningVersion = DetectVersionService2.GetRunningVersion();
-            string textVersion = ((runningVersion.Major > 1) ? "$v" + runningVersion : ThisAssembly.Git.BaseTag + "-" + ThisAssembly.Git.Commits + "-local");
+            string textVersion = ((runningVersion.Major > 1) ? "v" + runningVersion : ThisAssembly.Git.BaseTag + "-" + ThisAssembly.Git.Commits + "-local");
             return textVersion;
         }
 

src/MigrationTools.Host/MigrationToolHost.cs

@@ -73,7 +73,9 @@ public void RunStartupLogic(string[] args)
                     Log.Warning("Windows Server: If you are running on Windows Server you can use the experimental version of Winget, or you can still use Chocolatey to manage the install. Install chocolatey from https://chocolatey.org/install and then use `choco install vsts-sync-migrator` to install, and `choco upgrade vsts-sync-migrator` to upgrade to newer versions.", _detectVersionService.PackageId);
                 } else
                 {
-                    if (!_detectVersionService.IsPackageInstalled && !_detectVersionService.IsRunningInDebug)
+                    if (!_detectVersionService.IsRunningInDebug)
+                    { 
+                    if (!_detectVersionService.IsPackageInstalled)
                     {
                         Log.Information("It looks like this application has been installed from a zip, would you like to use the managed version?");
                         Console.WriteLine("Do you want install the managed version? (y/n)");
@@ -85,14 +87,14 @@ public void RunStartupLogic(string[] args)
                     }
                     if (_detectVersionService.IsUpdateAvailable && _detectVersionService.IsPackageInstalled)
                     {
-                        Log.Information("It looks like this application has been installed from a zip, would you like to use the managed version from Winget?");
+                        Log.Information("It looks like an updated version is available from Winget, would you like to update?");
                         Console.WriteLine("Do you want install the managed version? (y/n)");
                         if (Console.ReadKey().Key == ConsoleKey.Y)
                         {
                             _detectVersionService.UpdateFromSource();
                         }
                     }
-                    if ((_detectVersionService.IsNewLocalVersionAvailable && _detectVersionService.IsPackageInstalled) && !_detectVersionService.IsRunningInDebug)
+                    if (_detectVersionService.IsNewLocalVersionAvailable && _detectVersionService.IsPackageInstalled)
                     {
                         Log.Information("It looks like this package ({PackageId}) has been updated locally to version {InstalledVersion} and you are not running the latest version?", _detectVersionService.PackageId, _detectVersionService.InstalledVersion);
                         Console.WriteLine("Do you want to quit and restart? (y/n)");
@@ -104,6 +106,10 @@ public void RunStartupLogic(string[] args)
                             Environment.Exit(0);
                         }
                     }
+                    } else
+                    {
+                        Log.Information("Running in Debug! No further version checkes.....");
+                    }
                 }
             } else
             {