Merge pull request #1111 from solidify/feature/vs402625-bump-changeddate

Feature/vs402625 bump changeddate

Author: Alexander-Hjelm

docs/config.md

@@ -38,7 +38,7 @@ The migration configuration file is defined in a json file with the properties d
 |**suppress-notifications**|False|boolean|Set to True to suppress all notifications in Azure DevOps about created and updated Work Items. Default = False.|
 |**include-development-links**|False|boolean|Set to True to migrated commit links from Jira to Azure DevOps. You will also need to fill out the **repository-map** property. Default = False.|
 |**sleep-time-between-revision-import-milliseconds**|False|integer|How many milliseconds to sleep between each revision import. Use this if throttling is an issue for ADO Services. Default = 0 (no sleep).|
-|**buffer-revisions-succeeding-attachment-imports-milliseconds**|False|integer|How many milliseconds to buffer each subsequent revision if there is a negative revision timestamp offset. Increase this if you get problems with VS402625 error messages. Default = 5 (ms).|
+|**changeddate-bump-ms**|False|integer|How many milliseconds to buffer each subsequent revision if there is a negative revision timestamp offset. Increase this if you get a lot of VS402625 warning messages during the import. Default = 2 (ms).|
 |**process-template**|False|string|Process template in the target DevOps project. Supported values: Scrum, Agile or CMMI. Default = "Scrum".|
 |**link-map**|True|json|List of **links** to map between Jira and Azure DevOps/TFS work item link types.|
 |**type-map**|True|json|List of the work item **types** you want to migrate from Jira to Azure DevOps/TFS.|

docs/faq.md

@@ -430,28 +430,28 @@ curl -D-
  "http://johnie:8081/rest/api/2/search"
 ```
 
-## 19. I get the error message "VS402625: Dates must be increasing with each revision."
+## 19. I get the warning message "VS402625: Dates must be increasing with each revision."
 
-This error message will show up if the tool attempts to import a subsequent revision with a changedDate that is less than the current changedDate of the current state of the Work Item.
+This warning message will show up if the tool attempts to import a subsequent revision with a changedDate that is less than the current changedDate of the current state of the Work Item.
 
 This can have multiple causes:
 
 - Two consecutive revisions have the same date stamp, or the difference is 1ms or less (i.e. the issue is in the Jira issue data itself).
 - ADO can sometimes add a few milliseconds to the work item changedDate when adding an attachment.
+- A link change in Jira has created a situation where the Link Import happens earlier in time than the timestamp of one of the revisions in the associated issues.
 
 You may end up receiving an error message similar to this one:
 
 ```txt
-[E][18:32:06] VS402625: Dates must be increasing with each revision.
-[E][18:32:06] Work Item 15312 failed to save.
+[W][11:15:29] Received response while updating Work Item: VS402625: Dates must be increasing with each revision.. Bumped Changed Date by 2ms and trying again... New ChangedDate: 3/31/2016 3:21:38 PM, ms: 172
 ```
 
-The solution is to buffer the subsequent revision's changedDate by a few miliseconds. This is the purpose of the configuration parameter `buffer-revisions-succeeding-attachment-imports-milliseconds` (default: 5). Add this parameter to your `config.json` file and try increasing the value by 1 (5, 6, 7, and so on...) until the import succeeds without errors.
+The tool is attempting to buffer the subsequent revision's changedDate by a few miliseconds in order to get around the error response from the ADO Rest API. The exact number of miliseconds to buffer can be controlled with the config parameter `changeddate-bump-ms` (default: 2). If you experience a lot of this warning message and believe that your import is slowing down because of it, go ahead and add this parameter to your `config.json` file and try increasing the value by 1 (3, 4, 5, and so on...) until the import succeeds without too many warnings.
 
 Example `config.json`:
 
 ```json
-  "buffer-revisions-succeeding-attachment-imports-milliseconds": 6,
+  "changeddate-bump-ms": 5,
 ```
 
 ## 20. Sprint names are corrupted. ADO Iteration paths are named "[ synced = false  ]"

src/WorkItemMigrator/Migration.Common/Config/ConfigJson.cs

@@ -81,8 +81,8 @@ public class ConfigJson
         [JsonProperty(PropertyName = "include-jira-css-styles")]
         public bool IncludeJiraCssStyles { get; set; } = false;
 
-        [JsonProperty(PropertyName = "buffer-revisions-succeeding-attachment-imports-milliseconds")]
-        public int BufferRevisionsSucceedingAttachmentImportsMilliseconds { get; set; } = 5;
+        [JsonProperty(PropertyName = "changeddate-bump-ms")]
+        public int ChangedDateBumpMS { get; set; } = 2;
 
         [JsonProperty(PropertyName = "ignore-empty-revisions")]
         public bool IgnoreEmptyRevisions { get; set; } = false;

src/WorkItemMigrator/WorkItemImport/Agent.cs

@@ -67,7 +67,7 @@ public bool ImportRevision(WiRevision rev, WorkItem wi, Settings settings)
                 if (rev.Index == 0)
                     _witClientUtils.EnsureClassificationFields(rev);
 
-                _witClientUtils.EnsureDateFields(rev, wi, settings.BufferRevisionsSucceedingAttachmentImportsMilliseconds);
+                _witClientUtils.EnsureDateFields(rev, wi);
                 _witClientUtils.EnsureAuthorFields(rev);
                 _witClientUtils.EnsureAssigneeField(rev, wi);
                 _witClientUtils.EnsureFieldsOnStateChange(rev, wi);
@@ -228,7 +228,7 @@ internal static Agent Initialize(MigrationContext context, Settings settings)
 
             var agent = new Agent(context, settings, restConnection);
 
-            var witClientWrapper = new WitClientWrapper(settings.Account, settings.Project, settings.Pat);
+            var witClientWrapper = new WitClientWrapper(settings.Account, settings.Project, settings.Pat, settings.ChangedDateBumpMS);
             agent._witClientUtils = new WitClientUtils(witClientWrapper);
 
             // check if projects exists, if not create it

src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs

@@ -94,7 +94,7 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt
                     IncludeDevelopmentLinks = config.IncludeDevelopmentLinks,
                     FieldMap = config.FieldMap,
                     SuppressNotifications = config.SuppressNotifications,
-                    BufferRevisionsSucceedingAttachmentImportsMilliseconds = config.BufferRevisionsSucceedingAttachmentImportsMilliseconds
+                    ChangedDateBumpMS = config.ChangedDateBumpMS
                 };
 
                 // initialize Azure DevOps/TFS connection. Creates/fetches project, fills area and iteration caches.

src/WorkItemMigrator/WorkItemImport/Settings.cs

@@ -22,6 +22,6 @@ public Settings(string account, string project, string pat)
         public bool IncludeDevelopmentLinks { get; internal set; }
         public FieldMap FieldMap { get; internal set; }
         public bool SuppressNotifications { get; internal set; }
-        public int BufferRevisionsSucceedingAttachmentImportsMilliseconds { get; set; }
+        public int ChangedDateBumpMS { get; set; }
     }
 }

src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs

@@ -240,7 +240,7 @@ public void EnsureAssigneeField(WiRevision rev, WorkItem wi)
             rev.Fields.Add(new WiField() { ReferenceName = WiFieldReference.AssignedTo, Value = assignedTo });
         }
 
-        public void EnsureDateFields(WiRevision rev, WorkItem wi, int bufferMS)
+        public void EnsureDateFields(WiRevision rev, WorkItem wi)
         {
             if (rev == null)
             {
@@ -263,17 +263,7 @@ public void EnsureDateFields(WiRevision rev, WorkItem wi, int bufferMS)
             }
             if (!rev.Fields.HasAnyByRefName(WiFieldReference.ChangedDate))
             {
-                DateTime workItemChangedDate = (DateTime)(wi.Fields[WiFieldReference.ChangedDate]);
-                if (workItemChangedDate.ToUniversalTime() == rev.Time.ToUniversalTime())
-                {
-                    rev.Fields.Add(new WiField() { ReferenceName = WiFieldReference.ChangedDate, Value = rev.Time.AddMilliseconds(1).ToString("o") });
-                }
-                else
-                {
-                    // ADO can add a few milliseconds to work item createdDate when adding an attachment, hence adding more here to the revision time
-                    rev.Fields.Add(new WiField() { ReferenceName = WiFieldReference.ChangedDate, Value = rev.Time.AddMilliseconds(bufferMS).ToString("o") });
-                    wi.Fields[WiFieldReference.ChangedDate] = rev.Time.AddMilliseconds(bufferMS);
-                }
+                rev.Fields.Add(new WiField() { ReferenceName = WiFieldReference.ChangedDate, Value = rev.Time.ToString("o") });
             }
 
         }

src/WorkItemMigrator/WorkItemImport/WitClient/WitClientWrapper.cs

@@ -28,15 +28,17 @@ public class WitClientWrapper : IWitClientWrapper
         private VssConnection Connection { get; }
         private TeamProjectReference TeamProject { get; }
         private GitHttpClient GitClient { get; }
+        private int ChangedDateBumpMS { get; }
 
-        public WitClientWrapper(string collectionUri, string project, string personalAccessToken)
+        public WitClientWrapper(string collectionUri, string project, string personalAccessToken, int changedDateBumpMS)
         {
             var credentials = new VssBasicCredential("", personalAccessToken);
             Connection = new VssConnection(new Uri(collectionUri), credentials);
             WitClient = Connection.GetClient<WorkItemTrackingHttpClient>();
             ProjectClient = Connection.GetClient<ProjectHttpClient>();
             TeamProject = ProjectClient.GetProject(project).Result;
             GitClient = Connection.GetClient<GitHttpClient>();
+            ChangedDateBumpMS = changedDateBumpMS;
         }
 
         public WorkItem CreateWorkItem(string wiType, bool suppressNotifications, DateTime? createdDate = null, string createdBy = "")
@@ -104,13 +106,48 @@ public WorkItem GetWorkItem(int wiId)
 
         public WorkItem UpdateWorkItem(JsonPatchDocument patchDocument, int workItemId, bool suppressNotifications)
         {
-            return WitClient.UpdateWorkItemAsync(
-                document: patchDocument,
-                id: workItemId,
-                suppressNotifications: suppressNotifications,
-                bypassRules: true,
-                expand: WorkItemExpand.All
-            ).Result;
+            while (true)
+            {
+                try
+                {
+                    var result = WitClient.UpdateWorkItemAsync(
+                        document: patchDocument,
+                        id: workItemId,
+                        suppressNotifications: suppressNotifications,
+                        bypassRules: true,
+                        expand: WorkItemExpand.All
+                    ).Result;
+                    return result;
+                }
+                catch (AggregateException ex)
+                {
+                    bool datesMustIncreaseError = false;
+                    foreach (Exception ex2 in ex.InnerExceptions)
+                    {
+                        // Handle 'VS402625' error responses, the supplied ChangedDate was older than the latest revision already in ADO.
+                        // We must bump the ChangedDate by a small factor and try again.
+                        if (ex2.Message.Contains("VS402625"))
+                        {
+                            foreach (var patchOp in patchDocument)
+                            {
+                                if (patchOp.Path == "/fields/System.ChangedDate")
+                                {
+                                    patchOp.Value = ((DateTime)patchOp.Value).AddMilliseconds(ChangedDateBumpMS);
+                                    Logger.Log(LogLevel.Warning, $"Received response while updating Work Item: {ex2.Message}." +
+                                        $" Bumped ChangedDate by {ChangedDateBumpMS}ms and trying again... New ChangedDate: {patchOp.Value}, ms: " +
+                                        ((DateTime)patchOp.Value).Millisecond);
+                                    break;
+                                }
+                            }
+                            datesMustIncreaseError = true;
+                        }
+                    }
+                    if (!datesMustIncreaseError)
+                    {
+                        throw;
+                    }
+                }
+            }
         }
 
         public TeamProject GetProject(string projectId)

src/WorkItemMigrator/tests/Migration.Wi-Import.Tests/WitClient/WitClientUtilsTests.cs

@@ -280,7 +280,7 @@ public void When_calling_ensure_date_fields_with_empty_args_Then_an_exception_is
             MockedWitClientWrapper witClientWrapper = new MockedWitClientWrapper();
             WitClientUtils wiUtils = new WitClientUtils(witClientWrapper);
             Assert.That(
-                () => wiUtils.EnsureDateFields(null, null, 0),
+                () => wiUtils.EnsureDateFields(null, null),
                 Throws.InstanceOf<ArgumentException>());
         }
 
@@ -303,7 +303,7 @@ public void When_calling_ensure_date_fields_with_first_revision_Then_dates_are_a
 
             createdWI.Fields[WiFieldReference.ChangedDate] = now;
 
-            wiUtils.EnsureDateFields(rev, createdWI, 0);
+            wiUtils.EnsureDateFields(rev, createdWI);
 
             Assert.Multiple(() =>
             {
@@ -315,7 +315,7 @@ public void When_calling_ensure_date_fields_with_first_revision_Then_dates_are_a
                 Assert.That(rev.Fields[1].ReferenceName, Is.EqualTo(WiFieldReference.ChangedDate));
                 Assert.That(
                     DateTime.Parse(rev.Fields[1].Value.ToString()),
-                    Is.EqualTo(DateTime.Parse(rev.Fields[0].Value.ToString()).AddMilliseconds(1)));
+                    Is.EqualTo(DateTime.Parse(rev.Fields[0].Value.ToString())));
             });
         }