Cache Tagger in TextView not TextBuffer

See #2382 for why.

Author: jcansdale

src/GitHub.InlineReviews/Tags/InlineCommentTaggerProvider.cs

@@ -1,9 +1,9 @@
 using System;
 using System.ComponentModel.Composition;
 using GitHub.Extensions;
-using GitHub.InlineReviews.Services;
 using GitHub.Services;
 using Microsoft.VisualStudio.Text;
+using Microsoft.VisualStudio.Text.Differencing;
 using Microsoft.VisualStudio.Text.Editor;
 using Microsoft.VisualStudio.Text.Tagging;
 using Microsoft.VisualStudio.Utilities;
@@ -31,11 +31,22 @@ public InlineCommentTaggerProvider(
 
         public ITagger<T> CreateTagger<T>(ITextView view, ITextBuffer buffer) where T : ITag
         {
-            return buffer.Properties.GetOrCreateSingletonProperty(() =>
-                new InlineCommentTagger(
-                    view,
-                    buffer,
-                    sessionManager)) as ITagger<T>;
+            if (view.TextViewModel is IDifferenceTextViewModel model)
+            {
+                if (buffer == model.Viewer.DifferenceBuffer.BaseLeftBuffer)
+                {
+                    return view.Properties.GetOrCreateSingletonProperty("InlineTaggerForLeftBuffer",
+                        () => new InlineCommentTagger(view, buffer, sessionManager) as ITagger<T>);
+                }
+
+                if (buffer == model.Viewer.DifferenceBuffer.BaseRightBuffer)
+                {
+                    return view.Properties.GetOrCreateSingletonProperty("InlineTaggerForRightBuffer",
+                        () => new InlineCommentTagger(view, buffer, sessionManager) as ITagger<T>);
+                }
+            }
+
+            return null;
         }
     }
 }