Merged PR 684749: Add support for async publishing to remote in GetCHL and AddOrGetCHL

Publish CHLs to remote for GetCHL and AddOrGetCHL (that results in a GetCHL) calls if the CHLs differ between local and remote. If the CHLs match but the remote is not backed, then pin the CHLs to remote.

Related [query](https://dataexplorer.azure.com/clusters/cbuild/databases/CloudBuildCBTest?query=H4sIAAAAAAAAA2WRwW7CMAxA7/sKq+KwSc3UplkhB6YJmLRKjKHBbrukiduGlaRKAxPSPn6lQmyIq+1nP9vT2u7UVMgK57Z83qPxNz/wXaFDWDqUusW13uLKi20DjyBKexuru3PJZKdrlSkYjyFQCUuilCeEJZQSFudIOM8l4RhLpVJBWYTBmZxa57AWXltz4nlCIyxGHcAfkDCa5mRUJJykRao4kyrK6fDIb6w28KWNGmtj0MFauBJ9ZgoL1sCgxsLf//MaOF1Wp8gK27YfeNZYWIXrQ4O9weQjm8/+FN97vz0uha+O+aegcXovPH7OrF+gZ0OamdaLukZ1pBpnNyj91d3Ci07h5eohvIi2WnmnTRnCW4OuT4Tw2rmKEru+1qluzfxw/RDRyl8Q2oGqwAEAAA==) for a successful GetCHL call + publishing scenario.

Author: mruxmohan4

Public/Src/Cache/MemoizationStore/Distributed/Sessions/PublishingCacheSession.cs

@@ -110,12 +110,47 @@ public async Task<AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Co
             return result;
         }
 
+        /// <inheritdoc />
+        public async Task<GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
+        {
+            var operationContext = TrackShutdown(new OperationContext(context, cts));
+            var cancellationForPublish = _publishAsynchronously
+                ? ShutdownStartedCancellationToken
+                : operationContext.Context.Token;
+
+            var localResult = await _local.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint);
+            if (localResult.Succeeded && localResult.ContentHashListWithDeterminism.ContentHashList is not null)
+            {
+                var publishTask = PublishContentHashListAsync(
+                    context,
+                    strongFingerprint,
+                    localResult.ContentHashListWithDeterminism,
+                    cancellationForPublish);
+
+                if (_publishAsynchronously)
+                {
+                    publishTask.FireAndForget(context, traceErrorResult: true, operation: nameof(PublishContentHashListAsync));
+                }
+                else
+                {
+                    var publishingResult = await publishTask;
+                    if (!publishingResult.Succeeded)
+                    {
+                        return new GetContentHashListResult(publishingResult);
+                    }
+                }
+            }
+
+            return localResult;
+        }
+
         private async Task<BoolResult> PublishContentHashListAsync(
             Context context,
             StrongFingerprint strongFingerprint,
             ContentHashListWithDeterminism contentHashList,
             CancellationToken token)
         {
+            await Task.Yield();
             Contract.Assert(!_hasBeenMarkedForShutdown, "Should not queue publish operations after the session has been marked for shutdown");
 
             var publishingOperation = new PublishingOperation
@@ -172,12 +207,6 @@ public IAsyncEnumerable<GetSelectorResult> GetSelectors(Context context, Fingerp
             return _local.GetSelectors(context, weakFingerprint, cts, urgencyHint);
         }
 
-        /// <inheritdoc />
-        public Task<GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
-        {
-            return _local.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint);
-        }
-
         /// <inheritdoc />
         public Task<PinResult> PinAsync(Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
         {

Public/Src/Cache/MemoizationStore/DistributedTest/BuildCachePublishingCacheTests.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 #if MICROSOFT_INTERNAL
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -96,6 +97,8 @@ public DummyPublisher(bool forceUpdate)
                 _forceUpdate = forceUpdate;
             }
 
+            public Guid CacheGuid => Guid.Empty;
+
             protected override Tracer Tracer { get; } = new Tracer(nameof(DummyPublisher));
 
             public Task<AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
@@ -126,6 +129,16 @@ public Task<AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
                 return Task.FromResult(new AddOrGetContentHashListResult(_storedHashLists[strongFingerprint]));
             }
 
+            public Task<GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
+            {
+                if (_storedHashLists.ContainsKey(strongFingerprint))
+                {
+                    return Task.FromResult(new GetContentHashListResult(_storedHashLists[strongFingerprint]));
+                };
+
+                return Task.FromResult(new GetContentHashListResult(default(ContentHashListWithDeterminism)));
+            }
+
             public Task<BoolResult> IncorporateStrongFingerprintsAsync(OperationContext context, IEnumerable<Task<StrongFingerprint>> strongFingerprints)
             {
                 return BoolResult.SuccessTask;

Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IPublishingSession.cs

@@ -43,6 +43,11 @@ Task<BoolResult> IncorporateStrongFingerprintsAsync(
     /// <nodoc />
     public interface ICachePublisher : IStartupShutdownSlim
     {
+        /// <summary>
+        /// Gets the unique GUID for the given cache.
+        /// </summary>
+        Guid CacheGuid { get; }
+
         /// <summary>
         /// Add a content hash list.
         /// </summary>
@@ -76,11 +81,21 @@ public Task<IEnumerable<Task<Indexed<PinResult>>>> PinAsync(
         Task<BoolResult> IncorporateStrongFingerprintsAsync(
             OperationContext context,
             IEnumerable<Task<StrongFingerprint>> strongFingerprints);
+
+        /// <nodoc />
+        Task<GetContentHashListResult> GetContentHashListAsync(
+            Context context,
+            StrongFingerprint strongFingerprint,
+            CancellationToken cts,
+            UrgencyHint urgencyHint = UrgencyHint.Nominal);
     }
 
     /// <nodoc />
     public class CacheSessionPublisherWrapper : StartupShutdownSlimBase, ICachePublisher
     {
+        /// <nodoc />
+        public Guid CacheGuid => _cache.Id;
+
         /// <nodoc />
         protected override Tracer Tracer { get; } = new Tracer(nameof(CacheSessionPublisherWrapper));
 
@@ -134,6 +149,12 @@ public Task<BoolResult> IncorporateStrongFingerprintsAsync(OperationContext cont
         {
             return _session.IncorporateStrongFingerprintsAsync(context, strongFingerprints, context.Token);
         }
+
+        /// <nodoc />
+        public Task<GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
+        {
+            return _session.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint);
+        }
     }
 
     /// <nodoc />
@@ -223,12 +244,27 @@ public Task<BoolResult> PublishContentHashListAsync(
                 (timeSpentWaiting, gateCount) =>
                 {
                     ContentHashList? hashListInRemote = null;
+                    bool publishSkipped = false;
                     return context.PerformOperationAsync(
                         Tracer,
                         async () =>
                         {
+                            var remoteResult = await _cachePublisher.GetContentHashListAsync(context, fingerprint, context.Token);
+                            var localContentHashList = contentHashList.ContentHashList;
+                            var remoteContentHashList = remoteResult.ContentHashListWithDeterminism.ContentHashList;
+                            var isRemoteBacked = remoteResult.Succeeded
+                                    && (remoteResult.ContentHashListWithDeterminism.Determinism.IsDeterministicTool
+                                        || remoteResult.ContentHashListWithDeterminism.Determinism.EffectiveGuid.Equals(_cachePublisher.CacheGuid));
+
+                            // Skip publishing when local CHL matches remote CHL & the remote is backed.
+                            if (localContentHashList.Equals(remoteContentHashList) && isRemoteBacked)
+                            {
+                                publishSkipped = true;
+                                return BoolResult.Success;
+                            }
+
                             // Make sure to push the blob in the selector if it exists.
-                            var hashesToPush = new List<ContentHash>(contentHashList.ContentHashList.Hashes);
+                            var hashesToPush = new List<ContentHash>(localContentHashList.Hashes);
                             if (!fingerprint.Selector.ContentHash.IsZero())
                             {
                                 hashesToPush.Add(fingerprint.Selector.ContentHash);
@@ -252,7 +288,8 @@ public Task<BoolResult> PublishContentHashListAsync(
                         },
                         traceOperationStarted: false,
                         extraEndMessage: result =>
-                            $"Added=[{result.Succeeded && hashListInRemote is null}], " +
+                            $"Skipped=[{publishSkipped}], " +
+                            $"Added=[{result.Succeeded && hashListInRemote is null && !publishSkipped}], " +
                             $"StrongFingerprint=[{fingerprint}], " +
                             $"ContentHashList=[{contentHashList.ToTraceString()}], " +
                             $"TimeSpentWaiting=[{timeSpentWaiting}], " +

Public/Src/Cache/MemoizationStore/Library/Stores/AzureBlobStoragePublishingSession.cs

@@ -62,6 +62,7 @@ protected override async Task<ICachePublisher> CreateCachePublisherCoreAsync(
                 Credentials = credentials,
             });
 
+            // TODO: Implement force publishing for blob L3
             var database = new MetadataStoreMemoizationDatabase(blobMetadataStore);
             var memoizationStore = new DatabaseMemoizationStore(database);
 

Public/Src/Cache/MemoizationStore/Vsts/BuildCachePublishingSession.cs

@@ -56,6 +56,7 @@ protected override async Task<ICachePublisher> CreateCachePublisherCoreAsync(Ope
                 helper: null,
                 logger: m => Tracer.Info(context, m));
 
+            configuration.BuildCacheConfiguration.ForceUpdateOnAddContentHashList = true;
             var cache = BuildCacheCacheFactory.Create(
                 PassThroughFileSystem.Default,
                 context.TracingContext.Logger,