Merge pull request #10742 from NuGet/dev

[ReleasePrep][2026.03.16]RI of dev into main

Author: zhhyu

python/StatsLogParser/loginterpretation/knownclients.yaml

@@ -223,4 +223,8 @@
 
   # NuGet - Keep this one at the bottom of this file as a catch-all resolver
   - regex: '(NuGet)/?(\d+)\.(\d+)\.?(\d+)?'
-    family_replacement: 'NuGet'
+    family_replacement: 'NuGet'
+
+  # GetNuTool
+  - regex: '(GetNuTool)/(\d+)\.(\d+)\.?(\d+)?'
+    family_replacement: 'GetNuTool'

python/StatsLogParser/tests/test_useragentparser.py

@@ -65,7 +65,8 @@
     ("Bazel/release 6.0.0", "Bazel", "6", "0", "0"),
     ("Visual Studio/6.4.0", "Visual Studio", "6", "4", "0"),
     ("NuGetMirror/6.0.0", "NuGetMirror", "6", "0", "0"),
-    ("BaGet/1.0.0", "BaGet", "1", "0", "0")
+    ("BaGet/1.0.0", "BaGet", "1", "0", "0"),
+    ("GetNuTool/1.10.0.5939+e5f5c45", "GetNuTool", "1", "10", "0")
     ])
 def test_recognizes_custom_clients(user_agent, expected_client, expected_major, expected_minor, expected_patch):
     parsed = UserAgentParser.parse(user_agent)

src/Catalog/Dnx/DnxCatalogCollector.cs

@@ -79,13 +79,15 @@ protected override Task<IEnumerable<CatalogCommitItemBatch>> CreateBatchesAsync(
             return Task.FromResult(batches);
         }
 
-        protected override Task<bool> FetchAsync(
+        protected override async Task<bool> FetchAsync(
             CollectorHttpClient client,
             ReadWriteCursor front,
             ReadCursor back,
             CancellationToken cancellationToken)
         {
-            return CatalogCommitUtilities.ProcessCatalogCommitsAsync(
+            await DnxPackageVersionIndexCacheControl.LoadPackageIdsToIncludeAsync(_storageFactory.Create(), _logger, cancellationToken);
+
+            return await CatalogCommitUtilities.ProcessCatalogCommitsAsync(
                 client,
                 front,
                 back,
@@ -156,7 +158,7 @@ await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async ca
                             cancellationToken);
                         var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri);
 
-                        if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent)
+                        if (isNupkgSynchronized && areRequiredPropertiesPresent && isPackageInIndex && !DnxPackageVersionIndexCacheControl.PackageIdsToInclude.Contains(packageId))
                         {
                             _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion);
 

src/Catalog/Dnx/DnxMaker.cs

@@ -219,7 +219,7 @@ public async Task UpdatePackageVersionIndexAsync(string id, Action<HashSet<NuGet
                 // Store versions (sorted)
                 result.Sort();
 
-                await storage.SaveAsync(resourceUri, CreateContent(result.Select(version => version.ToNormalizedString())), cancellationToken);
+                await storage.SaveAsync(resourceUri, CreateContentForPackageVersionIndex(id, result.Select(version => version.ToNormalizedString())), cancellationToken);
             }
             else
             {
@@ -260,10 +260,10 @@ private static HashSet<NuGetVersion> GetVersions(string json)
             return result;
         }
 
-        private StorageContent CreateContent(IEnumerable<string> versions)
+        private StorageContent CreateContentForPackageVersionIndex(string id, IEnumerable<string> versions)
         {
             JObject obj = new JObject { { "versions", new JArray(versions) } };
-            return new StringStorageContent(obj.ToString(), "application/json", Constants.NoStoreCacheControl);
+            return new StringStorageContent(obj.ToString(), "application/json", DnxPackageVersionIndexCacheControl.GetCacheControl(id, _logger));
         }
 
         private async Task<Uri> SaveNupkgAsync(Stream nupkgStream, CatalogStorage storage, string id, string version, CancellationToken cancellationToken)

src/Catalog/Dnx/DnxPackageVersionIndexCacheControl.cs

@@ -0,0 +1,63 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+    public static class DnxPackageVersionIndexCacheControl
+    {
+        private const string DefaultCacheControlForPackageVersionIndex = "max-age=10";
+        private const string BlobNameOfPackageIdsToInclude = "PackageIdsToIncludeForCachingPackageVersionIndex.json";
+
+        public static HashSet<string> PackageIdsToInclude = new HashSet<string>();
+
+        public static string GetCacheControl(string id, ILogger logger)
+        {
+            if (PackageIdsToInclude.Contains(id))
+            {
+                logger.LogInformation("Add caching to the package version index of Package Id: {id}.", id);
+
+                return DefaultCacheControlForPackageVersionIndex;
+            }
+            else
+            {
+                return Constants.NoStoreCacheControl;
+            }
+        }
+
+        public static async Task LoadPackageIdsToIncludeAsync(IStorage storage, ILogger logger, CancellationToken cancellationToken)
+        {
+            if (!storage.Exists(BlobNameOfPackageIdsToInclude))
+            {
+                logger.LogInformation("{BlobName} does not exist, at {Address}.", BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+
+                return;
+            }
+
+            logger.LogInformation("Loading the list of Package Ids from {BlobName}, at {Address}.", BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+
+            PackageIdsToInclude = new HashSet<string>();
+            string jsonFile = await storage.LoadStringAsync(storage.ResolveUri(BlobNameOfPackageIdsToInclude), cancellationToken);
+            if (jsonFile != null)
+            {
+                JObject obj = JObject.Parse(jsonFile);
+                JArray ids = obj["ids"] as JArray;
+                if (ids != null)
+                {
+                    foreach (JToken id in ids)
+                    {
+                        PackageIdsToInclude.Add(id.ToString().ToLowerInvariant());
+                    }
+                }
+            }
+
+            logger.LogInformation("Loaded the list of Package Ids (Count: {Count}) from {BlobName}, at {Address}.", PackageIdsToInclude.Count, BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+        }
+    }
+}

src/Catalog/DurableCursorWithUpdates.cs

@@ -19,9 +19,10 @@ public class DurableCursorWithUpdates : DurableCursor
 
         private readonly int _maxNumberOfUpdatesToKeep;
         private readonly TimeSpan _minIntervalBetweenTwoUpdates;
+        private readonly TimeSpan _minIntervalBeforeToReadUpdate;
 
         public DurableCursorWithUpdates(Uri address, Persistence.Storage storage, DateTime defaultValue, ILogger logger,
-            int maxNumberOfUpdatesToKeep, TimeSpan minIntervalBetweenTwoUpdates)
+            int maxNumberOfUpdatesToKeep, TimeSpan minIntervalBetweenTwoUpdates, TimeSpan minIntervalBeforeToReadUpdate)
             : base(address, storage, defaultValue)
         {
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -36,8 +37,14 @@ public DurableCursorWithUpdates(Uri address, Persistence.Storage storage, DateTi
                 throw new ArgumentOutOfRangeException(nameof(minIntervalBetweenTwoUpdates), $"{nameof(minIntervalBetweenTwoUpdates)} must be equal or larger than 0.");
             }
 
+            if (minIntervalBeforeToReadUpdate < TimeSpan.Zero)
+            {
+                throw new ArgumentOutOfRangeException(nameof(minIntervalBeforeToReadUpdate), $"{nameof(minIntervalBeforeToReadUpdate)} must be equal or larger than 0.");
+            }
+
             _maxNumberOfUpdatesToKeep = maxNumberOfUpdatesToKeep;
             _minIntervalBetweenTwoUpdates = minIntervalBetweenTwoUpdates;
+            _minIntervalBeforeToReadUpdate = minIntervalBeforeToReadUpdate;
         }
 
         public override async Task SaveAsync(CancellationToken cancellationToken)
@@ -54,6 +61,7 @@ public override async Task SaveAsync(CancellationToken cancellationToken)
             }
 
             cursorValueWithUpdates.Value = Value.ToString("O");
+            cursorValueWithUpdates.MinIntervalBeforeToReadUpdate = _minIntervalBeforeToReadUpdate;
             if (storageContent != null)
             {
                 cursorValueWithUpdates.Updates = GetUpdates(cursorValueWithUpdates, storageContent.StorageDateTimeInUtc);

src/Catalog/Helpers/CursorValueWithUpdates.cs

@@ -16,6 +16,12 @@ public class CursorValueWithUpdates
 
         [JsonProperty("value")]
         public string Value { get; set; }
+
+        // This is for the cursor reader to determine which update (in the list of updates) of the cursor value to read.
+        // The timestamp of the update to read should be at least before the current timestamp minus this interval.
+        [JsonProperty("minIntervalBeforeToReadUpdate")]
+        public TimeSpan MinIntervalBeforeToReadUpdate { get; set; }
+
         [JsonProperty("updates")]
         public IList<CursorValueUpdate> Updates { get; set; } = new List<CursorValueUpdate>();
     }
@@ -30,6 +36,7 @@ public CursorValueUpdate(DateTime timeStamp, string value)
 
         [JsonProperty("timeStamp")]
         public DateTime TimeStamp { get; set; }
+
         [JsonProperty("value")]
         public string Value { get; set; }
     }

src/Catalog/HttpReadCursor.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -14,9 +14,9 @@ namespace NuGet.Services.Metadata.Catalog
 {
     public class HttpReadCursor : ReadCursor
     {
-        private readonly Uri _address;
-        private readonly DateTime? _defaultValue;
-        private readonly Func<HttpMessageHandler> _handlerFunc;
+        protected readonly Uri _address;
+        protected readonly DateTime? _defaultValue;
+        protected readonly Func<HttpMessageHandler> _handlerFunc;
 
         public HttpReadCursor(Uri address, DateTime defaultValue, Func<HttpMessageHandler> handlerFunc = null)
         {
@@ -52,9 +52,9 @@ await Retry.IncrementalAsync(
                         {
                             response.EnsureSuccessStatusCode();
 
-                            string json = await response.Content.ReadAsStringAsync();
+                            string valueInJson = await GetValueInJsonAsync(response);
 
-                            JObject obj = JObject.Parse(json);
+                            JObject obj = JObject.Parse(valueInJson);
                             Value = obj["value"].ToObject<DateTime>();
                         }
                     }
@@ -66,5 +66,10 @@ await Retry.IncrementalAsync(
                 initialWaitInterval: TimeSpan.Zero,
                 waitIncrement: TimeSpan.FromSeconds(10));
         }
+
+        public virtual async Task<string> GetValueInJsonAsync(HttpResponseMessage response)
+        {
+            return await response.Content.ReadAsStringAsync();
+        }
     }
-}
+}

src/Catalog/HttpReadCursorWithUpdates.cs

@@ -0,0 +1,85 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    public class HttpReadCursorWithUpdates : HttpReadCursor
+    {
+        private readonly ILogger _logger;
+
+        public HttpReadCursorWithUpdates(Uri address, ILogger logger, Func<HttpMessageHandler> handlerFunc = null)
+            : base(address, handlerFunc)
+        {
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public override async Task<string> GetValueInJsonAsync(HttpResponseMessage response)
+        {
+            var storageDateTimeOffset = response.Headers.Date;
+            DateTime? storageDateTimeInUtc = null;
+            if (storageDateTimeOffset.HasValue)
+            {
+                storageDateTimeInUtc = storageDateTimeOffset.Value.UtcDateTime;
+            }
+
+            var update = await GetUpdate(response, storageDateTimeInUtc);
+
+            return JsonConvert.SerializeObject(new { value = update.Value });
+        }
+
+        private async Task<CursorValueUpdate> GetUpdate(HttpResponseMessage response, DateTime? storageDateTimeInUtc)
+        {
+            if (!storageDateTimeInUtc.HasValue)
+            {
+                throw new ArgumentNullException(nameof(storageDateTimeInUtc));
+            }
+
+            var content = await response.Content.ReadAsStringAsync();
+
+            var cursorValueWithUpdates = JsonConvert.DeserializeObject<CursorValueWithUpdates>(content, CursorValueWithUpdates.SerializerSettings);
+            var minIntervalBeforeToReadUpdate = cursorValueWithUpdates.MinIntervalBeforeToReadUpdate;
+            var updates = cursorValueWithUpdates.Updates.OrderByDescending(u => u.TimeStamp).ToList();
+
+            foreach (var update in updates)
+            {
+                if (update.TimeStamp <= storageDateTimeInUtc.Value - minIntervalBeforeToReadUpdate)
+                {
+                    _logger.LogInformation("Read the cursor update with timeStamp: {TimeStamp} and value: {UpdateValue}, at {Address}. " +
+                        "(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadUpdate: {MinIntervalBeforeToReadUpdate})",
+                        update.TimeStamp.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
+                        update.Value,
+                        _address.AbsoluteUri,
+                        storageDateTimeInUtc.Value.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
+                        minIntervalBeforeToReadUpdate);
+
+                    return update;
+                }
+            }
+
+            if (updates.Count > 0)
+            {
+                _logger.LogWarning("Unable to find the cursor update and the oldest cursor update has timeStamp: {TimeStamp}, at {Address}. " +
+                    "(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadUpdate: {MinIntervalBeforeToReadUpdate})",
+                    updates.Last().TimeStamp.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
+                    _address.AbsoluteUri,
+                    storageDateTimeInUtc.Value.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
+                    minIntervalBeforeToReadUpdate);
+            }
+            else
+            {
+                _logger.LogWarning("Unable to find the cursor update and the count of updates is {CursorUpdatesCount}, at {Address}.",
+                    updates.Count,
+                    _address.AbsoluteUri);
+            }
+
+            throw new InvalidOperationException("Unable to find the cursor update.");
+        }
+    }
+}

src/Catalog/Persistence/IStorage.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -27,6 +27,7 @@ Task<OptimisticConcurrencyControlToken> GetOptimisticConcurrencyControlTokenAsyn
         Task<string> LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken);
         Uri ResolveUri(string relativeUri);
         Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken);
+        bool Exists(string fileName);
 
         /// <summary>
         /// Updates the cache control header on the provided resource URI (blob). This method throws an exception if
@@ -38,4 +39,4 @@ Task<OptimisticConcurrencyControlToken> GetOptimisticConcurrencyControlTokenAsyn
         /// <returns>True if the Cache-Control changes, false if the Cache-Control already matched the provided value.</returns>
         Task<bool> UpdateCacheControlAsync(Uri resourceUri, string cacheControl, CancellationToken cancellationToken);
     }
-}
+}