Rework support for @latest version

Currently, unpkg and jsdelivr support using 'latest' as a version.  When files are requested by libman, they redirect (HTTP302) to the URL for the current latest files.  We put those files into the cache like any other version (in a folder name 'latest') and all is well.

However, when a new version is released, libman continues to use the now-stale downloaded assets from the cache.  There isn't a signal to realize when to purge the latest version from the cache.

This change instead simply turns 'latest' into the latest version we determine from the provider.  There is a small risk that we are out of sync with the CDN: for example, since we get the version data from NPM, it might not match the version that e.g. jsdelivr redirects to.  But this should be extremely rare.

Doing it this way also allows cdnjs to support 'latest'.  I didn't add it to the completion items, but I did verify that it works (CLI install, restore, and the file list in VS all support foo@latest for cdnjs now too).

Author: jimmylewis

src/LibraryManager/ManifestConstants.cs

@@ -48,5 +48,10 @@ public static class ManifestConstants
         /// libman.json files element 
         /// </summary>
         public const string Files = "files";
+
+        /// <summary>
+        /// For providers that support versioned libraries, this represents the evergreen latest version
+        /// </summary>
+        public const string LatestVersion = "latest";
     }
 }

src/LibraryManager/Providers/BaseProvider.cs

@@ -98,6 +98,18 @@ public virtual async Task<ILibraryOperationResult> UpdateStateAsync(ILibraryInst
             try
             {
                 ILibraryCatalog catalog = GetCatalog();
+
+                if (string.Equals(desiredState.Version, ManifestConstants.LatestVersion, StringComparison.Ordinal))
+                {
+                    // replace the @latest version with the latest version from the catalog.  This redirect
+                    // ensures that as new versions are released, we will not reuse stale "latest" assets
+                    // from the cache.
+                    string latestVersion = await catalog.GetLatestVersion(libraryId, includePreReleases: false, cancellationToken).ConfigureAwait(false);
+                    LibraryInstallationState newState = LibraryInstallationState.FromInterface(desiredState);
+                    newState.Version = latestVersion;
+                    desiredState = newState;
+                }
+
                 ILibrary library = await catalog.GetLibraryAsync(desiredState.Name, desiredState.Version, cancellationToken).ConfigureAwait(false);
 
                 if (library == null)

src/LibraryManager/Providers/Cdnjs/CdnjsCatalog.cs

@@ -20,7 +20,7 @@ internal class CdnjsCatalog : ILibraryCatalog
         // TO DO: These should become Provider properties to be passed to CacheService
         private const string FileName = "cache.json";
         public const string CatalogUrl = "https://api.cdnjs.com/libraries?fields=name,description,version";
-        public const string MetaPackageUrlFormat =    "https://api.cdnjs.com/libraries/{0}?fields=filename,versions"; // {libraryName}
+        public const string MetaPackageUrlFormat = "https://api.cdnjs.com/libraries/{0}?fields=filename,versions"; // {libraryName}
         public const string PackageVersionUrlFormat = "https://api.cdnjs.com/libraries/{0}/{1}?fields=files";         // {libraryName}/{version}
 
         private readonly string _cacheFile;
@@ -150,7 +150,13 @@ public async Task<ILibrary> GetLibraryAsync(string libraryName, string version,
                 throw new InvalidLibraryException(libraryId, _provider.Id);
             }
 
-            if (!(await GetLibraryVersionsAsync(libraryName, cancellationToken)).Contains(version))
+            if (string.Equals(version, ManifestConstants.LatestVersion, StringComparison.Ordinal))
+            {
+                version = await GetLatestVersion(libraryName, includePreReleases: false, cancellationToken);
+            }
+
+            IEnumerable<string> enumerable = await GetLibraryVersionsAsync(libraryName, cancellationToken);
+            if (!enumerable.Contains(version))
             {
                 throw new InvalidLibraryException(libraryId, _provider.Id);
             }
@@ -160,7 +166,7 @@ public async Task<ILibrary> GetLibraryAsync(string libraryName, string version,
                 JObject groupMetadata = await GetLibraryGroupMetadataAsync(libraryName, cancellationToken);
                 string defaultFile = groupMetadata?["filename"].Value<string>() ?? string.Empty;
 
-                IEnumerable<string> libraryFiles= await GetLibraryFilesAsync(libraryName, version, cancellationToken).ConfigureAwait(false);
+                IEnumerable<string> libraryFiles = await GetLibraryFilesAsync(libraryName, version, cancellationToken).ConfigureAwait(false);
 
                 return new CdnjsLibrary
                 {

src/LibraryManager/Providers/Unpkg/UnpkgCatalog.cs

@@ -20,7 +20,6 @@ internal class UnpkgCatalog : ILibraryCatalog
         public const string CacheFileName = "cache.json";
         public const string LibraryFileListUrlFormat = "https://unpkg.com/{0}@{1}/?meta"; // e.g. https://unpkg.com/[email protected]/?meta
         public const string LatestLibraryVersonUrl = "https://unpkg.com/{0}/package.json"; // e.g. https://unpkg.com/jquery/package.json
-        public const string LatestVersionTag = "latest";
 
         private readonly INpmPackageInfoFactory _packageInfoFactory;
         private readonly INpmPackageSearch _packageSearch;
@@ -58,7 +57,7 @@ public async Task<string> GetLatestVersion(string libraryName, bool includePreRe
             try
             {
                 string latestLibraryVersionUrl = string.Format(LatestLibraryVersonUrl, libraryName);
-                string latestCacheFile = Path.Combine(_cacheFolder, libraryName, $"{LatestVersionTag}.json");
+                string latestCacheFile = Path.Combine(_cacheFolder, libraryName, $"{ManifestConstants.LatestVersion}.json");
 
                 string latestJson = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(latestLibraryVersionUrl,
                                                                                                  latestCacheFile,
@@ -88,7 +87,7 @@ public async Task<ILibrary> GetLibraryAsync(string libraryName, string version,
             }
 
             string libraryId = _libraryNamingScheme.GetLibraryId(libraryName, version);
-            if (string.Equals(version, LatestVersionTag, StringComparison.Ordinal))
+            if (string.Equals(version, ManifestConstants.LatestVersion, StringComparison.Ordinal))
             {
                 string latestVersion = await GetLatestVersion(libraryId, includePreReleases: false, cancellationToken).ConfigureAwait(false);
                 libraryId = _libraryNamingScheme.GetLibraryId(libraryName, latestVersion);
@@ -263,8 +262,8 @@ public async Task<CompletionSet> GetLibraryCompletionSetAsync(string libraryName
                     // support @latest version
                     completions.Add(new CompletionItem
                     {
-                        DisplayText = LatestVersionTag,
-                        InsertionText = _libraryNamingScheme.GetLibraryId(name, LatestVersionTag),
+                        DisplayText = ManifestConstants.LatestVersion,
+                        InsertionText = _libraryNamingScheme.GetLibraryId(name, ManifestConstants.LatestVersion),
                     });
 
                     completionSet.CompletionType = CompletionSortOrder.Version;

src/LibraryManager/Providers/jsDelivr/JsDelivrCatalog.cs

@@ -23,7 +23,6 @@ internal class JsDelivrCatalog : ILibraryCatalog
         public const string LatestLibraryVersionUrl = "https://data.jsdelivr.com/v1/package/npm/{0}";
         public const string LibraryFileListUrlFormatGH = "https://data.jsdelivr.com/v1/package/gh/{0}/flat";
         public const string LatestLibraryVersionUrlGH = "https://data.jsdelivr.com/v1/package/gh/{0}";
-        public const string LatestVersionTag = "latest";
 
         private readonly INpmPackageInfoFactory _packageInfoFactory;
         private readonly INpmPackageSearch _packageSearch;
@@ -61,7 +60,7 @@ public async Task<string> GetLatestVersion(string libraryId, bool includePreRele
                 bool isGitHub = IsGitHub(libraryId);
                 string latestLibraryVersionUrl = string.Format(isGitHub ? LatestLibraryVersionUrlGH : LatestLibraryVersionUrl, name);
                 string cacheFileType = isGitHub ? "github" : "npm";
-                string latestLibraryVersionCacheFile = Path.Combine(_cacheFolder, name, $"{cacheFileType}-{LatestVersionTag}.json");
+                string latestLibraryVersionCacheFile = Path.Combine(_cacheFolder, name, $"{cacheFileType}-{ManifestConstants.LatestVersion}.json");
 
                 string latestVersionContent = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(latestLibraryVersionUrl,
                                                                                                            latestLibraryVersionCacheFile,
@@ -116,7 +115,7 @@ public async Task<ILibrary> GetLibraryAsync(string name, string version, Cancell
             }
 
             string libraryId = _libraryNamingScheme.GetLibraryId(name, version);
-            if (string.Equals(version, LatestVersionTag, StringComparison.Ordinal))
+            if (string.Equals(version, ManifestConstants.LatestVersion, StringComparison.Ordinal))
             {
                 string latestVersion = await GetLatestVersion(libraryId, includePreReleases: false, cancellationToken).ConfigureAwait(false);
                 libraryId = _libraryNamingScheme.GetLibraryId(name, latestVersion);
@@ -288,8 +287,8 @@ public async Task<CompletionSet> GetLibraryCompletionSetAsync(string libraryName
                     // support @latest version
                     completions.Add(new CompletionItem
                     {
-                        DisplayText = LatestVersionTag,
-                        InsertionText = _libraryNamingScheme.GetLibraryId(name, LatestVersionTag),
+                        DisplayText = ManifestConstants.LatestVersion,
+                        InsertionText = _libraryNamingScheme.GetLibraryId(name, ManifestConstants.LatestVersion),
                     });
 
                     completionSet.CompletionType = CompletionSortOrder.Version;

test/LibraryManager.Test/Providers/Cdnjs/CdnjsCatalogTest.cs

@@ -134,6 +134,19 @@ public async Task GetLibraryAsync_Success()
             Assert.IsNotNull(library.Files);
         }
 
+        [TestMethod]
+        public async Task GetLibraryAsync_LatestVersion_Success()
+        {
+            CdnjsCatalog sut = SetupCatalog();
+
+            ILibrary library = await sut.GetLibraryAsync("sampleLibrary", "latest", CancellationToken.None);
+
+            Assert.IsNotNull(library);
+            Assert.AreEqual("sampleLibrary", library.Name);
+            Assert.AreEqual("3.1.4", library.Version);
+            Assert.IsNotNull(library.Files);
+        }
+
         [TestMethod]
         public async Task GetLibraryAsync_InvalidLibraryId()
         {