Merge pull request #1558 from paulvanbrenk/removeLocalNpmCacheDev14

paulvanbrenk · web-flow · commit 9fd4eaac6a1c · 2017-05-05T12:36:08.000-07:00
Minimal work to support the new NPM protocol for package search.
diff --git a/Nodejs/Product/Nodejs/NodejsPackage.cs b/Nodejs/Product/Nodejs/NodejsPackage.cs
@@ -64,7 +64,6 @@ namespace Microsoft.NodejsTools {
     [InstalledProductRegistration("#110", "#112", AssemblyVersionInfo.Version, IconResourceID = 400)]
     [Guid(Guids.NodejsPackageString)]
     [ProvideOptionPage(typeof(NodejsGeneralOptionsPage), "Node.js Tools", "General", 114, 115, true)]
-    [ProvideOptionPage(typeof(NodejsNpmOptionsPage), "Node.js Tools", "Npm", 114, 116, true)]
     [ProvideDebugEngine("Node.js Debugging", typeof(AD7ProgramProvider), typeof(AD7Engine), AD7Engine.DebugEngineId, setNextStatement: false, hitCountBp: true, justMyCodeStepping: false)]
 #if DEV14
     [ProvideLanguageService(typeof(NodejsLanguageInfo), NodejsConstants.Nodejs, 106, RequestStockColors = true, ShowSmartIndent = true, ShowCompletion = true, DefaultToInsertSpaces = true, HideAdvancedMembersByDefault = true, EnableAdvancedMembersOption = true, ShowDropDownOptions = true)]
diff --git a/Nodejs/Product/Nodejs/NpmUI/NpmPackageInstallWindow.xaml b/Nodejs/Product/Nodejs/NpmUI/NpmPackageInstallWindow.xaml
@@ -359,31 +359,6 @@
             </Grid>
             <GridSplitter Grid.Column="1" Grid.Row="0" Grid.RowSpan="3" Width="6" Background="Transparent" VerticalAlignment="Stretch" HorizontalAlignment="Center" ShowsPreview="True" />
 
-            <Grid Grid.Row="2" VerticalAlignment="Top" Margin="0 4 0 0">
-                <Grid.ColumnDefinitions>
-                    <ColumnDefinition Width="*" />
-                    <ColumnDefinition Width="Auto" />
-                </Grid.ColumnDefinitions>
-
-                <!-- Package cache control -->
-
-
-                <ContentControl Grid.Column="0"
-                                    Content="{Binding LastRefreshedMessage}"
-                                    Foreground="{DynamicResource {x:Static wpf:Controls.ForegroundKey}}"
-                                    VerticalAlignment="Center"
-                                    HorizontalAlignment="Stretch"
-                                    Focusable="False"
-                                    IsTabStop="False"/>
-
-                <Button Grid.Column="1" HorizontalAlignment="Right" Margin="0"
-                        Name="RefreshButton"
-                        Command="{x:Static npmUi:NpmPackageInstallViewModel.RefreshCatalogCommand}"
-                        KeyboardNavigation.TabIndex="8"
-                        xml:space="preserve"
-                        Content="{x:Static resx:NpmInstallWindowResources.RefreshButtonContent}" />
-            </Grid>
-
             <Grid Grid.Column="2" Grid.RowSpan="2" Margin="8 0 0 0" >
                 <Grid.RowDefinitions>
                     <RowDefinition Height="*"/>
diff --git a/Nodejs/Product/Npm/IPackageCatalog.cs b/Nodejs/Product/Npm/IPackageCatalog.cs
@@ -16,16 +16,150 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
 using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace Microsoft.NodejsTools.Npm {
     public interface IPackageCatalog {
         DateTime LastRefreshed { get; }
 
-        Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText, Uri registryUrl = null);
-
-        IPackage this[string name] { get; }
+        Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText);
 
         long? ResultsCount { get; }
     }
+
+    // This class is a wrapper catalog to directly query the NPM repo
+    // instead of downloading the entire catalog (which is no longer supported).
+    internal sealed class EmptyPackageCatalog : IPackageCatalog {
+
+        private static readonly Uri defaultRegistryUri = new Uri("https://registry.npmjs.org/");
+
+        public static readonly IPackageCatalog Instance = new EmptyPackageCatalog();
+
+        private EmptyPackageCatalog() { }
+
+        public DateTime LastRefreshed => DateTime.Now;
+
+        public long? ResultsCount => 0;
+
+        public async Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText) {
+
+            // All exceptions thrown here and in the called methods are handled by the
+            // NPM search dialog, so we don't have to do any exception handling here.
+
+            var relativeUri = string.Format("/-/v1/search?text={0}", filterText);
+            var searchUri = new Uri(defaultRegistryUri, relativeUri);
+
+            var request = WebRequest.Create(searchUri);
+            using (var response = await request.GetResponseAsync()) {
+                var reader = new StreamReader(response.GetResponseStream());
+                using (var jsonReader = new JsonTextReader(reader)) {
+                    while (jsonReader.Read()) {
+                        switch (jsonReader.TokenType) {
+                            case JsonToken.StartObject:
+                            case JsonToken.PropertyName:
+                                continue;
+                            case JsonToken.StartArray:
+                                return ReadPackagesFromArray(jsonReader);
+                            default:
+                                throw new InvalidOperationException("Unexpected json token.");
+                        }
+                    }
+                }
+            }
+
+            // should never get here
+            throw new InvalidOperationException("Unexpected json token.");
+        }
+
+        private IEnumerable<IPackage> ReadPackagesFromArray(JsonTextReader jsonReader) {
+            var pkgList = new List<IPackage>();
+
+            // Inside the array, each object is an NPM package
+            var builder = new NodeModuleBuilder();
+            while (jsonReader.Read()) {
+                switch (jsonReader.TokenType) {
+                    case JsonToken.PropertyName:
+                        if (StringComparer.OrdinalIgnoreCase.Equals(jsonReader.Value, "package")) {
+                            var token = (JProperty)JToken.ReadFrom(jsonReader);
+                            var package = ReadPackage(token.Value, builder);
+                            if (package != null) {
+                                pkgList.Add(package);
+                            }
+                        }
+                        continue;
+                    case JsonToken.EndArray:
+                        // This is the spot the function should always exit on valid data
+                        return pkgList;
+                    default:
+                        continue;
+                }
+            }
+            throw new JsonException("Unexpected end of stream reading the NPM catalog data array");
+        }
+
+        private IPackage ReadPackage(JToken package, NodeModuleBuilder builder) {
+            builder.Reset();
+
+            try {
+                builder.Name = (string)package["name"];
+                if (string.IsNullOrEmpty(builder.Name)) {
+                    // I don't believe this should ever happen if the data returned is
+                    // well formed. Could throw an exception, but just skip instead for
+                    // resiliency on the NTVS side.
+                    return null;
+                }
+
+                builder.AppendToDescription((string)package["description"] ?? string.Empty);
+
+                var date = package["date"];
+                if (date != null) {
+                    builder.AppendToDate((string)date);
+                }
+
+                var version = package["version"];
+                if (version != null) {
+                    var semver = SemverVersion.Parse((string)version);
+                    builder.AddVersion(semver);
+                }
+
+                AddKeywords(builder, package["keywords"]);
+                AddAuthor(builder, package["author"]);
+                AddHomepage(builder, package["links"]);
+
+                return builder.Build();
+            } catch (InvalidOperationException) {
+                // Occurs if a JValue appears where we expect JProperty
+                return null;
+            } catch (ArgumentException) {
+                return null;
+            }
+        }
+
+        private static void AddKeywords(NodeModuleBuilder builder, JToken keywords) {
+            if (keywords != null) {
+                foreach (var keyword in keywords.Select(v => (string)v)) {
+                    builder.AddKeyword(keyword);
+                }
+            }
+        }
+
+        private static void AddHomepage(NodeModuleBuilder builder, JToken links) {
+            var homepage = links?["homepage"];
+            if (homepage != null) {
+                builder.AddHomepage((string)homepage);
+            }
+        }
+
+        private static void AddAuthor(NodeModuleBuilder builder, JToken author) {
+            var name = author?["name"];
+            if (author != null) {
+                builder.AddAuthor((string)name);
+            }
+        }
+    }
 }
diff --git a/Nodejs/Product/Npm/NodeModuleBuilder.cs b/Nodejs/Product/Npm/NodeModuleBuilder.cs
@@ -144,6 +144,13 @@ public IEnumerable<string> Keywords {
             }
         }
 
+        public void AddVersion(SemverVersion version) {
+            if( version > this.LatestVersion) {
+                this.LatestVersion = version;
+            }
+            this._availableVersions.Add(version);
+        }
+
         public IPackage Build() {
             var proxy = new PackageProxy {
                 Author = Author,
diff --git a/Nodejs/Product/Npm/SPI/NpmController.cs b/Nodejs/Product/Npm/SPI/NpmController.cs
@@ -22,7 +22,7 @@
 
 namespace Microsoft.NodejsTools.Npm.SPI {
     internal class NpmController : AbstractNpmLogSource, INpmController {
-        private IPackageCatalog _sRepoCatalog;
+        private IPackageCatalog _sRepoCatalog = EmptyPackageCatalog.Instance;
         private string _fullPathToRootPackageDirectory;
         private string _cachePath;
         private bool _showMissingDevOptionalSubPackages;
@@ -174,28 +174,8 @@ public void LogCommandCompleted(object sender, NpmCommandCompletedEventArgs e) {
             OnCommandCompleted(e.Arguments, e.WithErrors, e.Cancelled);
         }
 
-        public async Task<IPackageCatalog> GetRepositoryCatalogAsync(bool forceDownload, IProgress<string> progress) {
-            //  This should really be thread-safe but await can't be inside a lock so
-            //  we'll just have to hope and pray this doesn't happen concurrently. Worst
-            //  case is we'll end up with two retrievals, one of which will be binned,
-            //  which isn't the end of the world.
-            _sRepoCatalog = null;
-            if (null == _sRepoCatalog || _sRepoCatalog.ResultsCount == 0 || forceDownload) {
-                Exception ex = null;
-                using (var commander = CreateNpmCommander()) {
-                    EventHandler<NpmExceptionEventArgs> exHandler = (sender, args) => { LogException(sender, args); ex = args.Exception; };
-                    commander.ErrorLogged += LogError;
-                    commander.ExceptionLogged += exHandler;
-                    _sRepoCatalog = await commander.GetCatalogAsync(forceDownload, progress);
-                    commander.ErrorLogged -= LogError;
-                    commander.ExceptionLogged -= exHandler;
-                }
-                if (null != ex) {
-                    OnOutputLogged(ex.ToString());
-                    throw ex;
-                }
-            }
-            return _sRepoCatalog;
+        public Task<IPackageCatalog> GetRepositoryCatalogAsync(bool forceDownload, IProgress<string> progress) {
+            return Task.FromResult(_sRepoCatalog);
         }
 
         public IPackageCatalog MostRecentlyLoadedCatalog {
diff --git a/Nodejs/Product/Npm/SPI/NpmGetCatalogCommand.cs b/Nodejs/Product/Npm/SPI/NpmGetCatalogCommand.cs
@@ -622,7 +622,7 @@ private async Task<string> UpdatePackageCache(Uri registry, string cachePath, lo
 
         public IPackageCatalog Catalog { get { return this; } }
 
-        public async Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText, Uri registryUrl = null) {
+        public async Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText) {
             IEnumerable<IPackage> packages = null;
             using (var semaphore = GetDatabaseSemaphore()) {
                 // Wait until file is downloaded/parsed if another download is already in session.
@@ -635,7 +635,7 @@ public async Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterTe
                 }
 
                 try {
-                    registryUrl = registryUrl ?? await GetRegistryUrl();
+                    var registryUrl = await GetRegistryUrl();
                     RegistryFileMapping registryFileMapping = null;
                     using (var db = new SQLiteConnection(DatabaseCacheFilePath)) {
                         registryFileMapping = db.Table<RegistryFileMapping>().FirstOrDefault(info => info.RegistryUrl == registryUrl.ToString());
diff --git a/Nodejs/Tests/NpmTests/MockPackageCatalog.cs b/Nodejs/Tests/NpmTests/MockPackageCatalog.cs
@@ -36,7 +36,7 @@ public MockPackageCatalog(IList<IPackage> results) {
 
         public DateTime LastRefreshed { get; private set; }
 
-        public Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText, Uri registryUrl = null) {
+        public Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText) {
             return Task.FromResult(_results.AsEnumerable());
         }
 
diff --git a/Nodejs/Tests/NpmTests/NpmSearchTests.cs b/Nodejs/Tests/NpmTests/NpmSearchTests.cs
@@ -115,7 +115,7 @@ private void CheckPackage(
         private IList<IPackage> GetTestPackageList(
             string cachePath,
             out IDictionary<string, IPackage> byName) {
-            IList<IPackage> target = new NpmGetCatalogCommand(string.Empty, cachePath, false, RegistryUrl).GetCatalogPackagesAsync(string.Empty, new Uri(RegistryUrl)).GetAwaiter().GetResult().ToList();
+            IList<IPackage> target = new NpmGetCatalogCommand(string.Empty, cachePath, false, RegistryUrl).GetCatalogPackagesAsync(string.Empty).GetAwaiter().GetResult().ToList();
 
             //  Do this after because package names can be split across multiple
             //  lines and therefore may change after the IPackage is initially created.
@@ -133,7 +133,7 @@ private IPackageCatalog GetTestPackageCatalog(string filename) {
             return new MockPackageCatalog(GetTestPackageList(filename, out byName));
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckDatabaseCreation() {
             string databaseFilename = NpmGetCatalogCommand.DatabaseCacheFilename;
             string registryFilename = NpmGetCatalogCommand.RegistryCacheFilename;
@@ -175,7 +175,7 @@ public void CheckDatabaseCreation() {
                 );
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckDatabaseUpdate() {
             string cachePath = "NpmCacheUpdate";
             string registryPath = Path.Combine(cachePath, "registry", NpmGetCatalogCommand.RegistryCacheFilename);
@@ -226,7 +226,7 @@ public void CheckDatabaseUpdate() {
 
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckDatabaseUpdateArray() {
             string cachePath = "NpmCacheUpdate";
             string registryPath = Path.Combine(cachePath, "registry", NpmGetCatalogCommand.RegistryCacheFilename);
@@ -277,7 +277,7 @@ public void CheckDatabaseUpdateArray() {
 
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckPackageWithBuildPreReleaseInfo() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
@@ -359,35 +359,35 @@ private void CheckOnlyOneOfEachPackage(IEnumerable<IPackage> target) {
             }
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckNonZeroPackageVersionsExist() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
             CheckSensibleNumberOfNonZeroVersions(target);
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckCorrectPackageCount() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
             Assert.AreEqual(89924, target.Count, "Unexpected package count in catalogue list.");
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckNoDuplicatePackages() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
             CheckOnlyOneOfEachPackage(target);
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckListAndDictByNameSameSize() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
             Assert.AreEqual(target.Count, byName.Count, "Number of packages should be same in list and dictionary.");
         }
         
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckFirstPackageInCatalog() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
@@ -406,7 +406,7 @@ public void CheckFirstPackageInCatalog() {
                 Enumerable.Empty<string>());
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckLastPackageInCatalog_zzz() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
@@ -425,7 +425,7 @@ public void CheckLastPackageInCatalog_zzz() {
                 Enumerable.Empty<string>());
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckPackageEqualsInDescription() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
@@ -444,7 +444,7 @@ public void CheckPackageEqualsInDescription() {
                 Enumerable.Empty<string>());
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckPackageNoDescriptionAuthorVersion() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);
@@ -462,7 +462,7 @@ public void CheckPackageNoDescriptionAuthorVersion() {
                 Enumerable.Empty<string>());
         }
 
-        [TestMethod, Priority(0)]
+        //[TestMethod, Priority(0)]
         public void CheckPackageNoDescription() {
             IDictionary<string, IPackage> byName;
             var target = GetTestPackageList(PackageCacheDirectory, out byName);

Original file line number	Diff line number	Diff line change
`@@ -144,6 +144,13 @@ public IEnumerable<string> Keywords {`
`144`	`144`	`}`
`145`	`145`	`}`
`146`	`146`
	`147`	`+ public void AddVersion(SemverVersion version) {`
	`148`	`+ if( version > this.LatestVersion) {`
	`149`	`+ this.LatestVersion = version;`
	`150`	`+ }`
	`151`	`+ this._availableVersions.Add(version);`
	`152`	`+ }`
	`153`	`+`
`147`	`154`	`public IPackage Build() {`
`148`	`155`	`var proxy = new PackageProxy {`
`149`	`156`	`Author = Author,`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ public MockPackageCatalog(IList<IPackage> results) {`
`36`	`36`
`37`	`37`	`public DateTime LastRefreshed { get; private set; }`
`38`	`38`
`39`		`- public Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText, Uri registryUrl = null) {`
	`39`	`+ public Task<IEnumerable<IPackage>> GetCatalogPackagesAsync(string filterText) {`
`40`	`40`	`return Task.FromResult(_results.AsEnumerable());`
`41`	`41`	`}`
`42`	`42`