Fix Cloudsmith query (#315)

***NO_CI***

Author: josesimoes

nanoFirmwareFlasher.Library/FirmwarePackage.cs

@@ -147,74 +147,75 @@ protected FirmwarePackage(
         /// <param name="verbosity">VerbosityLevel to use when outputting progress and error messages.</param>
         /// <returns>List of <see cref="CloudSmithPackageDetail"/> with details on target firmware packages.</returns>
         public static List<CloudSmithPackageDetail> GetTargetList(
-            bool communityTargets,
-            bool preview,
-            SupportedPlatform? platform,
-            VerbosityLevel verbosity)
+    bool communityTargets,
+    bool preview,
+    SupportedPlatform? platform,
+    VerbosityLevel verbosity)
         {
             string repoName = communityTargets ? CommunityTargetsRepo : preview ? RefTargetsDevRepo : RefTargetsStableRepo;
 
             // NOTE: the query seems to be the opposite, it should be LESS THAN.
             // this has been reported to Cloudsmith and it's being checked. Maybe need to revisit this if changes are made in their API.
             // Because new stable releases are published on a regular basis and preview very rarely, we query for stable versions published in past month and preview versions published during the past 6 months.
-            string requestUri = $"{repoName}/?page_size=500&q=uploaded:'>{(preview ? "6" : "1")} month ago' {(platform.HasValue ? "AND tag:" + platform.Value : "")}";
+            // This is a paged query with page size 100 to improve performance.
+            string requestUri = $"{repoName}/?page_size=100&q=uploaded:'>{(preview ? "6" : "1")} month ago' {(platform.HasValue ? "AND tag:" + platform.Value : "")}";
 
             List<CloudSmithPackageDetail> targetPackages = [];
+            int page = 1;
+            bool morePages = true;
 
-            if (verbosity > VerbosityLevel.Normal)
+            while (morePages)
             {
-                OutputWriter.ForegroundColor = ConsoleColor.White;
-
-                OutputWriter.Write($"Listing {platform} targets from '{repoName}' repository");
-
-                if (!communityTargets)
+                if (verbosity > VerbosityLevel.Normal)
                 {
-                    if (preview)
-                    {
-                        OutputWriter.Write(" [PREVIEW]");
-                    }
-                    else
+                    OutputWriter.ForegroundColor = ConsoleColor.White;
+                    OutputWriter.Write($"Listing {platform} targets from '{repoName}' repository, page {page}...");
+
+                    if (!communityTargets)
                     {
-                        OutputWriter.Write(" [STABLE]");
+                        if (preview)
+                        {
+                            OutputWriter.Write(" [PREVIEW]");
+                        }
+                        else
+                        {
+                            OutputWriter.Write(" [STABLE]");
+                        }
                     }
-                }
 
-                OutputWriter.WriteLine("...");
-            }
-
-            HttpResponseMessage response = s_cloudsmithClient.GetAsync(requestUri).GetAwaiter().GetResult();
+                    OutputWriter.WriteLine("...");
+                }
 
-            string responseBody = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+                HttpResponseMessage response = s_cloudsmithClient.GetAsync($"{requestUri}&page={page}").GetAwaiter().GetResult();
+                string responseBody = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
 
-            // check for empty array 
-            if (responseBody == "[]")
-            {
-                if (verbosity > VerbosityLevel.Quiet)
+                if (responseBody == "[]" || responseBody.Contains("\"Invalid page.\""))
                 {
-                    OutputWriter.WriteLine("");
+                    morePages = false;
+                    continue;
                 }
 
-                // can't find this target
-                return targetPackages;
+                var options = new JsonSerializerOptions
+                {
+                    PropertyNameCaseInsensitive = true,
+                };
+
+                List<CloudSmithPackageDetailJson> deserializedPackages = JsonSerializer.Deserialize<List<CloudSmithPackageDetailJson>>(responseBody, options);
+                targetPackages.AddRange(from p in deserializedPackages
+                                        select new CloudSmithPackageDetail()
+                                        {
+                                            Name = p.Name,
+                                            Version = p.Version,
+                                            Platform = p.Tags?.Info is null
+                                                ? null
+                                                : (from t in p.Tags.Info
+                                                   where s_supportedPlatforms.Contains(t)
+                                                   select t).FirstOrDefault(),
+                                        });
+
+                page++;
             }
 
-            var options = new JsonSerializerOptions
-            {
-                PropertyNameCaseInsensitive = true,
-            };
-
-            List<CloudSmithPackageDetailJson> deserializedPackages = JsonSerializer.Deserialize<List<CloudSmithPackageDetailJson>>(responseBody, options);
-            targetPackages.AddRange(from p in deserializedPackages
-                                    select new CloudSmithPackageDetail()
-                                    {
-                                        Name = p.Name,
-                                        Version = p.Version,
-                                        Platform = p.Tags?.Info is null
-                                            ? null
-                                            : (from t in p.Tags.Info
-                                               where s_supportedPlatforms.Contains(t)
-                                               select t).FirstOrDefault(),
-                                    });
             return targetPackages;
         }
 

nanoFirmwareFlasher.Tests/FirmwarePackageTests.cs

@@ -23,17 +23,28 @@ public void FirmwarePackage_ListReferenceTargets()
             using var output = new OutputWriterHelper();
 
             #region Get the stable packages
-            List<CloudSmithPackageDetail> stable = FirmwarePackage.GetTargetList(false, false, null, VerbosityLevel.Diagnostic);
+
+            List<CloudSmithPackageDetail> stable = FirmwarePackage.GetTargetList(
+                false,
+                false,
+                null,
+                VerbosityLevel.Diagnostic);
+
             Assert.IsNotNull(stable);
             Assert.AreNotEqual(0, stable.Count);
-            output.AssertAreEqual("Listing  targets from 'nanoframework-images' repository [STABLE]...");
+
             #endregion
 
             #region Get the preview packages
+
             output.Reset();
-            List<CloudSmithPackageDetail> preview = FirmwarePackage.GetTargetList(false, true, null, VerbosityLevel.Quiet);
+            List<CloudSmithPackageDetail> preview = FirmwarePackage.GetTargetList(
+                false,
+                true,
+                null,
+                VerbosityLevel.Quiet);
+
             Assert.IsNotNull(preview);
-            output.AssertAreEqual("");
 
             // Assert that the preview packages are not part of the stable package list
             foreach (CloudSmithPackageDetail previewPackage in preview)
@@ -42,25 +53,31 @@ public void FirmwarePackage_ListReferenceTargets()
                                 where s.Name == previewPackage.Name && s.Version == previewPackage.Version
                                 select s).Any());
             }
+
             #endregion
 
             #region Get the stable esp32 packages
+
             output.Reset();
-            List<CloudSmithPackageDetail> stableEsp32 = FirmwarePackage.GetTargetList(false, false, SupportedPlatform.esp32, VerbosityLevel.Diagnostic);
+            List<CloudSmithPackageDetail> stableEsp32 = FirmwarePackage.GetTargetList(
+                false,
+                false,
+                SupportedPlatform.esp32,
+                VerbosityLevel.Diagnostic);
+
             Assert.IsNotNull(stableEsp32);
             Assert.AreNotEqual(0, stableEsp32.Count);
-            output.AssertAreEqual("Listing esp32 targets from 'nanoframework-images' repository [STABLE]...");
 
             // Assert that there are more stable packages than for the esp32
             Assert.IsTrue(stableEsp32.Count < stable.Count);
 
             // Assert that all esp32 packages are in the stable list
             foreach (CloudSmithPackageDetail esp32Package in stableEsp32)
             {
-                Assert.IsTrue((from s in stable
-                               where s.Name == esp32Package.Name && s.Version == esp32Package.Version
-                               select s).Any());
+                Assert.IsTrue(stable.Any(s => s.Name == esp32Package.Name && s.Version == esp32Package.Version),
+                              $"Package {esp32Package.Name} with version {esp32Package.Version} is not in the stable list.");
             }
+
             #endregion
         }
 
@@ -74,7 +91,7 @@ public void FirmwarePackage_ListCommunityTargets()
             List<CloudSmithPackageDetail> stable = FirmwarePackage.GetTargetList(true, false, null, VerbosityLevel.Diagnostic);
             Assert.IsNotNull(stable);
             Assert.AreNotEqual(0, stable.Count);
-            output.AssertAreEqual("Listing  targets from 'nanoframework-images-community-targets' repository...");
+            
             #endregion
 
             #region Get the stable esp32 packages