[PackageIndexer] Special case two AI packages (#470)

Author: gewarren

PackageIndexer/CsvUtils.cs

@@ -159,6 +159,7 @@ string frameworkName
             // Special case for packages from dotnet/extensions repo - include XML files.
             string[] reposToIncludeXmlComments = [
                 "https://github.com/dotnet/extensions",
+                "https://github.com/microsoft/semantic-kernel",
                 "https://devdiv.visualstudio.com/DevDiv/_git/AITestingTools"
                 ];
 

PackageIndexer/DotnetPackageIndex.cs

@@ -94,6 +94,9 @@ params string[] feedUrls
                 packageIds.Add(packageId);
         }
 
+        // Special case since not owned by dotnetframework (yet)
+        packageIds.Add("Microsoft.Extensions.VectorData.Abstractions");
+
         Console.WriteLine($"Found {packageIds.Count:N0} package IDs owned by .NET.");
 
         Console.WriteLine("Getting versions...");
@@ -163,8 +166,12 @@ bool usePreviewVersions
 
             if (usePreviewVersions)
             {
+                // TODO - this seems clunky.
                 // Make sure it's a .NET 9 preview version.
-                if ((latestPrerelease != default) && latestPrerelease.packageId.Version.Major == 9)
+                if (latestPrerelease != default &&
+                    (latestPrerelease.packageId.Version.Major == 9 ||
+                    // Special case for major version (0) of Microsoft.Extensions.AI.Evaluation.
+                    latestPrerelease.packageId.Id.StartsWith("Microsoft.Extensions.AI.Evaluation")))
                     result.Add(latestPrerelease.packageId);
             }
         }

PackageIndexer/NuGet/NuGetStore.cs

@@ -7,7 +7,6 @@ namespace PackageIndexer;
 public class NuGetStore
 {
     private readonly NuGetFeed[] _feeds;
-    private readonly Dictionary<string, IReadOnlyList<NuGetVersion>> _packageVersionCache = new();
 
     public NuGetStore(string packagesCachePath, params NuGetFeed[] feeds)
     {

PackageIndexer/PackageFilter.cs

@@ -72,46 +72,4 @@ public bool IsMatch(string packageId)
         return Includes.Any(e => e.IsMatch(packageId)) &&
                !Excludes.Any(e => e.IsMatch(packageId));
     }
-
-    public static PackageFilter Default { get; } = new(
-        includes:
-        [
-            PackageFilterExpression.Parse("Microsoft.Bcl.*"),
-            PackageFilterExpression.Parse("Microsoft.Extensions.*"),
-            PackageFilterExpression.Parse("Microsoft.IO.Redist"),
-            PackageFilterExpression.Parse("Microsoft.Win32.*"),
-            PackageFilterExpression.Parse("System.*"),
-        ],
-        excludes:
-        [
-            PackageFilterExpression.Parse("System.Private.ServiceModel"),
-            PackageFilterExpression.Parse("System.Runtime.WindowsRuntime"),
-            PackageFilterExpression.Parse("System.Runtime.WindowsRuntime.UI.Xaml"),
-            // Documented under Azure SDK for .NET moniker.
-            PackageFilterExpression.Parse("System.ClientModel"),
-            // Documented under ASP.NET Core moniker.
-            PackageFilterExpression.Parse("System.Threading.RateLimiting"),
-            PackageFilterExpression.Parse("Microsoft.Extensions.Features"),
-            PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Core"),
-            PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Stores"),
-            // Documented under ML.NET moniker.
-            PackageFilterExpression.Parse("Microsoft.Extensions.ML"),
-            // Documented under .NET Aspire moniker.
-            PackageFilterExpression.Parse("Microsoft.Extensions.ServiceDiscovery*"),
-            // Suffixes.
-            PackageFilterExpression.Parse("*.cs"),
-            PackageFilterExpression.Parse("*.de"),
-            PackageFilterExpression.Parse("*.es"),
-            PackageFilterExpression.Parse("*.fr"),
-            PackageFilterExpression.Parse("*.it"),
-            PackageFilterExpression.Parse("*.ja"),
-            PackageFilterExpression.Parse("*.ko"),
-            PackageFilterExpression.Parse("*.pl"),
-            PackageFilterExpression.Parse("*.pt-br"),
-            PackageFilterExpression.Parse("*.ru"),
-            PackageFilterExpression.Parse("*.tr"),
-            PackageFilterExpression.Parse("*.zh-Hans"),
-            PackageFilterExpression.Parse("*.zh-Hant"),
-        ]
-    );
 }

PackageIndexer/PlatformPackageDefinition.cs

@@ -61,8 +61,7 @@ internal static class PlatformPackageDefinition
         "System.Formats.Nrbf",
         "System.Net.ServerSentEvents",
         "System.Numerics.Tensors",
-        "System.Runtime.Serialization.Schema",
-        "System.Speech"
+        "System.Runtime.Serialization.Schema"
     ];
 
     public static FrozenSet<string> Owners = FrozenSet.ToFrozenSet(

PackageIndexer/Program.cs

@@ -8,7 +8,7 @@ internal static class Program
     private static async Task<int> Main(string[] args)
     {
 #if DEBUG
-        args = [@"c:\users\gewarren\desktop\Package Index 1106", "preview"];
+        args = [@"c:\users\gewarren\desktop\Package Index 0130", "preview"];
 #endif
 
         if ((args.Length == 0) || (args.Length > 2))
@@ -101,9 +101,9 @@ string indexPackagesPath
             string disabledPath = Path.Join(indexPackagesPath, $"{id}-all.disabled");
             string failedVersionPath = Path.Join(indexPackagesPath, $"{id}-{version}.failed");
 
-            bool alreadyIndexed = !retryIndexed && File.Exists(path) ||
-                                 !retryDisabled && File.Exists(disabledPath) ||
-                                 !retryFailed && File.Exists(failedVersionPath);
+            bool alreadyIndexed = (!retryIndexed && File.Exists(path)) ||
+                                 (!retryDisabled && File.Exists(disabledPath)) ||
+                                 (!retryFailed && File.Exists(failedVersionPath));
 
             if (alreadyIndexed)
             {