diff --git a/PackageIndexer/DotnetPackageIndex.cs b/PackageIndexer/DotnetPackageIndex.cs index 154636ef..8af4919c 100644 --- a/PackageIndexer/DotnetPackageIndex.cs +++ b/PackageIndexer/DotnetPackageIndex.cs @@ -73,20 +73,305 @@ params string[] feedUrls private static async Task> GetPackagesFromNuGetOrgAsync(NuGetFeed feed) { Console.WriteLine("Fetching owner information..."); - Dictionary ownerInformation = await feed.GetOwnerMappingAsync(); - string[] packageIds = ownerInformation.Keys - .ToHashSet(StringComparer.OrdinalIgnoreCase) - .Where(id => IsOwnedByDotNet(ownerInformation, id) && - PackageFilter.Default.IsMatch(id)) - .ToArray(); + // TODO + // 9/11/24 - Commented out until I can get owner info from Kusto. + //Dictionary ownerInformation = await feed.GetOwnerMappingAsync(); + + //string[] packageIds = ownerInformation.Keys + // .ToHashSet(StringComparer.OrdinalIgnoreCase) + // .Where(id => IsOwnedByDotNet(ownerInformation, id) && + // PackageFilter.Default.IsMatch(id)) + // .ToArray(); + + // Temporarily hard-code the .NET package IDs. + string[] packageIds = [ + "Microsoft.Bcl.AsyncInterfaces", + "Microsoft.Bcl.Build", + "Microsoft.Bcl.Cryptography", + "Microsoft.Bcl.HashCode", + "Microsoft.Bcl.Numerics", + "Microsoft.Bcl.TimeProvider", + "Microsoft.Extensions.AmbientMetadata.Application", + "Microsoft.Extensions.ApiDescription.Client", + "Microsoft.Extensions.ApiDescription.Server", + "Microsoft.Extensions.AsyncState", + "Microsoft.Extensions.AuditReports", + "Microsoft.Extensions.Caching.Abstractions", + "Microsoft.Extensions.Caching.Hybrid", + "Microsoft.Extensions.Caching.Memory", + "Microsoft.Extensions.Caching.SqlServer", + "Microsoft.Extensions.Caching.StackExchangeRedis", + "Microsoft.Extensions.Compliance.Abstractions", + "Microsoft.Extensions.Compliance.Redaction", + "Microsoft.Extensions.Compliance.Testing", + "Microsoft.Extensions.Configuration", + "Microsoft.Extensions.Configuration.Abstractions", + "Microsoft.Extensions.Configuration.Binder", + "Microsoft.Extensions.Configuration.CommandLine", + "Microsoft.Extensions.Configuration.EnvironmentVariables", + "Microsoft.Extensions.Configuration.FileExtensions", + "Microsoft.Extensions.Configuration.Ini", + "Microsoft.Extensions.Configuration.Json", + "Microsoft.Extensions.Configuration.KeyPerFile", + "Microsoft.Extensions.Configuration.UserSecrets", + "Microsoft.Extensions.Configuration.Xml", + "Microsoft.Extensions.DependencyInjection", + "Microsoft.Extensions.DependencyInjection.Abstractions", + "Microsoft.Extensions.DependencyInjection.AutoActivation", + "Microsoft.Extensions.DependencyInjection.Specification.Tests", + "Microsoft.Extensions.DependencyModel", + "Microsoft.Extensions.DiagnosticAdapter", + "Microsoft.Extensions.Diagnostics", + "Microsoft.Extensions.Diagnostics.Abstractions", + "Microsoft.Extensions.Diagnostics.ExceptionSummarization", + "Microsoft.Extensions.Diagnostics.HealthChecks", + "Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions", + "Microsoft.Extensions.Diagnostics.HealthChecks.Common", + "Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore", + "Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization", + "Microsoft.Extensions.Diagnostics.Probes", + "Microsoft.Extensions.Diagnostics.ResourceMonitoring", + "Microsoft.Extensions.Diagnostics.Testing", + "Microsoft.Extensions.FileProviders.Abstractions", + "Microsoft.Extensions.FileProviders.Composite", + "Microsoft.Extensions.FileProviders.Embedded", + "Microsoft.Extensions.FileProviders.Physical", + "Microsoft.Extensions.FileSystemGlobbing", + "Microsoft.Extensions.Hosting", + "Microsoft.Extensions.Hosting.Abstractions", + "Microsoft.Extensions.Hosting.Systemd", + "Microsoft.Extensions.Hosting.Testing", + "Microsoft.Extensions.Hosting.WindowsServices", + "Microsoft.Extensions.Http", + "Microsoft.Extensions.Http.Diagnostics", + "Microsoft.Extensions.Http.Polly", + "Microsoft.Extensions.Http.Resilience", + "Microsoft.Extensions.Localization", + "Microsoft.Extensions.Localization.Abstractions", + "Microsoft.Extensions.Logging", + "Microsoft.Extensions.Logging.Abstractions", + "Microsoft.Extensions.Logging.AzureAppServices", + "Microsoft.Extensions.Logging.Configuration", + "Microsoft.Extensions.Logging.Console", + "Microsoft.Extensions.Logging.Debug", + "Microsoft.Extensions.Logging.EventLog", + "Microsoft.Extensions.Logging.EventSource", + "Microsoft.Extensions.Logging.TraceSource", + "Microsoft.Extensions.ObjectPool", + "Microsoft.Extensions.ObjectPool.DependencyInjection", + "Microsoft.Extensions.Options", + "Microsoft.Extensions.Options.ConfigurationExtensions", + "Microsoft.Extensions.Options.Contextual", + "Microsoft.Extensions.Options.DataAnnotations", + "Microsoft.Extensions.Primitives", + "Microsoft.Extensions.Resilience", + "Microsoft.Extensions.StaticAnalysis", + "Microsoft.Extensions.Telemetry", + "Microsoft.Extensions.Telemetry.Abstractions", + "Microsoft.Extensions.TimeProvider.Testing", + "Microsoft.Extensions.WebEncoders", + "Microsoft.Win32.Primitives", + "Microsoft.Win32.Registry", + "Microsoft.Win32.Registry.AccessControl", + "Microsoft.Win32.SystemEvents", + "System.AppContext", + "System.Buffers", + "System.CodeDom", + "System.Collections", + "System.Collections.Concurrent", + "System.Collections.Immutable", + "System.Collections.NonGeneric", + "System.Collections.Specialized", + "System.ComponentModel", + "System.ComponentModel.Annotations", + "System.ComponentModel.Composition", + "System.ComponentModel.Composition.Registration", + "System.ComponentModel.EventBasedAsync", + "System.ComponentModel.Primitives", + "System.ComponentModel.TypeConverter", + "System.Composition", + "System.Composition.AttributedModel", + "System.Composition.Convention", + "System.Composition.Hosting", + "System.Composition.Runtime", + "System.Composition.TypedParts", + "System.Configuration.ConfigurationManager", + "System.Console", + "System.Data.Common", + "System.Data.DataSetExtensions", + "System.Data.Odbc", + "System.Data.OleDb", + "System.Data.SqlClient", + "System.Diagnostics.Contracts", + "System.Diagnostics.Debug", + "System.Diagnostics.DiagnosticSource", + "System.Diagnostics.EventLog", + "System.Diagnostics.FileVersionInfo", + "System.Diagnostics.PerformanceCounter", + "System.Diagnostics.Process", + "System.Diagnostics.StackTrace", + "System.Diagnostics.TextWriterTraceListener", + "System.Diagnostics.Tools", + "System.Diagnostics.TraceSource", + "System.Diagnostics.Tracing", + "System.DirectoryServices", + "System.DirectoryServices.AccountManagement", + "System.DirectoryServices.Protocols", + "System.Drawing.Common", + "System.Drawing.Primitives", + "System.Dynamic.Runtime", + "System.Formats.Asn1", + "System.Formats.Cbor", + "System.Formats.Nrbf", + "System.Globalization", + "System.Globalization.Calendars", + "System.Globalization.Extensions", + "System.IO", + "System.IO.Compression", + "System.IO.Compression.clrcompression-arm", + "System.IO.Compression.clrcompression-x64", + "System.IO.Compression.clrcompression-x86", + "System.IO.Compression.ZipFile", + "System.IO.FileSystem", + "System.IO.FileSystem.AccessControl", + "System.IO.FileSystem.DriveInfo", + "System.IO.FileSystem.Primitives", + "System.IO.FileSystem.Watcher", + "System.IO.Hashing", + "System.IO.IsolatedStorage", + "System.IO.MemoryMappedFiles", + "System.IO.Packaging", + "System.IO.Pipelines", + "System.IO.Pipes", + "System.IO.Pipes.AccessControl", + "System.IO.Ports", + "System.IO.UnmanagedMemoryStream", + "System.Json", + "System.Linq", + "System.Linq.Expressions", + "System.Linq.Parallel", + "System.Linq.Queryable", + "System.Management", + "System.Memory", + "System.Memory.Data", + "System.Net.Http", + "System.Net.Http.Json", + "System.Net.Http.Rtc", + "System.Net.Http.WinHttpHandler", + "System.Net.NameResolution", + "System.Net.NetworkInformation", + "System.Net.Ping", + "System.Net.Primitives", + "System.Net.Requests", + "System.Net.Security", + "System.Net.ServerSentEvents", + "System.Net.Sockets", + "System.Net.WebHeaderCollection", + "System.Net.WebSockets", + "System.Net.WebSockets.Client", + "System.Net.WebSockets.WebSocketProtocol", + "System.Numerics.Tensors", + "System.Numerics.Vectors", + "System.Numerics.Vectors.WindowsRuntime", + "System.ObjectModel", + "System.Private.DataContractSerialization", + "System.Private.Networking", + "System.Private.Uri", + "System.Reflection", + "System.Reflection.Context", + "System.Reflection.DispatchProxy", + "System.Reflection.Emit", + "System.Reflection.Emit.ILGeneration", + "System.Reflection.Emit.Lightweight", + "System.Reflection.Extensions", + "System.Reflection.Metadata", + "System.Reflection.MetadataLoadContext", + "System.Reflection.Primitives", + "System.Reflection.TypeExtensions", + "System.Resources.Extensions", + "System.Resources.Reader", + "System.Resources.ResourceManager", + "System.Resources.Writer", + "System.Runtime", + "System.Runtime.Caching", + "System.Runtime.CompilerServices.Unsafe", + "System.Runtime.CompilerServices.VisualC", + "System.Runtime.Experimental", + "System.Runtime.Extensions", + "System.Runtime.Handles", + "System.Runtime.InteropServices", + "System.Runtime.InteropServices.NFloat.Internal", + "System.Runtime.InteropServices.RuntimeInformation", + "System.Runtime.InteropServices.WindowsRuntime", + "System.Runtime.Loader", + "System.Runtime.Numerics", + "System.Runtime.Serialization.Formatters", + "System.Runtime.Serialization.Json", + "System.Runtime.Serialization.Primitives", + "System.Runtime.Serialization.Schema", + "System.Runtime.Serialization.Xml", + "System.Security.AccessControl", + "System.Security.Claims", + "System.Security.Cryptography.Algorithms", + "System.Security.Cryptography.Cng", + "System.Security.Cryptography.Cose", + "System.Security.Cryptography.Csp", + "System.Security.Cryptography.Encoding", + "System.Security.Cryptography.OpenSsl", + "System.Security.Cryptography.Pkcs", + "System.Security.Cryptography.Primitives", + "System.Security.Cryptography.ProtectedData", + "System.Security.Cryptography.X509Certificates", + "System.Security.Cryptography.Xml", + "System.Security.Permissions", + "System.Security.Principal", + "System.Security.Principal.Windows", + "System.Security.SecureString", + "System.ServiceModel.Duplex", + "System.ServiceModel.Federation", + "System.ServiceModel.Http", + "System.ServiceModel.NetFramingBase", + "System.ServiceModel.NetNamedPipe", + "System.ServiceModel.NetTcp", + "System.ServiceModel.Primitives", + "System.ServiceModel.Security", + "System.ServiceModel.Syndication", + "System.ServiceModel.UnixDomainSocket", + "System.ServiceProcess.ServiceController", + "System.Speech", + "System.Text.Encoding", + "System.Text.Encoding.CodePages", + "System.Text.Encoding.Extensions", + "System.Text.Encodings.Web", + "System.Text.Json", + "System.Text.RegularExpressions", + "System.Threading", + "System.Threading.AccessControl", + "System.Threading.Channels", + "System.Threading.Overlapped", + "System.Threading.Tasks", + "System.Threading.Tasks.Dataflow", + "System.Threading.Tasks.Extensions", + "System.Threading.Tasks.Parallel", + "System.Threading.Thread", + "System.Threading.ThreadPool", + "System.Threading.Timer", + "System.ValueTuple", + "System.Web.Http.Common", + "System.Web.Services.Description", + "System.Windows.Extensions", + "System.Xml.ReaderWriter", + "System.Xml.XDocument", + "System.Xml.XmlDocument", + "System.Xml.XmlSerializer", + "System.Xml.XPath", + "System.Xml.XPath.XDocument", + "System.Xml.XPath.XmlDocument" + ]; Console.WriteLine($"Found {packageIds.Length:N0} package IDs owned by .NET."); - // Are these packages already filtered somehow? Yes - see PackageFilter.cs. - // https://www.nuget.org/packages/Microsoft.NETCore.App.Ref - // https://www.nuget.org/packages/Microsoft.WindowsDesktop.App.Ref - Console.WriteLine("Getting versions..."); ConcurrentBag<(PackageIdentity, bool)> identities = []; @@ -134,6 +419,8 @@ bool usePreviewVersions if (latestPrerelease.isDeprecated) latestPrerelease = default; + // If the latest stable version is newer than the latest + // prerelease version, don't include the prerelease version. if (latestStable != default && latestPrerelease != default) { bool stableIsNewer = VersionComparer.VersionReleaseMetadata.Compare( diff --git a/PackageIndexer/NuGet/NuGetFeed.cs b/PackageIndexer/NuGet/NuGetFeed.cs index 24550e13..b19eed15 100644 --- a/PackageIndexer/NuGet/NuGetFeed.cs +++ b/PackageIndexer/NuGet/NuGetFeed.cs @@ -241,8 +241,9 @@ public Task> GetOwnerMappingAsync() throw new NotSupportedException("We can only retrieve owner information for nuget.org"); var httpClient = new HttpClient(); + // TODO - this URL is no longer accessible. string url = "https://nugetprodusncazuresearch.blob.core.windows.net/v3-azuresearch-017/owners/owners.v2.json"; - return httpClient.GetFromJsonAsync>(url); + return httpClient.GetFromJsonAsync>(url)!; } private static bool TryGetAzureDevOpsFeed(string url, out string organization, out string project, out string feed) diff --git a/PackageIndexer/PackageIndexer.csproj b/PackageIndexer/PackageIndexer.csproj index 763b72e9..7b9d5341 100644 --- a/PackageIndexer/PackageIndexer.csproj +++ b/PackageIndexer/PackageIndexer.csproj @@ -11,6 +11,7 @@ + diff --git a/PackageIndexer/Program.cs b/PackageIndexer/Program.cs index acd1b7b3..599f4216 100644 --- a/PackageIndexer/Program.cs +++ b/PackageIndexer/Program.cs @@ -82,7 +82,7 @@ internal static class Program private static async Task Main(string[] args) { #if DEBUG - args = [@"c:\users\gewarren\desktop\Package Index 0814", "preview"]; + args = [@"c:\users\gewarren\desktop\Package Index 0911b", "preview"]; #endif if ((args.Length == 0) || (args.Length > 2)) @@ -265,6 +265,14 @@ FrameworkEntry tfm StringComparison.InvariantCultureIgnoreCase ); + // Except don't include XML file for Microsoft.Extensions.Diagnostics.ResourceMonitoring + // See https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787. + if (string.Equals( + packageEntry.Name, + "Microsoft.Extensions.Diagnostics.ResourceMonitoring", + StringComparison.InvariantCultureIgnoreCase)) + includeXml = false; + // Special case for newer assemblies - include XML documentation files. if (s_packagesWithTruthDocs.Contains(packageEntry.Name)) includeXml = true;