Added basic version of filter by manifest

darrelmiller · darrelmiller · commit 86262b6259fe · 2023-05-14T22:35:59.000-04:00
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -5,18 +5,36 @@
             // Use IntelliSense to find out which attributes exist for C# debugging
             // Use hover for the description of the existing attributes
             // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
-            "name": ".NET Core Launch (console)",
+            "name": "Launch Hidi",
             "type": "coreclr",
             "request": "launch",
             "preLaunchTask": "build",
             // If you have changed target frameworks, make sure to update the program path.
             "program": "${workspaceFolder}/src/Microsoft.OpenApi.Hidi/bin/Debug/net7.0/Microsoft.OpenApi.Hidi.dll",
-            "args": [],
+            "args": ["transform", 
+                    "-m","C:\\Users\\darrmi\\src\\github\\microsoft\\openapi.net\\test\\Microsoft.OpenApi.Hidi.Tests\\UtilityFiles\\exampleapimanifest.json", 
+                    "-o","minimoose.yaml"],
             "cwd": "${workspaceFolder}/src/Microsoft.OpenApi.Hidi",
             // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
             "console": "internalConsole",
             "stopAtEntry": false
         },
+        {
+            // Use IntelliSense to find out which attributes exist for C# debugging
+            // Use hover for the description of the existing attributes
+            // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+            "name": "Launch Workbench",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/src/Microsoft.OpenApi.WorkBench/bin/Debug/net7.0-windows/Microsoft.OpenApi.Workbench.exe",
+            "args": [],
+            "cwd": "${workspaceFolder}/src/Microsoft.OpenApi.Workbench",
+            // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+            "console": "internalConsole",
+            "stopAtEntry": false
+        },        
         {
             "name": ".NET Core Attach",
             "type": "coreclr",
diff --git a/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj b/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj
@@ -51,6 +51,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.OpenApi.Readers\Microsoft.OpenApi.Readers.csproj" />
     <ProjectReference Include="..\Microsoft.OpenApi\Microsoft.OpenApi.csproj" />
+    <ProjectReference Include="..\..\..\OpenApi.ApiManifest\src\lib\apimanifest.csproj" />
   </ItemGroup>
 
   <!-- Make internals available for Unit Testing -->
diff --git a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs
@@ -29,6 +29,8 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.OpenApi.Hidi.Utilities;
 using Microsoft.OpenApi.Hidi.Formatters;
+using Microsoft.OpenApi.ApiManifest;
+using System.Linq;
 
 namespace Microsoft.OpenApi.Hidi
 {
@@ -39,7 +41,7 @@ internal class OpenApiService
         /// </summary>
         public static async Task TransformOpenApiDocument(HidiOptions options, ILogger logger, CancellationToken cancellationToken)
         {
-            if (string.IsNullOrEmpty(options.OpenApi) && string.IsNullOrEmpty(options.Csdl))
+            if (string.IsNullOrEmpty(options.OpenApi) && string.IsNullOrEmpty(options.Csdl) && string.IsNullOrEmpty(options.FilterOptions?.FilterByApiManifest))
             {
                 throw new ArgumentException("Please input a file path or URL");
             }
@@ -65,11 +67,34 @@ public static async Task TransformOpenApiDocument(HidiOptions options, ILogger l
                 OpenApiFormat openApiFormat = options.OpenApiFormat ?? (!string.IsNullOrEmpty(options.OpenApi) ? GetOpenApiFormat(options.OpenApi, logger) : OpenApiFormat.Yaml);
                 OpenApiSpecVersion openApiVersion = options.Version != null ? TryParseOpenApiSpecVersion(options.Version) : OpenApiSpecVersion.OpenApi3_0;
 
+                // If API Manifest is provided, load it, use it get the OpenAPI path
+                ApiManifestDocument apiManifest = null;
+                if (!string.IsNullOrEmpty(options.FilterOptions?.FilterByApiManifest))
+                {
+                    using(var fileStream = await GetStream(options.FilterOptions.FilterByApiManifest, logger, cancellationToken)) {
+                        apiManifest = ApiManifestDocument.Load(JsonDocument.Parse(fileStream).RootElement);
+                    }
+                    options.OpenApi = apiManifest.ApiDependencies[0].ApiDescripionUrl;
+                }
+
+                // If Postman Collection is provided, load it
+                JsonDocument postmanCollection = null;
+                if (!String.IsNullOrEmpty(options.FilterOptions?.FilterByCollection))
+                {
+                    using (var collectionStream = await GetStream(options.FilterOptions.FilterByCollection, logger, cancellationToken)) {
+                        postmanCollection = JsonDocument.Parse(collectionStream);
+                    }
+                }
+
+                // Load OpenAPI document
                 OpenApiDocument document = await GetOpenApi(options.OpenApi, options.Csdl, options.CsdlFilter, options.SettingsConfig, options.InlineExternal, logger, cancellationToken, options.MetadataVersion);
+
                 if (options.FilterOptions != null)
-                    document = await FilterOpenApiDocument(options.FilterOptions.FilterByOperationIds, options.FilterOptions.FilterByTags, options.FilterOptions.FilterByCollection, document, logger, cancellationToken);
+                {
+                    document = ApplyFilters(options, logger, apiManifest, postmanCollection, document, cancellationToken);
+                }
 
-                var languageFormat = options.SettingsConfig.GetSection("LanguageFormat").Value;
+                var languageFormat = options.SettingsConfig?.GetSection("LanguageFormat")?.Value;
                 if (Extensions.StringExtensions.IsEquals(languageFormat, "PowerShell"))
                 {
                     // PowerShell Walker.
@@ -93,6 +118,38 @@ public static async Task TransformOpenApiDocument(HidiOptions options, ILogger l
             }
         }
 
+        private static OpenApiDocument ApplyFilters(HidiOptions options, ILogger logger, ApiManifestDocument apiManifest, JsonDocument postmanCollection, OpenApiDocument document, CancellationToken cancellationToken)
+        {
+            Dictionary<string, List<string>> requestUrls = null;
+            if (apiManifest != null)
+            {
+                requestUrls = GetRequestUrlsFromManifest(apiManifest, document);
+            }
+            else if (postmanCollection != null)
+            {
+                requestUrls = EnumerateJsonDocument(postmanCollection.RootElement, requestUrls);
+                logger.LogTrace("Finished fetching the list of paths and Http methods defined in the Postman collection.");
+            }
+
+
+            logger.LogTrace("Creating predicate from filter options.");
+            var predicate = FilterOpenApiDocument(options.FilterOptions.FilterByOperationIds,
+                                                    options.FilterOptions.FilterByTags,
+                                                    requestUrls,
+                                                    document,
+                                                     logger, cancellationToken);
+            if (predicate != null)
+            {
+                var stopwatch = new Stopwatch();
+                stopwatch.Start();
+                document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
+                stopwatch.Stop();
+                logger.LogTrace("{timestamp}ms: Creating filtered OpenApi document with {paths} paths.", stopwatch.ElapsedMilliseconds, document.Paths.Count);
+            }
+
+            return document;
+        }
+
         private static void WriteOpenApi(FileInfo output, bool terseOutput, bool inlineLocal, bool inlineExternal, OpenApiFormat openApiFormat, OpenApiSpecVersion openApiVersion, OpenApiDocument document, ILogger logger)
         {
             using (logger.BeginScope("Output"))
@@ -163,12 +220,12 @@ private static async Task<OpenApiDocument> GetOpenApi(string openapi, string csd
             return document;
         }
 
-        private static async Task<OpenApiDocument> FilterOpenApiDocument(string filterbyoperationids, string filterbytags, string filterbycollection, OpenApiDocument document, ILogger logger, CancellationToken cancellationToken)
+        private static Func<string, OperationType?, OpenApiOperation, bool> FilterOpenApiDocument(string filterbyoperationids, string filterbytags, Dictionary<string, List<string>> requestUrls, OpenApiDocument document, ILogger logger, CancellationToken cancellationToken)
         {
-            using (logger.BeginScope("Filter"))
-            {
-                Func<string, OperationType?, OpenApiOperation, bool> predicate = null;
+            Func<string, OperationType?, OpenApiOperation, bool> predicate = null;
 
+            using (logger.BeginScope("Create Filter"))
+            {
                 // Check if filter options are provided, then slice the OpenAPI document
                 if (!string.IsNullOrEmpty(filterbyoperationids) && !string.IsNullOrEmpty(filterbytags))
                 {
@@ -186,25 +243,30 @@ private static async Task<OpenApiDocument> FilterOpenApiDocument(string filterby
                     predicate = OpenApiFilterService.CreatePredicate(tags: filterbytags);
 
                 }
-                if (!string.IsNullOrEmpty(filterbycollection))
+                if (requestUrls != null)
                 {
-                    var fileStream = await GetStream(filterbycollection, logger, cancellationToken);
-                    var requestUrls = ParseJsonCollectionFile(fileStream, logger);
-
                     logger.LogTrace("Creating predicate based on the paths and Http methods defined in the Postman collection.");
                     predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source: document);
                 }
-                if (predicate != null)
-                {
-                    var stopwatch = new Stopwatch();
-                    stopwatch.Start();
-                    document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
-                    stopwatch.Stop();
-                    logger.LogTrace("{timestamp}ms: Creating filtered OpenApi document with {paths} paths.", stopwatch.ElapsedMilliseconds, document.Paths.Count);
-                }
             }
 
-            return document;
+            return predicate;
+        }
+
+        private static Dictionary<string, List<string>> GetRequestUrlsFromManifest(ApiManifestDocument apiManifestDocument, OpenApiDocument document)
+        {
+            // Get the request URLs from the API Dependencies in the API manifest that have a baseURL that matches the server URL in the OpenAPI document
+            var serversUrls = document.Servers.Select(s => s.Url);
+            var requests = apiManifestDocument.ApiDependencies
+                    .Where(a => serversUrls.Any(s => s == a.BaseUrl))
+                    .SelectMany(ad => ad.Requests.Where(r => r.Exclude == false)
+                                .Select(r=> new { BaseUrl=ad.BaseUrl, UriTemplate= r.UriTemplate, Method=r.Method } ))
+                    .GroupBy(r => r.BaseUrl.TrimEnd('/') + r.UriTemplate)   // The OpenApiFilterService expects non-relative URLs.
+                    .ToDictionary(g => g.Key, g => g.Select(r => r.Method).ToList());
+            // This makes the assumption that the UriTemplate in the ApiManifest matches exactly the UriTemplate in the OpenAPI document
+            // This does not need to be the case.  The URI template in the API manifest could map to a set of OpenAPI paths.
+            // Additional logic will be required to handle this scenario.  I sugggest we build this into the OpenAPI.Net library at some point.
+            return requests;
         }
 
         private static XslCompiledTransform GetFilterTransform()
diff --git a/src/Microsoft.OpenApi.Hidi/Options/CommandOptions.cs b/src/Microsoft.OpenApi.Hidi/Options/CommandOptions.cs
@@ -22,6 +22,7 @@ internal class CommandOptions
         public Option<string> FilterByOperationIdsOption = new("--filter-by-operationids", "Filters OpenApiDocument by comma delimited list of OperationId(s) provided");
         public Option<string> FilterByTagsOption = new("--filter-by-tags", "Filters OpenApiDocument by comma delimited list of Tag(s) provided. Also accepts a single regex.");
         public Option<string> FilterByCollectionOption = new("--filter-by-collection", "Filters OpenApiDocument by Postman collection provided. Provide path to collection file.");
+        public Option<string> FilterByApiManifestOption = new("--filter-by-manifest", "Filters OpenApiDocument by API Manifest provided. Provide path to API Manifest file.");
         public Option<bool> InlineLocalOption = new("--inline-local", "Inline local $ref instances");
         public Option<bool> InlineExternalOption = new("--inline-external", "Inline external $ref instances");
 
@@ -41,6 +42,7 @@ public CommandOptions()
             FilterByOperationIdsOption.AddAlias("--op");
             FilterByTagsOption.AddAlias("--t");
             FilterByCollectionOption.AddAlias("-c");
+            FilterByApiManifestOption.AddAlias("-m");
             InlineLocalOption.AddAlias("--il");
             InlineExternalOption.AddAlias("--ie");
         }
@@ -63,6 +65,7 @@ public IReadOnlyList<Option> GetAllCommandOptions()
                 FilterByOperationIdsOption,
                 FilterByTagsOption,
                 FilterByCollectionOption,
+                FilterByApiManifestOption,
                 InlineLocalOption,
                 InlineExternalOption
             };
diff --git a/src/Microsoft.OpenApi.Hidi/Options/FilterOptions.cs b/src/Microsoft.OpenApi.Hidi/Options/FilterOptions.cs
@@ -8,5 +8,6 @@ internal class FilterOptions
         public string FilterByOperationIds { get; internal set; }
         public string FilterByTags { get; internal set; }
         public string FilterByCollection { get; internal set; }
+        public string FilterByApiManifest { get; internal set; }        
     }
 }
diff --git a/src/Microsoft.OpenApi.Hidi/Options/HidiOptions.cs b/src/Microsoft.OpenApi.Hidi/Options/HidiOptions.cs
@@ -55,7 +55,8 @@ private void ParseHidiOptions(ParseResult parseResult, CommandOptions options)
             {
                 FilterByOperationIds = parseResult.GetValueForOption(options.FilterByOperationIdsOption),
                 FilterByTags = parseResult.GetValueForOption(options.FilterByTagsOption),
-                FilterByCollection = parseResult.GetValueForOption(options.FilterByCollectionOption)
+                FilterByCollection = parseResult.GetValueForOption(options.FilterByCollectionOption),
+                FilterByApiManifest = parseResult.GetValueForOption(options.FilterByApiManifestOption)
             };
         }
     }
diff --git a/src/Microsoft.OpenApi.Hidi/Properties/AssemblyInfo.cs b/src/Microsoft.OpenApi.Hidi/Properties/AssemblyInfo.cs
@@ -0,0 +1,4 @@
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
diff --git a/src/Microsoft.OpenApi/Services/OpenApiFilterService.cs b/src/Microsoft.OpenApi/Services/OpenApiFilterService.cs
@@ -22,8 +22,11 @@ public static class OpenApiFilterService
         /// <param name="requestUrls">A dictionary of requests from a postman collection.</param>
         /// <param name="source">The input OpenAPI document.</param>
         /// <returns>A predicate.</returns>
-        public static Func<string, OperationType?, OpenApiOperation, bool> CreatePredicate(string operationIds = null,
-            string tags = null, Dictionary<string, List<string>> requestUrls = null, OpenApiDocument source = null)
+        public static Func<string, OperationType?, OpenApiOperation, bool> CreatePredicate(
+                string operationIds = null,
+                string tags = null, 
+                Dictionary<string, List<string>> requestUrls = null, 
+                OpenApiDocument source = null)
         {
             Func<string, OperationType?, OpenApiOperation, bool> predicate;
 
diff --git a/test/Microsoft.OpenApi.Hidi.Tests/Services/OpenApiServiceTests.cs b/test/Microsoft.OpenApi.Hidi.Tests/Services/OpenApiServiceTests.cs
@@ -282,7 +282,7 @@ public void InvokeTransformCommand()
             var handler = rootCommand.Subcommands.Where(c => c.Name == "transform").First().Handler;
             var context = new InvocationContext(parseResult);
 
-            handler.Invoke(context);
+            handler!.Invoke(context);
 
             var output = File.ReadAllText("sample.json");
             Assert.NotEmpty(output);
@@ -298,7 +298,7 @@ public void InvokeShowCommand()
             var handler = rootCommand.Subcommands.Where(c => c.Name == "show").First().Handler;
             var context = new InvocationContext(parseResult);
 
-            handler.Invoke(context);
+            handler!.Invoke(context);
 
             var output = File.ReadAllText("sample.md");
             Assert.Contains("graph LR", output);
diff --git a/test/Microsoft.OpenApi.Hidi.Tests/UtilityFiles/exampleapimanifest.json b/test/Microsoft.OpenApi.Hidi.Tests/UtilityFiles/exampleapimanifest.json
@@ -0,0 +1,27 @@
+{
+    "publisher": {
+        "name": "Alice",
+        "contactEmail": "alice@example.org"
+    },
+    "apiDependencies": [
+        {
+            "apiDescripionUrl": "https://raw.githubusercontent.com/APIPatterns/Moostodon/main/spec/tsp-output/%40typespec/openapi3/openapi.yaml",
+            "baseUrl": "https://mastodon.example/",
+            "auth": {
+                "clientIdentifier": "some-uuid-here",
+                "access": [ "resourceA.ReadWrite",
+                "resourceB.ReadWrite","resourceB.Read"]
+            },
+            "requests": [
+                {
+                    "method": "GET",
+                    "uriTemplate": "/api/v1/accounts/search"
+                },
+                {
+                    "method": "GET",
+                    "uriTemplate": "/api/v1/accounts/{id}"
+                }
+            ]
+        }
+    ]
+}

Original file line number	Diff line number	Diff line change
`@@ -8,5 +8,6 @@ internal class FilterOptions`
`8`	`8`	`public string FilterByOperationIds { get; internal set; }`
`9`	`9`	`public string FilterByTags { get; internal set; }`
`10`	`10`	`public string FilterByCollection { get; internal set; }`
	`11`	`+ public string FilterByApiManifest { get; internal set; }`
`11`	`12`	`}`
`12`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,8 @@ private void ParseHidiOptions(ParseResult parseResult, CommandOptions options)`
`55`	`55`	`{`
`56`	`56`	`FilterByOperationIds = parseResult.GetValueForOption(options.FilterByOperationIdsOption),`
`57`	`57`	`FilterByTags = parseResult.GetValueForOption(options.FilterByTagsOption),`
`58`		`- FilterByCollection = parseResult.GetValueForOption(options.FilterByCollectionOption)`
	`58`	`+ FilterByCollection = parseResult.GetValueForOption(options.FilterByCollectionOption),`
	`59`	`+ FilterByApiManifest = parseResult.GetValueForOption(options.FilterByApiManifestOption)`
`59`	`60`	`};`
`60`	`61`	`}`
`61`	`62`	`}`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
++
 +using System.Runtime.CompilerServices;
++
 +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]