Skip to content

Commit 77ce0bc

Browse files
authored
Merge pull request #663 from microsoft/mk/filter-by-collection
Allow Filtering by postman collection
2 parents ff584c7 + 63b7373 commit 77ce0bc

File tree

11 files changed

+24098
-38
lines changed

11 files changed

+24098
-38
lines changed

src/Microsoft.OpenApi.Hidi/OpenApiService.cs

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22
// Licensed under the MIT license.
33

44
using System;
5+
using System.Collections.Generic;
56
using System.IO;
67
using System.Linq;
78
using System.Net;
89
using System.Net.Http;
910
using System.Text;
11+
using System.Text.Json;
1012
using Microsoft.OpenApi.Extensions;
1113
using Microsoft.OpenApi.Models;
1214
using Microsoft.OpenApi.Readers;
@@ -16,7 +18,7 @@
1618

1719
namespace Microsoft.OpenApi.Hidi
1820
{
19-
static class OpenApiService
21+
public static class OpenApiService
2022
{
2123
public static void ProcessOpenApiDocument(
2224
string input,
@@ -25,6 +27,7 @@ public static void ProcessOpenApiDocument(
2527
OpenApiFormat format,
2628
string filterByOperationIds,
2729
string filterByTags,
30+
string filterByCollection,
2831
bool inline,
2932
bool resolveExternal)
3033
{
@@ -49,23 +52,30 @@ public static void ProcessOpenApiDocument(
4952
}
5053
).ReadAsync(stream).GetAwaiter().GetResult();
5154

52-
OpenApiDocument document;
53-
document = result.OpenApiDocument;
55+
var document = result.OpenApiDocument;
56+
Func<string, OperationType?, OpenApiOperation, bool> predicate;
5457

5558
// Check if filter options are provided, then execute
5659
if (!string.IsNullOrEmpty(filterByOperationIds) && !string.IsNullOrEmpty(filterByTags))
5760
{
5861
throw new InvalidOperationException("Cannot filter by operationIds and tags at the same time.");
5962
}
60-
6163
if (!string.IsNullOrEmpty(filterByOperationIds))
6264
{
63-
var predicate = OpenApiFilterService.CreatePredicate(operationIds: filterByOperationIds);
65+
predicate = OpenApiFilterService.CreatePredicate(operationIds: filterByOperationIds);
6466
document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
6567
}
6668
if (!string.IsNullOrEmpty(filterByTags))
6769
{
68-
var predicate = OpenApiFilterService.CreatePredicate(tags: filterByTags);
70+
predicate = OpenApiFilterService.CreatePredicate(tags: filterByTags);
71+
document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
72+
}
73+
74+
if (!string.IsNullOrEmpty(filterByCollection))
75+
{
76+
var fileStream = GetStream(filterByCollection);
77+
var requestUrls = ParseJsonCollectionFile(fileStream);
78+
predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source:document);
6979
document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
7080
}
7181

@@ -125,6 +135,38 @@ private static Stream GetStream(string input)
125135
return stream;
126136
}
127137

138+
/// <summary>
139+
/// Takes in a file stream, parses the stream into a JsonDocument and gets a list of paths and Http methods
140+
/// </summary>
141+
/// <param name="stream"> A file stream.</param>
142+
/// <returns> A dictionary of request urls and http methods from a collection.</returns>
143+
public static Dictionary<string, List<string>> ParseJsonCollectionFile(Stream stream)
144+
{
145+
var requestUrls = new Dictionary<string, List<string>>();
146+
147+
// Convert file to JsonDocument
148+
using var document = JsonDocument.Parse(stream);
149+
var root = document.RootElement;
150+
var itemElement = root.GetProperty("item");
151+
foreach (var requestObject in itemElement.EnumerateArray().Select(item => item.GetProperty("request")))
152+
{
153+
// Fetch list of methods and urls from collection, store them in a dictionary
154+
var path = requestObject.GetProperty("url").GetProperty("raw").ToString();
155+
var method = requestObject.GetProperty("method").ToString();
156+
157+
if (!requestUrls.ContainsKey(path))
158+
{
159+
requestUrls.Add(path, new List<string> { method });
160+
}
161+
else
162+
{
163+
requestUrls[path].Add(method);
164+
}
165+
}
166+
167+
return requestUrls;
168+
}
169+
128170
internal static void ValidateOpenApiDocument(string input)
129171
{
130172
if (input == null)

src/Microsoft.OpenApi.Hidi/Program.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ static async Task<int> Main(string[] args)
3030
new Option("--inline", "Inline $ref instances", typeof(bool) ),
3131
new Option("--resolveExternal","Resolve external $refs", typeof(bool)),
3232
new Option("--filterByOperationIds", "Filters OpenApiDocument by OperationId(s) provided", typeof(string)),
33-
new Option("--filterByTags", "Filters OpenApiDocument by Tag(s) provided", typeof(string))
33+
new Option("--filterByTags", "Filters OpenApiDocument by Tag(s) provided", typeof(string)),
34+
new Option("--filterByCollection", "Filters OpenApiDocument by Postman collection provided", typeof(string))
3435
};
35-
transformCommand.Handler = CommandHandler.Create<string, FileInfo, OpenApiSpecVersion, OpenApiFormat, string, string, bool, bool>(
36+
transformCommand.Handler = CommandHandler.Create<string, FileInfo, OpenApiSpecVersion, OpenApiFormat, string, string, string, bool, bool>(
3637
OpenApiService.ProcessOpenApiDocument);
3738

3839
rootCommand.Add(transformCommand);

src/Microsoft.OpenApi/Microsoft.OpenApi.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838

3939
<ItemGroup>
4040
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
41+
<PackageReference Include="System.Text.Json" Version="6.0.0" />
4142
</ItemGroup>
4243

4344
<ItemGroup>

src/Microsoft.OpenApi/Services/OpenApiFilterService.cs

Lines changed: 179 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@ public static class OpenApiFilterService
1919
/// </summary>
2020
/// <param name="operationIds">Comma delimited list of operationIds or * for all operations.</param>
2121
/// <param name="tags">Comma delimited list of tags or a single regex.</param>
22+
/// <param name="requestUrls">A dictionary of requests from a postman collection.</param>
23+
/// <param name="source">The input OpenAPI document.</param>
2224
/// <returns>A predicate.</returns>
23-
public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds = null, string tags = null)
25+
public static Func<string, OperationType?, OpenApiOperation, bool> CreatePredicate(string operationIds = null,
26+
string tags = null, Dictionary<string, List<string>> requestUrls = null, OpenApiDocument source = null)
2427
{
25-
Func<OpenApiOperation, bool> predicate;
28+
Func<string, OperationType?, OpenApiOperation, bool> predicate;
29+
30+
if (requestUrls != null && (operationIds != null || tags != null))
31+
{
32+
throw new InvalidOperationException("Cannot filter by Postman collection and either operationIds and tags at the same time.");
33+
}
2634
if (!string.IsNullOrEmpty(operationIds) && !string.IsNullOrEmpty(tags))
2735
{
2836
throw new InvalidOperationException("Cannot specify both operationIds and tags at the same time.");
@@ -31,12 +39,12 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
3139
{
3240
if (operationIds == "*")
3341
{
34-
predicate = (o) => true; // All operations
42+
predicate = (url, operationType, operation) => true; // All operations
3543
}
3644
else
3745
{
3846
var operationIdsArray = operationIds.Split(',');
39-
predicate = (o) => operationIdsArray.Contains(o.OperationId);
47+
predicate = (url, operationType, operation) => operationIdsArray.Contains(operation.OperationId);
4048
}
4149
}
4250
else if (tags != null)
@@ -46,16 +54,59 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
4654
{
4755
var regex = new Regex(tagsArray[0]);
4856

49-
predicate = (o) => o.Tags.Any(t => regex.IsMatch(t.Name));
57+
predicate = (url, operationType, operation) => operation.Tags.Any(tag => regex.IsMatch(tag.Name));
5058
}
5159
else
5260
{
53-
predicate = (o) => o.Tags.Any(t => tagsArray.Contains(t.Name));
61+
predicate = (url, operationType, operation) => operation.Tags.Any(tag => tagsArray.Contains(tag.Name));
62+
}
63+
}
64+
else if (requestUrls != null)
65+
{
66+
var operationTypes = new List<string>();
67+
68+
if (source != null)
69+
{
70+
var apiVersion = source.Info.Version;
71+
72+
var sources = new Dictionary<string, OpenApiDocument> {{ apiVersion, source}};
73+
var rootNode = CreateOpenApiUrlTreeNode(sources);
74+
75+
// Iterate through urls dictionary and fetch operations for each url
76+
foreach (var path in requestUrls)
77+
{
78+
var serverList = source.Servers;
79+
var url = FormatUrlString(path.Key, serverList);
80+
81+
var openApiOperations = GetOpenApiOperations(rootNode, url, apiVersion);
82+
if (openApiOperations == null)
83+
{
84+
continue;
85+
}
86+
87+
// Add the available ops if they are in the postman collection. See path.Value
88+
foreach (var ops in openApiOperations)
89+
{
90+
if (path.Value.Contains(ops.Key.ToString().ToUpper()))
91+
{
92+
operationTypes.Add(ops.Key + url);
93+
}
94+
}
95+
}
96+
}
97+
98+
if (!operationTypes.Any())
99+
{
100+
throw new ArgumentException("The urls in the Postman collection supplied could not be found.");
54101
}
102+
103+
// predicate for matching url and operationTypes
104+
predicate = (path, operationType, operation) => operationTypes.Contains(operationType + path);
55105
}
106+
56107
else
57108
{
58-
throw new InvalidOperationException("Either operationId(s) or tag(s) need to be specified.");
109+
throw new InvalidOperationException("Either operationId(s),tag(s) or Postman collection need to be specified.");
59110
}
60111

61112
return predicate;
@@ -67,12 +118,12 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
67118
/// <param name="source">The target <see cref="OpenApiDocument"/>.</param>
68119
/// <param name="predicate">A predicate function.</param>
69120
/// <returns>A partial OpenAPI document.</returns>
70-
public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Func<OpenApiOperation, bool> predicate)
121+
public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Func<string, OperationType?, OpenApiOperation, bool> predicate)
71122
{
72123
// Fetch and copy title, graphVersion and server info from OpenApiDoc
73124
var subset = new OpenApiDocument
74125
{
75-
Info = new OpenApiInfo()
126+
Info = new OpenApiInfo
76127
{
77128
Title = source.Info.Title + " - Subset",
78129
Description = source.Info.Description,
@@ -83,13 +134,11 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
83134
Extensions = source.Info.Extensions
84135
},
85136

86-
Components = new OpenApiComponents()
137+
Components = new OpenApiComponents {SecuritySchemes = source.Components.SecuritySchemes},
138+
SecurityRequirements = source.SecurityRequirements,
139+
Servers = source.Servers
87140
};
88141

89-
subset.Components.SecuritySchemes = source.Components.SecuritySchemes;
90-
subset.SecurityRequirements = source.SecurityRequirements;
91-
subset.Servers = source.Servers;
92-
93142
var results = FindOperations(source, predicate);
94143
foreach (var result in results)
95144
{
@@ -111,7 +160,10 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
111160
}
112161
}
113162

114-
pathItem.Operations.Add((OperationType)result.CurrentKeys.Operation, result.Operation);
163+
if (result.CurrentKeys.Operation != null)
164+
{
165+
pathItem.Operations.Add((OperationType)result.CurrentKeys.Operation, result.Operation);
166+
}
115167
}
116168

117169
if (subset.Paths == null)
@@ -124,11 +176,103 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
124176
return subset;
125177
}
126178

127-
private static IList<SearchResult> FindOperations(OpenApiDocument graphOpenApi, Func<OpenApiOperation, bool> predicate)
179+
/// <summary>
180+
/// Creates an <see cref="OpenApiUrlTreeNode"/> from a collection of <see cref="OpenApiDocument"/>.
181+
/// </summary>
182+
/// <param name="sources">Dictionary of labels and their corresponding <see cref="OpenApiDocument"/> objects.</param>
183+
/// <returns>The created <see cref="OpenApiUrlTreeNode"/>.</returns>
184+
public static OpenApiUrlTreeNode CreateOpenApiUrlTreeNode(Dictionary<string, OpenApiDocument> sources)
185+
{
186+
var rootNode = OpenApiUrlTreeNode.Create();
187+
foreach (var source in sources)
188+
{
189+
rootNode.Attach(source.Value, source.Key);
190+
}
191+
return rootNode;
192+
}
193+
194+
private static IDictionary<OperationType, OpenApiOperation> GetOpenApiOperations(OpenApiUrlTreeNode rootNode, string relativeUrl, string label)
195+
{
196+
if (relativeUrl.Equals("/", StringComparison.Ordinal) && rootNode.HasOperations(label))
197+
{
198+
return rootNode.PathItems[label].Operations;
199+
}
200+
201+
var urlSegments = relativeUrl.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
202+
203+
IDictionary<OperationType, OpenApiOperation> operations = null;
204+
205+
var targetChild = rootNode;
206+
207+
/* This will help keep track of whether we've skipped a segment
208+
* in the target url due to a possible parameter naming mismatch
209+
* with the corresponding OpenApiUrlTreeNode target child segment.
210+
*/
211+
var parameterNameOffset = 0;
212+
213+
for (var i = 0; i < urlSegments?.Length; i++)
214+
{
215+
var tempTargetChild = targetChild?.Children?
216+
.FirstOrDefault(x => x.Key.Equals(urlSegments[i],
217+
StringComparison.OrdinalIgnoreCase)).Value;
218+
219+
// Segment name mismatch
220+
if (tempTargetChild == null)
221+
{
222+
if (i == 0)
223+
{
224+
/* If no match and we are at the 1st segment of the relative url,
225+
* exit; no need to continue matching subsequent segments.
226+
*/
227+
break;
228+
}
229+
230+
/* Attempt to get the parameter segment from the children of the current node:
231+
* We are assuming a failed match because of different parameter namings
232+
* between the relative url segment and the corresponding OpenApiUrlTreeNode segment name
233+
* ex.: matching '/users/12345/messages' with '/users/{user-id}/messages'
234+
*/
235+
tempTargetChild = targetChild?.Children?
236+
.FirstOrDefault(x => x.Value.IsParameter).Value;
237+
238+
/* If no parameter segment exists in the children of the
239+
* current node or we've already skipped a parameter
240+
* segment in the relative url from the last pass,
241+
* then exit; there's no match.
242+
*/
243+
if (tempTargetChild == null || parameterNameOffset > 0)
244+
{
245+
break;
246+
}
247+
248+
/* To help us know we've skipped a
249+
* corresponding segment in the relative url.
250+
*/
251+
parameterNameOffset++;
252+
}
253+
else
254+
{
255+
parameterNameOffset = 0;
256+
}
257+
258+
// Move to the next segment
259+
targetChild = tempTargetChild;
260+
261+
// We want the operations of the last segment of the path.
262+
if (i == urlSegments.Length - 1 && targetChild.HasOperations(label))
263+
{
264+
operations = targetChild.PathItems[label].Operations;
265+
}
266+
}
267+
268+
return operations;
269+
}
270+
271+
private static IList<SearchResult> FindOperations(OpenApiDocument sourceDocument, Func<string, OperationType?, OpenApiOperation, bool> predicate)
128272
{
129273
var search = new OperationSearch(predicate);
130274
var walker = new OpenApiWalker(search);
131-
walker.Walk(graphOpenApi);
275+
walker.Walk(sourceDocument);
132276
return search.SearchResults;
133277
}
134278

@@ -177,5 +321,23 @@ private static bool AddReferences(OpenApiComponents newComponents, OpenApiCompon
177321
}
178322
return moreStuff;
179323
}
324+
325+
private static string FormatUrlString(string url, IList<OpenApiServer> serverList)
326+
{
327+
var queryPath = string.Empty;
328+
foreach (var server in serverList)
329+
{
330+
var serverUrl = server.Url.TrimEnd('/');
331+
if (!url.Contains(serverUrl))
332+
{
333+
continue;
334+
}
335+
336+
var querySegments = url.Split(new[]{ serverUrl }, StringSplitOptions.None);
337+
queryPath = querySegments[1];
338+
}
339+
340+
return queryPath;
341+
}
180342
}
181343
}

0 commit comments

Comments
 (0)