Skip to content

Commit a487f60

Browse files
waldekmastykarzwaldekmastykarz
authored
Adds the HttpFileGeneratorPlugin. Closes #780 (#798)
1 parent 8b1a7a6 commit a487f60

File tree

4 files changed

+328
-0
lines changed

4 files changed

+328
-0
lines changed

‎dev-proxy-plugins/Reporters/MarkdownReporter.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class MarkdownReporter : BaseReporter
2121
{ typeof(ApiCenterProductionVersionPluginReport), TransformApiCenterProductionVersionReport },
2222
{ typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummaryByUrl },
2323
{ typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummaryByMessageType },
24+
{ typeof(HttpFileGeneratorPlugin), TransformHttpFileGeneratorReport },
2425
{ typeof(MinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport },
2526
{ typeof(MinimalPermissionsPluginReport), TransformMinimalPermissionsReport },
2627
{ typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }
@@ -476,4 +477,19 @@ private static void AddExecutionSummaryReportSummary(IEnumerable<RequestLog> req
476477

477478
return sb.ToString();
478479
}
480+
481+
private static string? TransformHttpFileGeneratorReport(object report)
482+
{
483+
var httpFileGeneratorReport = (HttpFileGeneratorPluginReport)report;
484+
485+
var sb = new StringBuilder();
486+
487+
sb.AppendLine("# Generated HTTP files");
488+
sb.AppendLine();
489+
sb.AppendJoin(Environment.NewLine, $"- {httpFileGeneratorReport}");
490+
sb.AppendLine();
491+
sb.AppendLine();
492+
493+
return sb.ToString();
494+
}
479495
}

‎dev-proxy-plugins/Reporters/PlainTextReporter.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class PlainTextReporter : BaseReporter
2121
{ typeof(ApiCenterProductionVersionPluginReport), TransformApiCenterProductionVersionReport },
2222
{ typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummaryByUrl },
2323
{ typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummaryByMessageType },
24+
{ typeof(HttpFileGeneratorPluginReport), TransformHttpFileGeneratorReport },
2425
{ typeof(MinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport },
2526
{ typeof(MinimalPermissionsPluginReport), TransformMinimalPermissionsReport },
2627
{ typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }
@@ -52,6 +53,19 @@ public PlainTextReporter(IPluginEvents pluginEvents, IProxyContext context, ILog
5253
}
5354
}
5455

56+
private static string? TransformHttpFileGeneratorReport(object report)
57+
{
58+
var httpFileGeneratorReport = (HttpFileGeneratorPluginReport)report;
59+
60+
var sb = new StringBuilder();
61+
62+
sb.AppendLine("Generated HTTP files:");
63+
sb.AppendLine();
64+
sb.AppendJoin(Environment.NewLine, httpFileGeneratorReport);
65+
66+
return sb.ToString();
67+
}
68+
5569
private static string? TransformOpenApiSpecGeneratorReport(object report)
5670
{
5771
var openApiSpecGeneratorReport = (OpenApiSpecGeneratorPluginReport)report;

‎dev-proxy-plugins/RequestLogs/HttpFileGeneratorPlugin.cs‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
5+
// Copyright (c) Microsoft Corporation.
6+
// Licensed under the MIT License.
7+
8+
using System.Text;
9+
using System.Web;
10+
using Microsoft.DevProxy.Abstractions;
11+
using Microsoft.Extensions.Configuration;
12+
using Microsoft.Extensions.Logging;
13+
14+
namespace Microsoft.DevProxy.Plugins.RequestLogs;
15+
16+
internal class HttpFile
17+
{
18+
public Dictionary<string, string> Variables { get; set; } = new();
19+
public List<HttpFileRequest> Requests { get; set; } = new();
20+
21+
public string Serialize()
22+
{
23+
var sb = new StringBuilder();
24+
25+
foreach (var variable in Variables)
26+
{
27+
sb.AppendLine($"@{variable.Key} = {variable.Value}");
28+
}
29+
30+
foreach (var request in Requests)
31+
{
32+
sb.AppendLine();
33+
sb.AppendLine("###");
34+
sb.AppendLine();
35+
sb.AppendLine($"# @name {GetRequestName(request)}");
36+
sb.AppendLine();
37+
38+
sb.AppendLine($"{request.Method} {request.Url}");
39+
40+
foreach (var header in request.Headers)
41+
{
42+
sb.AppendLine($"{header.Name}: {header.Value}");
43+
}
44+
45+
if (!string.IsNullOrEmpty(request.Body))
46+
{
47+
sb.AppendLine();
48+
sb.AppendLine(request.Body);
49+
}
50+
}
51+
52+
return sb.ToString();
53+
}
54+
55+
private string GetRequestName(HttpFileRequest request)
56+
{
57+
var url = new Uri(request.Url);
58+
return $"{request.Method.ToLower()}{url.Segments.Last().Replace("/", "").ToPascalCase()}";
59+
}
60+
}
61+
62+
internal class HttpFileRequest
63+
{
64+
public string Method { get; set; } = string.Empty;
65+
public string Url { get; set; } = string.Empty;
66+
public string? Body { get; set; }
67+
public List<HttpFileRequestHeader> Headers { get; set; } = new();
68+
}
69+
70+
internal class HttpFileRequestHeader
71+
{
72+
public string Name { get; set; } = string.Empty;
73+
public string Value { get; set; } = string.Empty;
74+
}
75+
76+
public class HttpFileGeneratorPluginReport : List<string>
77+
{
78+
public HttpFileGeneratorPluginReport() : base() { }
79+
80+
public HttpFileGeneratorPluginReport(IEnumerable<string> collection) : base(collection) { }
81+
}
82+
83+
internal class HttpFileGeneratorPluginConfiguration
84+
{
85+
public bool IncludeOptionsRequests { get; set; } = false;
86+
}
87+
88+
public class HttpFileGeneratorPlugin : BaseReportingPlugin
89+
{
90+
public override string Name => nameof(HttpFileGeneratorPlugin);
91+
public static readonly string GeneratedHttpFilesKey = "GeneratedHttpFiles";
92+
private HttpFileGeneratorPluginConfiguration _configuration = new();
93+
private readonly string[] headersToExtract = ["authorization", "key"];
94+
private readonly string[] queryParametersToExtract = ["key"];
95+
96+
public HttpFileGeneratorPlugin(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet<UrlToWatch> urlsToWatch, IConfigurationSection? configSection = null) : base(pluginEvents, context, logger, urlsToWatch, configSection)
97+
{
98+
}
99+
100+
public override void Register()
101+
{
102+
base.Register();
103+
104+
ConfigSection?.Bind(_configuration);
105+
106+
PluginEvents.AfterRecordingStop += AfterRecordingStop;
107+
}
108+
109+
private async Task AfterRecordingStop(object? sender, RecordingArgs e)
110+
{
111+
Logger.LogInformation("Creating HTTP file from recorded requests...");
112+
113+
if (!e.RequestLogs.Any())
114+
{
115+
Logger.LogDebug("No requests to process");
116+
return;
117+
}
118+
119+
var httpFile = await GetHttpRequests(e.RequestLogs);
120+
DeduplicateRequests(httpFile);
121+
ExtractVariables(httpFile);
122+
123+
var fileName = $"requests_{DateTime.Now:yyyyMMddHHmmss}.http";
124+
Logger.LogDebug("Writing HTTP file to {fileName}...", fileName);
125+
File.WriteAllText(fileName, httpFile.Serialize());
126+
Logger.LogInformation("Created HTTP file {fileName}", fileName);
127+
128+
var generatedHttpFiles = new[] { fileName };
129+
StoreReport(new HttpFileGeneratorPluginReport(generatedHttpFiles), e);
130+
131+
// store the generated HTTP files in the global data
132+
// for use by other plugins
133+
e.GlobalData[GeneratedHttpFilesKey] = generatedHttpFiles;
134+
}
135+
136+
private async Task<HttpFile> GetHttpRequests(IEnumerable<RequestLog> requestLogs)
137+
{
138+
var httpFile = new HttpFile();
139+
140+
foreach (var request in requestLogs)
141+
{
142+
if (request.MessageType != MessageType.InterceptedResponse ||
143+
request.Context is null ||
144+
request.Context.Session is null)
145+
{
146+
continue;
147+
}
148+
149+
if (!_configuration.IncludeOptionsRequests &&
150+
request.Context.Session.HttpClient.Request.Method.ToUpperInvariant() == "OPTIONS")
151+
{
152+
Logger.LogDebug("Skipping OPTIONS request {url}...", request.Context.Session.HttpClient.Request.RequestUri);
153+
continue;
154+
}
155+
156+
var methodAndUrlString = request.MessageLines.First();
157+
Logger.LogDebug("Adding request {methodAndUrl}...", methodAndUrlString);
158+
159+
var methodAndUrl = methodAndUrlString.Split(' ');
160+
httpFile.Requests.Add(new HttpFileRequest
161+
{
162+
Method = methodAndUrl[0],
163+
Url = methodAndUrl[1],
164+
Body = request.Context.Session.HttpClient.Request.HasBody ? await request.Context.Session.GetRequestBodyAsString() : null,
165+
Headers = request.Context.Session.HttpClient.Request.Headers
166+
.Select(h => new HttpFileRequestHeader { Name = h.Name, Value = h.Value })
167+
.ToList()
168+
});
169+
}
170+
171+
return httpFile;
172+
}
173+
174+
private void DeduplicateRequests(HttpFile httpFile)
175+
{
176+
Logger.LogDebug("Deduplicating requests...");
177+
178+
// remove duplicate requests
179+
// if the request doesn't have a body, dedupe on method + URL
180+
// if it has a body, dedupe on method + URL + body
181+
var uniqueRequests = new List<HttpFileRequest>();
182+
foreach (var request in httpFile.Requests)
183+
{
184+
Logger.LogDebug(" Checking request {method} {url}...", request.Method, request.Url);
185+
186+
var existingRequest = uniqueRequests.FirstOrDefault(r =>
187+
{
188+
if (r.Method != request.Method || r.Url != request.Url)
189+
{
190+
return false;
191+
}
192+
193+
if (r.Body is null && request.Body is null)
194+
{
195+
return true;
196+
}
197+
198+
if (r.Body is not null && request.Body is not null)
199+
{
200+
return r.Body == request.Body;
201+
}
202+
203+
return false;
204+
});
205+
206+
if (existingRequest is null)
207+
{
208+
Logger.LogDebug(" Keeping request {method} {url}...", request.Method, request.Url);
209+
uniqueRequests.Add(request);
210+
}
211+
else
212+
{
213+
Logger.LogDebug(" Skipping duplicate request {method} {url}...", request.Method, request.Url);
214+
}
215+
}
216+
217+
httpFile.Requests = uniqueRequests;
218+
}
219+
220+
private void ExtractVariables(HttpFile httpFile)
221+
{
222+
Logger.LogDebug("Extracting variables...");
223+
224+
foreach (var request in httpFile.Requests)
225+
{
226+
Logger.LogDebug(" Processing request {method} {url}...", request.Method, request.Url);
227+
228+
foreach (var headerName in headersToExtract)
229+
{
230+
Logger.LogDebug(" Extracting header {headerName}...", headerName);
231+
232+
var headers = request.Headers.Where(h => h.Name.Contains(headerName, StringComparison.OrdinalIgnoreCase));
233+
if (headers is not null)
234+
{
235+
Logger.LogDebug(" Found {numHeaders} matching headers...", headers.Count());
236+
237+
foreach (var header in headers)
238+
{
239+
var variableName = GetVariableName(request, headerName);
240+
Logger.LogDebug(" Extracting variable {variableName}...", variableName);
241+
httpFile.Variables[variableName] = header.Value;
242+
header.Value = $"{{{{{variableName}}}}}";
243+
}
244+
}
245+
}
246+
247+
var url = new Uri(request.Url);
248+
var query = HttpUtility.ParseQueryString(url.Query);
249+
if (query.Count > 0)
250+
{
251+
Logger.LogDebug(" Processing query parameters...");
252+
253+
foreach (var queryParameterName in queryParametersToExtract)
254+
{
255+
Logger.LogDebug(" Extracting query parameter {queryParameterName}...", queryParameterName);
256+
257+
var queryParams = query.AllKeys.Where(k => k is not null && k.Contains(queryParameterName, StringComparison.OrdinalIgnoreCase));
258+
if (queryParams is not null)
259+
{
260+
Logger.LogDebug(" Found {numQueryParams} matching query parameters...", queryParams.Count());
261+
262+
foreach (var queryParam in queryParams)
263+
{
264+
var variableName = GetVariableName(request, queryParam!);
265+
Logger.LogDebug(" Extracting variable {variableName}...", variableName);
266+
httpFile.Variables[variableName] = queryParam!;
267+
query[queryParam] = $"{{{{{variableName}}}}}";
268+
}
269+
}
270+
}
271+
request.Url = $"{url.GetLeftPart(UriPartial.Path)}?{query}"
272+
.Replace("%7b", "{")
273+
.Replace("%7d", "}");
274+
Logger.LogDebug(" Updated URL to {url}...", request.Url);
275+
}
276+
else
277+
{
278+
Logger.LogDebug(" No query parameters to process...");
279+
}
280+
}
281+
}
282+
283+
private string GetVariableName(HttpFileRequest request, string variableName)
284+
{
285+
var url = new Uri(request.Url);
286+
return $"{url.Host.Replace(".", "_").Replace("-", "_")}_{variableName.Replace("-", "_")}";
287+
}
288+
}

‎dev-proxy-plugins/StringExtensions.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,14 @@ internal static string MaxLength(this string input, int maxLength)
99
{
1010
return input.Length <= maxLength ? input : input[..maxLength];
1111
}
12+
13+
internal static string ToPascalCase(this string input)
14+
{
15+
if (string.IsNullOrEmpty(input))
16+
{
17+
return input;
18+
}
19+
20+
return char.ToUpper(input[0]) + input[1..];
21+
}
1222
}

0 commit comments

Comments
 (0)