From 91247f4e8740825e9552926a165cee75a9bc285f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:07:19 +0000 Subject: [PATCH 01/12] Initial plan From de7e82ae314c182875d2e72c5d991d43c66ecde6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:15:34 +0000 Subject: [PATCH 02/12] Add HAR capture extension for BiDi Network module Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 330 ++++++++++++ .../webdriver/BiDi/Network/Har/HarEntry.cs | 469 ++++++++++++++++++ .../src/webdriver/BiDi/Network/Har/HarLog.cs | 163 ++++++ .../common/BiDi/Network/HarCaptureTest.cs | 114 +++++ 4 files changed, 1076 insertions(+) create mode 100644 dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs create mode 100644 dotnet/src/webdriver/BiDi/Network/Har/HarEntry.cs create mode 100644 dotnet/src/webdriver/BiDi/Network/Har/HarLog.cs create mode 100644 dotnet/test/common/BiDi/Network/HarCaptureTest.cs diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs new file mode 100644 index 0000000000000..e7ed7e4d1353d --- /dev/null +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -0,0 +1,330 @@ +// +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace OpenQA.Selenium.BiDi.Network.Har; + +/// +/// Extension methods for BiDi class to capture network traffic as HAR. +/// +public static class BiDiHarExtensions +{ + /// + /// Captures network traffic and returns a HAR recorder that can be used to save the captured traffic. + /// + /// The BiDi instance. + /// Optional configuration options. + /// A task that represents the asynchronous operation and returns a HarRecorder. + public static async Task CaptureNetworkTrafficAsync(this BiDi bidi, HarCaptureOptions? options = null) + { + if (bidi is null) throw new ArgumentNullException(nameof(bidi)); + + var recorder = new HarRecorder(bidi, options ?? new HarCaptureOptions()); + await recorder.StartAsync().ConfigureAwait(false); + return recorder; + } +} + +/// +/// Options for HAR capture. +/// +public sealed class HarCaptureOptions +{ + /// + /// Gets or sets a value indicating whether to include response content in the HAR file. + /// + public bool IncludeResponseContent { get; set; } = false; + + /// + /// Gets or sets a value indicating whether to include request/response body content in the HAR file. + /// + public bool IncludeContent { get; set; } = false; + + /// + /// Gets or sets the browser name to include in the HAR file. + /// + public string? BrowserName { get; set; } + + /// + /// Gets or sets the browser version to include in the HAR file. + /// + public string? BrowserVersion { get; set; } +} + +/// +/// Records network traffic and provides methods to save it as HAR format. +/// +public sealed class HarRecorder : IAsyncDisposable +{ + private readonly BiDi _bidi; + private readonly HarCaptureOptions _options; + private readonly HarFile _harFile; + private readonly Dictionary _pendingRequests; + private Subscription? _beforeRequestSubscription; + private Subscription? _responseStartedSubscription; + private Subscription? _responseCompletedSubscription; + + internal HarRecorder(BiDi bidi, HarCaptureOptions options) + { + _bidi = bidi ?? throw new ArgumentNullException(nameof(bidi)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _harFile = new HarFile(); + _pendingRequests = new Dictionary(); + + if (!string.IsNullOrEmpty(options.BrowserName)) + { + _harFile.Log.Browser = new HarBrowser + { + Name = options.BrowserName, + Version = options.BrowserVersion ?? string.Empty + }; + } + } + + internal async Task StartAsync() + { + _beforeRequestSubscription = await _bidi.Network.OnBeforeRequestSentAsync(OnBeforeRequestSent).ConfigureAwait(false); + _responseStartedSubscription = await _bidi.Network.OnResponseStartedAsync(OnResponseStarted).ConfigureAwait(false); + _responseCompletedSubscription = await _bidi.Network.OnResponseCompletedAsync(OnResponseCompleted).ConfigureAwait(false); + } + + private void OnBeforeRequestSent(BeforeRequestSentEventArgs args) + { + var entry = new HarEntry + { + StartedDateTime = args.Timestamp.ToString("o"), + Request = ConvertRequest(args.Request), + Timings = ConvertTimings(args.Request.Timings), + Time = 0 + }; + + if (args.Context != null) + { + entry.Pageref = args.Context.Id; + } + + lock (_pendingRequests) + { + _pendingRequests[args.Request.Request.Id] = entry; + } + } + + private void OnResponseStarted(ResponseStartedEventArgs args) + { + lock (_pendingRequests) + { + if (_pendingRequests.TryGetValue(args.Request.Request.Id, out var entry)) + { + entry.Response = ConvertResponse(args.Response); + } + } + } + + private void OnResponseCompleted(ResponseCompletedEventArgs args) + { + lock (_pendingRequests) + { + if (_pendingRequests.TryGetValue(args.Request.Request.Id, out var entry)) + { + entry.Response = ConvertResponse(args.Response); + + // Calculate total time + var timings = args.Request.Timings; + entry.Time = CalculateTotalTime(timings); + + _harFile.Log.Entries.Add(entry); + _pendingRequests.Remove(args.Request.Request.Id); + } + } + } + + private HarRequest ConvertRequest(RequestData request) + { + var harRequest = new HarRequest + { + Method = request.Method, + Url = request.Url, + HttpVersion = "HTTP/1.1", + HeadersSize = request.HeadersSize ?? -1, + BodySize = request.BodySize ?? -1 + }; + + foreach (var header in request.Headers) + { + harRequest.Headers.Add(new HarHeader + { + Name = header.Name, + Value = header.Value.ToString() + }); + } + + foreach (var cookie in request.Cookies) + { + harRequest.Cookies.Add(new HarCookie + { + Name = cookie.Name, + Value = cookie.Value.ToString() + }); + } + + // Parse query string from URL + var uri = new Uri(request.Url); + if (!string.IsNullOrEmpty(uri.Query)) + { + var queryString = uri.Query.TrimStart('?'); + var queryParams = queryString.Split('&'); + foreach (var param in queryParams) + { + var parts = param.Split('=', 2); + harRequest.QueryString.Add(new HarQueryParam + { + Name = Uri.UnescapeDataString(parts[0]), + Value = parts.Length > 1 ? Uri.UnescapeDataString(parts[1]) : string.Empty + }); + } + } + + return harRequest; + } + + private HarResponse ConvertResponse(ResponseData response) + { + var harResponse = new HarResponse + { + Status = response.Status, + StatusText = response.StatusText, + HttpVersion = response.Protocol, + HeadersSize = response.HeadersSize ?? -1, + BodySize = response.BodySize ?? -1, + Content = new HarContent + { + Size = response.BodySize ?? 0, + MimeType = response.MimeType + } + }; + + foreach (var header in response.Headers) + { + harResponse.Headers.Add(new HarHeader + { + Name = header.Name, + Value = header.Value.ToString() + }); + + // Check for redirect URL + if (header.Name.Equals("Location", StringComparison.OrdinalIgnoreCase)) + { + harResponse.RedirectURL = header.Value.ToString(); + } + } + + return harResponse; + } + + private HarTimings ConvertTimings(FetchTimingInfo timings) + { + return new HarTimings + { + Blocked = -1, + Dns = CalculateDuration(timings.DnsStart, timings.DnsEnd), + Connect = CalculateDuration(timings.ConnectStart, timings.ConnectEnd), + Ssl = CalculateDuration(timings.TlsStart, timings.ConnectEnd), + Send = CalculateDuration(timings.RequestStart, timings.RequestStart), + Wait = CalculateDuration(timings.RequestStart, timings.ResponseStart), + Receive = CalculateDuration(timings.ResponseStart, timings.ResponseEnd) + }; + } + + private double CalculateDuration(double start, double end) + { + if (start < 0 || end < 0 || end < start) + { + return -1; + } + return end - start; + } + + private double CalculateTotalTime(FetchTimingInfo timings) + { + if (timings.FetchStart >= 0 && timings.ResponseEnd >= 0 && timings.ResponseEnd >= timings.FetchStart) + { + return timings.ResponseEnd - timings.FetchStart; + } + return 0; + } + + /// + /// Gets the captured HAR file. + /// + /// The HAR file containing all captured network traffic. + public HarFile GetHar() + { + return _harFile; + } + + /// + /// Saves the captured network traffic to a HAR file. + /// + /// The path where the HAR file should be saved. + /// A task that represents the asynchronous operation. + public async Task SaveAsync(string filePath) + { + if (string.IsNullOrEmpty(filePath)) + { + throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); + } + + var options = new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + var json = JsonSerializer.Serialize(_harFile, options); + await File.WriteAllTextAsync(filePath, json).ConfigureAwait(false); + } + + /// + /// Disposes the recorder and unsubscribes from network events. + /// + public async ValueTask DisposeAsync() + { + if (_beforeRequestSubscription != null) + { + await _beforeRequestSubscription.DisposeAsync().ConfigureAwait(false); + } + + if (_responseStartedSubscription != null) + { + await _responseStartedSubscription.DisposeAsync().ConfigureAwait(false); + } + + if (_responseCompletedSubscription != null) + { + await _responseCompletedSubscription.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/dotnet/src/webdriver/BiDi/Network/Har/HarEntry.cs b/dotnet/src/webdriver/BiDi/Network/Har/HarEntry.cs new file mode 100644 index 0000000000000..e633874fca361 --- /dev/null +++ b/dotnet/src/webdriver/BiDi/Network/Har/HarEntry.cs @@ -0,0 +1,469 @@ +// +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +using System.Collections.Generic; + +namespace OpenQA.Selenium.BiDi.Network.Har; + +/// +/// Represents a HAR entry object containing request and response information. +/// +public sealed class HarEntry +{ + /// + /// Gets or sets the reference to the parent page. + /// + public string? Pageref { get; set; } + + /// + /// Gets or sets the date and time stamp of the request start. + /// + public string StartedDateTime { get; set; } = string.Empty; + + /// + /// Gets or sets the total elapsed time in milliseconds. + /// + public double Time { get; set; } + + /// + /// Gets or sets the request information. + /// + public HarRequest Request { get; set; } = new HarRequest(); + + /// + /// Gets or sets the response information. + /// + public HarResponse Response { get; set; } = new HarResponse(); + + /// + /// Gets or sets the cache information. + /// + public HarCache Cache { get; set; } = new HarCache(); + + /// + /// Gets or sets the timing information. + /// + public HarTimings Timings { get; set; } = new HarTimings(); + + /// + /// Gets or sets the server IP address. + /// + public string? ServerIPAddress { get; set; } + + /// + /// Gets or sets the connection information. + /// + public string? Connection { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR request object. +/// +public sealed class HarRequest +{ + /// + /// Gets or sets the request method. + /// + public string Method { get; set; } = string.Empty; + + /// + /// Gets or sets the absolute URL of the request. + /// + public string Url { get; set; } = string.Empty; + + /// + /// Gets or sets the request HTTP version. + /// + public string HttpVersion { get; set; } = string.Empty; + + /// + /// Gets or sets the list of cookie objects. + /// + public List Cookies { get; set; } = new List(); + + /// + /// Gets or sets the list of header objects. + /// + public List Headers { get; set; } = new List(); + + /// + /// Gets or sets the list of query parameter objects. + /// + public List QueryString { get; set; } = new List(); + + /// + /// Gets or sets the posted data information. + /// + public HarPostData? PostData { get; set; } + + /// + /// Gets or sets the total number of bytes from the start of the HTTP request message. + /// + public long HeadersSize { get; set; } = -1; + + /// + /// Gets or sets the size of the request body in bytes. + /// + public long BodySize { get; set; } = -1; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR response object. +/// +public sealed class HarResponse +{ + /// + /// Gets or sets the response status code. + /// + public int Status { get; set; } + + /// + /// Gets or sets the response status description. + /// + public string StatusText { get; set; } = string.Empty; + + /// + /// Gets or sets the response HTTP version. + /// + public string HttpVersion { get; set; } = string.Empty; + + /// + /// Gets or sets the list of cookie objects. + /// + public List Cookies { get; set; } = new List(); + + /// + /// Gets or sets the list of header objects. + /// + public List Headers { get; set; } = new List(); + + /// + /// Gets or sets the response body content. + /// + public HarContent Content { get; set; } = new HarContent(); + + /// + /// Gets or sets the redirection target URL from the Location response header. + /// + public string RedirectURL { get; set; } = string.Empty; + + /// + /// Gets or sets the total number of bytes from the start of the HTTP response message. + /// + public long HeadersSize { get; set; } = -1; + + /// + /// Gets or sets the size of the received response body in bytes. + /// + public long BodySize { get; set; } = -1; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR cookie object. +/// +public sealed class HarCookie +{ + /// + /// Gets or sets the cookie name. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the cookie value. + /// + public string Value { get; set; } = string.Empty; + + /// + /// Gets or sets the path pertaining to the cookie. + /// + public string? Path { get; set; } + + /// + /// Gets or sets the host of the cookie. + /// + public string? Domain { get; set; } + + /// + /// Gets or sets the cookie expiration time. + /// + public string? Expires { get; set; } + + /// + /// Gets or sets a value indicating whether the cookie is HTTP only. + /// + public bool? HttpOnly { get; set; } + + /// + /// Gets or sets a value indicating whether the cookie is secure. + /// + public bool? Secure { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR header object. +/// +public sealed class HarHeader +{ + /// + /// Gets or sets the header name. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the header value. + /// + public string Value { get; set; } = string.Empty; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR query parameter object. +/// +public sealed class HarQueryParam +{ + /// + /// Gets or sets the query parameter name. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the query parameter value. + /// + public string Value { get; set; } = string.Empty; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents HAR post data information. +/// +public sealed class HarPostData +{ + /// + /// Gets or sets the MIME type of the posted data. + /// + public string MimeType { get; set; } = string.Empty; + + /// + /// Gets or sets the list of posted parameters. + /// + public List? Params { get; set; } + + /// + /// Gets or sets the plain text posted data. + /// + public string? Text { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR post parameter object. +/// +public sealed class HarPostParam +{ + /// + /// Gets or sets the parameter name. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the parameter value. + /// + public string? Value { get; set; } + + /// + /// Gets or sets the name of the uploaded file. + /// + public string? FileName { get; set; } + + /// + /// Gets or sets the content type of the uploaded file. + /// + public string? ContentType { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents HAR content information. +/// +public sealed class HarContent +{ + /// + /// Gets or sets the length of the content in bytes. + /// + public long Size { get; set; } + + /// + /// Gets or sets the length of the returned content in bytes. + /// + public long? Compression { get; set; } + + /// + /// Gets or sets the MIME type of the response. + /// + public string MimeType { get; set; } = string.Empty; + + /// + /// Gets or sets the response body text. + /// + public string? Text { get; set; } + + /// + /// Gets or sets the encoding used for the response text. + /// + public string? Encoding { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents HAR cache information. +/// +public sealed class HarCache +{ + /// + /// Gets or sets the state of the cache entry before the request. + /// + public HarCacheEntry? BeforeRequest { get; set; } + + /// + /// Gets or sets the state of the cache entry after the request. + /// + public HarCacheEntry? AfterRequest { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a HAR cache entry object. +/// +public sealed class HarCacheEntry +{ + /// + /// Gets or sets the expiration time of the cache entry. + /// + public string? Expires { get; set; } + + /// + /// Gets or sets the last accessed time of the cache entry. + /// + public string LastAccess { get; set; } = string.Empty; + + /// + /// Gets or sets the ETag. + /// + public string ETag { get; set; } = string.Empty; + + /// + /// Gets or sets the number of times the entry has been opened. + /// + public int HitCount { get; set; } + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents HAR timing information. +/// +public sealed class HarTimings +{ + /// + /// Gets or sets the time spent in a queue waiting for a network connection. + /// + public double Blocked { get; set; } = -1; + + /// + /// Gets or sets the DNS resolution time. + /// + public double Dns { get; set; } = -1; + + /// + /// Gets or sets the time required to create a TCP connection. + /// + public double Connect { get; set; } = -1; + + /// + /// Gets or sets the time required for SSL/TLS negotiation. + /// + public double Ssl { get; set; } = -1; + + /// + /// Gets or sets the time required to send the HTTP request to the server. + /// + public double Send { get; set; } = -1; + + /// + /// Gets or sets the waiting for a response from the server. + /// + public double Wait { get; set; } = -1; + + /// + /// Gets or sets the time required to read the entire response from the server. + /// + public double Receive { get; set; } = -1; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} diff --git a/dotnet/src/webdriver/BiDi/Network/Har/HarLog.cs b/dotnet/src/webdriver/BiDi/Network/Har/HarLog.cs new file mode 100644 index 0000000000000..213e75daf48dd --- /dev/null +++ b/dotnet/src/webdriver/BiDi/Network/Har/HarLog.cs @@ -0,0 +1,163 @@ +// +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +using System.Collections.Generic; + +namespace OpenQA.Selenium.BiDi.Network.Har; + +/// +/// Represents the root object of exported HAR data. +/// +public sealed class HarFile +{ + /// + /// Gets or sets the log object. + /// + public HarLog Log { get; set; } = new HarLog(); +} + +/// +/// Represents a HAR log object. +/// +public sealed class HarLog +{ + /// + /// Gets or sets the version of the HAR format. + /// + public string Version { get; set; } = "1.2"; + + /// + /// Gets or sets the creator information. + /// + public HarCreator Creator { get; set; } = new HarCreator(); + + /// + /// Gets or sets the browser information. + /// + public HarBrowser? Browser { get; set; } + + /// + /// Gets or sets the list of page objects. + /// + public List Pages { get; set; } = new List(); + + /// + /// Gets or sets the list of entry objects. + /// + public List Entries { get; set; } = new List(); + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents the creator information. +/// +public sealed class HarCreator +{ + /// + /// Gets or sets the name of the creator. + /// + public string Name { get; set; } = "Selenium"; + + /// + /// Gets or sets the version of the creator. + /// + public string Version { get; set; } = "4.0"; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents the browser information. +/// +public sealed class HarBrowser +{ + /// + /// Gets or sets the name of the browser. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the version of the browser. + /// + public string Version { get; set; } = string.Empty; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents a page object. +/// +public sealed class HarPage +{ + /// + /// Gets or sets the unique identifier for the page. + /// + public string Id { get; set; } = string.Empty; + + /// + /// Gets or sets the page title. + /// + public string Title { get; set; } = string.Empty; + + /// + /// Gets or sets the date and time stamp for the page. + /// + public string StartedDateTime { get; set; } = string.Empty; + + /// + /// Gets or sets the page timings. + /// + public HarPageTimings PageTimings { get; set; } = new HarPageTimings(); + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} + +/// +/// Represents page timing information. +/// +public sealed class HarPageTimings +{ + /// + /// Gets or sets the time in milliseconds for the page to load. + /// + public double OnContentLoad { get; set; } = -1; + + /// + /// Gets or sets the time in milliseconds for the page to finish loading. + /// + public double OnLoad { get; set; } = -1; + + /// + /// Gets or sets an optional comment. + /// + public string? Comment { get; set; } +} diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs new file mode 100644 index 0000000000000..455498537dd54 --- /dev/null +++ b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs @@ -0,0 +1,114 @@ +// +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +using NUnit.Framework; +using OpenQA.Selenium.BiDi.BrowsingContext; +using OpenQA.Selenium.BiDi.Network.Har; +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace OpenQA.Selenium.BiDi.Network; + +class HarCaptureTest : BiDiTestFixture +{ + [Test] + public async Task CanCaptureNetworkTrafficToHar() + { + await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions + { + BrowserName = "TestBrowser", + BrowserVersion = "1.0" + }); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + var har = recorder.GetHar(); + + Assert.That(har, Is.Not.Null); + Assert.That(har.Log, Is.Not.Null); + Assert.That(har.Log.Version, Is.EqualTo("1.2")); + Assert.That(har.Log.Creator.Name, Is.EqualTo("Selenium")); + Assert.That(har.Log.Browser, Is.Not.Null); + Assert.That(har.Log.Browser.Name, Is.EqualTo("TestBrowser")); + Assert.That(har.Log.Browser.Version, Is.EqualTo("1.0")); + Assert.That(har.Log.Entries, Is.Not.Empty); + + var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + Assert.That(entry, Is.Not.Null); + Assert.That(entry.Request.Method, Is.EqualTo("GET")); + Assert.That(entry.Response.Status, Is.EqualTo(200)); + } + + [Test] + public async Task CanSaveHarToFile() + { + var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); + + try + { + await using var recorder = await bidi.CaptureNetworkTrafficAsync(); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + await recorder.SaveAsync(tempFile); + + Assert.That(File.Exists(tempFile), Is.True); + + var jsonContent = await File.ReadAllTextAsync(tempFile); + Assert.That(jsonContent, Is.Not.Empty); + + var harFile = JsonSerializer.Deserialize(jsonContent, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }); + + Assert.That(harFile, Is.Not.Null); + Assert.That(harFile.Log, Is.Not.Null); + Assert.That(harFile.Log.Entries, Is.Not.Empty); + } + finally + { + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + } + } + + [Test] + public async Task HarEntriesContainRequestDetails() + { + await using var recorder = await bidi.CaptureNetworkTrafficAsync(); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + var har = recorder.GetHar(); + var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + + Assert.That(entry, Is.Not.Null); + Assert.That(entry.StartedDateTime, Is.Not.Empty); + Assert.That(entry.Time, Is.GreaterThanOrEqualTo(0)); + Assert.That(entry.Request.Headers, Is.Not.Empty); + Assert.That(entry.Response.Headers, Is.Not.Empty); + Assert.That(entry.Timings, Is.Not.Null); + } +} From a8e0e28e80d7e2560cb888a4f1b33cc44e2a99ed Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:17:05 +0000 Subject: [PATCH 03/12] Fix BytesValue conversion and add documentation Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 14 ++-- .../src/webdriver/BiDi/Network/Har/README.md | 70 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 dotnet/src/webdriver/BiDi/Network/Har/README.md diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index e7ed7e4d1353d..2fb13c1da4ada 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -176,7 +176,7 @@ private HarRequest ConvertRequest(RequestData request) harRequest.Headers.Add(new HarHeader { Name = header.Name, - Value = header.Value.ToString() + Value = (string)header.Value }); } @@ -185,7 +185,12 @@ private HarRequest ConvertRequest(RequestData request) harRequest.Cookies.Add(new HarCookie { Name = cookie.Name, - Value = cookie.Value.ToString() + Value = (string)cookie.Value, + Domain = cookie.Domain, + Path = cookie.Path, + HttpOnly = cookie.HttpOnly, + Secure = cookie.Secure, + Expires = cookie.Expiry?.ToString("o") }); } @@ -227,16 +232,17 @@ private HarResponse ConvertResponse(ResponseData response) foreach (var header in response.Headers) { + var headerValue = (string)header.Value; harResponse.Headers.Add(new HarHeader { Name = header.Name, - Value = header.Value.ToString() + Value = headerValue }); // Check for redirect URL if (header.Name.Equals("Location", StringComparison.OrdinalIgnoreCase)) { - harResponse.RedirectURL = header.Value.ToString(); + harResponse.RedirectURL = headerValue; } } diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md new file mode 100644 index 0000000000000..6ac5163bb2482 --- /dev/null +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -0,0 +1,70 @@ +# HAR Capture Extension for BiDi + +This extension provides the ability to capture network traffic using the BiDi protocol and export it to HAR (HTTP Archive) format. + +## Usage Example + +```csharp +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.BiDi; +using OpenQA.Selenium.BiDi.Network.Har; + +// Create a WebDriver instance with BiDi enabled +var options = new ChromeOptions(); +options.AddArgument("--remote-allow-origins=*"); +options.EnableBiDi(); + +using var driver = new ChromeDriver(options); + +// Connect to BiDi +await using var bidi = await driver.AsBiDiAsync(); + +// Start capturing network traffic +await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions +{ + BrowserName = "Chrome", + BrowserVersion = "120.0" +}); + +// Navigate to a page +driver.Navigate().GoToUrl("https://www.example.com"); + +// Wait for some network activity +await Task.Delay(2000); + +// Save the captured traffic to a HAR file +await recorder.SaveAsync("network-traffic.har"); + +// Or get the HAR object directly +var har = recorder.GetHar(); +Console.WriteLine($"Captured {har.Log.Entries.Count} network requests"); +``` + +## HAR Capture Options + +The `HarCaptureOptions` class allows you to configure the capture: + +- `IncludeResponseContent`: Whether to include response content in the HAR file (default: false) +- `IncludeContent`: Whether to include request/response body content (default: false) +- `BrowserName`: The browser name to include in the HAR metadata +- `BrowserVersion`: The browser version to include in the HAR metadata + +## HAR File Format + +The generated HAR file follows the HAR 1.2 specification and includes: + +- Request details (method, URL, headers, cookies, query parameters) +- Response details (status code, headers, content type) +- Timing information (DNS, connect, SSL, send, wait, receive) +- Metadata (browser info, timestamps) + +## Disposing the Recorder + +The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events: + +```csharp +await using var recorder = await bidi.CaptureNetworkTrafficAsync(); +// ... capture network traffic ... +// Dispose is called automatically when leaving the using block +``` From 9d58d5610061f3e191451a24b7cd86c853c44a03 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:20:46 +0000 Subject: [PATCH 04/12] Add HAR files to gitignore Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 246f2c4bdaf7d..6bfe8311431a8 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,6 @@ javascript/selenium-webdriver/.vscode/settings.json dotnet-bin .metadata/ .npmrc + +# HAR files generated during tests +*.har From 9e4f5eee79157ad9032228aa8fcfec80bd77c3b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 17:53:41 +0000 Subject: [PATCH 05/12] Add request/response body collection support Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 73 +++++++++++++++++-- .../src/webdriver/BiDi/Network/Har/README.md | 13 ++-- .../common/BiDi/Network/HarCaptureTest.cs | 19 +++++ 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index 2fb13c1da4ada..de13a1ed8d7d4 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -52,11 +52,6 @@ public static async Task CaptureNetworkTrafficAsync(this BiDi bidi, /// public sealed class HarCaptureOptions { - /// - /// Gets or sets a value indicating whether to include response content in the HAR file. - /// - public bool IncludeResponseContent { get; set; } = false; - /// /// Gets or sets a value indicating whether to include request/response body content in the HAR file. /// @@ -85,6 +80,7 @@ public sealed class HarRecorder : IAsyncDisposable private Subscription? _beforeRequestSubscription; private Subscription? _responseStartedSubscription; private Subscription? _responseCompletedSubscription; + private Collector? _dataCollector; internal HarRecorder(BiDi bidi, HarCaptureOptions options) { @@ -105,6 +101,12 @@ internal HarRecorder(BiDi bidi, HarCaptureOptions options) internal async Task StartAsync() { + // Add data collector if content capture is enabled + if (_options.IncludeContent) + { + _dataCollector = await _bidi.Network.AddDataCollectorAsync([DataType.Request, DataType.Response], 200000000).ConfigureAwait(false); + } + _beforeRequestSubscription = await _bidi.Network.OnBeforeRequestSentAsync(OnBeforeRequestSent).ConfigureAwait(false); _responseStartedSubscription = await _bidi.Network.OnResponseStartedAsync(OnResponseStarted).ConfigureAwait(false); _responseCompletedSubscription = await _bidi.Network.OnResponseCompletedAsync(OnResponseCompleted).ConfigureAwait(false); @@ -142,24 +144,76 @@ private void OnResponseStarted(ResponseStartedEventArgs args) } } - private void OnResponseCompleted(ResponseCompletedEventArgs args) + private async void OnResponseCompleted(ResponseCompletedEventArgs args) { + HarEntry? entry = null; + lock (_pendingRequests) { - if (_pendingRequests.TryGetValue(args.Request.Request.Id, out var entry)) + if (_pendingRequests.TryGetValue(args.Request.Request.Id, out entry)) { entry.Response = ConvertResponse(args.Response); // Calculate total time var timings = args.Request.Timings; entry.Time = CalculateTotalTime(timings); + } + } + if (entry != null) + { + // Retrieve request and response bodies if content capture is enabled + if (_options.IncludeContent && _dataCollector != null) + { + try + { + // Get request body + var requestBody = await _bidi.Network.GetDataAsync(DataType.Request, args.Request.Request).ConfigureAwait(false); + if (requestBody != null) + { + entry.Request.PostData = new HarPostData + { + MimeType = GetContentType(entry.Request.Headers), + Text = (string)requestBody + }; + } + } + catch + { + // Request body may not be available for all requests (e.g., GET requests) + } + + try + { + // Get response body + var responseBody = await _bidi.Network.GetDataAsync(DataType.Response, args.Request.Request).ConfigureAwait(false); + if (responseBody != null) + { + var bodyText = (string)responseBody; + entry.Response.Content.Text = bodyText; + entry.Response.Content.Size = System.Text.Encoding.UTF8.GetByteCount(bodyText); + } + } + catch + { + // Response body may not be available for all responses + } + } + + lock (_pendingRequests) + { _harFile.Log.Entries.Add(entry); _pendingRequests.Remove(args.Request.Request.Id); } } } + private string GetContentType(List headers) + { + var contentTypeHeader = headers.FirstOrDefault(h => h.Name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase)); + return contentTypeHeader?.Value ?? "application/octet-stream"; + } + private HarRequest ConvertRequest(RequestData request) { var harRequest = new HarRequest @@ -332,5 +386,10 @@ public async ValueTask DisposeAsync() { await _responseCompletedSubscription.DisposeAsync().ConfigureAwait(false); } + + if (_dataCollector != null) + { + await _bidi.Network.RemoveDataCollectorAsync(_dataCollector).ConfigureAwait(false); + } } } diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index 6ac5163bb2482..ebb570a45c893 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -20,11 +20,12 @@ using var driver = new ChromeDriver(options); // Connect to BiDi await using var bidi = await driver.AsBiDiAsync(); -// Start capturing network traffic +// Start capturing network traffic with body content await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions { BrowserName = "Chrome", - BrowserVersion = "120.0" + BrowserVersion = "120.0", + IncludeContent = true // Enable request/response body capture }); // Navigate to a page @@ -45,11 +46,12 @@ Console.WriteLine($"Captured {har.Log.Entries.Count} network requests"); The `HarCaptureOptions` class allows you to configure the capture: -- `IncludeResponseContent`: Whether to include response content in the HAR file (default: false) -- `IncludeContent`: Whether to include request/response body content (default: false) +- `IncludeContent`: Whether to include request/response body content (default: false). When enabled, a data collector is created to capture request and response bodies. - `BrowserName`: The browser name to include in the HAR metadata - `BrowserVersion`: The browser version to include in the HAR metadata +**Note:** Setting `IncludeContent = true` will create a network data collector that captures request and response bodies. This may increase memory usage for large requests/responses. + ## HAR File Format The generated HAR file follows the HAR 1.2 specification and includes: @@ -57,11 +59,12 @@ The generated HAR file follows the HAR 1.2 specification and includes: - Request details (method, URL, headers, cookies, query parameters) - Response details (status code, headers, content type) - Timing information (DNS, connect, SSL, send, wait, receive) +- Request/response body content (when `IncludeContent` is enabled) - Metadata (browser info, timestamps) ## Disposing the Recorder -The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events: +The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: ```csharp await using var recorder = await bidi.CaptureNetworkTrafficAsync(); diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs index 455498537dd54..990c395cd72f0 100644 --- a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs +++ b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs @@ -111,4 +111,23 @@ public async Task HarEntriesContainRequestDetails() Assert.That(entry.Response.Headers, Is.Not.Empty); Assert.That(entry.Timings, Is.Not.Null); } + + [Test] + public async Task CanCaptureRequestAndResponseBodies() + { + await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions + { + IncludeContent = true + }); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + var har = recorder.GetHar(); + var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + + Assert.That(entry, Is.Not.Null); + Assert.That(entry.Response.Content.Text, Is.Not.Null); + Assert.That(entry.Response.Content.Text, Is.Not.Empty); + Assert.That(entry.Response.Content.Size, Is.GreaterThan(0)); + } } From 0af30527663e9edbcfce964f9148f75324a624f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 18:04:15 +0000 Subject: [PATCH 06/12] Make body collection unconditional and enabled by default Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 16 ++++------------ .../src/webdriver/BiDi/Network/Har/README.md | 18 ++++++++++-------- .../test/common/BiDi/Network/HarCaptureTest.cs | 5 +---- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index de13a1ed8d7d4..7d86020c720fa 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -52,11 +52,6 @@ public static async Task CaptureNetworkTrafficAsync(this BiDi bidi, /// public sealed class HarCaptureOptions { - /// - /// Gets or sets a value indicating whether to include request/response body content in the HAR file. - /// - public bool IncludeContent { get; set; } = false; - /// /// Gets or sets the browser name to include in the HAR file. /// @@ -101,11 +96,8 @@ internal HarRecorder(BiDi bidi, HarCaptureOptions options) internal async Task StartAsync() { - // Add data collector if content capture is enabled - if (_options.IncludeContent) - { - _dataCollector = await _bidi.Network.AddDataCollectorAsync([DataType.Request, DataType.Response], 200000000).ConfigureAwait(false); - } + // Always create data collector for capturing request and response bodies + _dataCollector = await _bidi.Network.AddDataCollectorAsync([DataType.Request, DataType.Response], 200000000).ConfigureAwait(false); _beforeRequestSubscription = await _bidi.Network.OnBeforeRequestSentAsync(OnBeforeRequestSent).ConfigureAwait(false); _responseStartedSubscription = await _bidi.Network.OnResponseStartedAsync(OnResponseStarted).ConfigureAwait(false); @@ -162,8 +154,8 @@ private async void OnResponseCompleted(ResponseCompletedEventArgs args) if (entry != null) { - // Retrieve request and response bodies if content capture is enabled - if (_options.IncludeContent && _dataCollector != null) + // Retrieve request and response bodies + if (_dataCollector != null) { try { diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index ebb570a45c893..11be6736a69db 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -1,6 +1,6 @@ # HAR Capture Extension for BiDi -This extension provides the ability to capture network traffic using the BiDi protocol and export it to HAR (HTTP Archive) format. +This extension provides the ability to capture network traffic using the BiDi protocol and export it to HAR (HTTP Archive) format, including request and response body content. ## Usage Example @@ -20,12 +20,11 @@ using var driver = new ChromeDriver(options); // Connect to BiDi await using var bidi = await driver.AsBiDiAsync(); -// Start capturing network traffic with body content +// Start capturing network traffic (includes request/response bodies by default) await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions { BrowserName = "Chrome", - BrowserVersion = "120.0", - IncludeContent = true // Enable request/response body capture + BrowserVersion = "120.0" }); // Navigate to a page @@ -46,12 +45,9 @@ Console.WriteLine($"Captured {har.Log.Entries.Count} network requests"); The `HarCaptureOptions` class allows you to configure the capture: -- `IncludeContent`: Whether to include request/response body content (default: false). When enabled, a data collector is created to capture request and response bodies. - `BrowserName`: The browser name to include in the HAR metadata - `BrowserVersion`: The browser version to include in the HAR metadata -**Note:** Setting `IncludeContent = true` will create a network data collector that captures request and response bodies. This may increase memory usage for large requests/responses. - ## HAR File Format The generated HAR file follows the HAR 1.2 specification and includes: @@ -59,9 +55,15 @@ The generated HAR file follows the HAR 1.2 specification and includes: - Request details (method, URL, headers, cookies, query parameters) - Response details (status code, headers, content type) - Timing information (DNS, connect, SSL, send, wait, receive) -- Request/response body content (when `IncludeContent` is enabled) +- Request/response body content (automatically captured) - Metadata (browser info, timestamps) +## Body Content Capture + +By default, the HAR recorder captures request and response bodies for all network traffic. A network data collector is automatically created when you start capturing traffic. This provides complete visibility into all request and response payloads. + +**Note:** Capturing request/response bodies may increase memory usage for large requests/responses. + ## Disposing the Recorder The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs index 990c395cd72f0..a43c6c2b163c4 100644 --- a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs +++ b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs @@ -115,10 +115,7 @@ public async Task HarEntriesContainRequestDetails() [Test] public async Task CanCaptureRequestAndResponseBodies() { - await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions - { - IncludeContent = true - }); + await using var recorder = await bidi.CaptureNetworkTrafficAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); From 8a9e201a84b5d16cf07e105c4abf9941ae459932 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 18:20:20 +0000 Subject: [PATCH 07/12] Optimize memory usage by flushing entries to temp file Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .gitignore | 1 + .../BiDi/Network/Har/BiDi.HarExtensions.cs | 68 ++++++++++++++++++- .../src/webdriver/BiDi/Network/Har/README.md | 2 + 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6bfe8311431a8..6dc8f1d656228 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,4 @@ dotnet-bin # HAR files generated during tests *.har +selenium-har-*.jsonl diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index 7d86020c720fa..44ede2357d600 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -72,6 +72,8 @@ public sealed class HarRecorder : IAsyncDisposable private readonly HarCaptureOptions _options; private readonly HarFile _harFile; private readonly Dictionary _pendingRequests; + private readonly string _tempFilePath; + private readonly object _tempFileLock = new object(); private Subscription? _beforeRequestSubscription; private Subscription? _responseStartedSubscription; private Subscription? _responseCompletedSubscription; @@ -83,6 +85,7 @@ internal HarRecorder(BiDi bidi, HarCaptureOptions options) _options = options ?? throw new ArgumentNullException(nameof(options)); _harFile = new HarFile(); _pendingRequests = new Dictionary(); + _tempFilePath = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.jsonl"); if (!string.IsNullOrEmpty(options.BrowserName)) { @@ -194,12 +197,28 @@ private async void OnResponseCompleted(ResponseCompletedEventArgs args) lock (_pendingRequests) { - _harFile.Log.Entries.Add(entry); + // Flush entry to temp file instead of keeping in memory + FlushEntryToTempFile(entry); _pendingRequests.Remove(args.Request.Request.Id); } } } + private void FlushEntryToTempFile(HarEntry entry) + { + lock (_tempFileLock) + { + var jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + var entryJson = JsonSerializer.Serialize(entry, jsonOptions); + File.AppendAllText(_tempFilePath, entryJson + Environment.NewLine); + } + } + private string GetContentType(List headers) { var contentTypeHeader = headers.FirstOrDefault(h => h.Name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase)); @@ -333,9 +352,40 @@ private double CalculateTotalTime(FetchTimingInfo timings) /// The HAR file containing all captured network traffic. public HarFile GetHar() { + // Read entries from temp file + LoadEntriesFromTempFile(); return _harFile; } + private void LoadEntriesFromTempFile() + { + lock (_tempFileLock) + { + _harFile.Log.Entries.Clear(); + + if (File.Exists(_tempFilePath)) + { + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }; + + var lines = File.ReadAllLines(_tempFilePath); + foreach (var line in lines) + { + if (!string.IsNullOrWhiteSpace(line)) + { + var entry = JsonSerializer.Deserialize(line, jsonOptions); + if (entry != null) + { + _harFile.Log.Entries.Add(entry); + } + } + } + } + } + } + /// /// Saves the captured network traffic to a HAR file. /// @@ -348,6 +398,9 @@ public async Task SaveAsync(string filePath) throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); } + // Load entries from temp file before saving + LoadEntriesFromTempFile(); + var options = new JsonSerializerOptions { WriteIndented = true, @@ -383,5 +436,18 @@ public async ValueTask DisposeAsync() { await _bidi.Network.RemoveDataCollectorAsync(_dataCollector).ConfigureAwait(false); } + + // Clean up temp file + if (File.Exists(_tempFilePath)) + { + try + { + File.Delete(_tempFilePath); + } + catch + { + // Ignore errors when deleting temp file + } + } } } diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index 11be6736a69db..57607e5438eb5 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -62,6 +62,8 @@ The generated HAR file follows the HAR 1.2 specification and includes: By default, the HAR recorder captures request and response bodies for all network traffic. A network data collector is automatically created when you start capturing traffic. This provides complete visibility into all request and response payloads. +**Memory Optimization:** To minimize memory usage, captured network entries are written to a temporary file as they are completed. The entries are only loaded into memory when you call `GetHar()` or `SaveAsync()`. This allows for capturing large amounts of network traffic without consuming excessive memory. + **Note:** Capturing request/response bodies may increase memory usage for large requests/responses. ## Disposing the Recorder From 627a1afd8d24fe8441ab6d076079bd26b0de99c6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 18:31:09 +0000 Subject: [PATCH 08/12] Rename CaptureNetworkTrafficAsync to CaptureHarAsync Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs | 2 +- dotnet/src/webdriver/BiDi/Network/Har/README.md | 4 ++-- dotnet/test/common/BiDi/Network/HarCaptureTest.cs | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index 44ede2357d600..2a573e787ac80 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -37,7 +37,7 @@ public static class BiDiHarExtensions /// The BiDi instance. /// Optional configuration options. /// A task that represents the asynchronous operation and returns a HarRecorder. - public static async Task CaptureNetworkTrafficAsync(this BiDi bidi, HarCaptureOptions? options = null) + public static async Task CaptureHarAsync(this BiDi bidi, HarCaptureOptions? options = null) { if (bidi is null) throw new ArgumentNullException(nameof(bidi)); diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index 57607e5438eb5..d297f6620acee 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -21,7 +21,7 @@ using var driver = new ChromeDriver(options); await using var bidi = await driver.AsBiDiAsync(); // Start capturing network traffic (includes request/response bodies by default) -await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions +await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions { BrowserName = "Chrome", BrowserVersion = "120.0" @@ -71,7 +71,7 @@ By default, the HAR recorder captures request and response bodies for all networ The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: ```csharp -await using var recorder = await bidi.CaptureNetworkTrafficAsync(); +await using var recorder = await bidi.CaptureHarAsync(); // ... capture network traffic ... // Dispose is called automatically when leaving the using block ``` diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs index a43c6c2b163c4..638f19db85527 100644 --- a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs +++ b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs @@ -33,7 +33,7 @@ class HarCaptureTest : BiDiTestFixture [Test] public async Task CanCaptureNetworkTrafficToHar() { - await using var recorder = await bidi.CaptureNetworkTrafficAsync(new HarCaptureOptions + await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions { BrowserName = "TestBrowser", BrowserVersion = "1.0" @@ -65,7 +65,7 @@ public async Task CanSaveHarToFile() try { - await using var recorder = await bidi.CaptureNetworkTrafficAsync(); + await using var recorder = await bidi.CaptureHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); @@ -97,7 +97,7 @@ public async Task CanSaveHarToFile() [Test] public async Task HarEntriesContainRequestDetails() { - await using var recorder = await bidi.CaptureNetworkTrafficAsync(); + await using var recorder = await bidi.CaptureHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); @@ -115,7 +115,7 @@ public async Task HarEntriesContainRequestDetails() [Test] public async Task CanCaptureRequestAndResponseBodies() { - await using var recorder = await bidi.CaptureNetworkTrafficAsync(); + await using var recorder = await bidi.CaptureHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); From e0cf3f24f762764f457a07a2b298e523643f0e60 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 18:42:16 +0000 Subject: [PATCH 09/12] Extract IHarRecorder interface and remove GetHar method Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 30 ++-- .../src/webdriver/BiDi/Network/Har/README.md | 8 +- .../common/BiDi/Network/HarCaptureTest.cs | 140 +++++++++++++----- 3 files changed, 118 insertions(+), 60 deletions(-) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index 2a573e787ac80..3cd7cd2c7724f 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -36,8 +36,8 @@ public static class BiDiHarExtensions /// /// The BiDi instance. /// Optional configuration options. - /// A task that represents the asynchronous operation and returns a HarRecorder. - public static async Task CaptureHarAsync(this BiDi bidi, HarCaptureOptions? options = null) + /// A task that represents the asynchronous operation and returns a IHarRecorder. + public static async Task CaptureHarAsync(this BiDi bidi, HarCaptureOptions? options = null) { if (bidi is null) throw new ArgumentNullException(nameof(bidi)); @@ -63,10 +63,23 @@ public sealed class HarCaptureOptions public string? BrowserVersion { get; set; } } +/// +/// Interface for recording network traffic and saving it as HAR format. +/// +public interface IHarRecorder : IAsyncDisposable +{ + /// + /// Saves the captured network traffic to a HAR file. + /// + /// The path where the HAR file should be saved. + /// A task that represents the asynchronous operation. + Task SaveAsync(string filePath); +} + /// /// Records network traffic and provides methods to save it as HAR format. /// -public sealed class HarRecorder : IAsyncDisposable +public sealed class HarRecorder : IHarRecorder { private readonly BiDi _bidi; private readonly HarCaptureOptions _options; @@ -346,17 +359,6 @@ private double CalculateTotalTime(FetchTimingInfo timings) return 0; } - /// - /// Gets the captured HAR file. - /// - /// The HAR file containing all captured network traffic. - public HarFile GetHar() - { - // Read entries from temp file - LoadEntriesFromTempFile(); - return _harFile; - } - private void LoadEntriesFromTempFile() { lock (_tempFileLock) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index d297f6620acee..6b1e4369512d7 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -35,10 +35,6 @@ await Task.Delay(2000); // Save the captured traffic to a HAR file await recorder.SaveAsync("network-traffic.har"); - -// Or get the HAR object directly -var har = recorder.GetHar(); -Console.WriteLine($"Captured {har.Log.Entries.Count} network requests"); ``` ## HAR Capture Options @@ -62,13 +58,13 @@ The generated HAR file follows the HAR 1.2 specification and includes: By default, the HAR recorder captures request and response bodies for all network traffic. A network data collector is automatically created when you start capturing traffic. This provides complete visibility into all request and response payloads. -**Memory Optimization:** To minimize memory usage, captured network entries are written to a temporary file as they are completed. The entries are only loaded into memory when you call `GetHar()` or `SaveAsync()`. This allows for capturing large amounts of network traffic without consuming excessive memory. +**Memory Optimization:** To minimize memory usage, captured network entries are written to a temporary file as they are completed. The entries are only loaded into memory when you call `SaveAsync()`. This allows for capturing large amounts of network traffic without consuming excessive memory. **Note:** Capturing request/response bodies may increase memory usage for large requests/responses. ## Disposing the Recorder -The `HarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: +The `IHarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: ```csharp await using var recorder = await bidi.CaptureHarAsync(); diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs index 638f19db85527..1bbbadad45610 100644 --- a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs +++ b/dotnet/test/common/BiDi/Network/HarCaptureTest.cs @@ -33,29 +33,51 @@ class HarCaptureTest : BiDiTestFixture [Test] public async Task CanCaptureNetworkTrafficToHar() { - await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions + var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); + + try + { + await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions + { + BrowserName = "TestBrowser", + BrowserVersion = "1.0" + }); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + await recorder.SaveAsync(tempFile); + + Assert.That(File.Exists(tempFile), Is.True); + + var jsonContent = await File.ReadAllTextAsync(tempFile); + Assert.That(jsonContent, Is.Not.Empty); + + var harFile = JsonSerializer.Deserialize(jsonContent, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }); + + Assert.That(harFile, Is.Not.Null); + Assert.That(harFile.Log, Is.Not.Null); + Assert.That(harFile.Log.Version, Is.EqualTo("1.2")); + Assert.That(harFile.Log.Creator.Name, Is.EqualTo("Selenium")); + Assert.That(harFile.Log.Browser, Is.Not.Null); + Assert.That(harFile.Log.Browser.Name, Is.EqualTo("TestBrowser")); + Assert.That(harFile.Log.Browser.Version, Is.EqualTo("1.0")); + Assert.That(harFile.Log.Entries, Is.Not.Empty); + + var entry = harFile.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + Assert.That(entry, Is.Not.Null); + Assert.That(entry.Request.Method, Is.EqualTo("GET")); + Assert.That(entry.Response.Status, Is.EqualTo(200)); + } + finally { - BrowserName = "TestBrowser", - BrowserVersion = "1.0" - }); - - await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); - - var har = recorder.GetHar(); - - Assert.That(har, Is.Not.Null); - Assert.That(har.Log, Is.Not.Null); - Assert.That(har.Log.Version, Is.EqualTo("1.2")); - Assert.That(har.Log.Creator.Name, Is.EqualTo("Selenium")); - Assert.That(har.Log.Browser, Is.Not.Null); - Assert.That(har.Log.Browser.Name, Is.EqualTo("TestBrowser")); - Assert.That(har.Log.Browser.Version, Is.EqualTo("1.0")); - Assert.That(har.Log.Entries, Is.Not.Empty); - - var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); - Assert.That(entry, Is.Not.Null); - Assert.That(entry.Request.Method, Is.EqualTo("GET")); - Assert.That(entry.Response.Status, Is.EqualTo(200)); + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + } } [Test] @@ -97,34 +119,72 @@ public async Task CanSaveHarToFile() [Test] public async Task HarEntriesContainRequestDetails() { - await using var recorder = await bidi.CaptureHarAsync(); + var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); + + try + { + await using var recorder = await bidi.CaptureHarAsync(); + + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + await recorder.SaveAsync(tempFile); - await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + var jsonContent = await File.ReadAllTextAsync(tempFile); + var harFile = JsonSerializer.Deserialize(jsonContent, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }); - var har = recorder.GetHar(); - var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + var entry = harFile.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); - Assert.That(entry, Is.Not.Null); - Assert.That(entry.StartedDateTime, Is.Not.Empty); - Assert.That(entry.Time, Is.GreaterThanOrEqualTo(0)); - Assert.That(entry.Request.Headers, Is.Not.Empty); - Assert.That(entry.Response.Headers, Is.Not.Empty); - Assert.That(entry.Timings, Is.Not.Null); + Assert.That(entry, Is.Not.Null); + Assert.That(entry.StartedDateTime, Is.Not.Empty); + Assert.That(entry.Time, Is.GreaterThanOrEqualTo(0)); + Assert.That(entry.Request.Headers, Is.Not.Empty); + Assert.That(entry.Response.Headers, Is.Not.Empty); + Assert.That(entry.Timings, Is.Not.Null); + } + finally + { + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + } } [Test] public async Task CanCaptureRequestAndResponseBodies() { - await using var recorder = await bidi.CaptureHarAsync(); + var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); + + try + { + await using var recorder = await bidi.CaptureHarAsync(); - await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); + + await recorder.SaveAsync(tempFile); - var har = recorder.GetHar(); - var entry = har.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + var jsonContent = await File.ReadAllTextAsync(tempFile); + var harFile = JsonSerializer.Deserialize(jsonContent, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }); - Assert.That(entry, Is.Not.Null); - Assert.That(entry.Response.Content.Text, Is.Not.Null); - Assert.That(entry.Response.Content.Text, Is.Not.Empty); - Assert.That(entry.Response.Content.Size, Is.GreaterThan(0)); + var entry = harFile.Log.Entries.FirstOrDefault(e => e.Request.Url.Contains("logEntryAdded.html")); + + Assert.That(entry, Is.Not.Null); + Assert.That(entry.Response.Content.Text, Is.Not.Null); + Assert.That(entry.Response.Content.Text, Is.Not.Empty); + Assert.That(entry.Response.Content.Size, Is.GreaterThan(0)); + } + finally + { + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } + } } } From a26d88e40fe9c19a9d3babaeef054e619e68c22c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 18:54:35 +0000 Subject: [PATCH 10/12] Rename from Capture to Record terminology for consistency Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 14 +++++----- .../src/webdriver/BiDi/Network/Har/README.md | 28 +++++++++---------- ...{HarCaptureTest.cs => HarRecordingTest.cs} | 16 +++++------ 3 files changed, 29 insertions(+), 29 deletions(-) rename dotnet/test/common/BiDi/Network/{HarCaptureTest.cs => HarRecordingTest.cs} (92%) diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index 3cd7cd2c7724f..d250fe8735409 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -32,25 +32,25 @@ namespace OpenQA.Selenium.BiDi.Network.Har; public static class BiDiHarExtensions { /// - /// Captures network traffic and returns a HAR recorder that can be used to save the captured traffic. + /// Records network traffic and returns a HAR recorder that can be used to save the recorded traffic. /// /// The BiDi instance. /// Optional configuration options. /// A task that represents the asynchronous operation and returns a IHarRecorder. - public static async Task CaptureHarAsync(this BiDi bidi, HarCaptureOptions? options = null) + public static async Task RecordHarAsync(this BiDi bidi, HarRecordingOptions? options = null) { if (bidi is null) throw new ArgumentNullException(nameof(bidi)); - var recorder = new HarRecorder(bidi, options ?? new HarCaptureOptions()); + var recorder = new HarRecorder(bidi, options ?? new HarRecordingOptions()); await recorder.StartAsync().ConfigureAwait(false); return recorder; } } /// -/// Options for HAR capture. +/// Options for HAR recording. /// -public sealed class HarCaptureOptions +public sealed class HarRecordingOptions { /// /// Gets or sets the browser name to include in the HAR file. @@ -82,7 +82,7 @@ public interface IHarRecorder : IAsyncDisposable public sealed class HarRecorder : IHarRecorder { private readonly BiDi _bidi; - private readonly HarCaptureOptions _options; + private readonly HarRecordingOptions _options; private readonly HarFile _harFile; private readonly Dictionary _pendingRequests; private readonly string _tempFilePath; @@ -92,7 +92,7 @@ public sealed class HarRecorder : IHarRecorder private Subscription? _responseCompletedSubscription; private Collector? _dataCollector; - internal HarRecorder(BiDi bidi, HarCaptureOptions options) + internal HarRecorder(BiDi bidi, HarRecordingOptions options) { _bidi = bidi ?? throw new ArgumentNullException(nameof(bidi)); _options = options ?? throw new ArgumentNullException(nameof(options)); diff --git a/dotnet/src/webdriver/BiDi/Network/Har/README.md b/dotnet/src/webdriver/BiDi/Network/Har/README.md index 6b1e4369512d7..8179a1b5c7d06 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/README.md +++ b/dotnet/src/webdriver/BiDi/Network/Har/README.md @@ -1,6 +1,6 @@ -# HAR Capture Extension for BiDi +# HAR Recording Extension for BiDi -This extension provides the ability to capture network traffic using the BiDi protocol and export it to HAR (HTTP Archive) format, including request and response body content. +This extension provides the ability to record network traffic using the BiDi protocol and export it to HAR (HTTP Archive) format, including request and response body content. ## Usage Example @@ -20,8 +20,8 @@ using var driver = new ChromeDriver(options); // Connect to BiDi await using var bidi = await driver.AsBiDiAsync(); -// Start capturing network traffic (includes request/response bodies by default) -await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions +// Start recording network traffic (includes request/response bodies by default) +await using var recorder = await bidi.RecordHarAsync(new HarRecordingOptions { BrowserName = "Chrome", BrowserVersion = "120.0" @@ -33,13 +33,13 @@ driver.Navigate().GoToUrl("https://www.example.com"); // Wait for some network activity await Task.Delay(2000); -// Save the captured traffic to a HAR file +// Save the recorded traffic to a HAR file await recorder.SaveAsync("network-traffic.har"); ``` -## HAR Capture Options +## HAR Recording Options -The `HarCaptureOptions` class allows you to configure the capture: +The `HarRecordingOptions` class allows you to configure the recording: - `BrowserName`: The browser name to include in the HAR metadata - `BrowserVersion`: The browser version to include in the HAR metadata @@ -51,23 +51,23 @@ The generated HAR file follows the HAR 1.2 specification and includes: - Request details (method, URL, headers, cookies, query parameters) - Response details (status code, headers, content type) - Timing information (DNS, connect, SSL, send, wait, receive) -- Request/response body content (automatically captured) +- Request/response body content (automatically recorded) - Metadata (browser info, timestamps) -## Body Content Capture +## Body Content Recording -By default, the HAR recorder captures request and response bodies for all network traffic. A network data collector is automatically created when you start capturing traffic. This provides complete visibility into all request and response payloads. +By default, the HAR recorder records request and response bodies for all network traffic. A network data collector is automatically created when you start recording traffic. This provides complete visibility into all request and response payloads. -**Memory Optimization:** To minimize memory usage, captured network entries are written to a temporary file as they are completed. The entries are only loaded into memory when you call `SaveAsync()`. This allows for capturing large amounts of network traffic without consuming excessive memory. +**Memory Optimization:** To minimize memory usage, recorded network entries are written to a temporary file as they are completed. The entries are only loaded into memory when you call `SaveAsync()`. This allows for recording large amounts of network traffic without consuming excessive memory. -**Note:** Capturing request/response bodies may increase memory usage for large requests/responses. +**Note:** Recording request/response bodies may increase memory usage for large requests/responses. ## Disposing the Recorder The `IHarRecorder` implements `IAsyncDisposable` and should be disposed properly to unsubscribe from network events and clean up the data collector: ```csharp -await using var recorder = await bidi.CaptureHarAsync(); -// ... capture network traffic ... +await using var recorder = await bidi.RecordHarAsync(); +// ... record network traffic ... // Dispose is called automatically when leaving the using block ``` diff --git a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs b/dotnet/test/common/BiDi/Network/HarRecordingTest.cs similarity index 92% rename from dotnet/test/common/BiDi/Network/HarCaptureTest.cs rename to dotnet/test/common/BiDi/Network/HarRecordingTest.cs index 1bbbadad45610..9c0db50188be6 100644 --- a/dotnet/test/common/BiDi/Network/HarCaptureTest.cs +++ b/dotnet/test/common/BiDi/Network/HarRecordingTest.cs @@ -1,4 +1,4 @@ -// +// // Licensed to the Software Freedom Conservancy (SFC) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -28,16 +28,16 @@ namespace OpenQA.Selenium.BiDi.Network; -class HarCaptureTest : BiDiTestFixture +class HarRecordingTest : BiDiTestFixture { [Test] - public async Task CanCaptureNetworkTrafficToHar() + public async Task CanRecordNetworkTrafficToHar() { var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); try { - await using var recorder = await bidi.CaptureHarAsync(new HarCaptureOptions + await using var recorder = await bidi.RecordHarAsync(new HarRecordingOptions { BrowserName = "TestBrowser", BrowserVersion = "1.0" @@ -87,7 +87,7 @@ public async Task CanSaveHarToFile() try { - await using var recorder = await bidi.CaptureHarAsync(); + await using var recorder = await bidi.RecordHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); @@ -123,7 +123,7 @@ public async Task HarEntriesContainRequestDetails() try { - await using var recorder = await bidi.CaptureHarAsync(); + await using var recorder = await bidi.RecordHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); @@ -154,13 +154,13 @@ public async Task HarEntriesContainRequestDetails() } [Test] - public async Task CanCaptureRequestAndResponseBodies() + public async Task CanRecordRequestAndResponseBodies() { var tempFile = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.har"); try { - await using var recorder = await bidi.CaptureHarAsync(); + await using var recorder = await bidi.RecordHarAsync(); await context.NavigateAsync(UrlBuilder.WhereIs("bidi/logEntryAdded.html"), new() { Wait = ReadinessState.Complete }); From 413276ec9d0723332bbd30c1fe1e650bbf6692f2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 19:13:05 +0000 Subject: [PATCH 11/12] Move HarRecorder to separate file Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .../BiDi/Network/Har/BiDi.HarExtensions.cs | 382 ----------------- .../webdriver/BiDi/Network/Har/HarRecorder.cs | 405 ++++++++++++++++++ 2 files changed, 405 insertions(+), 382 deletions(-) create mode 100644 dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs diff --git a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs index d250fe8735409..5499681e0963c 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/BiDi.HarExtensions.cs @@ -18,10 +18,6 @@ // using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.Json; using System.Threading.Tasks; namespace OpenQA.Selenium.BiDi.Network.Har; @@ -75,381 +71,3 @@ public interface IHarRecorder : IAsyncDisposable /// A task that represents the asynchronous operation. Task SaveAsync(string filePath); } - -/// -/// Records network traffic and provides methods to save it as HAR format. -/// -public sealed class HarRecorder : IHarRecorder -{ - private readonly BiDi _bidi; - private readonly HarRecordingOptions _options; - private readonly HarFile _harFile; - private readonly Dictionary _pendingRequests; - private readonly string _tempFilePath; - private readonly object _tempFileLock = new object(); - private Subscription? _beforeRequestSubscription; - private Subscription? _responseStartedSubscription; - private Subscription? _responseCompletedSubscription; - private Collector? _dataCollector; - - internal HarRecorder(BiDi bidi, HarRecordingOptions options) - { - _bidi = bidi ?? throw new ArgumentNullException(nameof(bidi)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _harFile = new HarFile(); - _pendingRequests = new Dictionary(); - _tempFilePath = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.jsonl"); - - if (!string.IsNullOrEmpty(options.BrowserName)) - { - _harFile.Log.Browser = new HarBrowser - { - Name = options.BrowserName, - Version = options.BrowserVersion ?? string.Empty - }; - } - } - - internal async Task StartAsync() - { - // Always create data collector for capturing request and response bodies - _dataCollector = await _bidi.Network.AddDataCollectorAsync([DataType.Request, DataType.Response], 200000000).ConfigureAwait(false); - - _beforeRequestSubscription = await _bidi.Network.OnBeforeRequestSentAsync(OnBeforeRequestSent).ConfigureAwait(false); - _responseStartedSubscription = await _bidi.Network.OnResponseStartedAsync(OnResponseStarted).ConfigureAwait(false); - _responseCompletedSubscription = await _bidi.Network.OnResponseCompletedAsync(OnResponseCompleted).ConfigureAwait(false); - } - - private void OnBeforeRequestSent(BeforeRequestSentEventArgs args) - { - var entry = new HarEntry - { - StartedDateTime = args.Timestamp.ToString("o"), - Request = ConvertRequest(args.Request), - Timings = ConvertTimings(args.Request.Timings), - Time = 0 - }; - - if (args.Context != null) - { - entry.Pageref = args.Context.Id; - } - - lock (_pendingRequests) - { - _pendingRequests[args.Request.Request.Id] = entry; - } - } - - private void OnResponseStarted(ResponseStartedEventArgs args) - { - lock (_pendingRequests) - { - if (_pendingRequests.TryGetValue(args.Request.Request.Id, out var entry)) - { - entry.Response = ConvertResponse(args.Response); - } - } - } - - private async void OnResponseCompleted(ResponseCompletedEventArgs args) - { - HarEntry? entry = null; - - lock (_pendingRequests) - { - if (_pendingRequests.TryGetValue(args.Request.Request.Id, out entry)) - { - entry.Response = ConvertResponse(args.Response); - - // Calculate total time - var timings = args.Request.Timings; - entry.Time = CalculateTotalTime(timings); - } - } - - if (entry != null) - { - // Retrieve request and response bodies - if (_dataCollector != null) - { - try - { - // Get request body - var requestBody = await _bidi.Network.GetDataAsync(DataType.Request, args.Request.Request).ConfigureAwait(false); - if (requestBody != null) - { - entry.Request.PostData = new HarPostData - { - MimeType = GetContentType(entry.Request.Headers), - Text = (string)requestBody - }; - } - } - catch - { - // Request body may not be available for all requests (e.g., GET requests) - } - - try - { - // Get response body - var responseBody = await _bidi.Network.GetDataAsync(DataType.Response, args.Request.Request).ConfigureAwait(false); - if (responseBody != null) - { - var bodyText = (string)responseBody; - entry.Response.Content.Text = bodyText; - entry.Response.Content.Size = System.Text.Encoding.UTF8.GetByteCount(bodyText); - } - } - catch - { - // Response body may not be available for all responses - } - } - - lock (_pendingRequests) - { - // Flush entry to temp file instead of keeping in memory - FlushEntryToTempFile(entry); - _pendingRequests.Remove(args.Request.Request.Id); - } - } - } - - private void FlushEntryToTempFile(HarEntry entry) - { - lock (_tempFileLock) - { - var jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull - }; - - var entryJson = JsonSerializer.Serialize(entry, jsonOptions); - File.AppendAllText(_tempFilePath, entryJson + Environment.NewLine); - } - } - - private string GetContentType(List headers) - { - var contentTypeHeader = headers.FirstOrDefault(h => h.Name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase)); - return contentTypeHeader?.Value ?? "application/octet-stream"; - } - - private HarRequest ConvertRequest(RequestData request) - { - var harRequest = new HarRequest - { - Method = request.Method, - Url = request.Url, - HttpVersion = "HTTP/1.1", - HeadersSize = request.HeadersSize ?? -1, - BodySize = request.BodySize ?? -1 - }; - - foreach (var header in request.Headers) - { - harRequest.Headers.Add(new HarHeader - { - Name = header.Name, - Value = (string)header.Value - }); - } - - foreach (var cookie in request.Cookies) - { - harRequest.Cookies.Add(new HarCookie - { - Name = cookie.Name, - Value = (string)cookie.Value, - Domain = cookie.Domain, - Path = cookie.Path, - HttpOnly = cookie.HttpOnly, - Secure = cookie.Secure, - Expires = cookie.Expiry?.ToString("o") - }); - } - - // Parse query string from URL - var uri = new Uri(request.Url); - if (!string.IsNullOrEmpty(uri.Query)) - { - var queryString = uri.Query.TrimStart('?'); - var queryParams = queryString.Split('&'); - foreach (var param in queryParams) - { - var parts = param.Split('=', 2); - harRequest.QueryString.Add(new HarQueryParam - { - Name = Uri.UnescapeDataString(parts[0]), - Value = parts.Length > 1 ? Uri.UnescapeDataString(parts[1]) : string.Empty - }); - } - } - - return harRequest; - } - - private HarResponse ConvertResponse(ResponseData response) - { - var harResponse = new HarResponse - { - Status = response.Status, - StatusText = response.StatusText, - HttpVersion = response.Protocol, - HeadersSize = response.HeadersSize ?? -1, - BodySize = response.BodySize ?? -1, - Content = new HarContent - { - Size = response.BodySize ?? 0, - MimeType = response.MimeType - } - }; - - foreach (var header in response.Headers) - { - var headerValue = (string)header.Value; - harResponse.Headers.Add(new HarHeader - { - Name = header.Name, - Value = headerValue - }); - - // Check for redirect URL - if (header.Name.Equals("Location", StringComparison.OrdinalIgnoreCase)) - { - harResponse.RedirectURL = headerValue; - } - } - - return harResponse; - } - - private HarTimings ConvertTimings(FetchTimingInfo timings) - { - return new HarTimings - { - Blocked = -1, - Dns = CalculateDuration(timings.DnsStart, timings.DnsEnd), - Connect = CalculateDuration(timings.ConnectStart, timings.ConnectEnd), - Ssl = CalculateDuration(timings.TlsStart, timings.ConnectEnd), - Send = CalculateDuration(timings.RequestStart, timings.RequestStart), - Wait = CalculateDuration(timings.RequestStart, timings.ResponseStart), - Receive = CalculateDuration(timings.ResponseStart, timings.ResponseEnd) - }; - } - - private double CalculateDuration(double start, double end) - { - if (start < 0 || end < 0 || end < start) - { - return -1; - } - return end - start; - } - - private double CalculateTotalTime(FetchTimingInfo timings) - { - if (timings.FetchStart >= 0 && timings.ResponseEnd >= 0 && timings.ResponseEnd >= timings.FetchStart) - { - return timings.ResponseEnd - timings.FetchStart; - } - return 0; - } - - private void LoadEntriesFromTempFile() - { - lock (_tempFileLock) - { - _harFile.Log.Entries.Clear(); - - if (File.Exists(_tempFilePath)) - { - var jsonOptions = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; - - var lines = File.ReadAllLines(_tempFilePath); - foreach (var line in lines) - { - if (!string.IsNullOrWhiteSpace(line)) - { - var entry = JsonSerializer.Deserialize(line, jsonOptions); - if (entry != null) - { - _harFile.Log.Entries.Add(entry); - } - } - } - } - } - } - - /// - /// Saves the captured network traffic to a HAR file. - /// - /// The path where the HAR file should be saved. - /// A task that represents the asynchronous operation. - public async Task SaveAsync(string filePath) - { - if (string.IsNullOrEmpty(filePath)) - { - throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); - } - - // Load entries from temp file before saving - LoadEntriesFromTempFile(); - - var options = new JsonSerializerOptions - { - WriteIndented = true, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull - }; - - var json = JsonSerializer.Serialize(_harFile, options); - await File.WriteAllTextAsync(filePath, json).ConfigureAwait(false); - } - - /// - /// Disposes the recorder and unsubscribes from network events. - /// - public async ValueTask DisposeAsync() - { - if (_beforeRequestSubscription != null) - { - await _beforeRequestSubscription.DisposeAsync().ConfigureAwait(false); - } - - if (_responseStartedSubscription != null) - { - await _responseStartedSubscription.DisposeAsync().ConfigureAwait(false); - } - - if (_responseCompletedSubscription != null) - { - await _responseCompletedSubscription.DisposeAsync().ConfigureAwait(false); - } - - if (_dataCollector != null) - { - await _bidi.Network.RemoveDataCollectorAsync(_dataCollector).ConfigureAwait(false); - } - - // Clean up temp file - if (File.Exists(_tempFilePath)) - { - try - { - File.Delete(_tempFilePath); - } - catch - { - // Ignore errors when deleting temp file - } - } - } -} diff --git a/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs b/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs new file mode 100644 index 0000000000000..60be9c3daa0f6 --- /dev/null +++ b/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs @@ -0,0 +1,405 @@ +// +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace OpenQA.Selenium.BiDi.Network.Har; + +/// +/// Records network traffic and provides methods to save it as HAR format. +/// +public sealed class HarRecorder : IHarRecorder +{ + private readonly BiDi _bidi; + private readonly HarRecordingOptions _options; + private readonly HarFile _harFile; + private readonly Dictionary _pendingRequests; + private readonly string _tempFilePath; + private readonly object _tempFileLock = new object(); + private Subscription? _beforeRequestSubscription; + private Subscription? _responseStartedSubscription; + private Subscription? _responseCompletedSubscription; + private Collector? _dataCollector; + + internal HarRecorder(BiDi bidi, HarRecordingOptions options) + { + _bidi = bidi ?? throw new ArgumentNullException(nameof(bidi)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _harFile = new HarFile(); + _pendingRequests = new Dictionary(); + _tempFilePath = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.jsonl"); + + if (!string.IsNullOrEmpty(options.BrowserName)) + { + _harFile.Log.Browser = new HarBrowser + { + Name = options.BrowserName, + Version = options.BrowserVersion ?? string.Empty + }; + } + } + + internal async Task StartAsync() + { + // Always create data collector for capturing request and response bodies + _dataCollector = await _bidi.Network.AddDataCollectorAsync([DataType.Request, DataType.Response], 200000000).ConfigureAwait(false); + + _beforeRequestSubscription = await _bidi.Network.OnBeforeRequestSentAsync(OnBeforeRequestSent).ConfigureAwait(false); + _responseStartedSubscription = await _bidi.Network.OnResponseStartedAsync(OnResponseStarted).ConfigureAwait(false); + _responseCompletedSubscription = await _bidi.Network.OnResponseCompletedAsync(OnResponseCompleted).ConfigureAwait(false); + } + + private void OnBeforeRequestSent(BeforeRequestSentEventArgs args) + { + var entry = new HarEntry + { + StartedDateTime = args.Timestamp.ToString("o"), + Request = ConvertRequest(args.Request), + Timings = ConvertTimings(args.Request.Timings), + Time = 0 + }; + + if (args.Context != null) + { + entry.Pageref = args.Context.Id; + } + + lock (_pendingRequests) + { + _pendingRequests[args.Request.Request.Id] = entry; + } + } + + private void OnResponseStarted(ResponseStartedEventArgs args) + { + lock (_pendingRequests) + { + if (_pendingRequests.TryGetValue(args.Request.Request.Id, out var entry)) + { + entry.Response = ConvertResponse(args.Response); + } + } + } + + private async void OnResponseCompleted(ResponseCompletedEventArgs args) + { + HarEntry? entry = null; + + lock (_pendingRequests) + { + if (_pendingRequests.TryGetValue(args.Request.Request.Id, out entry)) + { + entry.Response = ConvertResponse(args.Response); + + // Calculate total time + var timings = args.Request.Timings; + entry.Time = CalculateTotalTime(timings); + } + } + + if (entry != null) + { + // Retrieve request and response bodies + if (_dataCollector != null) + { + try + { + // Get request body + var requestBody = await _bidi.Network.GetDataAsync(DataType.Request, args.Request.Request).ConfigureAwait(false); + if (requestBody != null) + { + entry.Request.PostData = new HarPostData + { + MimeType = GetContentType(entry.Request.Headers), + Text = (string)requestBody + }; + } + } + catch + { + // Request body may not be available for all requests (e.g., GET requests) + } + + try + { + // Get response body + var responseBody = await _bidi.Network.GetDataAsync(DataType.Response, args.Request.Request).ConfigureAwait(false); + if (responseBody != null) + { + var bodyText = (string)responseBody; + entry.Response.Content.Text = bodyText; + entry.Response.Content.Size = System.Text.Encoding.UTF8.GetByteCount(bodyText); + } + } + catch + { + // Response body may not be available for all responses + } + } + + lock (_pendingRequests) + { + // Flush entry to temp file instead of keeping in memory + FlushEntryToTempFile(entry); + _pendingRequests.Remove(args.Request.Request.Id); + } + } + } + + private void FlushEntryToTempFile(HarEntry entry) + { + lock (_tempFileLock) + { + var jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + var entryJson = JsonSerializer.Serialize(entry, jsonOptions); + File.AppendAllText(_tempFilePath, entryJson + Environment.NewLine); + } + } + + private string GetContentType(List headers) + { + var contentTypeHeader = headers.FirstOrDefault(h => h.Name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase)); + return contentTypeHeader?.Value ?? "application/octet-stream"; + } + + private HarRequest ConvertRequest(RequestData request) + { + var harRequest = new HarRequest + { + Method = request.Method, + Url = request.Url, + HttpVersion = "HTTP/1.1", + HeadersSize = request.HeadersSize ?? -1, + BodySize = request.BodySize ?? -1 + }; + + foreach (var header in request.Headers) + { + harRequest.Headers.Add(new HarHeader + { + Name = header.Name, + Value = (string)header.Value + }); + } + + foreach (var cookie in request.Cookies) + { + harRequest.Cookies.Add(new HarCookie + { + Name = cookie.Name, + Value = (string)cookie.Value, + Domain = cookie.Domain, + Path = cookie.Path, + HttpOnly = cookie.HttpOnly, + Secure = cookie.Secure, + Expires = cookie.Expiry?.ToString("o") + }); + } + + // Parse query string from URL + var uri = new Uri(request.Url); + if (!string.IsNullOrEmpty(uri.Query)) + { + var queryString = uri.Query.TrimStart('?'); + var queryParams = queryString.Split('&'); + foreach (var param in queryParams) + { + var parts = param.Split('=', 2); + harRequest.QueryString.Add(new HarQueryParam + { + Name = Uri.UnescapeDataString(parts[0]), + Value = parts.Length > 1 ? Uri.UnescapeDataString(parts[1]) : string.Empty + }); + } + } + + return harRequest; + } + + private HarResponse ConvertResponse(ResponseData response) + { + var harResponse = new HarResponse + { + Status = response.Status, + StatusText = response.StatusText, + HttpVersion = response.Protocol, + HeadersSize = response.HeadersSize ?? -1, + BodySize = response.BodySize ?? -1, + Content = new HarContent + { + Size = response.BodySize ?? 0, + MimeType = response.MimeType + } + }; + + foreach (var header in response.Headers) + { + var headerValue = (string)header.Value; + harResponse.Headers.Add(new HarHeader + { + Name = header.Name, + Value = headerValue + }); + + // Check for redirect URL + if (header.Name.Equals("Location", StringComparison.OrdinalIgnoreCase)) + { + harResponse.RedirectURL = headerValue; + } + } + + return harResponse; + } + + private HarTimings ConvertTimings(FetchTimingInfo timings) + { + return new HarTimings + { + Blocked = -1, + Dns = CalculateDuration(timings.DnsStart, timings.DnsEnd), + Connect = CalculateDuration(timings.ConnectStart, timings.ConnectEnd), + Ssl = CalculateDuration(timings.TlsStart, timings.ConnectEnd), + Send = CalculateDuration(timings.RequestStart, timings.RequestStart), + Wait = CalculateDuration(timings.RequestStart, timings.ResponseStart), + Receive = CalculateDuration(timings.ResponseStart, timings.ResponseEnd) + }; + } + + private double CalculateDuration(double start, double end) + { + if (start < 0 || end < 0 || end < start) + { + return -1; + } + return end - start; + } + + private double CalculateTotalTime(FetchTimingInfo timings) + { + if (timings.FetchStart >= 0 && timings.ResponseEnd >= 0 && timings.ResponseEnd >= timings.FetchStart) + { + return timings.ResponseEnd - timings.FetchStart; + } + return 0; + } + + private void LoadEntriesFromTempFile() + { + lock (_tempFileLock) + { + _harFile.Log.Entries.Clear(); + + if (File.Exists(_tempFilePath)) + { + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }; + + var lines = File.ReadAllLines(_tempFilePath); + foreach (var line in lines) + { + if (!string.IsNullOrWhiteSpace(line)) + { + var entry = JsonSerializer.Deserialize(line, jsonOptions); + if (entry != null) + { + _harFile.Log.Entries.Add(entry); + } + } + } + } + } + } + + /// + /// Saves the captured network traffic to a HAR file. + /// + /// The path where the HAR file should be saved. + /// A task that represents the asynchronous operation. + public async Task SaveAsync(string filePath) + { + if (string.IsNullOrEmpty(filePath)) + { + throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); + } + + // Load entries from temp file before saving + LoadEntriesFromTempFile(); + + var options = new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + var json = JsonSerializer.Serialize(_harFile, options); + await File.WriteAllTextAsync(filePath, json).ConfigureAwait(false); + } + + /// + /// Disposes the recorder and unsubscribes from network events. + /// + public async ValueTask DisposeAsync() + { + if (_beforeRequestSubscription != null) + { + await _beforeRequestSubscription.DisposeAsync().ConfigureAwait(false); + } + + if (_responseStartedSubscription != null) + { + await _responseStartedSubscription.DisposeAsync().ConfigureAwait(false); + } + + if (_responseCompletedSubscription != null) + { + await _responseCompletedSubscription.DisposeAsync().ConfigureAwait(false); + } + + if (_dataCollector != null) + { + await _bidi.Network.RemoveDataCollectorAsync(_dataCollector).ConfigureAwait(false); + } + + // Clean up temp file + if (File.Exists(_tempFilePath)) + { + try + { + File.Delete(_tempFilePath); + } + catch + { + // Ignore errors when deleting temp file + } + } + } +} From 4ed692e5d166e4251a2650e78455520fd39a0c9b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 18 Oct 2025 19:22:32 +0000 Subject: [PATCH 12/12] Use dedicated file per network response for better concurrency Co-authored-by: nvborisenko <22616990+nvborisenko@users.noreply.github.com> --- .gitignore | 4 +- .../webdriver/BiDi/Network/Har/HarRecorder.cs | 81 ++++++++++--------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 6dc8f1d656228..aeb238e4e07a0 100644 --- a/.gitignore +++ b/.gitignore @@ -144,6 +144,6 @@ dotnet-bin .metadata/ .npmrc -# HAR files generated during tests +# HAR files and directories generated during tests *.har -selenium-har-*.jsonl +selenium-har-*/ diff --git a/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs b/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs index 60be9c3daa0f6..34e62d7b3a392 100644 --- a/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs +++ b/dotnet/src/webdriver/BiDi/Network/Har/HarRecorder.cs @@ -35,8 +35,7 @@ public sealed class HarRecorder : IHarRecorder private readonly HarRecordingOptions _options; private readonly HarFile _harFile; private readonly Dictionary _pendingRequests; - private readonly string _tempFilePath; - private readonly object _tempFileLock = new object(); + private readonly string _tempDirectoryPath; private Subscription? _beforeRequestSubscription; private Subscription? _responseStartedSubscription; private Subscription? _responseCompletedSubscription; @@ -48,7 +47,8 @@ internal HarRecorder(BiDi bidi, HarRecordingOptions options) _options = options ?? throw new ArgumentNullException(nameof(options)); _harFile = new HarFile(); _pendingRequests = new Dictionary(); - _tempFilePath = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}.jsonl"); + _tempDirectoryPath = Path.Combine(Path.GetTempPath(), $"selenium-har-{Guid.NewGuid()}"); + Directory.CreateDirectory(_tempDirectoryPath); if (!string.IsNullOrEmpty(options.BrowserName)) { @@ -158,28 +158,31 @@ private async void OnResponseCompleted(ResponseCompletedEventArgs args) } } + // Flush entry to dedicated temp file (outside of lock for better concurrency) + FlushEntryToTempFile(entry, args.Request.Request.Id); + lock (_pendingRequests) { - // Flush entry to temp file instead of keeping in memory - FlushEntryToTempFile(entry); _pendingRequests.Remove(args.Request.Request.Id); } } } - private void FlushEntryToTempFile(HarEntry entry) + private void FlushEntryToTempFile(HarEntry entry, string requestId) { - lock (_tempFileLock) + // Use dedicated file per entry for better concurrency safety + var sanitizedId = string.Concat(requestId.Where(c => char.IsLetterOrDigit(c) || c == '-' || c == '_')); + var fileName = $"{sanitizedId}_{Guid.NewGuid()}.json"; + var filePath = Path.Combine(_tempDirectoryPath, fileName); + + var jsonOptions = new JsonSerializerOptions { - var jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull - }; + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; - var entryJson = JsonSerializer.Serialize(entry, jsonOptions); - File.AppendAllText(_tempFilePath, entryJson + Environment.NewLine); - } + var entryJson = JsonSerializer.Serialize(entry, jsonOptions); + File.WriteAllText(filePath, entryJson); } private string GetContentType(List headers) @@ -309,31 +312,33 @@ private double CalculateTotalTime(FetchTimingInfo timings) return 0; } - private void LoadEntriesFromTempFile() + private void LoadEntriesFromTempFiles() { - lock (_tempFileLock) - { - _harFile.Log.Entries.Clear(); + _harFile.Log.Entries.Clear(); - if (File.Exists(_tempFilePath)) + if (Directory.Exists(_tempDirectoryPath)) + { + var jsonOptions = new JsonSerializerOptions { - var jsonOptions = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; + PropertyNameCaseInsensitive = true + }; - var lines = File.ReadAllLines(_tempFilePath); - foreach (var line in lines) + var files = Directory.GetFiles(_tempDirectoryPath, "*.json"); + foreach (var file in files) + { + try { - if (!string.IsNullOrWhiteSpace(line)) + var entryJson = File.ReadAllText(file); + var entry = JsonSerializer.Deserialize(entryJson, jsonOptions); + if (entry != null) { - var entry = JsonSerializer.Deserialize(line, jsonOptions); - if (entry != null) - { - _harFile.Log.Entries.Add(entry); - } + _harFile.Log.Entries.Add(entry); } } + catch + { + // Skip corrupted or incomplete files + } } } } @@ -350,8 +355,8 @@ public async Task SaveAsync(string filePath) throw new ArgumentException("File path cannot be null or empty.", nameof(filePath)); } - // Load entries from temp file before saving - LoadEntriesFromTempFile(); + // Load entries from temp files before saving + LoadEntriesFromTempFiles(); var options = new JsonSerializerOptions { @@ -389,16 +394,16 @@ public async ValueTask DisposeAsync() await _bidi.Network.RemoveDataCollectorAsync(_dataCollector).ConfigureAwait(false); } - // Clean up temp file - if (File.Exists(_tempFilePath)) + // Clean up temp directory + if (Directory.Exists(_tempDirectoryPath)) { try { - File.Delete(_tempFilePath); + Directory.Delete(_tempDirectoryPath, recursive: true); } catch { - // Ignore errors when deleting temp file + // Ignore errors when deleting temp directory } } }