[dotnet] [bidi] Support network collectors and get response body

dotnet/src/webdriver/BiDi/Communication/Broker.cs

@@ -75,6 +75,7 @@ internal Broker(BiDi bidi, Uri url)
                 new BrowserUserContextConverter(bidi),
                 new BrowserClientWindowConverter(),
                 new NavigationConverter(),
+                new CollectorConverter(_bidi),
                 new InterceptConverter(_bidi),
                 new RequestConverter(),
                 new ChannelConverter(),

dotnet/src/webdriver/BiDi/Communication/Json/BiDiJsonSerializerContext.cs

@@ -118,13 +118,18 @@ namespace OpenQA.Selenium.BiDi.Communication.Json;
 [JsonSerializable(typeof(BrowsingContext.UserPromptOpenedEventArgs))]
 [JsonSerializable(typeof(BrowsingContext.UserPromptClosedEventArgs))]
 
+[JsonSerializable(typeof(Network.AddDataCollectorCommand))]
+[JsonSerializable(typeof(Network.AddDataCollectorResult))]
 [JsonSerializable(typeof(Network.AddInterceptCommand))]
 [JsonSerializable(typeof(Network.AddInterceptResult))]
 [JsonSerializable(typeof(Network.ContinueRequestCommand))]
 [JsonSerializable(typeof(Network.ContinueResponseCommand))]
 [JsonSerializable(typeof(Network.ContinueWithAuthCommand))]
 [JsonSerializable(typeof(Network.FailRequestCommand))]
+[JsonSerializable(typeof(Network.GetDataCommand))]
+[JsonSerializable(typeof(Network.GetDataResult))]
 [JsonSerializable(typeof(Network.ProvideResponseCommand))]
+[JsonSerializable(typeof(Network.RemoveDataCollectorCommand))]
 [JsonSerializable(typeof(Network.RemoveInterceptCommand))]
 [JsonSerializable(typeof(Network.SetCacheBehaviorCommand))]
 

dotnet/src/webdriver/BiDi/Communication/Json/Converters/CollectorConverter.cs

@@ -0,0 +1,47 @@
+// <copyright file="CollectorConverter.cs" company="Selenium Committers">
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// </copyright>
+
+using OpenQA.Selenium.BiDi.Network;
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace OpenQA.Selenium.BiDi.Communication.Json.Converters;
+
+internal class CollectorConverter : JsonConverter<Collector>
+{
+    private readonly BiDi _bidi;
+
+    public CollectorConverter(BiDi bidi)
+    {
+        _bidi = bidi;
+    }
+
+    public override Collector? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        var id = reader.GetString();
+
+        return new Collector(_bidi, id!);
+    }
+
+    public override void Write(Utf8JsonWriter writer, Collector value, JsonSerializerOptions options)
+    {
+        writer.WriteStringValue(value.Id);
+    }
+}

dotnet/src/webdriver/BiDi/Network/AddDataCollectorCommand.cs

@@ -0,0 +1,49 @@
+// <copyright file="AddDataCollectorCommand.cs" company="Selenium Committers">
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// </copyright>
+
+using System.Collections.Generic;
+using OpenQA.Selenium.BiDi.Communication;
+
+namespace OpenQA.Selenium.BiDi.Network;
+
+internal sealed class AddDataCollectorCommand(AddDataCollectorParameters @params)
+    : Command<AddDataCollectorParameters, AddDataCollectorResult>(@params, "network.addDataCollector");
+
+internal sealed record AddDataCollectorParameters(IEnumerable<DataType> DataTypes, int MaxEncodedDataSize, CollectorType? CollectorType, IEnumerable<BrowsingContext.BrowsingContext>? Contexts, IEnumerable<Browser.UserContext>? UserContexts) : Parameters;
+
+public class AddDataCollectorOptions : CommandOptions
+{
+    public CollectorType? CollectorType { get; set; }
+
+    public IEnumerable<BrowsingContext.BrowsingContext>? Contexts { get; set; }
+
+    public IEnumerable<Browser.UserContext>? UserContexts { get; set; }
+}
+
+public sealed record AddDataCollectorResult(Collector Collector) : EmptyResult;
+
+public enum DataType
+{
+    Response
+}
+
+public enum CollectorType
+{
+    Blob
+}

dotnet/src/webdriver/BiDi/Network/BytesValue.cs

@@ -29,6 +29,13 @@ public abstract record BytesValue
 {
     public static implicit operator BytesValue(string value) => new StringBytesValue(value);
     public static implicit operator BytesValue(byte[] value) => new Base64BytesValue(Convert.ToBase64String(value));
+
+    public static explicit operator string(BytesValue value) => value switch
+    {
+        StringBytesValue stringBytesValue => stringBytesValue.Value,
+        Base64BytesValue base64BytesValue => System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(base64BytesValue.Value)),
+        _ => throw new InvalidCastException($"Cannot cast '{value.GetType()}' to '{typeof(string)}'.")
+    };
 }
 
 public sealed record StringBytesValue(string Value) : BytesValue;

dotnet/src/webdriver/BiDi/Network/Collector.cs

@@ -0,0 +1,58 @@
+// <copyright file="Collector.cs" company="Selenium Committers">
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// </copyright>
+
+using System;
+using System.Threading.Tasks;
+
+namespace OpenQA.Selenium.BiDi.Network;
+
+public sealed class Collector : IAsyncDisposable
+{
+    private readonly BiDi _bidi;
+
+    internal Collector(BiDi bidi, string id)
+    {
+        _bidi = bidi;
+        Id = id;
+    }
+
+    internal string Id { get; }
+
+    public async Task RemoveAsync()
+    {
+        await _bidi.Network.RemoveDataCollectorAsync(this).ConfigureAwait(false);
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await RemoveAsync();
+    }
+
+    public override bool Equals(object? obj)
+    {
+        if (obj is Collector collectortObj) return collectortObj.Id == Id;
+
+        return false;
+    }
+
+    public override int GetHashCode()
+    {
+        return Id.GetHashCode();
+    }
+}

dotnet/src/webdriver/BiDi/Network/GetDataCommand.cs

@@ -0,0 +1,36 @@
+// <copyright file="GetDataCommand.cs" company="Selenium Committers">
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// </copyright>
+
+using OpenQA.Selenium.BiDi.Communication;
+
+namespace OpenQA.Selenium.BiDi.Network;
+
+internal sealed class GetDataCommand(GetDataParameters @params)
+    : Command<GetDataParameters, GetDataResult>(@params, "network.getData");
+
+internal sealed record GetDataParameters(DataType DataType, Request Request, Collector? Collector, bool? Disown) : Parameters;
+
+public sealed class GetDataOptions : CommandOptions
+{
+    public Collector? Collector { get; set; }
+
+    public bool? Disown { get; set; }
+}
+
+public sealed record GetDataResult(BytesValue Bytes) : EmptyResult;

dotnet/src/webdriver/BiDi/Network/NetworkModule.cs

@@ -26,6 +26,15 @@ namespace OpenQA.Selenium.BiDi.Network;
 
 public sealed partial class NetworkModule(Broker broker) : Module(broker)
 {
+    public async Task<Collector> AddDataCollectorAsync(IEnumerable<DataType> DataTypes, int MaxEncodedDataSize, AddDataCollectorOptions? options = null)
+    {
+        var @params = new AddDataCollectorParameters(DataTypes, MaxEncodedDataSize, options?.CollectorType, options?.Contexts, options?.UserContexts);
+
+        var result = await Broker.ExecuteCommandAsync<AddDataCollectorCommand, AddDataCollectorResult>(new AddDataCollectorCommand(@params), options).ConfigureAwait(false);
+
+        return result.Collector;
+    }
+
     public async Task<Intercept> AddInterceptAsync(IEnumerable<InterceptPhase> phases, AddInterceptOptions? options = null)
     {
         var @params = new AddInterceptParameters(phases, options?.Contexts, options?.UrlPatterns);
@@ -35,6 +44,13 @@ public async Task<Intercept> AddInterceptAsync(IEnumerable<InterceptPhase> phase
         return result.Intercept;
     }
 
+    public async Task<EmptyResult> RemoveDataCollectorAsync(Collector collector, RemoveDataCollectorOptions? options = null)
+    {
+        var @params = new RemoveDataCollectorParameters(collector);
+
+        return await Broker.ExecuteCommandAsync<RemoveDataCollectorCommand, EmptyResult>(new RemoveDataCollectorCommand(@params), options).ConfigureAwait(false);
+    }
+
     public async Task<EmptyResult> RemoveInterceptAsync(Intercept intercept, RemoveInterceptOptions? options = null)
     {
         var @params = new RemoveInterceptParameters(intercept);
@@ -70,6 +86,15 @@ public async Task<EmptyResult> FailRequestAsync(Request request, FailRequestOpti
         return await Broker.ExecuteCommandAsync<FailRequestCommand, EmptyResult>(new FailRequestCommand(@params), options).ConfigureAwait(false);
     }
 
+    public async Task<BytesValue> GetDataAsync(DataType dataType, Request request, GetDataOptions? options = null)
+    {
+        var @params = new GetDataParameters(dataType, request, options?.Collector, options?.Disown);
+
+        var result = await Broker.ExecuteCommandAsync<GetDataCommand, GetDataResult>(new GetDataCommand(@params), options).ConfigureAwait(false);
+
+        return result.Bytes;
+    }
+
     public async Task<EmptyResult> ProvideResponseAsync(Request request, ProvideResponseOptions? options = null)
     {
         var @params = new ProvideResponseParameters(request, options?.Body, options?.Cookies, options?.Headers, options?.ReasonPhrase, options?.StatusCode);

dotnet/src/webdriver/BiDi/Network/RemoveDataCollectorCommand.cs

@@ -0,0 +1,29 @@
+// <copyright file="RemoveDataCollectorCommand.cs" company="Selenium Committers">
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// </copyright>
+
+using OpenQA.Selenium.BiDi.Communication;
+
+namespace OpenQA.Selenium.BiDi.Network;
+
+internal sealed class RemoveDataCollectorCommand(RemoveDataCollectorParameters @params)
+    : Command<RemoveDataCollectorParameters, EmptyResult>(@params, "network.removeDataCollector");
+
+internal sealed record RemoveDataCollectorParameters(Collector Collector) : Parameters;
+
+public class RemoveDataCollectorOptions : CommandOptions;

dotnet/test/common/BiDi/Network/NetworkTest.cs

@@ -19,12 +19,23 @@
 
 using NUnit.Framework;
 using OpenQA.Selenium.BiDi.BrowsingContext;
+using System;
 using System.Threading.Tasks;
 
 namespace OpenQA.Selenium.BiDi.Network;
 
 class NetworkTest : BiDiTestFixture
 {
+    [Test]
+    public async Task CanAddDataCollector()
+    {
+        // Firefox doesn't like int.MaxValue as max encoded data size
+        // invalid argument: Expected "maxEncodedDataSize" to be less than the max total data size available (200000000), got 2147483647
+        await using var collector = await bidi.Network.AddDataCollectorAsync([DataType.Response], 200000000);
+
+        Assert.That(collector, Is.Not.Null);
+    }
+
     [Test]
     public async Task CanAddIntercept()
     {
@@ -210,6 +221,30 @@ public async Task CanFailRequest()
         Assert.That(action, Throws.TypeOf<BiDiException>().With.Message.Contain("net::ERR_FAILED").Or.Message.Contain("NS_ERROR_ABORT"));
     }
 
+    [Test]
+    public async Task CanGetData()
+    {
+        // Firefox doesn't like int.MaxValue as max encoded data size
+        // invalid argument: Expected "maxEncodedDataSize" to be less than the max total data size available (200000000), got 2147483647
+        await using var collector = await bidi.Network.AddDataCollectorAsync([DataType.Response], 200000000);
+
+        TaskCompletionSource<string> responseBodyCompletionSource = new();
+
+        await using var _ = await bidi.Network.OnResponseCompletedAsync(async e =>
+        {
+            if (e.Response.Url.Contains("simpleTest.html"))
+            {
+                responseBodyCompletionSource.SetResult((string)await bidi.Network.GetDataAsync(DataType.Response, e.Request.Request));
+            }
+        });
+
+        await context.NavigateAsync(UrlBuilder.WhereIs("simpleTest.html"), new() { Wait = ReadinessState.Complete });
+
+        var responseBody = await responseBodyCompletionSource.Task.WaitAsync(TimeSpan.FromSeconds(5));
+
+        Assert.That(responseBody, Contains.Substring("Hello WebDriver"));
+    }
+
     [Test]
     public void CanSetCacheBehavior()
     {