Skip to content

Commit fcdd4e4

Browse files
FlorianRapplFlorianRappl
committed
Integrated data requester and adjusted to config change
1 parent e75bef4 commit fcdd4e4

File tree

10 files changed

+264
-20
lines changed

10 files changed

+264
-20
lines changed

‎src/AngleSharp.Io.Tests/AngleSharp.Io.Tests.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,10 @@
7474
<ItemGroup>
7575
<Compile Include="DisposableStream.cs" />
7676
<Compile Include="Helper.cs" />
77+
<Compile Include="Integration\DocumentTests.cs" />
7778
<Compile Include="Network\CookieHandlingTests.cs" />
7879
<Compile Include="Network\AboutTests.cs" />
80+
<Compile Include="Network\DataRequesterTests.cs" />
7981
<Compile Include="Network\FileRequesterTests.cs" />
8082
<Compile Include="Network\FtpRequesterTests.cs" />
8183
<Compile Include="Network\HttpClientRequesterTests.cs" />

‎src/AngleSharp.Io.Tests/Integration/DocumentTests.cs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
namespace AngleSharp.Io.Tests.Integration
2+
{
3+
using AngleSharp.Dom;
4+
using AngleSharp.Html.Dom;
5+
using NUnit.Framework;
6+
using System;
7+
using System.Threading.Tasks;
8+
9+
[TestFixture]
10+
public class DocumentTests
11+
{
12+
[Test]
13+
public async Task IframeWithDocumentViaDataSrc()
14+
{
15+
var cfg = Configuration.Default.WithRequesters().WithDefaultLoader(setup => setup.IsResourceLoadingEnabled = true);
16+
var html = @"<!doctype html><iframe id=myframe src='data:text/html,<span>Hello World!</span>'></iframe></script>";
17+
var document = await BrowsingContext.New(cfg).OpenAsync(m => m.Content(html));
18+
var iframe = document.QuerySelector<IHtmlInlineFrameElement>("#myframe");
19+
Assert.IsNotNull(iframe);
20+
Assert.IsNotNull(iframe.ContentDocument);
21+
Assert.AreEqual("Hello World!", iframe.ContentDocument.Body.TextContent);
22+
Assert.AreEqual(iframe.ContentDocument, iframe.ContentWindow.Document);
23+
}
24+
25+
public async Task ImportPageFromDataRequest()
26+
{
27+
var receivedRequest = new TaskCompletionSource<Boolean>();
28+
var config = Configuration.Default.WithRequesters().WithDefaultLoader(setup =>
29+
{
30+
setup.IsResourceLoadingEnabled = true;
31+
setup.Filter = request =>
32+
{
33+
receivedRequest.SetResult(true);
34+
return true;
35+
};
36+
});
37+
38+
var document = await BrowsingContext.New(config).OpenAsync(m => m.Content("<!doctype html><link rel=import href='data:text/html,<div>foo</div>'>"));
39+
var link = document.QuerySelector<IHtmlLinkElement>("link");
40+
await receivedRequest.Task;
41+
42+
Assert.AreEqual("import", link.Relation);
43+
Assert.IsNotNull(link.Import);
44+
Assert.AreEqual("foo", link.Import.QuerySelector("div").TextContent);
45+
}
46+
}
47+
}

‎src/AngleSharp.Io.Tests/Network/CookieHandlingTests.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public async Task SettingOneCookiesInOneRequestAppearsInDocument()
1313
if (Helper.IsNetworkAvailable())
1414
{
1515
var url = "https://httpbin.org/cookies/set?k1=v1";
16-
var config = Configuration.Default.WithCookies().WithRequesters();
16+
var config = Configuration.Default.WithCookies().WithRequesters().WithDefaultLoader();
1717
var context = BrowsingContext.New(config);
1818
var document = await context.OpenAsync(url);
1919

@@ -27,7 +27,7 @@ public async Task SettingTwoCookiesInOneRequestAppearsInDocument()
2727
if (Helper.IsNetworkAvailable())
2828
{
2929
var url = "https://httpbin.org/cookies/set?k2=v2&k1=v1";
30-
var config = Configuration.Default.WithCookies().WithRequesters();
30+
var config = Configuration.Default.WithCookies().WithRequesters().WithDefaultLoader();
3131
var context = BrowsingContext.New(config);
3232
var document = await context.OpenAsync(url);
3333

@@ -41,7 +41,7 @@ public async Task SettingThreeCookiesInOneRequestAppearsInDocument()
4141
if (Helper.IsNetworkAvailable())
4242
{
4343
var url = "https://httpbin.org/cookies/set?test=baz&k2=v2&k1=v1&foo=bar";
44-
var config = Configuration.Default.WithCookies().WithRequesters();
44+
var config = Configuration.Default.WithCookies().WithRequesters().WithDefaultLoader();
4545
var context = BrowsingContext.New(config);
4646
var document = await context.OpenAsync(url);
4747

@@ -56,7 +56,7 @@ public async Task SettingThreeCookiesInOneRequestAreTransportedToNextRequest()
5656
{
5757
var baseUrl = "https://httpbin.org/cookies";
5858
var url = baseUrl + "/set?test=baz&k2=v2&k1=v1&foo=bar";
59-
var config = Configuration.Default.WithCookies().WithRequesters();
59+
var config = Configuration.Default.WithCookies().WithRequesters().WithDefaultLoader();
6060
var context = BrowsingContext.New(config);
6161
await context.OpenAsync(url);
6262
var document = await context.OpenAsync(baseUrl);
@@ -80,7 +80,7 @@ public async Task SettingCookieIsPreservedViaRedirect()
8080
{
8181
var cookieUrl = "https://httpbin.org/cookies/set?test=baz";
8282
var redirectUrl = "http://httpbin.org/redirect-to?url=http%3A%2F%2Fhttpbin.org%2Fcookies";
83-
var config = Configuration.Default.WithCookies().WithRequesters();
83+
var config = Configuration.Default.WithCookies().WithRequesters().WithDefaultLoader();
8484
var context = BrowsingContext.New(config);
8585
await context.OpenAsync(cookieUrl);
8686
var document = await context.OpenAsync(redirectUrl);

‎src/AngleSharp.Io.Tests/Network/DataRequesterTests.cs

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
namespace AngleSharp.Io.Tests.Network
2+
{
3+
using AngleSharp.Io;
4+
using AngleSharp.Io.Network;
5+
using AngleSharp.Text;
6+
using NUnit.Framework;
7+
using System;
8+
using System.IO;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
12+
[TestFixture]
13+
public class DataRequesterTests
14+
{
15+
[Test]
16+
public async Task DataGetRequestForPlainText()
17+
{
18+
var http = new DataRequester();
19+
var request = new Request { Address = new Url("data:,Hello+there"), Method = HttpMethod.Get };
20+
21+
using (var response = await http.RequestAsync(request, CancellationToken.None))
22+
{
23+
Assert.IsNotNull(response);
24+
Assert.AreEqual(200, (int)response.StatusCode);
25+
Assert.IsTrue(response.Content.CanRead);
26+
Assert.IsTrue(response.Headers.Count > 0);
27+
Assert.AreEqual("text/plain", response.Headers[HeaderNames.ContentType]);
28+
29+
var content = new StreamReader(response.Content);
30+
Assert.AreEqual("Hello there", content.ReadToEnd());
31+
}
32+
}
33+
34+
[Test]
35+
public async Task DataGetRequestForHtmlWithUtf8UrlEncoded()
36+
{
37+
var http = new DataRequester();
38+
var request = new Request();
39+
var source = "<!DOCTYPE html><html lang='en'><head><title>Embedded Window</title></head><body><h1>42</h1></body></html>";
40+
var content = TextEncoding.Utf8.GetBytes(source).UrlEncode();
41+
request.Address = new Url("data:text/html;charset=utf-8," + content);
42+
request.Method = HttpMethod.Get;
43+
44+
using (var response = await http.RequestAsync(request, CancellationToken.None))
45+
{
46+
Assert.IsNotNull(response);
47+
Assert.AreEqual(200, (int)response.StatusCode);
48+
Assert.IsTrue(response.Content.CanRead);
49+
Assert.IsTrue(response.Headers.Count > 0);
50+
Assert.AreEqual("text/html;charset=utf-8", response.Headers[HeaderNames.ContentType]);
51+
52+
var reader = new StreamReader(response.Content);
53+
Assert.AreEqual(source, reader.ReadToEnd());
54+
}
55+
}
56+
57+
[Test]
58+
public async Task DataGetRequestForHtmlWithUtf8Base64Encoded()
59+
{
60+
var http = new DataRequester();
61+
var request = new Request();
62+
var source = "<!DOCTYPE html><html lang='en'><head><title>Embedded Window</title></head><body><h1>42</h1></body></html>";
63+
var content = Convert.ToBase64String(TextEncoding.Utf8.GetBytes(source));
64+
request.Address = new Url("data:text/html;charset=utf-8;base64," + content);
65+
request.Method = HttpMethod.Get;
66+
67+
using (var response = await http.RequestAsync(request, CancellationToken.None))
68+
{
69+
Assert.IsNotNull(response);
70+
Assert.AreEqual(200, (int)response.StatusCode);
71+
Assert.IsTrue(response.Content.CanRead);
72+
Assert.IsTrue(response.Headers.Count > 0);
73+
Assert.AreEqual("text/html;charset=utf-8", response.Headers[HeaderNames.ContentType]);
74+
75+
var reader = new StreamReader(response.Content);
76+
Assert.AreEqual(source, reader.ReadToEnd());
77+
}
78+
}
79+
80+
[Test]
81+
public async Task DataGetRequestForMiddleImageBase64Encoded()
82+
{
83+
var http = new DataRequester();
84+
var request = new Request();
85+
var content = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD///+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4Ug9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC";
86+
request.Address = new Url(content);
87+
request.Method = HttpMethod.Get;
88+
89+
using (var response = await http.RequestAsync(request, CancellationToken.None))
90+
{
91+
Assert.IsNotNull(response);
92+
Assert.AreEqual(200, (int)response.StatusCode);
93+
Assert.IsTrue(response.Content.CanRead);
94+
Assert.IsTrue(response.Headers.Count > 0);
95+
Assert.AreEqual("image/png", response.Headers[HeaderNames.ContentType]);
96+
}
97+
}
98+
99+
[Test]
100+
public async Task DataGetRequestForSmallImageBase64Encoded()
101+
{
102+
var http = new DataRequester();
103+
var request = new Request();
104+
var content = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==";
105+
request.Address = new Url(content);
106+
request.Method = HttpMethod.Get;
107+
108+
using (var response = await http.RequestAsync(request, CancellationToken.None))
109+
{
110+
Assert.IsNotNull(response);
111+
Assert.AreEqual(200, (int)response.StatusCode);
112+
Assert.IsTrue(response.Content.CanRead);
113+
Assert.IsTrue(response.Headers.Count > 0);
114+
Assert.AreEqual("image/png", response.Headers[HeaderNames.ContentType]);
115+
}
116+
}
117+
}
118+
}

‎src/AngleSharp.Io.Tests/Network/FileRequesterTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public async Task GetLocalFileViaFileRequester()
3939
public async Task FollowLinkToUseFileRequesterUsingAllRequesters()
4040
{
4141
var url = GetLocalPath();
42-
var config = Configuration.Default.WithRequesters();
42+
var config = Configuration.Default.WithRequesters().WithDefaultLoader();
4343
var context = BrowsingContext.New(config);
4444
var document = await context.OpenAsync(res => res.Content("<a href='" + url + "'>Download</a>"));
4545
var result = await document.QuerySelector<IHtmlAnchorElement>("a").NavigateAsync();

‎src/AngleSharp.Io.Tests/Network/FtpRequesterTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public async Task DownloadFtpRfcViaFtpRequester()
2828
[Test]
2929
public async Task FollowLinkToUseFtpRequesterUsingAllRequesters()
3030
{
31-
var config = Configuration.Default.WithRequesters();
31+
var config = Configuration.Default.WithRequesters().WithDefaultLoader();
3232
var context = BrowsingContext.New(config);
3333
var document = await context.OpenAsync(res => res.Content("<a href='ftp://ftp.funet.fi/pub/standards/RFC/rfc959.txt'>Download</a>"));
3434
var result = await document.QuerySelector<IHtmlAnchorElement>("a").NavigateAsync();

‎src/AngleSharp.Io.Tests/Network/HttpClientRequesterTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public async Task EndToEnd()
122122
// ARRANGE
123123
var httpClient = new HttpClient();
124124
var requester = new HttpClientRequester(httpClient);
125-
var configuration = Configuration.Default.WithDefaultLoader(requesters: new[] { requester });
125+
var configuration = Configuration.Default.With(requester).WithDefaultLoader();
126126
var context = BrowsingContext.New(configuration);
127127
var request = DocumentRequest.Get(Url.Create("http://httpbin.org/html"));
128128

‎src/AngleSharp.Io/AngleSharp.Io.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
<Compile Include="Dom\WebSocket.cs" />
5454
<Compile Include="Interfaces\IStorage.cs" />
5555
<Compile Include="Network\AboutRequester.cs" />
56+
<Compile Include="Network\DataRequester.cs" />
5657
<Compile Include="Network\FileRequester.cs" />
5758
<Compile Include="Network\FtpRequester.cs" />
5859
<Compile Include="Network\HttpClientRequester.cs" />

‎src/AngleSharp.Io/IoConfigurationExtensions.cs

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,43 @@
11
namespace AngleSharp
22
{
3-
using AngleSharp.Io;
43
using AngleSharp.Io.Network;
54
using System;
65
using System.Net.Http;
7-
using LoaderSetup = AngleSharp.ConfigurationExtensions.LoaderSetup;
86

97
/// <summary>
108
/// Additional extensions for improved requesters.
119
/// </summary>
1210
public static class IoConfigurationExtensions
1311
{
1412
/// <summary>
15-
/// Adds a loader service that comes with all (improved) requesters.
13+
/// Adds the requesters from the AngleSharp.Io package.
1614
/// </summary>
1715
/// <param name="configuration">The configuration to use.</param>
18-
/// <param name="setup">Optional setup for the loader service.</param>
1916
/// <returns>The new configuration.</returns>
20-
public static IConfiguration WithRequesters(this IConfiguration configuration, Action<LoaderSetup> setup = null)
17+
public static IConfiguration WithRequesters(this IConfiguration configuration)
2118
{
22-
return configuration.WithRequesters(new HttpClientHandler { UseCookies = false, AllowAutoRedirect = false }, setup);
19+
return configuration.WithRequesters(new HttpClientHandler { UseCookies = false, AllowAutoRedirect = false });
2320
}
2421

2522
/// <summary>
26-
/// Adds a loader service that comes with all (improved) requesters.
23+
/// Adds the requesters from the AngleSharp.Io package.
2724
/// </summary>
2825
/// <param name="configuration">The configuration to use.</param>
2926
/// <param name="httpMessageHandler">
3027
/// The HTTP handler stack to use for sending requests.
3128
/// </param>
32-
/// <param name="setup">Optional setup for the loader service.</param>
3329
/// <returns>The new configuration.</returns>
34-
public static IConfiguration WithRequesters(this IConfiguration configuration, HttpMessageHandler httpMessageHandler, Action<LoaderSetup> setup = null)
30+
public static IConfiguration WithRequesters(this IConfiguration configuration, HttpMessageHandler httpMessageHandler)
3531
{
3632
var httpClient = new HttpClient(httpMessageHandler);
37-
var requesters = new IRequester[]
33+
return configuration.With(new Object[]
3834
{
3935
new HttpClientRequester(httpClient),
4036
new DataRequester(),
4137
new FtpRequester(),
4238
new FileRequester(),
4339
new AboutRequester()
44-
};
45-
return configuration.WithDefaultLoader(setup, requesters);
40+
});
4641
}
4742
}
4843
}

‎src/AngleSharp.Io/Network/DataRequester.cs

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
namespace AngleSharp.Io.Network
2+
{
3+
using AngleSharp.Text;
4+
using System;
5+
using System.IO;
6+
using System.Net;
7+
using System.Threading;
8+
using System.Threading.Tasks;
9+
10+
/// <summary>
11+
/// The default (ready-to-use) data scheme requester.
12+
/// </summary>
13+
public sealed class DataRequester : BaseRequester
14+
{
15+
#region Fields
16+
17+
private static readonly String Base64Section = ";base64";
18+
19+
#endregion
20+
21+
#region Methods
22+
23+
/// <summary>Checks if the data protocol is given.</summary>
24+
/// <param name="protocol">The protocol to check for data.</param>
25+
/// <returns>True if data is matched, otherwise false..</returns>
26+
public override Boolean SupportsProtocol(String protocol)
27+
{
28+
return protocol.Is(ProtocolNames.Data);
29+
}
30+
31+
/// <summary>
32+
/// Performs an asynchronous request that can be cancelled.
33+
/// </summary>
34+
/// <param name="request">The options to consider.</param>
35+
/// <param name="cancel">The token for cancelling the task.</param>
36+
/// <returns>
37+
/// The task that will eventually give the response data.
38+
/// </returns>
39+
protected override Task<IResponse> PerformRequestAsync(Request request, CancellationToken cancel)
40+
{
41+
var content = new MemoryStream();
42+
var data = request.Address.Data;
43+
44+
if (data.StartsWith(","))
45+
{
46+
data = MimeTypeNames.Plain + data;
47+
}
48+
49+
var parts = data.SplitCommas();
50+
var response = new DefaultResponse
51+
{
52+
Address = request.Address,
53+
Content = content,
54+
StatusCode = HttpStatusCode.BadRequest
55+
};
56+
57+
if (parts.Length == 2)
58+
{
59+
var index = parts[0].IndexOf(Base64Section);
60+
var base64 = index >= 0;
61+
var mimeType = base64 ? parts[0].Remove(index, Base64Section.Length) : parts[0];
62+
63+
try
64+
{
65+
var raw = base64 ? Convert.FromBase64String(parts[1]) : parts[1].UrlDecode();
66+
content.Write(raw, 0, raw.Length);
67+
content.Position = 0;
68+
response.Headers.Add(HeaderNames.ContentType, mimeType);
69+
response.StatusCode = HttpStatusCode.OK;
70+
}
71+
catch (FormatException)
72+
{
73+
}
74+
}
75+
76+
return Task.FromResult<IResponse>(response);
77+
}
78+
79+
#endregion
80+
}
81+
}

0 commit comments

Comments
 (0)