Skip to content

Commit ef4acfe

Browse files
DecimationDecimation
committed
CF work
1 parent 4c11b66 commit ef4acfe

File tree

13 files changed

+756
-98
lines changed

13 files changed

+756
-98
lines changed

‎SmartImage.Lib/Clients/ChallengeDetector.cs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
2+
// Author: Deci | Project: SmartImage.Lib | Name: ChallengeDetector.cs
3+
// Date: 2024/10/16 @ 15:10:44
4+
5+
using System.Net;
6+
7+
namespace SmartImage.Lib.Clients;
8+
9+
public static class ChallengeDetector
10+
{
11+
private static readonly HashSet<string> CloudflareServerNames = new HashSet<string>{
12+
"cloudflare",
13+
"cloudflare-nginx",
14+
"ddos-guard"
15+
};
16+
17+
/// <summary>
18+
/// Checks if clearance is required.
19+
/// </summary>
20+
/// <param name="response">The HttpResponseMessage to check.</param>
21+
/// <returns>True if the site requires clearance</returns>
22+
public static bool IsClearanceRequired(HttpResponseMessage response) => IsCloudflareProtected(response);
23+
24+
/// <summary>
25+
/// Checks if the site is protected by Cloudflare
26+
/// </summary>
27+
/// <param name="response">The HttpResponseMessage to check.</param>
28+
/// <returns>True if the site is protected</returns>
29+
private static bool IsCloudflareProtected(HttpResponseMessage response)
30+
{
31+
// check response headers
32+
if (!response.Headers.Server.Any(i =>
33+
i.Product != null && CloudflareServerNames.Contains(i.Product.Name.ToLower())))
34+
return false;
35+
36+
// detect CloudFlare and DDoS-GUARD
37+
if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) ||
38+
response.StatusCode.Equals(HttpStatusCode.Forbidden)) {
39+
var responseHtml = response.Content.ReadAsStringAsync().Result;
40+
if (responseHtml.Contains("<title>Just a moment...</title>") || // Cloudflare
41+
responseHtml.Contains("<title>Access denied</title>") || // Cloudflare Blocked
42+
responseHtml.Contains("<title>Attention Required! | Cloudflare</title>") || // Cloudflare Blocked
43+
responseHtml.Trim().Equals("error code: 1020") || // Cloudflare Blocked
44+
responseHtml.IndexOf("<title>DDOS-GUARD</title>", StringComparison.OrdinalIgnoreCase) > -1) // DDOS-GUARD
45+
return true;
46+
}
47+
48+
// detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
49+
if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" &&
50+
response.Content.Headers.ContentEncoding.ToString() == "" &&
51+
response.Content.ReadAsStringAsync().Result.ToLower().Contains("ddos"))
52+
return true;
53+
54+
return false;
55+
}
56+
57+
}

‎SmartImage.Lib/Clients/FlareSolverr.cs‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
// Author: Deci | Project: SmartImage.Lib | Name: FlareSolverr.cs
2+
// Date: 2024/10/16 @ 15:10:40
3+
4+
using System.Net;
5+
using System.Net.Http.Json;
6+
using System.Text;
7+
using System.Text.Json;
8+
using System.Text.Json.Serialization;
9+
10+
namespace SmartImage.Lib.Clients;
11+
12+
public class FlareSolverr
13+
{
14+
15+
private static readonly SemaphoreLocker Locker = new SemaphoreLocker();
16+
private HttpClient _httpClient;
17+
private readonly Uri _flareSolverrUri;
18+
19+
public int MaxTimeout = 60000;
20+
public string ProxyUrl = "";
21+
public string ProxyUsername = null;
22+
public string ProxyPassword = null;
23+
24+
public FlareSolverr(string flareSolverrApiUrl)
25+
{
26+
var apiUrl = flareSolverrApiUrl;
27+
28+
if (!apiUrl.EndsWith("/"))
29+
apiUrl += "/";
30+
_flareSolverrUri = new Uri(apiUrl + "v1");
31+
}
32+
33+
public Task<FlareSolverrRoot> Solve(HttpRequestMessage request, string sessionId = "")
34+
{
35+
return SendFlareSolverrRequestAsync(GenerateFlareSolverrRequest(request, sessionId));
36+
}
37+
38+
public Task<FlareSolverrRoot> CreateSession()
39+
{
40+
var req = new FlareSolverrRequest
41+
{
42+
Command = "sessions.create",
43+
MaxTimeout = MaxTimeout,
44+
Proxy = GetProxy()
45+
};
46+
return SendFlareSolverrRequestAsync(GetSolverRequestContent(req));
47+
}
48+
49+
public Task<FlareSolverrRoot> ListSessionsAsync()
50+
{
51+
var req = new FlareSolverrRequest
52+
{
53+
Command = "sessions.list",
54+
MaxTimeout = MaxTimeout,
55+
Proxy = GetProxy()
56+
};
57+
return SendFlareSolverrRequestAsync(GetSolverRequestContent(req));
58+
}
59+
60+
public async Task<FlareSolverrRoot> DestroySessionAsync(string sessionId)
61+
{
62+
var req = new FlareSolverrRequest
63+
{
64+
Command = "sessions.destroy",
65+
MaxTimeout = MaxTimeout,
66+
Proxy = GetProxy(),
67+
Session = sessionId
68+
};
69+
return await SendFlareSolverrRequestAsync(GetSolverRequestContent(req));
70+
}
71+
72+
private async Task<FlareSolverrRoot> SendFlareSolverrRequestAsync(HttpContent flareSolverrRequest)
73+
{
74+
FlareSolverrRoot result = null;
75+
76+
await Locker.LockAsync(async () =>
77+
{
78+
HttpResponseMessage response;
79+
80+
try {
81+
_httpClient = new HttpClient();
82+
83+
// wait 5 more seconds to make sure we return the FlareSolverr timeout message
84+
_httpClient.Timeout = TimeSpan.FromMilliseconds(MaxTimeout + 5000);
85+
response = await _httpClient.PostAsync(_flareSolverrUri, flareSolverrRequest);
86+
}
87+
catch (HttpRequestException e) {
88+
throw new FlareSolverrException("Error connecting to FlareSolverr server: " + e);
89+
}
90+
catch (Exception e) {
91+
throw new FlareSolverrException("Exception: " + e);
92+
}
93+
finally {
94+
_httpClient.Dispose();
95+
}
96+
97+
// Don't try parsing if FlareSolverr hasn't returned 200 or 500
98+
if (response.StatusCode != HttpStatusCode.OK
99+
&& response.StatusCode != HttpStatusCode.InternalServerError) {
100+
throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :"
101+
+ response.StatusCode);
102+
}
103+
104+
var resContent = await response.Content.ReadAsStringAsync();
105+
106+
try {
107+
result = JsonSerializer.Deserialize<FlareSolverrRoot>(resContent, FlareSolverrHandler.s_jsonSerializerOptions);
108+
}
109+
catch (Exception) {
110+
throw new FlareSolverrException("Error parsing response, check FlareSolverr. Response: "
111+
+ resContent);
112+
}
113+
114+
try {
115+
Enum.TryParse(result.Status, true, out FlareSolverrStatusCode returnStatusCode);
116+
117+
if (returnStatusCode.Equals(FlareSolverrStatusCode.ok)) {
118+
return result;
119+
}
120+
121+
if (returnStatusCode.Equals(FlareSolverrStatusCode.warning)) {
122+
throw new FlareSolverrException(
123+
"FlareSolverr was able to process the request, but a captcha was detected. Message: "
124+
+ result.Message);
125+
}
126+
127+
if (returnStatusCode.Equals(FlareSolverrStatusCode.error)) {
128+
throw new FlareSolverrException(
129+
"FlareSolverr was unable to process the request, please check FlareSolverr logs. Message: "
130+
+ result.Message);
131+
}
132+
133+
throw new FlareSolverrException("Unable to map FlareSolverr returned status code, received code: "
134+
+ result.Status + ". Message: " + result.Message);
135+
}
136+
catch (ArgumentException) {
137+
throw new FlareSolverrException("Error parsing status code, check FlareSolverr log. Status: "
138+
+ result.Status + ". Message: " + result.Message);
139+
}
140+
});
141+
142+
return result;
143+
}
144+
145+
private FlareSolverrRequestProxy GetProxy()
146+
{
147+
FlareSolverrRequestProxy proxy = null;
148+
149+
if (!string.IsNullOrWhiteSpace(ProxyUrl)) {
150+
proxy = new FlareSolverrRequestProxy
151+
{
152+
Url = ProxyUrl,
153+
};
154+
155+
if (!string.IsNullOrWhiteSpace(ProxyUsername)) {
156+
proxy.Username = ProxyUsername;
157+
}
158+
159+
;
160+
161+
if (!string.IsNullOrWhiteSpace(ProxyPassword)) {
162+
proxy.Password = ProxyPassword;
163+
}
164+
165+
;
166+
}
167+
168+
return proxy;
169+
}
170+
171+
private static HttpContent GetSolverRequestContent(FlareSolverrRequest request)
172+
{
173+
var content = JsonContent.Create(request, options: FlareSolverrHandler.s_jsonSerializerOptions);
174+
// HttpContent content = new StringContent(payload, Encoding.UTF8, "application/json");
175+
return content;
176+
}
177+
178+
private HttpContent GenerateFlareSolverrRequest(HttpRequestMessage request, string sessionId = "")
179+
{
180+
FlareSolverrRequest req;
181+
182+
if (string.IsNullOrWhiteSpace(sessionId))
183+
sessionId = null;
184+
185+
var url = request.RequestUri.ToString();
186+
187+
FlareSolverrRequestProxy proxy = GetProxy();
188+
189+
if (request.Method == HttpMethod.Get) {
190+
req = new FlareSolverrRequest
191+
{
192+
Command = "request.get",
193+
Url = url,
194+
MaxTimeout = MaxTimeout,
195+
Proxy = proxy,
196+
Session = sessionId
197+
};
198+
}
199+
else if (request.Method == HttpMethod.Post) {
200+
// request.Content.GetType() doesn't work well when encoding != utf-8
201+
var contentMediaType = request.Content.Headers.ContentType?.MediaType.ToLower() ?? "<null>";
202+
203+
if (contentMediaType.Contains("application/x-www-form-urlencoded")) {
204+
req = new FlareSolverrRequest
205+
{
206+
Command = "request.post",
207+
Url = url,
208+
PostData = request.Content.ReadAsStringAsync().Result,
209+
MaxTimeout = MaxTimeout,
210+
Proxy = proxy,
211+
Session = sessionId
212+
};
213+
}
214+
else if (contentMediaType.Contains("multipart/form-data")
215+
|| contentMediaType.Contains("text/html")) {
216+
//TODO Implement - check if we just need to pass the content-type with the relevant headers
217+
throw new FlareSolverrException("Unimplemented POST Content-Type: " + contentMediaType);
218+
}
219+
else {
220+
throw new FlareSolverrException("Unsupported POST Content-Type: " + contentMediaType);
221+
}
222+
}
223+
else {
224+
throw new FlareSolverrException("Unsupported HttpMethod: " + request.Method);
225+
}
226+
227+
return GetSolverRequestContent(req);
228+
}
229+
230+
}

0 commit comments

Comments
 (0)