Skip to content

Commit 9d33192

Browse files
CuadrixCuadrix
committed
v1.2.9
1 parent e42602b commit 9d33192

File tree

8 files changed

+105
-72
lines changed

8 files changed

+105
-72
lines changed

‎changelog.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# Change log
2+
### [1.2.9] - 2022-10-10
3+
#### Fixes
4+
- Allow ports in request url for `host` request header ([#61](https://github.com/Cuadrix/puppeteer-page-proxy/issues/61)) ([#62](https://github.com/Cuadrix/puppeteer-page-proxy/pull/62))
5+
- Take into account how `CDPSession` client is exposed in latest versions of Puppeteer ([#78](https://github.com/Cuadrix/puppeteer-page-proxy/issues/78)) ([#79](https://github.com/Cuadrix/puppeteer-page-proxy/pull/79))
6+
- Allow domain cookies to be unset ([#48](https://github.com/Cuadrix/puppeteer-page-proxy/issues/48)) ([#48#issuecomment-729802384](https://github.com/Cuadrix/puppeteer-page-proxy/issues/48#issuecomment-729802384))
7+
- Take into account that `request.frame()` might return `null` ([#36](https://github.com/Cuadrix/puppeteer-page-proxy/issues/36)) ([#43](https://github.com/Cuadrix/puppeteer-page-proxy/issues/43)) ([#59](https://github.com/Cuadrix/puppeteer-page-proxy/issues/59)) ([#36#issuecomment-814520620](https://github.com/Cuadrix/puppeteer-page-proxy/issues/36#issuecomment-814520620))
8+
- Update differentiation between page and http request objects for latest versions of Puppeteer (`Page` -> `CDPPage`)
9+
- Update `lookup` method for latest versions of Puppeteer
210
### [1.2.8] - 2020-07-21
311
#### Changes
412
- Fixed silent failure when there was an invalid host in the cookies set by the server ([#32](https://github.com/Cuadrix/puppeteer-page-proxy/issues/32))

‎package.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "puppeteer-page-proxy",
33
"description": "Additional Node.js module to use with 'puppeteer' for setting proxies per page basis.",
4-
"version": "1.2.8",
4+
"version": "1.2.9",
55
"author": "Cuadrix <[email protected]> (https://github.com/Cuadrix)",
66
"homepage": "https://github.com/Cuadrix/puppeteer-page-proxy",
77
"main": "./src/index.js",
@@ -22,10 +22,10 @@
2222
],
2323
"license": "MIT",
2424
"dependencies": {
25-
"got": "^11.5.1",
26-
"http-proxy-agent": "^4.0.1",
27-
"https-proxy-agent": "^5.0.0",
28-
"socks-proxy-agent": "^5.0.0",
29-
"tough-cookie": "^4.0.0"
25+
"got": "^11.8.5",
26+
"http-proxy-agent": "^5.0.0",
27+
"https-proxy-agent": "^5.0.1",
28+
"socks-proxy-agent": "^7.0.0",
29+
"tough-cookie": "^4.1.2"
3030
}
31-
}
31+
}

‎src/core/lookup.js‎

Lines changed: 44 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,61 @@
1-
const lookup = async (page, lookupService = "https://api64.ipify.org?format=json", isJSON = true, timeout = 30000) => {
1+
const defaults = {
2+
url: "https://api64.ipify.org?format=json",
3+
json: true,
4+
timeout: 30000
5+
};
6+
7+
const onLookupFail = (message) => {console.error(message)}
8+
const isOnLookupFailExposed = new WeakMap();
9+
10+
const lookup = async (page, lookupServiceUrl = defaults.url, isJSON = defaults.json, timeout = defaults.timeout) => {
211
const doLookup = async () => {
3-
return await page.evaluate((lookupService, timeout, isJSON) => {
12+
// Wait for network to be idle before evaluating code in page context
13+
await page.waitForNetworkIdle();
14+
return await page.evaluate((pageUrl, lookupServiceUrl, timeout, isJSON) => {
415
return new Promise((resolve) => {
516
const request = new XMLHttpRequest();
617
request.timeout = timeout;
718
request.onload = () => {
819
if (request.status >= 200 && request.status <= 299) {
920
resolve(isJSON ? JSON.parse(request.responseText) : request.responseText);
10-
} else {resolve(onLookupFailed(
11-
`Request from ${window.location.href} to ` +
12-
`${lookupService} failed with status code ${request.status}`
13-
))}
21+
} else {
22+
// Print message to browser and NodeJS console
23+
const failMessage =
24+
`Lookup request from ${pageUrl} to ${lookupServiceUrl} ` +
25+
`failed with status code ${request.status}`;
26+
console.error(failMessage);
27+
$ppp_onLookupFail(failMessage);
28+
resolve();
29+
}
30+
};
31+
request.ontimeout = () => {
32+
// Print message to browser and NodeJS console
33+
const timeOutMessage =
34+
`Lookup request from ${pageUrl} to ${lookupServiceUrl} ` +
35+
`timed out at ${request.timeout} ms`;
36+
console.error(timeOutMessage);
37+
$ppp_onLookupFail(timeOutMessage);
38+
resolve();
1439
};
15-
request.ontimeout = (error) => {resolve(onLookupFailed(
16-
`Request from ${window.location.href} to ` +
17-
`${lookupService} timed out at ${request.timeout} ms`
18-
))};
19-
request.open("GET", lookupService, true);
40+
request.open("GET", lookupServiceUrl, true);
2041
request.send();
2142
});
22-
}, lookupService, timeout, isJSON);
43+
}, page.url(), lookupServiceUrl, timeout, isJSON);
2344
};
2445
try {
25-
await page.setBypassCSP(true);
26-
const functionName = "$ppp_on_lookup_failed";
27-
if (!page._pageBindings.has(functionName)) {
28-
await page.exposeFunction(functionName, (failReason) => {
29-
console.error(failReason); return;
30-
});
46+
// Expose function to log error on NodeJS side
47+
// Deal with already exposed error by explicitly keeping track of function exposure
48+
if (!isOnLookupFailExposed.get(page)) {
49+
await page.exposeFunction("$ppp_onLookupFail", onLookupFail);
50+
isOnLookupFailExposed.set(page, true);
3151
}
52+
// Stop keeping track of exposure if page is closed
53+
if (page.isClosed()) {
54+
isOnLookupFailExposed.delete(page);
55+
}
56+
await page.setBypassCSP(true);
3257
return await doLookup();
33-
} catch(error) {console.error(error)}
58+
} catch(error) {console.log(error)}
3459
};
3560

3661
module.exports = lookup;

‎src/core/proxy.js‎

Lines changed: 31 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ const requestHandler = async (request, proxy, overrides = {}) => {
1010
request.continue(); return;
1111
}
1212
const cookieHandler = new CookieHandler(request);
13-
// Request options for Got accounting for overrides
13+
// Request options for GOT accounting for overrides
1414
const options = {
1515
cookieJar: await cookieHandler.getCookies(),
1616
method: overrides.method || request.method(),
@@ -57,42 +57,39 @@ const removeRequestListener = (page, listenerName) => {
5757
}
5858
};
5959

60-
// Calls this if request object passed
61-
const proxyPerRequest = async (request, data) => {
62-
let proxy, overrides;
63-
// Separate proxy and overrides
64-
if (type(data) === "object") {
65-
if (Object.keys(data).length !== 0) {
66-
proxy = data.proxy;
67-
delete data.proxy;
68-
overrides = data;
69-
}
70-
} else {proxy = data}
71-
// Skip request if proxy omitted
72-
if (proxy) {await requestHandler(request, proxy, overrides)}
73-
else {request.continue(overrides)}
74-
};
60+
const useProxyPer = {
61+
// Call this if request object passed
62+
HTTPRequest: async (request, data) => {
63+
let proxy, overrides;
64+
// Separate proxy and overrides
65+
if (type(data) === "object") {
66+
if (Object.keys(data).length !== 0) {
67+
proxy = data.proxy;
68+
delete data.proxy;
69+
overrides = data;
70+
}
71+
} else {proxy = data}
72+
// Skip request if proxy omitted
73+
if (proxy) {await requestHandler(request, proxy, overrides)}
74+
else {request.continue(overrides)}
75+
},
7576

76-
// Calls this if page object passed
77-
const proxyPerPage = async (page, proxy) => {
78-
await page.setRequestInterception(true);
79-
const listener = "$ppp_request_listener";
80-
removeRequestListener(page, listener);
81-
const f = {[listener]: async (request) => {
82-
await requestHandler(request, proxy);
83-
}};
84-
if (proxy) {page.on("request", f[listener])}
85-
else {await page.setRequestInterception(false)}
86-
};
77+
// Call this if page object passed
78+
CDPPage: async (page, proxy) => {
79+
await page.setRequestInterception(true);
80+
const listener = "$ppp_requestListener";
81+
removeRequestListener(page, listener);
82+
const f = {[listener]: async (request) => {
83+
await requestHandler(request, proxy);
84+
}};
85+
if (proxy) {page.on("request", f[listener])}
86+
else {await page.setRequestInterception(false)}
87+
}
88+
}
8789

8890
// Main function
8991
const useProxy = async (target, data) => {
90-
const targetType = target.constructor.name;
91-
if (targetType === "HTTPRequest") {
92-
await proxyPerRequest(target, data);
93-
} else if (targetType === "Page") {
94-
await proxyPerPage(target, data);
95-
}
92+
useProxyPer[target.constructor.name](target, data);
9693
};
9794

98-
module.exports = useProxy;
95+
module.exports = useProxy;

‎src/index.d.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
export = puppeteer_page_proxy;
1+
export = useProxy;
22
/**
33
* **Set a proxy to use in a given page or request.**
44
*
@@ -11,8 +11,8 @@ export = puppeteer_page_proxy;
1111
* @param page 'Page' or 'Request' object to set a proxy for.
1212
* @param proxy Proxy to use in the current page. Must begin with a protocol e.g. **http://**, **https://**, **socks://**.
1313
*/
14-
declare function puppeteer_page_proxy(page: object, proxy: string | object): Promise<any>;
15-
declare namespace puppeteer_page_proxy {
14+
declare function useProxy(page: object, proxy: string | object): Promise<any>;
15+
declare namespace useProxy {
1616
/**
1717
* **Request data from a lookupservice.**
1818
*
@@ -23,9 +23,9 @@ declare namespace puppeteer_page_proxy {
2323
* console.log(data.ip);
2424
* ```
2525
* @param page 'Page' object to execute the request on.
26-
* @param lookupService External lookup service to request data from. Fetches data from `api64.ipify.org` by default.
26+
* @param lookupServiceUrl External lookup service to request data from. Fetches data from `api64.ipify.org` by default.
2727
* @param isJSON Whether to JSON.parse the received response. Defaults to `true`.
2828
* @param timeout Time in milliseconds after which the request times out. Defaults to `30000` ms.
2929
*/
30-
function lookup(page: object, lookupService?: string, isJSON?: boolean, timeout?: number | string): Promise<any>;
30+
function lookup(page: object, lookupServiceUrl?: string, isJSON?: boolean, timeout?: number | string): Promise<any>;
3131
}

‎src/lib/cdp.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ class CDP {
1515
}
1616
}
1717

18-
module.exports = CDP;
18+
module.exports = CDP;

‎src/lib/cookies.js‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,11 @@ const formatCookie = (cookie) => {
6464
class CookieHandler extends CDP {
6565
constructor(request) {
6666
super(request._client || request.client);
67-
this.url = request.isNavigationRequest() ? request.url() : request.frame().url();
68-
this.domain = new URL(this.url).hostname;
67+
this.url =
68+
(request.isNavigationRequest() || request.frame() == null)
69+
? request.url()
70+
: request.frame().url();
71+
this.domain = (this.url) ? new URL(this.url).hostname : "";
6972
}
7073
// Parse an array of raw cookies to an array of cookie objects
7174
parseCookies(rawCookies) {
@@ -85,7 +88,7 @@ class CookieHandler extends CDP {
8588
const toughCookies = this.formatCookies(browserCookies);
8689
// Add cookies to cookieJar
8790
const cookieJar = CookieJar.deserializeSync({
88-
version: 'tough-cookie@4.0.0',
91+
version: 'tough-cookie@4.1.2',
8992
storeType: 'MemoryCookieStore',
9093
rejectPublicSuffixes: true,
9194
cookies: toughCookies
@@ -111,4 +114,4 @@ class CookieHandler extends CDP {
111114
}
112115
}
113116

114-
module.exports = CookieHandler;
117+
module.exports = CookieHandler;

‎src/lib/options.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const setHeaders = (request) => {
99
...request.headers(),
1010
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
1111
"accept-encoding": "gzip, deflate, br",
12-
"host": new URL(request.url()).host // hostname + port
12+
"host": new URL(request.url()).host
1313
}
1414
if (request.isNavigationRequest()) {
1515
headers["sec-fetch-mode"] = "navigate";
@@ -36,4 +36,4 @@ const setAgent = (proxy) => {
3636
};
3737
};
3838

39-
module.exports = {setHeaders, setAgent};
39+
module.exports = {setHeaders, setAgent};

0 commit comments

Comments
 (0)