v1.2.6

Author: Cuadrix

changelog.md

@@ -1,4 +1,8 @@
 # Change log
+### [1.2.6] - 2020-06-18
+#### Changes
+- Updated for Puppeteer's v4.0.0 [breaking changes](https://github.com/puppeteer/puppeteer/releases/tag/v4.0.0) ([#22](https://github.com/Cuadrix/puppeteer-page-proxy/issues/22), [#23](https://github.com/Cuadrix/puppeteer-page-proxy/issues/23))
+- Modified cookie handling to fix ([#20](https://github.com/Cuadrix/puppeteer-page-proxy/issues/20)) among other cookie related errors
 ### [1.2.5] - 2020-05-21
 #### Changes
 - Added ability to override requests

package.json

@@ -1,7 +1,7 @@
 {
   "name": "puppeteer-page-proxy",
   "description": "Additional Node.js module to use with 'puppeteer' for setting proxies per page basis.",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "author": "Cuadrix <[email protected]> (https://github.com/Cuadrix)",
   "homepage": "https://github.com/Cuadrix/puppeteer-page-proxy",
   "main": "./src/index.js",
@@ -22,7 +22,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "got": "^11.1.4",
+    "got": "^11.3.0",
     "http-proxy-agent": "^4.0.1",
     "https-proxy-agent": "^5.0.0",
     "socks-proxy-agent": "^5.0.0",

src/core/lookup.js

@@ -1,37 +1,39 @@
 const lookup = async (page, lookupService = "https://api.ipify.org?format=json", isJSON = true, timeout = 30000) => {
-    const XMLHttpRequest = async () => {
+    const doLookup = async () => {
         return await page.evaluate((lookupService, timeout, isJSON) => {
-            return new Promise(resolve => {
-                var req = new XMLHttpRequest();
-                req.open("GET", lookupService, true);
+            return new Promise((resolve) => {
+                const req = new XMLHttpRequest();
                 req.timeout = timeout;
                 req.onload = () => {
                     if (req.status >= 200 && req.status <= 299) {
                         resolve(isJSON ? JSON.parse(req.responseText) : req.responseText);
                     } else {
-                        resolve(xhrFailed(`Request from [${window.location.href.slice(0, -1)}] to [${lookupService}] failed with status code ${req.status}`));
+                        resolve(onLookupFailed(`Request from ${window.location.href} to ${lookupService} failed with status code ${req.status}`));
                     }
                 };
-                req.ontimeout = e => {
-                    resolve(xhrFailed(`Request from [${window.location.href.slice(0, -1)}] to [${lookupService}] timed out -> ${req.timeout} ms`));
+                req.ontimeout = (error) => {
+                    resolve(onLookupFailed(`Request from ${window.location.href} to ${lookupService} timed out -> ${req.timeout} ms`));
                 };
+                req.open("GET", lookupService, true);
                 req.send();
             });
         }, lookupService, timeout, isJSON);
     };
     try {
         await page.setBypassCSP(true);
-        const errName = "xhrFailed";
-        if (!page._pageBindings.has(errName)) {
-            await page.exposeFunction(errName, reason => {
+        const functionName = "onLookupFailed";
+        if (!page._pageBindings.has(functionName)) {
+            await page.exposeFunction(functionName, (reason) => {
                 console.error(reason);
                 return;
             });
         }
-        return await XMLHttpRequest();
+        return await doLookup();
     } catch(error) {
-        if (error.message === "Execution context was destroyed, most likely because of a navigation." || error.message === "Execution context was destroyed.")
-            return await XMLHttpRequest();
+        if (error.message.startsWith("Execution context was destroyed")) {
+            return await doLookup();
+        }
     }
 };
+
 module.exports = lookup;

src/core/proxy.js

@@ -1,64 +1,81 @@
 const request = require("got");
-const {type} = require("../lib/types");
-const {getCookies, cookieStore} = require("../lib/cookies");
+const type = require("../lib/types");
+const cookieJar = require("../lib/cookies");
 const {setOverrides, setHeaders, setAgent} = require("../lib/options");
 
-const useProxy = async (target, proxy) => {
-    // Listener responsible for applying proxy
-    const $puppeteerPageProxyHandler = async req => {       
-        endpoint = req._client._connection._url;
-        targetId = req._frame._id;
-        const cookieJar = cookieStore(
-            await getCookies(endpoint, targetId)
-        );
-        const options = {
-            cookieJar,
-            method: req.method(),
-            body: req.postData(),
-            headers: setHeaders(req),
-            agent: setAgent(proxy),
-            responseType: "buffer",
-            throwHttpErrors: false
-        };
-        try {
-            const res = await request(req.url(), options);
-            await req.respond({
-                status: res.statusCode,
-                headers: res.headers,
-                body: res.body
-            });
-        } catch(error) {
-            await req.abort();
-        }
-    };
-    // Remove existing listener for reassigning proxy of current page
-    const removeRequestListener = (page, listenerName) => {
-        const listeners = page.listeners("request");
-        for (let i = 0; i < listeners.length; i++) {
-            if (listeners[i].name === listenerName) {
-                page.removeListener("request", listeners[i]);
-            }
-        }
+// Responsible for applying proxy
+const proxyHandler = async (req, proxy) => {
+    const options = {
+        cookieJar,
+        method: req.method(),
+        body: req.postData(),
+        headers: setHeaders(req),
+        agent: setAgent(proxy),
+        responseType: "buffer",
+        maxRedirects: 15,
+        throwHttpErrors: false
     };
-    // Proxy per request
-    if (target.constructor.name.indexOf("Request")!=-1) {
-        if (type(proxy) == "object") {
-            target = setOverrides(target, proxy);
-            proxy = proxy.proxy;
-        }
-        await $puppeteerPageProxyHandler(target);
-    // Page-wide proxy
-    } else if (target.constructor.name === "Page") {
-        if (type(proxy) == "object") {
-            proxy = proxy.proxy;
+    try {
+        const res = await request(req.url(), options);
+        await req.respond({
+            status: res.statusCode,
+            headers: res.headers,
+            body: res.body
+        });
+    } catch(error) {await req.abort()}
+};
+
+// For reassigning proxy of page
+const removeRequestListener = (page, listenerName) => {
+    const eventName = "request";
+    const listeners = page.eventsMap.get(eventName);
+    if (listeners) {
+        const i = listeners.findIndex((listener) => {
+            return listener.name === listenerName
+        });
+        listeners.splice(i, 1);
+        if (!listeners.length) {
+            page.eventsMap.delete(eventName);
         }
-        await target.setRequestInterception(true);
-        removeRequestListener(target, "$puppeteerPageProxyHandler");
-        if (proxy) {
-            target.on("request", $puppeteerPageProxyHandler);
-        } else {
-            await target.setRequestInterception(false);
+    }
+};
+
+// Calls this if request object passed
+const proxyPerRequest = async (req, data) => {
+    let proxy, overrides;
+    // Separate proxy and overrides
+    if (type(data) === "object") {
+        if (Object.keys(data).length !== 0) {
+            proxy = data.proxy;
+            delete data.proxy;
+            overrides = data;
         }
+    } else {proxy = data}
+    req = setOverrides(req, overrides);
+    // Skip request if proxy omitted
+    if (proxy) {await proxyHandler(req, proxy)}
+    else {req.continue(overrides)}
+};
+
+// Calls this if page object passed
+const proxyPerPage = async (page, proxy) => {
+    await page.setRequestInterception(true);
+    removeRequestListener(page, "$ppp");
+    if (proxy) {
+        page.on("request", $ppp = async (req) => {
+            await proxyHandler(req, proxy);
+        });
+    } else {await page.setRequestInterception(false)}
+};
+
+// Main function
+const useProxy = async (target, data) => {
+    const targetType = target.constructor.name;
+    if (targetType === "HTTPRequest") {
+        await proxyPerRequest(target, data);
+    } else if (targetType === "Page") {
+        await proxyPerPage(target, data)
     }
 };
-module.exports = useProxy;
+
+module.exports = useProxy;

src/lib/cdp.js

@@ -1,40 +1,3 @@
-const WebSocket = require("ws");
 const {CookieJar} = require("tough-cookie");
-const {Target, Network} = require("./cdp");
 
-const cookies = {
-    async getCookies(endpoint, targetId) {
-        const ws = new WebSocket(endpoint, {
-            perMessageDeflate: false,
-            maxPayload: 180 * 4096 // 0.73728Mb
-        });
-        await new Promise(resolve => ws.once("open", resolve));
-        // Attach to target then get cookies
-        const sessionId = await Target.attachToTarget(ws, targetId);
-        return await Network.getCookies(ws, sessionId);
-    },
-    cookieStore(cookies) {
-        if (!cookies)
-            return;
-        return CookieJar.deserializeSync({
-            version: '[email protected]',
-            storeType: 'MemoryCookieStore',
-            rejectPublicSuffixes: true,
-            cookies: cookies.map((cookie) => {
-                return {
-                    key: cookie.name,
-                    value: cookie.value,
-                    expires: cookie.expires === -1 ? Infinity : new Date(cookie.expires * 1000).toISOString(),
-                    domain: cookie.domain.replace(/^\./, ""),
-                    path: cookie.path,
-                    secure: cookie.secure,
-                    httpOnly: cookie.httpOnly,
-                    sameSite: cookie.sameSite,
-                    creation: new Date().toISOString(),
-                    hostOnly: !(/^\./).test(cookie.domain)
-                };
-            })
-        });
-    }
-};
-module.exports = cookies;
+module.exports = new CookieJar();