Factorize proxy logic & store creds in browser

Author: LVerneyEC

src/archivist/fetcher/fullDomFetcher.js

@@ -1,10 +1,11 @@
 import puppeteer from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 
+import { resolveProxyConfiguration, extractProxyCredentials } from './proxyUtils.js';
+
 puppeteer.use(stealthPlugin());
 
 let browser;
-let proxyCredentials = {};
 
 export default async function fetch(url, cssSelectors, config) {
   let page;
@@ -26,8 +27,8 @@ export default async function fetch(url, cssSelectors, config) {
 
     await client.send('Network.clearBrowserCookies'); // Clear cookies to ensure clean state between fetches and prevent session persistence across different URLs
 
-    if (proxyCredentials.username && proxyCredentials.password) {
-      await page.authenticate(proxyCredentials);
+    if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) {
+      await page.authenticate(browser.proxyCredentials);
     }
 
     response = await page.goto(url, { waitUntil: 'load' }); // Using `load` instead of `networkidle0` as it's more reliable and faster. The 'load' event fires when the page and all its resources (stylesheets, scripts, images) have finished loading. `networkidle0` can be problematic as it waits for 500ms of network inactivity, which may never occur on dynamic pages and then triggers a navigation timeout.
@@ -96,37 +97,15 @@ export async function launchHeadlessBrowser() {
     headless: !process.env.FETCHER_NO_HEADLESS,
   };
 
-  // Handle http_proxy/https_proxy environment variables precedence
-  let http_proxy = null;
-  let https_proxy = null;
+  const { httpProxy, httpsProxy } = resolveProxyConfiguration();
 
-  if (process.env.http_proxy) {
-    http_proxy = process.env.http_proxy;
-  }
-  else if (process.env.HTTP_PROXY) {
-    http_proxy = process.env.HTTP_PROXY;
-  }
-
-  if (process.env.https_proxy) {
-    https_proxy = process.env.https_proxy;
-  }
-  else if (process.env.HTTPS_PROXY) {
-    https_proxy = process.env.HTTPS_PROXY;
-  }
-  else if (http_proxy) {
-    https_proxy = http_proxy;
-  }
+  let proxyCredentials = null;
 
-  // Set proxy in Puppeteer and eventually store credentials
-  if (http_proxy) {
-    const httpProxyUrl = new URL(http_proxy);
-    const httpsProxyUrl = new URL(https_proxy);
-    proxyCredentials.username = httpProxyUrl.username;
-    proxyCredentials.password = httpProxyUrl.password;
+  if (httpProxy) {
+    const httpProxyUrl = new URL(httpProxy);
+    const httpsProxyUrl = new URL(httpsProxy);
 
-    if (httpProxyUrl.username != httpsProxyUrl.username || httpProxyUrl.password != httpsProxyUrl.password) {
-      throw new Error('Unsupported proxies specified, http and https proxy should have the same credentials.');
-    }
+    proxyCredentials = extractProxyCredentials(httpProxy, httpsProxy);
 
     options.args = [].concat(options.args, `--proxy-server=http=${httpProxyUrl.host};https=${httpsProxyUrl.host}`);
   }
@@ -137,6 +116,10 @@ export async function launchHeadlessBrowser() {
 
   browser = await puppeteer.launch(options);
 
+  if (proxyCredentials) {
+    browser.proxyCredentials = proxyCredentials;
+  }
+
   return browser;
 }
 

src/archivist/fetcher/htmlOnlyFetcher.js

@@ -4,6 +4,8 @@ import HttpProxyAgent from 'http-proxy-agent';
 import HttpsProxyAgent from 'https-proxy-agent';
 import nodeFetch, { AbortError } from 'node-fetch';
 
+import { resolveProxyConfiguration } from './proxyUtils.js';
+
 export default async function fetch(url, config) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), config.navigationTimeout);
@@ -14,31 +16,12 @@ export default async function fetch(url, config) {
     headers: { 'Accept-Language': config.language },
   };
 
-  // Handle http_proxy/https_proxy environment variables precedence
-  let http_proxy = null;
-  let https_proxy = null;
-
-  if (process.env.http_proxy) {
-    http_proxy = process.env.http_proxy;
-  }
-  else if (process.env.HTTP_PROXY) {
-    http_proxy = process.env.HTTP_PROXY;
-  }
-
-  if (process.env.https_proxy) {
-    https_proxy = process.env.https_proxy;
-  }
-  else if (process.env.HTTPS_PROXY) {
-    https_proxy = process.env.HTTPS_PROXY;
-  }
-  else if (http_proxy) {
-    https_proxy = http_proxy;
-  }
+  const { httpProxy, httpsProxy } = resolveProxyConfiguration();
 
-  if (url.startsWith('https:') && https_proxy) {
-    nodeFetchOptions.agent = new HttpsProxyAgent(https_proxy);
-  } else if (url.startsWith('http:') && http_proxy) {
-    nodeFetchOptions.agent = new HttpProxyAgent(http_proxy);
+  if (url.startsWith('https:') && httpsProxy) {
+    nodeFetchOptions.agent = new HttpsProxyAgent(httpsProxy);
+  } else if (url.startsWith('http:') && httpProxy) {
+    nodeFetchOptions.agent = new HttpProxyAgent(httpProxy);
   }
 
   let response;

src/archivist/fetcher/proxyUtils.js

@@ -0,0 +1,41 @@
+export function resolveProxyConfiguration() {
+  let httpProxy = null;
+  let httpsProxy = null;
+
+  if (process.env.http_proxy) {
+    httpProxy = process.env.http_proxy;
+  } else if (process.env.HTTP_PROXY) {
+    httpProxy = process.env.HTTP_PROXY;
+  }
+
+  if (process.env.https_proxy) {
+    httpsProxy = process.env.https_proxy;
+  } else if (process.env.HTTPS_PROXY) {
+    httpsProxy = process.env.HTTPS_PROXY;
+  } else if (httpProxy) {
+    httpsProxy = httpProxy;
+  }
+
+  return { httpProxy, httpsProxy };
+}
+
+export function extractProxyCredentials(httpProxy, httpsProxy) {
+  if (!httpProxy) {
+    return null;
+  }
+
+  const httpProxyUrl = new URL(httpProxy);
+  const httpsProxyUrl = new URL(httpsProxy);
+
+  const { username, password } = httpProxyUrl;
+
+  if (!username || !password) {
+    return null;
+  }
+
+  if (httpProxyUrl.username !== httpsProxyUrl.username || httpProxyUrl.password !== httpsProxyUrl.password) {
+    throw new Error('Unsupported proxies specified, http and https proxy should have the same credentials.');
+  }
+
+  return { username, password };
+}