Move fullDomFetcher to Playwright

Author: LVerneyEC

package.json

@@ -94,10 +94,8 @@
     "morgan": "^1.10.0",
     "node-fetch": "^3.1.0",
     "octokit": "2.0.2",
+    "patchright": "1.50.1",
     "pdfjs-dist": "^2.9.359",
-    "puppeteer": "^22.8.1",
-    "puppeteer-extra": "^3.3.6",
-    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sib-api-v3-sdk": "^8.2.1",
     "simple-git": "^3.27.0",
     "swagger-jsdoc": "^6.2.8",

src/archivist/fetcher/fullDomFetcher.js

@@ -1,94 +1,103 @@
-import { TimeoutError } from 'puppeteer';
-import puppeteer from 'puppeteer-extra';
-import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { chromium, errors } from 'patchright';
 
-puppeteer.use(stealthPlugin());
 
 let browser;
 
 export default async function fetch(url, cssSelectors, config) {
-  let page;
-  let response;
-  const selectors = [].concat(cssSelectors);
-
-  if (!browser) {
-    throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".');
-  }
-
-  try {
-    page = await browser.newPage();
-
-    await page.setDefaultNavigationTimeout(config.navigationTimeout);
-    await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
-
-    response = await page.goto(url, { waitUntil: 'networkidle0' });
+    let page;
+    let response;
+    const selectors = [].concat(cssSelectors);
 
-    if (!response) {
-      throw new Error(`Response is empty when trying to fetch '${url}'`);
+    if (!browser) {
+        throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".');
     }
 
-    const statusCode = response.status();
-
-    if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
-      throw new Error(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
+    try {
+        page = await browser.newPage();
+
+        await page.setDefaultNavigationTimeout(config.navigationTimeout);
+        await page.setExtraHTTPHeaders({ 'Accept-Language': config.language });
+
+        response = await page.goto(url);
+
+        if (!response) {
+            throw new Error(`Response is empty when trying to fetch '${url}'`);
+        }
+
+        const statusCode = response.status();
+
+        if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+            throw new Error(`Received HTTP code ${statusCode} when trying to fetch '${url}'`);
+        }
+
+        const waitForSelectorsPromises = selectors.map(selector => page.waitForSelector(selector, { timeout: config.waitForElementsTimeout }));
+
+        // We expect all elements to be present on the page…
+        await Promise.all(waitForSelectorsPromises).catch(error => {
+            if (error.name == 'TimeoutError') {
+                // however, if they are not, this is not considered as an error since selectors may be out of date
+                // and the whole content of the page should still be returned.
+                return;
+            }
+
+            throw error;
+        });
+
+        return {
+            mimeType: 'text/html',
+            content: await page.content(),
+        };
+    } catch (error) {
+        if (error instanceof errors.TimeoutError) {
+            throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+        }
+        throw new Error(error.message);
+    } finally {
+        if (page) {
+            await page.close();
+        }
     }
+}
 
-    const waitForSelectorsPromises = selectors.map(selector => page.waitForSelector(selector, { timeout: config.waitForElementsTimeout }));
-
-    // We expect all elements to be present on the page…
-    await Promise.all(waitForSelectorsPromises).catch(error => {
-      if (error.name == 'TimeoutError') {
-        // however, if they are not, this is not considered as an error since selectors may be out of date
-        // and the whole content of the page should still be returned.
-        return;
-      }
-
-      throw error;
-    });
+export async function launchHeadlessBrowser() {
+    if (browser) {
+        return browser;
+    }
 
-    return {
-      mimeType: 'text/html',
-      content: await page.content(),
+    let options = {
+        channel: "chrome",
+        viewport: null,
     };
-  } catch (error) {
-    if (error instanceof TimeoutError) {
-      throw new Error(`Timed out after ${config.navigationTimeout / 1000} seconds when trying to fetch '${url}'`);
+    if (process.env.http_proxy) {
+        let proxyUrl = new URL(process.env.http_proxy);
+        options['proxy'] = {
+            server: proxyUrl.origin,
+            username: proxyUrl.username,
+            password: proxyUrl.password
+        }
     }
-    throw new Error(error.message);
-  } finally {
-    if (page) {
-      await page.close();
+    if (process.env.PLAYWRIGHT_NO_SANDBOX) {
+        options['args'] = ["--no-sandbox", "--disable-setuid-sandbox"];
     }
-  }
-}
+    if (process.env.PLAYWRIGHT_NO_HEADLESS) {
+        options['headless'] = false;
+    }
+    const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ota'));
+    browser = await chromium.launchPersistentContext(userDataDir, options);
 
-/**
- * Launches a headless browser instance using Puppeteer if one is not already running. Returns the existing browser instance if one is already running, otherwise creates and returns a new instance.
- * @function launchHeadlessBrowser
- * @returns {Promise<puppeteer.Browser>} The Puppeteer browser instance.
- * @async
- */
-export async function launchHeadlessBrowser() {
-  if (browser) {
     return browser;
-  }
-
-  browser = await puppeteer.launch({ headless: true });
-
-  return browser;
 }
 
-/**
- * Stops the headless browser instance if one is running. If no instance exists, it does nothing.
- * @function stopHeadlessBrowser
- * @returns {Promise<void>}
- * @async
- */
 export async function stopHeadlessBrowser() {
-  if (!browser) {
-    return;
-  }
+    if (!browser) {
+        return;
+    }
 
-  await browser.close();
-  browser = null;
+    await browser.close();
+    browser = null;
 }
+