Refactor CMPCollector and CookiePopupCollector (#143)

* Inject autoconsent in iframes

* Rely on unique execution context ids in CMPCollector and CookiePopupCollector

* Fix CMPCollector tests

* Add exception handlers around CDP calls

* tweak cmp timeouts

* helpful log message

* Log which site timed out

* Refactor: Merge CookiePopupCollector and CMPCollector

* lint fixes

* Encode unique context id in the binding name to prevent collisions

* remove unused variable

* Rename CMPCollector into CookiePopupsCollector

* Remove redundant CDP calls

* remove shadowing variables

* clean up when execution contexts are destroyed

* do not log stack trace

* lint fix

* Fix tests

* Limit cookie consent scraping and add timing logs

* Increase the total timeout (allow more slow pages) and add more timing logs

* Do not let remote browser throw during closing

* Limit browser close timeout

* Clean up getData()

* lint

* Template function for autoconsetn script

* Protect agains screenshot timeouts

* Add more logs

* Lint fix

Author: muodov

README.md

@@ -32,7 +32,7 @@ Available options:
 - `--chromium-version <version_number>` - use custom version of Chromium (e.g. "843427") instead of using the default
 - `--selenium-hub <url>` - If provided, browsers will be requested from selenium hub instead of spawning local processes (e.g. `--selenium-hub http://my-selenium-hub-host:4444`).
 - `--config <path>` - path to a config file that allows to set all the above settings (and more). Note that CLI flags have a higher priority than settings passed via config. You can find a sample config file in `tests/cli/sampleConfig.json`.
-- `--autoconsent-action <action>` - automatic autoconsent action (requires the `cmps` collector). Possible values: optIn, optOut
+- `--autoconsent-action <action>` - automatic autoconsent action (requires the `cookiepopups` collector). Possible values: optIn, optOut
 
 ### Use it as a module
 
@@ -97,7 +97,7 @@ const data = await crawler(new URL('https://example.com'), {
 ## Output format
 
 Each successfully crawled website will create a separate file named after the website (when using the CLI tool). Output data format is specified in `crawler.js` (see `CollectResult` type definition).
-Additionally, for each crawl `metadata.json` file will be created containing crawl configuration, system configuration and some high-level stats. 
+Additionally, for each crawl `metadata.json` file will be created containing crawl configuration, system configuration and some high-level stats.
 
 ## Data post-processing
 

browser/BaseBrowser.js

@@ -6,6 +6,9 @@ class BaseBrowser {
         throw new Error('Not implemented');
     }
 
+    /**
+     * @returns {Promise<void>}
+     */
     close() {
         throw new Error('Not implemented');
     }

browser/RemoteChrome.js

@@ -98,11 +98,19 @@ class RemoteChrome extends BaseBrowser {
             try {
                 await this.connection.send('Browser.close');
             } catch (error) {
-                console.error('Error when closing browser connection', error);
+                console.error('Error when closing browser connection', error.message);
             }
-            this.connection.dispose();
+            try {
+                this.connection.dispose();
+            } catch (error) {
+                console.error('Error when disposing browser connection', error.message);
+            }
+        }
+        try {
+            await this.driver?.quit();
+        } catch (error) {
+            console.error('Error when quitting driver', error.message);
         }
-        await this.driver?.quit();
     }
 
     /**

cli/crawl-cli.js

@@ -27,7 +27,7 @@ program
     .option('-r, --region-code <region>', 'optional 2 letter region code. Used for metadata only.')
     .option('-a, --disable-anti-bot', 'disable anti bot detection protections injected to every frame')
     .option('--config <path>', 'crawl configuration file')
-    .option('--autoconsent-action <action>', 'dismiss cookie popups. Possible values: optOut, optIn. Works only when cmps collector is enabled.')
+    .option('--autoconsent-action <action>', 'dismiss cookie popups. Possible values: optOut, optIn. Works only when cookiepopups collector is enabled.')
     .option('--chromium-version <version_number>', 'use custom version of chromium')
     .option('--selenium-hub <url>', 'selenium hub endpoint to request browsers from')
     .parse(process.argv);
@@ -137,7 +137,7 @@ async function run({
      * @type {Array<Array<number>>}
      */
     let crawlTimes = [];
-    
+
     // eslint-disable-next-line arrow-parens
     const updateProgress = (/** @type {string} */site = '', /** @type {import('../crawler').CollectResult} */data) => {
         reporters.forEach(reporter => {
@@ -274,7 +274,7 @@ if (!config.urls || !config.output) {
                 dataCollectors: item.dataCollectors.map(id => createCollector(id))
             };
         }
-        
+
         return item;
     });
 

collectors/ContentScriptCollector.js

@@ -0,0 +1,110 @@
+const BaseCollector = require('./BaseCollector');
+
+const ISOLATED_WORLD_PREFIX = 'iw_for_';
+
+/**
+ * @param {String|Error} e
+ */
+function isIgnoredCDPError(e) {
+    // ignore evaluation errors (sometimes frames reload too fast)
+    const error = (typeof e === 'string') ? e : e.message;
+    return (
+        error.includes('TargetCloseError:') ||
+        error.includes('No frame for given id found') ||
+        error.includes('Target closed') ||
+        error.includes('Session closed') ||
+        error.includes('Cannot find context with specified id') ||
+        error.includes('uniqueContextId not found')
+    );
+}
+
+/**
+ * @abstract
+ * Base class for collectors that need to create isolated worlds for each frame.
+ */
+class ContentScriptCollector extends BaseCollector {
+    /**
+     * @param {import('./BaseCollector').CollectorInitOptions} options
+     */
+    init(options) {
+        this.log = options.log;
+        /**
+         * maps isolated world uniqueId to page world uniqueId
+         * @type {Map<import('devtools-protocol/types/protocol').Protocol.Runtime.ExecutionContextDescription['uniqueId'], import('devtools-protocol/types/protocol').Protocol.Runtime.ExecutionContextDescription['uniqueId']>}
+         */
+        this.isolated2pageworld = new Map();
+        /**
+         * maps isolated world uniqueId to CDPSession
+         * @type {Map<import('devtools-protocol/types/protocol').Protocol.Runtime.ExecutionContextDescription['uniqueId'], import('puppeteer-core').CDPSession>}
+         */
+        this.cdpSessions = new Map();
+        this.iwPrefix = `${ISOLATED_WORLD_PREFIX}${this.id()}_`;
+    }
+
+    /**
+     * @param {import('puppeteer-core').CDPSession} session
+     * @param {import('devtools-protocol/types/protocol').Protocol.Target.TargetInfo} targetInfo
+     */
+    addTarget(session, targetInfo) {
+        if (targetInfo.type !== 'page' && targetInfo.type !== 'iframe') {
+            return;
+        }
+
+        session.on('Runtime.executionContextDestroyed', ({executionContextUniqueId}) => {
+            this.log(`context destroyed ${executionContextUniqueId}`);
+            this.isolated2pageworld.delete(executionContextUniqueId);
+            this.cdpSessions.delete(executionContextUniqueId);
+        });
+
+        // inject the content script into every frame in isolated world
+        session.on('Runtime.executionContextCreated', async ({context}) => {
+            // new isolated world for our content script
+            if (context.auxData.type === 'isolated' && context.name.startsWith(this.iwPrefix)) {
+                const pageWorldUniqueId = context.name.slice(this.iwPrefix.length);
+                this.log(`isolated world created ${context.uniqueId} for ${pageWorldUniqueId}`);
+                this.isolated2pageworld.set(context.uniqueId, pageWorldUniqueId);
+                this.cdpSessions.set(context.uniqueId, session);
+                await this.onIsolatedWorldCreated(session, context);
+                return;
+            }
+
+            // ignore other special contexts
+            if (!context.origin || context.origin === '://' || context.auxData.type !== 'default') {
+                return;
+            }
+
+            this.log(`creating isolated world for ${context.uniqueId}`);
+            // request an isolated world for this frame
+            try {
+                await session.send('Page.createIsolatedWorld', {
+                    frameId: context.auxData.frameId,
+                    worldName: `${this.iwPrefix}${context.uniqueId}`,
+                });
+            } catch (e) {
+                if (!this.isIgnoredCdpError(e)) {
+                    this.log(`Error creating isolated world for ${context.uniqueId}: ${e}`);
+                }
+            }
+        });
+    }
+
+    /**
+     * @abstract
+     * @param {import('puppeteer-core').CDPSession} session
+     * @param {import('devtools-protocol/types/protocol').Protocol.Runtime.ExecutionContextDescription} context
+     * @returns {Promise<void>}
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    onIsolatedWorldCreated(session, context) {
+        throw new Error('Not implemented');
+    }
+
+    /**
+     * @param {Error|string} e
+     */
+    isIgnoredCdpError(e) {
+        return isIgnoredCDPError(e);
+    }
+}
+
+module.exports = ContentScriptCollector;

collectors/CookiePopupCollector.js

@@ -212,7 +212,7 @@ function getUniqueSelector(el) {
 }
 
 /**
- * @returns {import('../CookiePopupCollector').ContentScriptResult}
+ * @returns {import('../CookiePopupsCollector').ScrapeScriptResult}
  */
 function collectPotentialPopups() {
     const isFramed = window.top !== window || location.ancestorOrigins?.length > 0;