chore: remove direct dependency on idcac-playwright (#3259)

Due to licensing issues, Crawlee now imports `idcac-playwright`
dynamically. If the dependency is missing, using the `closeCookieModals`
util method will print an error message and do nothing.

Author: barjin

packages/crawlee/package.json

@@ -70,10 +70,14 @@
         "tslib": "^2.4.0"
     },
     "peerDependencies": {
+        "idcac-playwright": "*",
         "playwright": "*",
         "puppeteer": "*"
     },
     "peerDependenciesMeta": {
+        "idcac-playwright": {
+            "optional": true
+        },
         "playwright": {
             "optional": true
         },

packages/playwright-crawler/package.json

@@ -62,7 +62,6 @@
         "@crawlee/types": "3.15.3",
         "@crawlee/utils": "3.15.3",
         "cheerio": "1.0.0-rc.12",
-        "idcac-playwright": "^0.1.2",
         "jquery": "^3.6.0",
         "lodash.isequal": "^4.5.0",
         "ml-logistic-regression": "^2.0.0",
@@ -72,9 +71,13 @@
         "tslib": "^2.4.0"
     },
     "peerDependencies": {
+        "idcac-playwright": "^0.1.2",
         "playwright": "*"
     },
     "peerDependenciesMeta": {
+        "idcac-playwright": {
+            "optional": true
+        },
         "playwright": {
             "optional": true
         }

packages/playwright-crawler/src/internals/utils/playwright-utils.ts

@@ -33,7 +33,6 @@ import {
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import { type CheerioRoot, type Dictionary, expandShadowRoots, sleep } from '@crawlee/utils';
 import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
 import ow from 'ow';
 import type { Page, Response, Route } from 'playwright';
 
@@ -656,10 +655,35 @@ export async function parseWithCheerio(
     return cheerio.load(pageContent);
 }
 
+let idcacPlaywright: null | { getInjectableScript: () => string } = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright) return idcacPlaywright;
+
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    } catch (error: any) {
+        log.warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
+}
+
 export async function closeCookieModals(page: Page): Promise<void> {
     ow(page, ow.object.validate(validators.browserPage));
+    const idcac = await getIdcacPlaywright();
 
-    await page.evaluate(getCookieClosingScript());
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 
 interface HandleCloudflareChallengeOptions {
@@ -993,6 +1017,15 @@ export interface PlaywrightContextUtils {
 
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
 

packages/puppeteer-crawler/package.json

@@ -67,9 +67,13 @@
         "tslib": "^2.4.0"
     },
     "peerDependencies": {
+        "idcac-playwright": "^0.1.2",
         "puppeteer": "*"
     },
     "peerDependenciesMeta": {
+        "idcac-playwright": {
+            "optional": true
+        },
         "puppeteer": {
             "optional": true
         }

packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts

@@ -27,7 +27,6 @@ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type CheerioRoot, expandShadowRoots, sleep } from '@crawlee/utils';
 import * as cheerio from 'cheerio';
 import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
-import { getInjectableScript } from 'idcac-playwright';
 import ow from 'ow';
 import type { HTTPRequest as PuppeteerRequest, HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
 
@@ -778,8 +777,35 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
     }
 }
 
+let idcacPlaywright: null | { getInjectableScript: () => string } = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright) return idcacPlaywright;
+
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    } catch (error: any) {
+        log.warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
+}
+
 export async function closeCookieModals(page: Page): Promise<void> {
-    await page.evaluate(getInjectableScript());
+    ow(page, ow.object.validate(validators.browserPage));
+    const idcac = await getIdcacPlaywright();
+
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 
 /** @internal */
@@ -1057,6 +1083,15 @@ export interface PuppeteerContextUtils {
 
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
 }

yarn.lock

@@ -686,7 +686,6 @@ __metadata:
     "@crawlee/types": "npm:3.15.3"
     "@crawlee/utils": "npm:3.15.3"
     cheerio: "npm:1.0.0-rc.12"
-    idcac-playwright: "npm:^0.1.2"
     jquery: "npm:^3.6.0"
     lodash.isequal: "npm:^4.5.0"
     ml-logistic-regression: "npm:^2.0.0"
@@ -695,8 +694,11 @@ __metadata:
     string-comparison: "npm:^1.3.0"
     tslib: "npm:^2.4.0"
   peerDependencies:
+    idcac-playwright: ^0.1.2
     playwright: "*"
   peerDependenciesMeta:
+    idcac-playwright:
+      optional: true
     playwright:
       optional: true
   languageName: unknown
@@ -719,8 +721,11 @@ __metadata:
     ow: "npm:^0.28.1"
     tslib: "npm:^2.4.0"
   peerDependencies:
+    idcac-playwright: ^0.1.2
     puppeteer: "*"
   peerDependenciesMeta:
+    idcac-playwright:
+      optional: true
     puppeteer:
       optional: true
   languageName: unknown
@@ -4934,9 +4939,12 @@ __metadata:
     import-local: "npm:^3.1.0"
     tslib: "npm:^2.4.0"
   peerDependencies:
+    idcac-playwright: "*"
     playwright: "*"
     puppeteer: "*"
   peerDependenciesMeta:
+    idcac-playwright:
+      optional: true
     playwright:
       optional: true
     puppeteer: