Merge branch 'v4' into copilot/add-abortsignal-option-basehttpclient

Author: barjin

docs/guides/http-clients.mdx

@@ -49,7 +49,7 @@ BaseHttpClient --|> GotScrapingHttpClient
 
 ## Switching between HTTP clients
 
-Crawlee currently provides two main HTTP clients: <ApiLink to="core/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink>, which uses the `got-scraping` library, and <ApiLink to="impit-client/class/ImpitHttpClient">`ImpitHttpClient`</ApiLink>, which uses the `impit` library. You can switch between them by setting the `BasehttpClient` parameter when initializing a crawler class. The default HTTP client is <ApiLink to="core/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink>. For more details on anti-blocking features, see our [avoid getting blocked guide](./avoid-blocking).
+Crawlee currently provides two main HTTP clients: <ApiLink to="got-scraping-client/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink>, which uses the `got-scraping` library, and <ApiLink to="impit-client/class/ImpitHttpClient">`ImpitHttpClient`</ApiLink>, which uses the `impit` library. You can switch between them by setting the `BasehttpClient` parameter when initializing a crawler class. The default HTTP client is <ApiLink to="got-scraping-client/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink>. For more details on anti-blocking features, see our [avoid getting blocked guide](./avoid-blocking).
 
 Below are examples of how to configure the HTTP client for the <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>:
 
@@ -68,7 +68,7 @@ Below are examples of how to configure the HTTP client for the <ApiLink to="chee
 
 ## Installation requirements
 
-Since <ApiLink to="core/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink> is the default HTTP client, it's included with the base Crawlee installation and requires no additional packages.
+Since <ApiLink to="got-scraping-client/class/GotScrapingHttpClient">`GotScrapingHttpClient`</ApiLink> is the default HTTP client, it's included with the base Crawlee installation and requires no additional packages.
 
 For <ApiLink to="impit-client/class/ImpitHttpClient">`ImpitHttpClient`</ApiLink>, you need to install a separate `@crawlee/impit-client` package:
 
@@ -78,7 +78,7 @@ npm i @crawlee/impit-client
 
 ## Creating custom HTTP clients
 
-Crawlee provides an interface, <ApiLink to="core/interface/BaseHttpClient">`BaseHttpClient`</ApiLink>, which defines the interface that all HTTP clients must implement. This allows you to create custom HTTP clients tailored to your specific requirements.
+Crawlee provides an interface, <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink>, which defines the interface that all HTTP clients must implement. This allows you to create custom HTTP clients tailored to your specific requirements.
 
 HTTP clients are responsible for several key operations:
 
@@ -88,10 +88,10 @@ HTTP clients are responsible for several key operations:
 - managing proxy configurations,
 - connection pooling with timeout management.
 
-To create a custom HTTP client, you need to implement the <ApiLink to="core/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> interface. Your implementation must be async-compatible and include proper cleanup and resource management to work seamlessly with Crawlee's concurrent processing model.
+To create a custom HTTP client, you need to implement the <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> interface. Your implementation must be async-compatible and include proper cleanup and resource management to work seamlessly with Crawlee's concurrent processing model.
 
 ## Conclusion
 
-This guide introduced you to the HTTP clients available in Crawlee and demonstrated how to switch between them, including their installation requirements and usage examples. You also learned about the responsibilities of HTTP clients and how to implement your own custom HTTP client by inheriting from the <ApiLink to="core/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> base class.
+This guide introduced you to the HTTP clients available in Crawlee and demonstrated how to switch between them, including their installation requirements and usage examples. You also learned about the responsibilities of HTTP clients and how to implement your own custom HTTP client by inheriting from the <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> base class.
 
 If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping!

docs/guides/http-clients/cheerio-got-scraping-example.ts

@@ -1,4 +1,5 @@
-import { CheerioCrawler, GotScrapingHttpClient } from 'crawlee';
+import { CheerioCrawler } from 'crawlee';
+import { GotScrapingHttpClient } from '@crawlee/got-scraping-client';
 
 const crawler = new CheerioCrawler({
     httpClient: new GotScrapingHttpClient(),

docs/guides/impit-http-client/basic-usage.ts

@@ -7,7 +7,7 @@ const crawler = new BasicCrawler({
     }),
     async requestHandler({ sendRequest, log }) {
         const response = await sendRequest();
-        log.info('Received response', { statusCode: response.statusCode });
+        log.info('Received response', { status: response.status });
     },
 });
 

package.json

@@ -102,6 +102,7 @@
         "globby": "^15.0.0",
         "got": "^14.4.7",
         "husky": "^9.1.7",
+        "iconv-lite": "^0.7.2",
         "is-ci": "^4.1.0",
         "lerna": "^9.0.0",
         "lint-staged": "^16.0.0",

packages/http-crawler/package.json

@@ -56,7 +56,7 @@
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
-        "iconv-lite": "^0.7.0",
+        "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",

packages/http-crawler/src/internals/http-crawler.ts

@@ -722,11 +722,9 @@ export class HttpCrawler<
         if (Buffer.isEncoding(encoding)) return { response, encoding };
 
         // Try to re-encode a variety of unsupported encodings to utf-8
-        if (iconv.default.encodingExists(encoding)) {
-            const encodeStream = iconv.default.encodeStream(utf8);
-            const decodeStream = iconv.default
-                .decodeStream(encoding)
-                .on('error', (err) => encodeStream.emit('error', err));
+        if (iconv.encodingExists(encoding)) {
+            const encodeStream = iconv.encodeStream(utf8);
+            const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
             const reencodedBody = response.body
                 ? Readable.toWeb(
                       Readable.from(

packages/stagehand-crawler/src/internals/stagehand-plugin.ts

@@ -1,12 +1,13 @@
-import log from '@apify/log';
 import type { Stagehand, V3Options } from '@browserbasehq/stagehand';
 import type { BrowserController, BrowserPluginOptions, LaunchContext } from '@crawlee/browser-pool';
 import { anonymizeProxySugar, BrowserPlugin } from '@crawlee/browser-pool';
-import type { BrowserType, LaunchOptions, Browser as PlaywrightBrowser } from 'playwright';
+import type { Browser as PlaywrightBrowser, BrowserType, LaunchOptions } from 'playwright';
 // Stagehand is built on CDP (Chrome DevTools Protocol), which only works with Chromium-based browsers.
 // Firefox and WebKit are not supported by Stagehand.
 import { chromium } from 'playwright';
 
+import log from '@apify/log';
+
 import { StagehandController } from './stagehand-controller';
 import type { StagehandOptions } from './stagehand-crawler';
 

test/core/crawlers/playwright_crawler.test.ts

@@ -4,6 +4,7 @@ import os from 'node:os';
 
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions, Request } from '@crawlee/playwright';
 import { PlaywrightCrawler, RequestList } from '@crawlee/playwright';
+import type { Cheerio, CheerioAPI, CheerioRoot, Element } from '@crawlee/utils';
 import express from 'express';
 import playwright from 'playwright';
 import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';

test/core/crawlers/rendering_type_predictor.test.ts

@@ -2,7 +2,7 @@ import { Request } from '@crawlee/core';
 import { RenderingTypePredictor } from '@crawlee/playwright';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';
+import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
 
 describe('RenderingTypePredictor', () => {
     const localStorageEmulator = new MemoryStorageEmulator();

test/stagehand-crawler/stagehand-controller.test.ts

@@ -1,7 +1,7 @@
 import log from '@apify/log';
 
-import { StagehandController } from '../../packages/stagehand-crawler/src/internals/stagehand-controller';
-import type { StagehandPlugin } from '../../packages/stagehand-crawler/src/internals/stagehand-plugin';
+import { StagehandController } from '../../packages/stagehand-crawler/src/internals/stagehand-controller.js';
+import type { StagehandPlugin } from '../../packages/stagehand-crawler/src/internals/stagehand-plugin.js';
 
 describe('StagehandController', () => {
     let mockPlugin: StagehandPlugin;