fix: Fix skipNavigation behavior in HttpCrawler subclasses (#3381)

janbuchar · web-flow · commit 340252af4462 · 2026-03-26T13:10:20.000+01:00
closes #3304
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -44,6 +44,7 @@ import {
     KeyValueStore,
     LogLevel,
     mergeCookies,
+    NavigationSkippedError,
     NonRetryableError,
     purgeDefaultStorages,
     RequestHandlerError,
@@ -914,13 +915,13 @@ export class BasicCrawler<
                     try {
                         await this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request));
                     } catch (error) {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
                         // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
                         if (error instanceof ContextPipelineInterruptedError) {
                             await this._timeoutAndRetry(
-                                async () => this.requestManager?.markRequestHandled(crawlingContext.request!),
+                                async () => this.requestManager?.markRequestHandled(request),
                                 this.internalTimeoutMillis,
                                 `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${
                                     this.internalTimeoutMillis / 1e3
@@ -939,6 +940,7 @@ export class BasicCrawler<
                             await this._requestFunctionErrorHandler(
                                 unwrappedError,
                                 crawlingContext as CrawlingContext,
+                                request,
                                 this.requestManager!,
                             );
                             crawlingContext.session?.markBad();
@@ -1838,9 +1840,7 @@ export class BasicCrawler<
     }
 
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
-    protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager) {
-        const { request } = crawlingContext;
-
+    protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {
         const statisticsId = request.id || request.uniqueKey;
         this.stats.startJob(statisticsId);
 
@@ -1871,7 +1871,7 @@ export class BasicCrawler<
             try {
                 request.state = RequestState.ERROR_HANDLER;
                 await addTimeoutToPromise(
-                    async () => this._requestFunctionErrorHandler(err, crawlingContext, requestSource),
+                    async () => this._requestFunctionErrorHandler(err, crawlingContext, request, requestSource),
                     this.internalTimeoutMillis,
                     `Handling request failure of ${request.url} (${request.id}) timed out after ${
                         this.internalTimeoutMillis / 1e3
@@ -2052,13 +2052,15 @@ export class BasicCrawler<
 
     /**
      * Handles errors thrown by user provided requestHandler()
+     *
+     * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
     protected async _requestFunctionErrorHandler(
         error: Error,
         crawlingContext: CrawlingContext,
+        request: Request,
         source: IRequestList | IRequestManager,
     ): Promise<void> {
-        const { request } = crawlingContext;
         request.pushErrorMessage(error);
 
         if (error instanceof CriticalError) {
@@ -2256,6 +2258,18 @@ export class BasicCrawler<
     }
 
     private requestMatchesEnqueueStrategy(request: Request) {
+        // If `skipNavigation` was used, just return `true`
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+            request.loadedUrl;
+        } catch (err) {
+            if (err instanceof NavigationSkippedError) {
+                return true;
+            }
+
+            throw err;
+        }
+
         const { url, loadedUrl } = request;
 
         // eslint-disable-next-line dot-notation -- private access
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -20,6 +20,7 @@ import {
     enqueueLinks,
     EVENT_SESSION_RETIRED,
     handleRequestTimeout,
+    NavigationSkippedError,
     RequestState,
     resolveBaseUrlForEnqueueLinksFiltering,
     SessionError,
@@ -508,15 +509,17 @@ export abstract class BrowserCrawler<
                 request: new Proxy(crawlingContext.request, {
                     get(target, propertyName, receiver) {
                         if (propertyName === 'loadedUrl') {
-                            throw new Error(
+                            throw new NavigationSkippedError(
                                 'The `request.loadedUrl` property is not available - `skipNavigation` was used',
                             );
                         }
                         return Reflect.get(target, propertyName, receiver);
                     },
                 }) as LoadedRequest<Request>,
                 get response(): Response {
-                    throw new Error('The `response` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `response` property is not available - `skipNavigation` was used',
+                    );
                 },
             };
         }
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -11,7 +11,13 @@ import type {
     RouterRoutes,
     SkippedRequestCallback,
 } from '@crawlee/http';
-import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router } from '@crawlee/http';
+import {
+    enqueueLinks,
+    HttpCrawler,
+    NavigationSkippedError,
+    resolveBaseUrlForEnqueueLinksFiltering,
+    Router,
+} from '@crawlee/http';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type CheerioRoot, extractUrlsFromCheerio, type RobotsTxtFile } from '@crawlee/utils';
 import type { CheerioAPI, CheerioOptions } from 'cheerio';
@@ -194,19 +200,40 @@ export class CheerioCrawler<
     }
 
     private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
-        const isXml = crawlingContext.contentType.type.includes('xml');
-        const body = Buffer.isBuffer(crawlingContext.body)
-            ? crawlingContext.body.toString(crawlingContext.contentType.encoding)
-            : crawlingContext.body;
-        const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
-        const $ = cheerio.load(dom, {
-            xml: { decodeEntities: true, xmlMode: isXml },
-        } as CheerioOptions);
+        try {
+            const isXml = crawlingContext.contentType.type.includes('xml');
+            const body = Buffer.isBuffer(crawlingContext.body)
+                ? crawlingContext.body.toString(crawlingContext.contentType.encoding)
+                : crawlingContext.body;
+            const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
+            const $ = cheerio.load(dom, {
+                xml: { decodeEntities: true, xmlMode: isXml },
+            } as CheerioOptions);
 
-        return {
-            $,
-            body,
-        };
+            return {
+                $,
+                body,
+            };
+        } catch (err) {
+            if (err instanceof NavigationSkippedError) {
+                return {
+                    get body(): string {
+                        throw new NavigationSkippedError(
+                            'The `body` property is not available - `skipNavigation` was used',
+                            { cause: err },
+                        );
+                    },
+                    get $(): CheerioAPI {
+                        throw new NavigationSkippedError(
+                            'The `$` property is not available - `skipNavigation` was used',
+                            { cause: err },
+                        );
+                    },
+                };
+            }
+
+            throw err;
+        }
     }
 
     private async addHelpers(crawlingContext: InternalHttpCrawlingContext & { $: CheerioAPI }) {
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
@@ -78,3 +78,9 @@ export class ServiceConflictError extends Error {
         );
     }
 }
+
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export class NavigationSkippedError extends NonRetryableError {}
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
@@ -275,12 +275,24 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
         });
     }
 
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
+     */
     get skipNavigation(): boolean {
         return this.userData.__crawlee?.skipNavigation ?? false;
     }
 
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value: boolean) {
         if (!this.userData.__crawlee) {
             (this.userData as Dictionary).__crawlee = { skipNavigation: value };
@@ -548,6 +560,10 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
@@ -13,7 +13,15 @@ import type {
     RouterRoutes,
     Session,
 } from '@crawlee/basic';
-import { BasicCrawler, ContextPipeline, mergeCookies, RequestState, Router, SessionError } from '@crawlee/basic';
+import {
+    BasicCrawler,
+    ContextPipeline,
+    mergeCookies,
+    NavigationSkippedError,
+    RequestState,
+    Router,
+    SessionError,
+} from '@crawlee/basic';
 import type { LoadedRequest } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import type { Awaitable, Dictionary } from '@crawlee/types';
@@ -392,15 +400,17 @@ export class HttpCrawler<
                 request: new Proxy(request, {
                     get(target, propertyName, receiver) {
                         if (propertyName === 'loadedUrl') {
-                            throw new Error(
+                            throw new NavigationSkippedError(
                                 'The `request.loadedUrl` property is not available - `skipNavigation` was used',
                             );
                         }
                         return Reflect.get(target, propertyName, receiver);
                     },
                 }) as LoadedRequest<CrawleeRequest>,
                 get response(): InternalHttpCrawlingContext['response'] {
-                    throw new Error('The `response` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `response` property is not available - `skipNavigation` was used',
+                    );
                 },
             };
         }
@@ -440,19 +450,29 @@ export class HttpCrawler<
         if (crawlingContext.request.skipNavigation) {
             return {
                 get contentType(): InternalHttpCrawlingContext['contentType'] {
-                    throw new Error('The `contentType` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `contentType` property is not available - `skipNavigation` was used',
+                    );
                 },
                 get body(): InternalHttpCrawlingContext['body'] {
-                    throw new Error('The `body` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `body` property is not available - `skipNavigation` was used',
+                    );
                 },
                 get json(): InternalHttpCrawlingContext['json'] {
-                    throw new Error('The `json` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `json` property is not available - `skipNavigation` was used',
+                    );
                 },
                 get waitForSelector(): InternalHttpCrawlingContext['waitForSelector'] {
-                    throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `waitForSelector` method is not available - `skipNavigation` was used',
+                    );
                 },
                 get parseWithCheerio(): InternalHttpCrawlingContext['parseWithCheerio'] {
-                    throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError(
+                        'The `parseWithCheerio` method is not available - `skipNavigation` was used',
+                    );
                 },
             };
         }
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts

Original file line number	Diff line number	Diff line change
`@@ -78,3 +78,9 @@ export class ServiceConflictError extends Error {`
`78`	`78`	`);`
`79`	`79`	`}`
`80`	`80`	`}`
	`81`	`+`
	`82`	`+/**`
	`83`	+ * Thrown by crawlers when `skipNavigation` is used on a request.
	`84`	`+ * Subclasses can catch this error to skip their own navigation-dependent logic.`
	`85`	`+ */`
	`86`	`+export class NavigationSkippedError extends NonRetryableError {}`