chore: Fix HttpCrawler context types (#3291)

Author: janbuchar

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -119,7 +119,7 @@ export type RequireContextPipeline<
 
 export interface BasicCrawlerOptions<
     Context extends CrawlingContext = CrawlingContext,
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
 > {
     /**
@@ -471,7 +471,7 @@ export interface CrawlerExperiments {
  */
 export class BasicCrawler<
     Context extends CrawlingContext = CrawlingContext,
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
 > {
     protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -102,7 +102,7 @@ export interface BrowserCrawlerOptions<
         ProvidedController,
         Dictionary
     >,
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
     InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
     __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,
@@ -297,7 +297,7 @@ export abstract class BrowserCrawler<
         ProvidedController,
         Dictionary
     >,
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
     GoToOptions extends Dictionary = Dictionary,
 > extends BasicCrawler<Context, ContextExtension, ExtendedContext> {

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -25,10 +25,11 @@ export type CheerioErrorHandler<
 > = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
 
 export interface CheerioCrawlerOptions<
-    ExtendedContext extends CheerioCrawlingContext,
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ExtendedContext> {}
+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
 
 export type CheerioHook<
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -169,12 +170,13 @@ export type CheerioRequestHandler<
  * @category Crawlers
  */
 export class CheerioCrawler<
-    ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext,
-> extends HttpCrawler<CheerioCrawlingContext, ExtendedContext> {
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
+> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options?: CheerioCrawlerOptions<ExtendedContext>, config?: Configuration) {
+    constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>, config?: Configuration) {
         super(
             {
                 ...options,

packages/http-crawler/src/internals/http-crawler.ts

@@ -75,8 +75,9 @@ export type HttpErrorHandler<
 
 export interface HttpCrawlerOptions<
     Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,
-    ExtendedContext extends Context = Context,
-> extends BasicCrawlerOptions<Context, ExtendedContext> {
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends Context = Context & ContextExtension,
+> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
     /**
      * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
      */
@@ -325,9 +326,9 @@ export type HttpRequestHandler<
  */
 export class HttpCrawler<
     Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawler<Context, ExtendedContext> {
+> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
     protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
     protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
     protected persistCookiesPerSession: boolean;
@@ -360,7 +361,7 @@ export class HttpCrawler<
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(
-        options: HttpCrawlerOptions<Context, ExtendedContext> &
+        options: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> &
             RequireContextPipeline<InternalHttpCrawlingContext, Context> = {} as any,
         override readonly config = Configuration.getGlobalConfig(),
     ) {

packages/jsdom-crawler/src/internals/jsdom-crawler.ts

@@ -34,10 +34,11 @@ export type JSDOMErrorHandler<
 > = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
 
 export interface JSDOMCrawlerOptions<
-    ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext,
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {
+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
     /**
      * Download and run scripts.
      */
@@ -177,10 +178,10 @@ const resources = new ResourceLoader({
         'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
 });
 
-export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<
-    JSDOMCrawlingContext,
-    ExtendedContext
-> {
+export class JSDOMCrawler<
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
+> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
     protected static override optionsShape = {
         ...HttpCrawler.optionsShape,
         runScripts: ow.optional.boolean,
@@ -191,7 +192,7 @@ export class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCr
     protected hideInternalConsole: boolean;
     protected virtualConsole: VirtualConsole | null = null;
 
-    constructor(options: JSDOMCrawlerOptions<ExtendedContext> = {}, config?: Configuration) {
+    constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}, config?: Configuration) {
         const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
 
         super(

packages/linkedom-crawler/src/internals/linkedom-crawler.ts

@@ -29,10 +29,11 @@ export type LinkeDOMErrorHandler<
 > = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
 
 export interface LinkeDOMCrawlerOptions<
-    ExtendedContext extends LinkeDOMCrawlingContext,
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {}
+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
 
 export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
 
@@ -160,11 +161,12 @@ export type LinkeDOMRequestHandler<
  */
 
 export class LinkeDOMCrawler<
-    ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,
-> extends HttpCrawler<LinkeDOMCrawlingContext, ExtendedContext> {
+    ContextExtension = Dictionary<never>,
+    ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
+> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
     private static parser = new DOMParser();
 
-    constructor(options: LinkeDOMCrawlerOptions<ExtendedContext>) {
+    constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {
         super({
             ...options,
             contextPipelineBuilder: () =>

packages/playwright-crawler/src/internals/playwright-crawler.ts

@@ -33,7 +33,7 @@ export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, P
 export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
 
 export interface PlaywrightCrawlerOptions<
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
 > extends BrowserCrawlerOptions<
         Page,
@@ -176,7 +176,7 @@ export interface PlaywrightCrawlerOptions<
  * @category Crawlers
  */
 export class PlaywrightCrawler<
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
 > extends BrowserCrawler<
     Page,

packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts

@@ -32,7 +32,7 @@ export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, Pup
 export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
 
 export interface PuppeteerCrawlerOptions<
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
 > extends BrowserCrawlerOptions<
         Page,
@@ -149,7 +149,7 @@ export interface PuppeteerCrawlerOptions<
  * @category Crawlers
  */
 export class PuppeteerCrawler<
-    ContextExtension = {},
+    ContextExtension = Dictionary<never>,
     ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
 > extends BrowserCrawler<
     Page,