feat: Report links skipped because of various filter conditions (#3026)

- closes #3016
- [x] consider also reporting links skipped due to after-redirect checks

---------

Co-authored-by: Jindřich Bär <jindrichbar@gmail.com>

Author: janbuchar

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -354,7 +354,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
 
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired only for requests skipped based on robots.txt file.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
+     * 4. or because the {@apilink BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
 

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -573,6 +573,8 @@ export abstract class BrowserCrawler<
             request.noRetry = true;
             request.state = RequestState.SKIPPED;
 
+            await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
+
             return;
         }
 

packages/core/src/enqueue_links/enqueue_links.ts

@@ -1,4 +1,4 @@
-import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 import { type RobotsTxtFile } from '@crawlee/utils';
 import ow from 'ow';
 import { getDomain } from 'tldts';
@@ -13,7 +13,15 @@ import type {
     RequestProvider,
     RequestQueueOperationOptions,
 } from '../storages';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, UrlPatternObject } from './shared';
+import type {
+    GlobInput,
+    PseudoUrlInput,
+    RegExpInput,
+    RequestTransform,
+    SkippedRequestCallback,
+    SkippedRequestReason,
+    UrlPatternObject,
+} from './shared';
 import {
     constructGlobObjectsFromGlobs,
     constructRegExpObjectsFromPseudoUrls,
@@ -23,8 +31,6 @@ import {
     filterRequestsByPatterns,
 } from './shared';
 
-export type SkippedRequestCallback = (args: { url: string; reason: 'robotsTxt' | 'limit' }) => Awaitable<void>;
-
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
@@ -175,7 +181,10 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
 
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired only for requests skipped based on robots.txt file.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. or because the maxRequestsPerCrawl limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
@@ -392,6 +401,16 @@ export async function enqueueLinks(
         }
     }
 
+    async function reportSkippedRequests(skippedRequests: { url: string }[], reason: SkippedRequestReason) {
+        if (onSkippedRequest && skippedRequests.length > 0) {
+            await Promise.all(
+                skippedRequests.map((request) => {
+                    return onSkippedRequest({ url: request.url, reason });
+                }),
+            );
+        }
+    }
+
     let requestOptions = createRequestOptions(urls, options);
 
     if (robotsTxtFile) {
@@ -406,25 +425,37 @@ export async function enqueueLinks(
             return false;
         });
 
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(
-                skippedRequests.map((request) => {
-                    return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
-                }),
-            );
-        }
+        await reportSkippedRequests(skippedRequests, 'robotsTxt');
     }
 
     if (transformRequestFunction) {
+        const skippedRequests: RequestOptions[] = [];
+
         requestOptions = requestOptions
-            .map((request) => transformRequestFunction(request))
-            .filter((r) => !!r) as RequestOptions[];
+            .map((request) => {
+                const transformedRequest = transformRequestFunction(request);
+                if (!transformedRequest) {
+                    skippedRequests.push(request);
+                }
+                return transformedRequest;
+            })
+            .filter((r) => Boolean(r)) as RequestOptions[];
+
+        await reportSkippedRequests(skippedRequests, 'filters');
     }
 
-    function createFilteredRequests() {
+    async function createFilteredRequests() {
+        const skippedRequests: string[] = [];
+
         // No user provided patterns means we can skip an extra filtering step
         if (urlPatternObjects.length === 0) {
-            return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);
+            return createRequests(
+                requestOptions,
+                enqueueStrategyPatterns,
+                urlExcludePatternObjects,
+                options.strategy,
+                (url) => skippedRequests.push(url),
+            );
         }
 
         // Generate requests based on the user patterns first
@@ -433,19 +464,24 @@ export async function enqueueLinks(
             urlPatternObjects,
             urlExcludePatternObjects,
             options.strategy,
+            (url) => skippedRequests.push(url),
         );
         // ...then filter them by the enqueue links strategy (making this an AND check)
-        return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
+        const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) =>
+            skippedRequests.push(url),
+        );
+
+        await reportSkippedRequests(
+            skippedRequests.map((url) => ({ url })),
+            'filters',
+        );
+
+        return filtered;
     }
 
-    let requests = createFilteredRequests();
+    let requests = await createFilteredRequests();
     if (limit && limit < requests.length) {
-        if (onSkippedRequest) {
-            for (const request of requests.slice(limit)) {
-                await onSkippedRequest({ url: request.url, reason: 'limit' });
-            }
-        }
-
+        await reportSkippedRequests(requests.slice(limit), 'limit');
         requests = requests.slice(0, limit);
     }
 

packages/core/src/enqueue_links/shared.ts

@@ -1,5 +1,6 @@
 import { URL } from 'node:url';
 
+import type { Awaitable } from '@crawlee/types';
 import { minimatch } from 'minimatch';
 
 import { purlToRegExp } from '@apify/pseudo_url';
@@ -46,6 +47,13 @@ export type RegExpObject = { regexp: RegExp } & Pick<
 
 export type RegExpInput = RegExp | RegExpObject;
 
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect';
+
+export type SkippedRequestCallback = (args: {
+    url: string;
+    reason: SkippedRequestReason;
+}) => Awaitable<void>;
+
 /**
  * @ignore
  */
@@ -166,14 +174,21 @@ export function createRequests(
     urlPatternObjects?: UrlPatternObject[],
     excludePatternObjects: UrlPatternObject[] = [],
     strategy?: EnqueueLinksOptions['strategy'],
+    onSkippedUrl?: (url: string) => void,
 ): Request[] {
     return requestOptions
         .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
         .filter(({ url }) => {
-            return !excludePatternObjects.some((excludePatternObject) => {
+            const matchesExcludePatterns = excludePatternObjects.some((excludePatternObject) => {
                 const { regexp, glob } = excludePatternObject;
                 return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
             });
+
+            if (matchesExcludePatterns) {
+                onSkippedUrl?.(url);
+            }
+
+            return !matchesExcludePatterns;
         })
         .map(({ url, opts }) => {
             if (!urlPatternObjects || !urlPatternObjects.length) {
@@ -193,27 +208,33 @@ export function createRequests(
             }
 
             // didn't match any positive pattern
+            onSkippedUrl?.(url);
             return null;
         })
         .filter((request) => request) as Request[];
 }
 
-export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[] {
+export function filterRequestsByPatterns(
+    requests: Request[],
+    patterns?: UrlPatternObject[],
+    onSkippedUrl?: (url: string) => void,
+): Request[] {
     if (!patterns?.length) {
         return requests;
     }
 
     const filtered: Request[] = [];
 
     for (const request of requests) {
-        for (const urlPatternObject of patterns) {
-            const { regexp, glob } = urlPatternObject;
+        const matchingPattern = patterns.find(
+            ({ regexp, glob }) =>
+                (regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true })),
+        );
 
-            if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
-                filtered.push(request);
-                // Break the pattern loop, as we already matched this request once
-                break;
-            }
+        if (matchingPattern !== undefined) {
+            filtered.push(request);
+        } else {
+            onSkippedUrl?.(request.url);
         }
     }
 

packages/http-crawler/src/internals/http-crawler.ts

@@ -557,6 +557,8 @@ export class HttpCrawler<
                 request.noRetry = true;
                 request.state = RequestState.SKIPPED;
 
+                await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
+
                 return;
             }
 

packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts

@@ -600,6 +600,8 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
                                 request.noRetry = true;
                                 request.state = RequestState.SKIPPED;
 
+                                await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
+
                                 return;
                             }