diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index a9eeb1461fda..4f60768fede0 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -536,6 +536,7 @@ export class BasicCrawler; @@ -562,8 +563,8 @@ export class BasicCrawler; private _experimentWarnings: Partial> = {}; @@ -810,12 +811,23 @@ export class BasicCrawler { if (isMaxPagesExceeded()) { - if (this.shouldLogMaxProcessedRequestsExceeded) { + if (this.shouldLogShuttingDown) { log.info( 'Crawler reached the maxRequestsPerCrawl limit of ' + `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`, ); - this.shouldLogMaxProcessedRequestsExceeded = false; + this.shouldLogShuttingDown = false; + } + return false; + } + + if (this.unexpectedStop) { + if (this.shouldLogShuttingDown) { + this.log.info( + 'No new requests are allowed because the `stop()` method has been called. ' + + 'Ongoing requests will be allowed to complete.', + ); + this.shouldLogShuttingDown = false; } return false; } @@ -829,6 +841,15 @@ export class BasicCrawler this.autoscaledPool?.abort()) - .then(() => this.log.info(message)) - .catch((err) => { - this.log.error('An error occurred when stopping the crawler:', err); - }); + stop(reason = 'The crawler has been gracefully stopped.'): void { + if (this.unexpectedStop) { + return; + } + this.log.info(reason); + this.unexpectedStop = true; } async getRequestQueue(): Promise {