feat: add navigationTimeoutSecs to input

metalwarrior665 · metalwarrior665 · commit 287d37e4fae7 · 2022-11-15T09:56:06.000+01:00
diff --git a/.DS_Store b/.DS_Store
diff --git a/checker-cheerio/src/main.ts b/checker-cheerio/src/main.ts
@@ -29,6 +29,7 @@ Actor.main(async () => {
         proxyConfiguration,
         urlsToCheck,
         repeatChecksOnProvidedUrls,
+        navigationTimeoutSecs,
     } = input;
 
     const proxy = await Actor.createProxyConfiguration({
@@ -67,6 +68,7 @@ Actor.main(async () => {
 
     const crawler = new CheerioCrawler({
         maxRequestRetries: 0,
+        navigationTimeoutSecs,
         maxRequestsPerCrawl: maxNumberOfPagesCheckedPerDomain,
         maxConcurrency: maxConcurrentPagesCheckedPerDomain,
         requestQueue,
diff --git a/checker-cheerio/src/typedefs.ts b/checker-cheerio/src/typedefs.ts
@@ -59,6 +59,7 @@ export interface ActorInputData {
     maxConcurrentPagesCheckedPerDomain: number;
     maxConcurrentDomainsChecked: number;
     retireBrowserInstanceAfterRequestCount: number;
+    navigationTimeoutSecs: number;
 
     // Pass only to puppeteer
     'puppeteer.headfull'?: boolean;
diff --git a/starter/.DS_Store b/starter/.DS_Store
diff --git a/starter/INPUT_SCHEMA.json b/starter/INPUT_SCHEMA.json
@@ -116,6 +116,13 @@
       "editor": "number",
       "minimum": 1
     },
+    "navigationTimeoutSecs": {
+        "title": "Navigation timeout (seconds)",
+        "type": "integer",
+        "description": "Specifies the maximum time in seconds the request will wait for the page to load. If the page is not loaded within this time, the browser will throw an error and the page will be marked as failed.",
+        "default": 60,
+        "minimum": 1
+    },
     "puppeteer.headfull": {
       "title": "Headfull browser (XVFB)",
       "type": "boolean",
diff --git a/starter/src/configs.ts b/starter/src/configs.ts
@@ -64,6 +64,7 @@ function* createActorRunConfigForCrawler({ input, urlData, checkerId, playwright
                 maxConcurrentPagesCheckedPerDomain: input.maxConcurrentPagesCheckedPerDomain,
                 maxConcurrentDomainsChecked: input.maxConcurrentDomainsChecked,
                 retireBrowserInstanceAfterRequestCount: input.retireBrowserInstanceAfterRequestCount,
+                navigationTimeoutSecs: input.navigationTimeoutSecs,
             },
             params: {
                 memory: checkerId === ACTOR_CHEERIO_CHECKER_NAME ? 4096 : 8192,
diff --git a/starter/src/typedefs.ts b/starter/src/typedefs.ts
@@ -47,6 +47,7 @@ export interface ActorInputData {
     maxConcurrentPagesCheckedPerDomain: number;
     maxConcurrentDomainsChecked: number;
     retireBrowserInstanceAfterRequestCount: number;
+    navigationTimeoutSecs: number;
 
     // Pass only to puppeteer
     'puppeteer.headfull'?: boolean;
