ensure urls from autoclick also treated as links!

update to warcio 2.4.8
bump to 1.11.2

Author: ikreymer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-crawler",
-  "version": "1.11.1",
+  "version": "1.11.2",
   "main": "browsertrix-crawler",
   "type": "module",
   "repository": "https://github.com/webrecorder/browsertrix-crawler",
@@ -42,7 +42,7 @@
     "tsc": "^2.0.4",
     "undici": "^7.16.0",
     "uuid": "8.3.2",
-    "warcio": "^2.4.7",
+    "warcio": "git+https://github.com/webrecorder/warcio.js#multi-value-allow-always",
     "ws": "^7.4.4",
     "yargs": "^17.7.2"
   },

src/crawler.ts

@@ -973,8 +973,14 @@ self.__bx_behaviors.selectMainBehavior();
       }
     }
 
-    await page.exposeFunction(BxFunctionBindings.AddToSeenSet, (data: string) =>
-      this.crawlState.addToUserSet(data),
+    await page.exposeFunction(
+      BxFunctionBindings.AddToSeenSet,
+      (data: string) => {
+        if (data && (data.startsWith("https:") || data.startsWith("http:"))) {
+          void callbacks.addLink(data);
+        }
+        return this.crawlState.addToUserSet(data);
+      },
     );
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any

src/util/recorder.ts

@@ -1776,7 +1776,7 @@ class AsyncFetcher {
     } catch (e) {
       logger.warn(
         "Async load headers failed",
-        { ...formatErr(e), ...this.recorder.logDetails },
+        { ...formatErr(e), url: this.reqresp.url, ...this.recorder.logDetails },
         "fetch",
       );
     }

yarn.lock

@@ -7167,6 +7167,19 @@ warcio@^2.4.7:
     uuid-random "^1.3.2"
     yargs "^17.7.2"
 
+"warcio@git+https://github.com/webrecorder/warcio.js#multi-value-allow-always":
+  version "2.4.8"
+  resolved "git+https://github.com/webrecorder/warcio.js#07f8e3805766083304f3c02735b735d19c6320cc"
+  dependencies:
+    "@types/pako" "^1.0.7"
+    "@types/stream-buffers" "^3.0.7"
+    base32-encode "^2.0.0"
+    hash-wasm "^4.9.0"
+    pako "^1.0.11"
+    tempy "^3.1.0"
+    uuid-random "^1.3.2"
+    yargs "^17.7.2"
+
 [email protected]:
   version "0.3.10"
   resolved "https://registry.yarnpkg.com/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.10.tgz#437405564ff7e200371468f4f1eba1ff5537e119"