rec pending:

- track how many in-flight pending requests there are
- add api to get num pending requests
deps: bump to wabac.js 2.14.1
bump to 0.2.0

Author: ikreymer

package.json

@@ -7,7 +7,7 @@
     "type": "git",
     "url": "git+https://github.com/webrecorder/awp-sw.git"
   },
-  "version": "0.1.9",
+  "version": "0.2.0",
   "license": "AGPL-3.0-or-later",
   "exports": {
     ".": "./src/index.js"
@@ -18,7 +18,7 @@
   "dependencies": {
     "@ipld/car": "^5.0.1",
     "@ipld/unixfs": "^2.0.0",
-    "@webrecorder/wabac": "^2.14.0",
+    "@webrecorder/wabac": "^2.14.1",
     "client-zip": "^2.3.0",
     "hash-wasm": "^4.9.0",
     "idb": "^7.1.1",

src/api.js

@@ -4,6 +4,7 @@ import { tsToDate } from "@webrecorder/wabac/src/utils.js";
 import { Downloader } from "./downloader.js";
 import { Signer } from "./keystore.js";
 import { ipfsAdd, ipfsRemove, setAutoIPFSUrl } from "./ipfsutils.js";
+import { RecProxy } from "./recproxy.js";
 
 // eslint-disable-next-line no-undef
 const DEFAULT_SOFTWARE_STRING = `Webrecorder ArchiveWeb.page ${__AWP_VERSION__}, using warcio.js ${__WARCIO_VERSION__}`;
@@ -20,6 +21,7 @@ class ExtAPI extends API
     return {
       ...super.routes,
       "downloadPages": "c/:coll/dl",
+      "recPending": "c/:coll/recPending",
       "pageTitle": ["c/:coll/pageTitle", "POST"],
       "ipfsAdd": ["c/:coll/ipfs", "POST"],
       "ipfsRemove": ["c/:coll/ipfs", "DELETE"],
@@ -38,21 +40,11 @@ class ExtAPI extends API
 
   async handleApi(request, params, event) {
     switch (params._route) {
-    case "downloadPages": {
-      const coll = await this.collections.loadColl(params.coll);
-      if (!coll) {
-        return {error: "collection_not_found"};
-      }
-
-      const pageQ = params._query.get("pages");
-      const pageList = pageQ === "all" ? null : pageQ.split(",");
-
-      const format = params._query.get("format") || "wacz";
-      let filename = params._query.get("filename");
+    case "downloadPages":
+      return await this.handleDownload(params);
 
-      const dl = new Downloader({...this.downloaderOpts(), coll, format, filename, pageList});
-      return dl.download();
-    }
+    case "recPending":
+      return await this.recordingPending(params);
 
     case "pageTitle":
       return await this.updatePageTitle(params.coll, request);
@@ -66,11 +58,42 @@ class ExtAPI extends API
     case "ipfsRemove":
       return await this.ipfsRemove(request, params.coll);
 
-
     default:
       return await super.handleApi(request, params);
     }
   }
+
+  async handleDownload(params) {
+    const coll = await this.collections.loadColl(params.coll);
+    if (!coll) {
+      return {error: "collection_not_found"};
+    }
+
+    const pageQ = params._query.get("pages");
+    const pageList = pageQ === "all" ? null : pageQ.split(",");
+
+    const format = params._query.get("format") || "wacz";
+    let filename = params._query.get("filename");
+
+    const dl = new Downloader({...this.downloaderOpts(), coll, format, filename, pageList});
+    return dl.download();
+  }
+
+  async recordingPending(params) {
+    const coll = await this.collections.loadColl(params.coll);
+    if (!coll) {
+      return {error: "collection_not_found"};
+    }
+
+    if (!(coll.store instanceof RecProxy)) {
+      return {error: "invalid_collection"};
+    }
+
+    const numPending = await coll.store.getCounter();
+
+    return { numPending };
+  }
+
   async prepareColl(collId, request) {
     const coll = await this.collections.loadColl(collId);
     if (!coll) {

src/recproxy.js

@@ -6,7 +6,7 @@ import { postToGetUrl } from "warcio";
 
 
 // ===========================================================================
-class RecProxy extends ArchiveDB
+export class RecProxy extends ArchiveDB
 {
   constructor(config, collLoader) {
     super(config.dbname);
@@ -23,7 +23,22 @@ class RecProxy extends ArchiveDB
     this.isNew = true;
     this.firstPageOnly = config.extraConfig.firstPageOnly || false;
 
-    //this.cookie = "";
+    this.counter = 0;
+  }
+
+  _initDB(db, oldV, newV, tx) {
+    super._initDB(db, oldV, newV, tx);
+    db.createObjectStore("rec");
+  }
+
+  async decCounter() {
+    this.counter--;
+    //console.log("rec counter", this.counter);
+    await this.db.put("rec", this.counter, "numPending");
+  }
+
+  async getCounter() {
+    return await this.db.get("rec", "numPending");
   }
 
   async getResource(request, prefix) {
@@ -35,21 +50,28 @@ class RecProxy extends ArchiveDB
       req = request.request;
     }
 
-    request.headers["User-Agent"] = "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.91 Mobile Safari/537.36";
+    let response = null;
 
-    const response = await this.liveProxy.getResource(request, prefix);
+    try {
+      this.counter++;
+      response = await this.liveProxy.getResource(request, prefix);
+    } catch (e) {
+      await this.decCounter();
+      return null;
+    }
 
     //this.cookie = response.headers.get("x-wabac-preset-cookie");
 
     // don't record content proxied from specified hosts
     if (!this.recordProxied && this.liveProxy.hostProxy) {
       const parsedUrl = new URL(request.url);
       if (this.liveProxy.hostProxy[parsedUrl.host]) {
+        await this.decCounter();
         return response;
       }
     }
 
-    this.doRecord(response, req);
+    this.doRecord(response, req).finally(() => this.decCounter());
 
     return response;
   }

yarn.lock

@@ -495,10 +495,10 @@
   resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.0.tgz#f08ea194e01ed45379383a8886e8c85a65a5f26a"
   integrity sha512-Rumq5mHvGXamnOh3O8yLk1sjx8dB30qF1OeR6VC00DIR6SLJ4bwwUGKC4pE7qBFoQyyh0H9sAg3fikYgAqVR0w==
 
-"@webrecorder/wabac@^2.14.0":
-  version "2.14.0"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.14.0.tgz#b2fe5adcaefcdf3731a9e05cc11bbb2c63ae59c0"
-  integrity sha512-KAO4n4sY1HtKeW98Gy88UlHlpy86x2D6b+KdKvhnHkwkwN/0IKOBUipYN7/45u7v5QRZyJf5CYI9mXZhrvUaUw==
+"@webrecorder/wabac@^2.14.1":
+  version "2.14.1"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.14.1.tgz#dd7c92e4c5892cc6d865a3cb568b8f16eb6d9761"
+  integrity sha512-fADNPmUlqtwQA7jsomKjyTwMqjx6Le5YYLsyxHPVU1HzEIxzEh+7v0dazz3LYSjnPzlbjsG4jbB/jMlJtXncwg==
   dependencies:
     "@peculiar/asn1-ecc" "^2.3.4"
     "@peculiar/asn1-schema" "^2.3.3"