ts conversion, first pass

Author: ikreymer

package.json

@@ -16,16 +16,13 @@
     "src/*"
   ],
   "dependencies": {
-    "@ipld/car": "^5.3.1",
-    "@ipld/unixfs": "^3.0.0",
-    "@webrecorder/wabac": "^2.18.1",
+    "@webrecorder/wabac": "^2.20.0-beta.1",
     "auto-js-ipfs": "^2.3.0",
     "client-zip": "^2.3.0",
     "hash-wasm": "^4.9.0",
     "idb": "^7.1.1",
-    "p-queue": "^7.3.4",
     "uuid": "^9.0.0",
-    "warcio": "^2.2.1"
+    "warcio": "^2.3.1"
   },
   "scripts": {
     "build": "webpack --mode production",
@@ -34,8 +31,12 @@
     "lint": "eslint ./src webpack.config.cjs"
   },
   "devDependencies": {
+    "@types/uuid": "^10.0.0",
     "eslint": "^8.28.0",
     "raw-loader": "^4.0.2",
+    "ts-loader": "^9.5.1",
+    "tsconfig-paths-webpack-plugin": "^4.1.0",
+    "typescript": "^5.5.4",
     "webpack": "^5.91.0",
     "webpack-cli": "^5.1.4"
   },

src/api.js renamed to src/api.ts

@@ -1,5 +1,4 @@
-import { API } from "@webrecorder/wabac/src/api.js";
-import { tsToDate } from "@webrecorder/wabac/src/utils.js";
+import { API, SWCollections, tsToDate } from "@webrecorder/wabac/swlib";
 
 import { Downloader } from "./downloader.js";
 import { Signer } from "./keystore.js";
@@ -12,14 +11,17 @@ const DEFAULT_SOFTWARE_STRING = `Webrecorder ArchiveWeb.page ${__AWP_VERSION__},
 // ===========================================================================
 class ExtAPI extends API
 {
-  constructor(collections, {softwareString = "", replaceSoftwareString = false} = {}) {
+  softwareString = "";
+  uploading = new Map();
+
+  constructor(collections: SWCollections, {softwareString = "", replaceSoftwareString = false} = {}) {
     super(collections);
     this.softwareString = replaceSoftwareString ? softwareString : softwareString + DEFAULT_SOFTWARE_STRING;
 
     this.uploading = new Map();
   }
 
-  get routes() {
+  override get routes(): Record<string, string | [string, string]> {
     return {
       ...super.routes,
       "downloadPages": "c/:coll/dl",
@@ -43,7 +45,7 @@ class ExtAPI extends API
     return {softwareString, signer};
   }
 
-  async handleApi(request, params, event) {
+  override async handleApi(request: Request, params: any, event: FetchEvent){
     switch (params._route) {
     case "downloadPages":
       return await this.handleDownload(params);
@@ -67,10 +69,12 @@ class ExtAPI extends API
       return await this.getPublicKey();
 
     case "ipfsAdd":
-      return await this.startIpfsAdd(event, request, params.coll);
+      //return await this.startIpfsAdd(event, request, params.coll);
+      return {};
 
     case "ipfsRemove":
-      return await this.ipfsRemove(request, params.coll);
+      //return await this.ipfsRemove(request, params.coll);
+      return {};
 
     case "ipfsDaemonUrl":
       return await this.setIPFSDaemonUrlFromBody(request);

src/downloader.js renamed to src/downloader.ts

@@ -5,10 +5,14 @@ import { Deflate } from "pako";
 import { v5 as uuidv5 } from "uuid";
 
 import { createSHA256 } from "hash-wasm";
+import { type IHasher } from "hash-wasm/dist/lib/WASMInterface.js";
 
 import { getSurt, WARCRecord, WARCSerializer } from "warcio";
 
-import { getTSMillis, getStatusText, digestMessage } from "@webrecorder/wabac/src/utils.js";
+import { getTSMillis, getStatusText, digestMessage } from "@webrecorder/wabac";
+import { ArchiveDB, ResourceEntry } from "@webrecorder/wabac/swlib";
+import { Signer } from "./keystore";
+import { Collection } from "../../wabac.js/dist/types/collection";
 
 
 // ===========================================================================
@@ -28,11 +32,11 @@ const encoder = new TextEncoder();
 
 const EMPTY = new Uint8Array([]);
 
-async function* getPayload(payload) {
+async function* getPayload(payload: Uint8Array) {
   yield payload;
 }
 
-async function* hashingGen(gen, stats, hasher, sizeCallback, zipMarker) {
+async function* hashingGen(gen: AsyncIterable<Uint8Array | string>, stats: any, hasher: IHasher, sizeCallback: any, zipMarker: Uint8Array) {
   stats.size = 0;
 
   hasher.init();
@@ -61,31 +65,89 @@ async function* hashingGen(gen, stats, hasher, sizeCallback, zipMarker) {
   stats.hash = hasher.digest("hex");
 }
 
+type DownloaderOpts = {
+  coll: Collection;
+  format: string;
+  filename?: string;
+  pageList?: string[];
+  signer?: Signer;
+  softwareString?: string;
+  gzip?: boolean;
+  uuidNamespace?: string;
+  markers?: Record<string, Uint8Array>;
+};
+
 // ===========================================================================
 class Downloader
 {
-  constructor({coll, format = "wacz", filename = null, pageList = null, signer = null,
-    softwareString = null, gzip = true, uuidNamespace = null, markers = null}) {
+  db: ArchiveDB;
+  pageList: string[] | null;
+  collId: string;
+  metadata: Record<string, string>;
+  gzip: boolean;
+
+  markers: Record<string, Uint8Array>;
+  warcName: string;
+  alreadyDecoded: boolean;
+
+  softwareString: string;
+  uuidNamespace: string;
+
+  createdDateDt: Date;
+  createdDate: string;
+  modifiedDate: string | null;
+
+  format: string;
+  warcVersion: string;
+
+  digestOpts: {
+    algo: string;
+    prefix: string;
+    base32?: boolean;
+  };
+
+  filename: string;
+
+  signer: Signer | null;
+
+  offset = 0;
+  firstResources = [];
+  textResources = [];
+  cdxjLines = [];
+
+  // compressed index (idx) entries
+  indexLines = [];
+
+  digestsVisted = {};
+  fileHasher = null;
+  recordHasher = null;
+
+  datapackageDigest = null;
+
+  fileStats = [];
+
+  constructor({coll, format = "wacz", filename, pageList, signer,
+    softwareString, gzip = true, uuidNamespace, markers} : DownloaderOpts) {
 
     this.db = coll.store;
-    this.pageList = pageList;
+    this.pageList = pageList || [];
     this.collId = coll.name;
-    this.metadata = coll.config.metadata;
+    this.metadata = coll.config["metadata"];
     this.gzip = gzip;
 
     this.markers = markers || {};
 
     this.warcName = this.gzip ? "data.warc.gz" : "data.warc";
 
-    this.alreadyDecoded = !coll.config.decode && !coll.config.loadUrl;
+    this.alreadyDecoded = !coll.config["decode"] && !coll.config["loadUrl"];
 
     this.softwareString = softwareString || "ArchiveWeb.page";
 
     this.uuidNamespace = uuidNamespace || DEFAULT_UUID_NAMESPACE;
 
-    this.createdDateDt = new Date(coll.config.ctime);
+    this.createdDateDt = new Date(coll.config["ctime"]);
     this.createdDate = this.createdDateDt.toISOString();
-    this.modifiedDate = coll.config.metadata.mtime ? new Date(coll.config.metadata.mtime).toISOString() : null;
+    this.modifiedDate = coll.config["metadata"].mtime ? new Date(coll.config["metadata"].mtime).toISOString() : null;
 
     this.format = format;
     this.warcVersion = (format === "warc1.0") ? "WARC/1.0" : "WARC/1.1";
@@ -96,33 +158,17 @@ class Downloader
       this.digestOpts = {algo: "sha-256", prefix: "sha256:"};
     }
 
-    this.filename = filename;
-
     // determine filename from title, if it exists
-    if (!this.filename && coll.config.metadata.title) {
-      this.filename = encodeURIComponent(coll.config.metadata.title.toLowerCase().replace(/\s/g, "-"));
+    if (!filename && coll.config["metadata"].title) {
+      filename = encodeURIComponent(coll.config["metadata"].title.toLowerCase().replace(/\s/g, "-"));
     }
 
-    if (!this.filename) {
-      this.filename = "webarchive";
+    if (!filename) {
+      filename = "webarchive";
     }
+    this.filename = filename;
 
-    this.offset = 0;
-    this.firstResources = [];
-    this.textResources = [];
-    this.cdxjLines = [];
-
-    // compressed index (idx) entries
-    this.indexLines = [];
-
-    this.digestsVisted = {};
-    this.fileHasher = null;
-    this.recordHasher = null;
-
-    this.datapackageDigest = null;
-    this.signer = signer;
-
-    this.fileStats = [];
+    this.signer = signer || null;
   }
 
   download(sizeCallback = null) {
@@ -139,7 +185,7 @@ class Downloader
     }
   }
 
-  downloadWARC(filename, sizeCallback = null) {
+  downloadWARC(filename: string, sizeCallback = null) {
     filename = (filename || "webarchive").split(".")[0] + ".warc";
 
     const dl = this;
@@ -160,19 +206,19 @@ class Downloader
     return resp;
   }
 
-  async loadResourcesBlock(start = []) {
-    return await this.db.db.getAll("resources", IDBKeyRange.lowerBound(start, true), RESOURCE_BATCH_SIZE);
+  async loadResourcesBlock(start : [string, number] | [] = []) {
+    return await this.db.db!.getAll("resources", IDBKeyRange.lowerBound(start, true), RESOURCE_BATCH_SIZE);
   }
 
-  async* iterResources(resources) {
-    let start = [];
+  async* iterResources(resources: ResourceEntry[]) {
+    let start : [string, number] | [] = [];
     //let count = 0;
 
     while (resources.length) {
-      const last = resources[resources.length - 1];
+      const last : ResourceEntry = resources[resources.length - 1] as ResourceEntry;
 
       if (this.pageList) {
-        resources = resources.filter((res) => this.pageList.includes(res.pageId));
+        resources = resources.filter((res) => this.pageList!.includes(res.pageId || ""));
       }
       //count += resources.length;
       yield* resources;
@@ -185,7 +231,7 @@ class Downloader
     // }
   }
 
-  async queueWARC(controller, filename, sizeCallback) {
+  async queueWARC(controller, filename: string, sizeCallback: any) {
     this.firstResources = await this.loadResourcesBlock();
 
     for await (const chunk of this.generateWARC(filename)) {

src/index.js renamed to src/index.ts

@@ -2,9 +2,9 @@ import { Downloader } from "./downloader.js";
 
 import { create as createAutoIPFS } from "auto-js-ipfs";
 
-import * as UnixFS from "@ipld/unixfs";
-import { CarWriter } from "@ipld/car/writer";
-import Queue from "p-queue";
+//import * as UnixFS from "@ipld/unixfs";
+//import { CarWriter } from "@ipld/car/writer";
+//import Queue from "p-queue";
 
 // eslint-disable-next-line no-undef
 const autoipfsOpts = {web3StorageToken: __WEB3_STORAGE_TOKEN__};