bump version to 0.5.0-beta.0

Author: ikreymer

package.json

@@ -1,24 +1,26 @@
 {
   "name": "@webrecorder/awp-sw",
   "browser": "dist/sw.js",
-  "version": "0.4.4",
-  "main": "index.js",
+  "version": "0.5.0-beta.0",
   "type": "module",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/webrecorder/awp-sw.git"
   },
   "license": "AGPL-3.0-or-later",
   "exports": {
-    ".": "./src/index.js"
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/index.js"
+    }
   },
   "files": [
     "src/*"
   ],
   "dependencies": {
     "@ipld/car": "^5.3.2",
     "@ipld/unixfs": "^3.0.0",
-    "@webrecorder/wabac": "^2.20.0-beta.1",
+    "@webrecorder/wabac": "^2.20.0-beta.2",
     "auto-js-ipfs": "^2.3.0",
     "client-zip": "^2.3.0",
     "hash-wasm": "^4.9.0",
@@ -46,7 +48,7 @@
     "ts-loader": "^9.5.1",
     "tsconfig-paths-webpack-plugin": "^4.1.0",
     "typescript": "^5.5.4",
-    "webpack": "^5.91.0",
+    "webpack": "^5.94.0",
     "webpack-cli": "^5.1.4"
   },
   "description": "This library has been factored out of [ArchiveWeb.page](https://webrecorder/archiveweb.page) and represents the core service worker implementation necessarily for high-fidelity web archiving.",

src/api.ts

@@ -170,12 +170,12 @@ class ExtAPI extends API {
         signal,
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
       } as any);
-      uploading.set(params["coll"], counter);
+      uploading.set(params.coll, counter);
       if (event.waitUntil) {
         event.waitUntil(
           this.uploadFinished(
             fetchPromise,
-            params["coll"],
+            params.coll,
             dl.metadata,
             filename,
             counter,
@@ -184,7 +184,7 @@ class ExtAPI extends API {
       }
       return { uploading: true };
     } catch (e: unknown) {
-      uploading.delete(params["coll"]);
+      uploading.delete(params.coll);
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       return { error: "upload_failed", details: (e as any).toString() };
     }
@@ -224,7 +224,7 @@ class ExtAPI extends API {
   }
 
   async deleteUpload(params: RouteMatch) {
-    const collId = params["coll"];
+    const collId = params.coll;
 
     this.uploading.delete(collId);
 
@@ -242,7 +242,7 @@ class ExtAPI extends API {
 
   async getUploadStatus(params: RouteMatch) {
     let result: Metadata = {};
-    const counter = this.uploading.get(params["coll"]);
+    const counter = this.uploading.get(params.coll);
 
     if (!counter) {
       result = { status: "idle" };
@@ -251,11 +251,11 @@ class ExtAPI extends API {
       result = { status, size, totalSize };
 
       if (status !== "uploading") {
-        this.uploading.delete(params["coll"]);
+        this.uploading.delete(params.coll);
       }
     }
 
-    const coll = await this.collections.loadColl(params["coll"]);
+    const coll = await this.collections.loadColl(params.coll);
 
     if (coll?.metadata) {
       result.uploadTime = coll.metadata.uploadTime;
@@ -268,7 +268,7 @@ class ExtAPI extends API {
   }
 
   async recordingPending(params: RouteMatch) {
-    const coll = await this.collections.loadColl(params["coll"]);
+    const coll = await this.collections.loadColl(params.coll);
     if (!coll) {
       return { error: "collection_not_found" };
     }
@@ -435,7 +435,9 @@ async function runIPFSAdd(
 
   sendMessage("ipfsAdd", result);
 
-  await collections.updateMetadata(coll.name, coll.config["metadata"]);
+  if (coll.config.metadata) {
+    await collections.updateMetadata(coll.name, coll.config.metadata);
+  }
 }
 
 // ===========================================================================

src/downloader.ts

@@ -268,7 +268,7 @@ class Downloader {
     this.db = coll.store;
     this.pageList = pageList || [];
     this.collId = coll.name;
-    this.metadata = coll.config["metadata"];
+    this.metadata = coll.config.metadata || {};
     this.gzip = gzip;
 
     this.markers = markers || {};
@@ -281,12 +281,10 @@ class Downloader {
 
     this.uuidNamespace = uuidNamespace || DEFAULT_UUID_NAMESPACE;
 
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-    this.createdDateDt = new Date(coll.config.ctime);
+    this.createdDateDt = new Date(coll.config.ctime!);
     this.createdDate = this.createdDateDt.toISOString();
-    this.modifiedDate = coll.config["metadata"].mtime
-      ? // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        new Date(coll.config.metadata.mtime).toISOString()
+    this.modifiedDate = coll.config.metadata!.mtime
+      ? new Date(coll.config.metadata!.mtime).toISOString()
       : null;
 
     this.format = format;
@@ -299,10 +297,9 @@ class Downloader {
     }
 
     // determine filename from title, if it exists
-    if (!filename && coll.config["metadata"].title) {
+    if (!filename && coll.config.metadata!.title) {
       filename = encodeURIComponent(
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        coll.config.metadata.title.toLowerCase().replace(/\s/g, "-"),
+        coll.config.metadata!.title.toLowerCase().replace(/\s/g, "-"),
       );
     }
 

src/ipfsutils.ts

@@ -1,4 +1,4 @@
-import { type Collection } from "@webrecorder/wabac/swlib";
+import { type CollMetadata, type Collection } from "@webrecorder/wabac/swlib";
 import { Downloader, type DownloaderOpts, type Markers } from "./downloader";
 
 // @ts-expect-error no types
@@ -31,6 +31,10 @@ type ReplayOpts = {
   loading?: boolean;
 };
 
+type MetadataWithIPFS = CollMetadata & {
+  ipfsPins?: {url: string, cid: string}[] | null;
+}
+
 export async function setAutoIPFSUrl(url: string) {
   if (autoipfsOpts.daemonURL !== url) {
     autoipfs = null;
@@ -66,8 +70,10 @@ export async function ipfsAdd(
     throw new Error(dlResponse.error);
   }
 
-  if (!coll.config.metadata.ipfsPins) {
-    coll.config.metadata.ipfsPins = [];
+  const metadata : MetadataWithIPFS = coll.config.metadata || {};
+
+  if (!metadata.ipfsPins) {
+    metadata.ipfsPins = [];
   }
 
   let concur;
@@ -120,8 +126,8 @@ export async function ipfsAdd(
 
   progress(0, totalSize);
 
-  let url,
-    cid = "";
+  let url = "";
+  let cid = "";
 
   let reject: ((reason?: string) => void) | null = null;
 
@@ -132,7 +138,7 @@ export async function ipfsAdd(
     .pipeThrough(new ShardStoringStream(autoipfs, concur, reject!))
     .pipeTo(
       new WritableStream({
-        write: (res) => {
+        write: (res: {url: string, cid: string, size: number}) => {
           if (res.url && res.cid) {
             url = res.url;
             cid = res.cid;
@@ -160,7 +166,7 @@ export async function ipfsAdd(
 
   const res = { cid: cid.toString(), url };
 
-  coll.config.metadata.ipfsPins.push(res);
+  metadata.ipfsPins.push(res);
 
   console.log("ipfs cid added " + url);
 
@@ -172,8 +178,10 @@ export async function ipfsRemove(coll: Collection) {
     autoipfs = await createAutoIPFS(autoipfsOpts);
   }
 
-  if (coll.config.metadata.ipfsPins) {
-    for (const { url } of coll.config.metadata.ipfsPins) {
+  const metadata : MetadataWithIPFS = coll.config.metadata || {};
+
+  if (metadata.ipfsPins) {
+    for (const { url } of metadata.ipfsPins) {
       try {
         await autoipfs.clear(url);
       } catch (_e) {
@@ -183,7 +191,7 @@ export async function ipfsRemove(coll: Collection) {
       }
     }
 
-    coll.config.metadata.ipfsPins = null;
+    metadata.ipfsPins = null;
     return true;
   }
 
@@ -205,6 +213,7 @@ async function ipfsWriteBuff(
   const file = UnixFS.createFileWriter(writer);
   if (content instanceof Uint8Array) {
     await file.write(content);
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   } else if (content[Symbol.asyncIterator]) {
     for await (const chunk of content) {
       await file.write(chunk);
@@ -223,7 +232,7 @@ export async function ipfsGenerateCar(
   uiContent: Uint8Array,
   htmlContent: string,
   replayOpts: ReplayOpts,
-  markers: Markers,
+  markers: Markers | null,
   favicon: Uint8Array | null,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): Promise<any> {
@@ -268,7 +277,7 @@ export async function ipfsGenerateCar(
 
   const { cid } = await rootDir.close();
 
-  writer.close();
+  await writer.close();
 
   return cid;
 }
@@ -338,6 +347,7 @@ async function splitByWarcRecordGroup(
         links = [];
       }
 
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
       fileLinks.push(link);
 
       const [dirName, filename] = getDirAndName(currName);
@@ -354,6 +364,7 @@ async function splitByWarcRecordGroup(
         dir = dirs[dirName];
       }
 
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
       dir.set(filename, link);
 
       inZipFile = false;
@@ -368,6 +379,7 @@ async function splitByWarcRecordGroup(
         file = UnixFS.createFileWriter(writer);
 
         if (chunk === WARC_GROUP) {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
           secondaryLinks.push(await concat(writer, links));
           links = [];
         }
@@ -376,7 +388,7 @@ async function splitByWarcRecordGroup(
       if (!inZipFile) {
         lastChunk = chunk;
       }
-      file.write(chunk);
+      await file.write(chunk);
       count++;
     }
   }
@@ -404,6 +416,7 @@ async function splitByWarcRecordGroup(
 
   rootDir.set("webarchive", await waczDir.close());
 
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
   rootDir.set(waczPath, await concat(writer, fileLinks));
 }
 
@@ -416,6 +429,7 @@ async function concat(
   const { fileEncoder, hasher, linker } = writer.settings;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const advanced = (fileEncoder as any).createAdvancedFile(links);
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
   const bytes = fileEncoder.encode(advanced);
   const hash = await hasher.digest(bytes);
   const cid = linker.createLink(fileEncoder.code, hash);
@@ -435,6 +449,7 @@ async function concat(
 
 export const iterate = async function* (stream: ReadableStream<Uint8Array>) {
   const reader = stream.getReader();
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   while (true) {
     const next = await reader.read();
     if (next.done) {
@@ -447,6 +462,7 @@ export const iterate = async function* (stream: ReadableStream<Uint8Array>) {
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function encodeBlocks(blocks: UnixFS.Block[], root?: any) {
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
   const { writer, out } = CarWriter.create(root);
   /** @type {Error?} */
   let error;
@@ -464,6 +480,7 @@ export async function encodeBlocks(blocks: UnixFS.Block[], root?: any) {
   })();
   const chunks = [];
   for await (const chunk of out) chunks.push(chunk);
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   if (error != null) throw error;
   const roots = root != null ? [root] : [];
   console.log("chunks", chunks.length);
@@ -537,6 +554,7 @@ export class ShardingStream extends TransformStream {
           shard = [];
           currSize = 0;
         }
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
         shard.push(block);
         currSize += block.bytes.length;
       },

webpack.config.cjs

@@ -8,16 +8,20 @@ const BANNER_TEXT = `'[name].js is part of the ArchiveWeb.page system (https://a
 
 
 module.exports = {
-  target: "webworker",
+  target: "web",
   entry: {
     "main": "./src/index.ts",
   },
   output: {
     filename: "sw.js",
+    globalObject: "self",
     library: {
-      type: "self"
+      type: "module"
     }
   },
+  experiments: {
+    outputModule: true
+  },
   optimization: {
     minimize: true,
     minimizer: [