Feature: @webdoc/default-template service worker (#190)

# Synopsis

This PR adds a service worker to the sites generated by the default template to
* cache all core assets (CSS & JavaScript)
* add support for a offline storage setting (in the new settings page)

It also adds an IndexedDB layer that holds the manifest data. The manifest is used to get all the pages in the site to be cached.

# Service Worker

The protocol/ folder in the site source contains the interface b/w the main & service-worker threads.

* `webdocService` lets the main application initialize and pass messages to the service worker.
* `webdocDB` is a wrapper around IndexedDB for data persistence. The "settings" store holds the application settings and the manifest url & hash.

The service worker accepts 2 types of messages:

* "lifecycle:init" − This is to be sent whenever the manifest changes (or the site is loaded for the first time). The manifest hash is to be fetched on each page load to check for changes.
* "trigger:settings" − This lets the service worker know if a setting changed (like offline storage).

## Caching

There are 2 types of caches:

* The main cache is permanent and holds the core assets like react, react-dom scripts & icons that are versioned or won't change.
* The ephemeral cache. This cache evicts items when the manifest hash changes, which is detected by attaching a "x-manifest-hash" header to cached responses and comparing it to the hash in the settings from IndexedDB (the exact settings object is found by the request origin).

The "offline storage" setting lets the user cache the whole website immediately (well all the pages in the manifest registry, which doesn't include things like README and settings pages). It's an opt-in setting; however, the service worker will cache each HTML page when the user visits them into the ephemeral cache anyway.

# IndexedDB

There are 2 object stores in the current database model:

* "settings" − This holds the application settings (identified by the application name or the site origin). It holds the manifest hash and the manifest URL.
* "hyperlinks" − This holds all the documentation hyperlinks in the website. Offline storage uses it to identify which pages need to be cached.

# UI

## Toolbar in explorer & settings page

This toolbar has the new settings page link [button]

<img width="297" alt="Screen Shot 2022-08-25 at 6 37 50 AM" src="https://user-images.githubusercontent.com/22450567/186643536-854af31e-9c94-412e-a764-4bb7f93f15c3.png">

The settings page lets you toggle offline storage:

<img width="510" alt="Screen Shot 2022-08-25 at 6 38 44 AM" src="https://user-images.githubusercontent.com/22450567/186643694-3d8647dd-6366-4ea2-8da0-87f0a68ed26d.png">

## Toasts for offline / stale pages

These toasts let the user know if
* the page was served from their offline cache (i.e. verifying the manifest hash failed) 
* the page is stale (the fetched hash was different from what was in IndexedDB).

<img width="371" alt="Screen Shot 2022-08-25 at 6 33 45 AM" src="https://user-images.githubusercontent.com/22450567/186643048-3da8e064-057b-408b-b664-5a6e48f63a7c.png">
<img width="357" alt="Screen Shot 2022-08-25 at 6 34 18 AM" src="https://user-images.githubusercontent.com/22450567/186643055-e71d348e-d33f-47ba-a08f-f42eeef2e60b.png">

Author: ShukantPal

.idea/runConfigurations.xml

@@ -1,4 +1,4 @@
 // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush.
 {
-  "pnpmShrinkwrapHash": "b24765a3c2eeef5de5c2127972cc6959228bc2f2"
+  "pnpmShrinkwrapHash": "9f03bf6671b1db458704136d5544b8a572f5f566"
 }

common/config/rush/pnpm-lock.yaml

@@ -1,6 +1,7 @@
 // @flow
 
 import * as Sentry from "@sentry/node";
+import * as crypto from "crypto";
 import * as external from "@webdoc/externalize";
 import * as yargs from "yargs";
 import {LogLevel, log, tag} from "missionlog";
@@ -134,7 +135,8 @@ export async function main(argv: yargs.Argv): Promise<void> {
   } catch (e) {
     // Make sure we get that API structure out so the user can debug the problem!
     if (config.opts && config.opts.export) {
-      fs.writeFileSync(config.opts.export, external.write(external.fromTree(documentTree), true));
+      fs.writeFileSync((config.opts.export: any),
+        external.write(external.fromTree(documentTree), true));
     }
 
     throw e;
@@ -181,7 +183,15 @@ export async function main(argv: yargs.Argv): Promise<void> {
   }
 
   if (config.opts && config.opts.export) {
-    fs.writeFileSync(config.opts.export, external.write(manifest, argv.verbose));
+    const file = config.opts.export;
+    const contents = external.write(manifest, argv.verbose);
+    const hash = crypto
+      .createHash("md5")
+      .update(contents)
+      .digest("hex");
+
+    fs.writeFileSync(file, contents);
+    fs.writeFileSync(`${file}.md5`, hash);
   }
 
   console.log(`@webdoc took ${Math.ceil(performance.now() - start)}ms to run!`);

common/config/rush/repo-state.json

@@ -2,5 +2,13 @@
     "ignorePatterns": "static/scripts/*",
     "extends": [
         "@webdoc/eslint-config"
+    ],
+    "overrides": [
+        {
+            "files": ["./src/service-worker/**"],
+            "env": {
+                "serviceworker": true
+            }
+        }
     ]
 }

core/webdoc-cli/src/main.js

@@ -1,2 +1,4 @@
 static/scripts/default-template.js
 static/scripts/default-template.js.LICENSE.txt
+static/service-worker.js
+static/service-worker.js.LICENSE.txt

core/webdoc-default-template/.eslintrc.json

@@ -8,7 +8,7 @@ import type {
   Doc,
   DocType,
   PackageDoc,
-  RootDoc
+  RootDoc,
 } from "@webdoc/types";
 import type {CategorizedDocumentList} from "./crawl";
 */
@@ -30,7 +30,7 @@ const HINTS = {
 };
 
 // Crawls the tree searching for the API reference
-function crawlReference(doc /*: Doc */, index /*: string */) {
+function crawlReference(doc /*: RootDoc */, index /*: string */) {
   if (!doc.members.length) {
     return null;
   }
@@ -52,6 +52,7 @@ function getPage(doc /*: Doc */) {
 type ExplorerNode = {
   doc: Doc,
   children: CategorizedDocumentList,
+};
 */
 
 function buildExplorerHierarchy(rootDoc /*: RootDoc */, multiPackage = false) /*: ExplorerNode */ {

core/webdoc-default-template/.gitignore

@@ -79,6 +79,13 @@ function buildLinks(manifest /*: Manifest */) /*: void */ {
 
         registry[packageDoc.id].uri = linker.getURI(packageDoc);
       });
+      for (const tutorialDoc of doc.tutorials) {
+        if (tutorialDoc.route) {
+          if (!registry[tutorialDoc.id]) registry[tutorialDoc.id] = {};
+          registry[tutorialDoc.id].uri = linker.getURI(tutorialDoc);
+        }
+      }
+
 
       return;
     }

core/webdoc-default-template/helper/crawl/crawl-reference-explorer.js

@@ -9,7 +9,7 @@
     "webdoc-template"
   ],
   "author": "Shukant Pal <[email protected]>",
-  "homepage": "https://github.com/SukantPal/webdoc/tree/master/packages/default-template#readme",
+  "homepage": "https://github.com/webdoc-labs/webdoc/tree/master/packages/default-template#readme",
   "license": "The Prosperity Public License 3.0.0",
   "main": "publish.js",
   "directories": {
@@ -31,6 +31,7 @@
   },
   "scripts": {
     "build": "webpack build --config ./webpack.config.js",
+    "flow_": "flow check",
     "lint": "eslint -c .eslintrc.json .",
     "unit-test": "",
     "test": "echo \"Error: run tests from root\" && exit 1"
@@ -70,6 +71,7 @@
     "eslint": "8.18.0",
     "extract-loader": "^5.1.0",
     "file-loader": "^6.0.0",
+    "flow-bin": "^0.130.0",
     "flow-typed": "^3.2.1",
     "lodash": "^4.17.20",
     "mini-css-extract-plugin": "^0.9.0",

core/webdoc-default-template/helper/crawl/crawl.js

@@ -78,6 +78,12 @@ exports.publish = async function publish(options /*: PublishOptions */) {
   const assetsDir = path.join(outDir, "./assets");
   const index = config.template.readme ? linker.createURI("index") : null;
   const indexRelative = index ? index.replace(`/${linker.siteRoot}/`, "") : null;
+  const settings = linker.createURI("settings");
+  const settingsRelative = settings.replace(`/${linker.siteRoot}/`, "");
+  const manifestNormalized = config.opts.export ? path.normalize(config.opts.export) : null;
+  const manifest = config.opts.export && manifestNormalized.startsWith(outDir) ?
+    path.join(linker.siteRoot, manifestNormalized.replace(outDir, "")) :
+    null;
 
   fse.ensureDir(outDir);
 
@@ -133,6 +139,13 @@ exports.publish = async function publish(options /*: PublishOptions */) {
       appBar: {
         items: appBarItems,
       },
+      manifest: {url: manifest},
+      pages: {
+        settings,
+        relative: {
+          settings: settingsRelative,
+        },
+      },
       variant: config.template.variant,
     });
   if (config.template.repository) {
@@ -169,6 +182,7 @@ exports.publish = async function publish(options /*: PublishOptions */) {
     outSource(outDir, pipeline, options.config, source, options.cmdLine.mainThread || false),
     outExplorerData(outDir, crawlData),
     outMainPage(indexRelative ? path.join(outDir, indexRelative) : null, pipeline, options.config),
+    outPages(outDir, pipeline, options.config),
     outIndexes(outDir, pipeline, options.config, crawlData.index),
     outReference(outDir, pipeline, options.config, docTree, crawlData.reference),
     outTutorials(outDir, pipeline, options.config, docTree, crawlData.tutorials),
@@ -291,6 +305,19 @@ async function outMainPage(
   }
 }
 
+async function outPages(
+  outDir /*: string */,
+  pipeline /*: TemplatePipeline */,
+  config /*: WebdocConfig */,
+)/*: Promise<void> */ {
+  pipeline.render("pages/settings.tmpl", {
+    env: config,
+    title: "Site Settings",
+  }, {
+    outputFile: path.join(outDir, pipeline.renderer.data.pages.relative.settings),
+  });
+}
+
 async function outSource(
   outDir /*: string */,
   pipeline /*: TemplatePipeline */,