page stats (#1291)

* log page stats

* tree output

* localized size format

* fancier layout

* magenta if ≥10 MB

* tidier code

Author: mbostock

package.json

@@ -66,6 +66,7 @@
     "cross-env": "^7.0.3",
     "cross-spawn": "^7.0.3",
     "d3-array": "^3.2.4",
+    "d3-hierarchy": "^3.1.2",
     "esbuild": "^0.20.1",
     "fast-array-diff": "^1.1.0",
     "gray-matter": "^4.0.3",

src/build.ts

@@ -1,6 +1,6 @@
 import {createHash} from "node:crypto";
 import {existsSync} from "node:fs";
-import {access, constants, copyFile, readFile, writeFile} from "node:fs/promises";
+import {access, constants, copyFile, readFile, stat, writeFile} from "node:fs/promises";
 import {basename, dirname, extname, join} from "node:path/posix";
 import type {Config} from "./config.js";
 import {CliError, isEnoent} from "./error.js";
@@ -12,15 +12,16 @@ import type {MarkdownPage} from "./markdown.js";
 import {parseMarkdown} from "./markdown.js";
 import {extractNodeSpecifier} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport} from "./npm.js";
-import {isPathImport, relativePath, resolvePath} from "./path.js";
+import {isAssetPath, isPathImport, relativePath, resolvePath} from "./path.js";
 import {renderPage} from "./render.js";
 import type {Resolvers} from "./resolvers.js";
 import {getModuleResolver, getResolvers} from "./resolvers.js";
 import {resolveImportPath, resolveStylesheetPath} from "./resolvers.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
 import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
-import {faint, yellow} from "./tty.js";
+import {tree} from "./tree.js";
+import {faint, green, magenta, red, yellow} from "./tty.js";
 
 export interface BuildOptions {
   config: Config;
@@ -218,15 +219,12 @@ export async function build(
     await effects.writeFile(alias, contents);
   }
 
-  // Render pages, resolving against content-hashed file names!
-  for (const [sourceFile, {page, resolvers}] of pages) {
-    const sourcePath = join(root, sourceFile);
-    const outputPath = join(dirname(sourceFile), basename(sourceFile, ".md") + ".html");
+  // Wrap the resolvers to apply content-hashed file names.
+  for (const [sourceFile, page] of pages) {
     const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
-    const options = {path, ...config};
-    effects.output.write(`${faint("render")} ${sourcePath} ${faint("→")} `);
-    const html = await renderPage(page, {
-      ...options,
+    const {resolvers} = page;
+    pages.set(sourceFile, {
+      ...page,
       resolvers: {
         ...resolvers,
         resolveFile(specifier) {
@@ -251,12 +249,86 @@ export async function build(
         }
       }
     });
+  }
+
+  // Render pages!
+  for (const [sourceFile, {page, resolvers}] of pages) {
+    const sourcePath = join(root, sourceFile);
+    const outputPath = join(dirname(sourceFile), basename(sourceFile, ".md") + ".html");
+    const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
+    effects.output.write(`${faint("render")} ${sourcePath} ${faint("→")} `);
+    const html = await renderPage(page, {...config, path, resolvers});
     await effects.writeFile(outputPath, html);
   }
 
+  // Log page sizes.
+  const columnWidth = 12;
+  effects.logger.log("");
+  for (const [indent, name, description, node] of tree(pages)) {
+    if (node.children) {
+      effects.logger.log(
+        `${faint(indent)}${name}${faint(description)} ${
+          node.depth ? "" : ["Page", "Imports", "Files"].map((name) => name.padStart(columnWidth)).join(" ")
+        }`
+      );
+    } else {
+      const [sourceFile, {resolvers}] = node.data!;
+      const outputPath = join(dirname(sourceFile), basename(sourceFile, ".md") + ".html");
+      const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
+      const resolveOutput = (name: string) => join(config.output, resolvePath(path, name));
+      const pageSize = (await stat(join(config.output, outputPath))).size;
+      const importSize = await accumulateSize(resolvers.staticImports, resolvers.resolveImport, resolveOutput);
+      const fileSize =
+        (await accumulateSize(resolvers.files, resolvers.resolveFile, resolveOutput)) +
+        (await accumulateSize(resolvers.assets, resolvers.resolveFile, resolveOutput)) +
+        (await accumulateSize(resolvers.stylesheets, resolvers.resolveStylesheet, resolveOutput));
+      effects.logger.log(
+        `${faint(indent)}${name}${description} ${[pageSize, importSize, fileSize]
+          .map((size) => formatBytes(size, columnWidth))
+          .join(" ")}`
+      );
+    }
+  }
+  effects.logger.log("");
+
   Telemetry.record({event: "build", step: "finish", pageCount});
 }
 
+async function accumulateSize(
+  files: Iterable<string>,
+  resolveFile: (path: string) => string,
+  resolveOutput: (path: string) => string
+): Promise<number> {
+  let size = 0;
+  for (const file of files) {
+    const fileResolution = resolveFile(file);
+    if (isAssetPath(fileResolution)) {
+      try {
+        size += (await stat(resolveOutput(fileResolution))).size;
+      } catch {
+        // ignore missing file
+      }
+    }
+  }
+  return size;
+}
+
+function formatBytes(size: number, length: number, locale: Intl.LocalesArgument = "en-US"): string {
+  let color: (text: string) => string;
+  let text: string;
+  if (size < 1e3) {
+    text = "<1 kB";
+    color = faint;
+  } else if (size < 1e6) {
+    text = (size / 1e3).toLocaleString(locale, {maximumFractionDigits: 0}) + " kB";
+    color = green;
+  } else {
+    text = (size / 1e6).toLocaleString(locale, {minimumFractionDigits: 3, maximumFractionDigits: 3}) + " MB";
+    color = size < 10e6 ? yellow : size < 50e6 ? magenta : red;
+  }
+  return color(text.padStart(length));
+}
+
 export class FileBuildEffects implements BuildEffects {
   private readonly outputRoot: string;
   readonly logger: Logger;

src/dataloader.ts

@@ -9,6 +9,7 @@ import JSZip from "jszip";
 import {extract} from "tar-stream";
 import {maybeStat, prepareOutput} from "./files.js";
 import {FileWatchers} from "./fileWatchers.js";
+import {formatByteSize} from "./format.js";
 import {getFileInfo} from "./javascript/module.js";
 import type {Logger, Writer} from "./logger.js";
 import {cyan, faint, green, red, yellow} from "./tty.js";
@@ -267,8 +268,9 @@ export abstract class Loader {
     const start = performance.now();
     command.then(
       (path) => {
+        const {size} = statSync(join(this.root, path));
         effects.logger.log(
-          `${green("success")} ${cyan(formatSize(statSync(join(this.root, path)).size))} ${faint(
+          `${green("success")} ${size ? cyan(formatByteSize(size)) : yellow("empty output")} ${faint(
             `in ${formatElapsed(start)}`
           )}`
         );
@@ -395,12 +397,6 @@ const extractors = [
   [".tgz", TarGzExtractor]
 ] as const;
 
-function formatSize(size: number): string {
-  if (!size) return yellow("empty output");
-  const e = Math.floor(Math.log(size) / Math.log(1024));
-  return `${+(size / 1024 ** e).toFixed(2)} ${["bytes", "KiB", "MiB", "GiB", "TiB"][e]}`;
-}
-
 function formatElapsed(start: number): string {
   const elapsed = performance.now() - start;
   return `${Math.floor(elapsed)}ms`;

src/format.ts

@@ -14,14 +14,16 @@ function pad(number: number, length: number): string {
   return String(number).padStart(length, "0");
 }
 
-export function formatByteSize(n: number): string {
-  const suffixes = ["bytes", "KB", "MB", "GB"];
-  for (const suffix of suffixes) {
-    if (n < 1300) {
-      if (suffix === "bytes") return `${n} ${suffix}`;
-      return `${n > 100 ? n.toFixed(0) : n.toFixed(1)} ${suffix}`;
+export function formatByteSize(x: number, locale: Intl.LocalesArgument = "en-US"): string {
+  const formatOptions = {maximumSignificantDigits: 3, maximumFractionDigits: 2};
+  for (const [k, suffix] of [
+    [1e9, " GB"],
+    [1e6, " MB"],
+    [1e3, " kB"]
+  ] as const) {
+    if (Math.round((x / k) * 1e3) >= 1e3) {
+      return (x / k).toLocaleString(locale, formatOptions) + suffix;
     }
-    n /= 1000;
   }
-  return `${n.toFixed(1)} TB`;
+  return x.toLocaleString(locale, formatOptions) + " B";
 }

src/tree.ts

@@ -0,0 +1,65 @@
+import {ascending, max} from "d3-array";
+import {stratify} from "d3-hierarchy";
+
+export type TreeItem<T> = [path: string, value: T];
+
+export interface TreeNode<T> {
+  data?: TreeItem<T>;
+  parent?: TreeNode<T>;
+  children?: TreeNode<T>[];
+  value: number;
+  height: number;
+  depth: number;
+  id: string;
+}
+
+export function tree<T>(
+  items: Iterable<TreeItem<T>>
+): [indent: string, name: string, description: string, node: TreeNode<T>][] {
+  const lines: [indent: string, name: string, description: string, node: TreeNode<T>][] = [];
+  stratify()
+    .path(
+      ([path]) =>
+        path.replace(/\.md$/, "") + // remove .md extension
+        (path.endsWith("/") ? "" : "?") // distinguish files from folders
+    )([...items, ["/"]]) // add root to avoid implicit truncation
+    .sort(treeOrder)
+    .count()
+    .eachBefore((node: TreeNode<T>) => {
+      let p = node;
+      let indent = "";
+      if (p.parent) {
+        indent = (hasFollowingSibling(p) ? "├── " : "└── ") + indent;
+        while ((p = p.parent!).parent) {
+          indent = (hasFollowingSibling(p) ? "│   " : "    ") + indent;
+        }
+      }
+      lines.push([
+        indent || "┌",
+        `${node.id.split("/").pop()?.replace(/\?$/, "")}`,
+        node.height ? ` (${node.value.toLocaleString("en-US")} page${node.value === 1 ? "" : "s"})` : "",
+        node
+      ]);
+    });
+  const width =
+    (max(lines, ([indent, name, description]) => indent.length + description.length + stringLength(name)) || 0) + 1;
+  return lines.map(([indent, name, description, node]) => [
+    indent,
+    name,
+    description + " ".repeat(width - stringLength(name) - description.length - indent.length),
+    node
+  ]);
+}
+
+/** Counts the number of graphemes in the specified string. */
+function stringLength(string: string): number {
+  return [...new Intl.Segmenter().segment(string)].length;
+}
+
+function hasFollowingSibling(node: TreeNode<unknown>): boolean {
+  return node.parent != null && node.parent.children!.indexOf(node) < node.parent.children!.length - 1;
+}
+
+function treeOrder(a: TreeNode<unknown>, b: TreeNode<unknown>): number {
+  return ascending(!a.children, !b.children) || ascending(a.id, b.id);
+}

test/format-test.ts

@@ -24,19 +24,22 @@ describe("formatLocaleDate", () => {
 
 describe("formatByteSize", () => {
   it("returns a human-readable byte size", () => {
-    assert.strictEqual(formatByteSize(0), "0 bytes");
-    assert.strictEqual(formatByteSize(500), "500 bytes");
-    assert.strictEqual(formatByteSize(1200), "1200 bytes");
-    assert.strictEqual(formatByteSize(1_500), "1.5 KB");
-    assert.strictEqual(formatByteSize(3_000), "3.0 KB");
-    assert.strictEqual(formatByteSize(400_000), "400 KB");
-    assert.strictEqual(formatByteSize(50_000_000), "50.0 MB");
-    assert.strictEqual(formatByteSize(1_250_000_000), "1250 MB");
+    assert.strictEqual(formatByteSize(0), "0 B");
+    assert.strictEqual(formatByteSize(500), "500 B");
+    assert.strictEqual(formatByteSize(999), "999 B");
+    assert.strictEqual(formatByteSize(999.49), "999 B");
+    assert.strictEqual(formatByteSize(999.51), "1 kB");
+    assert.strictEqual(formatByteSize(1200), "1.2 kB");
+    assert.strictEqual(formatByteSize(1_500), "1.5 kB");
+    assert.strictEqual(formatByteSize(3_000), "3 kB");
+    assert.strictEqual(formatByteSize(400_000), "400 kB");
+    assert.strictEqual(formatByteSize(50_000_000), "50 MB");
+    assert.strictEqual(formatByteSize(1_250_000_000), "1.25 GB");
     assert.strictEqual(formatByteSize(1_500_000_000), "1.5 GB");
-    assert.strictEqual(formatByteSize(3_000_000_000), "3.0 GB");
+    assert.strictEqual(formatByteSize(3_000_000_000), "3 GB");
     assert.strictEqual(formatByteSize(600_000_000_000), "600 GB");
     assert.strictEqual(formatByteSize(60_598_160), "60.6 MB");
     assert.strictEqual(formatByteSize(60_598_160_000), "60.6 GB");
-    assert.strictEqual(formatByteSize(60_598_160_000_000), "60.6 TB");
+    assert.strictEqual(formatByteSize(60_598_160_000_000), "60,600 GB");
   });
 });

test/markdown-test.ts

@@ -8,7 +8,7 @@ import {isEnoent} from "../src/error.js";
 import type {MarkdownPage} from "../src/markdown.js";
 import {makeLinkNormalizer, parseMarkdown} from "../src/markdown.js";
 
-const {md} = normalizeConfig();
+const {md} = normalizeConfig({root: "docs"});
 
 describe("parseMarkdown(input)", () => {
   const inputRoot = "test/input";