Skip to content

Commit 00639a0

Browse files
mythmonFilmbostock
authored
deploy from output directory instead of doing an in-memory build (#1042)
* deploy from output directory instead of doing an in-memory build * handle 429 retries * add rate limiter to projects with more than 300 files * transpose --build to --if-missing and --if-stale * prettier * lower the upload rate limit to better match the server * lower deploy concurrency in upload error test * normalize file path; deterministic file resolution * getSourceFileHash * fix archives.win32 snapshot? * change --ifMissing to -if-missing in some help output Co-authored-by: Mike Bostock <[email protected]> * move deferred into test/ * fix import order * maxConcurrency=1 in all deploy tests * Revert "maxConcurrency=1 in all deploy tests" This reverts commit 10bba90. * mock jsdelivr in getResolvers tests --------- Co-authored-by: Philippe Rivière <[email protected]> Co-authored-by: Mike Bostock <[email protected]>
1 parent efd8ba0 commit 00639a0

File tree

11 files changed

+505
-112
lines changed

11 files changed

+505
-112
lines changed

src/bin/observable.ts

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,19 +105,46 @@ try {
105105
break;
106106
}
107107
case "deploy": {
108+
const missingDescription = "one of 'build', 'cancel', or 'prompt' (the default)";
109+
const staleDescription = "one of 'build', 'cancel', 'deploy', or 'prompt' (the default)";
108110
const {
109-
values: {config, root, message}
111+
values: {config, root, message, "if-stale": ifStale, "if-missing": ifMissing}
110112
} = helpArgs(command, {
111113
options: {
112114
...CONFIG_OPTION,
113115
message: {
114116
type: "string",
115117
short: "m"
118+
},
119+
"if-stale": {
120+
type: "string",
121+
description: `What to do if the output directory is stale: ${staleDescription}`
122+
},
123+
"if-missing": {
124+
type: "string",
125+
description: `What to do if the output directory is missing: ${missingDescription}`
116126
}
117127
}
118128
});
129+
if (ifStale && ifStale !== "prompt" && ifStale !== "build" && ifStale !== "cancel" && ifStale !== "deploy") {
130+
console.log(`Invalid --if-stale option: ${ifStale}, expected ${staleDescription}`);
131+
process.exit(1);
132+
}
133+
if (ifMissing && ifMissing !== "prompt" && ifMissing !== "build" && ifMissing !== "cancel") {
134+
console.log(`Invalid --if-missing option: ${ifMissing}, expected ${missingDescription}`);
135+
process.exit(1);
136+
}
137+
if (!process.stdin.isTTY && (ifStale === "prompt" || ifMissing === "prompt")) {
138+
throw new CliError("Cannot prompt for input in non-interactive mode");
139+
}
140+
119141
await import("../deploy.js").then(async (deploy) =>
120-
deploy.deploy({config: await readConfig(config, root), message})
142+
deploy.deploy({
143+
config: await readConfig(config, root),
144+
message,
145+
ifBuildMissing: (ifMissing ?? "prompt") as "prompt" | "build" | "cancel",
146+
ifBuildStale: (ifStale ?? "prompt") as "prompt" | "build" | "cancel" | "deploy"
147+
})
121148
);
122149
break;
123150
}
@@ -235,10 +262,25 @@ try {
235262
process.exit(1);
236263
}
237264

265+
type DescribableParseArgsConfig = ParseArgsConfig & {
266+
options?: {
267+
[longOption: string]: {
268+
type: "string" | "boolean";
269+
multiple?: boolean | undefined;
270+
short?: string | undefined;
271+
default?: string | boolean | string[] | boolean[] | undefined;
272+
description?: string;
273+
};
274+
};
275+
};
276+
238277
// A wrapper for parseArgs that adds --help functionality with automatic usage.
239278
// TODO It’d be nicer nice if we could change the return type to denote
240279
// arguments with default values, and to enforce required arguments, if any.
241-
function helpArgs<T extends ParseArgsConfig>(command: string | undefined, config: T): ReturnType<typeof parseArgs<T>> {
280+
function helpArgs<T extends DescribableParseArgsConfig>(
281+
command: string | undefined,
282+
config: T
283+
): ReturnType<typeof parseArgs<T>> {
242284
let result: ReturnType<typeof parseArgs<T>>;
243285
try {
244286
result = parseArgs<T>({
@@ -259,6 +301,16 @@ function helpArgs<T extends ParseArgsConfig>(command: string | undefined, config
259301
.map(([name, {default: def}]) => ` [--${name}${def === undefined ? "" : `=${def}`}]`)
260302
.join("")}`
261303
);
304+
if (Object.values(config.options ?? {}).some((spec) => spec.description)) {
305+
console.log();
306+
for (const [long, spec] of Object.entries(config.options ?? {})) {
307+
if (spec.description) {
308+
const left = ` ${spec.short ? `-${spec.short}, ` : ""}--${long}`.padEnd(20);
309+
console.log(`${left}${spec.description}`);
310+
}
311+
}
312+
console.log();
313+
}
262314
process.exit(0);
263315
}
264316
return result;

src/concurrency.ts

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import os from "node:os";
2+
3+
/** Double the number of CPUs, up to 8.
4+
*
5+
* This number of chosen for IO-bound tasks, with
6+
* the expectation that the other side will be a server that wouldn't appreciate
7+
* unbounded concurrency. */
8+
//`os.cpus()` can return an empty array, so use a minimum of 2 as well.
9+
const DEFAULT_CONCURRENCY = Math.max(Math.min(os.cpus().length * 2, 8), 2);
10+
11+
export async function runAllWithConcurrencyLimit<T>(
12+
tasks: Iterable<T>,
13+
worker: (task: T, index: number) => Promise<void>,
14+
{maxConcurrency = DEFAULT_CONCURRENCY}: {maxConcurrency?: number} = {}
15+
) {
16+
const queue = Array.from(tasks);
17+
const pending = new Set();
18+
let index = 0;
19+
20+
while (queue.length) {
21+
if (pending.size >= maxConcurrency) {
22+
await Promise.race(pending);
23+
continue;
24+
}
25+
26+
const item = queue.shift();
27+
if (!item) throw new Error("unexpectedly out of items");
28+
const promise = worker(item, index++);
29+
pending.add(promise);
30+
promise.finally(() => pending.delete(promise));
31+
}
32+
33+
await Promise.all(pending);
34+
}
35+
36+
export class RateLimiter {
37+
// This works by chaining together promises, one for each call to `this.wait`.
38+
// This implicitly forms a queue of callers. The important thing is that we
39+
// never have two of this function's `setTimeout`s running concurrently, since
40+
// that could cause us to exceed the rate limit.
41+
42+
private _nextTick: Promise<void>;
43+
44+
constructor(private ratePerSecond: number) {
45+
this._nextTick = Promise.resolve();
46+
}
47+
48+
/** Wait long enough to avoid going over the rate limit. */
49+
async wait() {
50+
const nextTick = this._nextTick;
51+
this._nextTick = nextTick.then(() => new Promise((res) => setTimeout(res, 1000 / this.ratePerSecond)));
52+
await nextTick;
53+
}
54+
}

0 commit comments

Comments
 (0)