Skip to content

Commit 3f9d576

Browse files
mythmonmythmon
authored
use file manifests during uploads (#1278)
* use file manifests during uploads * prettier * review feedback
1 parent db5fd18 commit 3f9d576

File tree

7 files changed

+275
-63
lines changed

7 files changed

+275
-63
lines changed

‎src/bin/observable.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ try {
108108
}
109109
case "deploy": {
110110
const {
111-
values: {config, root, message, build, "no-build": noBuild}
111+
values: {config, root, message, build}
112112
} = helpArgs(command, {
113113
options: {
114114
...CONFIG_OPTION,
@@ -130,7 +130,7 @@ try {
130130
deploy.deploy({
131131
config: await readConfig(config, root),
132132
message,
133-
force: build ? "build" : noBuild ? "deploy" : null
133+
force: build === true ? "build" : build === false ? "deploy" : null
134134
})
135135
);
136136
break;

‎src/deploy.ts

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
import {createHash} from "node:crypto";
12
import type {Stats} from "node:fs";
2-
import {stat} from "node:fs/promises";
3+
import {readFile, stat} from "node:fs/promises";
34
import {join} from "node:path/posix";
45
import * as clack from "@clack/prompts";
56
import wrapAnsi from "wrap-ansi";
@@ -16,6 +17,7 @@ import type {AuthEffects} from "./observableApiAuth.js";
1617
import {defaultEffects as defaultAuthEffects, formatUser, loginInner, validWorkspaces} from "./observableApiAuth.js";
1718
import {ObservableApiClient} from "./observableApiClient.js";
1819
import type {
20+
DeployManifestFile,
1921
GetCurrentUserResponse,
2022
GetDeployResponse,
2123
GetProjectResponse,
@@ -359,23 +361,61 @@ export async function deploy(
359361
throw error;
360362
}
361363

362-
// Upload the files
363-
const uploadSpinner = clack.spinner();
364-
uploadSpinner.start("");
364+
const progressSpinner = clack.spinner();
365+
progressSpinner.start("");
366+
367+
// upload a manifest before uploading the files
368+
progressSpinner.message("Hashing local files");
369+
const manifestFileInfo: DeployManifestFile[] = [];
370+
await runAllWithConcurrencyLimit(buildFilePaths, async (path) => {
371+
const fullPath = join(config.output, path);
372+
const statInfo = await stat(fullPath);
373+
const hash = createHash("sha512")
374+
.update(await readFile(fullPath))
375+
.digest("base64");
376+
manifestFileInfo.push({path, size: statInfo.size, hash});
377+
});
378+
progressSpinner.message("Sending file manifest to server");
379+
const instructions = await apiClient.postDeployManifest(deployId, manifestFileInfo);
380+
const fileErrors: {path: string; detail: string | null}[] = [];
381+
for (const fileInstruction of instructions.files) {
382+
if (fileInstruction.status === "error") {
383+
fileErrors.push({path: fileInstruction.path, detail: fileInstruction.detail});
384+
}
385+
}
386+
if (fileErrors.length) {
387+
clack.log.error(
388+
"The server rejected some files from the upload:\n\n" +
389+
fileErrors.map(({path, detail}) => ` - ${path} - ${detail ? `(${detail})` : "no details"}`).join("\n")
390+
);
391+
}
392+
if (instructions.status === "error" || fileErrors.length) {
393+
throw new CliError(`Server rejected deploy manifest: ${instructions.detail ?? "no details"}`);
394+
}
395+
const filesToUpload: string[] = instructions.files
396+
.filter((instruction) => instruction.status === "upload")
397+
.map((instruction) => instruction.path);
365398

399+
// Upload the files
366400
const rateLimiter = new RateLimiter(5);
367-
const waitForRateLimit = buildFilePaths.length <= 300 ? async () => {} : () => rateLimiter.wait();
401+
const waitForRateLimit = filesToUpload.length <= 300 ? async () => {} : () => rateLimiter.wait();
368402

369403
await runAllWithConcurrencyLimit(
370-
buildFilePaths,
404+
filesToUpload,
371405
async (path, i) => {
372406
await waitForRateLimit();
373-
uploadSpinner.message(`${i + 1} / ${buildFilePaths!.length} ${path.slice(0, effects.outputColumns - 10)}`);
407+
progressSpinner.message(
408+
`${i + 1} / ${filesToUpload.length} ${faint("uploading")} ${path.slice(0, effects.outputColumns - 17)}`
409+
);
374410
await apiClient.postDeployFile(deployId, join(config.output, path), path);
375411
},
376412
{maxConcurrency}
377413
);
378-
uploadSpinner.stop(`${buildFilePaths.length} uploaded`);
414+
progressSpinner.stop(
415+
`${filesToUpload.length} uploaded, ${buildFilePaths.length - filesToUpload.length} unchanged, ${
416+
buildFilePaths.length
417+
} total.`
418+
);
379419

380420
// Mark the deploy as uploaded
381421
await apiClient.postDeployUploaded(deployId);

‎src/observableApiClient.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,14 @@ export class ObservableApiClient {
196196
}
197197
}
198198

199+
async postDeployManifest(deployId: string, files: DeployManifestFile[]): Promise<PostDeployManifestResponse> {
200+
return await this._fetch<PostDeployManifestResponse>(new URL(`/cli/deploy/${deployId}/manifest`, this._apiOrigin), {
201+
method: "POST",
202+
headers: {"content-type": "application/json"},
203+
body: JSON.stringify({files})
204+
});
205+
}
206+
199207
async postDeployUploaded(deployId: string): Promise<DeployInfo> {
200208
return await this._fetch<DeployInfo>(new URL(`/cli/deploy/${deployId}/uploaded`, this._apiOrigin), {
201209
method: "POST",
@@ -305,3 +313,19 @@ export interface GetDeployResponse {
305313
status: string;
306314
url: string;
307315
}
316+
317+
export interface DeployManifestFile {
318+
path: string;
319+
size: number;
320+
hash: string;
321+
}
322+
323+
export interface PostDeployManifestResponse {
324+
status: "ok" | "error";
325+
detail: string | null;
326+
files: {
327+
path: string;
328+
status: "upload" | "skip" | "error";
329+
detail: string | null;
330+
}[];
331+
}

‎src/tty.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export const defaultEffects: TtyEffects = {
3333
outputColumns: Math.min(80, process.stdout.columns ?? 80)
3434
};
3535

36-
function stripColor(s: string): string {
36+
export function stripColor(s: string): string {
3737
// eslint-disable-next-line no-control-regex
3838
return s.replace(/\x1b\[[0-9;]*m/g, "");
3939
}

‎test/deploy-test.ts

Lines changed: 88 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type {ObservableApiClientOptions} from "../src/observableApiClient.js";
1111
import type {GetCurrentUserResponse} from "../src/observableApiClient.js";
1212
import {ObservableApiClient} from "../src/observableApiClient.js";
1313
import type {DeployConfig} from "../src/observableApiConfig.js";
14+
import {stripColor} from "../src/tty.js";
1415
import {MockAuthEffects} from "./mocks/authEffects.js";
1516
import {TestClackEffects} from "./mocks/clack.js";
1617
import {MockConfigEffects} from "./mocks/configEffects.js";
@@ -178,11 +179,11 @@ describe("deploy", () => {
178179
.handleGetCurrentUser()
179180
.handleGetProject(DEPLOY_CONFIG)
180181
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
181-
.handlePostDeployFile({deployId, clientName: "index.html"})
182-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
183-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
184-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
185-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
182+
.expectFileUpload({deployId, path: "index.html"})
183+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
184+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
185+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
186+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
186187
.handlePostDeployUploaded({deployId})
187188
.handleGetDeploy({deployId, deployStatus: "uploaded"})
188189
.start();
@@ -206,11 +207,11 @@ describe("deploy", () => {
206207
.handleGetProject({...DEPLOY_CONFIG, title: oldTitle})
207208
.handleUpdateProject({projectId: DEPLOY_CONFIG.projectId, title: TEST_CONFIG.title!})
208209
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
209-
.handlePostDeployFile({deployId, clientName: "index.html"})
210-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
211-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
212-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
213-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
210+
.expectFileUpload({deployId, path: "index.html"})
211+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
212+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
213+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
214+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
214215
.handlePostDeployUploaded({deployId})
215216
.handleGetDeploy({deployId})
216217
.start();
@@ -234,11 +235,11 @@ describe("deploy", () => {
234235
.handleGetCurrentUser()
235236
.handleGetProject(deployConfig)
236237
.handlePostDeploy({projectId: deployConfig.projectId, deployId})
237-
.handlePostDeployFile({deployId, clientName: "index.html"})
238-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
239-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
240-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
241-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
238+
.expectFileUpload({deployId, path: "index.html"})
239+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
240+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
241+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
242+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
242243
.handlePostDeployUploaded({deployId})
243244
.handleGetDeploy({deployId})
244245
.start();
@@ -261,11 +262,11 @@ describe("deploy", () => {
261262
})
262263
.handlePostProject({projectId: DEPLOY_CONFIG.projectId})
263264
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
264-
.handlePostDeployFile({deployId, clientName: "index.html"})
265-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
266-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
267-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
268-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
265+
.expectFileUpload({deployId, path: "index.html"})
266+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
267+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
268+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
269+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
269270
.handlePostDeployUploaded({deployId})
270271
.handleGetDeploy({deployId})
271272
.start();
@@ -472,6 +473,7 @@ describe("deploy", () => {
472473
.handleGetCurrentUser()
473474
.handleGetProject(DEPLOY_CONFIG)
474475
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
476+
.handlePostDeployManifest({deployId, files: [{deployId, path: "index.html", action: "upload"}]})
475477
.handlePostDeployFile({deployId, status: 500})
476478
.start();
477479
const effects = new MockDeployEffects({deployConfig: DEPLOY_CONFIG});
@@ -495,11 +497,11 @@ describe("deploy", () => {
495497
.handleGetCurrentUser()
496498
.handleGetProject(DEPLOY_CONFIG)
497499
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
498-
.handlePostDeployFile({deployId, clientName: "index.html"})
499-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
500-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
501-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
502-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
500+
.expectFileUpload({deployId, path: "index.html"})
501+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
502+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
503+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
504+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
503505
.handlePostDeployUploaded({deployId, status: 500})
504506
.start();
505507
const effects = new MockDeployEffects({deployConfig: DEPLOY_CONFIG});
@@ -612,11 +614,11 @@ describe("deploy", () => {
612614
projectId: newProjectId
613615
})
614616
.handlePostDeploy({projectId: newProjectId, deployId})
615-
.handlePostDeployFile({deployId, clientName: "index.html"})
616-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
617-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
618-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
619-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
617+
.expectFileUpload({deployId, path: "index.html"})
618+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
619+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
620+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
621+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
620622
.handlePostDeployUploaded({deployId})
621623
.handleGetDeploy({deployId})
622624
.start();
@@ -700,11 +702,11 @@ describe("deploy", () => {
700702
.handleGetCurrentUser()
701703
.handleGetProject(DEPLOY_CONFIG)
702704
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
703-
.handlePostDeployFile({deployId, clientName: "index.html"})
704-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
705-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
706-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
707-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
705+
.expectFileUpload({deployId, path: "index.html"})
706+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
707+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
708+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
709+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
708710
.handlePostDeployUploaded({deployId})
709711
.handleGetDeploy({deployId})
710712
.start();
@@ -727,11 +729,11 @@ describe("deploy", () => {
727729
.handleGetCurrentUser()
728730
.handleGetProject(DEPLOY_CONFIG)
729731
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
730-
.handlePostDeployFile({deployId, clientName: "index.html"})
731-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
732-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
733-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
734-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
732+
.expectFileUpload({deployId, path: "index.html"})
733+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
734+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
735+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
736+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
735737
.handlePostDeployUploaded({deployId})
736738
.handleGetDeploy({deployId, deployStatus: "created"})
737739
.handleGetDeploy({deployId, deployStatus: "pending"})
@@ -831,11 +833,11 @@ describe("deploy", () => {
831833
.handleGetCurrentUser()
832834
.handleGetProject(DEPLOY_CONFIG)
833835
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
834-
.handlePostDeployFile({deployId, clientName: "index.html"})
835-
.handlePostDeployFile({deployId, clientName: "_observablehq/theme-air,near-midnight.css"})
836-
.handlePostDeployFile({deployId, clientName: "_observablehq/client.js"})
837-
.handlePostDeployFile({deployId, clientName: "_observablehq/runtime.js"})
838-
.handlePostDeployFile({deployId, clientName: "_observablehq/stdlib.js"})
836+
.expectFileUpload({deployId, path: "index.html"})
837+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css"})
838+
.expectFileUpload({deployId, path: "_observablehq/client.js"})
839+
.expectFileUpload({deployId, path: "_observablehq/runtime.js"})
840+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js"})
839841
.handlePostDeployUploaded({deployId})
840842
.handleGetDeploy({deployId})
841843
.start();
@@ -860,6 +862,47 @@ describe("deploy", () => {
860862

861863
effects.close();
862864
});
865+
866+
it("will skip file uploads if instructed by the server", async () => {
867+
const deployId = "deploy456";
868+
getCurrentObservableApi()
869+
.handleGetCurrentUser()
870+
.handleGetProject(DEPLOY_CONFIG)
871+
.handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId})
872+
.expectFileUpload({deployId, path: "index.html", action: "upload"})
873+
.expectFileUpload({deployId, path: "_observablehq/theme-air,near-midnight.css", action: "skip"})
874+
.expectFileUpload({deployId, path: "_observablehq/client.js", action: "skip"})
875+
.expectFileUpload({deployId, path: "_observablehq/runtime.js", action: "skip"})
876+
.expectFileUpload({deployId, path: "_observablehq/stdlib.js", action: "skip"})
877+
.handlePostDeployUploaded({deployId})
878+
.handleGetDeploy({deployId, deployStatus: "uploaded"})
879+
.start();
880+
881+
const effects = new MockDeployEffects({
882+
deployConfig: DEPLOY_CONFIG,
883+
fixedInputStatTime: new Date("2024-03-09"),
884+
fixedOutputStatTime: new Date("2024-03-10")
885+
});
886+
effects.clack.inputs = ["fix some bugs"]; // "what changed?"
887+
await deploy(TEST_OPTIONS, effects);
888+
889+
effects.close();
890+
891+
// first is the upload spinner, second is the server processing spinner
892+
assert.equal(effects.clack.spinners.length, 2, JSON.stringify(effects.clack.spinners, null, 2));
893+
const events = effects.clack.spinners[0]._events.map((e) => {
894+
const r: {method: string; message?: string} = {method: e.method};
895+
if (e.message) r.message = stripColor(e.message);
896+
return r;
897+
});
898+
assert.deepEqual(events, [
899+
{method: "start"},
900+
{method: "message", message: "Hashing local files"},
901+
{method: "message", message: "Sending file manifest to server"},
902+
{method: "message", message: "1 / 1 uploading index.html"},
903+
{method: "stop", message: "1 uploaded, 4 unchanged, 5 total."}
904+
]);
905+
});
863906
});
864907

865908
describe("promptDeployTarget", () => {

0 commit comments

Comments
 (0)