Skip to content

Commit 5bc5510

Browse files
mbostockFil
mbostock
and
Fil
authored
observable convert (#764)
* observable convert * convert --output specifies the output directory, defaults to . convert the file names (scatterplot/2 should be "scatterplot,2") nicer log effects * download attachments skip documents and files we already have * touch (preserve dates) * Update bin/observable.ts * clack, --force * effects.touch * prompt to overwrite * files is optional * disable coverage for convert --------- Co-authored-by: Philippe Rivière <[email protected]>
1 parent ca4218a commit 5bc5510

File tree

4 files changed

+254
-2
lines changed

4 files changed

+254
-2
lines changed

‎.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
node-version: ${{ matrix.version }}
2020
cache: yarn
2121
- run: yarn --frozen-lockfile
22-
- run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client --lines 80 --per-file yarn test:mocha
22+
- run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client -x src/convert.ts --lines 80 --per-file yarn test:mocha
2323
- run: yarn test:tsc
2424
- run: |
2525
echo ::add-matcher::.github/eslint.json

‎bin/observable.ts

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ else if (values.help) {
6262

6363
/** Commands that use Clack formatting. When handling CliErrors, clack.outro()
6464
* will be used for these commands. */
65-
const CLACKIFIED_COMMANDS = ["create", "deploy", "login"];
65+
const CLACKIFIED_COMMANDS = ["create", "deploy", "login", "convert"];
6666

6767
try {
6868
switch (command) {
@@ -78,6 +78,7 @@ try {
7878
logout sign-out of Observable
7979
deploy deploy a project to Observable
8080
whoami check authentication status
81+
convert convert an Observable notebook to Markdown
8182
help print usage information
8283
version print the version`
8384
);
@@ -177,6 +178,17 @@ try {
177178
await import("../src/observableApiAuth.js").then((auth) => auth.whoami());
178179
break;
179180
}
181+
case "convert": {
182+
const {
183+
positionals,
184+
values: {output, force}
185+
} = helpArgs(command, {
186+
options: {output: {type: "string", default: "."}, force: {type: "boolean", short: "f"}},
187+
allowPositionals: true
188+
});
189+
await import("../src/convert.js").then((convert) => convert.convert(positionals, {output: output!, force}));
190+
break;
191+
}
180192
default: {
181193
console.error(`observable: unknown command '${command}'. See 'observable help'.`);
182194
process.exit(1);

‎src/convert.ts

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import {existsSync} from "node:fs";
2+
import {utimes, writeFile} from "node:fs/promises";
3+
import {join} from "node:path";
4+
import * as clack from "@clack/prompts";
5+
import wrapAnsi from "wrap-ansi";
6+
import type {ClackEffects} from "./clack.js";
7+
import {CliError} from "./error.js";
8+
import {prepareOutput} from "./files.js";
9+
import {getObservableUiOrigin} from "./observableApiClient.js";
10+
import {type TtyEffects, bold, cyan, faint, inverse, link, reset, defaultEffects as ttyEffects} from "./tty.js";
11+
12+
export interface ConvertEffects extends TtyEffects {
13+
clack: ClackEffects;
14+
prepareOutput(outputPath: string): Promise<void>;
15+
existsSync(outputPath: string): boolean;
16+
writeFile(outputPath: string, contents: Buffer | string): Promise<void>;
17+
touch(outputPath: string, date: Date | string | number): Promise<void>;
18+
}
19+
20+
const defaultEffects: ConvertEffects = {
21+
...ttyEffects,
22+
clack,
23+
async prepareOutput(outputPath: string): Promise<void> {
24+
await prepareOutput(outputPath);
25+
},
26+
existsSync(outputPath: string): boolean {
27+
return existsSync(outputPath);
28+
},
29+
async writeFile(outputPath: string, contents: Buffer | string): Promise<void> {
30+
await writeFile(outputPath, contents);
31+
},
32+
async touch(outputPath: string, date: Date | string | number): Promise<void> {
33+
await utimes(outputPath, (date = new Date(date)), date);
34+
}
35+
};
36+
37+
export async function convert(
38+
inputs: string[],
39+
{output, force = false, files: includeFiles = true}: {output: string; force?: boolean; files?: boolean},
40+
effects: ConvertEffects = defaultEffects
41+
): Promise<void> {
42+
const {clack} = effects;
43+
clack.intro(`${inverse(" observable convert ")}`);
44+
let n = 0;
45+
for (const input of inputs) {
46+
let start = Date.now();
47+
let s = clack.spinner();
48+
const url = resolveInput(input);
49+
const name = inferFileName(url);
50+
const path = join(output, name);
51+
if (await maybeFetch(path, force, effects)) {
52+
s.start(`Downloading ${bold(path)}`);
53+
const response = await fetch(url);
54+
if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`);
55+
const {nodes, files, update_time} = await response.json();
56+
s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`);
57+
await effects.prepareOutput(path);
58+
await effects.writeFile(path, convertNodes(nodes));
59+
await effects.touch(path, update_time);
60+
n++;
61+
if (includeFiles) {
62+
for (const file of files) {
63+
const path = join(output, file.name);
64+
if (await maybeFetch(path, force, effects)) {
65+
start = Date.now();
66+
s = clack.spinner();
67+
s.start(`Downloading ${bold(file.name)}`);
68+
const response = await fetch(file.download_url);
69+
if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`);
70+
const buffer = Buffer.from(await response.arrayBuffer());
71+
s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`);
72+
await effects.prepareOutput(path);
73+
await effects.writeFile(path, buffer);
74+
await effects.touch(path, file.create_time);
75+
n++;
76+
}
77+
}
78+
}
79+
}
80+
}
81+
clack.note(
82+
wrapAnsi(
83+
"Due to syntax differences between Observable notebooks and " +
84+
"Observable Framework, converted notebooks may require further " +
85+
"changes to function correctly. To learn more about JavaScript " +
86+
"in Framework, please read:\n\n" +
87+
reset(cyan(link("https://observablehq.com/framework/javascript"))),
88+
Math.min(64, effects.outputColumns)
89+
),
90+
"Note"
91+
);
92+
clack.outro(
93+
`${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted; ${n} file${n === 1 ? "" : "s"} written`
94+
);
95+
}
96+
97+
async function maybeFetch(path: string, force: boolean, effects: ConvertEffects): Promise<boolean> {
98+
const {clack} = effects;
99+
if (effects.existsSync(path) && !force) {
100+
const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false});
101+
if (!choice) return false;
102+
if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false});
103+
}
104+
return true;
105+
}
106+
107+
export function convertNodes(nodes): string {
108+
let string = "";
109+
let first = true;
110+
for (const node of nodes) {
111+
if (first) first = false;
112+
else string += "\n";
113+
string += convertNode(node);
114+
}
115+
return string;
116+
}
117+
118+
export function convertNode(node): string {
119+
let string = "";
120+
if (node.mode !== "md") string += `\`\`\`${node.mode}${node.pinned ? " echo" : ""}\n`;
121+
string += `${node.value}\n`;
122+
if (node.mode !== "md") string += "```\n";
123+
return string;
124+
}
125+
126+
export function inferFileName(input: string): string {
127+
return new URL(input).pathname.replace(/^\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md";
128+
}
129+
130+
export function resolveInput(input: string): string {
131+
let url: URL;
132+
if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin());
133+
else if (isSlugSpecifier(input)) url = new URL(`/${input}`, getObservableUiOrigin());
134+
else url = new URL(input);
135+
url.host = `api.${url.host}`;
136+
url.pathname = `/document${url.pathname.replace(/^\/d\//, "/")}`;
137+
return String(url);
138+
}
139+
140+
function isIdSpecifier(string: string) {
141+
return /^([0-9a-f]{16})(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string);
142+
}
143+
144+
function isSlugSpecifier(string: string) {
145+
return /^(?:@([0-9a-z_-]+))\/([0-9a-z_-]+(?:\/[0-9]+)?)(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string);
146+
}

‎test/convert-test.ts

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import assert from "node:assert";
2+
import {convertNode, convertNodes, inferFileName, resolveInput} from "../src/convert.js";
3+
4+
describe("convertNodes", () => {
5+
it("converts multiple nodes", () => {
6+
assert.strictEqual(
7+
convertNodes([
8+
{mode: "md", value: "Hello, world!"},
9+
{mode: "js", value: "1 + 2"}
10+
]),
11+
"Hello, world!\n\n```js\n1 + 2\n```\n"
12+
);
13+
});
14+
});
15+
16+
describe("convertNode", () => {
17+
it("passes through Markdown, adding a newline", () => {
18+
assert.strictEqual(convertNode({mode: "md", value: "Hello, world!"}), "Hello, world!\n");
19+
assert.strictEqual(convertNode({mode: "md", value: "# Hello, world!"}), "# Hello, world!\n");
20+
assert.strictEqual(convertNode({mode: "md", value: "# Hello, ${'world'}!"}), "# Hello, ${'world'}!\n");
21+
});
22+
it("wraps JavaScript in a fenced code block", () => {
23+
assert.strictEqual(convertNode({mode: "js", value: "1 + 2"}), "```js\n1 + 2\n```\n");
24+
});
25+
it("converts pinned to echo", () => {
26+
assert.strictEqual(convertNode({mode: "js", pinned: true, value: "1 + 2"}), "```js echo\n1 + 2\n```\n");
27+
});
28+
});
29+
30+
describe("inferFileName", () => {
31+
it("infers a suitable file name based on identifier", () => {
32+
assert.strictEqual(inferFileName("https://api.observablehq.com/document/1111111111111111"), "1111111111111111.md");
33+
});
34+
it("infers a suitable file name based on slug", () => {
35+
assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart"), "bar-chart.md");
36+
});
37+
it("handles a slug with a suffix", () => {
38+
assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart/2"), "bar-chart,2.md");
39+
});
40+
it("handles a different origin", () => {
41+
assert.strictEqual(inferFileName("https://api.example.com/document/@d3/bar-chart"), "bar-chart.md");
42+
});
43+
});
44+
45+
/* prettier-ignore */
46+
describe("resolveInput", () => {
47+
it("resolves document identifiers", () => {
48+
assert.strictEqual(resolveInput("1111111111111111"), "https://api.observablehq.com/document/1111111111111111");
49+
assert.strictEqual(resolveInput("1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef");
50+
});
51+
it("resolves document slugs", () => {
52+
assert.strictEqual(resolveInput("@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart");
53+
assert.strictEqual(resolveInput("@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2");
54+
});
55+
it("resolves document versions", () => {
56+
assert.strictEqual(resolveInput("1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123");
57+
assert.strictEqual(resolveInput("1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest");
58+
assert.strictEqual(resolveInput("1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0");
59+
assert.strictEqual(resolveInput("@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123");
60+
assert.strictEqual(resolveInput("@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest");
61+
assert.strictEqual(resolveInput("@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0");
62+
assert.strictEqual(resolveInput("@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123");
63+
assert.strictEqual(resolveInput("@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest");
64+
assert.strictEqual(resolveInput("@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0");
65+
});
66+
it("resolves urls", () => {
67+
assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef");
68+
assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123");
69+
assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest");
70+
assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0");
71+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart");
72+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123");
73+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest");
74+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0");
75+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2");
76+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123");
77+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest");
78+
assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0");
79+
});
80+
it("preserves the specified host", () => {
81+
assert.strictEqual(resolveInput("https://example.com/1234567890abcdef"), "https://api.example.com/document/1234567890abcdef");
82+
assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@123"), "https://api.example.com/document/1234567890abcdef@123");
83+
assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@latest"), "https://api.example.com/document/1234567890abcdef@latest");
84+
assert.strictEqual(resolveInput("https://example.com/1234567890abcdef~0"), "https://api.example.com/document/1234567890abcdef~0");
85+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart"), "https://api.example.com/document/@d3/bar-chart");
86+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@123"), "https://api.example.com/document/@d3/bar-chart@123");
87+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@latest"), "https://api.example.com/document/@d3/bar-chart@latest");
88+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart~0"), "https://api.example.com/document/@d3/bar-chart~0");
89+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2"), "https://api.example.com/document/@d3/bar-chart/2");
90+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@123"), "https://api.example.com/document/@d3/bar-chart/2@123");
91+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@latest"), "https://api.example.com/document/@d3/bar-chart/2@latest");
92+
assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2~0"), "https://api.example.com/document/@d3/bar-chart/2~0");
93+
});
94+
});

0 commit comments

Comments
 (0)