Skip to content

Commit 2c037b7

Browse files
mbostockmbostock
committed
baked DatabaseClient
1 parent 7c2c044 commit 2c037b7

File tree

4 files changed

+164
-23
lines changed

4 files changed

+164
-23
lines changed

‎src/runtime/index.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ export * from "./display.js";
88
export * from "./inspect.js";
99
export * from "./stdlib/index.js";
1010

11+
export type * from "./stdlib/databaseClient.js";
12+
export type * from "./stdlib/fileAttachment.js";
13+
export {DatabaseClient} from "./stdlib/databaseClient.js";
14+
export {FileAttachment} from "./stdlib/fileAttachment.js";
15+
1116
export const runtime = Object.assign(new Runtime({...library, __ojs_runtime: () => runtime}), {fileAttachments});
1217
export const main = (runtime as typeof runtime & {main: Module}).main = runtime.module();
1318

‎src/runtime/stdlib/databaseClient.ts

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/* eslint-disable @typescript-eslint/no-explicit-any */
2+
3+
/** A serializable value that can be interpolated into a query. */
4+
export type QueryParam = any;
5+
6+
/** @see https://observablehq.com/@observablehq/database-client-specification#%C2%A71 */
7+
export type QueryResult = Record<string, any>[] & {schema: ColumnSchema[]; date: Date};
8+
9+
/** @see https://observablehq.com/@observablehq/database-client-specification#%C2%A72.2 */
10+
export interface ColumnSchema {
11+
/** The name of the column. */
12+
name: string;
13+
/** The type of the column. */
14+
type:
15+
| "string"
16+
| "number"
17+
| "integer"
18+
| "bigint"
19+
| "date"
20+
| "boolean"
21+
| "object"
22+
| "array"
23+
| "buffer"
24+
| "other";
25+
/** If present, the nullability of the column is known. */
26+
nullable?: boolean;
27+
}
28+
29+
export interface QueryOptionsSpec {
30+
/**
31+
* If specified, query results are at least as fresh as the specified date.
32+
* If null, results are as fresh as possible (never pulled from the cache).
33+
*/
34+
since?: Date | string | number | null;
35+
/**
36+
* If specified, query results must be younger than the specified number of seconds.
37+
* If null, results are as fresh as possible (never pulled from the cache).
38+
*/
39+
maxAge?: number | null;
40+
}
41+
42+
export interface QueryOptions extends QueryOptionsSpec {
43+
since?: Date | null;
44+
maxAge?: number | null;
45+
}
46+
47+
export interface DatabaseClient {
48+
readonly name: string;
49+
readonly options: QueryOptions;
50+
sql(strings: string[], ...params: QueryParam[]): Promise<QueryResult>;
51+
}
52+
53+
export const DatabaseClient = (name: string, options?: QueryOptionsSpec): DatabaseClient => {
54+
if (!/^[\w-]+$/.test(name)) throw new Error(`invalid database: ${name}`);
55+
return new DatabaseClientImpl(name, normalizeQueryOptions(options));
56+
};
57+
58+
function normalizeQueryOptions({since, maxAge}: QueryOptionsSpec = {}): QueryOptions {
59+
const options: QueryOptions = {};
60+
if (since !== undefined) options.since = since == null ? since : new Date(since);
61+
if (maxAge !== undefined) options.maxAge = maxAge == null ? maxAge : Number(maxAge);
62+
return options;
63+
}
64+
65+
class DatabaseClientImpl implements DatabaseClient {
66+
readonly name!: string;
67+
readonly options!: QueryOptions;
68+
constructor(name: string, options: QueryOptions) {
69+
Object.defineProperties(this, {
70+
name: {value: name, enumerable: true},
71+
options: {value: options, enumerable: true}
72+
});
73+
}
74+
async sql(strings: string[], ...params: QueryParam[]): Promise<QueryResult> {
75+
const path = `.observable/cache/${this.name}/${await hash(strings, ...params)}.json`;
76+
const response = await fetch(path);
77+
if (!response.ok) throw new Error(`failed to fetch: ${path}`);
78+
const {rows, schema, date} = await response.json();
79+
rows.schema = schema;
80+
rows.date = new Date(date);
81+
revive(rows);
82+
return rows;
83+
}
84+
}
85+
86+
async function hash(strings: string[], ...params: unknown[]): Promise<string> {
87+
const encoded = new TextEncoder().encode(JSON.stringify([strings, ...params]));
88+
const buffer = await crypto.subtle.digest("SHA-256", encoded);
89+
const int = new Uint8Array(buffer).reduce((i, byte) => (i << 8n) | BigInt(byte), 0n);
90+
return int.toString(36).padStart(24, "0").slice(0, 24);
91+
}
92+
93+
function revive(rows: QueryResult): void {
94+
for (const column of rows.schema) {
95+
switch (column.type) {
96+
case "bigint": {
97+
const {name} = column;
98+
for (const row of rows) {
99+
const value = row[name];
100+
if (value == null) continue;
101+
row[name] = BigInt(value);
102+
}
103+
break;
104+
}
105+
case "date": {
106+
const {name} = column;
107+
for (const row of rows) {
108+
const value = row[name];
109+
if (value == null) continue;
110+
row[name] = new Date(value);
111+
}
112+
break;
113+
}
114+
}
115+
}
116+
}
117+
118+
DatabaseClient.hash = hash;
119+
DatabaseClient.revive = revive;
120+
DatabaseClient.prototype = DatabaseClientImpl.prototype; // instanceof
121+
Object.defineProperty(DatabaseClientImpl, "name", {value: "DatabaseClient"}); // prevent mangling

‎src/runtime/stdlib/fileAttachment.ts

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,44 +2,62 @@
22
const files = new Map<string, FileAttachmentImpl>();
33

44
export type DsvOptions = {delimiter?: string; array?: boolean; typed?: boolean};
5-
export type DsvResult = any[] & {columns: string[]};
5+
export type DsvResult = (Record<string, any>[] | any[][]) & {columns: string[]};
66

77
export interface FileAttachment {
8+
/** The URL of the file. */
9+
href: string;
10+
/** The name of the file (not including the path), such as "test.csv". */
811
name: string;
12+
/** The MIME type, such as "text/csv". */
913
mimeType: string;
10-
href: string;
11-
lastModified: number | undefined;
12-
size: number | undefined;
14+
/** The time this file was most-recently modified, as milliseconds since epoch, if known. */
15+
lastModified?: number;
16+
/** The size of this file in bytes, if known. */
17+
size?: number;
1318
/** @deprecated use FileAttachment.href instead */
1419
url(): Promise<string>;
20+
/** Returns the contents of this file as a Blob. */
1521
blob(): Promise<Blob>;
22+
/** Returns the contents of this file as an ArrayBuffer. */
1623
arrayBuffer(): Promise<ArrayBuffer>;
24+
/** Returns the contents of this file as a string with the given encoding. */
1725
text(encoding?: string): Promise<string>;
26+
/** Returns the contents of this file as JSON. */
1827
json(): Promise<any>;
28+
/** Returns a byte stream to the contents of this file. */
1929
stream(): Promise<ReadableStream<Uint8Array<ArrayBufferLike>>>;
30+
/** Returns the contents of this file as delimiter-separated values. */
2031
dsv(options?: DsvOptions): Promise<DsvResult>;
32+
/** Returns the contents of this file as comma-separated values. */
2133
csv(options?: Omit<DsvOptions, "delimiter">): Promise<DsvResult>;
34+
/** Returns the contents of this file as tab-separated values. */
2235
tsv(options?: Omit<DsvOptions, "delimiter">): Promise<DsvResult>;
36+
/** Returns the contents of this file as an image. */
2337
image(props?: Partial<HTMLImageElement>): Promise<HTMLImageElement>;
38+
/** Returns the contents of this Arrow IPC file as an Apache Arrow table. */
2439
arrow(): Promise<any>;
40+
/** Returns the contents of this file as an Arquero table. */
2541
arquero(options?: any): Promise<any>;
42+
/** Returns the contents of this Parquet file as an Apache Arrow table. */
2643
parquet(): Promise<any>;
44+
/** Returns the contents of this file as an XML document. */
2745
xml(mimeType?: DOMParserSupportedType): Promise<Document>;
46+
/** Returns the contents of this file as an HTML document. */
2847
html(): Promise<Document>;
2948
}
3049

31-
export function FileAttachment(name: string, base = document.baseURI): FileAttachment {
32-
if (new.target !== undefined) throw new TypeError("FileAttachment is not a constructor");
50+
export const FileAttachment = (name: string, base = document.baseURI): FileAttachment => {
3351
const href = new URL(name, base).href;
3452
let file = files.get(href);
3553
if (!file) {
3654
file = new FileAttachmentImpl(href, name.split("/").pop()!);
3755
files.set(href, file);
3856
}
3957
return file;
40-
}
58+
};
4159

42-
async function remote_fetch(file: FileAttachment) {
60+
async function fetchFile(file: FileAttachment): Promise<Response> {
4361
const response = await fetch(file.href);
4462
if (!response.ok) throw new Error(`Unable to load file: ${file.name}`);
4563
return response;
@@ -51,12 +69,7 @@ export abstract class AbstractFile implements FileAttachment {
5169
lastModified!: number | undefined;
5270
size!: number | undefined;
5371
abstract href: string;
54-
constructor(
55-
name: string,
56-
mimeType = guessMimeType(name),
57-
lastModified?: number,
58-
size?: number
59-
) {
72+
constructor(name: string, mimeType = guessMimeType(name), lastModified?: number, size?: number) {
6073
Object.defineProperties(this, {
6174
name: {value: `${name}`, enumerable: true},
6275
mimeType: {value: `${mimeType}`, enumerable: true},
@@ -68,21 +81,21 @@ export abstract class AbstractFile implements FileAttachment {
6881
return this.href;
6982
}
7083
async blob(): Promise<Blob> {
71-
return (await remote_fetch(this)).blob();
84+
return (await fetchFile(this)).blob();
7285
}
7386
async arrayBuffer(): Promise<ArrayBuffer> {
74-
return (await remote_fetch(this)).arrayBuffer();
87+
return (await fetchFile(this)).arrayBuffer();
7588
}
7689
async text(encoding?: string): Promise<string> {
7790
return encoding === undefined
78-
? (await remote_fetch(this)).text()
91+
? (await fetchFile(this)).text()
7992
: new TextDecoder(encoding).decode(await this.arrayBuffer());
8093
}
8194
async json(): Promise<any> {
82-
return (await remote_fetch(this)).json();
95+
return (await fetchFile(this)).json();
8396
}
8497
async stream(): Promise<ReadableStream<Uint8Array<ArrayBufferLike>>> {
85-
return (await remote_fetch(this)).body!;
98+
return (await fetchFile(this)).body!;
8699
}
87100
async dsv({delimiter = ",", array = false, typed = false} = {}): Promise<DsvResult> {
88101
const [text, d3] = await Promise.all([this.text(), import("npm:d3-dsv")]);
@@ -108,7 +121,7 @@ export abstract class AbstractFile implements FileAttachment {
108121
});
109122
}
110123
async arrow(): Promise<any> {
111-
const [Arrow, response] = await Promise.all([import("npm:apache-arrow"), remote_fetch(this)]);
124+
const [Arrow, response] = await Promise.all([import("npm:apache-arrow"), fetchFile(this)]);
112125
return Arrow.tableFromIPC(response);
113126
}
114127
async arquero(options?: any): Promise<any> {

‎src/runtime/stdlib/index.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import {Mutable} from "./mutable.js";
2-
import * as Generators from "./generators/index.js";
3-
import {FileAttachment} from "./fileAttachment.js";
1+
import {DatabaseClient} from "./databaseClient.js";
42
import * as DOM from "./dom/index.js";
3+
import {FileAttachment} from "./fileAttachment.js";
4+
import * as Generators from "./generators/index.js";
5+
import {Mutable} from "./mutable.js";
56
import {Observer} from "./observer.js";
67
import * as recommendedLibraries from "./recommendedLibraries.js";
78
import {require} from "./require.js";
@@ -13,6 +14,7 @@ export const root = document.querySelector("main") ?? document.body;
1314
export const library = {
1415
now: () => Generators.now(),
1516
width: () => Generators.width(root),
17+
DatabaseClient: () => DatabaseClient,
1618
FileAttachment: () => FileAttachment,
1719
Generators: () => Generators,
1820
Mutable: () => Mutable,

0 commit comments

Comments
 (0)