Skip to content

Commit 7c33e2d

Browse files
mbostockmbostock
committed
duckdb connector
1 parent 6ec5f9f commit 7c33e2d

File tree

7 files changed

+199
-6
lines changed

7 files changed

+199
-6
lines changed

‎docs/duckdb-test.html

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<!doctype html>
2+
<notebook theme="slate">
3+
<title>Hello, DuckDB</title>
4+
<script type="text/markdown">
5+
# Hello, DuckDB
6+
7+
This is a test of the DuckDB database connector. First, let's add two numbers.
8+
</script>
9+
<script type="application/sql" hidden pinned="" database="duckdb" output="three">
10+
SELECT 1 + 2
11+
</script>
12+
<script type="text/markdown">
13+
The result is an array with a single object representing the row returned:
14+
</script>
15+
<script type="module">
16+
three
17+
</script>
18+
<script type="text/markdown">
19+
Now let's read a CSV file and pull out the top ten values.
20+
</script>
21+
<script type="application/sql" pinned="" database="duckdb" output="alphabet">
22+
SELECT * FROM 'ex/data/alphabet.csv' ORDER BY frequency DESC LIMIT 10
23+
</script>
24+
<script type="text/markdown">
25+
Can we put the numbers in a chart?
26+
</script>
27+
<script type="module" pinned="">
28+
Plot.barX(alphabet, {y: "letter", x: "frequency"}).plot()
29+
</script>
30+
<script type="text/markdown">
31+
Can we summarize a large Parquet file?
32+
</script>
33+
<script type="application/sql" pinned="" database="duckdb">
34+
SUMMARIZE 'ex/data/gaia-sample.parquet';
35+
</script>
36+
</notebook>

‎package.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
"vite": "^7.0.0"
6464
},
6565
"devDependencies": {
66+
"@duckdb/node-api": "^1.3.2-alpha.26",
6667
"@eslint/js": "^9.29.0",
6768
"@types/jsdom": "^21.1.7",
6869
"@types/markdown-it": "^14.1.2",
@@ -77,10 +78,14 @@
7778
"vitest": "^3.2.4"
7879
},
7980
"peerDependencies": {
81+
"@duckdb/node-api": "^1.3.2-alpha.26",
8082
"postgres": "^3.4.7",
8183
"snowflake-sdk": "^2.1.3"
8284
},
8385
"peerDependenciesMeta": {
86+
"@duckdb/node-api": {
87+
"optional": true
88+
},
8489
"postgres": {
8590
"optional": true
8691
},

‎src/databases/duckdb.ts

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import type {DuckDBResult, DuckDBType, Json} from "@duckdb/node-api";
2+
import {DuckDBConnection} from "@duckdb/node-api";
3+
import {BIGINT, BIT, BLOB, BOOLEAN, DATE, DOUBLE, FLOAT, HUGEINT, INTEGER, INTERVAL, SMALLINT, TIME, TIMESTAMP, TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S, TIMESTAMPTZ, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR, VARINT} from "@duckdb/node-api"; // prettier-ignore
4+
import type {DatabaseContext, DuckDBConfig, QueryTemplateFunction} from "./index.js";
5+
import type {ColumnSchema} from "../runtime/index.js";
6+
7+
export default function duckdb(
8+
_options: DuckDBConfig,
9+
context: DatabaseContext
10+
): QueryTemplateFunction {
11+
return async (strings, ...params) => {
12+
const connection = await DuckDBConnection.create();
13+
await connection.run(`SET file_search_path=$0`, [context.cwd]);
14+
const date = new Date();
15+
let result: DuckDBResult;
16+
let rows: Record<string, Json>[];
17+
try {
18+
result = await connection.run(
19+
strings.reduce((p, c, i) => `${p}$${i - 1}${c}`),
20+
params
21+
);
22+
rows = await result.getRowObjectsJson();
23+
} finally {
24+
connection.disconnectSync();
25+
}
26+
return {
27+
rows,
28+
schema: getResultSchema(result),
29+
duration: Date.now() - +date,
30+
date
31+
};
32+
};
33+
}
34+
35+
function getResultSchema(result: DuckDBResult): ColumnSchema[] {
36+
return result.columnNames().map((name, i) => ({name, type: getColumnType(result.columnType(i))}));
37+
}
38+
39+
function getColumnType(type: DuckDBType): ColumnSchema["type"] {
40+
switch (type) {
41+
case BOOLEAN:
42+
return "boolean";
43+
case BIT:
44+
case TINYINT:
45+
case SMALLINT:
46+
case INTEGER:
47+
case UTINYINT:
48+
case USMALLINT:
49+
case UINTEGER:
50+
case VARINT:
51+
return "integer";
52+
case BIGINT:
53+
case UBIGINT:
54+
case HUGEINT:
55+
case UHUGEINT:
56+
return "bigint";
57+
case FLOAT:
58+
case DOUBLE:
59+
return "number";
60+
case TIMESTAMP:
61+
case TIMESTAMP_S:
62+
case TIMESTAMP_MS:
63+
case TIMESTAMP_NS:
64+
case TIMESTAMPTZ:
65+
case DATE:
66+
return "date";
67+
case TIME:
68+
case VARCHAR:
69+
case UUID:
70+
return "string";
71+
case BLOB:
72+
return "buffer";
73+
case INTERVAL:
74+
return "array";
75+
default:
76+
return "other";
77+
}
78+
}

‎src/databases/index.ts

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import type {ColumnSchema, QueryParam} from "../runtime/index.js";
22

3-
export type DatabaseConfig = SnowflakeConfig | PostgresConfig;
3+
export type DatabaseConfig = DuckDBConfig | SnowflakeConfig | PostgresConfig;
4+
5+
export type DuckDBConfig = {
6+
type: "duckdb";
7+
};
48

59
export type SnowflakeConfig = {
610
type: "snowflake";
@@ -23,6 +27,10 @@ export type PostgresConfig = {
2327
ssl?: boolean;
2428
};
2529

30+
export type DatabaseContext = {
31+
cwd: string;
32+
};
33+
2634
export type QueryTemplateFunction = (
2735
strings: string[],
2836
...params: QueryParam[]
@@ -35,8 +43,13 @@ export type SerializableQueryResult = {
3543
date: Date;
3644
};
3745

38-
export async function getDatabase(config: DatabaseConfig): Promise<QueryTemplateFunction> {
46+
export async function getDatabase(
47+
config: DatabaseConfig,
48+
context: DatabaseContext
49+
): Promise<QueryTemplateFunction> {
3950
switch (config.type) {
51+
case "duckdb":
52+
return (await import("./duckdb.js")).default(config, context);
4053
case "snowflake":
4154
return (await import("./snowflake.js")).default(config);
4255
case "postgres":
@@ -45,3 +58,7 @@ export async function getDatabase(config: DatabaseConfig): Promise<QueryTemplate
4558
throw new Error(`unsupported database type: ${config["type"]}`);
4659
}
4760
}
61+
62+
export function isDefaultDatabase(name: string): name is "postgres" | "duckdb" {
63+
return name === "postgres" || name === "duckdb";
64+
}

‎src/lib/error.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
export function isEnoent(error: unknown): error is Error & {code: "ENOENT"} {
2+
return isSystemError(error) && error.code === "ENOENT";
3+
}
4+
5+
export function isSystemError(error: unknown): error is Error & {code: string} {
6+
return error instanceof Error && "code" in error;
7+
}

‎src/vite/observable.ts

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import type {TemplateLiteral} from "acorn";
77
import {JSDOM} from "jsdom";
88
import type {PluginOption, IndexHtmlTransformContext} from "vite";
99
import type {DatabaseConfig} from "../databases/index.js";
10-
import {getDatabase} from "../databases/index.js";
10+
import {getDatabase, isDefaultDatabase} from "../databases/index.js";
11+
import {isEnoent} from "../lib/error.js";
1112
import type {Cell, Notebook} from "../lib/notebook.js";
1213
import {deserialize} from "../lib/serialize.js";
1314
import {Sourcemap} from "../javascript/sourcemap.js";
@@ -120,13 +121,19 @@ export function observable({
120121
const cacheName = `${cell.database}-${hash}.json`;
121122
const cachePath = join(cacheDir, cacheName);
122123
if (!existsSync(cachePath)) {
124+
let config: DatabaseConfig | undefined;
123125
try {
124126
const configPath = join(dir, ".observable", "databases.json");
125127
const configStream = createReadStream(configPath, "utf-8");
126128
const configs = (await json(configStream)) as Record<string, DatabaseConfig>;
127-
const config = configs[cell.database];
128-
if (!config) throw new Error(`database not found: ${cell.database}`);
129-
const database = await getDatabase(config);
129+
config = configs[cell.database];
130+
} catch (error) {
131+
if (!isEnoent(error)) throw error;
132+
}
133+
if (isDefaultDatabase(cell.database)) config ??= {type: cell.database};
134+
if (!config) throw new Error(`database not found: ${cell.database}`);
135+
try {
136+
const database = await getDatabase(config, {cwd: dir});
130137
const results = await database.call(null, [value]);
131138
await mkdir(cacheDir, {recursive: true});
132139
await writeFile(cachePath, JSON.stringify(results));

‎yarn.lock

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -830,6 +830,49 @@
830830
enabled "2.0.x"
831831
kuler "^2.0.0"
832832

833+
"@duckdb/node-api@^1.3.2-alpha.26":
834+
version "1.3.2-alpha.26"
835+
resolved "https://registry.yarnpkg.com/@duckdb/node-api/-/node-api-1.3.2-alpha.26.tgz#3c0a69819343f565fad70ba187568e05f783bf13"
836+
integrity sha512-2gLtgJaiguAuPbXuS4NCthbIfEHo72FIF9514FKxg7sYY0gGJP1DgCpvBKdisyhWtBkNJ+jjn2p2PmRqkgZabQ==
837+
dependencies:
838+
"@duckdb/node-bindings" "1.3.2-alpha.26"
839+
840+
"@duckdb/[email protected]":
841+
version "1.3.2-alpha.26"
842+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.3.2-alpha.26.tgz#3ba44763941ae11257e1edee7879a815ec4e2ae9"
843+
integrity sha512-SioksFdehT2TWuGx6otWM2bCJ6kjD4Nq9r6xHCA4gvko1MBESIm+XhwvwcnGx2IaXTzKRZ+DPxNTPobPX18lpQ==
844+
845+
"@duckdb/[email protected]":
846+
version "1.3.2-alpha.26"
847+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.3.2-alpha.26.tgz#208ae84f4bf299742996de2d99c35732a5739d96"
848+
integrity sha512-RZ1n2Vtzit8YLl9kjfuH17nZd5MOLZoWTlDo0rv2EuIgslC7XLbxK3ZH6jiUQ+VEGRZAjXWvwtPOFNvQQ3+LhQ==
849+
850+
"@duckdb/[email protected]":
851+
version "1.3.2-alpha.26"
852+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.3.2-alpha.26.tgz#b588bae3d42eedba14adf577f173fe29a833a8ba"
853+
integrity sha512-vhHmoiQpkeXb0PbhsLEEmulG05BxO1elNhHVMLobdAwVfoGEPE3fZkH4bPgvTGywwZC7ccqMFwMjYTTOIIJvLg==
854+
855+
"@duckdb/[email protected]":
856+
version "1.3.2-alpha.26"
857+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.3.2-alpha.26.tgz#63475d3e54661f9ef9c058572233d473602e7ddf"
858+
integrity sha512-tLzo/lGu5DXrDZhogI8N9ohhByU8UzGnBFFE/0blBuZ/g5SonvTL0yq1zE/NCkLSmDSBtqibz3ehZ1rZuBBQTQ==
859+
860+
"@duckdb/[email protected]":
861+
version "1.3.2-alpha.26"
862+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.3.2-alpha.26.tgz#1c01fcde529cfc983993ef51ce22876d5d958c4c"
863+
integrity sha512-wNZmvYgnkUS3/pC9RQNO6yJeI8EANxB5LAyERJMHtMKrF6LB1nTEYmFbSlVJQ9n10qtNxKv/fF47AAAovOnG6g==
864+
865+
"@duckdb/[email protected]":
866+
version "1.3.2-alpha.26"
867+
resolved "https://registry.yarnpkg.com/@duckdb/node-bindings/-/node-bindings-1.3.2-alpha.26.tgz#cc01b752a1eb251ef24e672a8b2bcbb10b76d4e0"
868+
integrity sha512-066o5XrzesZnF/qhNlpmFwDxvIXNYQWr48bjw/ecyLfUcpcvqE0MyL1/zV5oScfcf7hSd/dSntRVzUnQmJCwXg==
869+
optionalDependencies:
870+
"@duckdb/node-bindings-darwin-arm64" "1.3.2-alpha.26"
871+
"@duckdb/node-bindings-darwin-x64" "1.3.2-alpha.26"
872+
"@duckdb/node-bindings-linux-arm64" "1.3.2-alpha.26"
873+
"@duckdb/node-bindings-linux-x64" "1.3.2-alpha.26"
874+
"@duckdb/node-bindings-win32-x64" "1.3.2-alpha.26"
875+
833876
"@esbuild/[email protected]":
834877
version "0.25.5"
835878
resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.25.5.tgz#4e0f91776c2b340e75558f60552195f6fad09f18"

0 commit comments

Comments
 (0)