Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 81 additions & 1 deletion api/pkgs/@duckdb/node-api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -925,6 +925,17 @@ for (let stmtIndex = 0; stmtIndex < statementCount; stmtIndex++) {

### Control Evaluation of Tasks

DuckDB splits work into relatively short tasks. By controlling the
evaluation of these tasks explicitly, better cooperative multithreading
can be accomplished. This is especially important for maximum throughput
in the Node environment, which has a small, fixed number of threads for
running asynchronous work.
(See https://docs.libuv.org/en/stable/threadpool.html)

Below is a low-level way of controlling the evaluation of processing.
See the "startThenRead" and "startStreamThenRead" methods for helpers
that do much of this for you.

```ts
import { DuckDBPendingResultState } from '@duckdb/node-api';

Expand All @@ -935,7 +946,7 @@ async function sleep(ms) {
}

const prepared = await connection.prepare('from range(10_000_000)');
const pending = prepared.start();
const pending = prepared.startStream();
while (pending.runTask() !== DuckDBPendingResultState.RESULT_READY) {
console.log('not ready');
await sleep(1);
Expand All @@ -945,6 +956,12 @@ const result = await pending.getResult();
// ...
```

Equivalently, using `startStreamThenRead`:

```ts
const reader = await connection.startStreamThenRead(sql);
```

### Ways to run SQL

```ts
Expand Down Expand Up @@ -1030,11 +1047,51 @@ const pending = await connection.start(sql);
const pending = await connection.start(sql, values);
const pending = await connection.start(sql, values, types);

// The methods beginning with "startThenRead" provide some, but not full,
// cooperative multithreading. They use pending results to split processing
// into short tasks, but they fully materialize the result, which can
// take some time (and memory). For full cooperative multithreading,
// see the "startStreamThenRead" methods below.
const reader = await connection.startThenRead(sql);
const reader = await connection.startThenRead(sql, values);
const reader = await connection.startThenRead(sql, values, types);

const reader = await connection.startThenReadAll(sql);
const reader = await connection.startThenReadAll(sql, values);
const reader = await connection.startThenReadAll(sql, values, types);

const reader = await connection.startThenReadUntil(sql, targetRowCount);
const reader =
await connection.startThenReadUntil(sql, targetRowCount, values);
const reader =
await connection.startThenReadUntil(sql, targetRowCount, values, types);

// Create a pending, streaming result.
const pending = await connection.startStream(sql);
const pending = await connection.startStream(sql, values);
const pending = await connection.startStream(sql, values, types);

// The methods beginning with "startStreamThenRead" are the best options
// for cooperative multithreading. By creating a streaming result, they
// prevent the result from being fully materialized. By using a pending
// result, they split processing into short tasks, preventing any single
// task from occupying a thread for too long.
const reader = await connection.startStreamThenRead(sql);
const reader = await connection.startStreamThenRead(sql, values);
const reader = await connection.startStreamThenRead(sql, values, types);

const reader = await connection.startStreamThenReadAll(sql);
const reader = await connection.startStreamThenReadAll(sql, values);
const reader =
await connection.startStreamThenReadAll(sql, values, types);

const reader =
await connection.startStreamThenReadUntil(sql, targetRowCount);
const reader =
await connection.startStreamThenReadUntil(sql, targetRowCount, values);
const reader = await connection.startStreamThenReadUntil(
sql, targetRowCount, values, types);

// Create a pending result from a prepared statement.
const pending = await prepared.start();
const pending = await prepared.startStream();
Expand All @@ -1043,6 +1100,9 @@ while (pending.runTask() !== DuckDBPendingResultState.RESULT_READY) {
// optionally sleep or do other work between tasks
}

// Or, run tasks (cooperatively) until the result is ready.
await pending.runAllTasks();

// Retrieve the result. If not yet READY, will run until it is.
const result = await pending.getResult();

Expand All @@ -1058,14 +1118,30 @@ const reader = await pending.readUntil(targetRowCount);

// Asynchronously retrieve data for all rows:
const columns = await result.getColumns();
const columnsJS = await result.getColumnsJS();
const columnsJson = await result.getColumnsJson();
const columnsObject = await result.getColumnsObject();
const columnsObjectJS = await result.getColumnsObjectJS();
const columnsObjectJson = await result.getColumnsObjectJson();
const rows = await result.getRows();
const rowsJS = await result.getRowsJS();
const rowsJson = await result.getRowsJson();
const rowObjects = await result.getRowObjects();
const rowObjectsJS = await result.getRowObjectsJS();
const rowObjectsJson = await result.getRowObjectsJson();

// Asynchronous iterators can be used to retrieve partial data:
for await (const chunk of result) {
// ...
}

// Each chunk can be converted to rows:
for await (const rows of result.yieldRows()) {
// ...
}
// See also variations of "yieldRow" for returning row objects
// (instead of row arrays), and with JS, JSON, or custom conversion.

// From a reader

// First, (asynchronously) read some rows:
Expand All @@ -1075,12 +1151,16 @@ await reader.readUntil(targetRowCount);

// Then, (synchronously) get result data for the rows read:
const columns = reader.getColumns();
const columnsJS = reader.getColumnsJS();
const columnsJson = reader.getColumnsJson();
const columnsObject = reader.getColumnsObject();
const columnsObjectJS = reader.getColumnsObjectJS();
const columnsObjectJson = reader.getColumnsObjectJson();
const rows = reader.getRows();
const rowsJS = reader.getRowsJS();
const rowsJson = reader.getRowsJson();
const rowObjects = reader.getRowObjects();
const rowObjectsJS = reader.getRowObjectsJS();
const rowObjectsJson = reader.getRowObjectsJson();

// Individual values can also be read directly:
Expand Down
2 changes: 1 addition & 1 deletion api/pkgs/@duckdb/node-api/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-api",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings-darwin-arm64",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"os": [
"darwin"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings-darwin-x64",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"os": [
"darwin"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings-linux-arm64",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"os": [
"linux"
Expand Down
2 changes: 1 addition & 1 deletion bindings/pkgs/@duckdb/node-bindings-linux-x64/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings-linux-x64",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"os": [
"linux"
Expand Down
2 changes: 1 addition & 1 deletion bindings/pkgs/@duckdb/node-bindings-win32-x64/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings-win32-x64",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"os": [
"win32"
Expand Down
2 changes: 1 addition & 1 deletion bindings/pkgs/@duckdb/node-bindings/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@duckdb/node-bindings",
"version": "1.4.3-r.1",
"version": "1.4.3-r.2",
"license": "MIT",
"main": "./duckdb.js",
"types": "./duckdb.d.ts",
Expand Down