Skip to content

core bindings for scalar functions #267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jraymakers merged 7 commits into from
Jul 27, 2025
Merged

core bindings for scalar functions #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c707884
core bindings for scalar functions
jraymakers Jul 3, 2025
e5035d3
clarify fn names for creation without finalizers
jraymakers Jul 26, 2025
43c1fc5
add cpp headers
jraymakers Jul 26, 2025
b3974bb
fix header order
jraymakers Jul 26, 2025
e974377
fix condition_variable segfault
jraymakers Jul 27, 2025
9b0a230
destroy_scalar_function_sync
jraymakers Jul 27, 2025
1662ade
typo in comment
jraymakers Jul 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions bindings/pkgs/@duckdb/node-bindings/duckdb.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ export interface TimestampParts {
time: TimeParts;
}

export interface FunctionInfo {
__duckdb_type: 'duckdb_function_info';
}

export interface Vector {
__duckdb_type: 'duckdb_vector';
}
Expand Down Expand Up @@ -228,6 +232,10 @@ export interface Result {
__duckdb_type: 'duckdb_result';
}

export interface ScalarFunction {
__duckdb_type: 'duckdb_scalar_function';
}

// export interface SelectionVector {
// __duckdb_type: 'duckdb_selection_vector';
// }
Expand All @@ -248,6 +256,7 @@ export interface ExtractedStatementsAndCount {
statement_count: number;
}

export type ScalarFunctionMainFunction = (info: FunctionInfo, input: DataChunk, output: Vector) => void;

// Functions

Expand Down Expand Up @@ -1013,23 +1022,39 @@ export function validity_set_row_invalid(validity: Uint8Array, row_index: number
export function validity_set_row_valid(validity: Uint8Array, row_index: number): void;

// DUCKDB_C_API duckdb_scalar_function duckdb_create_scalar_function();
export function create_scalar_function(): ScalarFunction;

// DUCKDB_C_API void duckdb_destroy_scalar_function(duckdb_scalar_function *scalar_function);
export function destroy_scalar_function_sync(scalar_function: ScalarFunction): void;

// DUCKDB_C_API void duckdb_scalar_function_set_name(duckdb_scalar_function scalar_function, const char *name);
export function scalar_function_set_name(scalar_function: ScalarFunction, name: string): void;

// DUCKDB_C_API void duckdb_scalar_function_set_varargs(duckdb_scalar_function scalar_function, duckdb_logical_type type);
// DUCKDB_C_API void duckdb_scalar_function_set_special_handling(duckdb_scalar_function scalar_function);
// DUCKDB_C_API void duckdb_scalar_function_set_volatile(duckdb_scalar_function scalar_function);
// DUCKDB_C_API void duckdb_scalar_function_add_parameter(duckdb_scalar_function scalar_function, duckdb_logical_type type);

// DUCKDB_C_API void duckdb_scalar_function_set_return_type(duckdb_scalar_function scalar_function, duckdb_logical_type type);
export function scalar_function_set_return_type(scalar_function: ScalarFunction, logical_type: LogicalType): void;

// DUCKDB_C_API void duckdb_scalar_function_set_extra_info(duckdb_scalar_function scalar_function, void *extra_info, duckdb_delete_callback_t destroy);
// DUCKDB_C_API void duckdb_scalar_function_set_bind(duckdb_scalar_function scalar_function, duckdb_scalar_function_bind_t bind);
// DUCKDB_C_API void duckdb_scalar_function_set_bind_data(duckdb_bind_info info, void *bind_data, duckdb_delete_callback_t destroy);
// DUCKDB_C_API void duckdb_scalar_function_bind_set_error(duckdb_bind_info info, const char *error);

// DUCKDB_C_API void duckdb_scalar_function_set_function(duckdb_scalar_function scalar_function, duckdb_scalar_function_t function);
export function scalar_function_set_function(scalar_function: ScalarFunction, func: ScalarFunctionMainFunction): void;

// DUCKDB_C_API duckdb_state duckdb_register_scalar_function(duckdb_connection con, duckdb_scalar_function scalar_function);
export function register_scalar_function(connection: Connection, scalar_function: ScalarFunction): void;

// DUCKDB_C_API void *duckdb_scalar_function_get_extra_info(duckdb_function_info info);
// DUCKDB_C_API void *duckdb_scalar_function_get_bind_data(duckdb_function_info info);
// DUCKDB_C_API void duckdb_scalar_function_get_client_context(duckdb_bind_info info, duckdb_client_context *out_context);

// DUCKDB_C_API void duckdb_scalar_function_set_error(duckdb_function_info info, const char *error);
export function scalar_function_set_error(function_info: FunctionInfo, error: string): void;

// DUCKDB_C_API duckdb_scalar_function_set duckdb_create_scalar_function_set(const char *name);
// DUCKDB_C_API void duckdb_destroy_scalar_function_set(duckdb_scalar_function_set *scalar_function_set);
Expand Down
215 changes: 208 additions & 7 deletions bindings/src/duckdb_node_bindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
#define NODE_API_NO_EXTERNAL_BUFFERS_ALLOWED
#include "napi.h"

#include <condition_variable>
#include <cstddef>
#include <mutex>
#include <optional>
#include <string>
#include <vector>
Expand Down Expand Up @@ -462,6 +465,10 @@ Napi::External<_duckdb_data_chunk> CreateExternalForDataChunk(Napi::Env env, duc
return CreateExternal<_duckdb_data_chunk>(env, DataChunkTypeTag, chunk, FinalizeDataChunk);
}

Napi::External<_duckdb_data_chunk> CreateExternalForDataChunkWithoutFinalizer(Napi::Env env, duckdb_data_chunk chunk) {
return CreateExternalWithoutFinalizer<_duckdb_data_chunk>(env, DataChunkTypeTag, chunk);
}

duckdb_data_chunk GetDataChunkFromExternal(Napi::Env env, Napi::Value value) {
return GetDataFromExternal<_duckdb_data_chunk>(env, DataChunkTypeTag, value, "Invalid data chunk argument");
}
Expand All @@ -485,6 +492,21 @@ duckdb_extracted_statements GetExtractedStatementsFromExternal(Napi::Env env, Na
return GetDataFromExternal<_duckdb_extracted_statements>(env, ExtractedStatementsTypeTag, value, "Invalid extracted statements argument");
}

static const napi_type_tag FunctionInfoTypeTag = {
0xB0E6739D698048EA, 0x9E79734E3E137AC3
};

Napi::External<_duckdb_function_info> CreateExternalForFunctionInfoWithoutFinalizer(Napi::Env env, duckdb_function_info function_info) {
// FunctionInfo objects are never explicitly created; they are passed in to function callbacks.
return CreateExternalWithoutFinalizer<_duckdb_function_info>(env, FunctionInfoTypeTag, function_info);
}

duckdb_function_info GetFunctionInfoFromExternal(Napi::Env env, Napi::Value value) {
return GetDataFromExternal<_duckdb_function_info>(env, FunctionInfoTypeTag, value, "Invalid function info argument");
}



static const napi_type_tag InstanceCacheTypeTag = {
0x2F3346E30FB5457C, 0xB9201EE5112EEF9F
};
Expand Down Expand Up @@ -592,6 +614,38 @@ duckdb_result *GetResultFromExternal(Napi::Env env, Napi::Value value) {
return GetDataFromExternal<duckdb_result>(env, ResultTypeTag, value, "Invalid result argument");
}

static const napi_type_tag ScalarFunctionTypeTag = {
0x95D48B7051D14994, 0x9F883D7DF5DEA86D
};

typedef struct {
duckdb_scalar_function scalar_function;
} duckdb_scalar_function_holder;

duckdb_scalar_function_holder *CreateScalarFunctionHolder(duckdb_scalar_function scalar_function) {
auto scalar_function_holder_ptr = reinterpret_cast<duckdb_scalar_function_holder*>(duckdb_malloc(sizeof(duckdb_scalar_function_holder)));
scalar_function_holder_ptr->scalar_function = scalar_function;
return scalar_function_holder_ptr;
}

void FinalizeScalarFunctionHolder(Napi::BasicEnv, duckdb_scalar_function_holder *scalar_function_holder_ptr) {
// duckdb_destroy_scalar_function is a no-op if already closed
duckdb_destroy_scalar_function(&scalar_function_holder_ptr->scalar_function);
duckdb_free(scalar_function_holder_ptr);
}

Napi::External<duckdb_scalar_function_holder> CreateExternalForScalarFunction(Napi::Env env, duckdb_scalar_function scalar_function) {
return CreateExternal<duckdb_scalar_function_holder>(env, ScalarFunctionTypeTag, CreateScalarFunctionHolder(scalar_function), FinalizeScalarFunctionHolder);
}

duckdb_scalar_function_holder *GetScalarFunctionHolderFromExternal(Napi::Env env, Napi::Value value) {
return GetDataFromExternal<duckdb_scalar_function_holder>(env, ScalarFunctionTypeTag, value, "Invalid scalar function argument");
}

duckdb_scalar_function GetScalarFunctionFromExternal(Napi::Env env, Napi::Value value) {
return GetScalarFunctionHolderFromExternal(env, value)->scalar_function;
}

static const napi_type_tag ValueTypeTag = {
0xC60F36613BF14E93, 0xBAA92848936FAA25
};
Expand All @@ -615,7 +669,7 @@ static const napi_type_tag VectorTypeTag = {
0x9FE56DE8E3124D07, 0x9ABF31145EDE1C9E
};

Napi::External<_duckdb_vector> CreateExternalForVector(Napi::Env env, duckdb_vector vector) {
Napi::External<_duckdb_vector> CreateExternalForVectorWithoutFinalizer(Napi::Env env, duckdb_vector vector) {
// Vectors live as long as their containing data chunk; they cannot be explicitly destroyed.
return CreateExternalWithoutFinalizer<_duckdb_vector>(env, VectorTypeTag, vector);
}
Expand All @@ -624,6 +678,78 @@ duckdb_vector GetVectorFromExternal(Napi::Env env, Napi::Value value) {
return GetDataFromExternal<_duckdb_vector>(env, VectorTypeTag, value, "Invalid vector argument");
}

// Scalar functions

using ScalarFunctionMainContext = std::nullptr_t;
struct ScalarFunctionMainData {
duckdb_function_info info;
duckdb_data_chunk input;
duckdb_vector output;
std::condition_variable *cv;
std::mutex *cv_mutex;
bool done;
};
void ScalarFunctionMainCallback(Napi::Env env, Napi::Function callback, ScalarFunctionMainContext *context, ScalarFunctionMainData *data) {
if (env != nullptr) {
if (callback != nullptr) {
callback.Call(
env.Undefined(),
{
CreateExternalForFunctionInfoWithoutFinalizer(env, data->info),
CreateExternalForDataChunkWithoutFinalizer(env, data->input),
CreateExternalForVectorWithoutFinalizer(env, data->output)
}
);
}
}
{
std::lock_guard lk(*data->cv_mutex);
data->done = true;
}
data->cv->notify_one();
}
using ScalarFunctionMainTSFN = Napi::TypedThreadSafeFunction<ScalarFunctionMainContext, ScalarFunctionMainData, ScalarFunctionMainCallback>;

struct ScalarFunctionMainExtraInfo {
ScalarFunctionMainTSFN tsfn;
// TODO: user extra info
};

ScalarFunctionMainExtraInfo *CreateScalarFunctionMainExtraInfo(Napi::Env env, Napi::Function func) {
auto extra_info = reinterpret_cast<ScalarFunctionMainExtraInfo*>(duckdb_malloc(sizeof(ScalarFunctionMainExtraInfo)));
extra_info->tsfn = ScalarFunctionMainTSFN::New(env, func, "ScalarFunctionMain", 0, 1);
return extra_info;
}

void DeleteScalarFunctionMainExtraInfo(ScalarFunctionMainExtraInfo *extra_info) {
extra_info->tsfn.Release();
duckdb_free(extra_info);
}

void ScalarFunctionMainFunction(duckdb_function_info info, duckdb_data_chunk input, duckdb_vector output) {
auto extra_info = reinterpret_cast<ScalarFunctionMainExtraInfo*>(duckdb_scalar_function_get_extra_info(info));
auto data = reinterpret_cast<ScalarFunctionMainData*>(duckdb_malloc(sizeof(ScalarFunctionMainData)));
data->info = info;
data->input = input;
data->output = output;
data->cv = new std::condition_variable;
data->cv_mutex = new std::mutex;
data->done = false;
// The "blocking" part of this call only waits for queue space, not for the JS function call to complete.
// Since we specify no limit to the queue space, it in fact never blocks.
auto status = extra_info->tsfn.BlockingCall(data);
if (status == napi_ok) {
// Wait for the JS function call to complete.
std::unique_lock<std::mutex> lk(*data->cv_mutex);
data->cv->wait(lk, [&]{ return data->done; });
} else {
duckdb_scalar_function_set_error(info, "BlockingCall returned not ok");
}
delete data->cv;
delete data->cv_mutex;
duckdb_free(data);
}

// Promise workers

class PromiseWorker : public Napi::AsyncWorker {
Expand Down Expand Up @@ -1421,6 +1547,14 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
InstanceMethod("validity_set_row_invalid", &DuckDBNodeAddon::validity_set_row_invalid),
InstanceMethod("validity_set_row_valid", &DuckDBNodeAddon::validity_set_row_valid),

InstanceMethod("create_scalar_function", &DuckDBNodeAddon::create_scalar_function),
InstanceMethod("destroy_scalar_function_sync", &DuckDBNodeAddon::destroy_scalar_function_sync),
InstanceMethod("scalar_function_set_name", &DuckDBNodeAddon::scalar_function_set_name),
InstanceMethod("scalar_function_set_return_type", &DuckDBNodeAddon::scalar_function_set_return_type),
InstanceMethod("scalar_function_set_function", &DuckDBNodeAddon::scalar_function_set_function),
InstanceMethod("register_scalar_function", &DuckDBNodeAddon::register_scalar_function),
InstanceMethod("scalar_function_set_error", &DuckDBNodeAddon::scalar_function_set_error),

InstanceMethod("appender_create", &DuckDBNodeAddon::appender_create),
InstanceMethod("appender_create_ext", &DuckDBNodeAddon::appender_create_ext),
InstanceMethod("appender_column_count", &DuckDBNodeAddon::appender_column_count),
Expand Down Expand Up @@ -3680,7 +3814,7 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
auto chunk = GetDataChunkFromExternal(env, info[0]);
auto column_index = info[1].As<Napi::Number>().Uint32Value();
auto vector = duckdb_data_chunk_get_vector(chunk, column_index);
return CreateExternalForVector(env, vector);
return CreateExternalForVectorWithoutFinalizer(env, vector);
}

// DUCKDB_C_API idx_t duckdb_data_chunk_get_size(duckdb_data_chunk chunk);
Expand Down Expand Up @@ -3779,7 +3913,7 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
auto env = info.Env();
auto vector = GetVectorFromExternal(env, info[0]);
auto child = duckdb_list_vector_get_child(vector);
return CreateExternalForVector(env, child);
return CreateExternalForVectorWithoutFinalizer(env, child);
}

// DUCKDB_C_API idx_t duckdb_list_vector_get_size(duckdb_vector vector);
Expand Down Expand Up @@ -3818,7 +3952,7 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
auto vector = GetVectorFromExternal(env, info[0]);
auto index = info[1].As<Napi::Number>().Uint32Value();
auto child = duckdb_struct_vector_get_child(vector, index);
return CreateExternalForVector(env, child);
return CreateExternalForVectorWithoutFinalizer(env, child);
}

// DUCKDB_C_API duckdb_vector duckdb_array_vector_get_child(duckdb_vector vector);
Expand All @@ -3827,7 +3961,7 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
auto env = info.Env();
auto vector = GetVectorFromExternal(env, info[0]);
auto child = duckdb_array_vector_get_child(vector);
return CreateExternalForVector(env, child);
return CreateExternalForVectorWithoutFinalizer(env, child);
}

// DUCKDB_C_API void duckdb_slice_vector(duckdb_vector vector, duckdb_selection_vector selection, idx_t len);
Expand Down Expand Up @@ -3878,23 +4012,90 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
}

// DUCKDB_C_API duckdb_scalar_function duckdb_create_scalar_function();
// function create_scalar_function(): ScalarFunction
Napi::Value create_scalar_function(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto scalar_function = duckdb_create_scalar_function();
return CreateExternalForScalarFunction(env, scalar_function);
}

// DUCKDB_C_API void duckdb_destroy_scalar_function(duckdb_scalar_function *scalar_function);
// function destroy_scalar_function_sync(scalar_function: ScalarFunction): void
Napi::Value destroy_scalar_function_sync(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto scalar_function_holder_ptr = GetScalarFunctionHolderFromExternal(env, info[0]);
// duckdb_destroy_scalar_function is a no-op if already closed
duckdb_destroy_scalar_function(&scalar_function_holder_ptr->scalar_function);
return env.Undefined();
}

// DUCKDB_C_API void duckdb_scalar_function_set_name(duckdb_scalar_function scalar_function, const char *name);
// function scalar_function_set_name(scalar_function: ScalarFunction, name: string): void
Napi::Value scalar_function_set_name(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto scalar_function = GetScalarFunctionFromExternal(env, info[0]);
std::string name = info[1].As<Napi::String>();
duckdb_scalar_function_set_name(scalar_function, name.c_str());
return env.Undefined();
}

// DUCKDB_C_API void duckdb_scalar_function_set_varargs(duckdb_scalar_function scalar_function, duckdb_logical_type type);
// DUCKDB_C_API void duckdb_scalar_function_set_special_handling(duckdb_scalar_function scalar_function);
// DUCKDB_C_API void duckdb_scalar_function_set_volatile(duckdb_scalar_function scalar_function);
// DUCKDB_C_API void duckdb_scalar_function_add_parameter(duckdb_scalar_function scalar_function, duckdb_logical_type type);

// DUCKDB_C_API void duckdb_scalar_function_set_return_type(duckdb_scalar_function scalar_function, duckdb_logical_type type);
// function scalar_function_set_return_type(scalar_function: ScalarFunction, logical_type: LogicalType): void
Napi::Value scalar_function_set_return_type(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto scalar_function = GetScalarFunctionFromExternal(env, info[0]);
auto logical_type = GetLogicalTypeFromExternal(env, info[1]);
duckdb_scalar_function_set_return_type(scalar_function, logical_type);
return env.Undefined();
}

// DUCKDB_C_API void duckdb_scalar_function_set_extra_info(duckdb_scalar_function scalar_function, void *extra_info, duckdb_delete_callback_t destroy);
// DUCKDB_C_API void duckdb_scalar_function_set_bind(duckdb_scalar_function scalar_function, duckdb_scalar_function_bind_t bind);
// DUCKDB_C_API void duckdb_scalar_function_set_bind_data(duckdb_bind_info info, void *bind_data, duckdb_delete_callback_t destroy);
// DUCKDB_C_API void duckdb_scalar_function_bind_set_error(duckdb_bind_info info, const char *error);

// DUCKDB_C_API void duckdb_scalar_function_set_function(duckdb_scalar_function scalar_function, duckdb_scalar_function_t function);
// function scalar_function_set_function(scalar_function: ScalarFunction, func: ScalarFunctionMainFunction): void
Napi::Value scalar_function_set_function(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto scalar_function = GetScalarFunctionFromExternal(env, info[0]);
auto func = info[1].As<Napi::Function>();
auto extra_info = CreateScalarFunctionMainExtraInfo(env, func);
duckdb_scalar_function_set_extra_info(scalar_function, extra_info, reinterpret_cast<duckdb_delete_callback_t>(DeleteScalarFunctionMainExtraInfo));
duckdb_scalar_function_set_function(scalar_function, &ScalarFunctionMainFunction);
return env.Undefined();
}

// DUCKDB_C_API duckdb_state duckdb_register_scalar_function(duckdb_connection con, duckdb_scalar_function scalar_function);
// function register_scalar_function(connection: Connection, scalar_function: ScalarFunction): void
Napi::Value register_scalar_function(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto connection = GetConnectionFromExternal(env, info[0]);
auto scalar_function = GetScalarFunctionFromExternal(env, info[1]);
if (duckdb_register_scalar_function(connection, scalar_function)) {
throw Napi::Error::New(env, "Failed to register scalar function");
}
return env.Undefined();
}

// DUCKDB_C_API void *duckdb_scalar_function_get_extra_info(duckdb_function_info info);
// DUCKDB_C_API void *duckdb_scalar_function_get_bind_data(duckdb_function_info info);
// DUCKDB_C_API void duckdb_scalar_function_get_client_context(duckdb_bind_info info, duckdb_client_context *out_context);

// DUCKDB_C_API void duckdb_scalar_function_set_error(duckdb_function_info info, const char *error);
// function scalar_function_set_error(function_info: FunctionInfo, error: string): void
Napi::Value scalar_function_set_error(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto function_info = GetFunctionInfoFromExternal(env, info[0]);
std::string error = info[1].As<Napi::String>();
duckdb_scalar_function_set_error(function_info, error.c_str());
return env.Undefined();
}

// DUCKDB_C_API duckdb_scalar_function_set duckdb_create_scalar_function_set(const char *name);
// DUCKDB_C_API void duckdb_destroy_scalar_function_set(duckdb_scalar_function_set *scalar_function_set);
Expand Down Expand Up @@ -4490,14 +4691,14 @@ NODE_API_ADDON(DuckDBNodeAddon)
---
431 total functions

245 instance methods
252 instance methods
3 unimplemented client context functions
1 unimplemented table names function
1 unimplemented value to string function
1 unimplemented logical type function
2 unimplemented vector creation functions
3 unimplemented vector manipulation functions
18 unimplemented scalar function functions
11 unimplemented scalar function functions
4 unimplemented scalar function set functions
3 unimplemented selection vector functions
12 unimplemented aggregate function functions
Expand Down
Loading
Loading