Skip to content

Commit 20dda76

Browse files
kentkwukentkwu
authored
feat(javascript): add Node.js ADBC driver manager (#4046)
This PR introduces an ADBC Driver Manager for JavaScript, specifically Node.js. It is architected as a [Node-API](https://nodejs.org/api/n-api.html) native module that wraps the Rust ADBC Driver Manager via [NAPI-RS](https://napi.rs). This was chosen to leverage existing Arrow infrastructure in Rust and the mature Rust/JS FFI ecosystem. This means that the driver manager will run on Node (and bun, but currently untested). - Rust layer (src/): Wraps adbc_driver_manager crate. - Async operations are run on the libuv thread pool via [NAPI-RS AsyncTask](https://napi.rs/docs/concepts/async-task). Results cross the boundary as Arrow IPC streams - IPC is used as JS does not currently have a C Data Interface implementation. In the future, we can switch the driver manager to use the C Data Interface for true zero-copy once that is implemented - TypeScript layer (lib/) - JS Binding API with AdbcDatabase, AdbcConnection, AdbcStatement - Platform binaries are built for - linux-x64 - linux-arm64 - darwin-x64 - darwin-arm64 - win32-x64 I've omitted final publishing to NPM in this PR. I will file a separate issue to set up publishing once this is merged. Closes #3734
1 parent 65afee9 commit 20dda76

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+8661
-2
lines changed

‎.codespellrc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@
1818
[codespell]
1919
dictionary = .codespell-dictionary,-
2020
ignore-words = .codespell-ignore
21-
skip = go/adbc/go.sum
21+
skip = go/adbc/go.sum,javascript/package-lock.json

‎.github/workflows/javascript.yml‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: JavaScript
19+
20+
env:
21+
DEBUG: napi:*
22+
APP_NAME: adbc-driver-manager
23+
MACOSX_DEPLOYMENT_TARGET: '10.15'
24+
CARGO_INCREMENTAL: '1'
25+
26+
on:
27+
workflow_dispatch:
28+
push:
29+
branches:
30+
- main
31+
paths:
32+
- 'javascript/**'
33+
- 'rust/**'
34+
- '.github/workflows/javascript.yml'
35+
pull_request:
36+
paths:
37+
- 'javascript/**'
38+
- 'rust/**'
39+
- '.github/workflows/javascript.yml'
40+
41+
concurrency:
42+
group: ${{ github.workflow }}-${{ github.ref }}
43+
cancel-in-progress: true
44+
45+
jobs:
46+
lint:
47+
name: Lint
48+
runs-on: ubuntu-latest
49+
steps:
50+
- uses: actions/checkout@v6
51+
with:
52+
fetch-depth: 0
53+
persist-credentials: false
54+
- name: Setup node
55+
uses: actions/setup-node@v4
56+
with:
57+
node-version: 22
58+
cache: 'npm'
59+
cache-dependency-path: javascript/package-lock.json
60+
- name: Use stable Rust
61+
run: |
62+
rustup toolchain install stable --no-self-update
63+
rustup default stable
64+
rustup component add clippy rustfmt
65+
- name: Install dependencies
66+
working-directory: javascript
67+
run: npm ci
68+
- name: Check JS
69+
working-directory: javascript
70+
run: npm run check:js
71+
- name: Check Rust
72+
working-directory: javascript
73+
run: npm run check:rust
74+
75+
build:
76+
strategy:
77+
fail-fast: false
78+
matrix:
79+
settings:
80+
- host: macos-15-intel
81+
target: x86_64-apple-darwin
82+
- host: macos-latest
83+
target: aarch64-apple-darwin
84+
- host: windows-latest
85+
target: x86_64-pc-windows-msvc
86+
- host: ubuntu-latest
87+
target: x86_64-unknown-linux-gnu
88+
- host: ubuntu-24.04-arm
89+
target: aarch64-unknown-linux-gnu
90+
name: Build Node.js ${{ matrix.settings.target }}
91+
runs-on: ${{ matrix.settings.host }}
92+
steps:
93+
- uses: actions/checkout@v6
94+
with:
95+
fetch-depth: 0
96+
persist-credentials: false
97+
- name: Setup node
98+
uses: actions/setup-node@v4
99+
with:
100+
node-version: 22
101+
cache: 'npm'
102+
cache-dependency-path: javascript/package-lock.json
103+
- name: Use stable Rust
104+
run: |
105+
rustup toolchain install stable --no-self-update
106+
rustup default stable
107+
rustup target add ${{ matrix.settings.target }}
108+
- name: Install dependencies
109+
working-directory: javascript
110+
run: npm ci
111+
- name: Build
112+
working-directory: javascript
113+
shell: bash
114+
run: |
115+
npx napi build --platform --release --target ${{ matrix.settings.target }}
116+
mv index.js binding.js
117+
mv index.d.ts binding.d.ts
118+
npx tsc
119+
- name: Ad-hoc sign binary (macOS)
120+
if: runner.os == 'macOS'
121+
working-directory: javascript
122+
run: codesign --sign - *.node
123+
- name: Upload artifact
124+
uses: actions/upload-artifact@v4
125+
with:
126+
name: bindings-${{ matrix.settings.target }}
127+
path: javascript/*.node
128+
if-no-files-found: error
129+
130+
test:
131+
name: Test ${{ matrix.settings.target }}
132+
needs: build
133+
runs-on: ${{ matrix.settings.host }}
134+
strategy:
135+
fail-fast: false
136+
matrix:
137+
settings:
138+
- host: ubuntu-latest
139+
target: x86_64-unknown-linux-gnu
140+
- host: ubuntu-24.04-arm
141+
target: aarch64-unknown-linux-gnu
142+
- host: macos-15-intel
143+
target: x86_64-apple-darwin
144+
- host: macos-latest
145+
target: aarch64-apple-darwin
146+
- host: windows-latest
147+
target: x86_64-pc-windows-msvc
148+
steps:
149+
- uses: actions/checkout@v6
150+
with:
151+
fetch-depth: 0
152+
persist-credentials: false
153+
- name: Setup node
154+
uses: actions/setup-node@v4
155+
with:
156+
node-version: 22
157+
cache: 'npm'
158+
cache-dependency-path: javascript/package-lock.json
159+
- name: Install dependencies
160+
working-directory: javascript
161+
run: npm ci --omit=optional
162+
- name: Download artifacts
163+
uses: actions/download-artifact@v4
164+
with:
165+
name: bindings-${{ matrix.settings.target }}
166+
path: javascript
167+
- name: Setup conda (Windows)
168+
if: runner.os == 'Windows'
169+
uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0
170+
with:
171+
miniforge-version: latest
172+
- name: Install C++ dependencies (Windows)
173+
if: runner.os == 'Windows'
174+
run: conda install -c conda-forge --file ci/conda_env_cpp.txt
175+
- name: Install C++ dependencies (Linux)
176+
if: runner.os == 'Linux'
177+
run: sudo apt-get install libsqlite3-dev
178+
- name: Set cmake args (Windows)
179+
if: runner.os == 'Windows'
180+
shell: pwsh
181+
run: echo "ADBC_CMAKE_ARGS=-DCMAKE_PREFIX_PATH=$env:CONDA_PREFIX/Library" >> $env:GITHUB_ENV
182+
- name: Build Driver
183+
# We need to build the SQLite driver for tests.
184+
shell: bash
185+
run: ci/scripts/node_build.sh "${{ github.workspace }}/javascript/build"
186+
- name: Set driver path (Linux)
187+
if: runner.os == 'Linux'
188+
run: echo "ADBC_DRIVER_MANAGER_TEST_LIB=${{ github.workspace }}/javascript/build/lib/libadbc_driver_sqlite.so" >> "$GITHUB_ENV"
189+
- name: Set driver path (macOS)
190+
if: runner.os == 'macOS'
191+
run: echo "ADBC_DRIVER_MANAGER_TEST_LIB=${{ github.workspace }}/javascript/build/lib/libadbc_driver_sqlite.dylib" >> "$GITHUB_ENV"
192+
- name: Set driver path (Windows)
193+
if: runner.os == 'Windows'
194+
shell: pwsh
195+
run: |
196+
echo "PATH=${{ github.workspace }}/javascript/build/bin;$env:CONDA_PREFIX\Library\bin;$env:PATH" >> $env:GITHUB_ENV
197+
echo "ADBC_DRIVER_MANAGER_TEST_LIB=${{ github.workspace }}/javascript/build/bin/adbc_driver_sqlite.dll" >> $env:GITHUB_ENV
198+
- name: Run Tests
199+
working-directory: javascript
200+
run: npm test

‎.github/workflows/packaging.yml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1096,6 +1096,136 @@ jobs:
10961096
docker compose run python-sdist-test
10971097
popd
10981098
1099+
node-binaries:
1100+
name: "Node.js ${{ matrix.settings.target }}"
1101+
runs-on: ${{ matrix.settings.host }}
1102+
needs:
1103+
- source
1104+
strategy:
1105+
fail-fast: false
1106+
matrix:
1107+
settings:
1108+
- host: macos-15-intel
1109+
target: x86_64-apple-darwin
1110+
- host: macos-latest
1111+
target: aarch64-apple-darwin
1112+
- host: windows-latest
1113+
target: x86_64-pc-windows-msvc
1114+
- host: ubuntu-latest
1115+
target: x86_64-unknown-linux-gnu
1116+
- host: ubuntu-24.04-arm
1117+
target: aarch64-unknown-linux-gnu
1118+
steps:
1119+
- uses: actions/download-artifact@v6
1120+
with:
1121+
name: source
1122+
1123+
- name: Extract source archive
1124+
run: |
1125+
source_archive=$(echo apache-arrow-adbc-*.tar.gz)
1126+
VERSION=${source_archive#apache-arrow-adbc-}
1127+
VERSION=${VERSION%.tar.gz}
1128+
1129+
tar xf apache-arrow-adbc-${VERSION}.tar.gz
1130+
mv apache-arrow-adbc-${VERSION} adbc
1131+
1132+
- name: Setup Node
1133+
uses: actions/setup-node@v4
1134+
with:
1135+
node-version: 22
1136+
cache: 'npm'
1137+
cache-dependency-path: adbc/javascript/package-lock.json
1138+
1139+
- name: Setup Rust
1140+
run: |
1141+
rustup toolchain install stable --no-self-update
1142+
rustup default stable
1143+
rustup target add ${{ matrix.settings.target }}
1144+
1145+
- name: Install Node dependencies
1146+
working-directory: adbc/javascript
1147+
run: npm ci
1148+
1149+
- name: Build Node.js binaries
1150+
working-directory: adbc/javascript
1151+
run: npx napi build --platform --release --target ${{ matrix.settings.target }}
1152+
shell: bash
1153+
1154+
- name: Sign binary (macOS)
1155+
if: runner.os == 'macOS'
1156+
working-directory: adbc/javascript
1157+
run: codesign --sign - *.node
1158+
1159+
- name: Upload Node.js binaries
1160+
uses: actions/upload-artifact@v5
1161+
with:
1162+
name: bindings-${{ matrix.settings.target }}
1163+
path: adbc/javascript/*.node
1164+
if-no-files-found: error
1165+
1166+
node-dist:
1167+
name: "Node.js Package"
1168+
runs-on: ubuntu-latest
1169+
needs:
1170+
- source
1171+
- node-binaries
1172+
steps:
1173+
- uses: actions/download-artifact@v6
1174+
with:
1175+
name: source
1176+
1177+
- name: Extract source archive
1178+
run: |
1179+
source_archive=$(echo apache-arrow-adbc-*.tar.gz)
1180+
VERSION=${source_archive#apache-arrow-adbc-}
1181+
VERSION=${VERSION%.tar.gz}
1182+
1183+
tar xf apache-arrow-adbc-${VERSION}.tar.gz
1184+
mv apache-arrow-adbc-${VERSION} adbc
1185+
1186+
- name: Setup Node
1187+
uses: actions/setup-node@v4
1188+
with:
1189+
node-version: 22
1190+
cache: 'npm'
1191+
cache-dependency-path: adbc/javascript/package-lock.json
1192+
1193+
- name: Install Node dependencies
1194+
working-directory: adbc/javascript
1195+
run: npm ci
1196+
1197+
- name: Build Node.js JS
1198+
working-directory: adbc/javascript
1199+
run: npm run build:ts
1200+
1201+
- name: Download Node.js binaries
1202+
uses: actions/download-artifact@v6
1203+
with:
1204+
pattern: bindings-*
1205+
path: adbc/javascript/artifacts
1206+
merge-multiple: true
1207+
1208+
- name: Package Node.js artifacts
1209+
working-directory: adbc/javascript
1210+
env:
1211+
HUSKY: "0"
1212+
run: |
1213+
npx napi artifacts
1214+
# Pack the main package
1215+
npm pack
1216+
# Pack the platform-specific packages
1217+
for dir in npm/*; do
1218+
npm pack "./$dir"
1219+
done
1220+
1221+
- name: Upload Node.js packages
1222+
uses: actions/upload-artifact@v5
1223+
with:
1224+
name: node-packages
1225+
retention-days: 7
1226+
path: |
1227+
adbc/javascript/*.tgz
1228+
10991229
release:
11001230
name: "Create release"
11011231
runs-on: ubuntu-latest
@@ -1106,6 +1236,7 @@ jobs:
11061236
- source
11071237
- java
11081238
- linux
1239+
- node-dist
11091240
- python-manylinux
11101241
- python-macos
11111242
- python-windows
@@ -1140,6 +1271,9 @@ jobs:
11401271
')' \
11411272
-exec mv '{}' upload-staging \;
11421273
1274+
# Handle Node.js packages (in tarball form)
1275+
find ./release-artifacts/ -name 'adbc-driver-manager-*.tgz' -exec mv '{}' upload-staging \;
1276+
11431277
UPLOAD=$(find upload-staging -type f | sort | uniq)
11441278
11451279
echo "Uploading files:" >> $GITHUB_STEP_SUMMARY

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,3 +131,10 @@ target/
131131
# Meson subproject support
132132
/c/subprojects/*
133133
!/c/subprojects/*.wrap
134+
135+
# Node.js specific ignores
136+
javascript/**/node_modules/
137+
javascript/**/target/
138+
javascript/**/dist/
139+
javascript/native/*.node
140+
javascript/native/build/

‎.pre-commit-config.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,12 @@ repos:
136136
hooks:
137137
- id: fmt
138138
name: rustfmt
139+
files: ^rust/
139140
args: ["--all", "--manifest-path", "rust/Cargo.toml", "--"]
141+
- id: fmt
142+
name: rustfmt-node
143+
files: ^javascript/
144+
args: ["--all", "--manifest-path", "javascript/Cargo.toml", "--"]
140145
- repo: https://github.com/codespell-project/codespell
141146
rev: 63c8f8312b7559622c0d82815639671ae42132ac # v2.4.1
142147
hooks:

‎c/driver/sqlite/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ if(EXISTS "${SQLite3_INCLUDE_DIRS}/sqlite3.h")
3232
ADBC_SQLITE_WITH_LOAD_EXTENSION)
3333
endif()
3434

35-
if(NOT ADBC_SQLITE_WITH_LOAD_EXTENSION)
35+
if(ADBC_SQLITE_WITH_LOAD_EXTENSION EQUAL -1)
3636
set(ADBC_SQLITE_COMPILE_DEFINES "-DADBC_SQLITE_WITH_NO_LOAD_EXTENSION")
3737
endif()
3838

0 commit comments

Comments
 (0)