diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ae7f60e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [main, master, 'liz/**'] + pull_request: + branches: [main, master] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18, 20, 22] + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v3 + with: + version: 9 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: pnpm install + - run: pnpm run typecheck + - run: pnpm test -- run diff --git a/examples/blob-crud/README.md b/examples/blob-crud/README.md index 662bd18..80112c0 100644 --- a/examples/blob-crud/README.md +++ b/examples/blob-crud/README.md @@ -1,20 +1,23 @@ -# Dexie Cloud SDK Example +# Dexie Cloud SDK — Blob CRUD Example -A Node.js example showing how to use the Dexie Cloud SDK for server-side data operations with blob support. +Server-side data operations with blob support using client credentials. ## What It Shows -- Authenticating with OTP +- Authenticating with client credentials (`clientId`/`clientSecret`) - CRUD operations via REST API - Uploading and downloading blobs -- Auto vs lazy blob handling modes +- Auto blob handling mode ## Run ```bash npm install -# Set your database URL: + +# Credentials from your dexie-cloud.key file: export DEXIE_CLOUD_DB_URL=https://xxxxxxxx.dexie.cloud -export DEXIE_CLOUD_EMAIL=your@email.com +export DEXIE_CLOUD_CLIENT_ID=your-client-id +export DEXIE_CLOUD_CLIENT_SECRET=your-client-secret + npm start ``` diff --git a/examples/blob-crud/src/main.ts b/examples/blob-crud/src/main.ts index b4f8bf4..727eda6 100644 --- a/examples/blob-crud/src/main.ts +++ b/examples/blob-crud/src/main.ts @@ -2,51 +2,45 @@ * Dexie Cloud SDK — Blob CRUD Example * * Demonstrates server-side data operations with blob offloading. + * Uses client credentials (clientId/clientSecret from dexie-cloud.key). */ import { DexieCloudClient } from 'dexie-cloud-sdk'; import * as fs from 'fs'; -import * as readline from 'readline/promises'; +// Read credentials from dexie-cloud.key or environment const DB_URL = process.env.DEXIE_CLOUD_DB_URL; -const EMAIL = process.env.DEXIE_CLOUD_EMAIL; +const CLIENT_ID = process.env.DEXIE_CLOUD_CLIENT_ID; +const CLIENT_SECRET = process.env.DEXIE_CLOUD_CLIENT_SECRET; -if (!DB_URL || !EMAIL) { - console.error('Set DEXIE_CLOUD_DB_URL and DEXIE_CLOUD_EMAIL environment variables'); +if (!DB_URL || !CLIENT_ID || !CLIENT_SECRET) { + console.error(`Set environment variables: + DEXIE_CLOUD_DB_URL — Your database URL (from dexie-cloud.key) + DEXIE_CLOUD_CLIENT_ID — Client ID (from dexie-cloud.key) + DEXIE_CLOUD_CLIENT_SECRET — Client secret (from dexie-cloud.key) + +Or source your dexie-cloud.key file directly.`); process.exit(1); } async function main() { - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - // --- Initialize SDK --- const client = new DexieCloudClient({ - serviceUrl: 'https://dexie.cloud', dbUrl: DB_URL, + clientId: CLIENT_ID, + clientSecret: CLIENT_SECRET, blobHandling: 'auto' // Binary data handled transparently }); - console.log('🔑 Authenticating...'); - - // --- Authenticate --- - - const { accessToken } = await client.auth.authenticateWithOTP( - EMAIL, - async () => { - const otp = await rl.question('Enter OTP from email: '); - return otp.trim(); - }, - ['ACCESS_DB'] - ); - + console.log('🔑 Authenticating with client credentials...'); + const accessToken = await client.auth.getToken(['ACCESS_DB', 'GLOBAL_WRITE']); console.log('✅ Authenticated!\n'); // --- Create item with binary data --- console.log('📝 Creating item with binary data...'); - // Create a sample image (or read from file) const imageData = new Uint8Array(1024); crypto.getRandomValues(imageData); // Random bytes for demo @@ -92,8 +86,6 @@ async function main() { console.log('🗑️ Cleaning up...'); await client.data.delete('files', item.id, accessToken); console.log('✅ Done!\n'); - - rl.close(); } main().catch(err => { diff --git a/src/blob.ts b/src/blob.ts index abf8a7b..ab04d61 100644 --- a/src/blob.ts +++ b/src/blob.ts @@ -9,26 +9,55 @@ import type { HttpAdapter } from './adapters.js'; import type { BlobHandling, BlobRef } from './types.js'; import { DexieCloudError } from './types.js'; +/** + * Minimum byte size for offloading a binary to blob storage. + * Binaries smaller than this threshold are kept inline (as base64). + * Must match the server-side threshold. + */ +export const BLOB_THRESHOLD = 4096; + +/** + * Maximum number of concurrent blob downloads in _walkForRead. + * Mirrors the client-side MAX_CONCURRENT pattern. + */ +const MAX_CONCURRENT_DOWNLOADS = 6; + /** Generate a unique blob ID */ function generateBlobId(): string { if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { return crypto.randomUUID().replace(/-/g, ''); } - // Fallback: timestamp + random hex - return Date.now().toString(16) + Math.random().toString(16).slice(2); + // Fallback: use getRandomValues for strong entropy + if (typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function') { + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return Array.from(bytes) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); + } + // Last resort (non-browser, non-Node env): still better than Math.random alone + const ts = Date.now().toString(16); + const rand = Math.floor(Math.random() * 0xffffffff).toString(16).padStart(8, '0'); + return ts + rand; } -/** Convert Blob/ArrayBuffer/TypedArray to Uint8Array */ -async function toUint8Array(data: Uint8Array | Blob | ArrayBuffer): Promise { +/** + * Convert Blob/ArrayBuffer/TypedArray/DataView to Uint8Array. + * Accepts any ArrayBufferView (TypedArrays + DataView) as well as + * Uint8Array, ArrayBuffer, and Blob. + */ +async function toUint8Array( + data: Uint8Array | Blob | ArrayBuffer | ArrayBufferView +): Promise { if (data instanceof Uint8Array) return data; if (data instanceof ArrayBuffer) return new Uint8Array(data); if (typeof Blob !== 'undefined' && data instanceof Blob) { const buf = await data.arrayBuffer(); return new Uint8Array(buf); } - // TypedArray (e.g. Int8Array, etc.) + // Handles all TypedArrays (Int8Array, Float32Array, etc.) and DataView if (ArrayBuffer.isView(data)) { - return new Uint8Array((data as ArrayBufferView).buffer); + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } throw new TypeError('Unsupported data type for blob upload'); } @@ -92,7 +121,7 @@ export class BlobManager { * Returns the blob ref (e.g. "1:abc123..."). */ async upload( - data: Uint8Array | Blob | ArrayBuffer, + data: Uint8Array | Blob | ArrayBuffer | ArrayBufferView, token: string, contentType = 'application/octet-stream' ): Promise { @@ -125,14 +154,20 @@ export class BlobManager { const parsed = JSON.parse(text); if (parsed?.ref) return parsed.ref as string; } catch { - // ignore parse errors, construct ref ourselves + // ignore parse errors, fall through } // If server returned "version:blobId" directly if (text.includes(':')) return text.trim(); } - // Fallback: assume version 1 - return `1:${blobId}`; + // Server response was unparseable — we cannot safely construct a ref + // because we don't know the server-assigned version. + throw new DexieCloudError( + `Blob upload succeeded (HTTP ${response.status}) but server returned no parseable ref. ` + + `Cannot construct a safe blob reference without the server-assigned version.`, + response.status, + text + ); } /** @@ -160,8 +195,9 @@ export class BlobManager { } /** - * Process an object before uploading: find inline blobs, upload them, - * replace with BlobRefs. Only active in 'auto' mode. + * Process an object before uploading: find inline blobs large enough to + * offload (≥ BLOB_THRESHOLD bytes), upload them, replace with BlobRefs. + * Small binaries are left inline. Only active in 'auto' mode. */ async processForUpload(obj: any, token: string): Promise { if (this.mode !== 'auto') return obj; @@ -179,8 +215,13 @@ export class BlobManager { private async _walkForUpload(val: any, token: string): Promise { if (isInlineBlob(val)) { - // Upload inline blob, replace with BlobRef const bytes = base64ToUint8Array(val.v); + // Only offload to blob storage if the binary meets the size threshold. + // Small binaries are cheaper to keep inline than to round-trip through + // the blob endpoint. + if (bytes.length < BLOB_THRESHOLD) { + return val; // keep as-is + } const contentType = val.ct ?? 'application/octet-stream'; const ref = await this.upload(bytes, token, contentType); const blobRef: BlobRef = { @@ -193,19 +234,14 @@ export class BlobManager { } if (Array.isArray(val)) { - const results: any[] = []; - for (const item of val) { - results.push(await this._walkForUpload(item, token)); - } - return results; + return Promise.all(val.map((item) => this._walkForUpload(item, token))); } if (val !== null && typeof val === 'object') { - const result: Record = {}; - for (const [k, v] of Object.entries(val)) { - result[k] = await this._walkForUpload(v, token); - } - return result; + const entries = await Promise.all( + Object.entries(val).map(async ([k, v]) => [k, await this._walkForUpload(v, token)] as const) + ); + return Object.fromEntries(entries); } return val; @@ -213,7 +249,6 @@ export class BlobManager { private async _walkForRead(val: any, token: string): Promise { if (isBlobRef(val)) { - // Download and replace with inline const { data, contentType } = await this.download(val.ref, token); return { _bt: val._bt, @@ -223,21 +258,48 @@ export class BlobManager { } if (Array.isArray(val)) { - const results: any[] = []; - for (const item of val) { - results.push(await this._walkForRead(item, token)); - } - return results; + // Download up to MAX_CONCURRENT_DOWNLOADS blobs in parallel + return this._parallelMap(val, (item) => this._walkForRead(item, token)); } if (val !== null && typeof val === 'object') { + const keys = Object.keys(val); + const resolvedValues = await this._parallelMap( + keys, + (k) => this._walkForRead(val[k], token) + ); const result: Record = {}; - for (const [k, v] of Object.entries(val)) { - result[k] = await this._walkForRead(v, token); + for (let i = 0; i < keys.length; i++) { + result[keys[i]!] = resolvedValues[i]; } return result; } return val; } + + /** + * Like Promise.all but with a concurrency cap. + */ + private async _parallelMap( + items: T[], + fn: (item: T) => Promise + ): Promise { + const results: R[] = new Array(items.length); + let index = 0; + + async function worker() { + while (index < items.length) { + const i = index++; + results[i] = await fn(items[i]!); + } + } + + const workers = Array.from( + { length: Math.min(MAX_CONCURRENT_DOWNLOADS, items.length) }, + () => worker() + ); + await Promise.all(workers); + return results; + } } diff --git a/src/client.ts b/src/client.ts index 698e102..ff80411 100644 --- a/src/client.ts +++ b/src/client.ts @@ -43,8 +43,9 @@ export class DexieCloudClient { // Use dbUrl if provided, otherwise fall back to serviceUrl const dbUrl = fullConfig.dbUrl ?? fullConfig.serviceUrl; - this.data = new DataManager(dbUrl, this.http); this.blobs = new BlobManager(dbUrl, this.http, fullConfig.blobHandling ?? 'auto'); + // Pass BlobManager to DataManager so create/get/list auto-process blobs + this.data = new DataManager(dbUrl, this.http, this.blobs); } /** diff --git a/src/data.ts b/src/data.ts index 7df18e6..46b8acb 100644 --- a/src/data.ts +++ b/src/data.ts @@ -3,9 +3,14 @@ * * Provides CRUD operations against the Dexie Cloud REST API with * automatic TSON serialization/deserialization. + * + * Blob integration: if a BlobManager is provided, create() will + * automatically call processForUpload before sending, and get()/list() + * will call processForRead on the response. */ import type { HttpAdapter } from './adapters.js'; +import type { BlobManager } from './blob.js'; import { DexieCloudError } from './types.js'; import { parseResponse, stringifyBody } from './http-utils.js'; @@ -22,10 +27,16 @@ async function handleResponse(response: Response): Promise { } export class DataManager { - constructor(private dbUrl: string, private http: HttpAdapter) {} + constructor( + private dbUrl: string, + private http: HttpAdapter, + private blobManager?: BlobManager + ) {} /** * List all objects in a table, optionally filtered by realm. + * BlobRefs in results are automatically resolved to inline data + * when a BlobManager is present. */ async list(table: string, token: string, options?: { realm?: string }): Promise { let url = `${this.dbUrl}/${encodeURIComponent(table)}`; @@ -37,12 +48,23 @@ export class DataManager { headers: { Authorization: `Bearer ${token}` }, }); const result = await handleResponse(response); - // Server may return array directly or wrapped - return Array.isArray(result) ? result : result?.data ?? result ?? []; + const items: any[] = Array.isArray(result) ? result : result?.data ?? result ?? []; + if (this.blobManager) { + // Process items sequentially to avoid unbounded parallel downloads. + // Each item already uses internal parallelism (MAX_CONCURRENT=6) for its blobs. + const resolved: any[] = []; + for (const item of items) { + resolved.push(await this.blobManager.processForRead(item, token)); + } + return resolved; + } + return items; } /** * Get a single object by id. + * BlobRefs in the result are automatically resolved to inline data + * when a BlobManager is present. */ async get(table: string, id: string, token: string): Promise { const url = `${this.dbUrl}/${encodeURIComponent(table)}/${encodeURIComponent(id)}`; @@ -50,41 +72,70 @@ export class DataManager { method: 'GET', headers: { Authorization: `Bearer ${token}` }, }); - return handleResponse(response); + const result = await handleResponse(response); + if (this.blobManager) { + return this.blobManager.processForRead(result, token); + } + return result; } /** * Create an object in a table. + * Inline blobs in obj are automatically uploaded and replaced with + * BlobRefs when a BlobManager is present. */ async create(table: string, obj: any, token: string): Promise { const url = `${this.dbUrl}/${encodeURIComponent(table)}`; + const body = this.blobManager + ? await this.blobManager.processForUpload(obj, token) + : obj; const response = await this.http.fetch(url, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, - body: stringifyBody(obj), + body: stringifyBody(body), }); return handleResponse(response); } /** - * Update (replace) an object by id. + * Replace (full update) an object by id using HTTP PUT. + * + * NOTE: This sends the complete object as a full replacement (PUT semantics). + * All fields not included in `obj` will be removed on the server. + * If you want partial updates, use a PATCH-based approach when the server + * supports it (not currently exposed here). + * + * Previously named `update()` — `update` is kept as an alias for backwards + * compatibility but may be deprecated in a future release. */ - async update(table: string, id: string, obj: any, token: string): Promise { + async replace(table: string, id: string, obj: any, token: string): Promise { const url = `${this.dbUrl}/${encodeURIComponent(table)}/${encodeURIComponent(id)}`; + const body = this.blobManager + ? await this.blobManager.processForUpload(obj, token) + : obj; const response = await this.http.fetch(url, { method: 'PUT', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, - body: stringifyBody(obj), + body: stringifyBody(body), }); return handleResponse(response); } + /** + * @deprecated Use `replace()` instead. This method performs a full object + * replacement (HTTP PUT), not a partial update (PATCH). The name `update` + * is misleading and kept only for backwards compatibility. + */ + update(table: string, id: string, obj: any, token: string): Promise { + return this.replace(table, id, obj, token); + } + /** * Delete an object by id. */ @@ -105,13 +156,13 @@ export class DataManager { } /** - * Bulk create objects in a table (sequential). + * Bulk create objects in a table in parallel. + * All requests are sent concurrently via Promise.all. + * + * NOTE: This is NOT atomic — if one create fails, others may have already + * succeeded on the server. There is no rollback for partial failures. */ async bulkCreate(table: string, objects: any[], token: string): Promise { - const results: any[] = []; - for (const obj of objects) { - results.push(await this.create(table, obj, token)); - } - return results; + return Promise.all(objects.map((obj) => this.create(table, obj, token))); } } diff --git a/tests/unit/blob.test.ts b/tests/unit/blob.test.ts index 22d9b4b..fb46a93 100644 --- a/tests/unit/blob.test.ts +++ b/tests/unit/blob.test.ts @@ -102,7 +102,7 @@ describe('BlobManager', () => { expect(ref).toBe('1:abc123'); }); - it('falls back to "1:{id}" when server returns empty body', async () => { + it('throws when server returns empty body (version cannot be determined)', async () => { const emptyResponse = { ok: true, status: 200, statusText: 'OK', text: async () => '', @@ -117,8 +117,8 @@ describe('BlobManager', () => { } as Response; fetchMock.mockResolvedValue(emptyResponse); - const ref = await manager.upload(uint8([1]), TOKEN); - expect(ref).toMatch(/^1:[a-f0-9]+$/); + // Server returned no parseable ref — we cannot safely construct a version-prefixed ref + await expect(manager.upload(uint8([1]), TOKEN)).rejects.toThrow(DexieCloudError); }); it('throws on upload error', async () => { @@ -157,7 +157,11 @@ describe('BlobManager', () => { }); }); - describe('processForUpload (auto mode)', () => { +describe('processForUpload (auto mode)', () => { + // Blobs must be >= BLOB_THRESHOLD (4096 bytes) to be offloaded. + // We create a base64 string representing 4096 zero bytes. + const bigB64 = btoa(String.fromCharCode(...new Array(4096).fill(0))); + it('replaces inline blobs with BlobRefs', async () => { fetchMock.mockResolvedValue(mockJsonResponse({ ref: '1:uploaded' })); @@ -165,7 +169,7 @@ describe('BlobManager', () => { name: 'test', avatar: { _bt: 'Blob', - v: btoa('fake-png-data'), + v: bigB64, ct: 'image/png', }, }; @@ -185,7 +189,7 @@ describe('BlobManager', () => { nested: { deep: { _bt: 'Uint8Array', - v: btoa('data'), + v: bigB64, }, }, }; @@ -201,8 +205,8 @@ describe('BlobManager', () => { .mockResolvedValueOnce(mockJsonResponse({ ref: '1:second' })); const arr = [ - { _bt: 'Blob', v: btoa('a') }, - { _bt: 'Blob', v: btoa('b') }, + { _bt: 'Blob', v: bigB64 }, + { _bt: 'Blob', v: bigB64 }, ]; const result = await manager.processForUpload(arr, TOKEN); @@ -210,6 +214,13 @@ describe('BlobManager', () => { expect(result[1].ref).toBe('1:second'); }); + it('keeps small blobs (< 4096 bytes) inline without uploading', async () => { + const smallObj = { _bt: 'Blob', v: btoa('tiny'), ct: 'image/png' }; + const result = await manager.processForUpload(smallObj, TOKEN); + expect(result).toEqual(smallObj); // unchanged + expect(fetchMock).not.toHaveBeenCalled(); + }); + it('passes through non-blob values unchanged', async () => { const obj = { id: '1', count: 42, tags: ['a', 'b'] }; const result = await manager.processForUpload(obj, TOKEN);