From 410d9da7ab82e72083e7cffe1e31aee0dae9a6d2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 21 Jun 2025 04:14:00 +0000 Subject: [PATCH 1/3] feat: implement multi-version PostgreSQL support with folder-based structure - Create lightweight versions for PG13-16 without deparse/scan functionality - Add full-featured PG17 version (libpg-query-full) - Implement libpg-query-multi package with PG15/16/17 and Parser class API - Add master Makefile for coordinated builds across all versions - Generate version-specific TypeScript types using PgProtoParser - Maintain existing test patterns for each version Key features: - Lightweight WASM builds exclude wasm_deparse_protobuf and wasm_scan functions - Parser class supports runtime version selection with error handling - Version-specific libpg_query tags: 13-2.2.0, 14-3.0.0, 15-4.2.4, 16-5.2.0, 17-6.1.0 - Type generation from GitHub proto files using pg-proto-parser - Comprehensive test suites for all packages - Bundle size optimization for lightweight versions Co-Authored-By: Dan Lynch --- PLAN.md | 859 ++++++++++++++++++++ libpg-query-13/Makefile | 60 ++ libpg-query-13/README.md | 34 + libpg-query-13/package.json | 33 + libpg-query-13/src/wasm_wrapper_light.c | 213 +++++ libpg-query-13/test/parsing.test.js | 95 +++ libpg-query-13/wasm/index.d.ts | 15 + libpg-query-13/wasm/index.js | 201 +++++ libpg-query-14/Makefile | 60 ++ libpg-query-14/README.md | 34 + libpg-query-14/package.json | 33 + libpg-query-14/src/wasm_wrapper_light.c | 213 +++++ libpg-query-14/test/parsing.test.js | 95 +++ libpg-query-14/wasm/index.d.ts | 15 + libpg-query-14/wasm/index.js | 201 +++++ libpg-query-15/Makefile | 60 ++ libpg-query-15/README.md | 34 + libpg-query-15/package.json | 33 + libpg-query-15/src/wasm_wrapper_light.c | 213 +++++ libpg-query-15/test/parsing.test.js | 95 +++ libpg-query-15/wasm/index.d.ts | 15 + libpg-query-15/wasm/index.js | 201 +++++ libpg-query-16/Makefile | 60 ++ libpg-query-16/README.md | 34 + libpg-query-16/package.json | 33 + libpg-query-16/src/wasm_wrapper_light.c | 213 +++++ libpg-query-16/test/parsing.test.js | 95 +++ libpg-query-16/wasm/index.d.ts | 15 + libpg-query-16/wasm/index.js | 201 +++++ libpg-query-full/Makefile | 60 ++ libpg-query-full/README.md | 48 ++ libpg-query-full/package.json | 37 + libpg-query-full/scripts/protogen.js | 36 + libpg-query-full/src/wasm_wrapper.c | 345 ++++++++ libpg-query-full/test/deparsing.test.js | 55 ++ libpg-query-full/test/parsing.test.js | 110 +++ libpg-query-full/test/scan.test.js | 68 ++ libpg-query-full/wasm/index.d.ts | 21 + libpg-query-full/wasm/index.js | 296 +++++++ libpg-query-multi/Makefile | 34 + libpg-query-multi/README.md | 99 +++ libpg-query-multi/index.d.ts | 66 ++ libpg-query-multi/index.js | 115 +++ libpg-query-multi/package.json | 48 ++ libpg-query-multi/scripts/generate-types.js | 50 ++ libpg-query-multi/test/parser.test.js | 136 ++++ 46 files changed, 5087 insertions(+) create mode 100644 PLAN.md create mode 100644 libpg-query-13/Makefile create mode 100644 libpg-query-13/README.md create mode 100644 libpg-query-13/package.json create mode 100644 libpg-query-13/src/wasm_wrapper_light.c create mode 100644 libpg-query-13/test/parsing.test.js create mode 100644 libpg-query-13/wasm/index.d.ts create mode 100644 libpg-query-13/wasm/index.js create mode 100644 libpg-query-14/Makefile create mode 100644 libpg-query-14/README.md create mode 100644 libpg-query-14/package.json create mode 100644 libpg-query-14/src/wasm_wrapper_light.c create mode 100644 libpg-query-14/test/parsing.test.js create mode 100644 libpg-query-14/wasm/index.d.ts create mode 100644 libpg-query-14/wasm/index.js create mode 100644 libpg-query-15/Makefile create mode 100644 libpg-query-15/README.md create mode 100644 libpg-query-15/package.json create mode 100644 libpg-query-15/src/wasm_wrapper_light.c create mode 100644 libpg-query-15/test/parsing.test.js create mode 100644 libpg-query-15/wasm/index.d.ts create mode 100644 libpg-query-15/wasm/index.js create mode 100644 libpg-query-16/Makefile create mode 100644 libpg-query-16/README.md create mode 100644 libpg-query-16/package.json create mode 100644 libpg-query-16/src/wasm_wrapper_light.c create mode 100644 libpg-query-16/test/parsing.test.js create mode 100644 libpg-query-16/wasm/index.d.ts create mode 100644 libpg-query-16/wasm/index.js create mode 100644 libpg-query-full/Makefile create mode 100644 libpg-query-full/README.md create mode 100644 libpg-query-full/package.json create mode 100644 libpg-query-full/scripts/protogen.js create mode 100644 libpg-query-full/src/wasm_wrapper.c create mode 100644 libpg-query-full/test/deparsing.test.js create mode 100644 libpg-query-full/test/parsing.test.js create mode 100644 libpg-query-full/test/scan.test.js create mode 100644 libpg-query-full/wasm/index.d.ts create mode 100644 libpg-query-full/wasm/index.js create mode 100644 libpg-query-multi/Makefile create mode 100644 libpg-query-multi/README.md create mode 100644 libpg-query-multi/index.d.ts create mode 100644 libpg-query-multi/index.js create mode 100644 libpg-query-multi/package.json create mode 100644 libpg-query-multi/scripts/generate-types.js create mode 100644 libpg-query-multi/test/parser.test.js diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 00000000..b3ef6df5 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,859 @@ +# Multi-Version PostgreSQL Support Plan + +## Overview + +This document outlines the architecture for supporting multiple PostgreSQL versions (PG13-17) in the libpg-query ecosystem using a folder-based structure. The plan creates lightweight versions for PG13-16 (without deparse/scan functionality), a full-featured PG17 version, and a multi-version package combining PG15/16/17. + +## Current Architecture Analysis + +### PG17 (Current Implementation) +- **WASM-only build system** using Emscripten +- **Full functionality**: parse, deparse, scan, fingerprint, normalize, parsePlPgSQL +- **Protocol Buffer integration** via 5.4MB proto.js file for deparse operations +- **Key WASM exports**: `wasm_deparse_protobuf`, `wasm_scan`, `wasm_parse_query_protobuf` +- **Dependencies**: `@pgsql/types@^17.0.0`, `@launchql/protobufjs@7.2.6` + +### Version Differences Identified +- **PG17**: Uses `LIBPG_QUERY_TAG := 17-6.1.0` with pure WASM wrapper (`src/wasm_wrapper.c`) +- **PG15**: Uses `LIBPG_QUERY_TAG := 15-4.2.4` with hybrid C++/WASM approach (`src/*.cc`) +- **Build system**: PG17 excludes native builds entirely, PG15 supports node-gyp fallback + +## Proposed Folder Structure + +``` +libpg-query-multi/ +├── libpg-query-13/ # Lightweight - no deparse/scan +│ ├── src/ +│ │ └── wasm_wrapper_light.c +│ ├── wasm/ +│ │ ├── index.js # Lightweight API +│ │ └── index.d.ts +│ ├── package.json # @pgsql/types@^13.0.0 +│ ├── Makefile # LIBPG_QUERY_TAG := 13-2.2.0 +│ └── README.md +├── libpg-query-14/ # Lightweight - no deparse/scan +│ ├── src/ +│ │ └── wasm_wrapper_light.c +│ ├── wasm/ +│ │ ├── index.js # Lightweight API +│ │ └── index.d.ts +│ ├── package.json # @pgsql/types@^14.0.0 +│ ├── Makefile # LIBPG_QUERY_TAG := 14-3.0.0 +│ └── README.md +├── libpg-query-15/ # Lightweight - no deparse/scan +│ ├── src/ +│ │ └── wasm_wrapper_light.c +│ ├── wasm/ +│ │ ├── index.js # Lightweight API +│ │ └── index.d.ts +│ ├── package.json # @pgsql/types@^15.0.0 +│ ├── Makefile # LIBPG_QUERY_TAG := 15-4.2.4 +│ └── README.md +├── libpg-query-16/ # Lightweight - no deparse/scan +│ ├── src/ +│ │ └── wasm_wrapper_light.c +│ ├── wasm/ +│ │ ├── index.js # Lightweight API +│ │ └── index.d.ts +│ ├── package.json # @pgsql/types@^16.0.0 +│ ├── Makefile # LIBPG_QUERY_TAG := 16-5.2.0 +│ └── README.md +├── libpg-query-full/ # PG17 - full functionality +│ ├── src/ +│ │ └── wasm_wrapper.c # Full WASM wrapper +│ ├── wasm/ +│ │ ├── index.js # Full API with deparse/scan +│ │ └── index.d.ts +│ ├── scripts/ +│ │ └── protogen.js # Protocol buffer generation +│ ├── proto.js # 5.4MB Protocol Buffer definitions +│ ├── package.json # @pgsql/types@^17.0.0 +│ ├── Makefile # LIBPG_QUERY_TAG := 17-6.1.0 +│ └── README.md +├── libpg-query-multi/ # PG15/16/17 in single package +│ ├── pg15/ # Lightweight PG15 +│ ├── pg16/ # Lightweight PG16 +│ ├── pg17/ # Full PG17 +│ ├── index.js # Version selector API +│ ├── package.json # Multi-version exports +│ ├── Makefile # Builds all three versions +│ └── README.md +├── scripts/ # Shared build utilities +│ ├── build-lightweight.sh +│ ├── build-full.sh +│ └── version-config.js +├── Makefile # Master build coordinator +└── README.md # Multi-version documentation +``` + +## Lightweight Version Implementation + +### WASM Wrapper Modifications (`src/wasm_wrapper_light.c`) + +**Excluded Functions:** +- `wasm_deparse_protobuf()` - Protocol buffer deparse functionality +- `wasm_scan()` - SQL tokenization functionality +- `wasm_parse_query_protobuf()` - Protocol buffer parsing +- `wasm_get_protobuf_len()` - Protocol buffer length calculation + +**Retained Functions:** +- `wasm_parse_query()` - Core SQL parsing +- `wasm_parse_plpgsql()` - PL/pgSQL parsing +- `wasm_fingerprint()` - Query fingerprinting +- `wasm_normalize_query()` - Query normalization +- `wasm_parse_query_detailed()` - Detailed parsing with error info +- `wasm_free_detailed_result()` - Memory cleanup +- `wasm_free_string()` - String memory cleanup + +### Makefile Configuration + +**Lightweight Versions (PG13-16):** +```makefile +LIBPG_QUERY_TAG := 13-2.2.0 # Version-specific +SRC_FILES := src/wasm_wrapper_light.c + +# Reduced WASM exports +-sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" +``` + +**Full Version (PG17):** +```makefile +LIBPG_QUERY_TAG := 17-6.1.0 +SRC_FILES := src/wasm_wrapper.c + +# Complete WASM exports +-sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_query_protobuf','_wasm_get_protobuf_len','_wasm_deparse_protobuf','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" +``` + +### JavaScript API Modifications (`wasm/index.js`) + +**Lightweight API (PG13-16):** +```javascript +// Excluded imports +// import { pg_query } from '../proto.js'; // NO proto.js dependency + +// Retained exports +export { parse, parseSync } from './core.js'; +export { parsePlPgSQL, parsePlPgSQLSync } from './core.js'; +export { fingerprint, fingerprintSync } from './core.js'; +export { normalize, normalizeSync } from './core.js'; +export { loadModule } from './core.js'; + +// Excluded exports (deparse/scan functionality) +// export { deparse, deparseSync } from './deparse.js'; +// export { scan, scanSync } from './scan.js'; +``` + +**Full API (PG17):** +```javascript +// Complete API including deparse/scan +export * from "@pgsql/types"; +import { pg_query } from '../proto.js'; + +export { parse, parseSync } from './core.js'; +export { deparse, deparseSync } from './deparse.js'; +export { scan, scanSync } from './scan.js'; +export { parsePlPgSQL, parsePlPgSQLSync } from './core.js'; +export { fingerprint, fingerprintSync } from './core.js'; +export { normalize, normalizeSync } from './core.js'; +export { loadModule } from './core.js'; +``` + +## libpg-query-multi Package Design + +### Version Selector API (`libpg-query-multi/index.js`) + +```javascript +// Named version exports +export { + parse as parse15, + fingerprint as fingerprint15, + normalize as normalize15, + parsePlPgSQL as parsePlPgSQL15 +} from './pg15/index.js'; + +export { + parse as parse16, + fingerprint as fingerprint16, + normalize as normalize16, + parsePlPgSQL as parsePlPgSQL16 +} from './pg16/index.js'; + +export { + parse as parse17, + fingerprint as fingerprint17, + deparse as deparse17, + scan as scan17, + normalize as normalize17, + parsePlPgSQL as parsePlPgSQL17 +} from './pg17/index.js'; + +// Default to latest (PG17) +export { + parse, + fingerprint, + deparse, + scan, + normalize, + parsePlPgSQL +} from './pg17/index.js'; + +### Type Generation Script for libpg-query-multi + +**scripts/generate-types.js:** +```javascript +import { PgProtoParser, PgProtoParserOptions } from 'pg-proto-parser'; +import { writeFileSync, mkdirSync } from 'fs'; +import { join } from 'path'; + +const versions = [ + { version: '15', tag: '15-4.2.4' }, + { version: '16', tag: '16-5.2.0' }, + { version: '17', tag: '17-6.1.0' } +]; + +async function generateTypes() { + for (const { version, tag } of versions) { + const protoUrl = `https://raw.githubusercontent.com/pganalyze/libpg_query/refs/tags/${tag}/protobuf/pg_query.proto`; + const outDir = `./pg${version}/types`; + + // Ensure output directory exists + mkdirSync(outDir, { recursive: true }); + + // Fetch proto file + const response = await fetch(protoUrl); + const protoContent = await response.text(); + const protoFile = join(outDir, 'pg_query.proto'); + writeFileSync(protoFile, protoContent); + + // Configure pg-proto-parser + const options = { + outDir, + types: { + enabled: true, + wrappedNodeTypeExport: true, + optionalFields: true, + filename: 'types.d.ts', + enumsSource: './enums.js', + }, + enums: { + enabled: true, + enumsAsTypeUnion: true, + filename: 'enums.d.ts', + }, + }; + + // Generate types + const parser = new PgProtoParser(protoFile, options); + await parser.write(); + + console.log(`Generated types for PostgreSQL ${version} (${tag})`); + } +} + +generateTypes().catch(console.error); +``` + + + +// Runtime version selector +export async function createParser(version) { + switch(version) { + case 15: return await import('./pg15/index.js'); + case 16: return await import('./pg16/index.js'); + case 17: return await import('./pg17/index.js'); + default: throw new Error(`Unsupported PostgreSQL version: ${version}`); + } +} + +// Version detection utility +export function getSupportedVersions() { + return [15, 16, 17]; +} + +// Feature detection +export function getVersionFeatures(version) { + const baseFeatures = ['parse', 'fingerprint', 'normalize', 'parsePlPgSQL']; + const fullFeatures = [...baseFeatures, 'deparse', 'scan']; + + switch(version) { + case 15: + case 16: return baseFeatures; + case 17: return fullFeatures; + default: throw new Error(`Unknown version: ${version}`); + } +} +``` + +### Package Configuration (`libpg-query-multi/package.json`) + +```json +{ + "name": "libpg-query-multi", + "version": "1.0.0", + "description": "Multi-version PostgreSQL query parser (PG15/16/17)", + "main": "./index.js", + "types": "./index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.js", + "require": "./index.cjs" + }, + "./pg15": { + "types": "./pg15/index.d.ts", + "import": "./pg15/index.js", + "require": "./pg15/index.cjs" + }, + "./pg16": { + "types": "./pg16/index.d.ts", + "import": "./pg16/index.js", + "require": "./pg16/index.cjs" + }, + "./pg17": { + "types": "./pg17/index.d.ts", + "import": "./pg17/index.js", + "require": "./pg17/index.cjs" + } + }, + "files": [ + "pg15/**/*", + "pg16/**/*", + "pg17/**/*", + "index.js", + "index.d.ts", + "index.cjs" + ], + "dependencies": { + "@pgsql/types": "^17.0.0", + "@launchql/protobufjs": "7.2.6" + }, + "keywords": [ + "sql", "postgres", "postgresql", "pg", "query", + "multi-version", "pg15", "pg16", "pg17" + ] +} +``` + +## Master Makefile System + +### Root Makefile (`Makefile`) + +```makefile +# Version configuration +VERSIONS := 13 14 15 16 17 +LIGHTWEIGHT_VERSIONS := 13 14 15 16 +FULL_VERSION := 17 + +# Directory configuration +LIGHTWEIGHT_DIRS := $(addprefix libpg-query-,$(LIGHTWEIGHT_VERSIONS)) +FULL_DIR := libpg-query-full +MULTI_DIR := libpg-query-multi + +# Docker configuration for consistent builds +EMSCRIPTEN_IMAGE := emscripten/emsdk:latest +DOCKER_RUN := docker run --rm -v $(PWD):/src -w /src -u $(shell id -u):$(shell id -g) + +.PHONY: build-all build-lightweight build-full build-multi clean-all test-all publish-all + +# Main targets +build-all: build-lightweight build-full build-multi + +build-lightweight: + @echo "Building lightweight versions (PG$(LIGHTWEIGHT_VERSIONS))..." + @for dir in $(LIGHTWEIGHT_DIRS); do \ + echo "Building $$dir..."; \ + $(MAKE) -C $$dir build || exit 1; \ + done + +build-full: + @echo "Building full version (PG$(FULL_VERSION))..." + $(MAKE) -C $(FULL_DIR) build + +build-multi: + @echo "Building multi-version package..." + $(MAKE) -C $(MULTI_DIR) build + +# Individual version targets +build-13: + $(MAKE) -C libpg-query-13 build + +build-14: + $(MAKE) -C libpg-query-14 build + +build-15: + $(MAKE) -C libpg-query-15 build + +build-16: + $(MAKE) -C libpg-query-16 build + +build-17: + $(MAKE) -C libpg-query-full build + +# Testing +test-all: + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + echo "Testing $$dir..."; \ + $(MAKE) -C $$dir test || exit 1; \ + done + +# Cleanup +clean-all: + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + echo "Cleaning $$dir..."; \ + $(MAKE) -C $$dir clean; \ + done + +# Docker-based builds for consistency +docker-build-all: + $(DOCKER_RUN) $(EMSCRIPTEN_IMAGE) make build-all + +docker-test-all: + $(DOCKER_RUN) $(EMSCRIPTEN_IMAGE) make test-all + +# Publishing (requires npm authentication) +publish-all: build-all test-all + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + echo "Publishing $$dir..."; \ + cd $$dir && npm publish && cd ..; \ + done + +# Development helpers +dev-setup: + @echo "Setting up development environment..." + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + echo "Installing dependencies for $$dir..."; \ + cd $$dir && npm install && cd ..; \ + done + +# Version bumping +bump-patch: + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + cd $$dir && npm version patch && cd ..; \ + done + +bump-minor: + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + cd $$dir && npm version minor && cd ..; \ + done + +bump-major: + @for dir in $(LIGHTWEIGHT_DIRS) $(FULL_DIR) $(MULTI_DIR); do \ + cd $$dir && npm version major && cd ..; \ + done +``` + +## Version-Specific Configuration + +### libpg_query Branch Mapping + +| PostgreSQL Version | libpg_query Branch | Package Name | Features | +|-------------------|-------------------|--------------|----------| +| PG13 | `13-2.2.0` | `libpg-query-13` | parse, fingerprint, normalize, parsePlPgSQL | +| PG14 | `14-3.0.0` | `libpg-query-14` | parse, fingerprint, normalize, parsePlPgSQL | +| PG15 | `15-4.2.4` | `libpg-query-15` | parse, fingerprint, normalize, parsePlPgSQL | +| PG16 | `16-5.2.0` | `libpg-query-16` | parse, fingerprint, normalize, parsePlPgSQL | +| PG17 | `17-6.1.0` | `libpg-query-full` | parse, deparse, scan, fingerprint, normalize, parsePlPgSQL | + +### Package Dependencies + +```json +// libpg-query-13/package.json +{ + "name": "libpg-query-13", + "version": "13.1.0", + "dependencies": { + "@pgsql/types": "^13.0.0" + } +} + +// libpg-query-14/package.json +{ + "name": "libpg-query-14", + "version": "14.1.0", + "dependencies": { + "@pgsql/types": "^14.0.0" + } +} + +// libpg-query-15/package.json +{ + "name": "libpg-query-15", + "version": "15.1.0", + "dependencies": { + "@pgsql/types": "^15.0.0" + } +} + +// libpg-query-16/package.json +{ + "name": "libpg-query-16", + "version": "16.1.0", + "dependencies": { + "@pgsql/types": "^16.0.0" + } +} + +// libpg-query-full/package.json +{ + "name": "libpg-query-full", + "version": "17.2.0", + "dependencies": { + "@pgsql/types": "^17.0.0", + "@launchql/protobufjs": "7.2.6" + } +} +``` + +## Proto.js Exclusion Strategy + +### Lightweight Versions (PG13-16) +- **No proto.js dependency** - excludes 5.4MB Protocol Buffer definitions +- **No protogen script** - skips Protocol Buffer generation step +- **Reduced package size** - significantly smaller WASM bundles +- **Build optimization** - faster builds without protobuf compilation + +### Full Version (PG17) & Multi Package +- **Includes proto.js** - full Protocol Buffer support for deparse operations +- **Protogen integration** - generates proto.js from libpg_query definitions +- **Complete functionality** - supports all AST serialization operations + +### Build Script Modifications + +```makefile +# Lightweight version Makefile +protogen: + @echo "Skipping protogen for lightweight version" + +build: wasm-build + @echo "Lightweight build complete (no proto.js)" + +# Full version Makefile +protogen: + node scripts/protogen.js + +build: protogen wasm-build + @echo "Full build complete (with proto.js)" +``` + +## API Consistency Strategy + +### Common API Surface +All versions provide consistent APIs for shared functionality: + +```typescript +// Common interface across all versions +interface CommonParser { + parse(query: string): Promise; + parseSync(query: string): ParseResult; + parsePlPgSQL(funcsSql: string): Promise; + parsePlPgSQLSync(funcsSql: string): ParseResult; + fingerprint(sql: string): Promise; + fingerprintSync(sql: string): string; + normalize(sql: string): Promise; + normalizeSync(sql: string): string; + loadModule(): Promise; +} + +// Extended interface for full version +interface FullParser extends CommonParser { + deparse(parseTree: ParseResult): Promise; + deparseSync(parseTree: ParseResult): string; + scan(sql: string): Promise; + scanSync(sql: string): ScanResult; +} +``` + +### Error Handling +Consistent error handling across versions: + +```javascript +// Lightweight versions +export function deparse() { + throw new Error('deparse() not available in lightweight version. Use libpg-query-full or libpg-query-multi for deparse functionality.'); +} + +export function scan() { + throw new Error('scan() not available in lightweight version. Use libpg-query-full or libpg-query-multi for scan functionality.'); +} +``` + +## Migration Guide + +### From Current Single-Version Approach + +**Before (current):** +```javascript +import { parse, deparse, scan } from 'libpg-query'; +``` + +**After (version-specific):** +```javascript +// Option 1: Use specific version +import { parse } from 'libpg-query-15'; // Lightweight +import { parse, deparse, scan } from 'libpg-query-full'; // Full PG17 + +// Option 2: Use multi-version package with Parser class +import { Parser } from 'libpg-query-multi'; + +const parser = new Parser(); // Defaults to latest version (17) +const { tree } = await parser.parse('SELECT * FROM users WHERE id = 1'); + +// Or specify a version +const pg15Parser = new Parser({ version: 15 }); // Use Postgres 15 parser +const result = await pg15Parser.parse('SELECT * FROM users'); + +// Option 3: Default to latest +import { parse, deparse, scan } from 'libpg-query-multi'; // Uses PG17 +``` + +### Package Selection Guide + +| Use Case | Recommended Package | Rationale | +|----------|-------------------|-----------| +| Parse-only applications | `libpg-query-13/14/15/16` | Smallest bundle size | +| Full SQL manipulation | `libpg-query-full` | Complete PG17 functionality | +| Multi-version support | `libpg-query-multi` | Runtime version selection | +| Legacy PG compatibility | `libpg-query-15` | Stable PG15 support | +| Latest features | `libpg-query-full` | Cutting-edge PG17 | + +## Testing Strategy + +### Unit Testing +Each version maintains its own test suite: + +```javascript +// libpg-query-13/test/parse.test.js +describe('PG13 Parser', () => { + it('should parse basic SELECT', async () => { + const result = await parse('SELECT * FROM users'); + expect(result.stmts).to.have.length(1); + }); + + it('should throw on deparse attempt', () => { + expect(() => deparse({})).to.throw('deparse() not available'); + }); +}); + +// libpg-query-full/test/deparse.test.js +describe('PG17 Full Parser', () => { + it('should deparse AST back to SQL', async () => { + const ast = await parse('SELECT * FROM users'); + const sql = await deparse(ast); + expect(sql).to.include('SELECT'); + }); +}); + +// libpg-query-multi/test/version-selector.test.js +describe('Multi-Version Selector', () => { + it('should load correct version', async () => { + const pg15 = new Parser({ version: 15 }); + expect(await pg15.parse('SELECT 1')).to.have.property('tree'); + + // Should throw error for unavailable functionality + try { + await pg15.deparse({}); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.include('Deparse not available for PostgreSQL 15'); + } + }); +}); +``` + +### Integration Testing +Cross-version compatibility testing: + +```javascript +// test/integration/cross-version.test.js +describe('Cross-Version Compatibility', () => { + it('should produce compatible ASTs', async () => { + const sql = 'SELECT id, name FROM users WHERE active = true'; + + const pg15Result = await pg15.parse(sql); + const pg17Result = await pg17.parse(sql); + + // Core AST structure should be compatible + expect(pg15Result.stmts[0].stmt_type).to.equal(pg17Result.stmts[0].stmt_type); + }); +}); +``` + +### Performance Testing +Bundle size and runtime performance validation: + +```javascript +// test/performance/bundle-size.test.js +describe('Bundle Size Validation', () => { + it('lightweight versions should be under 2MB', () => { + const lightweightSizes = [ + getPackageSize('libpg-query-13'), + getPackageSize('libpg-query-14'), + getPackageSize('libpg-query-15'), + getPackageSize('libpg-query-16') + ]; + + lightweightSizes.forEach(size => { + expect(size).to.be.below(2 * 1024 * 1024); // 2MB + }); + }); + + it('full version should include proto.js overhead', () => { + const fullSize = getPackageSize('libpg-query-full'); + expect(fullSize).to.be.above(5 * 1024 * 1024); // >5MB due to proto.js + }); +}); +``` + +## Deployment Strategy + +### NPM Publishing +Coordinated publishing across all packages: + +```bash +# Build all versions +make build-all + +# Test all versions +make test-all + +# Publish all packages +make publish-all +``` + +### Version Synchronization +Maintain consistent versioning across related packages: + +```json +// scripts/sync-versions.js +{ + "libpg-query-13": "13.1.0", + "libpg-query-14": "14.1.0", + "libpg-query-15": "15.1.0", + "libpg-query-16": "16.1.0", + "libpg-query-full": "17.2.0", + "libpg-query-multi": "1.0.0" +} +``` + +### CI/CD Integration +GitHub Actions workflow for automated builds: + +```yaml +# .github/workflows/multi-version-ci.yml +name: Multi-Version CI +on: [push, pull_request] + +jobs: + build-lightweight: + runs-on: ubuntu-latest + strategy: + matrix: + version: [13, 14, 15, 16] + steps: + - uses: actions/checkout@v3 + - name: Build PG${{ matrix.version }} + run: make build-${{ matrix.version }} + - name: Test PG${{ matrix.version }} + run: make -C libpg-query-${{ matrix.version }} test + + build-full: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build Full Version + run: make build-full + - name: Test Full Version + run: make -C libpg-query-full test + + build-multi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build Multi Package + run: make build-multi + - name: Test Multi Package + run: make -C libpg-query-multi test +``` + +## Implementation Timeline + +### Phase 1: Foundation (Week 1-2) +1. Create folder structure +2. Implement lightweight WASM wrapper (`wasm_wrapper_light.c`) +3. Create master Makefile +4. Set up shared build scripts + +### Phase 2: Lightweight Versions (Week 3-4) +1. Implement PG13-16 packages +2. Create lightweight JavaScript APIs +3. Configure version-specific Makefiles +4. Implement unit tests + +### Phase 3: Full Version (Week 5) +1. Migrate current PG17 implementation to `libpg-query-full` +2. Ensure proto.js integration works +3. Validate deparse/scan functionality +4. Update documentation + +### Phase 4: Multi-Version Package (Week 6) +1. Implement `libpg-query-multi` structure +2. Create version selector API +3. Build PG15/16/17 integration +4. Implement runtime version switching + +### Phase 5: Testing & Documentation (Week 7-8) +1. Comprehensive testing across all versions +2. Performance validation +3. Bundle size optimization +4. Documentation completion +5. Migration guide creation + +### Phase 6: Deployment (Week 9) +1. CI/CD pipeline setup +2. NPM package publishing +3. Version synchronization +4. Release coordination + +## Risk Mitigation + +### Compatibility Risks +- **AST Structure Changes**: Maintain compatibility matrices between versions +- **API Breaking Changes**: Use semantic versioning and deprecation warnings +- **Build Environment**: Use Docker for consistent build environments + +### Maintenance Overhead +- **Shared Components**: Extract common build logic to shared scripts +- **Automated Testing**: Comprehensive CI/CD for all versions +- **Documentation**: Keep version-specific docs synchronized + +### Performance Concerns +- **Bundle Size**: Monitor and optimize WASM bundle sizes +- **Runtime Overhead**: Benchmark version selection performance +- **Memory Usage**: Validate memory cleanup across versions + +## Success Metrics + +### Technical Metrics +- **Bundle Size Reduction**: Lightweight versions <2MB (vs current 7.4MB) +- **Build Time**: <5 minutes for all versions +- **Test Coverage**: >90% across all packages +- **API Compatibility**: 100% backward compatibility for shared functions + +### User Experience Metrics +- **Migration Effort**: <1 hour for typical applications +- **Documentation Quality**: Complete API docs for all versions +- **Error Messages**: Clear guidance for missing functionality +- **Performance**: No regression in parse performance + +## Conclusion + +This multi-version architecture provides: + +1. **Lightweight Options**: PG13-16 packages without deparse/scan overhead +2. **Full Functionality**: PG17 with complete feature set +3. **Flexible Integration**: Multi-version package for complex applications +4. **Maintainable Structure**: Folder-based organization with shared tooling +5. **Scalable Builds**: Master Makefile coordinating all versions + +The approach balances functionality, performance, and maintainability while providing clear migration paths for existing users and optimal package selection for new projects. diff --git a/libpg-query-13/Makefile b/libpg-query-13/Makefile new file mode 100644 index 00000000..2cd02d33 --- /dev/null +++ b/libpg-query-13/Makefile @@ -0,0 +1,60 @@ +LIBPG_QUERY_TAG := 13-2.2.0 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + PLATFORM := linux +endif +ifeq ($(UNAME_S),Darwin) + PLATFORM := darwin +endif + +ifeq ($(UNAME_M),x86_64) + ARCH := x64 +endif +ifeq ($(UNAME_M),arm64) + ARCH := arm64 +endif + +LIBPG_QUERY_DIR := libpg_query +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +WASM_DIR := wasm +WASM_FILE := $(WASM_DIR)/libpg-query.wasm +WASM_JS_FILE := $(WASM_DIR)/libpg-query.js + +.PHONY: build clean clean-cache + +build: $(WASM_FILE) + +$(WASM_FILE): $(LIBPG_QUERY_LIB) src/wasm_wrapper_light.c + @echo "Building WASM module for PostgreSQL 13..." + emcc -O3 \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/src/postgres/include \ + src/wasm_wrapper_light.c \ + $(LIBPG_QUERY_LIB) \ + -o $(WASM_FILE) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \ + -sEXPORTED_RUNTIME_METHODS="['UTF8ToString','stringToUTF8','lengthBytesUTF8']" \ + -sMODULARIZE=1 \ + -sEXPORT_NAME="PgQueryModule" \ + -sENVIRONMENT=web,node \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=16777216 \ + -sSTACK_SIZE=1048576 \ + -sNO_FILESYSTEM=1 \ + -sNO_EXIT_RUNTIME=1 + +$(LIBPG_QUERY_LIB): + @echo "Cloning and building libpg_query $(LIBPG_QUERY_TAG)..." + rm -rf $(LIBPG_QUERY_DIR) + git clone --depth 1 --branch $(LIBPG_QUERY_TAG) https://github.com/pganalyze/libpg_query.git $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && make build + +clean: + rm -rf $(WASM_DIR)/*.wasm $(WASM_DIR)/*.js $(WASM_DIR)/*.wast + +clean-cache: + rm -rf $(LIBPG_QUERY_DIR) diff --git a/libpg-query-13/README.md b/libpg-query-13/README.md new file mode 100644 index 00000000..97c50964 --- /dev/null +++ b/libpg-query-13/README.md @@ -0,0 +1,34 @@ +# libpg-query-13 + +PostgreSQL 13 query parser (lightweight version) + +## Features + +- Parse SQL queries into AST +- Generate query fingerprints +- Normalize queries +- Parse PL/pgSQL functions + +## Installation + +```bash +npm install libpg-query-13 +``` + +## Usage + +```javascript +import { parse, fingerprint, normalize } from 'libpg-query-13'; + +const ast = await parse('SELECT * FROM users'); +const fp = await fingerprint('SELECT * FROM users WHERE id = $1'); +const normalized = await normalize('SELECT * FROM users WHERE id = 123'); +``` + +## Limitations + +This is a lightweight version that does not include: +- Deparse functionality +- Scan functionality + +For full functionality, use `libpg-query-full` or `libpg-query-multi`. diff --git a/libpg-query-13/package.json b/libpg-query-13/package.json new file mode 100644 index 00000000..21fde94d --- /dev/null +++ b/libpg-query-13/package.json @@ -0,0 +1,33 @@ +{ + "name": "libpg-query-13", + "version": "13.2.0", + "description": "PostgreSQL 13 query parser (lightweight - no deparse/scan)", + "main": "wasm/index.js", + "types": "wasm/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./wasm/index.js", + "types": "./wasm/index.d.ts" + } + }, + "files": [ + "wasm/", + "README.md" + ], + "scripts": { + "build": "EMSCRIPTEN=1 make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000" + }, + "dependencies": { + "@pgsql/types": "^13.0.0" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + }, + "keywords": ["postgresql", "parser", "sql", "pg13"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-13/src/wasm_wrapper_light.c b/libpg-query-13/src/wasm_wrapper_light.c new file mode 100644 index 00000000..8429ef5c --- /dev/null +++ b/libpg-query-13/src/wasm_wrapper_light.c @@ -0,0 +1,213 @@ +#include "pg_query.h" +#include +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/libpg-query-13/test/parsing.test.js b/libpg-query-13/test/parsing.test.js new file mode 100644 index 00000000..a4e11ccd --- /dev/null +++ b/libpg-query-13/test/parsing.test.js @@ -0,0 +1,95 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("PostgreSQL 13 Query Parsing (Lightweight)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + expect(res.stmts).to.have.lengthOf(1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + expect(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival).to.eq(1); + expect(selectedDatas[1][0].ResTarget.val.A_Const.isnull).to.eq(true); + expect(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval).to.eq(""); + expect(selectedDatas[3]).to.have.lengthOf(2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + expect(res.stmts.map(removeLocationProperties)).to.deep.eq([ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + expect(() => query.parseSync("NOT A QUERY")).to.throw(Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + expect(resPromise).to.be.instanceof(Promise); + expect(res).to.deep.eq(query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + expect(e).instanceof(Error); + expect(e.message).to.match(/NOT/); + } + ); + }); + }); + + describe("Lightweight version restrictions", () => { + it("should not have deparse functionality", () => { + expect(query.deparse).to.be.undefined; + expect(query.deparseSync).to.be.undefined; + }); + + it("should not have scan functionality", () => { + expect(query.scan).to.be.undefined; + expect(query.scanSync).to.be.undefined; + }); + }); +}); diff --git a/libpg-query-13/wasm/index.d.ts b/libpg-query-13/wasm/index.d.ts new file mode 100644 index 00000000..3bdd7528 --- /dev/null +++ b/libpg-query-13/wasm/index.d.ts @@ -0,0 +1,15 @@ +export * from "@pgsql/types"; + +export function loadModule(): Promise; + +export function parse(query: string): Promise; +export function parseSync(query: string): any; + +export function parsePlPgSQL(query: string): Promise; +export function parsePlPgSQLSync(query: string): any; + +export function fingerprint(query: string): Promise; +export function fingerprintSync(query: string): string; + +export function normalize(query: string): Promise; +export function normalizeSync(query: string): string; diff --git a/libpg-query-13/wasm/index.js b/libpg-query-13/wasm/index.js new file mode 100644 index 00000000..88fd30d0 --- /dev/null +++ b/libpg-query-13/wasm/index.js @@ -0,0 +1,201 @@ +export * from "@pgsql/types"; +import PgQueryModule from './libpg-query.js'; + +let wasmModule; +const initPromise = PgQueryModule().then((module) => { + wasmModule = module; +}); + +export async function loadModule() { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit(fn) { + return (async (...args) => { + await initPromise; + return fn(...args); + }); +} + +function stringToPtr(str) { + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } + catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr) { + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function parseSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} diff --git a/libpg-query-14/Makefile b/libpg-query-14/Makefile new file mode 100644 index 00000000..569e34d4 --- /dev/null +++ b/libpg-query-14/Makefile @@ -0,0 +1,60 @@ +LIBPG_QUERY_TAG := 14-3.0.0 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + PLATFORM := linux +endif +ifeq ($(UNAME_S),Darwin) + PLATFORM := darwin +endif + +ifeq ($(UNAME_M),x86_64) + ARCH := x64 +endif +ifeq ($(UNAME_M),arm64) + ARCH := arm64 +endif + +LIBPG_QUERY_DIR := libpg_query +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +WASM_DIR := wasm +WASM_FILE := $(WASM_DIR)/libpg-query.wasm +WASM_JS_FILE := $(WASM_DIR)/libpg-query.js + +.PHONY: build clean clean-cache + +build: $(WASM_FILE) + +$(WASM_FILE): $(LIBPG_QUERY_LIB) src/wasm_wrapper_light.c + @echo "Building WASM module for PostgreSQL 14..." + emcc -O3 \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/src/postgres/include \ + src/wasm_wrapper_light.c \ + $(LIBPG_QUERY_LIB) \ + -o $(WASM_FILE) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \ + -sEXPORTED_RUNTIME_METHODS="['UTF8ToString','stringToUTF8','lengthBytesUTF8']" \ + -sMODULARIZE=1 \ + -sEXPORT_NAME="PgQueryModule" \ + -sENVIRONMENT=web,node \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=16777216 \ + -sSTACK_SIZE=1048576 \ + -sNO_FILESYSTEM=1 \ + -sNO_EXIT_RUNTIME=1 + +$(LIBPG_QUERY_LIB): + @echo "Cloning and building libpg_query $(LIBPG_QUERY_TAG)..." + rm -rf $(LIBPG_QUERY_DIR) + git clone --depth 1 --branch $(LIBPG_QUERY_TAG) https://github.com/pganalyze/libpg_query.git $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && make build + +clean: + rm -rf $(WASM_DIR)/*.wasm $(WASM_DIR)/*.js $(WASM_DIR)/*.wast + +clean-cache: + rm -rf $(LIBPG_QUERY_DIR) diff --git a/libpg-query-14/README.md b/libpg-query-14/README.md new file mode 100644 index 00000000..fd4c7dda --- /dev/null +++ b/libpg-query-14/README.md @@ -0,0 +1,34 @@ +# libpg-query-14 + +PostgreSQL 14 query parser (lightweight version) + +## Features + +- Parse SQL queries into AST +- Generate query fingerprints +- Normalize queries +- Parse PL/pgSQL functions + +## Installation + +```bash +npm install libpg-query-14 +``` + +## Usage + +```javascript +import { parse, fingerprint, normalize } from 'libpg-query-14'; + +const ast = await parse('SELECT * FROM users'); +const fp = await fingerprint('SELECT * FROM users WHERE id = $1'); +const normalized = await normalize('SELECT * FROM users WHERE id = 123'); +``` + +## Limitations + +This is a lightweight version that does not include: +- Deparse functionality +- Scan functionality + +For full functionality, use `libpg-query-full` or `libpg-query-multi`. diff --git a/libpg-query-14/package.json b/libpg-query-14/package.json new file mode 100644 index 00000000..dd4d9155 --- /dev/null +++ b/libpg-query-14/package.json @@ -0,0 +1,33 @@ +{ + "name": "libpg-query-14", + "version": "14.3.0", + "description": "PostgreSQL 14 query parser (lightweight - no deparse/scan)", + "main": "wasm/index.js", + "types": "wasm/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./wasm/index.js", + "types": "./wasm/index.d.ts" + } + }, + "files": [ + "wasm/", + "README.md" + ], + "scripts": { + "build": "EMSCRIPTEN=1 make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000" + }, + "dependencies": { + "@pgsql/types": "^14.0.0" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + }, + "keywords": ["postgresql", "parser", "sql", "pg14"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-14/src/wasm_wrapper_light.c b/libpg-query-14/src/wasm_wrapper_light.c new file mode 100644 index 00000000..8429ef5c --- /dev/null +++ b/libpg-query-14/src/wasm_wrapper_light.c @@ -0,0 +1,213 @@ +#include "pg_query.h" +#include +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/libpg-query-14/test/parsing.test.js b/libpg-query-14/test/parsing.test.js new file mode 100644 index 00000000..5130730c --- /dev/null +++ b/libpg-query-14/test/parsing.test.js @@ -0,0 +1,95 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("PostgreSQL 14 Query Parsing (Lightweight)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + expect(res.stmts).to.have.lengthOf(1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + expect(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival).to.eq(1); + expect(selectedDatas[1][0].ResTarget.val.A_Const.isnull).to.eq(true); + expect(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval).to.eq(""); + expect(selectedDatas[3]).to.have.lengthOf(2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + expect(res.stmts.map(removeLocationProperties)).to.deep.eq([ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + expect(() => query.parseSync("NOT A QUERY")).to.throw(Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + expect(resPromise).to.be.instanceof(Promise); + expect(res).to.deep.eq(query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + expect(e).instanceof(Error); + expect(e.message).to.match(/NOT/); + } + ); + }); + }); + + describe("Lightweight version restrictions", () => { + it("should not have deparse functionality", () => { + expect(query.deparse).to.be.undefined; + expect(query.deparseSync).to.be.undefined; + }); + + it("should not have scan functionality", () => { + expect(query.scan).to.be.undefined; + expect(query.scanSync).to.be.undefined; + }); + }); +}); diff --git a/libpg-query-14/wasm/index.d.ts b/libpg-query-14/wasm/index.d.ts new file mode 100644 index 00000000..3bdd7528 --- /dev/null +++ b/libpg-query-14/wasm/index.d.ts @@ -0,0 +1,15 @@ +export * from "@pgsql/types"; + +export function loadModule(): Promise; + +export function parse(query: string): Promise; +export function parseSync(query: string): any; + +export function parsePlPgSQL(query: string): Promise; +export function parsePlPgSQLSync(query: string): any; + +export function fingerprint(query: string): Promise; +export function fingerprintSync(query: string): string; + +export function normalize(query: string): Promise; +export function normalizeSync(query: string): string; diff --git a/libpg-query-14/wasm/index.js b/libpg-query-14/wasm/index.js new file mode 100644 index 00000000..88fd30d0 --- /dev/null +++ b/libpg-query-14/wasm/index.js @@ -0,0 +1,201 @@ +export * from "@pgsql/types"; +import PgQueryModule from './libpg-query.js'; + +let wasmModule; +const initPromise = PgQueryModule().then((module) => { + wasmModule = module; +}); + +export async function loadModule() { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit(fn) { + return (async (...args) => { + await initPromise; + return fn(...args); + }); +} + +function stringToPtr(str) { + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } + catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr) { + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function parseSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} diff --git a/libpg-query-15/Makefile b/libpg-query-15/Makefile new file mode 100644 index 00000000..6b4fd8e3 --- /dev/null +++ b/libpg-query-15/Makefile @@ -0,0 +1,60 @@ +LIBPG_QUERY_TAG := 15-4.2.4 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + PLATFORM := linux +endif +ifeq ($(UNAME_S),Darwin) + PLATFORM := darwin +endif + +ifeq ($(UNAME_M),x86_64) + ARCH := x64 +endif +ifeq ($(UNAME_M),arm64) + ARCH := arm64 +endif + +LIBPG_QUERY_DIR := libpg_query +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +WASM_DIR := wasm +WASM_FILE := $(WASM_DIR)/libpg-query.wasm +WASM_JS_FILE := $(WASM_DIR)/libpg-query.js + +.PHONY: build clean clean-cache + +build: $(WASM_FILE) + +$(WASM_FILE): $(LIBPG_QUERY_LIB) src/wasm_wrapper_light.c + @echo "Building WASM module for PostgreSQL 15..." + emcc -O3 \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/src/postgres/include \ + src/wasm_wrapper_light.c \ + $(LIBPG_QUERY_LIB) \ + -o $(WASM_FILE) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \ + -sEXPORTED_RUNTIME_METHODS="['UTF8ToString','stringToUTF8','lengthBytesUTF8']" \ + -sMODULARIZE=1 \ + -sEXPORT_NAME="PgQueryModule" \ + -sENVIRONMENT=web,node \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=16777216 \ + -sSTACK_SIZE=1048576 \ + -sNO_FILESYSTEM=1 \ + -sNO_EXIT_RUNTIME=1 + +$(LIBPG_QUERY_LIB): + @echo "Cloning and building libpg_query $(LIBPG_QUERY_TAG)..." + rm -rf $(LIBPG_QUERY_DIR) + git clone --depth 1 --branch $(LIBPG_QUERY_TAG) https://github.com/pganalyze/libpg_query.git $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && make build + +clean: + rm -rf $(WASM_DIR)/*.wasm $(WASM_DIR)/*.js $(WASM_DIR)/*.wast + +clean-cache: + rm -rf $(LIBPG_QUERY_DIR) diff --git a/libpg-query-15/README.md b/libpg-query-15/README.md new file mode 100644 index 00000000..072a3391 --- /dev/null +++ b/libpg-query-15/README.md @@ -0,0 +1,34 @@ +# libpg-query-15 + +PostgreSQL 15 query parser (lightweight version) + +## Features + +- Parse SQL queries into AST +- Generate query fingerprints +- Normalize queries +- Parse PL/pgSQL functions + +## Installation + +```bash +npm install libpg-query-15 +``` + +## Usage + +```javascript +import { parse, fingerprint, normalize } from 'libpg-query-15'; + +const ast = await parse('SELECT * FROM users'); +const fp = await fingerprint('SELECT * FROM users WHERE id = $1'); +const normalized = await normalize('SELECT * FROM users WHERE id = 123'); +``` + +## Limitations + +This is a lightweight version that does not include: +- Deparse functionality +- Scan functionality + +For full functionality, use `libpg-query-full` or `libpg-query-multi`. diff --git a/libpg-query-15/package.json b/libpg-query-15/package.json new file mode 100644 index 00000000..688e26d9 --- /dev/null +++ b/libpg-query-15/package.json @@ -0,0 +1,33 @@ +{ + "name": "libpg-query-15", + "version": "15.4.2", + "description": "PostgreSQL 15 query parser (lightweight - no deparse/scan)", + "main": "wasm/index.js", + "types": "wasm/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./wasm/index.js", + "types": "./wasm/index.d.ts" + } + }, + "files": [ + "wasm/", + "README.md" + ], + "scripts": { + "build": "EMSCRIPTEN=1 make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000" + }, + "dependencies": { + "@pgsql/types": "^15.0.0" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + }, + "keywords": ["postgresql", "parser", "sql", "pg15"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-15/src/wasm_wrapper_light.c b/libpg-query-15/src/wasm_wrapper_light.c new file mode 100644 index 00000000..8429ef5c --- /dev/null +++ b/libpg-query-15/src/wasm_wrapper_light.c @@ -0,0 +1,213 @@ +#include "pg_query.h" +#include +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/libpg-query-15/test/parsing.test.js b/libpg-query-15/test/parsing.test.js new file mode 100644 index 00000000..a4fd5bf5 --- /dev/null +++ b/libpg-query-15/test/parsing.test.js @@ -0,0 +1,95 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("PostgreSQL 15 Query Parsing (Lightweight)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + expect(res.stmts).to.have.lengthOf(1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + expect(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival).to.eq(1); + expect(selectedDatas[1][0].ResTarget.val.A_Const.isnull).to.eq(true); + expect(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval).to.eq(""); + expect(selectedDatas[3]).to.have.lengthOf(2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + expect(res.stmts.map(removeLocationProperties)).to.deep.eq([ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + expect(() => query.parseSync("NOT A QUERY")).to.throw(Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + expect(resPromise).to.be.instanceof(Promise); + expect(res).to.deep.eq(query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + expect(e).instanceof(Error); + expect(e.message).to.match(/NOT/); + } + ); + }); + }); + + describe("Lightweight version restrictions", () => { + it("should not have deparse functionality", () => { + expect(query.deparse).to.be.undefined; + expect(query.deparseSync).to.be.undefined; + }); + + it("should not have scan functionality", () => { + expect(query.scan).to.be.undefined; + expect(query.scanSync).to.be.undefined; + }); + }); +}); diff --git a/libpg-query-15/wasm/index.d.ts b/libpg-query-15/wasm/index.d.ts new file mode 100644 index 00000000..3bdd7528 --- /dev/null +++ b/libpg-query-15/wasm/index.d.ts @@ -0,0 +1,15 @@ +export * from "@pgsql/types"; + +export function loadModule(): Promise; + +export function parse(query: string): Promise; +export function parseSync(query: string): any; + +export function parsePlPgSQL(query: string): Promise; +export function parsePlPgSQLSync(query: string): any; + +export function fingerprint(query: string): Promise; +export function fingerprintSync(query: string): string; + +export function normalize(query: string): Promise; +export function normalizeSync(query: string): string; diff --git a/libpg-query-15/wasm/index.js b/libpg-query-15/wasm/index.js new file mode 100644 index 00000000..88fd30d0 --- /dev/null +++ b/libpg-query-15/wasm/index.js @@ -0,0 +1,201 @@ +export * from "@pgsql/types"; +import PgQueryModule from './libpg-query.js'; + +let wasmModule; +const initPromise = PgQueryModule().then((module) => { + wasmModule = module; +}); + +export async function loadModule() { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit(fn) { + return (async (...args) => { + await initPromise; + return fn(...args); + }); +} + +function stringToPtr(str) { + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } + catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr) { + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function parseSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} diff --git a/libpg-query-16/Makefile b/libpg-query-16/Makefile new file mode 100644 index 00000000..3f9d7afc --- /dev/null +++ b/libpg-query-16/Makefile @@ -0,0 +1,60 @@ +LIBPG_QUERY_TAG := 16-5.2.0 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + PLATFORM := linux +endif +ifeq ($(UNAME_S),Darwin) + PLATFORM := darwin +endif + +ifeq ($(UNAME_M),x86_64) + ARCH := x64 +endif +ifeq ($(UNAME_M),arm64) + ARCH := arm64 +endif + +LIBPG_QUERY_DIR := libpg_query +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +WASM_DIR := wasm +WASM_FILE := $(WASM_DIR)/libpg-query.wasm +WASM_JS_FILE := $(WASM_DIR)/libpg-query.js + +.PHONY: build clean clean-cache + +build: $(WASM_FILE) + +$(WASM_FILE): $(LIBPG_QUERY_LIB) src/wasm_wrapper_light.c + @echo "Building WASM module for PostgreSQL 16..." + emcc -O3 \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/src/postgres/include \ + src/wasm_wrapper_light.c \ + $(LIBPG_QUERY_LIB) \ + -o $(WASM_FILE) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \ + -sEXPORTED_RUNTIME_METHODS="['UTF8ToString','stringToUTF8','lengthBytesUTF8']" \ + -sMODULARIZE=1 \ + -sEXPORT_NAME="PgQueryModule" \ + -sENVIRONMENT=web,node \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=16777216 \ + -sSTACK_SIZE=1048576 \ + -sNO_FILESYSTEM=1 \ + -sNO_EXIT_RUNTIME=1 + +$(LIBPG_QUERY_LIB): + @echo "Cloning and building libpg_query $(LIBPG_QUERY_TAG)..." + rm -rf $(LIBPG_QUERY_DIR) + git clone --depth 1 --branch $(LIBPG_QUERY_TAG) https://github.com/pganalyze/libpg_query.git $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && make build + +clean: + rm -rf $(WASM_DIR)/*.wasm $(WASM_DIR)/*.js $(WASM_DIR)/*.wast + +clean-cache: + rm -rf $(LIBPG_QUERY_DIR) diff --git a/libpg-query-16/README.md b/libpg-query-16/README.md new file mode 100644 index 00000000..e53a5ecc --- /dev/null +++ b/libpg-query-16/README.md @@ -0,0 +1,34 @@ +# libpg-query-16 + +PostgreSQL 16 query parser (lightweight version) + +## Features + +- Parse SQL queries into AST +- Generate query fingerprints +- Normalize queries +- Parse PL/pgSQL functions + +## Installation + +```bash +npm install libpg-query-16 +``` + +## Usage + +```javascript +import { parse, fingerprint, normalize } from 'libpg-query-16'; + +const ast = await parse('SELECT * FROM users'); +const fp = await fingerprint('SELECT * FROM users WHERE id = $1'); +const normalized = await normalize('SELECT * FROM users WHERE id = 123'); +``` + +## Limitations + +This is a lightweight version that does not include: +- Deparse functionality +- Scan functionality + +For full functionality, use `libpg-query-full` or `libpg-query-multi`. diff --git a/libpg-query-16/package.json b/libpg-query-16/package.json new file mode 100644 index 00000000..5d92695a --- /dev/null +++ b/libpg-query-16/package.json @@ -0,0 +1,33 @@ +{ + "name": "libpg-query-16", + "version": "16.5.2", + "description": "PostgreSQL 16 query parser (lightweight - no deparse/scan)", + "main": "wasm/index.js", + "types": "wasm/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./wasm/index.js", + "types": "./wasm/index.d.ts" + } + }, + "files": [ + "wasm/", + "README.md" + ], + "scripts": { + "build": "EMSCRIPTEN=1 make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000" + }, + "dependencies": { + "@pgsql/types": "^16.0.0" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0" + }, + "keywords": ["postgresql", "parser", "sql", "pg16"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-16/src/wasm_wrapper_light.c b/libpg-query-16/src/wasm_wrapper_light.c new file mode 100644 index 00000000..8429ef5c --- /dev/null +++ b/libpg-query-16/src/wasm_wrapper_light.c @@ -0,0 +1,213 @@ +#include "pg_query.h" +#include +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/libpg-query-16/test/parsing.test.js b/libpg-query-16/test/parsing.test.js new file mode 100644 index 00000000..b103fe25 --- /dev/null +++ b/libpg-query-16/test/parsing.test.js @@ -0,0 +1,95 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("PostgreSQL 16 Query Parsing (Lightweight)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + expect(res.stmts).to.have.lengthOf(1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + expect(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival).to.eq(1); + expect(selectedDatas[1][0].ResTarget.val.A_Const.isnull).to.eq(true); + expect(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval).to.eq(""); + expect(selectedDatas[3]).to.have.lengthOf(2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + expect(res.stmts.map(removeLocationProperties)).to.deep.eq([ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + expect(() => query.parseSync("NOT A QUERY")).to.throw(Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + expect(resPromise).to.be.instanceof(Promise); + expect(res).to.deep.eq(query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + expect(e).instanceof(Error); + expect(e.message).to.match(/NOT/); + } + ); + }); + }); + + describe("Lightweight version restrictions", () => { + it("should not have deparse functionality", () => { + expect(query.deparse).to.be.undefined; + expect(query.deparseSync).to.be.undefined; + }); + + it("should not have scan functionality", () => { + expect(query.scan).to.be.undefined; + expect(query.scanSync).to.be.undefined; + }); + }); +}); diff --git a/libpg-query-16/wasm/index.d.ts b/libpg-query-16/wasm/index.d.ts new file mode 100644 index 00000000..3bdd7528 --- /dev/null +++ b/libpg-query-16/wasm/index.d.ts @@ -0,0 +1,15 @@ +export * from "@pgsql/types"; + +export function loadModule(): Promise; + +export function parse(query: string): Promise; +export function parseSync(query: string): any; + +export function parsePlPgSQL(query: string): Promise; +export function parsePlPgSQLSync(query: string): any; + +export function fingerprint(query: string): Promise; +export function fingerprintSync(query: string): string; + +export function normalize(query: string): Promise; +export function normalizeSync(query: string): string; diff --git a/libpg-query-16/wasm/index.js b/libpg-query-16/wasm/index.js new file mode 100644 index 00000000..88fd30d0 --- /dev/null +++ b/libpg-query-16/wasm/index.js @@ -0,0 +1,201 @@ +export * from "@pgsql/types"; +import PgQueryModule from './libpg-query.js'; + +let wasmModule; +const initPromise = PgQueryModule().then((module) => { + wasmModule = module; +}); + +export async function loadModule() { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit(fn) { + return (async (...args) => { + await initPromise; + return fn(...args); + }); +} + +function stringToPtr(str) { + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } + catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr) { + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function parseSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} diff --git a/libpg-query-full/Makefile b/libpg-query-full/Makefile new file mode 100644 index 00000000..f11d428d --- /dev/null +++ b/libpg-query-full/Makefile @@ -0,0 +1,60 @@ +LIBPG_QUERY_TAG := 17-6.1.0 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + PLATFORM := linux +endif +ifeq ($(UNAME_S),Darwin) + PLATFORM := darwin +endif + +ifeq ($(UNAME_M),x86_64) + ARCH := x64 +endif +ifeq ($(UNAME_M),arm64) + ARCH := arm64 +endif + +LIBPG_QUERY_DIR := libpg_query +LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a + +WASM_DIR := wasm +WASM_FILE := $(WASM_DIR)/libpg-query.wasm +WASM_JS_FILE := $(WASM_DIR)/libpg-query.js + +.PHONY: build clean clean-cache + +build: $(WASM_FILE) + +$(WASM_FILE): $(LIBPG_QUERY_LIB) src/wasm_wrapper.c + @echo "Building WASM module for PostgreSQL 17 (full functionality)..." + emcc -O3 \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/src/postgres/include \ + src/wasm_wrapper.c \ + $(LIBPG_QUERY_LIB) \ + -o $(WASM_FILE) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_query_protobuf','_wasm_get_protobuf_len','_wasm_deparse_protobuf','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \ + -sEXPORTED_RUNTIME_METHODS="['UTF8ToString','stringToUTF8','lengthBytesUTF8']" \ + -sMODULARIZE=1 \ + -sEXPORT_NAME="PgQueryModule" \ + -sENVIRONMENT=web,node \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=16777216 \ + -sSTACK_SIZE=1048576 \ + -sNO_FILESYSTEM=1 \ + -sNO_EXIT_RUNTIME=1 + +$(LIBPG_QUERY_LIB): + @echo "Cloning and building libpg_query $(LIBPG_QUERY_TAG)..." + rm -rf $(LIBPG_QUERY_DIR) + git clone --depth 1 --branch $(LIBPG_QUERY_TAG) https://github.com/pganalyze/libpg_query.git $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR) && make build + +clean: + rm -rf $(WASM_DIR)/*.wasm $(WASM_DIR)/*.js $(WASM_DIR)/*.wast + +clean-cache: + rm -rf $(LIBPG_QUERY_DIR) diff --git a/libpg-query-full/README.md b/libpg-query-full/README.md new file mode 100644 index 00000000..9ef1b9ff --- /dev/null +++ b/libpg-query-full/README.md @@ -0,0 +1,48 @@ +# libpg-query-full + +PostgreSQL 17 query parser with complete functionality + +## Features + +- Parse SQL queries into AST +- Generate query fingerprints +- Normalize queries +- Parse PL/pgSQL functions +- Deparse AST back to SQL +- Scan queries for tokens + +## Installation + +```bash +npm install libpg-query-full +``` + +## Usage + +```javascript +import { parse, deparse, scan, fingerprint, normalize } from 'libpg-query-full'; + +// Parse query +const ast = await parse('SELECT * FROM users'); + +// Deparse AST back to SQL +const sql = await deparse(ast); + +// Scan query for tokens +const tokens = await scan('SELECT * FROM users'); + +// Generate fingerprint +const fp = await fingerprint('SELECT * FROM users WHERE id = $1'); + +// Normalize query +const normalized = await normalize('SELECT * FROM users WHERE id = 123'); +``` + +## Complete Functionality + +This package includes all PostgreSQL 17 features: +- Full parsing capabilities +- Deparse functionality +- Token scanning +- Query fingerprinting +- Query normalization diff --git a/libpg-query-full/package.json b/libpg-query-full/package.json new file mode 100644 index 00000000..f620a0bf --- /dev/null +++ b/libpg-query-full/package.json @@ -0,0 +1,37 @@ +{ + "name": "libpg-query-full", + "version": "17.6.1", + "description": "PostgreSQL 17 query parser (full functionality with deparse/scan)", + "main": "wasm/index.js", + "types": "wasm/index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./wasm/index.js", + "types": "./wasm/index.d.ts" + } + }, + "files": [ + "wasm/", + "proto.js", + "README.md" + ], + "scripts": { + "build": "EMSCRIPTEN=1 make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000", + "protogen": "node ./scripts/protogen.js" + }, + "dependencies": { + "@pgsql/types": "^17.0.0", + "@launchql/protobufjs": "7.2.6" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0", + "pg-proto-parser": "^0.6.3" + }, + "keywords": ["postgresql", "parser", "sql", "pg17", "deparse", "scan"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-full/scripts/protogen.js b/libpg-query-full/scripts/protogen.js new file mode 100644 index 00000000..c279cd58 --- /dev/null +++ b/libpg-query-full/scripts/protogen.js @@ -0,0 +1,36 @@ +const { exec } = require('child_process'); + +const branchName = '17-6.1.0'; +const protoUrl = `https://raw.githubusercontent.com/pganalyze/libpg_query/${branchName}/protobuf/pg_query.proto`; +const inFile = 'libpg_query/protobuf/pg_query.proto'; +const outFile = 'proto.js'; + +const protogenCmd = [ + 'pg-proto-parser', + 'protogen', + '--protoUrl', + protoUrl, + '--inFile', + inFile, + '--outFile', + outFile, + '--originalPackageName', + 'protobufjs/minimal', + '--newPackageName', + '@launchql/protobufjs/minimal' +]; + +function generateProtoJS(callback) { + exec(protogenCmd.join(' '), (error, stdout, stderr) => { + if (error) { + console.error(`Error during code generation: ${error.message}`); + return; + } + console.log('Generated proto.js from proto file.'); + callback(); + }); +} + +generateProtoJS(() => { + console.log('all done 🎉'); +}); diff --git a/libpg-query-full/src/wasm_wrapper.c b/libpg-query-full/src/wasm_wrapper.c new file mode 100644 index 00000000..ee8ae9da --- /dev/null +++ b/libpg-query-full/src/wasm_wrapper.c @@ -0,0 +1,345 @@ +#include "pg_query.h" +#include +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_deparse_protobuf(const char* protobuf_data, size_t data_len) { + if (!protobuf_data || data_len == 0) { + return safe_strdup("Invalid input: protobuf data cannot be null or empty"); + } + + PgQueryDeparseResult result = pg_query_deparse_protobuf(protobuf_data, data_len); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_deparse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* query = safe_strdup(result.query); + pg_query_free_deparse_result(result); + return query; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query_protobuf(const char* input, int* out_len) { + if (!validate_input(input)) { + *out_len = 0; + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryProtobufParseResult result = pg_query_parse_protobuf(input); + + if (result.error) { + *out_len = 0; + char* error_msg = safe_strdup(result.error->message); + pg_query_free_protobuf_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + *out_len = result.parse_tree_len; + char* parse_tree = safe_malloc(result.parse_tree_len); + if (!parse_tree) { + *out_len = 0; + pg_query_free_protobuf_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + memcpy(parse_tree, result.parse_tree, result.parse_tree_len); + pg_query_free_protobuf_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +int wasm_get_protobuf_len(const char* input) { + if (!validate_input(input)) { + return -1; + } + + PgQueryProtobufParseResult result = pg_query_parse_protobuf(input); + + if (result.error) { + pg_query_free_protobuf_parse_result(result); + return -1; + } + + int len = result.parse_tree_len; + pg_query_free_protobuf_parse_result(result); + return len; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} + +static char* create_scan_json(PgQueryScanResult scan_result) { + if (!scan_result.tokens || scan_result.ntokens == 0) { + return safe_strdup("{\"version\":0,\"tokens\":[]}"); + } + + size_t json_size = 1024; + char* json = safe_malloc(json_size); + if (!json) return NULL; + + size_t pos = 0; + pos += snprintf(json + pos, json_size - pos, "{\"version\":%d,\"tokens\":[", scan_result.version); + + for (int i = 0; i < scan_result.ntokens; i++) { + PgQueryToken token = scan_result.tokens[i]; + + size_t needed = pos + 200; + if (needed >= json_size) { + json_size = needed * 2; + char* new_json = realloc(json, json_size); + if (!new_json) { + free(json); + return NULL; + } + json = new_json; + } + + if (i > 0) { + pos += snprintf(json + pos, json_size - pos, ","); + } + + pos += snprintf(json + pos, json_size - pos, + "{\"text\":\"%.*s\",\"start\":%d,\"end\":%d,\"tokenName\":\"%s\",\"keywordName\":\"%s\"}", + token.end - token.start, token.start, + token.start, token.end, + token.token_name ? token.token_name : "UNKNOWN", + token.keyword_name ? token.keyword_name : "NO_KEYWORD"); + } + + pos += snprintf(json + pos, json_size - pos, "]}"); + return json; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_scan(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryScanResult result = pg_query_scan(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_scan_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* json_result = create_scan_json(result); + pg_query_free_scan_result(result); + + if (!json_result) { + return safe_strdup("Memory allocation failed during JSON creation"); + } + + return json_result; +} diff --git a/libpg-query-full/test/deparsing.test.js b/libpg-query-full/test/deparsing.test.js new file mode 100644 index 00000000..1d1dba0a --- /dev/null +++ b/libpg-query-full/test/deparsing.test.js @@ -0,0 +1,55 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +describe("PostgreSQL 17 Query Deparsing (Full)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Deparsing", () => { + it("should deparse a simple query", () => { + const sql = 'SELECT * FROM users'; + const parseTree = query.parseSync(sql); + const deparsed = query.deparseSync(parseTree); + expect(deparsed).to.equal(sql); + }); + + it("should deparse a complex query", () => { + const sql = 'SELECT a, b, c FROM t1 JOIN t2 ON t1.id = t2.id WHERE t1.x > 10'; + const parseTree = query.parseSync(sql); + const deparsed = query.deparseSync(parseTree); + expect(deparsed).to.equal(sql); + }); + + it("should fail to deparse without protobuf data", () => { + expect(() => query.deparseSync({})).to.throw('No parseTree provided'); + }); + }); + + describe("Async Deparsing", () => { + it("should return a promise resolving to same result", async () => { + const sql = 'SELECT * FROM users'; + const parseTree = await query.parse(sql); + const deparsed = await query.deparse(parseTree); + expect(deparsed).to.equal(sql); + }); + + it("should reject when no protobuf data", async () => { + try { + await query.deparse({}); + throw new Error('should have rejected'); + } catch (err) { + expect(err.message).to.equal('No parseTree provided'); + } + }); + }); + + describe("Round-trip parsing and deparsing", () => { + it("should maintain query semantics through round-trip", async () => { + const sql = 'SELECT a, b, c FROM t1 JOIN t2 ON t1.id = t2.id WHERE t1.x > 10'; + const parseTree = await query.parse(sql); + const deparsed = await query.deparse(parseTree); + expect(deparsed).to.equal(sql); + }); + }); +}); diff --git a/libpg-query-full/test/parsing.test.js b/libpg-query-full/test/parsing.test.js new file mode 100644 index 00000000..60687cf5 --- /dev/null +++ b/libpg-query-full/test/parsing.test.js @@ -0,0 +1,110 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("PostgreSQL 17 Query Parsing (Full)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + expect(res.stmts).to.have.lengthOf(1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + expect(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival).to.eq(1); + expect(selectedDatas[1][0].ResTarget.val.A_Const.isnull).to.eq(true); + expect(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval).to.eq(""); + expect(selectedDatas[3]).to.have.lengthOf(2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + expect(res.stmts.map(removeLocationProperties)).to.deep.eq([ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + expect(() => query.parseSync("NOT A QUERY")).to.throw(Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + expect(resPromise).to.be.instanceof(Promise); + expect(res).to.deep.eq(query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + expect(e).instanceof(Error); + expect(e.message).to.match(/NOT/); + } + ); + }); + }); + + describe("Full version functionality", () => { + it("should have deparse functionality", () => { + expect(query.deparse).to.be.a('function'); + expect(query.deparseSync).to.be.a('function'); + }); + + it("should have scan functionality", () => { + expect(query.scan).to.be.a('function'); + expect(query.scanSync).to.be.a('function'); + }); + + it("should deparse a simple query", async () => { + const sql = 'SELECT * FROM users'; + const parseTree = await query.parse(sql); + const deparsed = await query.deparse(parseTree); + expect(deparsed).to.equal(sql); + }); + + it("should scan a simple query", async () => { + const result = await query.scan("SELECT 1"); + expect(result).to.be.an("object"); + expect(result).to.have.property("version"); + expect(result).to.have.property("tokens"); + expect(result.tokens).to.be.an("array"); + }); + }); +}); diff --git a/libpg-query-full/test/scan.test.js b/libpg-query-full/test/scan.test.js new file mode 100644 index 00000000..90541622 --- /dev/null +++ b/libpg-query-full/test/scan.test.js @@ -0,0 +1,68 @@ +const query = require("../wasm"); +const { expect } = require("chai"); + +describe("PostgreSQL 17 Query Scanning (Full)", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Scanning", () => { + it("should return a scan result with version and tokens", () => { + const result = query.scanSync("SELECT 1"); + + expect(result).to.be.an("object"); + expect(result).to.have.property("version"); + expect(result).to.have.property("tokens"); + expect(result.version).to.be.a("number"); + expect(result.tokens).to.be.an("array"); + }); + + it("should scan a simple SELECT query correctly", () => { + const result = query.scanSync("SELECT 1"); + + expect(result.tokens).to.have.lengthOf(2); + + const selectToken = result.tokens[0]; + expect(selectToken.text).to.eq("SELECT"); + expect(selectToken.start).to.eq(0); + expect(selectToken.end).to.eq(6); + + const numberToken = result.tokens[1]; + expect(numberToken.text).to.eq("1"); + expect(numberToken.start).to.eq(7); + expect(numberToken.end).to.eq(8); + }); + + it("should scan tokens with correct positions", () => { + const sql = "SELECT * FROM users"; + const result = query.scanSync(sql); + + expect(result.tokens).to.have.lengthOf(4); + + result.tokens.forEach(token => { + const actualText = sql.substring(token.start, token.end); + expect(token.text).to.eq(actualText); + }); + }); + }); + + describe("Async Scanning", () => { + it("should return a promise resolving to same result as sync", async () => { + const testQuery = "SELECT * FROM users WHERE id = $1"; + const resultPromise = query.scan(testQuery); + const result = await resultPromise; + + expect(resultPromise).to.be.instanceof(Promise); + expect(result).to.deep.eq(query.scanSync(testQuery)); + }); + + it("should handle complex queries asynchronously", async () => { + const testQuery = "SELECT COUNT(*) as total FROM orders WHERE status = 'completed'"; + const result = await query.scan(testQuery); + + expect(result).to.be.an("object"); + expect(result.tokens).to.be.an("array"); + expect(result.tokens.length).to.be.greaterThan(5); + }); + }); +}); diff --git a/libpg-query-full/wasm/index.d.ts b/libpg-query-full/wasm/index.d.ts new file mode 100644 index 00000000..35925af4 --- /dev/null +++ b/libpg-query-full/wasm/index.d.ts @@ -0,0 +1,21 @@ +export * from "@pgsql/types"; + +export function loadModule(): Promise; + +export function parse(query: string): Promise; +export function parseSync(query: string): any; + +export function parsePlPgSQL(query: string): Promise; +export function parsePlPgSQLSync(query: string): any; + +export function fingerprint(query: string): Promise; +export function fingerprintSync(query: string): string; + +export function normalize(query: string): Promise; +export function normalizeSync(query: string): string; + +export function deparse(tree: any): Promise; +export function deparseSync(tree: any): string; + +export function scan(query: string): Promise; +export function scanSync(query: string): any; diff --git a/libpg-query-full/wasm/index.js b/libpg-query-full/wasm/index.js new file mode 100644 index 00000000..0b38c561 --- /dev/null +++ b/libpg-query-full/wasm/index.js @@ -0,0 +1,296 @@ +export * from "@pgsql/types"; +import PgQueryModule from './libpg-query.js'; +import { pg_query } from '../proto.js'; + +let wasmModule; +const initPromise = PgQueryModule().then((module) => { + wasmModule = module; +}); + +export async function loadModule() { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit(fn) { + return (async (...args) => { + await initPromise; + return fn(...args); + }); +} + +function stringToPtr(str) { + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } + catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr) { + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const deparse = awaitInit(async (parseTree) => { + if (!parseTree || typeof parseTree !== 'object' || !Array.isArray(parseTree.stmts) || parseTree.stmts.length === 0) { + throw new Error('No parseTree provided'); + } + const msg = pg_query.ParseResult.fromObject(parseTree); + const data = pg_query.ParseResult.encode(msg).finish(); + const dataPtr = wasmModule._malloc(data.length); + let resultPtr = 0; + try { + wasmModule.HEAPU8.set(data, dataPtr); + resultPtr = wasmModule._wasm_deparse_protobuf(dataPtr, data.length); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(dataPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function parseSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function deparseSync(parseTree) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + if (!parseTree || typeof parseTree !== 'object' || !Array.isArray(parseTree.stmts) || parseTree.stmts.length === 0) { + throw new Error('No parseTree provided'); + } + const msg = pg_query.ParseResult.fromObject(parseTree); + const data = pg_query.ParseResult.encode(msg).finish(); + const dataPtr = wasmModule._malloc(data.length); + let resultPtr = 0; + try { + wasmModule.HEAPU8.set(data, dataPtr); + resultPtr = wasmModule._wasm_deparse_protobuf(dataPtr, data.length); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(dataPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return resultStr; + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export const scan = awaitInit(async (query) => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function scanSync(query) { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.includes('ERROR')) { + throw new Error(resultStr); + } + return JSON.parse(resultStr); + } + finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} diff --git a/libpg-query-multi/Makefile b/libpg-query-multi/Makefile new file mode 100644 index 00000000..c5545fa3 --- /dev/null +++ b/libpg-query-multi/Makefile @@ -0,0 +1,34 @@ +.PHONY: build clean test generate-types + +build: generate-types + @echo "Building libpg-query-multi package..." + @echo "Copying PG15 lightweight version..." + @mkdir -p pg15 + @cp -r ../libpg-query-15/wasm pg15/ + @cp ../libpg-query-15/package.json pg15/package.json.bak + + @echo "Copying PG16 lightweight version..." + @mkdir -p pg16 + @cp -r ../libpg-query-16/wasm pg16/ + @cp ../libpg-query-16/package.json pg16/package.json.bak + + @echo "Copying PG17 full version..." + @mkdir -p pg17 + @cp -r ../libpg-query-full/wasm pg17/ + @cp -r ../libpg-query-full/proto.js pg17/ 2>/dev/null || true + @cp ../libpg-query-full/package.json pg17/package.json.bak + + @echo "libpg-query-multi package built successfully!" + +generate-types: + @echo "Generating version-specific TypeScript types..." + @node scripts/generate-types.js + +clean: + @echo "Cleaning libpg-query-multi package..." + @rm -rf pg15 pg16 pg17 + @echo "Cleaned!" + +test: + @echo "Running tests for libpg-query-multi..." + @npm test diff --git a/libpg-query-multi/README.md b/libpg-query-multi/README.md new file mode 100644 index 00000000..7cfd974d --- /dev/null +++ b/libpg-query-multi/README.md @@ -0,0 +1,99 @@ +# libpg-query-multi + +Multi-version PostgreSQL query parser with runtime version selection + +## Features + +- Runtime PostgreSQL version selection (15, 16, 17) +- Parser class API for flexible usage +- Named exports for direct version access +- Automatic feature availability detection + +## Installation + +```bash +npm install libpg-query-multi +``` + +## Usage + +### Parser Class API + +```javascript +import { Parser } from 'libpg-query-multi'; + +// Default to latest (PostgreSQL 17) +const parser = new Parser(); +const result = await parser.parse('SELECT * FROM users'); + +// Specify PostgreSQL version +const pg15Parser = new Parser({ version: 15 }); +const result15 = await pg15Parser.parse('SELECT * FROM users'); + +// Feature availability is automatically handled +try { + await pg15Parser.deparse(result15); // Throws error - not available in PG15 +} catch (error) { + console.log(error.message); // "Deparse functionality not available for PostgreSQL 15" +} + +// PG17 has all features +const pg17Parser = new Parser({ version: 17 }); +const ast = await pg17Parser.parse('SELECT * FROM users'); +const sql = await pg17Parser.deparse(ast); +const tokens = await pg17Parser.scan('SELECT * FROM users'); +``` + +### Named Exports + +```javascript +import { + parse15, parse16, parse17, + deparse16, deparse17, + scan17, + parse, deparse, scan // Defaults to PG17 +} from 'libpg-query-multi'; + +// Direct version access +const result15 = await parse15('SELECT * FROM users'); +const result16 = await parse16('SELECT * FROM users'); +const result17 = await parse17('SELECT * FROM users'); + +// Deparse (available in PG16+) +const sql16 = await deparse16(result16); +const sql17 = await deparse17(result17); + +// Scan (available in PG17 only) +const tokens = await scan17('SELECT * FROM users'); + +// Default exports (PG17) +const ast = await parse('SELECT * FROM users'); +const sql = await deparse(ast); +const scanResult = await scan('SELECT * FROM users'); +``` + +## Feature Matrix + +| Feature | PG15 | PG16 | PG17 | +|---------|------|------|------| +| Parse | ✅ | ✅ | ✅ | +| Fingerprint | ✅ | ✅ | ✅ | +| Normalize | ✅ | ✅ | ✅ | +| PL/pgSQL | ✅ | ✅ | ✅ | +| Deparse | ❌ | ✅ | ✅ | +| Scan | ❌ | ❌ | ✅ | + +## Error Handling + +The Parser class automatically detects feature availability and provides helpful error messages: + +```javascript +const pg15Parser = new Parser({ version: 15 }); + +try { + await pg15Parser.scan('SELECT 1'); +} catch (error) { + console.log(error.message); + // "Scan functionality not available for PostgreSQL 15. Available in version 17 only." +} +``` diff --git a/libpg-query-multi/index.d.ts b/libpg-query-multi/index.d.ts new file mode 100644 index 00000000..1a00f1f0 --- /dev/null +++ b/libpg-query-multi/index.d.ts @@ -0,0 +1,66 @@ +export interface ParserOptions { + version?: 15 | 16 | 17; +} + +export interface ParseResult { + stmts: any[]; + version?: number; +} + +export interface ScanResult { + version: number; + tokens: Array<{ + text: string; + start: number; + end: number; + tokenName: string; + keywordName: string; + }>; +} + +export declare class Parser { + constructor(options?: ParserOptions); + + parse(query: string): Promise; + parseSync(query: string): ParseResult; + + parsePlPgSQL(query: string): Promise; + parsePlPgSQLSync(query: string): any; + + fingerprint(query: string): Promise; + fingerprintSync(query: string): string; + + normalize(query: string): Promise; + normalizeSync(query: string): string; + + deparse(tree: ParseResult): Promise; + deparseSync(tree: ParseResult): string; + + scan(query: string): Promise; + scanSync(query: string): ScanResult; +} + +export declare function parse15(query: string): Promise; +export declare function fingerprint15(query: string): Promise; +export declare function normalize15(query: string): Promise; +export declare function parsePlPgSQL15(query: string): Promise; + +export declare function parse16(query: string): Promise; +export declare function fingerprint16(query: string): Promise; +export declare function normalize16(query: string): Promise; +export declare function parsePlPgSQL16(query: string): Promise; +export declare function deparse16(tree: ParseResult): Promise; + +export declare function parse17(query: string): Promise; +export declare function fingerprint17(query: string): Promise; +export declare function normalize17(query: string): Promise; +export declare function parsePlPgSQL17(query: string): Promise; +export declare function deparse17(tree: ParseResult): Promise; +export declare function scan17(query: string): Promise; + +export declare function parse(query: string): Promise; +export declare function fingerprint(query: string): Promise; +export declare function normalize(query: string): Promise; +export declare function parsePlPgSQL(query: string): Promise; +export declare function deparse(tree: ParseResult): Promise; +export declare function scan(query: string): Promise; diff --git a/libpg-query-multi/index.js b/libpg-query-multi/index.js new file mode 100644 index 00000000..a2154623 --- /dev/null +++ b/libpg-query-multi/index.js @@ -0,0 +1,115 @@ +export class Parser { + constructor(options = {}) { + this.version = options.version || 17; + this._parser = null; + } + + async _getParser() { + if (!this._parser) { + switch(this.version) { + case 15: + this._parser = await import('./pg15/index.js'); + break; + case 16: + this._parser = await import('./pg16/index.js'); + break; + case 17: + this._parser = await import('./pg17/index.js'); + break; + default: + throw new Error(`Unsupported PostgreSQL version: ${this.version}. Supported versions: 15, 16, 17`); + } + } + return this._parser; + } + + async parse(query) { + const parser = await this._getParser(); + return parser.parse(query); + } + + parseSync(query) { + if (!this._parser) { + throw new Error('Parser not initialized. Call parse() first or use async methods.'); + } + return this._parser.parseSync(query); + } + + async parsePlPgSQL(query) { + const parser = await this._getParser(); + return parser.parsePlPgSQL(query); + } + + parsePlPgSQLSync(query) { + if (!this._parser) { + throw new Error('Parser not initialized. Call parsePlPgSQL() first or use async methods.'); + } + return this._parser.parsePlPgSQLSync(query); + } + + async fingerprint(query) { + const parser = await this._getParser(); + return parser.fingerprint(query); + } + + fingerprintSync(query) { + if (!this._parser) { + throw new Error('Parser not initialized. Call fingerprint() first or use async methods.'); + } + return this._parser.fingerprintSync(query); + } + + async normalize(query) { + const parser = await this._getParser(); + return parser.normalize(query); + } + + normalizeSync(query) { + if (!this._parser) { + throw new Error('Parser not initialized. Call normalize() first or use async methods.'); + } + return this._parser.normalizeSync(query); + } + + async deparse(tree) { + const parser = await this._getParser(); + if (!parser.deparse) { + throw new Error(`Deparse functionality not available for PostgreSQL ${this.version}. Available in versions 16 and 17.`); + } + return parser.deparse(tree); + } + + deparseSync(tree) { + if (!this._parser) { + throw new Error('Parser not initialized. Call deparse() first or use async methods.'); + } + if (!this._parser.deparseSync) { + throw new Error(`Deparse functionality not available for PostgreSQL ${this.version}. Available in versions 16 and 17.`); + } + return this._parser.deparseSync(tree); + } + + async scan(query) { + const parser = await this._getParser(); + if (!parser.scan) { + throw new Error(`Scan functionality not available for PostgreSQL ${this.version}. Available in version 17 only.`); + } + return parser.scan(query); + } + + scanSync(query) { + if (!this._parser) { + throw new Error('Parser not initialized. Call scan() first or use async methods.'); + } + if (!this._parser.scanSync) { + throw new Error(`Scan functionality not available for PostgreSQL ${this.version}. Available in version 17 only.`); + } + return this._parser.scanSync(query); + } +} + +export { parse as parse15, fingerprint as fingerprint15, normalize as normalize15, parsePlPgSQL as parsePlPgSQL15 } from './pg15/index.js'; +export { parse as parse16, fingerprint as fingerprint16, normalize as normalize16, parsePlPgSQL as parsePlPgSQL16, deparse as deparse16 } from './pg16/index.js'; +export { parse as parse17, fingerprint as fingerprint17, normalize as normalize17, parsePlPgSQL as parsePlPgSQL17, deparse as deparse17, scan as scan17 } from './pg17/index.js'; + +export { parse, fingerprint, normalize, parsePlPgSQL, deparse, scan } from './pg17/index.js'; diff --git a/libpg-query-multi/package.json b/libpg-query-multi/package.json new file mode 100644 index 00000000..e39ae1e4 --- /dev/null +++ b/libpg-query-multi/package.json @@ -0,0 +1,48 @@ +{ + "name": "libpg-query-multi", + "version": "1.0.0", + "description": "Multi-version PostgreSQL query parser (PG15/16/17 with runtime version selection)", + "main": "index.js", + "types": "index.d.ts", + "type": "module", + "exports": { + ".": { + "import": "./index.js", + "types": "./index.d.ts" + }, + "./pg15": { + "import": "./pg15/index.js", + "types": "./pg15/index.d.ts" + }, + "./pg16": { + "import": "./pg16/index.js", + "types": "./pg16/index.d.ts" + }, + "./pg17": { + "import": "./pg17/index.js", + "types": "./pg17/index.d.ts" + } + }, + "files": [ + "index.js", + "index.d.ts", + "pg15/", + "pg16/", + "pg17/", + "README.md" + ], + "scripts": { + "build": "make build", + "clean": "make clean", + "test": "mocha test/*.test.js --timeout 5000", + "generate-types": "node ./scripts/generate-types.js" + }, + "devDependencies": { + "chai": "^4.3.7", + "mocha": "^10.2.0", + "pg-proto-parser": "^0.6.3" + }, + "keywords": ["postgresql", "parser", "sql", "multi-version", "pg15", "pg16", "pg17"], + "author": "Dan Lynch", + "license": "MIT" +} diff --git a/libpg-query-multi/scripts/generate-types.js b/libpg-query-multi/scripts/generate-types.js new file mode 100644 index 00000000..1a7db4fc --- /dev/null +++ b/libpg-query-multi/scripts/generate-types.js @@ -0,0 +1,50 @@ +const { PgProtoParser } = require('pg-proto-parser'); +const fs = require('fs'); +const path = require('path'); + +const versions = [ + { version: '15', tag: '15-4.2.4' }, + { version: '16', tag: '16-5.2.0' }, + { version: '17', tag: '17-6.1.0' } +]; + +async function generateTypes() { + for (const { version, tag } of versions) { + console.log(`Generating types for PostgreSQL ${version}...`); + + const protoUrl = `https://raw.githubusercontent.com/pganalyze/libpg_query/refs/tags/${tag}/protobuf/pg_query.proto`; + const outDir = `./pg${version}/types`; + + if (!fs.existsSync(outDir)) { + fs.mkdirSync(outDir, { recursive: true }); + } + + const options = { + outDir, + types: { + enabled: true, + wrappedNodeTypeExport: true, + optionalFields: true, + filename: 'types.d.ts', + enumsSource: './enums.js', + }, + enums: { + enabled: true, + enumsAsTypeUnion: true, + filename: 'enums.d.ts', + }, + }; + + try { + const parser = new PgProtoParser(protoUrl, options); + await parser.write(); + console.log(`✓ Generated types for PostgreSQL ${version}`); + } catch (error) { + console.error(`✗ Failed to generate types for PostgreSQL ${version}:`, error.message); + } + } + + console.log('Type generation complete!'); +} + +generateTypes().catch(console.error); diff --git a/libpg-query-multi/test/parser.test.js b/libpg-query-multi/test/parser.test.js new file mode 100644 index 00000000..0d0ec2d2 --- /dev/null +++ b/libpg-query-multi/test/parser.test.js @@ -0,0 +1,136 @@ +const { Parser } = require("../"); +const { expect } = require("chai"); + +describe("Multi-Version Parser", () => { + describe("Parser class instantiation", () => { + it("should default to PostgreSQL 17", () => { + const parser = new Parser(); + expect(parser.version).to.eq(17); + }); + + it("should accept version 15", () => { + const parser = new Parser({ version: 15 }); + expect(parser.version).to.eq(15); + }); + + it("should accept version 16", () => { + const parser = new Parser({ version: 16 }); + expect(parser.version).to.eq(16); + }); + + it("should accept version 17", () => { + const parser = new Parser({ version: 17 }); + expect(parser.version).to.eq(17); + }); + + it("should throw error for unsupported version", () => { + expect(() => new Parser({ version: 14 })).to.throw(); + }); + }); + + describe("Version-specific functionality", () => { + it("should parse queries with PG15", async () => { + const parser = new Parser({ version: 15 }); + const result = await parser.parse("SELECT 1"); + expect(result.stmts).to.have.lengthOf(1); + }); + + it("should parse queries with PG16", async () => { + const parser = new Parser({ version: 16 }); + const result = await parser.parse("SELECT 1"); + expect(result.stmts).to.have.lengthOf(1); + }); + + it("should parse queries with PG17", async () => { + const parser = new Parser({ version: 17 }); + const result = await parser.parse("SELECT 1"); + expect(result.stmts).to.have.lengthOf(1); + }); + + it("should throw error when trying to deparse with PG15", async () => { + const parser = new Parser({ version: 15 }); + const parseTree = await parser.parse("SELECT 1"); + + try { + await parser.deparse(parseTree); + throw new Error("Should have thrown"); + } catch (error) { + expect(error.message).to.include("Deparse functionality not available for PostgreSQL 15"); + } + }); + + it("should allow deparse with PG16", async () => { + const parser = new Parser({ version: 16 }); + const parseTree = await parser.parse("SELECT 1"); + + try { + await parser.deparse(parseTree); + } catch (error) { + expect(error.message).to.not.include("not available"); + } + }); + + it("should allow deparse with PG17", async () => { + const parser = new Parser({ version: 17 }); + const parseTree = await parser.parse("SELECT 1"); + + try { + await parser.deparse(parseTree); + } catch (error) { + expect(error.message).to.not.include("not available"); + } + }); + + it("should throw error when trying to scan with PG15", async () => { + const parser = new Parser({ version: 15 }); + + try { + await parser.scan("SELECT 1"); + throw new Error("Should have thrown"); + } catch (error) { + expect(error.message).to.include("Scan functionality not available for PostgreSQL 15"); + } + }); + + it("should throw error when trying to scan with PG16", async () => { + const parser = new Parser({ version: 16 }); + + try { + await parser.scan("SELECT 1"); + throw new Error("Should have thrown"); + } catch (error) { + expect(error.message).to.include("Scan functionality not available for PostgreSQL 16"); + } + }); + + it("should allow scan with PG17", async () => { + const parser = new Parser({ version: 17 }); + + try { + const result = await parser.scan("SELECT 1"); + expect(result.tokens).to.be.an('array'); + } catch (error) { + expect(error.message).to.not.include("not available"); + } + }); + }); + + describe("Named exports", () => { + it("should export version-specific parse functions", async () => { + const { parse15, parse16, parse17 } = require("../"); + + expect(parse15).to.be.a('function'); + expect(parse16).to.be.a('function'); + expect(parse17).to.be.a('function'); + }); + + it("should export default functions from PG17", async () => { + const { parse, fingerprint, deparse, scan } = require("../"); + + expect(parse).to.be.a('function'); + expect(fingerprint).to.be.a('function'); + expect(deparse).to.be.a('function'); + expect(scan).to.be.a('function'); + }); + }); +}); From 41bb6ad926fa0a70b8f1634ca6d6a0843cfe7758 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 21 Jun 2025 04:47:20 +0000 Subject: [PATCH 2/3] refactor: update package.json files to use Docker-based build pattern - Move root package.json functionality to libpg-query-full (PG17) - Replace root with workspace coordinator package.json - Update all version packages to use Docker-based wasm:make scripts - Standardize exports, files, and dependency configurations - Lightweight versions exclude proto.js from files array - Version-specific @pgsql/types dependencies configured correctly Co-Authored-By: Dan Lynch --- libpg-query-13/package.json | 59 ++++++++++++++++++++--------- libpg-query-14/package.json | 59 ++++++++++++++++++++--------- libpg-query-15/package.json | 59 ++++++++++++++++++++--------- libpg-query-16/package.json | 59 ++++++++++++++++++++--------- libpg-query-full/package.json | 67 ++++++++++++++++++++++----------- libpg-query-multi/package.json | 66 ++++++++++++++++++++++---------- package.json | 69 ++++++++++------------------------ 7 files changed, 280 insertions(+), 158 deletions(-) diff --git a/libpg-query-13/package.json b/libpg-query-13/package.json index 21fde94d..d3b7a810 100644 --- a/libpg-query-13/package.json +++ b/libpg-query-13/package.json @@ -2,32 +2,57 @@ "name": "libpg-query-13", "version": "13.2.0", "description": "PostgreSQL 13 query parser (lightweight - no deparse/scan)", - "main": "wasm/index.js", - "types": "wasm/index.d.ts", - "type": "module", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*" + ], "exports": { ".": { + "types": "./wasm/index.d.ts", "import": "./wasm/index.js", - "types": "./wasm/index.d.ts" + "require": "./wasm/index.cjs" } }, - "files": [ - "wasm/", - "README.md" - ], "scripts": { - "build": "EMSCRIPTEN=1 make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000" }, - "dependencies": { - "@pgsql/types": "^13.0.0" + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" }, "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0" + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@pgsql/types": "^13.0.0" }, - "keywords": ["postgresql", "parser", "sql", "pg13"], - "author": "Dan Lynch", - "license": "MIT" + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] } diff --git a/libpg-query-14/package.json b/libpg-query-14/package.json index dd4d9155..43a7b8b3 100644 --- a/libpg-query-14/package.json +++ b/libpg-query-14/package.json @@ -2,32 +2,57 @@ "name": "libpg-query-14", "version": "14.3.0", "description": "PostgreSQL 14 query parser (lightweight - no deparse/scan)", - "main": "wasm/index.js", - "types": "wasm/index.d.ts", - "type": "module", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*" + ], "exports": { ".": { + "types": "./wasm/index.d.ts", "import": "./wasm/index.js", - "types": "./wasm/index.d.ts" + "require": "./wasm/index.cjs" } }, - "files": [ - "wasm/", - "README.md" - ], "scripts": { - "build": "EMSCRIPTEN=1 make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000" }, - "dependencies": { - "@pgsql/types": "^14.0.0" + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" }, "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0" + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@pgsql/types": "^14.0.0" }, - "keywords": ["postgresql", "parser", "sql", "pg14"], - "author": "Dan Lynch", - "license": "MIT" + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] } diff --git a/libpg-query-15/package.json b/libpg-query-15/package.json index 688e26d9..662408a8 100644 --- a/libpg-query-15/package.json +++ b/libpg-query-15/package.json @@ -2,32 +2,57 @@ "name": "libpg-query-15", "version": "15.4.2", "description": "PostgreSQL 15 query parser (lightweight - no deparse/scan)", - "main": "wasm/index.js", - "types": "wasm/index.d.ts", - "type": "module", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*" + ], "exports": { ".": { + "types": "./wasm/index.d.ts", "import": "./wasm/index.js", - "types": "./wasm/index.d.ts" + "require": "./wasm/index.cjs" } }, - "files": [ - "wasm/", - "README.md" - ], "scripts": { - "build": "EMSCRIPTEN=1 make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000" }, - "dependencies": { - "@pgsql/types": "^15.0.0" + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" }, "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0" + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@pgsql/types": "^15.0.0" }, - "keywords": ["postgresql", "parser", "sql", "pg15"], - "author": "Dan Lynch", - "license": "MIT" + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] } diff --git a/libpg-query-16/package.json b/libpg-query-16/package.json index 5d92695a..854a6e28 100644 --- a/libpg-query-16/package.json +++ b/libpg-query-16/package.json @@ -2,32 +2,57 @@ "name": "libpg-query-16", "version": "16.5.2", "description": "PostgreSQL 16 query parser (lightweight - no deparse/scan)", - "main": "wasm/index.js", - "types": "wasm/index.d.ts", - "type": "module", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*" + ], "exports": { ".": { + "types": "./wasm/index.d.ts", "import": "./wasm/index.js", - "types": "./wasm/index.d.ts" + "require": "./wasm/index.cjs" } }, - "files": [ - "wasm/", - "README.md" - ], "scripts": { - "build": "EMSCRIPTEN=1 make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000" }, - "dependencies": { - "@pgsql/types": "^16.0.0" + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" }, "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0" + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@pgsql/types": "^16.0.0" }, - "keywords": ["postgresql", "parser", "sql", "pg16"], - "author": "Dan Lynch", - "license": "MIT" + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] } diff --git a/libpg-query-full/package.json b/libpg-query-full/package.json index f620a0bf..7c93fb1f 100644 --- a/libpg-query-full/package.json +++ b/libpg-query-full/package.json @@ -1,37 +1,62 @@ { "name": "libpg-query-full", - "version": "17.6.1", - "description": "PostgreSQL 17 query parser (full functionality with deparse/scan)", - "main": "wasm/index.js", - "types": "wasm/index.d.ts", - "type": "module", + "version": "17.2.0", + "description": "The real PostgreSQL query parser (PostgreSQL 17 - full functionality)", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*", + "proto.js" + ], "exports": { ".": { + "types": "./wasm/index.d.ts", "import": "./wasm/index.js", - "types": "./wasm/index.d.ts" + "require": "./wasm/index.cjs" } }, - "files": [ - "wasm/", - "proto.js", - "README.md" - ], "scripts": { - "build": "EMSCRIPTEN=1 make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000", + "yamlize": "node ./scripts/yamlize.js", "protogen": "node ./scripts/protogen.js" }, + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" + }, + "devDependencies": { + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, "dependencies": { "@pgsql/types": "^17.0.0", "@launchql/protobufjs": "7.2.6" }, - "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0", - "pg-proto-parser": "^0.6.3" - }, - "keywords": ["postgresql", "parser", "sql", "pg17", "deparse", "scan"], - "author": "Dan Lynch", - "license": "MIT" + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] } diff --git a/libpg-query-multi/package.json b/libpg-query-multi/package.json index e39ae1e4..42bbc31d 100644 --- a/libpg-query-multi/package.json +++ b/libpg-query-multi/package.json @@ -2,13 +2,25 @@ "name": "libpg-query-multi", "version": "1.0.0", "description": "Multi-version PostgreSQL query parser (PG15/16/17 with runtime version selection)", - "main": "index.js", - "types": "index.d.ts", - "type": "module", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./index.cjs", + "typings": "./index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "index.js", + "index.cjs", + "index.d.ts", + "pg15/*", + "pg16/*", + "pg17/*" + ], "exports": { ".": { + "types": "./index.d.ts", "import": "./index.js", - "types": "./index.d.ts" + "require": "./index.cjs" }, "./pg15": { "import": "./pg15/index.js", @@ -23,26 +35,42 @@ "types": "./pg17/index.d.ts" } }, - "files": [ - "index.js", - "index.d.ts", - "pg15/", - "pg16/", - "pg17/", - "README.md" - ], "scripts": { - "build": "make build", - "clean": "make clean", + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", "test": "mocha test/*.test.js --timeout 5000", "generate-types": "node ./scripts/generate-types.js" }, + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" + }, "devDependencies": { - "chai": "^4.3.7", - "mocha": "^10.2.0", + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3", "pg-proto-parser": "^0.6.3" }, - "keywords": ["postgresql", "parser", "sql", "multi-version", "pg15", "pg16", "pg17"], - "author": "Dan Lynch", - "license": "MIT" + "dependencies": {}, + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database", + "multi-version" + ] } diff --git a/package.json b/package.json index 85117915..bb489423 100644 --- a/package.json +++ b/package.json @@ -1,62 +1,31 @@ { - "name": "libpg-query", - "version": "17.2.0", - "description": "The real PostgreSQL query parser", - "homepage": "https://github.com/launchql/libpg-query-node", - "main": "./wasm/index.cjs", - "typings": "./wasm/index.d.ts", - "publishConfig": { - "access": "public" - }, - "files": [ - "wasm/*", - "proto.js" - ], - "exports": { - ".": { - "types": "./wasm/index.d.ts", - "import": "./wasm/index.js", - "require": "./wasm/index.cjs" - } - }, + "name": "libpg-query-node-workspace", + "version": "1.0.0", + "description": "Multi-version PostgreSQL query parser workspace", + "private": true, "scripts": { - "clean": "yarn wasm:clean && rimraf cjs esm", - "build:js": "node scripts/build.js", - "build": "yarn clean; yarn wasm:build; yarn build:js", - "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", - "wasm:build": "yarn wasm:make build", - "wasm:rebuild": "yarn wasm:make rebuild", - "wasm:clean": "yarn wasm:make clean", - "wasm:clean-cache": "yarn wasm:make clean-cache", - "test": "mocha test/*.test.js --timeout 5000", - "yamlize": "node ./scripts/yamlize.js", - "protogen": "node ./scripts/protogen.js" - }, - "author": "Dan Lynch (http://github.com/pyramation)", - "license": "LICENSE IN LICENSE", - "repository": { - "type": "git", - "url": "git://github.com/launchql/libpg-query-node.git" + "build-all": "make build-all", + "clean-all": "make clean-all", + "test-all": "make test-all", + "install-deps": "make install-deps" }, "devDependencies": { - "@launchql/proto-cli": "1.25.0", - "@yamlize/cli": "^0.8.0", - "chai": "^3.5.0", - "mocha": "^11.7.0", - "rimraf": "5.0.0", - "typescript": "^5.3.3" - }, - "dependencies": { - "@pgsql/types": "^17.0.0", - "@launchql/protobufjs": "7.2.6" + "rimraf": "5.0.0" }, "keywords": [ "sql", - "postgres", + "postgres", "postgresql", "pg", "query", "plpgsql", - "database" - ] + "database", + "multi-version" + ], + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" + } } From e17e74b70d29735c53259b31729c2b5f4d7467de Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 21 Jun 2025 04:51:47 +0000 Subject: [PATCH 3/3] fix: restore root package.json with wasm:build script for CI - Revert root package.json to original libpg-query structure - Rename libpg-query-full to libpg-query-17 to avoid conflicts - Add workspace coordination scripts to root package.json - Maintain Docker-based build pattern across all packages - Fix CI Build WASM job that was failing due to missing wasm:build script Co-Authored-By: Dan Lynch --- libpg-query-full/package.json | 2 +- package.json | 67 ++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/libpg-query-full/package.json b/libpg-query-full/package.json index 7c93fb1f..a92e76b0 100644 --- a/libpg-query-full/package.json +++ b/libpg-query-full/package.json @@ -1,5 +1,5 @@ { - "name": "libpg-query-full", + "name": "libpg-query-17", "version": "17.2.0", "description": "The real PostgreSQL query parser (PostgreSQL 17 - full functionality)", "homepage": "https://github.com/launchql/libpg-query-node", diff --git a/package.json b/package.json index bb489423..3c658387 100644 --- a/package.json +++ b/package.json @@ -1,31 +1,66 @@ { - "name": "libpg-query-node-workspace", - "version": "1.0.0", - "description": "Multi-version PostgreSQL query parser workspace", - "private": true, + "name": "libpg-query", + "version": "17.2.0", + "description": "The real PostgreSQL query parser", + "homepage": "https://github.com/launchql/libpg-query-node", + "main": "./wasm/index.cjs", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "files": [ + "wasm/*", + "proto.js" + ], + "exports": { + ".": { + "types": "./wasm/index.d.ts", + "import": "./wasm/index.js", + "require": "./wasm/index.cjs" + } + }, "scripts": { + "clean": "yarn wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "yarn clean; yarn wasm:build; yarn build:js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", + "test": "mocha test/*.test.js --timeout 5000", + "yamlize": "node ./scripts/yamlize.js", + "protogen": "node ./scripts/protogen.js", "build-all": "make build-all", - "clean-all": "make clean-all", + "clean-all": "make clean-all", "test-all": "make test-all", "install-deps": "make install-deps" }, + "author": "Dan Lynch (http://github.com/pyramation)", + "license": "LICENSE IN LICENSE", + "repository": { + "type": "git", + "url": "git://github.com/launchql/libpg-query-node.git" + }, "devDependencies": { - "rimraf": "5.0.0" + "@launchql/proto-cli": "1.25.0", + "@yamlize/cli": "^0.8.0", + "chai": "^3.5.0", + "mocha": "^11.7.0", + "rimraf": "5.0.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@pgsql/types": "^17.0.0", + "@launchql/protobufjs": "7.2.6" }, "keywords": [ "sql", - "postgres", + "postgres", "postgresql", "pg", "query", "plpgsql", - "database", - "multi-version" - ], - "author": "Dan Lynch (http://github.com/pyramation)", - "license": "LICENSE IN LICENSE", - "repository": { - "type": "git", - "url": "git://github.com/launchql/libpg-query-node.git" - } + "database" + ] }