Merge pull request #106 from launchql/feat/wasm17-scan

Feat/wasm17 scan

Author: pyramation

.gitignore

@@ -7,4 +7,5 @@ wasm/libpg-query.js
 *.wasm
 .cache
 esm/
-cjs/
+cjs/
+.claude

Makefile

@@ -54,8 +54,9 @@ ifdef EMSCRIPTEN
 		-v \
 		$(CXXFLAGS) \
 		-I$(LIBPG_QUERY_DIR) \
+		-I$(LIBPG_QUERY_DIR)/vendor \
 		-L$(LIBPG_QUERY_DIR) \
-		-sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_query_protobuf','_wasm_get_protobuf_len','_wasm_deparse_protobuf','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \
+		-sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_query_protobuf','_wasm_get_protobuf_len','_wasm_deparse_protobuf','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string']" \
 		-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','UTF8ToString','HEAPU8','HEAPU32']" \
 		-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
 		-sENVIRONMENT="web,node" \

README.md

@@ -157,6 +157,29 @@ const normalized = normalizeSync('SELECT * FROM users WHERE active = true');
 // Returns: string - normalized SQL query
 ```
 
+### `scan(sql: string): Promise<ScanResult>`
+
+Scans (tokenizes) a SQL query and returns detailed information about each token. Returns a Promise for a ScanResult containing all tokens with their positions, types, and classifications.
+
+```typescript
+import { scan } from 'libpg-query';
+
+const result = await scan('SELECT * FROM users WHERE id = $1');
+// Returns: ScanResult - detailed tokenization information
+console.log(result.tokens[0]); // { start: 0, end: 6, text: "SELECT", tokenType: 651, tokenName: "UNKNOWN", keywordKind: 4, keywordName: "RESERVED_KEYWORD" }
+```
+
+### `scanSync(sql: string): ScanResult`
+
+Synchronous version that scans (tokenizes) a SQL query directly.
+
+```typescript
+import { scanSync } from 'libpg-query';
+
+const result = scanSync('SELECT * FROM users WHERE id = $1');
+// Returns: ScanResult - detailed tokenization information
+```
+
 ### Initialization
 
 The library provides both async and sync methods. Async methods handle initialization automatically, while sync methods require explicit initialization.
@@ -166,37 +189,40 @@ The library provides both async and sync methods. Async methods handle initializ
 Async methods handle initialization automatically and are always safe to use:
 
 ```typescript
-import { parse, deparse } from 'libpg-query';
+import { parse, deparse, scan } from 'libpg-query';
 
 // These handle initialization automatically
 const result = await parse('SELECT * FROM users');
 const sql = await deparse(result);
+const tokens = await scan('SELECT * FROM users');
 ```
 
 #### Sync Methods
 
 Sync methods require explicit initialization using `loadModule()`:
 
 ```typescript
-import { loadModule, parseSync } from 'libpg-query';
+import { loadModule, parseSync, scanSync } from 'libpg-query';
 
 // Initialize first
 await loadModule();
 
 // Now safe to use sync methods
 const result = parseSync('SELECT * FROM users');
+const tokens = scanSync('SELECT * FROM users');
 ```
 
 ### `loadModule(): Promise<void>`
 
 Explicitly initializes the WASM module. Required before using any sync methods.
 
 ```typescript
-import { loadModule, parseSync } from 'libpg-query';
+import { loadModule, parseSync, scanSync } from 'libpg-query';
 
 // Initialize before using sync methods
 await loadModule();
 const result = parseSync('SELECT * FROM users');
+const tokens = scanSync('SELECT * FROM users');
 ```
 
 Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first.
@@ -215,6 +241,21 @@ interface Statement {
   stmt_location: number;
   query: string;
 }
+
+interface ScanResult {
+  version: number;
+  tokens: ScanToken[];
+}
+
+interface ScanToken {
+  start: number;          // Starting position in the SQL string
+  end: number;            // Ending position in the SQL string
+  text: string;           // The actual token text
+  tokenType: number;      // Numeric token type identifier
+  tokenName: string;      // Human-readable token type name
+  keywordKind: number;    // Numeric keyword classification
+  keywordName: string;    // Human-readable keyword classification
+}
 ```
 
 **Note:** The return value is an array, as multiple queries may be provided in a single string (semicolon-delimited, as PostgreSQL expects).