Skip to content

Commit adb3c1d

Browse files
pyramationpyramation
committed
get ready for full parse()
1 parent 3ee0458 commit adb3c1d

File tree

6 files changed

+24925
-92640
lines changed

6 files changed

+24925
-92640
lines changed

‎__fixtures__/generated/generated.json‎

Lines changed: 23156 additions & 92624 deletions
Large diffs are not rendered by default.

‎__fixtures__/generated/upstream-diff.json‎

Lines changed: 1594 additions & 0 deletions
Large diffs are not rendered by default.

‎packages/deparser/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"fixtures:ast": "ts-node scripts/make-fixtures-ast.ts",
3131
"fixtures:sql": "ts-node scripts/make-fixtures-sql.ts",
3232
"fixtures": "ts-node scripts/make-fixtures.ts",
33+
"fixtures:upstream-diff": "ts-node scripts/make-upstream-diff.ts",
3334
"lint": "eslint . --fix",
3435
"test": "jest",
3536
"test:watch": "jest --watch"

‎packages/deparser/scripts/make-fixtures.ts‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import * as path from 'path';
33
import * as fs from 'fs';
44
import { sync as globSync } from 'glob';
5-
import { parse, deparse } from 'libpg-query';
5+
import { parse } from 'libpg-query';
66
import { ParseResult, RawStmt } from '@pgsql/types';
77

88
const FIXTURE_DIR = path.join(__dirname, '../../../__fixtures__/kitchen-sink');
@@ -80,8 +80,8 @@ function extractOriginalSQL(originalSQL: string, rawStmt: RawStmt, isFirst: bool
8080
}
8181

8282
async function main() {
83-
// Collect both deparsed and original SQL in a single JSON
84-
const results: Record<string, { deparsed: string; original?: string }> = {};
83+
// Collect original SQL in a single JSON
84+
const results: Record<string, string> = {};
8585

8686
for (const fixturePath of fixtures) {
8787
const relPath = path.relative(FIXTURE_DIR, fixturePath);
@@ -96,22 +96,14 @@ async function main() {
9696

9797
for (let idx = 0; idx < parseResult.stmts.length; idx++) {
9898
const stmt = parseResult.stmts[idx];
99-
let deparsedSql: string;
100-
try {
101-
deparsedSql = await deparse({ version: 170000, stmts: [stmt] });
102-
} catch (err: any) {
103-
console.error(`Failed to deparse statement ${idx + 1} in ${relPath}:`, err);
104-
continue;
105-
}
10699

107100
// Extract original SQL using location info
108101
const originalSql = extractOriginalSQL(sql, stmt, idx === 0);
109102

110103
const key = `${relPath.replace(/\.sql$/, '')}-${idx + 1}.sql`;
111-
results[key] = {
112-
deparsed: deparsedSql,
113-
...(originalSql && { original: originalSql })
114-
};
104+
if (originalSql) {
105+
results[key] = originalSql;
106+
}
115107
}
116108
}
117109

‎packages/deparser/scripts/make-upstream-diff.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/usr/bin/env ts-node
2+
import * as path from 'path';
3+
import * as fs from 'fs';
4+
import { sync as globSync } from 'glob';
5+
import { parse, deparse } from 'libpg-query';
6+
import { ParseResult, RawStmt } from '@pgsql/types';
7+
import { deparse as ourDeparse } from '../src';
8+
import { cleanTree } from '../src/utils';
9+
10+
const FIXTURE_DIR = path.join(__dirname, '../../../__fixtures__/kitchen-sink');
11+
const OUT_DIR = path.join(__dirname, '../../../__fixtures__/generated');
12+
13+
function ensureDir(dir: string) {
14+
if (!fs.existsSync(dir)) {
15+
fs.mkdirSync(dir, { recursive: true });
16+
}
17+
}
18+
19+
ensureDir(OUT_DIR);
20+
21+
const fixtures = globSync(path.join(FIXTURE_DIR, '**/*.sql'));
22+
23+
function extractOriginalSQL(originalSQL: string, rawStmt: RawStmt, isFirst: boolean = false): string | null {
24+
let extracted: string | null = null;
25+
26+
if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
27+
// Check if we need to adjust location - if the character before the location looks like part of a SQL keyword
28+
let adjustedLocation = rawStmt.stmt_location;
29+
if (rawStmt.stmt_location > 0) {
30+
const charBefore = originalSQL[rawStmt.stmt_location - 1];
31+
const charAtLocation = originalSQL[rawStmt.stmt_location];
32+
// If the char before looks like it should be part of the statement (e.g., 'C' before 'REATE')
33+
if (/[A-Za-z]/.test(charBefore) && /[A-Za-z]/.test(charAtLocation)) {
34+
adjustedLocation = rawStmt.stmt_location - 1;
35+
}
36+
}
37+
extracted = originalSQL.substring(adjustedLocation, adjustedLocation + rawStmt.stmt_len);
38+
} else if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len === undefined) {
39+
// We have location but no length - extract from location to end of file
40+
let adjustedLocation = rawStmt.stmt_location;
41+
if (rawStmt.stmt_location > 0) {
42+
const charBefore = originalSQL[rawStmt.stmt_location - 1];
43+
const charAtLocation = originalSQL[rawStmt.stmt_location];
44+
// If the char before looks like it should be part of the statement (e.g., 'C' before 'REATE')
45+
if (/[A-Za-z]/.test(charBefore) && /[A-Za-z]/.test(charAtLocation)) {
46+
adjustedLocation = rawStmt.stmt_location - 1;
47+
}
48+
}
49+
extracted = originalSQL.substring(adjustedLocation);
50+
} else if (isFirst && rawStmt.stmt_len !== undefined) {
51+
// For first statement when location is missing but we have length
52+
extracted = originalSQL.substring(0, rawStmt.stmt_len);
53+
} else if (isFirst && rawStmt.stmt_location === undefined && rawStmt.stmt_len === undefined) {
54+
// For first statement when both location and length are missing, use entire SQL
55+
extracted = originalSQL;
56+
}
57+
58+
if (extracted) {
59+
// Split into lines to handle leading whitespace and comments properly
60+
const lines = extracted.split('\n');
61+
let startLineIndex = 0;
62+
63+
// Find the first line that contains actual SQL content
64+
for (let i = 0; i < lines.length; i++) {
65+
const line = lines[i].trim();
66+
// Skip empty lines and comment-only lines
67+
if (line === '' || line.startsWith('--')) {
68+
continue;
69+
}
70+
startLineIndex = i;
71+
break;
72+
}
73+
74+
// Reconstruct from the first SQL line, preserving the original indentation of that line
75+
if (startLineIndex < lines.length) {
76+
const resultLines = lines.slice(startLineIndex);
77+
extracted = resultLines.join('\n').trim();
78+
}
79+
}
80+
81+
return extracted;
82+
}
83+
84+
async function main() {
85+
// Collect only files with differences between deparsers
86+
const results: Record<string, { upstream?: string; deparsed?: string; original: string }> = {};
87+
88+
for (const fixturePath of fixtures) {
89+
const relPath = path.relative(FIXTURE_DIR, fixturePath);
90+
const sql = fs.readFileSync(fixturePath, 'utf-8');
91+
let parseResult: ParseResult;
92+
try {
93+
parseResult = await parse(sql);
94+
} catch (err: any) {
95+
console.error(`Failed to parse ${relPath}:`, err);
96+
continue;
97+
}
98+
99+
for (let idx = 0; idx < parseResult.stmts.length; idx++) {
100+
const stmt = parseResult.stmts[idx];
101+
102+
// Extract original SQL using location info
103+
const originalSql = extractOriginalSQL(sql, stmt, idx === 0);
104+
if (!originalSql) {
105+
console.error(`Failed to extract original SQL for statement ${idx + 1} in ${relPath}`);
106+
continue;
107+
}
108+
109+
// Get source of truth: cleanTree(parse(original))
110+
let sourceOfTruthAst: any;
111+
try {
112+
const originalParsed = await parse(originalSql);
113+
sourceOfTruthAst = cleanTree(originalParsed.stmts?.[0]?.stmt);
114+
} catch (err: any) {
115+
console.error(`Failed to parse original SQL for statement ${idx + 1} in ${relPath}:`, err);
116+
continue;
117+
}
118+
119+
// Get upstream deparse and its AST
120+
let upstreamSql: string | undefined;
121+
let upstreamAst: any;
122+
try {
123+
upstreamSql = await deparse({ version: 170000, stmts: [stmt] });
124+
const upstreamParsed = await parse(upstreamSql);
125+
upstreamAst = cleanTree(upstreamParsed.stmts?.[0]?.stmt);
126+
} catch (err: any) {
127+
console.error(`Failed to process upstream deparse for statement ${idx + 1} in ${relPath}:`, err);
128+
continue;
129+
}
130+
131+
// Get our deparse and its AST
132+
let ourDeparsedSql: string | undefined;
133+
let ourAst: any;
134+
try {
135+
ourDeparsedSql = ourDeparse(stmt.stmt);
136+
const ourParsed = await parse(ourDeparsedSql);
137+
ourAst = cleanTree(ourParsed.stmts?.[0]?.stmt);
138+
} catch (err: any) {
139+
console.error(`Failed to process our deparse for statement ${idx + 1} in ${relPath}:`, err);
140+
// Continue with just upstream comparison
141+
}
142+
143+
// Compare ASTs to source of truth only
144+
const upstreamMatches = JSON.stringify(upstreamAst) === JSON.stringify(sourceOfTruthAst);
145+
const ourMatches = ourAst ? JSON.stringify(ourAst) === JSON.stringify(sourceOfTruthAst) : false;
146+
147+
// Only include if either deparser differs from original
148+
if (!upstreamMatches || !ourMatches) {
149+
const key = `${relPath.replace(/\.sql$/, '')}-${idx + 1}.sql`;
150+
results[key] = {
151+
original: originalSql,
152+
// Show upstream only if it differs from original
153+
...(!upstreamMatches && upstreamSql && { upstream: upstreamSql }),
154+
// Show our deparser only if it differs from original
155+
...(!ourMatches && ourDeparsedSql && { deparsed: ourDeparsedSql })
156+
};
157+
}
158+
}
159+
}
160+
161+
// Write aggregated JSON to output file
162+
const outputFile = path.join(OUT_DIR, 'upstream-diff.json');
163+
fs.writeFileSync(outputFile, JSON.stringify(results, null, 2));
164+
console.log(`Wrote JSON to ${outputFile}`);
165+
}
166+
167+
main().catch(console.error);

‎packages/deparser/test-utils/index.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,7 @@ export class FixtureTestUtils extends TestUtils {
219219
const entries = this.getTestEntries(filters);
220220
for (const [relativePath, sql] of entries) {
221221
try {
222-
// @ts-ignore
223-
await this.expectAstMatch(relativePath, sql.original);
222+
await this.expectAstMatch(relativePath, sql);
224223
} catch (err) {
225224
throw err;
226225
}

0 commit comments

Comments
 (0)