Skip to content

Commit 8e6657c

Browse files
samualtnormansamualtnorman
committed
rewrite preprocessor syntax "parser"
we're now using the acorn tokeniser to iterate over private identifier tokens, replacing them with regular identifiers, then parsing the resulting string to an AST using babel, and using that to turn the private identifiers back into private identifiers or recognise them as preprocessor syntax current drawback is acorn supports less stuff than babel does so lost tuples for example
1 parent ed6d72e commit 8e6657c

File tree

1 file changed

+130
-88
lines changed

1 file changed

+130
-88
lines changed

‎src/processScript/preprocess.ts‎

Lines changed: 130 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,19 @@ import babelTraverse from "@babel/traverse"
55
import type { Program } from "@babel/types"
66
import t from "@babel/types"
77
import type { LaxPartial } from "@samual/lib"
8-
import { assert } from "@samual/lib/assert"
8+
import { assert, ensure } from "@samual/lib/assert"
99
import { spliceString } from "@samual/lib/spliceString"
1010
import { tokenizer as tokenise, tokTypes as TokenTypes } from "acorn"
1111
import { resolve as resolveModule } from "import-meta-resolve"
12+
import { validDBMethods } from "../constants"
1213

1314
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
1415
const { default: traverse } = babelTraverse as any as typeof import("@babel/traverse")
1516
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
1617
const { default: generate } = babelGenerator as any as typeof import("@babel/generator")
1718

1819
const SUBSCRIPT_PREFIXES = [ `s`, `fs`, `4s`, `hs`, `3s`, `ms`, `2s`, `ls`, `1s`, `ns`, `0s` ]
20+
const PREPROCESSOR_NAMES = [ ...SUBSCRIPT_PREFIXES, `D`, `G`, `FMCL`, `db` ]
1921

2022
export type PreprocessOptions = LaxPartial<{ /** 11 a-z 0-9 characters */ uniqueId: string }>
2123

@@ -25,110 +27,150 @@ export async function preprocess(code: string, { uniqueId = `00000000000` }: Pre
2527
: Promise<{ code: string }> {
2628
assert(/^\w{11}$/.test(uniqueId), HERE)
2729

28-
const tokensIterable = tokenise(code, { ecmaVersion: `latest` })
29-
30-
for (const token of tokensIterable) {
31-
assert(`value` in token, HERE)
32-
33-
if (token.type != TokenTypes.privateId)
34-
continue
35-
36-
assert(typeof token.value == `string`, HERE)
37-
38-
if (!SUBSCRIPT_PREFIXES.includes(token.value))
39-
continue
30+
const sourceCode = code
31+
const tokens = [ ...tokenise(code, { ecmaVersion: `latest` }) ]
4032

41-
const nextToken = tokensIterable.getToken()
33+
const needExportDefault =
34+
ensure(tokens[0], HERE).type == TokenTypes._function && ensure(tokens[1], HERE).type == TokenTypes.parenL
4235

43-
if (nextToken.type != TokenTypes._in && nextToken.type != TokenTypes.dot)
44-
throw SyntaxError(`Subscripts must be in the form of #fs.foo.bar`)
45-
}
36+
const maybePrivatePrefix = `$${uniqueId}$MAYBE_PRIVATE$`
4637

47-
const sourceCode = code
48-
let lengthBefore
49-
50-
do {
51-
lengthBefore = code.length
52-
code = code.replace(/^\s+/, ``).replace(/^\/\/.*/, ``).replace(/^\/\*[\s\S]*?\*\//, ``)
53-
} while (code.length != lengthBefore)
54-
55-
code = code.replace(/^function\s*\(/, `export default function (`)
56-
57-
let file
58-
59-
while (true) {
60-
let error
61-
62-
try {
63-
file = parse(code, {
64-
plugins: [
65-
`typescript`,
66-
[ `decorators`, { decoratorsBeforeExport: true } ],
67-
`doExpressions`,
68-
`functionBind`,
69-
`functionSent`,
70-
`partialApplication`,
71-
[ `pipelineOperator`, { proposal: `hack`, topicToken: `%` } ],
72-
`throwExpressions`,
73-
[ `recordAndTuple`, { syntaxType: `hash` } ],
74-
`classProperties`,
75-
`classPrivateProperties`,
76-
`classPrivateMethods`,
77-
`logicalAssignment`,
78-
`numericSeparator`,
79-
`nullishCoalescingOperator`,
80-
`optionalChaining`,
81-
`optionalCatchBinding`,
82-
`objectRestSpread`
83-
],
84-
sourceType: `module`
85-
})
86-
87-
break
88-
} catch (error_) {
89-
assert(error_ instanceof SyntaxError, HERE)
90-
error = error_ as SyntaxError & { pos: number, code: string, reasonCode: string }
91-
}
38+
for (const token of [ ...tokens ].reverse()) {
39+
assert(`value` in token, HERE)
9240

93-
if (error.code != `BABEL_PARSER_SYNTAX_ERROR` || error.reasonCode != `PrivateInExpectedIn`) {
94-
console.log((/.+/.exec(code.slice(error.pos)))?.[0])
41+
if (token.type == TokenTypes.privateId) {
42+
assert(typeof token.value == `string`, HERE)
9543

96-
throw error
44+
if (PREPROCESSOR_NAMES.includes(token.value))
45+
code = spliceString(code, maybePrivatePrefix + token.value, token.start, token.end - token.start)
9746
}
98-
99-
const codeSlice = code.slice(error.pos)
100-
let match
101-
102-
if ((match = /^#[0-4fhmln]s\.scripts\.quine\(\)/.exec(codeSlice)))
103-
code = spliceString(code, JSON.stringify(sourceCode), error.pos, match[0]!.length)
104-
else if ((match = /^#[0-4fhmln]?s\./.exec(codeSlice)))
105-
code = spliceString(code, `$`, error.pos, 1)
106-
else if ((match = /^#D[^\w$]/.exec(codeSlice)))
107-
code = spliceString(code, `$`, error.pos, 1)
108-
else if ((match = /^#FMCL/.exec(codeSlice)))
109-
code = spliceString(code, `$${uniqueId}$FMCL$`, error.pos, match[0]!.length)
110-
else if ((match = /^#G/.exec(codeSlice)))
111-
code = spliceString(code, `$${uniqueId}$GLOBAL$`, error.pos, match[0]!.length)
112-
else if ((match = /^#db\./.exec(codeSlice)))
113-
code = spliceString(code, `$`, error.pos, 1)
114-
else
115-
throw error
11647
}
11748

49+
if (needExportDefault)
50+
code = `export default ${code}`
51+
11852
let program!: NodePath<Program>
11953

120-
traverse(file, {
54+
traverse(parse(code, {
55+
plugins: [
56+
`typescript`,
57+
[ `decorators`, { decoratorsBeforeExport: true } ],
58+
`doExpressions`,
59+
`functionBind`,
60+
`functionSent`,
61+
`partialApplication`,
62+
[ `pipelineOperator`, { proposal: `hack`, topicToken: `%` } ],
63+
`throwExpressions`,
64+
[ `recordAndTuple`, { syntaxType: `hash` } ],
65+
`classProperties`,
66+
`classPrivateProperties`,
67+
`classPrivateMethods`,
68+
`logicalAssignment`,
69+
`numericSeparator`,
70+
`nullishCoalescingOperator`,
71+
`optionalChaining`,
72+
`optionalCatchBinding`,
73+
`objectRestSpread`
74+
],
75+
sourceType: `module`
76+
}), {
12177
Program(path) {
12278
program = path
123-
path.skip()
79+
},
80+
Identifier(path) {
81+
if (!path.node.name.startsWith(maybePrivatePrefix))
82+
return
83+
84+
const name = path.node.name.slice(maybePrivatePrefix.length)
85+
86+
if (path.parent.type == `ClassProperty` && path.parent.key == path.node) {
87+
path.parentPath.replaceWith(t.classPrivateProperty(
88+
t.privateName(t.identifier(name)),
89+
path.parent.value,
90+
path.parent.decorators,
91+
path.parent.static
92+
))
93+
} else if (path.parent.type == `MemberExpression`) {
94+
if (path.parent.property == path.node) {
95+
assert(!path.parent.computed, HERE)
96+
path.replaceWith(t.privateName(t.identifier(name)))
97+
} else {
98+
assert(path.parent.object == path.node, HERE)
99+
100+
if (name == `db`) {
101+
if (path.parent.computed)
102+
throw Error(`Index notation cannot be used on #db, must be in the form of #db.<DB method name>`)
103+
104+
if (path.parent.property.type != `Identifier`)
105+
throw Error(`Expected DB method name to be an Identifier, got ${path.parent.property.type} instead`)
106+
107+
if (!validDBMethods.includes(path.parent.property.name))
108+
throw Error(`Invalid DB method #db.${path.parent.property.name}`)
109+
110+
path.node.name = `$db`
111+
} else {
112+
assert(SUBSCRIPT_PREFIXES.includes(name), HERE)
113+
114+
if (path.parent.computed)
115+
throw Error(`Index notation cannot be used for subscripts, must be in the form of #${name}.foo.bar`)
116+
117+
if (path.parent.property.type != `Identifier`)
118+
throw Error(`Expected subscript user name to be Identifier but got ${path.parent.property.type} instead`)
119+
120+
if (path.parentPath.parent.type != `MemberExpression`)
121+
throw Error(`Subscripts must be in the form of #${name}.foo.bar`)
122+
123+
if (path.parentPath.parent.computed)
124+
throw Error(`Index notation cannot be used for subscripts, must be in the form of #${name}.foo.bar`)
125+
126+
if (path.parentPath.parent.property.type != `Identifier`)
127+
throw Error(`Expected subscript subname to be Identifier but got ${path.parent.property.type} instead`)
128+
129+
if (
130+
path.parentPath.parentPath?.parent.type == `CallExpression` &&
131+
path.parent.property.name == `scripts` &&
132+
path.parentPath.parent.property.name == `quine`
133+
)
134+
ensure(path.parentPath.parentPath.parentPath, HERE).replaceWith(t.stringLiteral(sourceCode))
135+
else
136+
path.node.name = `$${name}`
137+
}
138+
}
139+
} else if (path.parent.type == `BinaryExpression` && path.parent.left == path.node && path.parent.operator == `in`)
140+
path.replaceWith(t.privateName(t.identifier(name)))
141+
else if (path.parent.type == `ClassMethod` && path.parent.key == path.node) {
142+
assert(path.parent.kind != `constructor`, HERE)
143+
144+
path.parentPath.replaceWith(t.classPrivateMethod(
145+
path.parent.kind,
146+
t.privateName(t.identifier(name)),
147+
path.parent.params,
148+
path.parent.body,
149+
path.parent.static
150+
))
151+
} else {
152+
if (name == `FMCL`)
153+
path.node.name = `$${uniqueId}$FMCL$`
154+
else if (name == `G`)
155+
path.node.name = `$${uniqueId}$GLOBAL$`
156+
else if (name == `D`)
157+
path.node.name = `$D`
158+
else if (name == `db`)
159+
throw Error(`Invalid #db syntax, must be in the form of #db.<DB method name>`)
160+
else {
161+
assert(SUBSCRIPT_PREFIXES.includes(name), `${HERE} ${name}`)
162+
163+
throw Error(`Invalid subscript syntax, must be in the form of #${name}.foo.bar`)
164+
}
165+
}
124166
}
125167
})
126168

127169
const needRecord = program.scope.hasGlobal(`Record`)
128170
const needTuple = program.scope.hasGlobal(`Tuple`)
129171

130172
if (needRecord || needTuple) {
131-
file.program.body.unshift(t.importDeclaration(
173+
program.node.body.unshift(t.importDeclaration(
132174
needRecord
133175
? (needTuple
134176
? [
@@ -143,7 +185,7 @@ export async function preprocess(code: string, { uniqueId = `00000000000` }: Pre
143185
}
144186

145187
if (program.scope.hasGlobal(`Proxy`)) {
146-
file.program.body.unshift(t.importDeclaration([
188+
program.node.body.unshift(t.importDeclaration([
147189
t.importDefaultSpecifier(t.identifier(`Proxy`))
148190
], t.stringLiteral(resolveModule(`proxy-polyfill/src/proxy.js`, import.meta.url).slice(7))))
149191
}
@@ -152,5 +194,5 @@ export async function preprocess(code: string, { uniqueId = `00000000000` }: Pre
152194
throw Error(`Scripts that only contain a single function declaration are no longer supported.\nPrefix the function declaration with \`export default\`.`)
153195
}
154196

155-
return { code: generate(file).code }
197+
return { code: generate(program.node).code }
156198
}

0 commit comments

Comments
 (0)