From 3f2b74e8de618839b1b594aad18c2a66ef02368a Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 15 Apr 2025 14:54:54 +0200 Subject: [PATCH 1/3] add regex precompilation to build step Signed-off-by: Karthik Ganeshram --- packages/build-tools/package-lock.json | 141 ++++++++++++++++++++++++- packages/build-tools/package.json | 10 +- packages/build-tools/src/index.ts | 15 +-- packages/build-tools/src/precompile.js | 62 +++++++++++ test/test-app/package-lock.json | 16 +-- test/test.sh | 3 +- 6 files changed, 227 insertions(+), 20 deletions(-) create mode 100644 packages/build-tools/src/precompile.js diff --git a/packages/build-tools/package-lock.json b/packages/build-tools/package-lock.json index 90e2c43b..68ba1be1 100644 --- a/packages/build-tools/package-lock.json +++ b/packages/build-tools/package-lock.json @@ -1,16 +1,20 @@ { "name": "@spinframework/build-tools", - "version": "1.0.0", + "version": "1.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@spinframework/build-tools", - "version": "1.0.0", + "version": "1.0.2", "license": "Apache-2.0 WITH LLVM-exception", "dependencies": { "@bytecodealliance/componentize-js": "^0.18.1", "@bytecodealliance/jco": "^1.10.2", + "acorn-walk": "^8.3.4", + "acron": "^1.0.5", + "magic-string": "^0.30.17", + "regexpu-core": "^6.2.0", "yargs": "^17.7.2" }, "bin": { @@ -849,7 +853,6 @@ "version": "8.3.4", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", - "dev": true, "license": "MIT", "dependencies": { "acorn": "^8.11.0" @@ -858,6 +861,18 @@ "node": ">=0.4.0" } }, + "node_modules/acron": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/acron/-/acron-1.0.5.tgz", + "integrity": "sha512-Jm3uey9PSUl5k9RsjBSfDJG35nG0vd9B4FUCV1J/NeVHDQ5HJPVpAhmGrPYtPKh0gLhEUIEy4DKqBy/p60B+4Q==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.11" + }, + "bin": { + "acron": "cli.js" + } + }, "node_modules/ansi-colors": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", @@ -1927,6 +1942,18 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsesc": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/just-extend": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", @@ -1950,6 +1977,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -2035,6 +2068,15 @@ "dev": true, "license": "ISC" }, + "node_modules/magic-string": { + "version": "0.30.17", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", + "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0" + } + }, "node_modules/make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -2510,6 +2552,59 @@ "node": ">=8.10.0" } }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "license": "MIT" + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.2.0.tgz", + "integrity": "sha512-DqHn3DwbmmPVzeKj9woBadqmXxLvQoQIwu7nopMc72ztvxVmVk2SBhSnx67zuye5TP+lJsb/TBQsjLKhnDf3MA==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regexpu-core": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-6.2.0.tgz", + "integrity": "sha512-H66BPQMrv+V16t8xtmq+UC0CBpiTBA60V8ibS1QVReIp8T1z8hwFxqcGzm9K6lgsN7sB5edVH8a+ze6Fqm4weA==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.2.0", + "regjsgen": "^0.8.0", + "regjsparser": "^0.12.0", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regjsgen": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.8.0.tgz", + "integrity": "sha512-RvwtGe3d7LvWiDQXeQw8p5asZUmfU1G/l6WbUXeHta7Y2PEIvBTwH6E2EfmYUK8pxcxEdEmaomqyp0vZZ7C+3Q==", + "license": "MIT" + }, + "node_modules/regjsparser": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.12.0.tgz", + "integrity": "sha512-cnE+y8bz4NhMjISKbgeVJtqNbtf5QpjZP+Bslo+UqkIt9QPnX9q095eiRRASJG1/tz6dlNr6Z5NsBiWYokp6EQ==", + "license": "BSD-2-Clause", + "dependencies": { + "jsesc": "~3.0.2" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3017,6 +3112,46 @@ "dev": true, "license": "MIT" }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", + "integrity": "sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "license": "MIT", + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.2.0.tgz", + "integrity": "sha512-4IehN3V/+kkr5YeSSDDQG8QLqO26XpL2XP3GQtqwlT/QYSECAwFztxVHjlbh0+gjJ3XmNLS0zDsbgs9jWKExLg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz", + "integrity": "sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/packages/build-tools/package.json b/packages/build-tools/package.json index 5e47353e..c752f73f 100644 --- a/packages/build-tools/package.json +++ b/packages/build-tools/package.json @@ -1,10 +1,10 @@ { "name": "@spinframework/build-tools", - "version": "1.0.1", + "version": "1.0.2", "description": "", "main": "index.js", "scripts": { - "build": "make && rm -rf dist && tsc && chmod +x dist/*", + "build": "make && rm -rf dist && tsc && chmod +x dist/* && cp src/*.js dist/", "fmt": "prettier --write \"src/**/*.{ts,tsx,js,jsx}\"", "test": "npm run build && mocha --require ts-node/register test/**/*.spec.ts" }, @@ -31,7 +31,11 @@ "dependencies": { "@bytecodealliance/componentize-js": "^0.18.1", "@bytecodealliance/jco": "^1.10.2", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "acorn-walk": "^8.3.4", + "acron": "^1.0.5", + "magic-string": "^0.30.17", + "regexpu-core": "^6.2.0" }, "files": [ "lib", diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index 7cfad1d4..c7d9c8a3 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -3,15 +3,16 @@ import { componentize } from '@bytecodealliance/componentize-js'; import { version as componentizeVersion } from '@bytecodealliance/componentize-js'; import { getPackagesWithWasiDeps, processWasiDeps } from './wasiDepsParser.js'; - import { calculateChecksum, saveBuildData, } from './utils.js'; import { getCliArgs } from './cli.js'; import { getBuildDataPath, ShouldComponentize } from './build.js'; -import { writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import { mergeWit } from '../lib/wit_tools.js'; +//@ts-ignore +import { precompile } from "./precompile.js" async function main() { try { @@ -61,10 +62,12 @@ async function main() { 'combined-wit:combined-wit@0.3.0', ); - const { component } = await componentize({ - sourcePath: src, - // @ts-ignore - witWorld: inlineWit, + const source = await readFile(src, 'utf8'); + const precompiledSource = precompile(source, src, true) as string; + + // Using the old syntax because the new sytnax does not allow passing in the precompiled string + //@ts-ignore + const { component } = await componentize(precompiledSource, inlineWit, { runtimeArgs, enableAot: CliArgs.aot, }); diff --git a/packages/build-tools/src/precompile.js b/packages/build-tools/src/precompile.js new file mode 100644 index 00000000..f529fb8f --- /dev/null +++ b/packages/build-tools/src/precompile.js @@ -0,0 +1,62 @@ +// Copyright 2024 Fastly Inc. +// License: the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import regexpuc from 'regexpu-core'; +import { parse } from 'acorn'; +import MagicString from 'magic-string'; +import { simple as simpleWalk } from 'acorn-walk'; + +const PREAMBLE = `;{ + // Precompiled regular expressions + const precompile = (r) => { r.exec('a'); r.exec('\\u1000'); };`; +const POSTAMBLE = '}'; + +/// Emit a block of javascript that will pre-compile the regular expressions given. As spidermonkey +/// will intern regular expressions, duplicating them at the top level and testing them with both +/// an ascii and utf8 string should ensure that they won't be re-compiled when run in the fetch +/// handler. +export function precompile(source, filename = '', moduleMode = false) { + const magicString = new MagicString(source, { + filename, + }); + + const ast = parse(source, { + ecmaVersion: 'latest', + sourceType: moduleMode ? 'module' : 'script', + }); + + const precompileCalls = []; + simpleWalk(ast, { + Literal(node) { + if (!node.regex) return; + let transpiledPattern; + try { + transpiledPattern = regexpuc(node.regex.pattern, node.regex.flags, { + unicodePropertyEscapes: 'transform', + }); + } catch { + // swallow regex parse errors here to instead throw them at the engine level + // this then also avoids regex parser bugs being thrown unnecessarily + transpiledPattern = node.regex.pattern; + } + const transpiledRegex = `/${transpiledPattern}/${node.regex.flags}`; + precompileCalls.push(`precompile(${transpiledRegex});`); + magicString.overwrite(node.start, node.end, transpiledRegex); + }, + }); + + if (!precompileCalls.length) return source; + + magicString.prepend(`${PREAMBLE}${precompileCalls.join('\n')}${POSTAMBLE}`); + + // When we're ready to pipe in source maps: + // const map = magicString.generateMap({ + // source: 'source.js', + // file: 'converted.js.map', + // includeContent: true + // }); + + return magicString.toString(); +} \ No newline at end of file diff --git a/test/test-app/package-lock.json b/test/test-app/package-lock.json index 80774be7..7b780fef 100644 --- a/test/test-app/package-lock.json +++ b/test/test-app/package-lock.json @@ -25,11 +25,15 @@ }, "../../packages/build-tools": { "name": "@spinframework/build-tools", - "version": "1.0.0", - "license": "ISC", + "version": "1.0.2", + "license": "Apache-2.0 WITH LLVM-exception", "dependencies": { - "@bytecodealliance/componentize-js": "^0.18.0", + "@bytecodealliance/componentize-js": "^0.18.1", "@bytecodealliance/jco": "^1.10.2", + "acorn-walk": "^8.3.4", + "acron": "^1.0.5", + "magic-string": "^0.30.17", + "regexpu-core": "^6.2.0", "yargs": "^17.7.2" }, "bin": { @@ -52,7 +56,7 @@ "../../packages/http-trigger": { "name": "@spinframework/wasi-http-proxy", "version": "1.0.0", - "license": "Apache-2.0", + "license": " Apache-2.0 WITH LLVM-exception", "devDependencies": { "typescript": "^5.7.3" } @@ -60,7 +64,7 @@ "../../packages/spin-kv": { "name": "@spinframework/spin-kv", "version": "1.0.0", - "license": "Apache-2.0", + "license": " Apache-2.0 WITH LLVM-exception", "devDependencies": { "typescript": "^5.7.3" } @@ -77,7 +81,7 @@ "../../packages/spin-variables": { "name": "@spinframework/spin-variables", "version": "1.0.0", - "license": "Apache-2.0", + "license": " Apache-2.0 WITH LLVM-exception", "devDependencies": { "typescript": "^5.7.3" } diff --git a/test/test.sh b/test/test.sh index 2fede960..9709f122 100755 --- a/test/test.sh +++ b/test/test.sh @@ -23,7 +23,6 @@ echo "built the test app successfully" # Start the spin app in the background echo "Starting Spin app" spin up & -SPIN_PID=$! # wait for app to be up and running echo "Waiting for Spin app to be ready" @@ -36,7 +35,7 @@ echo "\n\nTest completed" # kill the spin app echo "Stopping Spin" -kill -9 $SPIN_PID +killall spin if [ "$isFailed" = true ] ; then From 7ce6e828be57c386c0297d3e43b50c11c90b0a44 Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Wed, 23 Apr 2025 09:16:08 +0200 Subject: [PATCH 2/3] fix typo Signed-off-by: Karthik Ganeshram --- packages/build-tools/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index c7d9c8a3..196d1352 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -65,7 +65,7 @@ async function main() { const source = await readFile(src, 'utf8'); const precompiledSource = precompile(source, src, true) as string; - // Using the old syntax because the new sytnax does not allow passing in the precompiled string + // Using the old syntax because the new syntax does not allow passing in the precompiled string //@ts-ignore const { component } = await componentize(precompiledSource, inlineWit, { runtimeArgs, From dc5c773bf9b30baf7e7a95f6b904ea8bb14724dd Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Wed, 23 Apr 2025 09:21:21 +0200 Subject: [PATCH 3/3] write out precompiled source Signed-off-by: Karthik Ganeshram --- packages/build-tools/src/index.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index 196d1352..08cc9e46 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -13,6 +13,7 @@ import { readFile, writeFile } from 'node:fs/promises'; import { mergeWit } from '../lib/wit_tools.js'; //@ts-ignore import { precompile } from "./precompile.js" +import path from 'node:path' async function main() { try { @@ -65,9 +66,16 @@ async function main() { const source = await readFile(src, 'utf8'); const precompiledSource = precompile(source, src, true) as string; - // Using the old syntax because the new syntax does not allow passing in the precompiled string - //@ts-ignore - const { component } = await componentize(precompiledSource, inlineWit, { + // Write precompiled source to disk for debugging purposes In the future we + // will also write a source map to make debugging easier + let srcDir = path.dirname(src); + let precompiledSourcePath = path.join(srcDir, 'precompiled-source.js'); + await writeFile(precompiledSourcePath, precompiledSource); + + const { component } = await componentize({ + sourcePath: precompiledSourcePath, + // @ts-ignore + witWorld: inlineWit, runtimeArgs, enableAot: CliArgs.aot, });