Skip to content

Commit 584ef4d

Browse files
authored
New Workload: TypeScript standalone (#117)
Add an in-memory typescript compilation workload. - npm run build downloads and build src/gen in-memory file contents for 3 different sized libs (for local testing) - workload file data and tsconfig json files are parsed outside the measured time - webpack is used to bundle the results - Use the UnicodeEscapePlugin to avoid accidental 2-byte input strings
1 parent 7b374ea commit 584ef4d

18 files changed

+3342
-15
lines changed

JetStreamDriver.js

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ function getWorstCaseCount(plan) {
7373
return JetStreamParams.testWorstCaseCountMap.get(plan.name);
7474
if (JetStreamParams.testWorstCaseCount)
7575
return JetStreamParams.testWorstCaseCount;
76-
if (plan.worstCaseCount)
76+
if (plan.worstCaseCount !== undefined)
7777
return plan.worstCaseCount;
7878
return defaultWorstCaseCount;
7979
}
@@ -1180,8 +1180,9 @@ class DefaultBenchmark extends Benchmark {
11801180
this.worstScore = null;
11811181
this.averageTime = null;
11821182
this.averageScore = null;
1183-
1184-
console.assert(this.iterations > this.worstCaseCount);
1183+
if (this.worstCaseCount)
1184+
console.assert(this.iterations > this.worstCaseCount);
1185+
console.assert(this.worstCaseCount >= 0);
11851186
}
11861187

11871188
processResults(results) {
@@ -1195,21 +1196,24 @@ class DefaultBenchmark extends Benchmark {
11951196
for (let i = 0; i + 1 < results.length; ++i)
11961197
console.assert(results[i] >= results[i + 1]);
11971198

1198-
const worstCase = [];
1199-
for (let i = 0; i < this.worstCaseCount; ++i)
1200-
worstCase.push(results[i]);
1201-
this.worstTime = mean(worstCase);
1202-
this.worstScore = toScore(this.worstTime);
1199+
if (this.worstCaseCount) {
1200+
const worstCase = [];
1201+
for (let i = 0; i < this.worstCaseCount; ++i)
1202+
worstCase.push(results[i]);
1203+
this.worstTime = mean(worstCase);
1204+
this.worstScore = toScore(this.worstTime);
1205+
}
12031206
this.averageTime = mean(results);
12041207
this.averageScore = toScore(this.averageTime);
12051208
}
12061209

12071210
subScores() {
1208-
return {
1209-
"First": this.firstIterationScore,
1210-
"Worst": this.worstScore,
1211-
"Average": this.averageScore,
1212-
};
1211+
const scores = { "First": this.firstIterationScore }
1212+
if (this.worstCaseCount)
1213+
scores["Worst"] = this.worstScore;
1214+
if (this.iterations > 1)
1215+
scores["Average"] = this.averageScore;
1216+
return scores;
12131217
}
12141218
}
12151219

@@ -1757,7 +1761,7 @@ let BENCHMARKS = [
17571761
tags: ["Default", "Octane"],
17581762
}),
17591763
new DefaultBenchmark({
1760-
name: "typescript",
1764+
name: "typescript-octane",
17611765
files: [
17621766
"./Octane/typescript-compiler.js",
17631767
"./Octane/typescript-input.js",
@@ -1766,7 +1770,7 @@ let BENCHMARKS = [
17661770
iterations: 15,
17671771
worstCaseCount: 2,
17681772
deterministicRandom: true,
1769-
tags: ["Default", "Octane"],
1773+
tags: ["Octane", "typescript"],
17701774
}),
17711775
// RexBench
17721776
new DefaultBenchmark({
@@ -1996,6 +2000,24 @@ let BENCHMARKS = [
19962000
],
19972001
tags: ["Default", "ClassFields"],
19982002
}),
2003+
new AsyncBenchmark({
2004+
name: "typescript-lib",
2005+
files: [
2006+
"./TypeScript/src/mock/sys.js",
2007+
"./TypeScript/dist/bundle.js",
2008+
"./TypeScript/benchmark.js",
2009+
],
2010+
preload: {
2011+
// Large test project:
2012+
// "tsconfig": "./TypeScript/src/gen/zod-medium/tsconfig.json",
2013+
// "files": "./TypeScript/src/gen/zod-medium/files.json",
2014+
"tsconfig": "./TypeScript/src/gen/immer-tiny/tsconfig.json",
2015+
"files": "./TypeScript/src/gen/immer-tiny/files.json",
2016+
},
2017+
iterations: 1,
2018+
worstCaseCount: 0,
2019+
tags: ["Default", "typescript"],
2020+
}),
19992021
// Generators
20002022
new AsyncBenchmark({
20012023
name: "async-fs",

TypeScript/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules
2+
build/*.git

TypeScript/README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# TypeSCript test for JetStream
2+
3+
Measures the performance of running typescript on several framwork sources.
4+
5+
The build steps bundles sources from different frameworks:
6+
- [jestjs](https://github.com/jestjs/jest.git)
7+
- [zod](https://github.com/colinhacks/zod.git)
8+
- [immer](https://github.com/immerjs/immer.git)
9+
10+
## Build Instructions
11+
12+
```bash
13+
# install required node packages.
14+
npm ci
15+
# build the workload, output is ./dist
16+
npm run build
17+
```

TypeScript/benchmark.js

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright (C) 2025 Apple Inc. All rights reserved.
3+
*
4+
* Redistribution and use in source and binary forms, with or without
5+
* modification, are permitted provided that the following conditions
6+
* are met:
7+
* 1. Redistributions of source code must retain the above copyright
8+
* notice, this list of conditions and the following disclaimer.
9+
* 2. Redistributions in binary form must reproduce the above copyright
10+
* notice, this list of conditions and the following disclaimer in the
11+
* documentation and/or other materials provided with the distribution.
12+
*
13+
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14+
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16+
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17+
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18+
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19+
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20+
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21+
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23+
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24+
*/
25+
26+
// Prevent typescript logging output.
27+
console.log = () => {};
28+
29+
class Benchmark {
30+
tsConfig;
31+
fileData;
32+
33+
async init() {
34+
this.tsConfig = JSON.parse(await JetStream.getString(JetStream.preload.tsconfig));
35+
this.fileData = JSON.parse(await JetStream.getString(JetStream.preload.files));
36+
}
37+
38+
runIteration() {
39+
TypeScriptCompileTest.compileTest(this.tsConfig, this.fileData);
40+
}
41+
}
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import fs from "fs";
2+
import path from "path";
3+
import { spawnSync } from "child_process";
4+
import { globSync } from "glob";
5+
import assert from 'assert/strict';
6+
import { fileURLToPath } from "url";
7+
8+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
9+
10+
11+
class Importer {
12+
constructor({ projectName, size, repoUrl, srcFolder, extraFiles, extraDirs }) {
13+
this.projectName = projectName;
14+
assert(projectName.endsWith(`-${size}`), "missing size annotation in projectName");
15+
this.repoUrl = repoUrl;
16+
this.baseDir = path.resolve(__dirname);
17+
let repoName = path.basename(this.repoUrl);
18+
if (!repoName.endsWith(".git")) {
19+
repoName = `${repoName}.git`;
20+
}
21+
this.repoDir = path.resolve(__dirname, repoName);
22+
this.srcFolder = srcFolder;
23+
this.outputDir = path.resolve(__dirname, `../src/gen/${this.projectName}`);
24+
fs.mkdirSync(this.outputDir, { recursive: true });
25+
this.srcFileData = Object.create(null);
26+
this.extraFiles = extraFiles;
27+
this.extraDirs = extraDirs;
28+
}
29+
run() {
30+
this.cloneRepo();
31+
this.readSrcFileData();
32+
this.addExtraFilesFromDirs();
33+
this.addSpecificFiles();
34+
this.writeTsConfig();
35+
this.writeSrcFileData();
36+
}
37+
38+
cloneRepo() {
39+
if (fs.existsSync(this.repoDir)) return;
40+
console.info(`Cloning src data repository to ${this.repoDir}`);
41+
spawnSync("git", ["clone", this.repoUrl, this.repoDir]);
42+
}
43+
44+
readSrcFileData() {
45+
const patterns = [`${this.srcFolder}/**/*.ts`, `${this.srcFolder}/**/*.d.ts`, `${this.srcFolder}/*.d.ts`];
46+
patterns.forEach(pattern => {
47+
const files = globSync(pattern, { cwd: this.repoDir, nodir: true });
48+
files.forEach(file => {
49+
const filePath = path.join(this.repoDir, file);
50+
const relativePath = path.relative(this.repoDir, filePath).toLowerCase();
51+
const fileContents = fs.readFileSync(filePath, "utf8");
52+
this.addFileContents(relativePath, fileContents);
53+
});
54+
});
55+
}
56+
57+
addExtraFilesFromDirs() {
58+
this.extraDirs.forEach(({ dir, nameOnly = false }) => {
59+
const absoluteSourceDir = path.resolve(__dirname, dir);
60+
let allFiles = globSync("**/*.d.ts", { cwd: absoluteSourceDir, nodir: true });
61+
allFiles = allFiles.concat(globSync("**/*.d.mts", { cwd: absoluteSourceDir, nodir: true }));
62+
63+
allFiles.forEach(file => {
64+
const filePath = path.join(absoluteSourceDir, file);
65+
let relativePath = path.join(dir, path.relative(absoluteSourceDir, filePath));
66+
if (nameOnly) {
67+
relativePath = path.basename(relativePath);
68+
}
69+
this.addFileContents(relativePath, fs.readFileSync(filePath, "utf8"))
70+
});
71+
});
72+
}
73+
74+
addFileContents(relativePath, fileContents) {
75+
if (relativePath in this.srcFileData) {
76+
if (this.srcFileData[relativePath] !== fileContents) {
77+
throw new Error(`${relativePath} was previously added with different contents.`);
78+
}
79+
} else {
80+
this.srcFileData[relativePath] = fileContents;
81+
}
82+
}
83+
84+
addSpecificFiles() {
85+
this.extraFiles.forEach(file => {
86+
const filePath = path.join(this.baseDir, file);
87+
this.srcFileData[file] = fs.readFileSync(filePath, "utf8");
88+
});
89+
}
90+
91+
writeSrcFileData() {
92+
const filesDataPath = path.join(this.outputDir, "files.json");
93+
fs.writeFileSync(
94+
filesDataPath,
95+
JSON.stringify(this.srcFileData, null, 2)
96+
);
97+
const stats = fs.statSync(filesDataPath);
98+
const fileSizeInKiB = (stats.size / 1024) | 0;
99+
console.info(`Exported ${this.projectName}`);
100+
console.info(` File Contents: ${path.relative(process.cwd(), filesDataPath)}`);
101+
console.info(` Total Size: ${fileSizeInKiB} KiB`);
102+
}
103+
104+
writeTsConfig() {
105+
const tsconfigInputPath = path.join(this.repoDir, "tsconfig.json");
106+
const mergedTsconfig = this.loadAndMergeTsconfig(tsconfigInputPath);
107+
const tsconfigOutputPath = path.join(this.outputDir, "tsconfig.json");
108+
fs.writeFileSync(
109+
tsconfigOutputPath,
110+
JSON.stringify(mergedTsconfig, null, 2)
111+
);
112+
}
113+
114+
loadAndMergeTsconfig(configPath) {
115+
const tsconfigContent = fs.readFileSync(configPath, "utf8");
116+
const tsconfigContentWithoutComments = tsconfigContent.replace(/(?:^|\s)\/\/.*$|\/\*[\s\S]*?\*\//gm, "");
117+
const tsconfig = JSON.parse(tsconfigContentWithoutComments);
118+
let baseConfigPath = tsconfig.extends;
119+
if (!baseConfigPath) return tsconfig;
120+
if (!baseConfigPath.startsWith('./') && !baseConfigPath.startsWith('../')) return tsconfig;
121+
122+
baseConfigPath = path.resolve(path.dirname(configPath), baseConfigPath);
123+
const baseConfig = this.loadAndMergeTsconfig(baseConfigPath);
124+
125+
const mergedConfig = { ...baseConfig, ...tsconfig };
126+
if (baseConfig.compilerOptions && tsconfig.compilerOptions) {
127+
mergedConfig.compilerOptions = { ...baseConfig.compilerOptions, ...tsconfig.compilerOptions };
128+
}
129+
delete mergedConfig.extends;
130+
return mergedConfig;
131+
}
132+
}
133+
134+
const jest = new Importer({
135+
projectName: "jestjs-large",
136+
size: "large",
137+
repoUrl: "https://github.com/jestjs/jest.git",
138+
srcFolder: "packages",
139+
extraFiles: [
140+
"../../node_modules/@babel/types/lib/index.d.ts",
141+
"../../node_modules/callsites/index.d.ts",
142+
"../../node_modules/camelcase/index.d.ts",
143+
"../../node_modules/chalk/types/index.d.ts",
144+
"../../node_modules/execa/index.d.ts",
145+
"../../node_modules/fast-json-stable-stringify/index.d.ts",
146+
"../../node_modules/get-stream/index.d.ts",
147+
"../../node_modules/strip-json-comments/index.d.ts",
148+
"../../node_modules/tempy/index.d.ts",
149+
"../../node_modules/tempy/node_modules/type-fest/index.d.ts",
150+
"../node_modules/@jridgewell/trace-mapping/types/trace-mapping.d.mts",
151+
"../node_modules/@types/eslint/index.d.ts",
152+
"../node_modules/ansi-regex/index.d.ts",
153+
"../node_modules/ansi-styles/index.d.ts",
154+
"../node_modules/glob/dist/esm/index.d.ts",
155+
"../node_modules/jest-worker/build/index.d.ts",
156+
"../node_modules/lru-cache/dist/esm/index.d.ts",
157+
"../node_modules/minipass/dist/esm/index.d.ts",
158+
"../node_modules/p-limit/index.d.ts",
159+
"../node_modules/path-scurry/dist/esm/index.d.ts",
160+
"../node_modules/typescript/lib/lib.dom.d.ts",
161+
],
162+
extraDirs: [
163+
{ dir: "../node_modules/@types/" },
164+
{ dir: "../node_modules/typescript/lib/", nameOnly: true },
165+
{ dir: "../node_modules/jest-worker/build/" },
166+
{ dir: "../node_modules/@jridgewell/trace-mapping/types/" },
167+
{ dir: "../node_modules/minimatch/dist/esm/" },
168+
{ dir: "../node_modules/glob/dist/esm/" },
169+
{ dir: "../../node_modules/tempy/node_modules/type-fest/source/" }
170+
],
171+
});
172+
173+
const zod = new Importer({
174+
projectName: "zod-medium",
175+
size: "medium",
176+
repoUrl: "https://github.com/colinhacks/zod.git",
177+
srcFolder: "packages",
178+
extraFiles: [],
179+
extraDirs: [
180+
{ dir: "../node_modules/typescript/lib/", nameOnly: true },
181+
],
182+
});
183+
184+
const immer =new Importer({
185+
projectName: "immer-tiny",
186+
size: "tiny",
187+
repoUrl: "https://github.com/immerjs/immer.git",
188+
srcFolder: "src",
189+
extraFiles: [],
190+
extraDirs: [
191+
{ dir: "../node_modules/typescript/lib/", nameOnly: true },
192+
],
193+
});
194+
195+
// Skip jest since it produces a hugh in-memory FS.
196+
// jest.run();
197+
zod.run();
198+
immer.run();

TypeScript/dist/bundle.js

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

TypeScript/dist/bundle.js.map

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)