From 31b3a997d8980d230c56c073ca381d40f025ea16 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 1 Nov 2024 16:02:10 +0000 Subject: [PATCH 01/10] implement example apps benchmarking --- benchmarking/.gitignore | 1 + benchmarking/README.md | 7 ++ benchmarking/package.json | 14 +++ benchmarking/src/benchmarking.ts | 110 ++++++++++++++++++++ benchmarking/src/cloudflare.ts | 95 ++++++++++++++++++ benchmarking/src/index.ts | 41 ++++++++ benchmarking/src/utils.ts | 39 ++++++++ benchmarking/tsconfig.json | 15 +++ package.json | 3 +- pnpm-lock.yaml | 166 +++++++++++++++++++++++++++---- pnpm-workspace.yaml | 2 + 11 files changed, 474 insertions(+), 19 deletions(-) create mode 100644 benchmarking/.gitignore create mode 100644 benchmarking/README.md create mode 100644 benchmarking/package.json create mode 100644 benchmarking/src/benchmarking.ts create mode 100644 benchmarking/src/cloudflare.ts create mode 100644 benchmarking/src/index.ts create mode 100644 benchmarking/src/utils.ts create mode 100644 benchmarking/tsconfig.json diff --git a/benchmarking/.gitignore b/benchmarking/.gitignore new file mode 100644 index 00000000..fbca2253 --- /dev/null +++ b/benchmarking/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/benchmarking/README.md b/benchmarking/README.md new file mode 100644 index 00000000..638c0537 --- /dev/null +++ b/benchmarking/README.md @@ -0,0 +1,7 @@ +# Benchmarking + +This directory contains a script for running full end to end benchmarks again the example applications + +> [!note] +> This is the first cut at benchmarking our solution, later we can take the script in this directory, +> generalize it and make it more reusable if we want diff --git a/benchmarking/package.json b/benchmarking/package.json new file mode 100644 index 00000000..2563e5b2 --- /dev/null +++ b/benchmarking/package.json @@ -0,0 +1,14 @@ +{ + "name": "@opennextjs-cloudflare/benchmarking", + "private": true, + "type": "module", + "devDependencies": { + "tsx": "catalog:", + "@tsconfig/strictest": "catalog:", + "@types/node": "catalog:", + "ora": "^8.1.0" + }, + "scripts": { + "benchmark": "tsx src/index.ts" + } +} diff --git a/benchmarking/src/benchmarking.ts b/benchmarking/src/benchmarking.ts new file mode 100644 index 00000000..2568d58b --- /dev/null +++ b/benchmarking/src/benchmarking.ts @@ -0,0 +1,110 @@ +import nodeTimesPromises from "node:timers/promises"; +import nodeFsPromises from "node:fs/promises"; +import nodePath from "node:path"; + +export type FetchBenchmark = { + calls: number[]; + average: number; +}; + +export type BenchmarkingResults = { + name: string; + path: string; + fetchBenchmark: FetchBenchmark; +}[]; + +/** + * Benchmarks the response time of an application end-to-end by: + * - building the application + * - deploying it + * - and fetching from it (multiple times) + * + * @param options.build function implementing how the application is to be built + * @param options.deploy function implementing how the application is deployed (returning the url of the deployment) + * @param options.fetch function indicating how to fetch from the application (in case a specific route needs to be hit, cookies need to be applied, etc...) + * @returns the benchmarking results for the application + */ +export async function benchmarkApplicationResponseTime({ + build, + deploy, + fetch, +}: { + build: () => Promise; + deploy: () => Promise; + fetch: (deploymentUrl: string) => Promise; +}): Promise { + await build(); + const deploymentUrl = await deploy(); + return benchmarkFetch(deploymentUrl, { fetch }); +} + +type BenchmarkFetchOptions = { + numberOfCalls?: number; + randomDelayMax?: number; + fetch: (deploymentUrl: string) => Promise; +}; + +const defaultOptions: Required> = { + numberOfCalls: 20, + randomDelayMax: 15_000, +}; + +/** + * Benchmarks a fetch operation by running it multiple times and computing the average time (in milliseconds) such fetch operation takes. + * + * @param url The url to fetch from + * @param options options for the benchmarking + * @returns the computed average alongside all the single call times + */ +async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Promise { + const benchmarkFetchCall = async () => { + const preTime = performance.now(); + const resp = await options.fetch(url); + const postTime = performance.now(); + + if (!resp.ok) { + throw new Error(`Error: Failed to fetch from "${url}"`); + } + + return postTime - preTime; + }; + + const calls = await Promise.all( + new Array(options?.numberOfCalls ?? defaultOptions.numberOfCalls).fill(null).map(async () => { + // let's add a random delay before we make the fetch + await nodeTimesPromises.setTimeout( + Math.round(Math.random() * (options?.randomDelayMax ?? defaultOptions.randomDelayMax)) + ); + + return benchmarkFetchCall(); + }) + ); + + const average = calls.reduce((time, sum) => sum + time) / calls.length; + + return { + calls, + average, + }; +} + +/** + * Saves benchmarking results in a local json file + * + * @param results the benchmarking results to save + * @returns the path to the created json file + */ +export async function saveResultsToDisk(results: BenchmarkingResults): Promise { + const date = new Date(); + + const fileName = `${date.toISOString().split(".")[0]!.replace("T", "_").replaceAll(":", "-")}.json`; + + const outputFile = nodePath.resolve(`./results/${fileName}`); + + await nodeFsPromises.mkdir(nodePath.dirname(outputFile), { recursive: true }); + + const resultStr = JSON.stringify(results, null, 2); + await nodeFsPromises.writeFile(outputFile, resultStr); + + return outputFile; +} diff --git a/benchmarking/src/cloudflare.ts b/benchmarking/src/cloudflare.ts new file mode 100644 index 00000000..1a0e559f --- /dev/null +++ b/benchmarking/src/cloudflare.ts @@ -0,0 +1,95 @@ +import nodeFsPromises from "node:fs/promises"; +import nodeFs from "node:fs"; +import nodePath from "node:path"; +import nodeChildProcess from "node:child_process"; +import nodeUtil from "node:util"; + +const promiseExec = nodeUtil.promisify(nodeChildProcess.exec); + +await ensureWranglerSetup(); + +/** + * Collects name and absolute paths of apps (in this repository) that we want to benchmark + * + * @returns Array of objects containing the app's name and absolute path + */ +export async function collectAppPathsToBenchmark(): Promise< + { + name: string; + path: string; + }[] +> { + const allExampleNames = await nodeFsPromises.readdir("../examples"); + + const examplesToIgnore = new Set(["vercel-commerce"]); + + const examplePaths = allExampleNames + .filter((exampleName) => !examplesToIgnore.has(exampleName)) + .map((exampleName) => ({ + name: exampleName, + path: nodePath.resolve(`../examples/${exampleName}`), + })); + + return examplePaths; +} + +/** + * Builds an application using their "build:worker" script + * (an error is thrown if the application doesn't have such a script) + * + * @param dir Path to the application to build + */ +export async function buildApp(dir: string): Promise { + const packageJsonPath = `${dir}/package.json`; + if (!nodeFs.existsSync(packageJsonPath)) { + throw new Error(`Error: package.json for app at "${dir}" not found`); + } + + const packageJsonContent = JSON.parse(await nodeFsPromises.readFile(packageJsonPath, "utf8")); + + if (!("scripts" in packageJsonContent) || !("build:worker" in packageJsonContent.scripts)) { + throw new Error(`Error: package.json for app at "${dir}" does not include a "build:worker" script`); + } + + const command = "pnpm build:worker"; + + await promiseExec(command, { cwd: dir }); +} + +/** + * Deploys a built application using wrangler + * + * @param dir Path to the application to build + * @returns the url of the deployed application + */ +export async function deployBuiltApp(dir: string): Promise { + const { stdout } = await promiseExec("pnpm exec wrangler deploy", { cwd: dir }); + + const deploymentUrl = stdout.match(/\bhttps:\/\/(?:[a-zA-Z0-9.\-])*\.workers\.dev\b/)?.[0]; + + if (!deploymentUrl) { + throw new Error(`Could not obtain a deployment url for app at "${dir}"`); + } + + return deploymentUrl; +} + +/** + * Makes sure that everything is set up so that wrangler can actually deploy the applications. + * This means that: + * - the user has logged in + * - if they have more than one account they have set a CLOUDFLARE_ACCOUNT_ID env variable + */ +async function ensureWranglerSetup(): Promise { + const { stdout } = await promiseExec("pnpm dlx wrangler whoami"); + + if (stdout.includes("You are not authenticated")) { + throw new Error("Please log in using wrangler by running `pnpm dlx wrangler login`"); + } + + if (!(process.env as Record)["CLOUDFLARE_ACCOUNT_ID"]) { + throw new Error( + "Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications" + ); + } +} diff --git a/benchmarking/src/index.ts b/benchmarking/src/index.ts new file mode 100644 index 00000000..cfd0453f --- /dev/null +++ b/benchmarking/src/index.ts @@ -0,0 +1,41 @@ +import nodeTimesPromises from "node:timers/promises"; +import * as cloudflare from "./cloudflare"; +import { benchmarkApplicationResponseTime, BenchmarkingResults, saveResultsToDisk } from "./benchmarking"; +import { runOperationsWithSpinner } from "./utils"; + +const appPathsToBenchmark = await cloudflare.collectAppPathsToBenchmark(); + +const benchmarkingResults: BenchmarkingResults = await runOperationsWithSpinner( + "Benchmarking Apps", + appPathsToBenchmark.map(({ name, path }, i) => async () => { + await nodeTimesPromises.setTimeout(i * 1_000); + const fetchBenchmark = await benchmarkApplicationResponseTime({ + build: async () => cloudflare.buildApp(path), + deploy: async () => cloudflare.deployBuiltApp(path), + fetch, + }); + + return { + name, + path, + fetchBenchmark, + }; + }) +); + +console.log(); + +const outputFile = await saveResultsToDisk(benchmarkingResults); + +console.log(`The benchmarking results have been written in ${outputFile}`); + +console.log("\n\nSummary: "); +const summary = benchmarkingResults.map(({ name, fetchBenchmark }) => ({ + name, + "average fetch duration (ms)": Math.round(fetchBenchmark.average), +})); +console.table(summary); + +console.log(); + +process.exit(0); diff --git a/benchmarking/src/utils.ts b/benchmarking/src/utils.ts new file mode 100644 index 00000000..4d5a1ca7 --- /dev/null +++ b/benchmarking/src/utils.ts @@ -0,0 +1,39 @@ +import ora from "ora"; + +/** + * Runs a number of operations while presenting a loading spinner with some text + * + * @param spinnerText The text to add to the spinner + * @param operations The operations to run + * @returns The operations results + */ +export async function runOperationsWithSpinner( + spinnerText: string, + operations: (() => Promise)[] +): Promise { + const spinner = ora({ + discardStdin: false, + hideCursor: false, + }).start(); + + let doneCount = 0; + + const updateSpinnerText = () => { + doneCount++; + spinner.text = `${spinnerText} (${doneCount}/${operations.length})`; + }; + + updateSpinnerText(); + + const results = await Promise.all( + operations.map(async (operation) => { + const result = await operation(); + updateSpinnerText(); + return result; + }) + ); + + spinner.stop(); + + return results; +} diff --git a/benchmarking/tsconfig.json b/benchmarking/tsconfig.json new file mode 100644 index 00000000..756fa557 --- /dev/null +++ b/benchmarking/tsconfig.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@tsconfig/strictest/tsconfig.json", + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "lib": ["ESNext"], + "types": ["node"], + "moduleResolution": "Bundler", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": false, + "exactOptionalPropertyTypes": false + }, + "include": ["./src/**/*.ts"] +} diff --git a/package.json b/package.json index e107e974..759db9d8 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "postinstall": "pnpm --filter cloudflare build", "build": "pnpm --filter cloudflare build", "e2e": "pnpm build && pnpm -r e2e", - "e2e:dev": "pnpm build && pnpm -r e2e:dev" + "e2e:dev": "pnpm build && pnpm -r e2e:dev", + "benchmark": "pnpm run --filter benchmarking benchmark" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 935558f6..2e379814 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,6 +60,9 @@ catalogs: tsup: specifier: ^8.2.4 version: 8.2.4 + tsx: + specifier: ^4.19.2 + version: 4.19.2 typescript: specifier: ^5.5.4 version: 5.5.4 @@ -90,6 +93,21 @@ importers: specifier: 3.3.3 version: 3.3.3 + benchmarking: + devDependencies: + '@tsconfig/strictest': + specifier: 'catalog:' + version: 2.0.5 + '@types/node': + specifier: 'catalog:' + version: 22.2.0 + ora: + specifier: ^8.1.0 + version: 8.1.0 + tsx: + specifier: 'catalog:' + version: 4.19.2 + examples/api: dependencies: next: @@ -333,7 +351,7 @@ importers: version: 0.2.0 tsup: specifier: 'catalog:' - version: 8.2.4(jiti@1.21.6)(postcss@8.4.47)(tsx@4.17.0)(typescript@5.5.4)(yaml@2.5.1) + version: 8.2.4(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.5.4)(yaml@2.5.1) typescript: specifier: 'catalog:' version: 5.5.4 @@ -1873,6 +1891,10 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + chalk@5.3.0: + resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -1908,6 +1930,14 @@ packages: resolution: {integrity: sha512-GfisEZEJvzKrmGWkvfhgzcz/BllN1USeqD2V6tg14OAOgaCD2Z/PUEuxnAZ/nPvmaHRG7a8y77p1T/IRQ4D1Hw==} engines: {node: '>=4'} + cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} + + cli-spinners@2.9.2: + resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} + engines: {node: '>=6'} + client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} @@ -2089,6 +2119,9 @@ packages: electron-to-chromium@1.5.29: resolution: {integrity: sha512-PF8n2AlIhCKXQ+gTpiJi0VhcHDb69kYX4MtCiivctc2QD3XuNZ/XIOlbGzt7WAjjEev0TtaH6Cu3arZExm5DOw==} + emoji-regex@10.4.0: + resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} @@ -2477,6 +2510,10 @@ packages: peerDependencies: next: '>=13.2.0' + get-east-asian-width@1.3.0: + resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==} + engines: {node: '>=18'} + get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} @@ -2718,6 +2755,10 @@ packages: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} + is-interactive@2.0.0: + resolution: {integrity: sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==} + engines: {node: '>=12'} + is-map@2.0.3: resolution: {integrity: sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==} engines: {node: '>= 0.4'} @@ -2774,6 +2815,14 @@ packages: resolution: {integrity: sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==} engines: {node: '>= 0.4'} + is-unicode-supported@1.3.0: + resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} + engines: {node: '>=12'} + + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + is-weakmap@2.0.2: resolution: {integrity: sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==} engines: {node: '>= 0.4'} @@ -2920,6 +2969,10 @@ packages: lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} + log-symbols@6.0.0: + resolution: {integrity: sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==} + engines: {node: '>=18'} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -3044,6 +3097,10 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} + mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -3217,10 +3274,18 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + ora@8.1.0: + resolution: {integrity: sha512-GQEkNkH/GHOhPFXcqZs3IDahXEQcQxsSjEkK4KvEEST4t7eNzoMjxTzef+EZ+JluDEV+Raoi3WQ2CflnRdSVnQ==} + engines: {node: '>=18'} + os-tmpdir@1.0.2: resolution: {integrity: sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==} engines: {node: '>=0.10.0'} @@ -3625,6 +3690,10 @@ packages: resolution: {integrity: sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==} hasBin: true + restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} + reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3802,6 +3871,10 @@ packages: std-env@3.7.0: resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==} + stdin-discarder@0.2.2: + resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} + engines: {node: '>=18'} + stop-iteration-iterator@1.0.0: resolution: {integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==} engines: {node: '>= 0.4'} @@ -3826,6 +3899,10 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + string.prototype.includes@2.0.0: resolution: {integrity: sha512-E34CkBgyeqNDcrbU76cDjL5JLcVrtSdYq0MEh/B10r17pRP4ciHLwTgnuLV8Ay6cgEMLkcBkFCKyFZ43YldYzg==} @@ -4032,8 +4109,8 @@ packages: typescript: optional: true - tsx@4.17.0: - resolution: {integrity: sha512-eN4mnDA5UMKDt4YZixo9tBioibaMBpoxBkD+rIPAjVmYERSG0/dWEY1CEFuV89CgASlKL499q8AhmkMnnjtOJg==} + tsx@4.19.2: + resolution: {integrity: sha512-pOUl6Vo2LUq/bSa8S5q7b91cgNSjctn9ugq/+Mvow99qW6x/UZYwzxy/3NmqoT66eHYfCVvFvACC58UBPFf28g==} engines: {node: '>=18.0.0'} hasBin: true @@ -5787,6 +5864,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + chalk@5.3.0: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -5819,6 +5898,12 @@ snapshots: dependencies: escape-string-regexp: 1.0.5 + cli-cursor@5.0.0: + dependencies: + restore-cursor: 5.1.0 + + cli-spinners@2.9.2: {} + client-only@0.0.1: {} clsx@2.1.1: {} @@ -5994,6 +6079,8 @@ snapshots: electron-to-chromium@1.5.29: {} + emoji-regex@10.4.0: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} @@ -6208,8 +6295,8 @@ snapshots: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.5.4) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1) - eslint-plugin-import: 2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1) + eslint-plugin-import: 2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) eslint-plugin-jsx-a11y: 6.10.0(eslint@8.57.1) eslint-plugin-react: 7.36.1(eslint@8.57.1) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.1) @@ -6228,33 +6315,33 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1): + eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.3.6 enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) fast-glob: 3.3.2 get-tsconfig: 4.8.0 is-bun-module: 1.2.1 is-glob: 4.0.3 optionalDependencies: - eslint-plugin-import: 2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-plugin-import: 2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) transitivePeerDependencies: - '@typescript-eslint/parser' - eslint-import-resolver-node - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.5.4) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -6268,7 +6355,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.8 @@ -6279,7 +6366,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.5.4))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.15.1 is-glob: 4.0.3 @@ -6661,6 +6748,8 @@ snapshots: dependencies: next: 15.0.0-canary.113(@playwright/test@1.47.0)(react-dom@19.0.0-rc-3208e73e-20240730(react@19.0.0-rc-3208e73e-20240730))(react@19.0.0-rc-3208e73e-20240730) + get-east-asian-width@1.3.0: {} + get-func-name@2.0.2: {} get-intrinsic@1.2.4: @@ -6924,6 +7013,8 @@ snapshots: dependencies: is-extglob: 2.1.1 + is-interactive@2.0.0: {} + is-map@2.0.3: {} is-negative-zero@2.0.3: {} @@ -6967,6 +7058,10 @@ snapshots: dependencies: which-typed-array: 1.1.15 + is-unicode-supported@1.3.0: {} + + is-unicode-supported@2.1.0: {} + is-weakmap@2.0.2: {} is-weakref@1.0.2: @@ -7097,6 +7192,11 @@ snapshots: lodash.startcase@4.4.0: {} + log-symbols@6.0.0: + dependencies: + chalk: 5.3.0 + is-unicode-supported: 1.3.0 + longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -7319,6 +7419,8 @@ snapshots: mimic-fn@2.1.0: {} + mimic-function@5.0.1: {} + min-indent@1.0.1: {} miniflare@3.20240925.0: @@ -7554,6 +7656,10 @@ snapshots: dependencies: mimic-fn: 2.1.0 + onetime@7.0.0: + dependencies: + mimic-function: 5.0.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -7563,6 +7669,18 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + ora@8.1.0: + dependencies: + chalk: 5.3.0 + cli-cursor: 5.0.0 + cli-spinners: 2.9.2 + is-interactive: 2.0.0 + is-unicode-supported: 2.1.0 + log-symbols: 6.0.0 + stdin-discarder: 0.2.2 + string-width: 7.2.0 + strip-ansi: 7.1.0 + os-tmpdir@1.0.2: {} outdent@0.5.0: {} @@ -7698,13 +7816,13 @@ snapshots: optionalDependencies: postcss: 8.4.31 - postcss-load-config@6.0.1(jiti@1.21.6)(postcss@8.4.47)(tsx@4.17.0)(yaml@2.5.1): + postcss-load-config@6.0.1(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(yaml@2.5.1): dependencies: lilconfig: 3.1.2 optionalDependencies: jiti: 1.21.6 postcss: 8.4.47 - tsx: 4.17.0 + tsx: 4.19.2 yaml: 2.5.1 postcss-nested@6.2.0(postcss@8.4.31): @@ -7903,6 +8021,11 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + restore-cursor@5.1.0: + dependencies: + onetime: 7.0.0 + signal-exit: 4.1.0 + reusify@1.0.4: {} rimraf@3.0.2: @@ -8118,6 +8241,8 @@ snapshots: std-env@3.7.0: {} + stdin-discarder@0.2.2: {} + stop-iteration-iterator@1.0.0: dependencies: internal-slot: 1.0.7 @@ -8140,6 +8265,12 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.1.0 + string-width@7.2.0: + dependencies: + emoji-regex: 10.4.0 + get-east-asian-width: 1.3.0 + strip-ansi: 7.1.0 + string.prototype.includes@2.0.0: dependencies: define-properties: 1.2.1 @@ -8348,7 +8479,7 @@ snapshots: tslib@2.6.3: {} - tsup@8.2.4(jiti@1.21.6)(postcss@8.4.47)(tsx@4.17.0)(typescript@5.5.4)(yaml@2.5.1): + tsup@8.2.4(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.5.4)(yaml@2.5.1): dependencies: bundle-require: 5.0.0(esbuild@0.23.1) cac: 6.7.14 @@ -8360,7 +8491,7 @@ snapshots: globby: 11.1.0 joycon: 3.1.1 picocolors: 1.1.0 - postcss-load-config: 6.0.1(jiti@1.21.6)(postcss@8.4.47)(tsx@4.17.0)(yaml@2.5.1) + postcss-load-config: 6.0.1(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(yaml@2.5.1) resolve-from: 5.0.0 rollup: 4.21.0 source-map: 0.8.0-beta.0 @@ -8375,13 +8506,12 @@ snapshots: - tsx - yaml - tsx@4.17.0: + tsx@4.19.2: dependencies: esbuild: 0.23.1 get-tsconfig: 4.8.0 optionalDependencies: fsevents: 2.3.3 - optional: true type-check@0.4.0: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c6e4e8d9..64e0443e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - "packages/*" - "examples/*" + - "benchmarking" catalog: "@cloudflare/workers-types": ^4.20240925.0 @@ -22,6 +23,7 @@ catalog: "ts-morph": ^23.0.0 "tsup": ^8.2.4 "typescript": ^5.5.4 + "tsx": ^4.19.2 "typescript-eslint": ^8.7.0 "vitest": ^2.1.1 "wrangler": ^3.78.10 From f54f54fdfbd1fdf8c762b330cb21d997c7c8919c Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 15:32:25 +0000 Subject: [PATCH 02/10] Update benchmarking/src/utils.ts Co-authored-by: Victor Berchet --- benchmarking/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarking/src/utils.ts b/benchmarking/src/utils.ts index 4d5a1ca7..726bddeb 100644 --- a/benchmarking/src/utils.ts +++ b/benchmarking/src/utils.ts @@ -1,7 +1,7 @@ import ora from "ora"; /** - * Runs a number of operations while presenting a loading spinner with some text + * Runs a list of operations while presenting a loading spinner with some text * * @param spinnerText The text to add to the spinner * @param operations The operations to run From e7d289e4310f8eef69139570351ab5f4c3e6d952 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 18:26:09 +0000 Subject: [PATCH 03/10] add ms suffix to all duration variables --- benchmarking/src/benchmarking.ts | 24 ++++++++++++------------ benchmarking/src/index.ts | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarking/src/benchmarking.ts b/benchmarking/src/benchmarking.ts index 2568d58b..ab247696 100644 --- a/benchmarking/src/benchmarking.ts +++ b/benchmarking/src/benchmarking.ts @@ -3,8 +3,8 @@ import nodeFsPromises from "node:fs/promises"; import nodePath from "node:path"; export type FetchBenchmark = { - calls: number[]; - average: number; + callDurationsMs: number[]; + averageMs: number; }; export type BenchmarkingResults = { @@ -40,13 +40,13 @@ export async function benchmarkApplicationResponseTime({ type BenchmarkFetchOptions = { numberOfCalls?: number; - randomDelayMax?: number; + maxRandomDelayMs?: number; fetch: (deploymentUrl: string) => Promise; }; const defaultOptions: Required> = { numberOfCalls: 20, - randomDelayMax: 15_000, + maxRandomDelayMs: 15_000, }; /** @@ -58,33 +58,33 @@ const defaultOptions: Required> = { */ async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Promise { const benchmarkFetchCall = async () => { - const preTime = performance.now(); + const preTimeMs = performance.now(); const resp = await options.fetch(url); - const postTime = performance.now(); + const postTimeMs = performance.now(); if (!resp.ok) { throw new Error(`Error: Failed to fetch from "${url}"`); } - return postTime - preTime; + return postTimeMs - preTimeMs; }; - const calls = await Promise.all( + const callDurationsMs = await Promise.all( new Array(options?.numberOfCalls ?? defaultOptions.numberOfCalls).fill(null).map(async () => { // let's add a random delay before we make the fetch await nodeTimesPromises.setTimeout( - Math.round(Math.random() * (options?.randomDelayMax ?? defaultOptions.randomDelayMax)) + Math.round(Math.random() * (options?.maxRandomDelayMs ?? defaultOptions.maxRandomDelayMs)) ); return benchmarkFetchCall(); }) ); - const average = calls.reduce((time, sum) => sum + time) / calls.length; + const averageMs = callDurationsMs.reduce((time, sum) => sum + time) / callDurationsMs.length; return { - calls, - average, + callDurationsMs, + averageMs, }; } diff --git a/benchmarking/src/index.ts b/benchmarking/src/index.ts index cfd0453f..a3ab9bd2 100644 --- a/benchmarking/src/index.ts +++ b/benchmarking/src/index.ts @@ -32,7 +32,7 @@ console.log(`The benchmarking results have been written in ${outputFile}`); console.log("\n\nSummary: "); const summary = benchmarkingResults.map(({ name, fetchBenchmark }) => ({ name, - "average fetch duration (ms)": Math.round(fetchBenchmark.average), + "average fetch duration (ms)": Math.round(fetchBenchmark.averageMs), })); console.table(summary); From 552fd68d0d2c21e30991e4845ad009052b622e2a Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 18:27:24 +0000 Subject: [PATCH 04/10] rename `runOperationsWithSpinner` to `parallelRunWithSpinner` --- benchmarking/src/index.ts | 4 ++-- benchmarking/src/utils.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarking/src/index.ts b/benchmarking/src/index.ts index a3ab9bd2..cdf78788 100644 --- a/benchmarking/src/index.ts +++ b/benchmarking/src/index.ts @@ -1,11 +1,11 @@ import nodeTimesPromises from "node:timers/promises"; import * as cloudflare from "./cloudflare"; import { benchmarkApplicationResponseTime, BenchmarkingResults, saveResultsToDisk } from "./benchmarking"; -import { runOperationsWithSpinner } from "./utils"; +import { parallelRunWithSpinner } from "./utils"; const appPathsToBenchmark = await cloudflare.collectAppPathsToBenchmark(); -const benchmarkingResults: BenchmarkingResults = await runOperationsWithSpinner( +const benchmarkingResults: BenchmarkingResults = await parallelRunWithSpinner( "Benchmarking Apps", appPathsToBenchmark.map(({ name, path }, i) => async () => { await nodeTimesPromises.setTimeout(i * 1_000); diff --git a/benchmarking/src/utils.ts b/benchmarking/src/utils.ts index 726bddeb..6fc2f45a 100644 --- a/benchmarking/src/utils.ts +++ b/benchmarking/src/utils.ts @@ -1,13 +1,13 @@ import ora from "ora"; /** - * Runs a list of operations while presenting a loading spinner with some text + * Runs a list of operations in parallel while presenting a loading spinner with some text * * @param spinnerText The text to add to the spinner * @param operations The operations to run * @returns The operations results */ -export async function runOperationsWithSpinner( +export async function parallelRunWithSpinner( spinnerText: string, operations: (() => Promise)[] ): Promise { From 8577e4c4b21ba0c625a626092858628200959674 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 19:36:13 +0000 Subject: [PATCH 05/10] add comment as to when we don't benchmark the `vercel-commerce` app --- benchmarking/src/cloudflare.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarking/src/cloudflare.ts b/benchmarking/src/cloudflare.ts index 1a0e559f..ba63bc2a 100644 --- a/benchmarking/src/cloudflare.ts +++ b/benchmarking/src/cloudflare.ts @@ -21,10 +21,16 @@ export async function collectAppPathsToBenchmark(): Promise< > { const allExampleNames = await nodeFsPromises.readdir("../examples"); - const examplesToIgnore = new Set(["vercel-commerce"]); + /** + * Example applications that we don't want to benchmark + * + * Currently we only want to skip the `vercel-commerce` example, and that's simply + * because it requires a shopify specific setup and secrets. + */ + const exampleAppsNotToBenchmark = new Set(["vercel-commerce"]); const examplePaths = allExampleNames - .filter((exampleName) => !examplesToIgnore.has(exampleName)) + .filter((exampleName) => !exampleAppsNotToBenchmark.has(exampleName)) .map((exampleName) => ({ name: exampleName, path: nodePath.resolve(`../examples/${exampleName}`), From 3087239630dbc1c822945fefc55e2e3d301d4f2b Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 19:49:43 +0000 Subject: [PATCH 06/10] expand benchmarking README --- benchmarking/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarking/README.md b/benchmarking/README.md index 638c0537..500f815a 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -1,6 +1,16 @@ # Benchmarking -This directory contains a script for running full end to end benchmarks again the example applications +This directory contains a script for running full end to end benchmarks against the example applications. + +What the script does: + +- takes all the example applications from the [`./examples` directory](../examples/) + (excluding the ones specified in the `exampleAppsNotToBenchmark` set in [`./src/cloudflare.ts`](./src/cloudflare.ts)) +- in parallel for each application: + - builds the application by running its `build:worker` script + - deploys the application to production (with `wrangler deploy`) + - takes the production deployment url + - benchmarks the application's response time by fetching from the deployment url a number of times > [!note] > This is the first cut at benchmarking our solution, later we can take the script in this directory, From 455e0978ae0d930bc0c655abf39993d36528e603 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 4 Nov 2024 20:03:12 +0000 Subject: [PATCH 07/10] add new `toSimpleDateString` function --- benchmarking/src/benchmarking.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/benchmarking/src/benchmarking.ts b/benchmarking/src/benchmarking.ts index ab247696..c9f38e70 100644 --- a/benchmarking/src/benchmarking.ts +++ b/benchmarking/src/benchmarking.ts @@ -97,7 +97,7 @@ async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Prom export async function saveResultsToDisk(results: BenchmarkingResults): Promise { const date = new Date(); - const fileName = `${date.toISOString().split(".")[0]!.replace("T", "_").replaceAll(":", "-")}.json`; + const fileName = `${toSimpleDateString(date)}.json`; const outputFile = nodePath.resolve(`./results/${fileName}`); @@ -108,3 +108,20 @@ export async function saveResultsToDisk(results: BenchmarkingResults): Promise Date: Tue, 5 Nov 2024 11:17:40 +0000 Subject: [PATCH 08/10] add p90ms measurement --- benchmarking/src/benchmarking.ts | 5 +++++ benchmarking/src/index.ts | 1 + benchmarking/src/utils.ts | 22 ++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/benchmarking/src/benchmarking.ts b/benchmarking/src/benchmarking.ts index c9f38e70..5bf01f83 100644 --- a/benchmarking/src/benchmarking.ts +++ b/benchmarking/src/benchmarking.ts @@ -1,10 +1,12 @@ import nodeTimesPromises from "node:timers/promises"; import nodeFsPromises from "node:fs/promises"; import nodePath from "node:path"; +import { getPercentile } from "./utils"; export type FetchBenchmark = { callDurationsMs: number[]; averageMs: number; + p90Ms: number; }; export type BenchmarkingResults = { @@ -82,9 +84,12 @@ async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Prom const averageMs = callDurationsMs.reduce((time, sum) => sum + time) / callDurationsMs.length; + const p90Ms = getPercentile(callDurationsMs, 90); + return { callDurationsMs, averageMs, + p90Ms, }; } diff --git a/benchmarking/src/index.ts b/benchmarking/src/index.ts index cdf78788..26148ef2 100644 --- a/benchmarking/src/index.ts +++ b/benchmarking/src/index.ts @@ -33,6 +33,7 @@ console.log("\n\nSummary: "); const summary = benchmarkingResults.map(({ name, fetchBenchmark }) => ({ name, "average fetch duration (ms)": Math.round(fetchBenchmark.averageMs), + "90th percentile (ms)": Math.round(fetchBenchmark.p90Ms), })); console.table(summary); diff --git a/benchmarking/src/utils.ts b/benchmarking/src/utils.ts index 6fc2f45a..543fd3c6 100644 --- a/benchmarking/src/utils.ts +++ b/benchmarking/src/utils.ts @@ -37,3 +37,25 @@ export async function parallelRunWithSpinner( return results; } + +/** + * Gets a specific percentile for a given set of numbers + * + * @param data the data which percentile value needs to be computed + * @param percentile the requested percentile (a number between 0 and 100) + * @returns the computed percentile + */ +export function getPercentile(data: number[], percentile: number): number { + if (Number.isNaN(percentile) || percentile < 0 || percentile > 100) { + throw new Error(`A percentile needs to be between 0 and 100, found: ${percentile}`); + } + + data = data.sort((a, b) => a - b); + + const rank = (percentile / 100) * (data.length - 1); + + const rankInt = Math.floor(rank); + const rankFract = rank - rankInt; + + return Math.round(data[rankInt]! + rankFract * (data[rankInt + 1]! - data[rankInt]!)); +} From 63940093bee5913d047529ee178931a29a6f06dd Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 5 Nov 2024 11:54:27 +0000 Subject: [PATCH 09/10] apply minor PR suggestions --- benchmarking/src/benchmarking.ts | 40 ++++++++++++++++---------------- benchmarking/src/cloudflare.ts | 8 ++++--- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/benchmarking/src/benchmarking.ts b/benchmarking/src/benchmarking.ts index 5bf01f83..1da6a756 100644 --- a/benchmarking/src/benchmarking.ts +++ b/benchmarking/src/benchmarking.ts @@ -4,7 +4,7 @@ import nodePath from "node:path"; import { getPercentile } from "./utils"; export type FetchBenchmark = { - callDurationsMs: number[]; + iterationsMs: number[]; averageMs: number; p90Ms: number; }; @@ -15,6 +15,17 @@ export type BenchmarkingResults = { fetchBenchmark: FetchBenchmark; }[]; +type BenchmarkFetchOptions = { + numberOfIterations?: number; + maxRandomDelayMs?: number; + fetch: (deploymentUrl: string) => Promise; +}; + +const defaultOptions: Required> = { + numberOfIterations: 20, + maxRandomDelayMs: 15_000, +}; + /** * Benchmarks the response time of an application end-to-end by: * - building the application @@ -40,17 +51,6 @@ export async function benchmarkApplicationResponseTime({ return benchmarkFetch(deploymentUrl, { fetch }); } -type BenchmarkFetchOptions = { - numberOfCalls?: number; - maxRandomDelayMs?: number; - fetch: (deploymentUrl: string) => Promise; -}; - -const defaultOptions: Required> = { - numberOfCalls: 20, - maxRandomDelayMs: 15_000, -}; - /** * Benchmarks a fetch operation by running it multiple times and computing the average time (in milliseconds) such fetch operation takes. * @@ -71,23 +71,23 @@ async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Prom return postTimeMs - preTimeMs; }; - const callDurationsMs = await Promise.all( - new Array(options?.numberOfCalls ?? defaultOptions.numberOfCalls).fill(null).map(async () => { + const resolvedOptions = { ...defaultOptions, ...options }; + + const iterationsMs = await Promise.all( + new Array(resolvedOptions.numberOfIterations).fill(null).map(async () => { // let's add a random delay before we make the fetch - await nodeTimesPromises.setTimeout( - Math.round(Math.random() * (options?.maxRandomDelayMs ?? defaultOptions.maxRandomDelayMs)) - ); + await nodeTimesPromises.setTimeout(Math.round(Math.random() * resolvedOptions.maxRandomDelayMs)); return benchmarkFetchCall(); }) ); - const averageMs = callDurationsMs.reduce((time, sum) => sum + time) / callDurationsMs.length; + const averageMs = iterationsMs.reduce((time, sum) => sum + time) / iterationsMs.length; - const p90Ms = getPercentile(callDurationsMs, 90); + const p90Ms = getPercentile(iterationsMs, 90); return { - callDurationsMs, + iterationsMs, averageMs, p90Ms, }; diff --git a/benchmarking/src/cloudflare.ts b/benchmarking/src/cloudflare.ts index ba63bc2a..4c482cd5 100644 --- a/benchmarking/src/cloudflare.ts +++ b/benchmarking/src/cloudflare.ts @@ -53,11 +53,13 @@ export async function buildApp(dir: string): Promise { const packageJsonContent = JSON.parse(await nodeFsPromises.readFile(packageJsonPath, "utf8")); - if (!("scripts" in packageJsonContent) || !("build:worker" in packageJsonContent.scripts)) { - throw new Error(`Error: package.json for app at "${dir}" does not include a "build:worker" script`); + const buildScript = "build:worker"; + + if (!packageJsonContent.scripts?.[buildScript]) { + throw new Error(`Error: package.json for app at "${dir}" does not include a "${buildScript}" script`); } - const command = "pnpm build:worker"; + const command = `pnpm ${buildScript}`; await promiseExec(command, { cwd: dir }); } From 62e878b6b01b2704e74a3d83957e0a65291a8547 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 5 Nov 2024 12:21:15 +0000 Subject: [PATCH 10/10] remove `promiseExec` --- benchmarking/src/cloudflare.ts | 62 ++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/benchmarking/src/cloudflare.ts b/benchmarking/src/cloudflare.ts index 4c482cd5..e37a6c40 100644 --- a/benchmarking/src/cloudflare.ts +++ b/benchmarking/src/cloudflare.ts @@ -2,9 +2,6 @@ import nodeFsPromises from "node:fs/promises"; import nodeFs from "node:fs"; import nodePath from "node:path"; import nodeChildProcess from "node:child_process"; -import nodeUtil from "node:util"; - -const promiseExec = nodeUtil.promisify(nodeChildProcess.exec); await ensureWranglerSetup(); @@ -61,7 +58,14 @@ export async function buildApp(dir: string): Promise { const command = `pnpm ${buildScript}`; - await promiseExec(command, { cwd: dir }); + return new Promise((resolve, reject) => { + nodeChildProcess.exec(command, { cwd: dir }, (error) => { + if (error) { + return reject(error); + } + return resolve(); + }); + }); } /** @@ -71,15 +75,21 @@ export async function buildApp(dir: string): Promise { * @returns the url of the deployed application */ export async function deployBuiltApp(dir: string): Promise { - const { stdout } = await promiseExec("pnpm exec wrangler deploy", { cwd: dir }); + return new Promise((resolve, reject) => { + nodeChildProcess.exec("pnpm exec wrangler deploy", { cwd: dir }, (error, stdout) => { + if (error) { + return reject(error); + } - const deploymentUrl = stdout.match(/\bhttps:\/\/(?:[a-zA-Z0-9.\-])*\.workers\.dev\b/)?.[0]; + const deploymentUrl = stdout.match(/\bhttps:\/\/(?:[a-zA-Z0-9.\-])*\.workers\.dev\b/)?.[0]; - if (!deploymentUrl) { - throw new Error(`Could not obtain a deployment url for app at "${dir}"`); - } + if (!deploymentUrl) { + return reject(new Error(`Could not obtain a deployment url for app at "${dir}"`)); + } - return deploymentUrl; + return resolve(deploymentUrl); + }); + }); } /** @@ -89,15 +99,25 @@ export async function deployBuiltApp(dir: string): Promise { * - if they have more than one account they have set a CLOUDFLARE_ACCOUNT_ID env variable */ async function ensureWranglerSetup(): Promise { - const { stdout } = await promiseExec("pnpm dlx wrangler whoami"); - - if (stdout.includes("You are not authenticated")) { - throw new Error("Please log in using wrangler by running `pnpm dlx wrangler login`"); - } - - if (!(process.env as Record)["CLOUDFLARE_ACCOUNT_ID"]) { - throw new Error( - "Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications" - ); - } + return new Promise((resolve, reject) => { + nodeChildProcess.exec("pnpm dlx wrangler whoami", (error, stdout) => { + if (error) { + return reject(error); + } + + if (stdout.includes("You are not authenticated")) { + reject(new Error("Please log in using wrangler by running `pnpm dlx wrangler login`")); + } + + if (!(process.env as Record)["CLOUDFLARE_ACCOUNT_ID"]) { + reject( + new Error( + "Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications" + ) + ); + } + + return resolve(); + }); + }); }