chore: refactor benchmarking scripts and update user schema initialization for performance testing

Author: Shuunen

README.md

@@ -33,11 +33,11 @@ Here are the library used in this comparison :
 - [valibot](https://github.com/fabian-hiller/valibot) from Fabian Hiller
 - [zod](https://github.com/colinhacks/zod) from Colin McDonnell
 
-| Date       | Score | Library | Deps  |  Size  | Light | Input | Throw | Safe  |  Execution  | Fast  | Readability |
-| ---------- | :---: | :-----: | :---: | :----: | :---: | :---: | :---: | :---: | :---------: | :---: | :---------: |
-| 2025-03-04 |   3   | arktype |   2   | 140 KB |  _0_  | **1** |  _0_  | **1** |  155 ms_0_  |  _0_  |    **1**    |
-| 2025-03-04 |   6   | valibot | **0** |  5 KB  | **1** | **1** | **1** | **1** | 74 ms **1** | **1** |     _0_     |
-| 2025-03-04 |   6   |   zod   | **0** | 60 KB  |  _0_  | **1** | **1** | **1** | 75 ms **1** | **1** |    **1**    |
+| Date       | Score | Library | Deps  |  Size  | Light | Input | Throw | Safe  | Script exec |  Lib exec   | Fast  | Readability |
+| ---------- | :---: | :-----: | :---: | :----: | :---: | :---: | :---: | :---: | :---------: | :---------: | :---: | :---------: |
+| 2025-03-04 |   1   | arktype |   2   | 140 KB |  _0_  | **1** |  _0_  | **1** | 729 ms _-1_ | 586 ms _-1_ |  _0_  |    **1**    |
+| 2025-03-04 |   7   | valibot | **0** |  5 KB  | **1** | **1** | **1** | **1** | 70 ms **1** | 10 ms **1** | **1** |     _0_     |
+| 2025-03-04 |   6   |   zod   | **0** | 60 KB  |  _0_  | **1** | **1** | **1** | 98 ms **1** |  34 ms _0_  | **1** |    **1**    |
 
 Legend :
 
@@ -47,13 +47,18 @@ Legend :
 - Input : 1 point if the library can see that `age` is optional in the input but not optional in `type User` the output type
 - Throw : 1 point if the library have a parse or throw method, useful when we don't want to handle the error cases
 - Safe : 1 point if the library have a safe parse method that will not throw and usually return a `Result` type
-- Execution : 1 point if the average time in milliseconds to execute the test file with bun is under 100ms, check the `bun run bench` command output
+- Script exec : 1 point if the average time in milliseconds to execute the test file with bun is under `100ms`, check the `bun run bench` command output
+- Lib exec : 1 point if the average time in milliseconds to execute the library is under `30ms`, check the `bun run check:once` command output
 - Fast : 1 point if the library execution time is less than 100 ms
 - Readability : 1 point if the library is easy to write & read, the syntax need to be intuitive
 
 ## My favorite pick
 
-Zod is my favorite pick because it's fast as Valibot but provide a better readability. Ok the build size is bigger but it does not impact the performance.
+Zod is my favorite pick because it's fast and provide a better readability.
+
+Ok the build size is bigger but it does not impact the performance that much.
+
+The 34ms vs 10ms seems like a 3x difference but it's the time to run 1000 iterations. So in the end it's not that much of a difference in a real world scenario.
 
 ## Todo
 

package.json

@@ -42,13 +42,13 @@
     "url": "git+https://github.com/Shuunen/validation-lib-comparison.git"
   },
   "scripts": {
-    "bench": "bun bench:bun --warmup 5 --runs 25",
+    "bench": "bun bench:bun --warmup 5 --runs 10",
     "bench:bun": "hyperfine 'bun src/arktype.ts' 'bun src/valibot.ts' 'bun src/zod.ts'",
-    "bench:node": "hyperfine --runs 30 'node dist/arktype.js' 'node dist/valibot.js' 'node dist/zod.js'",
-    "bench:node:strip": "hyperfine --runs 30 'node --experimental-strip-types src/arktype.ts' 'node --experimental-strip-types src/valibot.ts' 'node --experimental-strip-types src/zod.ts'",
+    "bench:node": "hyperfine --runs 10 'node dist/arktype.js' 'node dist/valibot.js' 'node dist/zod.js'",
+    "bench:node:strip": "hyperfine --runs 10 'node --experimental-strip-types src/arktype.ts' 'node --experimental-strip-types src/valibot.ts' 'node --experimental-strip-types src/zod.ts'",
     "build": "bun scripts/build.ts && echo build success",
     "check": "repo-check && bun check:tsc && bun run build && bun check:once && echo check success",
-    "check:once": "bun bench:bun --runs 1 && echo check:once success",
+    "check:once": "bun src/arktype.ts && bun src/valibot.ts && bun src/zod.ts && echo check:once success",
     "check:tsc": "tsc --noEmit && echo check:tsc success"
   },
   "type": "module",

src/arktype.ts

@@ -1,31 +1,37 @@
 import { type } from "arktype"
-import { checkUserA, checkUserB, checkUserC } from './utils.ts'
+import { checkUserA, checkUserB, checkUserC, nbIterations } from './utils.ts'
 
-const userSchema = type({
-  name: type.string,
-  age: type.number.default(42),
-  phone: type.string.or(type.number).pipe(phone => typeof phone === 'number' ? phone.toString() : phone).default("123-456-7890"),
-})
+const startTime = performance.now()
 
-export type User = typeof userSchema.inferOut
+for (let i = 0; i < nbIterations; i++) {
 
-export type UserInput = typeof userSchema.inferIn
+  const userSchema = type({
+    name: type.string,
+    age: type.number.default(42),
+    phone: type.string.or(type.number).pipe(phone => typeof phone === 'number' ? phone.toString() : phone).default("123-456-7890"),
+  })
 
-const userA = userSchema({ name: "Jordan" })
-// @ts-expect-error Argument of type '{ name: string; age: number; } | ArkErrors' is not assignable to parameter of type 'User'.
-checkUserA(userA)
+  type User = typeof userSchema.inferOut
 
-function createUser (input: UserInput) {
-  const user = userSchema(input)
-  if (user instanceof type.errors) return userA as User // :'''''(
-  return user
-}
+  type UserInput = typeof userSchema.inferIn
+
+  const userA = userSchema({ name: "Jordan" })
+  // @ts-expect-error Argument of type '{ name: string; age: number; } | ArkErrors' is not assignable to parameter of type 'User'.
+  checkUserA(userA)
+
+  function createUser (input: UserInput) {
+    const user = userSchema(input)
+    if (user instanceof type.errors) return userA as User // :'''''(
+    return user
+  }
 
-const userB = createUser({ name: "Romain", age: 35 })
-checkUserB(userB)
+  const userB = createUser({ name: "Romain", age: 35, phone: 1234567890 })
+  checkUserB(userB)
 
-// @ts-expect-error age should be a number
-const userC = createUser({ name: "Romain", age: "35" })
-checkUserC(userC)
+  // @ts-expect-error age should be a number
+  const userC = createUser({ name: "Romain", age: "35" })
+  checkUserC(userC)
+
+}
 
-console.log('All tests passed successfully.')
+console.log(`ArkType exec time for ${nbIterations} iterations :`, performance.now() - startTime, 'ms')

src/utils.ts

@@ -16,3 +16,5 @@ export function checkUserB (userB: User) {
 export function checkUserC (userC: User) {
   console.assert(userC.name === "Jordan", 'userC.name should be Jordan')
 }
+
+export const nbIterations = 1000

src/valibot.ts

@@ -1,33 +1,40 @@
 import { number, union, object, optional, parse, safeParse, string, type InferInput, type InferOutput, pipe, transform } from 'valibot'
-import { checkUserA, checkUserB, checkUserC } from './utils.ts'
+import { checkUserA, checkUserB, checkUserC, nbIterations } from './utils.ts'
 
-const userSchema = object({
-  name: string(),
-  age: optional(number(), 42),
-  phone: pipe(
-    optional(union([string(), number()]), "123-456-7890"),
-    transform((phone) => typeof phone === 'number' ? phone.toString() : phone)
-  ),
-})
+const startTime = performance.now()
 
-export type User = InferOutput<typeof userSchema>
+for (let i = 0; i < nbIterations; i++) {
 
-export type UserInput = InferInput<typeof userSchema>
+  const userSchema = object({
+    name: string(),
+    age: optional(number(), 42),
+    phone: pipe(
+      optional(union([string(), number()]), "123-456-7890"),
+      transform((phone) => typeof phone === 'number' ? phone.toString() : phone)
+    ),
+  })
 
-const userA = parse(userSchema, { name: "Jordan" })
-checkUserA(userA)
+  // @ts-expect-error not exported, it's ok :p
+  type User = InferOutput<typeof userSchema>
 
-function createUser (input: UserInput) {
-  const result = safeParse(userSchema, input)
-  if (!result.success) return userA
-  return result.output
-}
+  type UserInput = InferInput<typeof userSchema>
+
+  const userA = parse(userSchema, { name: "Jordan" })
+  checkUserA(userA)
+
+  function createUser (input: UserInput) {
+    const result = safeParse(userSchema, input)
+    if (!result.success) return userA
+    return result.output
+  }
 
-const userB = createUser({ name: "Romain", age: 35, phone: 1234567890 })
-checkUserB(userB)
+  const userB = createUser({ name: "Romain", age: 35, phone: 1234567890 })
+  checkUserB(userB)
 
-// @ts-expect-error age should be a number
-const userC = createUser({ name: "Romain", age: "35" })
-checkUserC(userC)
+  // @ts-expect-error age should be a number
+  const userC = createUser({ name: "Romain", age: "35" })
+  checkUserC(userC)
+
+}
 
-console.log('All tests passed successfully.')
+console.log(`Valibot exec time for ${nbIterations} iterations :`, performance.now() - startTime, 'ms')

src/zod.ts

@@ -1,30 +1,37 @@
 import { string, number, object, type infer as InferOutput, type input as InferInput } from "zod"
-import { checkUserA, checkUserB, checkUserC } from './utils.ts'
+import { checkUserA, checkUserB, checkUserC, nbIterations } from './utils.ts'
 
-const userSchema = object({
-  name: string(),
-  age: number().default(42),
-  phone: string().or(number()).default("123-456-7890").transform(phone => typeof phone === 'number' ? phone.toString() : phone),
-})
+const startTime = performance.now()
 
-export type User = InferOutput<typeof userSchema>
+for (let i = 0; i < nbIterations; i++) {
 
-export type UserInput = InferInput<typeof userSchema>
+  const userSchema = object({
+    name: string(),
+    age: number().default(42),
+    phone: string().or(number()).default("123-456-7890").transform(phone => typeof phone === 'number' ? phone.toString() : phone),
+  })
 
-const userA = userSchema.parse({ name: "Jordan" })
-checkUserA(userA)
+  // @ts-expect-error not exported, it's ok :p
+  type User = InferOutput<typeof userSchema>
 
-function createUser (input: UserInput) {
-  const result = userSchema.safeParse(input)
-  if (!result.success) return userA
-  return result.data
-}
+  type UserInput = InferInput<typeof userSchema>
+
+  const userA = userSchema.parse({ name: "Jordan" })
+  checkUserA(userA)
+
+  function createUser (input: UserInput) {
+    const result = userSchema.safeParse(input)
+    if (!result.success) return userA
+    return result.data
+  }
 
-const userB = createUser({ name: "Romain", age: 35, phone: 1234567890 })
-checkUserB(userB)
+  const userB = createUser({ name: "Romain", age: 35, phone: 1234567890 })
+  checkUserB(userB)
 
-// @ts-expect-error age should be a number
-const userC = createUser({ name: "Romain", age: "35" })
-checkUserC(userC)
+  // @ts-expect-error age should be a number
+  const userC = createUser({ name: "Romain", age: "35" })
+  checkUserC(userC)
+
+}
 
-console.log('All tests passed successfully.')
+console.log(`Zod exec time for ${nbIterations} iterations :`, performance.now() - startTime, 'ms')