More benchmarks

Author: MichaReiser

scripts/ty_benchmark/pyproject.toml

@@ -3,7 +3,10 @@ name = "ty_benchmark"
 version = "0.0.1"
 description = "Package for running end-to-end ty benchmarks"
 requires-python = ">=3.12"
-dependencies = ["mypy", "pyright"]
+dependencies = [
+    "pyrefly>=0.41.3",
+    "pyright",
+]
 
 [project.scripts]
 benchmark = "benchmark.run:main"
@@ -19,3 +22,5 @@ packages = ["src/benchmark"]
 ignore = [
     "E501", # We use ruff format
 ]
+
+[tool.ty]

scripts/ty_benchmark/src/benchmark/__init__.py

@@ -3,11 +3,11 @@
 import logging
 import shlex
 import subprocess
-import typing
 from pathlib import Path
+from typing import Mapping, NamedTuple
 
 
-class Command(typing.NamedTuple):
+class Command(NamedTuple):
     name: str
     """The name of the command to benchmark."""
 
@@ -18,7 +18,7 @@ class Command(typing.NamedTuple):
     """The command to run before each benchmark run."""
 
 
-class Hyperfine(typing.NamedTuple):
+class Hyperfine(NamedTuple):
     name: str
     """The benchmark to run."""
 
@@ -37,13 +37,16 @@ class Hyperfine(typing.NamedTuple):
     json: bool
     """Whether to export results to JSON."""
 
-    def run(self, *, cwd: Path | None = None) -> None:
+    def run(self, *, cwd: Path | None = None, env: Mapping[str, str] = dict()) -> None:
         """Run the benchmark using `hyperfine`."""
         args = [
             "hyperfine",
-            # Most repositories have some typing errors.
-            # This is annoying because it prevents us from capturing "real" errors.
-            "-i",
+            # Ignore any warning/error diagnostics but fail if there are any fatal errors, incorrect configuration, etc.
+            # mypy exit codes: https://github.com/python/mypy/issues/14615#issuecomment-1420163253
+            # pyright exit codes: https://docs.basedpyright.com/v1.31.6/configuration/command-line/#pyright-exit-codes
+            # pyrefly exit codes: Not documented
+            # ty: https://docs.astral.sh/ty/reference/exit-codes/
+            "-i=1",
         ]
 
         # Export to JSON.
@@ -70,7 +73,4 @@ def run(self, *, cwd: Path | None = None) -> None:
 
         logging.info(f"Running {args}")
 
-        subprocess.run(
-            args,
-            cwd=cwd,
-        )
+        subprocess.run(args, cwd=cwd, env=env)