Skip to content

Tweaks to perf_compare.py script to reduce RAM usage #19321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 21, 2025
Merged

Tweaks to perf_compare.py script to reduce RAM usage #19321

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
95eff57
Tweaks to perf_compare.py script to reduce RAM usage
JukkaL Jun 21, 2025
bf82a65
Fix typo
JukkaL Jun 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions misc/perf_compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,13 @@ def heading(s: str) -> None:
print()


def build_mypy(target_dir: str) -> None:
def build_mypy(target_dir: str, multi_file: bool) -> None:
env = os.environ.copy()
env["CC"] = "clang"
env["MYPYC_OPT_LEVEL"] = "2"
env["PYTHONHASHSEED"] = "1"
if multi_file:
env["MYPYC_MULTI_FILE"] = "1"
cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
subprocess.run(cmd, env=env, check=True, cwd=target_dir)

Expand Down Expand Up @@ -110,6 +112,12 @@ def main() -> None:
action="store_true",
help="measure incremental run (fully cached)",
)
parser.add_argument(
"--multi-file",
default=False,
action="store_true",
help="compile each mypy module to a separate C file (reduces RAM use)",
)
parser.add_argument(
"--dont-setup",
default=False,
Expand All @@ -127,9 +135,9 @@ def main() -> None:
parser.add_argument(
"-j",
metavar="N",
default=8,
default=4,
type=int,
help="set maximum number of parallel builds (default=8)",
help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!",
)
parser.add_argument(
"-r",
Expand All @@ -155,6 +163,7 @@ def main() -> None:
args = parser.parse_args()
incremental: bool = args.incremental
dont_setup: bool = args.dont_setup
multi_file: bool = args.multi_file
commits = args.commit
num_runs: int = args.num_runs + 1
max_workers: int = args.j
Expand Down Expand Up @@ -185,7 +194,9 @@ def main() -> None:
print("(This will take a while...)")

with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs]
futures = [
executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs
]
for future in as_completed(futures):
future.result()

Expand Down