feat: compile with mypyc optionally

Author: jnoortheen

.asv/results/Noortheens-MacBook-Air.local/f772d735-virtualenv-py3.12.json

@@ -0,0 +1 @@
+{"commit_hash": "f772d735cf3d71636cd19b2855238689c67e1363", "env_name": "virtualenv-py3.12", "date": 1716994131000, "params": {"arch": "arm64", "cpu": "Apple M1", "machine": "Noortheens-MacBook-Air.local", "num_cpu": "8", "os": "Darwin 21.2.0", "ram": "8589934592", "python": "3.12"}, "python": "3.12", "requirements": {}, "env_vars": {}, "result_columns": ["result", "params", "version", "started_at", "duration", "stats_ci_99_a", "stats_ci_99_b", "stats_q_25", "stats_q_75", "stats_number", "stats_repeat", "samples", "profile"], "results": {"benchmarks.TimeSuite.time_parser_init": [[0.0009463538881391287], [], "f2c6ff395f3b5b2fa63f6bae6bfdc8bf1c5a82b6cfc1c4f86aea11182d6dc0ea", 1716994621799, 1.1429, [0.00064567], [0.0019985], [0.00064893], [0.0012532], [1], [10]], "benchmarks.MemSuite.mem_parser_init": [[0], [], "d1d13e9271ac2f54e4467d6fa696e8594e58c73a636054c7408b1f70d6993831", 1716994621574, 0.11643], "benchmarks.PeakMemSuite.peakmem_parser_init_": [[25837568], [], "47681fd0575e44856252c83a7fa6e6fe4fe73a76dd56202580ae9b2fedc0259d", 1716994621690, 0.1085]}, "durations": {}, "version": 2}

Taskfile.yml

@@ -12,7 +12,8 @@ tasks:
 
   profile:
     cmds:
-      - python tasks/profile_mem.py tee "logs/xonsh-parser-$(date "+%Y%m%d-%H%M%S").log"
+      - mkdir -p .local/logs
+      - python tasks/profile_mem.py | tee ".local/logs/xonsh-parser-$(date "+%Y%m%d-%H%M%S").$(git rev-parse --short HEAD).log"
 
   ply-add:
     cmds:
@@ -38,7 +39,8 @@ tasks:
 
   bench:
     cmds:
-      - asv run --python=same --show-stderr
+      - mkdir -p .local/logs
+      - asv run --python=same --show-stderr | tee .local/logs/bench.$(date "+%Y%m%d-%H%M%S").$(git rev-parse --short HEAD).txt
 
   bench-view:
     cmds:
@@ -52,4 +54,15 @@ tasks:
 
   mypy:
     cmds:
-      - watchexec -e py --clear -- mypy peg_parser
+      - watchexec -e py --clear -- mypy peg_parser
+
+  mypyc:
+    cmds:
+      - env COMPILE_WITH_MYPYC=1 pip install -e .
+    sources:
+      - peg_parser/subheader.py
+      - peg_parser/toke*.py
+
+  clean:
+    cmds:
+      - find peg_parser -name "*.so" -delete -print

experiments.md

@@ -113,3 +113,33 @@ current=2625.3KiB,  peak=9884.7KiB
 ...
 Total allocated size: 9799.6 KiB
 ```
+
+# Compiling with mypyc doesn't improve much
+
+needed to add following to use it successfully
+```py
+from mypy_extensions import mypyc_attr
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)
+class Parser():
+    ...
+```
+
+with mypyc
+```
+current=2125.4KiB,  peak=28725.6KiB
+...
+1836 other: 757.9 KiB
+Total allocated size: 1387.7 KiB
+Took:  4.03s
+```
+
+without mypyc
+```
+current=1904.7KiB,  peak=1947.3KiB
+...
+1948 other: 596.0 KiB
+Total allocated size: 1901.8 KiB
+Took:  1.39s
+```

setup.py

@@ -29,4 +29,21 @@ class CustomBuild(build):
     sub_commands = [("build_custom", None), *build.sub_commands]  # noqa: RUF012
 
 
-setup(cmdclass={"build": CustomBuild, "build_custom": CustomCommand})
+options = {}
+
+
+if os.environ.get("COMPILE_WITH_MYPYC"):
+    from mypyc.build import mypycify
+
+    options["ext_modules"] = mypycify(
+        [
+            "peg_parser/tokenize.py",
+            "peg_parser/tokenizer.py",
+            "peg_parser/subheader.py",
+        ]
+    )
+
+setup(
+    cmdclass={"build": CustomBuild, "build_custom": CustomCommand},
+    **options,
+)