feat: use optional cython compilation

Author: jnoortheen

Taskfile.yml

@@ -7,13 +7,13 @@ tasks:
   generate-meta:
     cmds:
       - git checkout pegen/grammar_parser.py
-      - python3 -m pegen pegen/metagrammar.gram -o pegen/grammar_parser.py
+      - uv run python3 -m pegen pegen/metagrammar.gram -o pegen/grammar_parser.py
       - ruff format pegen/grammar_parser.py
   generate:
     deps:
       - dev
     cmds:
-      - python3 tasks/generator.py
+      - uv run python3 tasks/generator.py
       - ruff check --fix peg_parser/parser.py
       - ruff format peg_parser/parser.py
       - scc peg_parser/parser.py --uloc
@@ -32,8 +32,8 @@ tasks:
 
   profile-tasks:
     cmds:
-      - python tasks/profile_mem.py | tee -a {{.LOG_FILE}}
-      - python tasks/simple.py | tee -a {{.LOG_FILE}}
+      - uv run python tasks/profile_mem.py | tee -a {{.LOG_FILE}}
+      - uv run python tasks/simple.py | tee -a {{.LOG_FILE}}
     vars:
       LOG_FILE:
         sh: echo ".local/logs/xonsh-parser-$(date "+%Y%m%d-%H%M%S").$(git rev-parse --short HEAD).log"
@@ -63,7 +63,7 @@ tasks:
     deps:
       - generate
     cmds:
-      - python -m pytest {{.CLI_ARGS}}
+      - uv run python -m pytest {{.CLI_ARGS}}
   #    sources:
   #      - '**/*.py'
 
@@ -82,23 +82,23 @@ tasks:
 
   memray:
     cmds:
-      - pytest --memray
+      - uv run pytest --memray
   mypy1:
     cmds:
-      - mypy peg_parser
+      - uv run mypy peg_parser
   mypy:
     cmds:
       - watchexec -e py,toml,gram --clear -- task generate mypy1
 
   build_dep:
     cmds:
-      - pip install setuptools_scm wheel
+      - uv run uv pip install setuptools_scm wheel
 
   mypycify:
     deps:
       - build_dep
     cmds:
-      - env COMPILE_WITH_MYPYC=1 pip install -e . --no-build-isolation
+      - env COMPILE_WITH_MYPYC=1 uv pip install -e . --no-build-isolation
     sources:
       - peg_parser/subheader.py
       - peg_parser/toke*.py
@@ -107,7 +107,7 @@ tasks:
     deps:
       - build_dep
     cmds:
-      - env COMPILE_WITH_CYTHON=1 pip install -e . --no-build-isolation
+      - env COMPILE_WITH_CYTHON=1 uv pip install -e . --no-build-isolation
     sources:
       - peg_parser/*.py
       - peg_parser/*.pxd
@@ -126,13 +126,13 @@ tasks:
 
   pytest-bench:
     cmds:
-      - pytest tests/benchmarks.py --benchmark-only --benchmark-autosave {{.CLI_ARGS}}
+      - uv run pytest tests/benchmarks.py --benchmark-only --benchmark-autosave {{.CLI_ARGS}}
 
   bench-lexer:
     deps:
       - dev
     cmds:
-      - pytest tests/bench_lexer.py --benchmark-only --benchmark-autosave {{.CLI_ARGS}}
+      - uv run pytest tests/bench_lexer.py --benchmark-only --benchmark-autosave {{.CLI_ARGS}}
 
   pytest-bench-plot:
     desc: Generate an SVG histogram of all recorded benchmarks

peg_parser/parser.pxd

@@ -0,0 +1,4 @@
+from peg_parser.subheader cimport Parser
+
+cdef class XonshParser(Parser):
+    pass

peg_parser/subheader.pxd

@@ -0,0 +1,19 @@
+
+import cython
+from peg_parser.tokenizer cimport Tokenizer
+
+cdef class Parser:
+    cdef public Tokenizer _tokenizer
+    cdef public bint _verbose
+    cdef public int _level
+    cdef public list _caches
+    cdef public object tok_cls
+    cdef public int in_recursive_rule
+    cdef public object _path_token
+    cdef public int _index     # index is int
+    cdef public str filename
+    cdef public object py_version
+    cdef public bint call_invalid_rules
+
+    cpdef _mark(self)
+    cpdef _reset(self, int index)

peg_parser/tokenize.pxd

@@ -0,0 +1,14 @@
+
+import cython
+
+cdef class TokenInfo:
+    cdef public tuple end
+    cdef public tuple start
+    cdef public str string
+    cdef public object type
+
+    cpdef is_exact_type(self, str typ)
+    cpdef loc_start(self)
+    cpdef loc_end(self)
+    cpdef loc(self)
+    cpdef is_next_to(self, TokenInfo prev)

peg_parser/tokenizer.pxd

@@ -0,0 +1,23 @@
+
+import cython
+
+cdef class Tokenizer:
+    cdef object _readline
+    cdef list _tokens
+    cdef public int _index
+    cdef bint _verbose
+    cdef dict _lines
+    cdef str _path
+    cdef list _stack
+    cdef bint _call_macro
+    cdef bint _proc_macro
+    cdef public list _raw_tokens
+    cdef public int _raw_index
+    cdef dict _end_parens
+    cdef public object tok_cls
+    cdef public object new_tok
+
+    cpdef getnext(self)
+    cpdef peek(self)
+    cpdef diagnose(self)
+    cpdef get_last_non_whitespace_token(self)

peg_parser/tokenizer.py

@@ -14,15 +14,15 @@
 SKIP_TOKENS = ("WS", "COMMENT", "NL")
 
 
-def rs_tokenize(source: str) -> list[TokenInfo]:
+def rs_tokenize(source: str) -> tuple:
     import winnow_parser as wp
 
     from peg_parser.tokenize import TokenInfo
 
     return wp.tokenize(source), wp.TokInfo, TokenInfo
 
 
-def py_tokenize(source: str) -> list[TokenInfo]:
+def py_tokenize(source: str) -> tuple:
     from peg_parser.tokenize import TokenInfo, generate_tokens
 
     return list(generate_tokens(source)), TokenInfo, TokenInfo

tests/conftest.py

@@ -56,7 +56,10 @@ def _py_lex_input(inp: str) -> list[TokenInfo]:
     return tokens
 
 
-LEXERS = ["rust", "py"]
+LEXERS = [
+    # "rust",
+    "py"
+]
 
 
 @pytest.fixture(params=LEXERS, name="lexer")