diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e55d7bf..d1365ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,32 +7,39 @@ on: branches: [main] jobs: - tests: - name: Tests on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - steps: - - uses: actions/checkout@v4 - - name: Install make on Windows - if: runner.os == 'Windows' - run: choco install make -y - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Run tests - shell: bash - run: make test - nix-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: cachix/install-nix-action@v27 with: - nix_path: nixpkgs=channel:nixos-24.05 + nix_path: nixpkgs=channel:nixos-25.05 extra_nix_config: | experimental-features = nix-command flakes - name: Run tests via Nix - run: nix develop --command make test + run: nix develop --command just test + + rust-tests: + name: Rust module test on ${{ matrix.os }} (Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["10", "11", "12", "13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.${{ matrix.python-version }} + - uses: astral-sh/setup-uv@v4 + - uses: messense/maturin-action@v1 + with: + command: build + args: --interpreter python3.${{ matrix.python-version }} -m crates/codetracer-python-recorder/Cargo.toml --release + - name: Install and test built wheel with uv (pytest) + shell: bash + run: | + v=${{matrix.python-version}} + file=(crates/codetracer-python-recorder/target/wheels/*.whl) + file="${file[0]}" + uv run -p python3.$v --with "${file}" --with pytest -- python -m pytest crates/codetracer-python-recorder/test -q diff --git a/.gitignore b/.gitignore index 6ffc067..48ffc7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ .direnv/ **/__pycache__/ +.aider* +.venv/ +**/target/ +build \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 690dcf2..feee796 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,13 +1,26 @@ # Instructions for Codex -To run the test suite, execute: +This repository contains two related projects: + +- codetracer-pure-python-recorder — the original pure-Python tracer. +- codetracer-python-recorder — a Rust-backed Python module built with PyO3 and maturin. + +To run the Python test suite for the pure-Python tracer, execute: ``` -make test +just test ``` The tester executes a number of sample programs in `tests/programs` and compares their outputs to the fixtures in `tests/fixtures`. +To build and locally develop-install the Rust-backed module: + +``` +just build-rust +# or: +maturin develop -m crates/codetracer-python-recorder/Cargo.toml +``` + # Code quality guidelines - Strive to achieve high code quality. @@ -33,4 +46,4 @@ https://www.conventionalcommits.org/en/v1.0.0/ In the remaining lines, provide a short description of the implemented functionality. Provide sufficient details for the justification of each design decision if multiple -approaches were considered. \ No newline at end of file +approaches were considered. diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..464ae9a --- /dev/null +++ b/Justfile @@ -0,0 +1,48 @@ +# Development helpers for the monorepo + +# Python versions used for multi-version testing/building with uv +PY_VERSIONS := "3.10 3.11 3.12 3.13" +PY_SHORT_VERSIONS := "10 11 12 13" +# Print toolchain versions to verify the dev environment +env: + python3 --version + cargo --version + rustc --version + maturin --version + +# Create a local virtualenv for Python tooling +venv: + test -d .venv || python3 -m venv .venv + +# Build and develop-install the Rust-backed Python module +build-rust: + test -d .venv || python3 -m venv .venv + VIRTUAL_ENV=.venv maturin develop -m crates/codetracer-python-recorder/Cargo.toml + +# Smoke test the Rust module after build +smoke-rust: + .venv/bin/python -m pip install -U pip pytest + .venv/bin/python -m pytest crates/codetracer-python-recorder/test -q + +# Run the Python test suite for the pure-Python recorder +test: + python3 -m unittest discover -v + +# Run the test suite across multiple Python versions using uv +test-uv-all: + uv python install {{PY_VERSIONS}} + for v in {{PY_VERSIONS}}; do uv run -p "$v" -m unittest discover -v; done + +# Build wheels for all target Python versions with maturin +build-rust-uv-all: + for v in {{PY_VERSIONS}}; do \ + maturin build --interpreter "python$v" -m crates/codetracer-python-recorder/Cargo.toml --release; \ + done + +# Smoke the built Rust wheels across versions using uv +smoke-rust-uv-all: + for v in {{PY_SHORT_VERSIONS}}; do \ + file=(crates/codetracer-python-recorder/target/wheels/codetracer_python_recorder-*-cp3$v-cp3$v-*.whl); \ + file="${file[0]}"; \ + uv run -p "python3.$v" --with "${file}" --with pytest -- python -m pytest crates/codetracer-python-recorder/test -q; \ + done diff --git a/Makefile b/Makefile index e2c0f86..0013d1f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ +# This Makefile is deprecated. Use 'just test' instead. .PHONY: test - test: - python3 -m unittest discover -v + @echo "Deprecated: Use 'just test' instead." && false diff --git a/README.md b/README.md index b8d4df4..9dfa927 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,22 @@ -## codetracer-python-recorder +## CodeTracer Recorders (Monorepo) -An unfinished prototype of a recorder of Python programs that produces [CodeTracer](https://github.com/metacraft-labs/CodeTracer) traces. +This repository now hosts two related projects: -> [!WARNING] -> Currently it is in a very early phase: we're welcoming contribution and discussion! +- codetracer-pure-python-recorder — the existing pure-Python prototype that records [CodeTracer](https://github.com/metacraft-labs/CodeTracer) traces using sys.settrace. +- codetracer-python-recorder — a new, Rust-backed Python extension module (PyO3) intended to provide a faster and more featureful recorder. +> [!WARNING] +> Both projects are early-stage prototypes. Contributions and discussion are welcome! -### Usage +### codetracer-pure-python-recorder -Install the package with `pip` or `uv`: +Install from PyPI: ```bash -pip install codetracer-python-recorder +pip install codetracer-pure-python-recorder ``` -Then invoke the recorder as a command line tool: +CLI usage: ```bash codetracer-record @@ -22,23 +24,34 @@ codetracer-record # or in the folder of `$CODETRACER_DB_TRACE_PATH` if such an env var is defined ``` -During development you can also run it directly with +During development you can also run it directly: ```bash -python trace.py +python src/trace.py # produces several trace json files in the current directory # or in the folder of `$CODETRACER_DB_TRACE_PATH` if such an env var is defined ``` -however you probably want to use it in combination with CodeTracer, which would be released soon. +### codetracer-python-recorder (Rust-backed) + +A separate Python module implemented in Rust with PyO3 and built via maturin lives under: +crates/codetracer-python-recorder/ + +Basic workflow: + +- Build/dev install the Rust module: + - maturin develop -m crates/codetracer-python-recorder/Cargo.toml +- Use in Python: + - from codetracer_python_recorder import hello + - hello() -## Future directions +### Future directions The current Python support is an unfinished prototype. We can finish it. In the future, it may be expanded to function in a way to similar to the more complete implementations, e.g. [Noir](https://github.com/blocksense-network/noir/tree/blocksense/tooling/tracer). Currently it's very similar to our [Ruby tracer](https://github.com/metacraft-labs/ct-ruby-tracer) -### Current approach: sys.settrace API +#### Current approach: sys.settrace API Currently we're using the sys.settrace API: https://docs.python.org/3/library/sys.html#sys.settrace . This is very flexible and can function with probably multiple Python versions out of the box. @@ -49,7 +62,7 @@ However, this is limited: For other languages, we've used a more deeply integrated approach: patching the interpreter or VM itself (e.g. Noir). -### Patching the VM +#### Patching the VM This can be a good approach for Python as well: it can let us record more precisely subvalues, assignments and subexpressions and to let some CodeTracer features work in a deeper/better way. @@ -57,7 +70,7 @@ some CodeTracer features work in a deeper/better way. One usually needs to add additional logic to places where new opcodes/lines are being ran, and to call entries/exits. Additionally tracking assignments can be a great addition, but it really depends on the interpreter internals. -### Filtering +#### Filtering It would be useful to have a way to record in detail only certain periods of the program, or certain functions or modules: we plan on expanding the [trace format](https://github.com/metacraft-labs/runtime_tracing/) and CodeTracer' support, so that this is possible. It would let one be able to record interesting diff --git a/crates/codetracer-python-recorder/Cargo.lock b/crates/codetracer-python-recorder/Cargo.lock new file mode 100644 index 0000000..ff1e5ba --- /dev/null +++ b/crates/codetracer-python-recorder/Cargo.lock @@ -0,0 +1,164 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "codetracer-python-recorder" +version = "0.1.0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "proc-macro2" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/crates/codetracer-python-recorder/Cargo.toml b/crates/codetracer-python-recorder/Cargo.toml new file mode 100644 index 0000000..7150965 --- /dev/null +++ b/crates/codetracer-python-recorder/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "codetracer-python-recorder" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Rust-backed Python module for CodeTracer recording (PyO3)" +repository = "https://github.com/metacraft-labs/codetracer-python-recorder" + +[lib] +name = "codetracer_python_recorder" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.25.1", features = ["extension-module"] } diff --git a/crates/codetracer-python-recorder/pyproject.toml b/crates/codetracer-python-recorder/pyproject.toml new file mode 100644 index 0000000..b5a79fd --- /dev/null +++ b/crates/codetracer-python-recorder/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["maturin>=1.5,<2"] +build-backend = "maturin" + +[project] +name = "codetracer-python-recorder" +version = "0.1.0" +description = "Rust-backed Python module for CodeTracer recording (PyO3)" +authors = [{name = "Metacraft Labs Ltd"}] +license = {text = "MIT"} +requires-python = ">=3.8" +classifiers = [ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Rust", +] + +[tool.maturin] +# Build the PyO3 extension module +bindings = "pyo3" +# Use the library name as the Python import name: codetracer_python_recorder diff --git a/crates/codetracer-python-recorder/src/lib.rs b/crates/codetracer-python-recorder/src/lib.rs new file mode 100644 index 0000000..830e44f --- /dev/null +++ b/crates/codetracer-python-recorder/src/lib.rs @@ -0,0 +1,17 @@ +use pyo3::prelude::*; + +/// codetracer_python_recorder +/// +/// Minimal placeholder for the Rust-backed recorder. This exposes a trivial +/// function to verify the module builds and imports successfully. +#[pyfunction] +fn hello() -> PyResult { + Ok("Hello from codetracer-python-recorder (Rust)".to_string()) +} + +#[pymodule] +fn codetracer_python_recorder(_py: Python<'_>, m: Bound<'_, PyModule>) -> PyResult<()> { + let hello_fn = wrap_pyfunction!(hello, &m)?; + m.add_function(hello_fn)?; + Ok(()) +} diff --git a/crates/codetracer-python-recorder/test/smoke.py b/crates/codetracer-python-recorder/test/smoke.py new file mode 100644 index 0000000..a54a945 --- /dev/null +++ b/crates/codetracer-python-recorder/test/smoke.py @@ -0,0 +1,8 @@ +"""Smoke test for codetracer_python_recorder wheel.""" + +def main() -> None: + import codetracer_python_recorder as m + print(m.hello()) + +if __name__ == "__main__": + main() diff --git a/crates/codetracer-python-recorder/test/test_smoke.py b/crates/codetracer-python-recorder/test/test_smoke.py new file mode 100644 index 0000000..aab35b5 --- /dev/null +++ b/crates/codetracer-python-recorder/test/test_smoke.py @@ -0,0 +1,5 @@ +import codetracer_python_recorder as m + + +def test_hello_returns_expected_string() -> None: + assert m.hello() == "Hello from codetracer-python-recorder (Rust)" diff --git a/crates/codetracer-python-recorder/uv.lock b/crates/codetracer-python-recorder/uv.lock new file mode 100644 index 0000000..52f9c5d --- /dev/null +++ b/crates/codetracer-python-recorder/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 2 +requires-python = ">=3.8" + +[[package]] +name = "codetracer-python-recorder" +version = "0.1.0" +source = { editable = "." } diff --git a/flake.lock b/flake.lock index 9769642..0594294 100644 --- a/flake.lock +++ b/flake.lock @@ -2,16 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1735563628, - "narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=", + "lastModified": 1754937576, + "narHash": "sha256-3sWA5WJybUE16kIMZ3+uxcxKZY/JRR4DFBqLdSLBo7w=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b134951a4c9f3c995fd7be05f3243f8ecd65d798", + "rev": "ddae11e58c0c345bf66efbddbf2192ed0e58f896", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05", + "ref": "nixos-25.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index d4b3c19..20d149d 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { - description = "Development environment for codetracer-python-recorder"; + description = "Development environment for CodeTracer recorders (pure-python and rust-backed)"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; outputs = { self, nixpkgs }: let @@ -12,7 +12,32 @@ let pkgs = import nixpkgs { inherit system; }; in { default = pkgs.mkShell { - packages = with pkgs; [ python3 just git-lfs ]; + packages = with pkgs; [ + bashInteractive + python310 + python311 + python312 + python313 + just + git-lfs + + # Linters and type checkers for Python code + ruff + black + mypy + python3Packages.pytest + + # Rust toolchain for the Rust-backed Python module + cargo + rustc + rustfmt + clippy + + # Build tooling for Python extensions + maturin + uv + pkg-config + ]; }; }); }; diff --git a/pyproject.toml b/pyproject.toml index b16aff0..558ea65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=61"] build-backend = "setuptools.build_meta" [project] -name = "codetracer-python-recorder" +name = "codetracer-pure-python-recorder" version = "0.1.0" -description = "Prototype recorder of Python programs producing CodeTracer traces" +description = "Pure-Python prototype recorder producing CodeTracer traces" authors = [{name = "Metacraft Labs Ltd"}] license = {text = "MIT"} readme = "README.md" @@ -20,5 +20,9 @@ classifiers = [ py-modules = ["trace"] package-dir = {"" = "src"} +[tool.setuptools.packages.find] +where = ["src"] + [project.scripts] -codetracer-record = "trace:main" +codetracer-record = "codetracer_pure_python_recorder.cli:main" +codetracer-record-pure = "codetracer_pure_python_recorder.cli:main" diff --git a/src/codetracer_pure_python_recorder/__init__.py b/src/codetracer_pure_python_recorder/__init__.py new file mode 100644 index 0000000..8bba4c0 --- /dev/null +++ b/src/codetracer_pure_python_recorder/__init__.py @@ -0,0 +1,6 @@ +""" +codetracer_pure_python_recorder + +Namespaced package wrapper for the pure-Python recorder. +""" +__all__ = [] diff --git a/src/codetracer_pure_python_recorder/cli.py b/src/codetracer_pure_python_recorder/cli.py new file mode 100644 index 0000000..cc11252 --- /dev/null +++ b/src/codetracer_pure_python_recorder/cli.py @@ -0,0 +1,13 @@ +"""CLI entrypoint for the pure-Python recorder (namespaced). + +This defers to the existing trace.main implementation to preserve behavior +and compatibility with the current tests and CLI usage. +""" + +from typing import List, Optional + + +def main(argv: Optional[List[str]] = None) -> None: + from trace import main as trace_main + + return trace_main(argv) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..e4956c6 --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 2 +requires-python = ">=3.8" + +[[package]] +name = "codetracer-pure-python-recorder" +version = "0.1.0" +source = { editable = "." }