diff --git a/.cargo/config.toml b/.cargo/config.toml index 4b2316cb..50726a9b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,4 @@ [build] -rustflags = ["-Clink-arg=-fuse-ld=lld", "-Zthreads=16", "-Ctarget-cpu=native"] +rustflags = ["-Zthreads=16"] +[profile.dev] +rustflags = ["-Clink-arg=-fuse-ld=lld", "-Zthreads=16"] diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000..e8694aea --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,191 @@ +# This file is autogenerated by maturin v1.8.2 +# To update, run +# +# maturin generate-ci github +# +name: CI + +on: + push: + branches: + - main + - master + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest-8 + target: x86_64 + # - runner: ubuntu-22.04 + # target: x86 + - runner: ubuntu-latest-8 + target: aarch64 + # - runner: ubuntu-22.04 + # target: armv7 + # - runner: ubuntu-22.04 + # target: s390x + # - runner: ubuntu-22.04 + # target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: manylinux_2_34_${{matrix.platform.target}} + working-directory: bindings/python + rust-toolchain: nightly + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: bindings/python/dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest-8 + target: x86_64 + # - runner: ubuntu-22.04 + # target: x86 + - runner: ubuntu-latest-8 + target: aarch64 + # - runner: ubuntu-22.04 + # target: armv7 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: musllinux_1_2 + working-directory: bindings/python + rust-toolchain: nightly + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: bindings/python/dist + + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + # - runner: windows-latest + # target: x86 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: ${{ matrix.platform.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + working-directory: bindings/python + rust-toolchain: nightly + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: bindings/python/dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-13-large + target: x86_64 + - runner: macos-14-large + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + working-directory: bindings/python + rust-toolchain: nightly + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: bindings/python/dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + working-directory: bindings/python + rust-toolchain: nightly + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: bindings/python/dist + + release: + name: Release + environment: release + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [linux, musllinux, windows, macos, sdist] + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write + steps: + - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: 'bindings/python/wheels-*/*' + - name: Publish to PyPI + if: ${{ startsWith(github.ref, 'refs/tags/') }} + uses: PyO3/maturin-action@v1 + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* + working-directory: bindings/python diff --git a/Cargo.lock b/Cargo.lock index e3e0cb09..40182247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "autocfg" version = "1.4.0" @@ -209,9 +215,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cast" @@ -303,6 +309,16 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "codegen-bindings-generator" +version = "0.1.0" +dependencies = [ + "anyhow", + "codegen-sdk-common", + "quote", + "syn 2.0.98", +] + [[package]] name = "codegen-sdk-analyzer" version = "0.1.0" @@ -449,16 +465,10 @@ version = "0.1.0" dependencies = [ "bytes", "codegen-sdk-common", - "convert_case", "dashmap", "indextree", - "log", - "rkyv", "salsa", - "tempfile", - "test-log", "thiserror 2.0.11", - "tree-sitter", ] [[package]] @@ -645,13 +655,10 @@ version = "0.1.0" dependencies = [ "ambassador", "anyhow", - "codegen-sdk-ast", "codegen-sdk-common", "codegen-sdk-cst", "indicatif", - "log", "salsa", - "smallvec", ] [[package]] @@ -1462,6 +1469,12 @@ dependencies = [ "log", ] +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + [[package]] name = "inotify" version = "0.11.0" @@ -1700,6 +1713,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.8.3" @@ -2000,6 +2022,23 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pink-python" +version = "0.1.0" +dependencies = [ + "anyhow", + "codegen-bindings-generator", + "codegen-sdk-analyzer", + "codegen-sdk-common", + "codegen-sdk-python", + "codegen-sdk-resolution", + "env_logger", + "log", + "pyo3", + "pyo3-bytes", + "pyo3-log", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -2114,6 +2153,91 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "pyo3" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +dependencies = [ + "anyhow", + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-bytes" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4404970fe5cad3b196021a9cc555634735a4f11d1df53f8c482a329151e7dfa9" +dependencies = [ + "bytes", + "pyo3", +] + +[[package]] +name = "pyo3-ffi" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-log" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" +dependencies = [ + "arc-swap", + "log", + "pyo3", +] + +[[package]] +name = "pyo3-macros" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.98", +] + [[package]] name = "quote" version = "1.0.38" @@ -2343,8 +2467,9 @@ checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "salsa" -version = "0.18.0" -source = "git+https://github.com/salsa-rs/salsa?rev=dbb0e5f6ab2cd61e42b372f333ab694f24141cf1#dbb0e5f6ab2cd61e42b372f333ab694f24141cf1" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd55c6549513b2a42884dae31e3d4f4ac8a6cc51062e68e24d162133889f327c" dependencies = [ "boxcar", "crossbeam-queue", @@ -2364,13 +2489,15 @@ dependencies = [ [[package]] name = "salsa-macro-rules" -version = "0.18.0" -source = "git+https://github.com/salsa-rs/salsa?rev=dbb0e5f6ab2cd61e42b372f333ab694f24141cf1#dbb0e5f6ab2cd61e42b372f333ab694f24141cf1" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2619b4b451beab0a7e4364ff1e6f31950e7e418888fd9bf2f28889671563166a" [[package]] name = "salsa-macros" -version = "0.18.0" -source = "git+https://github.com/salsa-rs/salsa?rev=dbb0e5f6ab2cd61e42b372f333ab694f24141cf1#dbb0e5f6ab2cd61e42b372f333ab694f24141cf1" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be57a99b3896e8d26850428a6874fb86849e2db874e1db3528e5cee4337d277" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2541,7 +2668,7 @@ dependencies = [ [[package]] name = "subenum" version = "1.1.2" -source = "git+https://github.com/mrenow/subenum?branch=main#d623bc4c0e2a8ab9bc24f255e933411f0c4c9c72" +source = "git+https://github.com/mrenow/subenum?rev=d623bc4c0e2a8ab9bc24f255e933411f0c4c9c72#d623bc4c0e2a8ab9bc24f255e933411f0c4c9c72" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -2596,6 +2723,12 @@ dependencies = [ "windows 0.57.0", ] +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.17.1" @@ -2747,21 +2880,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "tracing-core" version = "0.1.33" @@ -2864,7 +2985,7 @@ checksum = "38eee4db33814de3d004de9d8d825627ed3320d0989cce0dea30efaf5be4736c" [[package]] name = "tree-sitter-md" version = "0.4.0" -source = "git+https://github.com/tree-sitter-grammars/tree-sitter-markdown#192407ab5a24bfc24f13332979b5e7967518754a" +source = "git+https://github.com/tree-sitter-grammars/tree-sitter-markdown?rev=192407ab5a24bfc24f13332979b5e7967518754a#192407ab5a24bfc24f13332979b5e7967518754a" dependencies = [ "cc", "tree-sitter-language", @@ -2883,7 +3004,7 @@ dependencies = [ [[package]] name = "tree-sitter-query" version = "0.5.0" -source = "git+https://github.com/tree-sitter-grammars/tree-sitter-query#0555ac0da902abff06076e40501102cee3ba68bd" +source = "git+https://github.com/tree-sitter-grammars/tree-sitter-query?rev=0555ac0da902abff06076e40501102cee3ba68bd#0555ac0da902abff06076e40501102cee3ba68bd" dependencies = [ "cc", "tree-sitter-language", @@ -2969,6 +3090,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "url" version = "2.5.4" diff --git a/Cargo.toml b/Cargo.toml index c96fe6dd..8e728e6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ -cargo-features = ["codegen-backend"] +cargo-features = ["codegen-backend", "profile-rustflags"] [package] name = "codegen-sdk-core" @@ -7,7 +7,7 @@ edition = "2024" [dependencies] clap = { version = "4.5.28", features = ["derive"] } -codegen-sdk-analyzer = { path = "codegen-sdk-analyzer", default-features = false } +codegen-sdk-analyzer = { workspace = true } codegen-sdk-ast = { workspace = true} codegen-sdk-common = { workspace = true} anyhow = { workspace = true} @@ -62,7 +62,7 @@ members = [ "languages/codegen-sdk-yaml", "languages/codegen-sdk-toml", "languages/codegen-sdk-ts_query", - "languages/codegen-sdk-typescript", "codegen-sdk-resolution", + "languages/codegen-sdk-typescript", "codegen-sdk-resolution", "bindings/python", "codegen-bindings-generator", ] resolver = "2" [workspace.dependencies] @@ -78,7 +78,7 @@ tree-sitter-java = "0.23.5" tree-sitter-ruby = "0.23.1" tree-sitter-rust = "0.23.2" tree-sitter-go = "0.23.4" -tree-sitter-md = {git="https://github.com/tree-sitter-grammars/tree-sitter-markdown"} +tree-sitter-md = {git="https://github.com/tree-sitter-grammars/tree-sitter-markdown", rev = "192407ab5a24bfc24f13332979b5e7967518754a"} tree-sitter-yaml = "0.7.0" tree-sitter-toml-ng = "0.7.0" bytes = "1.10.0" @@ -94,6 +94,8 @@ codegen-sdk-common = { path = "codegen-sdk-common" } codegen-sdk-cst = { path = "codegen-sdk-cst"} codegen-sdk-ast = { path = "codegen-sdk-ast" } codegen-sdk-resolution = { path = "codegen-sdk-resolution" } +codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" , default-features = false} +codegen-bindings-generator = { path = "codegen-bindings-generator" } codegen-sdk-cst-generator = { path = "codegen-sdk-cst-generator" } codegen-sdk-ast-generator = { path = "codegen-sdk-ast-generator" } codegen-sdk-ts_query = { path = "languages/codegen-sdk-ts_query" } @@ -117,8 +119,8 @@ insta = "1.42.1" prettyplease = "0.2.29" syn = { version = "2.0.98", features = ["proc-macro", "full"] } derive_more = { version = "2.0.1", features = ["debug", "display"] } -salsa = {git = "https://github.com/salsa-rs/salsa", rev ="dbb0e5f6ab2cd61e42b372f333ab694f24141cf1"} -subenum = {git = "https://github.com/mrenow/subenum", branch = "main"} +salsa = {version = "0.19.0"} +subenum = {git = "https://github.com/mrenow/subenum", rev = "d623bc4c0e2a8ab9bc24f255e933411f0c4c9c72"} indicatif-log-bridge = "0.2.3" indicatif = { version = "0.17.11", features = ["rayon"] } crossbeam-channel = "0.5.11" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml new file mode 100644 index 00000000..fcbee561 --- /dev/null +++ b/bindings/python/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "pink-python" +description = "Python bindings for pink" +version = "0.1.0" +edition = "2024" +# [lib] +# crate-type = ["cdylib", "rlib"] + +[dependencies] +codegen-sdk-analyzer = { workspace = true, features = ["python", "typescript", "toml", "yaml", "json"]} +codegen-sdk-resolution = { workspace = true } +pyo3 = { version = "0.23.5", features = ["anyhow", "abi3-py311"] } +pyo3-log = "0.12.1" +pyo3-bytes = "0.1.3" +codegen-sdk-python = { workspace = true } +codegen-sdk-common = { workspace = true } +anyhow = { workspace = true } +# pyo3-stub-gen = "0.7.0" +[build-dependencies] +codegen-bindings-generator = { workspace = true } +codegen-sdk-common = { workspace = true } +env_logger = { workspace = true } +log = { workspace = true } diff --git a/bindings/python/build.rs b/bindings/python/build.rs new file mode 100644 index 00000000..7376e774 --- /dev/null +++ b/bindings/python/build.rs @@ -0,0 +1,11 @@ +use codegen_bindings_generator::generate_python_bindings; +use codegen_sdk_common::language::LANGUAGES; +fn main() { + env_logger::init(); + for language in LANGUAGES.iter() { + generate_python_bindings(&language).unwrap_or_else(|e| { + log::error!("Error generating CST for {}: {}", language.name(), e); + panic!("Error generating CST for {}: {}", language.name(), e); + }); + } +} diff --git a/bindings/python/codegen_sdk_pink.pyi b/bindings/python/codegen_sdk_pink.pyi new file mode 100644 index 00000000..e69de29b diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml new file mode 100644 index 00000000..63dbc3e6 --- /dev/null +++ b/bindings/python/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "codegen-sdk-pink" +dynamic = ["version", "summary", "description", "authors", "license", "readme", "description_content_type", "keywords", "home_page", "author_email", "author", "license", "project_url"] +requires-python = ">=3.11" + +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +features = ["pyo3/extension-module"] +module-name = "codegen_sdk_pink" +[tool.uv] +# Rebuild package when any rust files change +cache-keys = [{file = "pyproject.toml"}, {file = "Cargo.toml"}, {file = "**/*.rs"}] diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs new file mode 100644 index 00000000..da7a200f --- /dev/null +++ b/bindings/python/src/file.rs @@ -0,0 +1,33 @@ +use std::{path::PathBuf, sync::Arc}; + +use pyo3::{prelude::*, sync::GILProtected}; +#[pyclass] +pub struct File { + path: PathBuf, + _codebase: Arc>, +} +impl File { + pub fn new(path: PathBuf, codebase: Arc>) -> Self { + Self { + path, + _codebase: codebase, + } + } +} +#[pymethods] +impl File { + pub fn path(&self) -> &PathBuf { + &self.path + } + pub fn content(&self) -> PyResult { + let content = std::fs::read_to_string(&self.path)?; + Ok(content) + } + pub fn content_bytes(&self) -> PyResult> { + let content = std::fs::read(&self.path)?; + Ok(content) + } + pub fn name(&self) -> String { + self.path.file_name().unwrap().to_str().unwrap().to_string() + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs new file mode 100644 index 00000000..0d28bf4f --- /dev/null +++ b/bindings/python/src/lib.rs @@ -0,0 +1,69 @@ +use std::{path::PathBuf, sync::Arc}; + +use codegen_sdk_common::language::LANGUAGES; +use codegen_sdk_resolution::CodebaseContext; +use file::File; +use pyo3::{prelude::*, sync::GILProtected}; +// use pyo3_stub_gen::{ +// define_stub_info_gatherer, +// derive::{gen_stub_pyclass, gen_stub_pyclass_enum, gen_stub_pymethods}, +// }; +mod file; +pub mod python { + include!(concat!(env!("OUT_DIR"), "/python-bindings.rs")); +} +// #[gen_stub_pyclass_enum] +#[derive(IntoPyObject)] +enum FileEnum { + Python(crate::python::PythonFile), + Unknown(crate::file::File), +} +// #[gen_stub_pyclass] +#[pyclass] +struct Codebase { + codebase: Arc>, +} +// #[gen_stub_pymethods] +#[pymethods] +impl Codebase { + #[new] + fn new(py: Python<'_>, repo_path: PathBuf) -> Self { + let codebase = py.allow_threads(|| codegen_sdk_analyzer::Codebase::new(repo_path)); + Self { + codebase: Arc::new(GILProtected::new(codebase)), + } + } + fn has_file(&self, py: Python<'_>, path: PathBuf) -> PyResult { + let path = path.canonicalize()?; + Ok(self.codebase.get(py).get_file(path).is_some()) + } + fn get_file(&self, py: Python<'_>, path: PathBuf) -> PyResult> { + let path = path.canonicalize()?; + if self.has_file(py, path.clone())? { + for language in LANGUAGES.iter() { + if language + .should_parse(&path) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))? + { + let file = python::PythonFile::new(path, self.codebase.clone()); + return Ok(Some(FileEnum::Python(file))); + } + } + let file = crate::file::File::new(path, self.codebase.clone()); + Ok(Some(FileEnum::Unknown(file))) + } else { + Ok(None) + } + } +} + +#[pymodule] +#[pyo3(name = "codegen_sdk_pink")] +fn codegen_sdk_pink(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let _ = pyo3_log::try_init(); + m.add_class::()?; + python::register_python(m)?; + m.add_class::()?; + Ok(()) +} +// define_stub_info_gatherer!(stub_info); diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock new file mode 100644 index 00000000..c7f8d699 --- /dev/null +++ b/bindings/python/uv.lock @@ -0,0 +1,7 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "pink-python" +version = "0.1.0" +source = { editable = "." } diff --git a/codegen-bindings-generator/Cargo.toml b/codegen-bindings-generator/Cargo.toml new file mode 100644 index 00000000..f6676a0a --- /dev/null +++ b/codegen-bindings-generator/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "codegen-bindings-generator" +description = "Generates bindings for the codegen-sdk" +version = "0.1.0" +edition = "2024" + +[dependencies] +quote = { workspace = true } +codegen-sdk-common = { workspace = true } +syn = { workspace = true } +anyhow = { workspace = true } diff --git a/codegen-bindings-generator/src/lib.rs b/codegen-bindings-generator/src/lib.rs new file mode 100644 index 00000000..1564a4a1 --- /dev/null +++ b/codegen-bindings-generator/src/lib.rs @@ -0,0 +1,35 @@ +#![feature(extend_one)] + +use codegen_sdk_common::{generator::format_code, language::Language}; +use quote::{ToTokens, quote}; +use syn::parse_quote; +mod python; +pub fn generate_python_bindings(language: &Language) -> anyhow::Result<()> { + let imports = quote! { + use pyo3::prelude::*; + use std::path::PathBuf; + use std::sync::Arc; + use pyo3::sync::GILProtected; + use codegen_sdk_resolution::CodebaseContext; + use codegen_sdk_common::traits::CSTNode; + // use pyo3_stub_gen::{impl_stub_type, derive::*}; + + }; + let bindings = python::generator::generate_bindings(language)?; + let ast: syn::File = parse_quote! { + #imports + #(#bindings)* + }; + let out_dir = std::env::var("OUT_DIR")?; + let out_file = format!("{}/{}-bindings.rs", out_dir, language.name()); + std::fs::write(&out_file, ast.to_token_stream().to_string())?; + let ast = format_code(&ast).unwrap_or_else(|_| { + panic!( + "Failed to format bindings for {} at {}", + language.name(), + out_file + ) + }); + std::fs::write(out_file, ast)?; + Ok(()) +} diff --git a/codegen-bindings-generator/src/python.rs b/codegen-bindings-generator/src/python.rs new file mode 100644 index 00000000..2a22e3d0 --- /dev/null +++ b/codegen-bindings-generator/src/python.rs @@ -0,0 +1 @@ +pub mod generator; diff --git a/codegen-bindings-generator/src/python/generator.rs b/codegen-bindings-generator/src/python/generator.rs new file mode 100644 index 00000000..b25b6949 --- /dev/null +++ b/codegen-bindings-generator/src/python/generator.rs @@ -0,0 +1,61 @@ +use codegen_sdk_common::Language; +use quote::format_ident; +use syn::parse_quote; +pub(crate) fn generate_bindings(language: &Language) -> anyhow::Result> { + let mut output = Vec::new(); + let struct_name = format_ident!("{}File", language.struct_name); + + output.push(parse_quote! { + // #[gen_stub_pyclass] + #[pyclass] + pub struct #struct_name { + path: PathBuf, + codebase: Arc>, + } + }); + let package_name = format_ident!("codegen_sdk_{}", language.name()); + let variant_name = format_ident!("{}", language.struct_name); + output.push(parse_quote! { + impl #struct_name { + pub fn new(path: PathBuf, codebase: Arc>) -> Self { + Self { path, codebase } + } + fn file<'db>(&'db self, py: Python<'db>) -> PyResult<&'db #package_name::ast::#struct_name<'db>>{ + let codebase = self.codebase.get(py); + if let codegen_sdk_analyzer::ParsedFile::#variant_name(file) = codebase.get_file(self.path.clone()).unwrap() { + Ok(file) + } else { + Err(pyo3::exceptions::PyValueError::new_err("File not found")) + } + } + } + }); + output.push(parse_quote! { + #[pymethods] + impl #struct_name { + pub fn content(&self, py: Python<'_>) -> PyResult { + let codebase = self.codebase.get(py); + let file = self.file(py)?.root(codebase.db()); + Ok(file.source()) + } + pub fn content_bytes(&self, py: Python<'_>) -> PyResult { + let codebase = self.codebase.get(py); + let file = self.file(py)?.root(codebase.db()); + Ok(pyo3_bytes::PyBytes::new(file.text())) + } + } + }); + let language_name = language.name(); + let register_name = format_ident!("register_{}", language_name); + + output.push(parse_quote! { + pub fn #register_name(parent_module: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(parent_module.py(), #language_name)?; + pyo3_log::init(); + child_module.add_class::<#struct_name>()?; + parent_module.add_submodule(&child_module)?; + Ok(()) + } + }); + Ok(output) +} diff --git a/codegen-sdk-analyzer/src/codebase/parser.rs b/codegen-sdk-analyzer/src/codebase/parser.rs index 3f2c4900..598f49f4 100644 --- a/codegen-sdk-analyzer/src/codebase/parser.rs +++ b/codegen-sdk-analyzer/src/codebase/parser.rs @@ -135,10 +135,9 @@ pub fn parse_files<'db>( #[cfg(feature = "serialization")] cache: &'db Cache, files_to_parse: FilesToParse, ) -> () { - rayon::ThreadPoolBuilder::new() + let _ = rayon::ThreadPoolBuilder::new() .stack_size(1024 * 1024 * 1024 * 10) - .build_global() - .unwrap(); + .build_global(); log_languages(); #[cfg(feature = "serialization")] let cache = Cache::new().unwrap(); diff --git a/codegen-sdk-common/Cargo.toml b/codegen-sdk-common/Cargo.toml index d8fcd0e1..f7ca7b65 100644 --- a/codegen-sdk-common/Cargo.toml +++ b/codegen-sdk-common/Cargo.toml @@ -23,7 +23,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } anyhow = { workspace = true } convert_case = { workspace = true } -tree-sitter-query = {git = "https://github.com/tree-sitter-grammars/tree-sitter-query", optional = true} +tree-sitter-query = {git = "https://github.com/tree-sitter-grammars/tree-sitter-query", rev = "0555ac0da902abff06076e40501102cee3ba68bd", optional = true} phf = { version = "0.11.3", features = ["macros"] } rkyv = { workspace = true } xdg = "2.5.2" diff --git a/codegen-sdk-common/src/language.rs b/codegen-sdk-common/src/language.rs index d7c6ffa1..12b29c24 100644 --- a/codegen-sdk-common/src/language.rs +++ b/codegen-sdk-common/src/language.rs @@ -1,4 +1,4 @@ -use std::{hash::Hash, num::NonZeroU16, sync::Arc}; +use std::{hash::Hash, num::NonZeroU16, path::PathBuf, sync::Arc}; use convert_case::{Case, Casing}; use mockall::automock; @@ -89,6 +89,15 @@ impl Language { .find(|node| normalize_type_name(&node.type_name, node.named) == struct_name) .cloned() } + pub fn should_parse(&self, file_path: &PathBuf) -> Result { + Ok(self.file_extensions.contains( + &file_path + .extension() + .ok_or(ParseError::Miscelaneous)? + .to_str() + .ok_or(ParseError::Miscelaneous)?, + )) + } } #[cfg(feature = "go")] pub mod go; diff --git a/codegen-sdk-cst/Cargo.toml b/codegen-sdk-cst/Cargo.toml index e4c13402..a0f7bca0 100644 --- a/codegen-sdk-cst/Cargo.toml +++ b/codegen-sdk-cst/Cargo.toml @@ -4,19 +4,12 @@ version = "0.1.0" edition = "2024" [dependencies] -tree-sitter = { workspace = true } bytes = { workspace = true } codegen-sdk-common = { path = "../codegen-sdk-common" } -convert_case = { workspace = true } -rkyv = { workspace = true } -log = { workspace = true } salsa = { workspace = true } dashmap = "6.1.0" thiserror = {workspace = true} indextree = { workspace = true } -[dev-dependencies] -tempfile = { workspace = true } -test-log = { workspace = true } [features] serialization = ["codegen-sdk-common/serialization"] diff --git a/codegen-sdk-cst/src/language.rs b/codegen-sdk-cst/src/language.rs index de3d9905..38452070 100644 --- a/codegen-sdk-cst/src/language.rs +++ b/codegen-sdk-cst/src/language.rs @@ -70,12 +70,6 @@ pub trait CSTLanguage { } fn should_parse(file_path: &PathBuf) -> Result { - Ok(Self::language().file_extensions.contains( - &file_path - .extension() - .ok_or(ParseError::Miscelaneous)? - .to_str() - .ok_or(ParseError::Miscelaneous)?, - )) + Self::language().should_parse(file_path) } } diff --git a/codegen-sdk-resolution/Cargo.toml b/codegen-sdk-resolution/Cargo.toml index 130b35de..753f7eae 100644 --- a/codegen-sdk-resolution/Cargo.toml +++ b/codegen-sdk-resolution/Cargo.toml @@ -6,11 +6,8 @@ edition = "2024" [dependencies] salsa = { workspace = true } -log = {workspace = true} -codegen-sdk-ast = { workspace = true } codegen-sdk-common = { workspace = true } anyhow = { workspace = true } indicatif = { workspace = true } ambassador = { workspace = true } codegen-sdk-cst = { workspace = true } -smallvec = { workspace = true }