diff --git a/Cargo.lock b/Cargo.lock index ac97cc03..8285e428 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,13 +285,19 @@ dependencies = [ "test-log", "thiserror", "tree-sitter", + "tree-sitter-go", "tree-sitter-java", "tree-sitter-javascript", "tree-sitter-json", "tree-sitter-language", + "tree-sitter-md", "tree-sitter-python", "tree-sitter-query", + "tree-sitter-ruby", + "tree-sitter-rust", + "tree-sitter-toml-ng", "tree-sitter-typescript", + "tree-sitter-yaml", "xdg", "zstd", ] @@ -1638,6 +1644,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-go" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13d476345220dbe600147dd444165c5791bf85ef53e28acbedd46112ee18431" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-java" version = "0.23.5" @@ -1674,6 +1690,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38eee4db33814de3d004de9d8d825627ed3320d0989cce0dea30efaf5be4736c" +[[package]] +name = "tree-sitter-md" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f968c22a01010b83fc960455ae729db08dbeb6388617d9113897cb9204b030" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-python" version = "0.23.6" @@ -1693,6 +1719,36 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-ruby" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4d64d449ca63e683c562c7743946a646671ca23947b9c925c0cfbe65051a4af" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-typescript" version = "0.23.2" @@ -1703,6 +1759,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-yaml" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c99f2b92b677f1a18b6b232fa9329afb5758118238a7d0b29cae324ef50d5e" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "typenum" version = "1.18.0" diff --git a/Cargo.toml b/Cargo.toml index 5dc819a4..6565262b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,22 @@ log = { workspace = true } rayon = { workspace = true} sysinfo = "0.33.1" rkyv.workspace = true +[features] +# python = [ "codegen-sdk-cst/python"] # TODO: Add python support +typescript = [ "codegen-sdk-cst/typescript"] +tsx = [ "codegen-sdk-cst/tsx"] +jsx = [ "codegen-sdk-cst/jsx"] +javascript = [ "codegen-sdk-cst/typescript"] +json = [ "codegen-sdk-cst/json"] +java = [ "codegen-sdk-cst/java"] +# ruby = [ "codegen-sdk-cst/ruby"] +# rust = [ "codegen-sdk-cst/rust"] +go = [ "codegen-sdk-cst/go"] +# markdown = [ "codegen-sdk-cst/markdown"] +yaml = [ "codegen-sdk-cst/yaml"] +toml = [ "codegen-sdk-cst/toml"] +ts_query = [] +default = ["json", "ts_query", "toml"] [dev-dependencies] test-log = { workspace = true } [workspace] @@ -35,6 +51,12 @@ tree-sitter-typescript = "0.23.2" tree-sitter-javascript = "0.23.1" tree-sitter-json = "0.24.0" tree-sitter-java = "0.23.5" +tree-sitter-ruby = "0.23.1" +tree-sitter-rust = "0.23.2" +tree-sitter-go = "0.23.4" +tree-sitter-md = "0.3.2" +tree-sitter-yaml = "0.7.0" +tree-sitter-toml-ng = "0.7.0" bytes = "1.10.0" convert_case = "0.7.1" serde = { version = "1.0.217", features = ["derive"] } @@ -45,7 +67,7 @@ test-log = "0.2.17" enum_delegate = "0.2.0" mockall = "0.13.1" codegen-sdk-common = { path = "codegen-sdk-common" } -codegen-sdk-cst = { path = "codegen-sdk-cst", features = ["typescript"]} +codegen-sdk-cst = { path = "codegen-sdk-cst"} codegen-sdk-ast = { path = "codegen-sdk-ast" } [profile.dev.package] diff --git a/codegen-sdk-ast/Cargo.toml b/codegen-sdk-ast/Cargo.toml index 7d46c2e7..ef580ae8 100644 --- a/codegen-sdk-ast/Cargo.toml +++ b/codegen-sdk-ast/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -codegen-sdk-cst = { workspace = true } +codegen-sdk-cst = { workspace = true, features = ["json"]} codegen-sdk-common = { workspace = true } [build-dependencies] @@ -12,3 +12,19 @@ codegen-sdk-common = { workspace = true } env_logger = { workspace = true } rayon = { workspace = true } codegen-sdk-ast-generator = { path = "../codegen-sdk-ast-generator"} +[features] +python = [ "codegen-sdk-cst/python"] +typescript = [ "codegen-sdk-cst/typescript"] +tsx = [ "codegen-sdk-cst/tsx"] +jsx = [ "codegen-sdk-cst/jsx"] +javascript = [ "codegen-sdk-cst/typescript"] +json = [ "codegen-sdk-cst/json"] +java = [ "codegen-sdk-cst/java"] +rust = [ "codegen-sdk-cst/rust"] +go = [ "codegen-sdk-cst/go"] +ruby = [ "codegen-sdk-cst/ruby"] +yaml = [ "codegen-sdk-cst/yaml"] +toml = [ "codegen-sdk-cst/toml"] +markdown = [ "codegen-sdk-cst/markdown"] +ts_query = [] +default = ["json", "ts_query"] diff --git a/codegen-sdk-ast/src/lib.rs b/codegen-sdk-ast/src/lib.rs index a1a6eada..006cdd11 100644 --- a/codegen-sdk-ast/src/lib.rs +++ b/codegen-sdk-ast/src/lib.rs @@ -10,4 +10,4 @@ impl Named for T { } } use std::path::PathBuf; -include!(concat!(env!("OUT_DIR"), "/typescript.rs")); +include!(concat!(env!("OUT_DIR"), "/json.rs")); diff --git a/codegen-sdk-common/Cargo.toml b/codegen-sdk-common/Cargo.toml index 4477e8cc..0edbff41 100644 --- a/codegen-sdk-common/Cargo.toml +++ b/codegen-sdk-common/Cargo.toml @@ -11,6 +11,12 @@ tree-sitter-typescript = { workspace = true, optional = true } tree-sitter-javascript = { workspace = true, optional = true } tree-sitter-json = { workspace = true, optional = true } tree-sitter-java = { workspace = true, optional = true } +tree-sitter-rust = { workspace = true, optional = true } +tree-sitter-go = { workspace = true, optional = true } +tree-sitter-ruby = { workspace = true, optional = true } +tree-sitter-yaml = { workspace = true, optional = true } +tree-sitter-toml-ng = { workspace = true, optional = true } +tree-sitter-md = { workspace = true, optional = true } lazy_static = "1.5.0" thiserror = "2.0.11" serde = { workspace = true, features = ["derive"] } @@ -35,6 +41,12 @@ python = ["dep:tree-sitter-python"] json = ["dep:tree-sitter-json"] java = ["dep:tree-sitter-java"] typescript = ["dep:tree-sitter-typescript", "dep:tree-sitter-javascript"] +rust = ["dep:tree-sitter-rust"] +go = ["dep:tree-sitter-go"] +ruby = ["dep:tree-sitter-ruby"] +yaml = ["dep:tree-sitter-yaml"] +toml = ["dep:tree-sitter-toml-ng"] +markdown = ["dep:tree-sitter-md"] ts_query = ["dep:tree-sitter-query"] default = ["all"] -all = ["python", "typescript", "json", "java", "ts_query"] +all = ["python", "typescript", "json", "java", "ts_query", "ruby", "rust", "go", "yaml", "toml"] diff --git a/codegen-sdk-common/src/language.rs b/codegen-sdk-common/src/language.rs index 7b62e254..a9f0e134 100644 --- a/codegen-sdk-common/src/language.rs +++ b/codegen-sdk-common/src/language.rs @@ -51,7 +51,7 @@ impl Language { self.nodes() .iter() .find(|node| node.root) - .unwrap() + .unwrap_or_else(|| panic!("No root node found for language: {}", self.name)) .type_name .to_case(Case::Pascal) } @@ -71,6 +71,8 @@ impl Language { self.name } } +#[cfg(feature = "go")] +pub mod go; #[cfg(feature = "java")] pub mod java; #[cfg(feature = "typescript")] @@ -79,14 +81,24 @@ pub mod javascript; pub mod json; #[cfg(feature = "typescript")] pub mod jsx; +#[cfg(feature = "markdown")] +pub mod markdown; #[cfg(feature = "python")] pub mod python; +#[cfg(feature = "ruby")] +pub mod ruby; +#[cfg(feature = "rust")] +pub mod rust; +#[cfg(feature = "toml")] +pub mod toml; #[cfg(feature = "ts_query")] pub mod ts_query; #[cfg(feature = "typescript")] pub mod tsx; #[cfg(feature = "typescript")] pub mod typescript; +#[cfg(feature = "yaml")] +pub mod yaml; lazy_static! { pub static ref LANGUAGES: Vec<&'static Language> = vec![ #[cfg(feature = "python")] @@ -99,6 +111,18 @@ lazy_static! { &jsx::JSX, #[cfg(feature = "typescript")] &javascript::Javascript, + #[cfg(feature = "rust")] + &rust::Rust, + #[cfg(feature = "go")] + &go::Go, + #[cfg(feature = "ruby")] + &ruby::Ruby, + #[cfg(feature = "yaml")] + &yaml::Yaml, + #[cfg(feature = "toml")] + &toml::TOML, + #[cfg(feature = "markdown")] + &markdown::Markdown, #[cfg(feature = "json")] &json::JSON, #[cfg(feature = "java")] diff --git a/codegen-sdk-common/src/language/markdown.rs b/codegen-sdk-common/src/language/markdown.rs new file mode 100644 index 00000000..91850eee --- /dev/null +++ b/codegen-sdk-common/src/language/markdown.rs @@ -0,0 +1,12 @@ +use super::Language; +lazy_static! { + pub static ref Markdown: Language = Language::new( + "markdown", + "Markdown", + tree_sitter_md::NODE_TYPES_BLOCK, + &["md"], + tree_sitter_md::LANGUAGE.into(), + "", + ) + .unwrap(); +} diff --git a/codegen-sdk-common/src/language/toml.rs b/codegen-sdk-common/src/language/toml.rs new file mode 100644 index 00000000..437bd21b --- /dev/null +++ b/codegen-sdk-common/src/language/toml.rs @@ -0,0 +1,12 @@ +use super::Language; +lazy_static! { + pub static ref TOML: Language = Language::new( + "toml", + "TOML", + tree_sitter_toml_ng::NODE_TYPES, + &["toml"], + tree_sitter_toml_ng::LANGUAGE.into(), + "", + ) + .unwrap(); +} diff --git a/codegen-sdk-common/src/language/yaml.rs b/codegen-sdk-common/src/language/yaml.rs new file mode 100644 index 00000000..ccfefbf7 --- /dev/null +++ b/codegen-sdk-common/src/language/yaml.rs @@ -0,0 +1,12 @@ +use super::Language; +lazy_static! { + pub static ref Yaml: Language = Language::new( + "yaml", + "Yaml", + tree_sitter_yaml::NODE_TYPES, + &["yaml", "yml"], + tree_sitter_yaml::LANGUAGE.into(), + "", + ) + .unwrap(); +} diff --git a/codegen-sdk-common/src/naming.rs b/codegen-sdk-common/src/naming.rs index 487dde22..1e31138d 100644 --- a/codegen-sdk-common/src/naming.rs +++ b/codegen-sdk-common/src/naming.rs @@ -36,6 +36,7 @@ static MAPPINGS: phf::Map = phf_map! { '"' => "DoubleQuote", '|' => "Pipe", ';' => "Semicolon", + '\0' => "Null", }; pub fn normalize_field_name(field_name: &str) -> String { if field_name == "type" { diff --git a/codegen-sdk-cst/Cargo.toml b/codegen-sdk-cst/Cargo.toml index 99dbacc6..2a91f166 100644 --- a/codegen-sdk-cst/Cargo.toml +++ b/codegen-sdk-cst/Cargo.toml @@ -32,5 +32,11 @@ jsx = [ "codegen-sdk-common/typescript"] javascript = [ "codegen-sdk-common/typescript"] json = [ "codegen-sdk-common/json"] java = [ "codegen-sdk-common/java"] +rust = [ "codegen-sdk-common/rust"] +go = [ "codegen-sdk-common/go"] +ruby = [ "codegen-sdk-common/ruby"] +yaml = [ "codegen-sdk-common/yaml"] +toml = [ "codegen-sdk-common/toml"] +markdown = [ "codegen-sdk-common/markdown"] ts_query = [] -default = ["json", "ts_query"] +default = ["json", "ts_query", "toml", "typescript"]