feat: more languages (#25)

* Update features

* Add a bunch of languages

* Disable languages

* Enable ts

Author: bagel897

Cargo.lock

@@ -15,6 +15,22 @@ log = { workspace = true }
 rayon = { workspace = true}
 sysinfo = "0.33.1"
 rkyv.workspace = true
+[features]
+# python = [ "codegen-sdk-cst/python"] # TODO: Add python support
+typescript = [ "codegen-sdk-cst/typescript"]
+tsx = [ "codegen-sdk-cst/tsx"]
+jsx = [ "codegen-sdk-cst/jsx"]
+javascript = [ "codegen-sdk-cst/typescript"]
+json = [ "codegen-sdk-cst/json"]
+java = [ "codegen-sdk-cst/java"]
+# ruby = [ "codegen-sdk-cst/ruby"]
+# rust = [ "codegen-sdk-cst/rust"]
+go = [ "codegen-sdk-cst/go"]
+# markdown = [ "codegen-sdk-cst/markdown"]
+yaml = [ "codegen-sdk-cst/yaml"]
+toml = [ "codegen-sdk-cst/toml"]
+ts_query = []
+default = ["json", "ts_query", "toml"]
 [dev-dependencies]
 test-log = { workspace = true }
 [workspace]
@@ -35,6 +51,12 @@ tree-sitter-typescript = "0.23.2"
 tree-sitter-javascript = "0.23.1"
 tree-sitter-json = "0.24.0"
 tree-sitter-java = "0.23.5"
+tree-sitter-ruby = "0.23.1"
+tree-sitter-rust = "0.23.2"
+tree-sitter-go = "0.23.4"
+tree-sitter-md = "0.3.2"
+tree-sitter-yaml = "0.7.0"
+tree-sitter-toml-ng = "0.7.0"
 bytes = "1.10.0"
 convert_case = "0.7.1"
 serde = { version = "1.0.217", features = ["derive"] }
@@ -45,7 +67,7 @@ test-log = "0.2.17"
 enum_delegate = "0.2.0"
 mockall = "0.13.1"
 codegen-sdk-common = { path = "codegen-sdk-common" }
-codegen-sdk-cst = { path = "codegen-sdk-cst", features = ["typescript"]}
+codegen-sdk-cst = { path = "codegen-sdk-cst"}
 codegen-sdk-ast = { path = "codegen-sdk-ast" }
 
 [profile.dev.package]

Cargo.toml

@@ -4,11 +4,27 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
-codegen-sdk-cst = { workspace = true }
+codegen-sdk-cst = { workspace = true, features = ["json"]}
 codegen-sdk-common = { workspace = true }
 
 [build-dependencies]
 codegen-sdk-common = { workspace = true }
 env_logger = { workspace = true }
 rayon = { workspace = true }
 codegen-sdk-ast-generator = { path = "../codegen-sdk-ast-generator"}
+[features]
+python = [ "codegen-sdk-cst/python"]
+typescript = [ "codegen-sdk-cst/typescript"]
+tsx = [ "codegen-sdk-cst/tsx"]
+jsx = [ "codegen-sdk-cst/jsx"]
+javascript = [ "codegen-sdk-cst/typescript"]
+json = [ "codegen-sdk-cst/json"]
+java = [ "codegen-sdk-cst/java"]
+rust = [ "codegen-sdk-cst/rust"]
+go = [ "codegen-sdk-cst/go"]
+ruby = [ "codegen-sdk-cst/ruby"]
+yaml = [ "codegen-sdk-cst/yaml"]
+toml = [ "codegen-sdk-cst/toml"]
+markdown = [ "codegen-sdk-cst/markdown"]
+ts_query = []
+default = ["json", "ts_query"]

codegen-sdk-ast/Cargo.toml

@@ -10,4 +10,4 @@ impl<T: File> Named for T {
     }
 }
 use std::path::PathBuf;
-include!(concat!(env!("OUT_DIR"), "/typescript.rs"));
+include!(concat!(env!("OUT_DIR"), "/json.rs"));

codegen-sdk-ast/src/lib.rs

@@ -11,6 +11,12 @@ tree-sitter-typescript = { workspace = true, optional = true }
 tree-sitter-javascript = { workspace = true, optional = true }
 tree-sitter-json = { workspace = true, optional = true }
 tree-sitter-java = { workspace = true, optional = true }
+tree-sitter-rust = { workspace = true, optional = true }
+tree-sitter-go = { workspace = true, optional = true }
+tree-sitter-ruby = { workspace = true, optional = true }
+tree-sitter-yaml = { workspace = true, optional = true }
+tree-sitter-toml-ng = { workspace = true, optional = true }
+tree-sitter-md = { workspace = true, optional = true }
 lazy_static = "1.5.0"
 thiserror = "2.0.11"
 serde = { workspace = true, features = ["derive"] }
@@ -35,6 +41,12 @@ python = ["dep:tree-sitter-python"]
 json = ["dep:tree-sitter-json"]
 java = ["dep:tree-sitter-java"]
 typescript = ["dep:tree-sitter-typescript", "dep:tree-sitter-javascript"]
+rust = ["dep:tree-sitter-rust"]
+go = ["dep:tree-sitter-go"]
+ruby = ["dep:tree-sitter-ruby"]
+yaml = ["dep:tree-sitter-yaml"]
+toml = ["dep:tree-sitter-toml-ng"]
+markdown = ["dep:tree-sitter-md"]
 ts_query = ["dep:tree-sitter-query"]
 default = ["all"]
-all = ["python", "typescript", "json", "java", "ts_query"]
+all = ["python", "typescript", "json", "java", "ts_query", "ruby", "rust", "go", "yaml", "toml"]

codegen-sdk-common/Cargo.toml

@@ -51,7 +51,7 @@ impl Language {
         self.nodes()
             .iter()
             .find(|node| node.root)
-            .unwrap()
+            .unwrap_or_else(|| panic!("No root node found for language: {}", self.name))
             .type_name
             .to_case(Case::Pascal)
     }
@@ -71,6 +71,8 @@ impl Language {
         self.name
     }
 }
+#[cfg(feature = "go")]
+pub mod go;
 #[cfg(feature = "java")]
 pub mod java;
 #[cfg(feature = "typescript")]
@@ -79,14 +81,24 @@ pub mod javascript;
 pub mod json;
 #[cfg(feature = "typescript")]
 pub mod jsx;
+#[cfg(feature = "markdown")]
+pub mod markdown;
 #[cfg(feature = "python")]
 pub mod python;
+#[cfg(feature = "ruby")]
+pub mod ruby;
+#[cfg(feature = "rust")]
+pub mod rust;
+#[cfg(feature = "toml")]
+pub mod toml;
 #[cfg(feature = "ts_query")]
 pub mod ts_query;
 #[cfg(feature = "typescript")]
 pub mod tsx;
 #[cfg(feature = "typescript")]
 pub mod typescript;
+#[cfg(feature = "yaml")]
+pub mod yaml;
 lazy_static! {
     pub static ref LANGUAGES: Vec<&'static Language> = vec![
         #[cfg(feature = "python")]
@@ -99,6 +111,18 @@ lazy_static! {
         &jsx::JSX,
         #[cfg(feature = "typescript")]
         &javascript::Javascript,
+        #[cfg(feature = "rust")]
+        &rust::Rust,
+        #[cfg(feature = "go")]
+        &go::Go,
+        #[cfg(feature = "ruby")]
+        &ruby::Ruby,
+        #[cfg(feature = "yaml")]
+        &yaml::Yaml,
+        #[cfg(feature = "toml")]
+        &toml::TOML,
+        #[cfg(feature = "markdown")]
+        &markdown::Markdown,
         #[cfg(feature = "json")]
         &json::JSON,
         #[cfg(feature = "java")]

codegen-sdk-common/src/language.rs

@@ -0,0 +1,12 @@
+use super::Language;
+lazy_static! {
+    pub static ref Markdown: Language = Language::new(
+        "markdown",
+        "Markdown",
+        tree_sitter_md::NODE_TYPES_BLOCK,
+        &["md"],
+        tree_sitter_md::LANGUAGE.into(),
+        "",
+    )
+    .unwrap();
+}

codegen-sdk-common/src/language/markdown.rs

@@ -0,0 +1,12 @@
+use super::Language;
+lazy_static! {
+    pub static ref TOML: Language = Language::new(
+        "toml",
+        "TOML",
+        tree_sitter_toml_ng::NODE_TYPES,
+        &["toml"],
+        tree_sitter_toml_ng::LANGUAGE.into(),
+        "",
+    )
+    .unwrap();
+}

codegen-sdk-common/src/language/toml.rs

@@ -0,0 +1,12 @@
+use super::Language;
+lazy_static! {
+    pub static ref Yaml: Language = Language::new(
+        "yaml",
+        "Yaml",
+        tree_sitter_yaml::NODE_TYPES,
+        &["yaml", "yml"],
+        tree_sitter_yaml::LANGUAGE.into(),
+        "",
+    )
+    .unwrap();
+}

codegen-sdk-common/src/language/yaml.rs

@@ -36,6 +36,7 @@ static MAPPINGS: phf::Map<char, &'static str> = phf_map! {
     '"' => "DoubleQuote",
     '|' => "Pipe",
     ';' => "Semicolon",
+    '\0' => "Null",
 };
 pub fn normalize_field_name(field_name: &str) -> String {
     if field_name == "type" {