basic AST generation support and other fixes (#10)

* fix parsing issues

* Add tags and root node support

* wip: ast

* wip: ast

* idk

* idk

* query lang

* Automate cst addition of new languages

* wip: query support/debugging

* query parsing pt1

* move naming

* kind id support

* Automated pre-commit update

Author: bagel897

Cargo.lock

@@ -17,7 +17,7 @@ sysinfo = "0.33.1"
 [workspace]
 members = [
   "codegen-sdk-analyzer",
-  "codegen-sdk-ast",
+  "codegen-sdk-ast", "codegen-sdk-ast-generator",
   "codegen-sdk-common",
   "codegen-sdk-cst",
   "codegen-sdk-cst-generator", "codegen-sdk-macros",
@@ -37,3 +37,4 @@ bytes = "1.10.0"
 convert_case = "0.7.1"
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.138"
+anyhow = { version = "1.0.95", features = ["backtrace"] }

Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "codegen-sdk-ast-generator"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+codegen-sdk-common = { path = "../codegen-sdk-common"}
+anyhow = { workspace = true }

codegen-sdk-ast-generator/Cargo.toml

@@ -0,0 +1,39 @@
+use codegen_sdk_common::language::Language;
+pub fn generate_ast(language: &Language) -> anyhow::Result<String> {
+    let content = format!(
+        "
+    #[derive(Debug)]
+    pub struct {language_struct_name}File {{
+        node: {language_name}::{root_node_name},
+        path: PathBuf
+    }}
+    impl File for {language_struct_name}File {{
+        fn path(&self) -> &PathBuf {{
+            &self.path
+        }}
+    }}
+    impl HasNode for {language_struct_name}File {{
+        type Node = {language_name}::{root_node_name};
+        fn node(&self) -> &Self::Node {{
+            &self.node
+        }}
+    }}
+
+    ",
+        language_struct_name = language.struct_name,
+        language_name = language.name,
+        root_node_name = language.root_node()
+    );
+    // for (name, query) in language.definitions() {
+    //     content.push_str(&format!("
+    //     impl {language_struct_name}File {{
+    //         pub fn {name}(&self) -> {language_struct_name}File {{
+    //             {language_struct_name}File {{
+    //                 node: self.node.children().find(|node| node.type_name == \"{name}\").unwrap(),
+    //                 path: self.path.clone()
+    //             }}
+    //         }}
+    //     "));
+    // }
+    Ok(content)
+}

codegen-sdk-ast-generator/src/generator.rs

@@ -0,0 +1,9 @@
+use codegen_sdk_common::language::Language;
+mod generator;
+pub fn generate_ast(language: &Language) -> anyhow::Result<()> {
+    let ast = generator::generate_ast(language)?;
+    let out_dir = std::env::var("OUT_DIR")?;
+    let out_file = format!("{}/{}.rs", out_dir, language.name);
+    std::fs::write(out_file, ast)?;
+    Ok(())
+}

codegen-sdk-ast-generator/src/lib.rs

@@ -5,3 +5,10 @@ edition = "2021"
 
 [dependencies]
 codegen-sdk-cst = { path = "../codegen-sdk-cst" }
+codegen-sdk-common = { path = "../codegen-sdk-common"}
+
+[build-dependencies]
+codegen-sdk-common = { path = "../codegen-sdk-common"}
+env_logger = { workspace = true }
+rayon = { workspace = true }
+codegen-sdk-ast-generator = { path = "../codegen-sdk-ast-generator"}

codegen-sdk-ast/Cargo.toml

@@ -0,0 +1,9 @@
+use codegen_sdk_ast_generator::generate_ast;
+use codegen_sdk_common::language::LANGUAGES;
+use rayon::prelude::*;
+fn main() {
+    env_logger::init();
+    LANGUAGES.par_iter().for_each(|language| {
+        generate_ast(language).unwrap();
+    });
+}

codegen-sdk-ast/build.rs

@@ -1 +1,12 @@
+use codegen_sdk_common::{File, HasNode};
 pub use codegen_sdk_cst::*;
+pub trait Named {
+    fn name(&self) -> &str;
+}
+impl<T: File> Named for T {
+    fn name(&self) -> &str {
+        self.path().file_name().unwrap().to_str().unwrap()
+    }
+}
+use std::path::PathBuf;
+include!(concat!(env!("OUT_DIR"), "/typescript.rs"));

codegen-sdk-ast/src/lib.rs

@@ -0,0 +1,6 @@
+pub use codegen_sdk_common::language::LANGUAGES;
+fn main() {
+    for language in LANGUAGES.iter() {
+        println!("{}", language.tags_query);
+    }
+}

codegen-sdk-ast/src/main.rs

@@ -14,9 +14,18 @@ tree-sitter-json = { workspace = true, optional = true }
 tree-sitter-java = { workspace = true, optional = true }
 lazy_static = "1.5.0"
 thiserror = "2.0.11"
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+anyhow = { workspace = true }
+convert_case = { workspace = true }
+tree-sitter-query = {git = "https://github.com/tree-sitter-grammars/tree-sitter-query", optional = true}
+tree-sitter-language = "0.1.4"
+phf = { version = "0.11.3", features = ["macros"] }
 [features]
 python = ["dep:tree-sitter-python"]
 json = ["dep:tree-sitter-json"]
 java = ["dep:tree-sitter-java"]
 typescript = ["dep:tree-sitter-typescript", "dep:tree-sitter-javascript"]
-all = ["python", "typescript", "json", "java"]
+ts_query = ["dep:tree-sitter-query"]
+default = ["all"]
+all = ["python", "typescript", "json", "java", "ts_query"]