refactor: everything (#6)

bagel897 · web-flow · commit 22740d48a71a · 2025-02-13T15:07:18.000-08:00
* refactor: everything

* Fixes

* Fix warnings
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -6,7 +6,8 @@ edition = "2021"
 [dependencies]
 clap = { version = "4.5.28", features = ["derive"] }
 codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" }
-codegen-sdk-cst = { path = "codegen-sdk-cst" }
+codegen-sdk-cst = { path = "codegen-sdk-cst" , features = ["typescript", "javascript", "tsx", "jsx"]}
+codegen-sdk-common = { path = "codegen-sdk-common" }
 crossbeam = "0.8.4"
 env_logger = "0.11.6"
 glob = "0.3.2"
@@ -19,14 +20,15 @@ members = [
   "codegen-sdk-ast",
   "codegen-sdk-common",
   "codegen-sdk-cst",
-  "codegen-sdk-cst-generator",
+  "codegen-sdk-cst-generator", "codegen-sdk-macros",
 ]
 [workspace.dependencies]
 log = "0.4.25"
 ouroboros = "0.18.5"
 tree-sitter = "0.25.1"
 tree-sitter-python = "0.23.6"
 tree-sitter-typescript = "0.23.2"
+tree-sitter-javascript = "0.23.1"
 bytes = "1.10.0"
 convert_case = "0.7.1"
 serde = { version = "1.0.217", features = ["derive"] }
diff --git a/codegen-sdk-common/Cargo.toml b/codegen-sdk-common/Cargo.toml
@@ -7,3 +7,10 @@ edition = "2021"
 bytes = { workspace = true }
 tree-sitter = { workspace = true }
 ouroboros = { workspace = true }
+tree-sitter-python = { workspace = true, optional = true }
+tree-sitter-typescript = { workspace = true, optional = true }
+tree-sitter-javascript = { workspace = true, optional = true }
+lazy_static = "1.5.0"
+[features]
+python = ["dep:tree-sitter-python"]
+typescript = ["dep:tree-sitter-typescript", "dep:tree-sitter-javascript"]
diff --git a/codegen-sdk-common/src/language.rs b/codegen-sdk-common/src/language.rs
@@ -0,0 +1,41 @@
+use tree_sitter::{LanguageError, Parser};
+
+pub struct Language {
+    pub name: &'static str,
+    pub struct_name: &'static str,
+    pub node_types: &'static str,
+    pub file_extensions: &'static [&'static str],
+    pub tree_sitter_language: tree_sitter::Language,
+}
+impl Language {
+    pub fn parse_tree_sitter(&self, content: &str) -> Result<tree_sitter::Tree, LanguageError> {
+        let mut parser = Parser::new();
+        parser.set_language(&self.tree_sitter_language)?;
+        let tree = parser.parse(content, None).unwrap();
+        Ok(tree)
+    }
+}
+#[cfg(feature = "typescript")]
+pub mod javascript;
+#[cfg(feature = "typescript")]
+pub mod jsx;
+#[cfg(feature = "python")]
+pub mod python;
+#[cfg(feature = "typescript")]
+pub mod tsx;
+#[cfg(feature = "typescript")]
+pub mod typescript;
+lazy_static! {
+    pub static ref LANGUAGES: Vec<&'static Language> = vec![
+        #[cfg(feature = "python")]
+        &python::Python,
+        #[cfg(feature = "typescript")]
+        &typescript::Typescript,
+        #[cfg(feature = "typescript")]
+        &tsx::TSX,
+        #[cfg(feature = "typescript")]
+        &jsx::JSX,
+        #[cfg(feature = "typescript")]
+        &javascript::Javascript,
+    ];
+}
diff --git a/codegen-sdk-common/src/language/javascript.rs b/codegen-sdk-common/src/language/javascript.rs
@@ -0,0 +1,11 @@
+use super::Language;
+
+lazy_static! {
+    pub static ref Javascript: Language = Language {
+        name: "javascript",
+        struct_name: "Javascript",
+        node_types: tree_sitter_javascript::NODE_TYPES,
+        file_extensions: &["js"],
+        tree_sitter_language: tree_sitter_javascript::LANGUAGE.into(),
+    };
+}
diff --git a/codegen-sdk-common/src/language/jsx.rs b/codegen-sdk-common/src/language/jsx.rs
@@ -0,0 +1,10 @@
+use super::Language;
+lazy_static! {
+    pub static ref JSX: Language = Language {
+        name: "jsx",
+        struct_name: "JSX",
+        node_types: tree_sitter_typescript::TSX_NODE_TYPES,
+        file_extensions: &["jsx"],
+        tree_sitter_language: tree_sitter_typescript::LANGUAGE_TSX.into(),
+    };
+}
diff --git a/codegen-sdk-common/src/language/python.rs b/codegen-sdk-common/src/language/python.rs
@@ -0,0 +1,10 @@
+use super::Language;
+lazy_static! {
+    pub static ref Python: Language = Language {
+        name: "python",
+        struct_name: "Python",
+        node_types: tree_sitter_python::NODE_TYPES,
+        file_extensions: &["py"],
+        tree_sitter_language: tree_sitter_python::LANGUAGE.into(),
+    };
+}
diff --git a/codegen-sdk-common/src/language/tsx.rs b/codegen-sdk-common/src/language/tsx.rs
@@ -0,0 +1,11 @@
+use super::Language;
+
+lazy_static! {
+    pub static ref TSX: Language = Language {
+        name: "tsx",
+        struct_name: "TSX",
+        node_types: tree_sitter_typescript::TSX_NODE_TYPES,
+        file_extensions: &["tsx"],
+        tree_sitter_language: tree_sitter_typescript::LANGUAGE_TSX.into(),
+    };
+}
diff --git a/codegen-sdk-common/src/language/typescript.rs b/codegen-sdk-common/src/language/typescript.rs
@@ -0,0 +1,11 @@
+use super::Language;
+
+lazy_static! {
+    pub static ref Typescript: Language = Language {
+        name: "typescript",
+        struct_name: "Typescript",
+        node_types: tree_sitter_typescript::TYPESCRIPT_NODE_TYPES,
+        file_extensions: &["ts"],
+        tree_sitter_language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
+    };
+}
diff --git a/codegen-sdk-common/src/lib.rs b/codegen-sdk-common/src/lib.rs
@@ -1,2 +1,5 @@
+pub mod language;
 pub mod traits;
 pub mod utils;
+#[macro_use]
+extern crate lazy_static;
diff --git a/codegen-sdk-common/src/traits.rs b/codegen-sdk-common/src/traits.rs
@@ -3,7 +3,7 @@ use tree_sitter::{self, Point};
 pub trait FromNode {
     fn from_node(node: tree_sitter::Node) -> Self;
 }
-pub trait CSTNode {
+pub trait CSTNode: Send {
     fn start_byte(&self) -> usize;
     fn end_byte(&self) -> usize;
     fn start_position(&self) -> Point;
@@ -13,3 +13,7 @@ pub trait CSTNode {
         String::from_utf8(self.text().to_vec()).unwrap()
     }
 }
+pub trait HasChildren {
+    type Child: Send;
+    fn children(&self) -> &Vec<Self::Child>;
+}
diff --git a/codegen-sdk-cst-generator/Cargo.toml b/codegen-sdk-cst-generator/Cargo.toml
@@ -14,4 +14,4 @@ tree-sitter = { workspace = true }
 log = { workspace = true }
 codegen-sdk-common = { path = "../codegen-sdk-common" }
 [dev-dependencies]
-tree-sitter-python = { workspace = true }
+codegen-sdk-common = { path = "../codegen-sdk-common" , features = ["python"] }
diff --git a/codegen-sdk-cst-generator/src/generator.rs b/codegen-sdk-cst-generator/src/generator.rs
@@ -57,9 +57,10 @@ mod tests {
     use crate::parser::parse_node_types;
 
     use super::*;
+    use codegen_sdk_common::language::python::Python;
     #[test]
     fn test_generate_cst() {
-        let node_types = parse_node_types(tree_sitter_python::NODE_TYPES).unwrap();
+        let node_types = parse_node_types(&Python).unwrap();
         let cst = generate_cst(&node_types).unwrap();
         log::info!("{}", cst);
     }
diff --git a/codegen-sdk-cst-generator/src/generator/struct_generator.rs b/codegen-sdk-cst-generator/src/generator/struct_generator.rs
@@ -36,6 +36,12 @@ impl CSTNode for {{name}} {
         &self.text
     }
 }
+impl HasChildren for {{name}} {
+    type Child = {{children}};
+    fn children(&self) -> &Vec<Self::Child> {
+       self.children.as_ref()
+    }
+}
 impl FromNode for {{name}} {
     fn from_node(node: tree_sitter::Node) -> Self {
         Self {
@@ -154,14 +160,11 @@ fn generate_children(
     state: &mut State,
     node_name: &str,
     constructor_fields: &mut Vec<String>,
-) {
+) -> String {
     let converted_type_name =
         convert_type_definition(&children.types, state, node_name, "children");
-    state.structs.push_str(&format!(
-        "    pub children: Vec<{}>,\n",
-        converted_type_name
-    ));
     constructor_fields.push(format!("    children: named_children_without_field_names(node).into_iter().map(|node| {converted_type_name}::from_node(node)).collect()", converted_type_name = converted_type_name));
+    converted_type_name
 }
 pub fn generate_struct(node: &Node, state: &mut State, name: &str) {
     state
@@ -171,13 +174,21 @@ pub fn generate_struct(node: &Node, state: &mut State, name: &str) {
     if let Some(fields) = &node.fields {
         generate_fields(fields, state, node, &mut constructor_fields);
     }
+    let mut children_type_name = "Self".to_string();
     if let Some(children) = &node.children {
-        generate_children(children, state, &node.type_name, &mut constructor_fields);
+        children_type_name =
+            generate_children(children, state, &node.type_name, &mut constructor_fields);
+    } else {
+        constructor_fields.push("    children: vec![]".to_string());
     }
+    state
+        .structs
+        .push_str(&format!("    pub children: Vec<{}>,\n", children_type_name));
     state.structs.push_str(FOOTER_TEMPLATE);
     state.structs.push_str(
         &CONSTRUCTOR_TEMPLATE
             .replace("{{fields}}", &constructor_fields.join(",\n       "))
-            .replace("{{name}}", name),
+            .replace("{{name}}", name)
+            .replace("{{children}}", &children_type_name),
     );
 }
diff --git a/codegen-sdk-cst-generator/src/lib.rs b/codegen-sdk-cst-generator/src/lib.rs
@@ -1,12 +1,13 @@
 mod generator;
 pub(crate) mod parser;
+use codegen_sdk_common::language::Language;
 use std::error::Error;
 
-pub fn generate_cst(source: &str, language: &str) -> Result<(), Box<dyn Error>> {
-    let node_types = parser::parse_node_types(source)?;
+pub fn generate_cst(language: &Language) -> Result<(), Box<dyn Error>> {
+    let node_types = parser::parse_node_types(language)?;
     let cst = generator::generate_cst(&node_types)?;
     let out_dir = std::env::var("OUT_DIR").unwrap();
-    let out_file = format!("{}/{}.rs", out_dir, language);
+    let out_file = format!("{}/{}.rs", out_dir, language.name);
     std::fs::write(out_file, cst)?;
     Ok(())
 }
diff --git a/codegen-sdk-cst-generator/src/parser.rs b/codegen-sdk-cst-generator/src/parser.rs
@@ -1,5 +1,6 @@
 use std::error::Error;
 
+use codegen_sdk_common::language::Language;
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -44,18 +45,17 @@ pub struct Children {
     pub types: Vec<TypeDefinition>,
 }
 
-pub fn parse_node_types(source: &str) -> Result<Vec<Node>, Box<dyn Error>> {
-    let parsed: Vec<Node> = serde_json::from_str(source)?;
+pub fn parse_node_types(language: &Language) -> Result<Vec<Node>, Box<dyn Error>> {
+    let parsed: Vec<Node> = serde_json::from_str(language.node_types)?;
     Ok(parsed)
 }
 #[cfg(test)]
 mod tests {
     use super::*;
-
+    use codegen_sdk_common::language::python::Python;
     #[test]
     fn test_parse_node_types() {
-        let source = tree_sitter_python::NODE_TYPES;
-        let cst = parse_node_types(source).unwrap();
+        let cst = parse_node_types(&Python).unwrap();
         assert!(!cst.is_empty());
     }
 }
diff --git a/codegen-sdk-cst/Cargo.toml b/codegen-sdk-cst/Cargo.toml
@@ -5,17 +5,20 @@ edition = "2021"
 
 [dependencies]
 tree-sitter = { workspace = true }
-tree-sitter-python = { workspace = true }
 ouroboros = { workspace = true }
-tree-sitter-typescript = { workspace = true }
 bytes = { workspace = true }
 codegen-sdk-common = { path = "../codegen-sdk-common" }
+codegen-sdk-macros = { path = "../codegen-sdk-macros" }
 convert_case = { workspace = true }
 [build-dependencies]
-codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator" }
-codegen-sdk-common = { path = "../codegen-sdk-common" }
-tree-sitter-python = { workspace = true }
-tree-sitter-typescript = { workspace = true }
+codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator"}
+codegen-sdk-common = { path = "../codegen-sdk-common", features = ["python", "typescript"] }
 
 [dev-dependencies]
 tempfile = "3.16.0"
+[features]
+python = [ "codegen-sdk-common/python"]
+typescript = [ "codegen-sdk-common/typescript"]
+tsx = [ "codegen-sdk-common/typescript"]
+jsx = [ "codegen-sdk-common/typescript"]
+javascript = [ "codegen-sdk-common/typescript"]
diff --git a/codegen-sdk-cst/build.rs b/codegen-sdk-cst/build.rs
diff --git a/codegen-sdk-cst/src/lib.rs b/codegen-sdk-cst/src/lib.rs
diff --git a/codegen-sdk-macros/Cargo.toml b/codegen-sdk-macros/Cargo.toml
diff --git a/codegen-sdk-macros/src/lib.rs b/codegen-sdk-macros/src/lib.rs
diff --git a/src/main.rs b/src/main.rs
