diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..d958320 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["-Clink-arg=-fuse-ld=lld", "-Zthreads=16"] diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml index 339e3ae..9d21893 100644 --- a/.github/actions/setup-environment/action.yml +++ b/.github/actions/setup-environment/action.yml @@ -4,4 +4,7 @@ runs: using: "composite" steps: - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly + components: rustc-codegen-cranelift-preview, rustfmt - uses: taiki-e/install-action@nextest diff --git a/Cargo.lock b/Cargo.lock index 8285e42..839742d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,8 +253,13 @@ dependencies = [ "codegen-sdk-ast-generator", "codegen-sdk-common", "codegen-sdk-cst", + "codegen-sdk-macros", + "derive-visitor", "env_logger", + "log", "rayon", + "tempfile", + "test-log", ] [[package]] @@ -263,6 +268,15 @@ version = "0.1.0" dependencies = [ "anyhow", "codegen-sdk-common", + "codegen-sdk-cst", + "codegen-sdk-cst-generator", + "convert_case 0.7.1", + "derive_more", + "insta", + "log", + "proc-macro2", + "quote", + "test-log", ] [[package]] @@ -278,10 +292,12 @@ dependencies = [ "lazy_static", "mockall", "phf", + "prettyplease", "rkyv", "serde", "serde_json", "sha2", + "syn 2.0.98", "test-log", "thiserror", "tree-sitter", diff --git a/Cargo.toml b/Cargo.toml index 6565262..df8e6b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,4 @@ +cargo-features = ["codegen-backend"] [package] name = "codegen-sdk-core" version = "0.1.0" @@ -6,8 +7,8 @@ edition = "2024" [dependencies] clap = { version = "4.5.28", features = ["derive"] } codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" } -codegen-sdk-cst = { workspace = true } -codegen-sdk-common = { workspace = true } +codegen-sdk-cst = { workspace = true , features = ["typescript"]} +codegen-sdk-common = { workspace = true} crossbeam = "0.8.4" glob = "0.3.2" env_logger = { workspace = true } @@ -69,7 +70,30 @@ mockall = "0.13.1" codegen-sdk-common = { path = "codegen-sdk-common" } codegen-sdk-cst = { path = "codegen-sdk-cst"} codegen-sdk-ast = { path = "codegen-sdk-ast" } +codegen-sdk-cst-generator = { path = "codegen-sdk-cst-generator" } +tempfile = "3.16.0" +quote = "1.0.38" +proc-macro2 = "1.0.93" +derive-visitor = "0.4.0" +insta = "1.42.1" +prettyplease = "0.2.29" +syn = { version = "2.0.98", features = ["proc-macro"] } +derive_more = { version = "2.0.1", features = ["debug", "display"] } + +[profile.dev] +debug = 0 +codegen-backend = "cranelift" [profile.dev.package] insta.opt-level = 3 similar.opt-level = 3 + +[profile.test] +inherits = "dev" +opt-level = 0 +debug = 0 +strip = "none" +lto = false +codegen-units = 256 +incremental = true +codegen-backend = "cranelift" diff --git a/codegen-sdk-ast-generator/Cargo.toml b/codegen-sdk-ast-generator/Cargo.toml index 8d8adc6..ffb2528 100644 --- a/codegen-sdk-ast-generator/Cargo.toml +++ b/codegen-sdk-ast-generator/Cargo.toml @@ -5,4 +5,14 @@ edition = "2024" [dependencies] codegen-sdk-common = { workspace = true } +codegen-sdk-cst = { workspace = true , features = ["ts_query"]} anyhow = { workspace = true } +quote = { workspace = true } +proc-macro2 = { workspace = true } +log = { workspace = true } +derive_more = { workspace = true } +codegen-sdk-cst-generator = { workspace = true } +convert_case = { workspace = true } +[dev-dependencies] +test-log = { workspace = true } +insta = { workspace = true } diff --git a/codegen-sdk-ast-generator/src/generator.rs b/codegen-sdk-ast-generator/src/generator.rs index e26d687..fc1514a 100644 --- a/codegen-sdk-ast-generator/src/generator.rs +++ b/codegen-sdk-ast-generator/src/generator.rs @@ -5,12 +5,20 @@ pub fn generate_ast(language: &Language) -> anyhow::Result { #[derive(Debug, Clone)] pub struct {language_struct_name}File {{ node: {language_name}::{root_node_name}, - path: PathBuf + path: PathBuf, + pub visitor: QueryExecutor }} impl File for {language_struct_name}File {{ fn path(&self) -> &PathBuf {{ &self.path }} + fn parse(path: &PathBuf) -> Result {{ + log::debug!(\"Parsing {language_name} file: {{}}\", path.display()); + let ast = {language_name}::{language_struct_name}::parse_file(path)?; + let mut visitor = QueryExecutor::default(); + ast.drive(&mut visitor); + Ok({language_struct_name}File {{ node: ast, path: path.clone(), visitor }}) + }} }} impl HasNode for {language_struct_name}File {{ type Node = {language_name}::{root_node_name}; @@ -18,11 +26,10 @@ pub fn generate_ast(language: &Language) -> anyhow::Result { &self.node }} }} - ", language_struct_name = language.struct_name, language_name = language.name(), - root_node_name = language.root_node() + root_node_name = language.root_node(), ); // for (name, query) in language.definitions() { // content.push_str(&format!(" diff --git a/codegen-sdk-ast-generator/src/lib.rs b/codegen-sdk-ast-generator/src/lib.rs index 8f6c623..2230b03 100644 --- a/codegen-sdk-ast-generator/src/lib.rs +++ b/codegen-sdk-ast-generator/src/lib.rs @@ -1,7 +1,31 @@ -use codegen_sdk_common::language::Language; +#![feature(extend_one)] + +use codegen_sdk_common::{generator::format_code, language::Language}; +use quote::quote; + +use crate::query::HasQuery; mod generator; +mod query; +mod visitor; pub fn generate_ast(language: &Language) -> anyhow::Result<()> { - let ast = generator::generate_ast(language)?; + let imports = quote! { + use derive_visitor::{Visitor, Drive}; + use codegen_sdk_common::*; + use std::path::PathBuf; + use codegen_sdk_cst::CSTLanguage; + }; + let mut ast = generator::generate_ast(language)?; + let visitor = visitor::generate_visitor( + &language + .definitions() + .values() + .into_iter() + .flatten() + .collect(), + language, + ); + ast = imports.to_string() + &ast + &visitor.to_string(); + ast = format_code(&ast).unwrap(); let out_dir = std::env::var("OUT_DIR")?; let out_file = format!("{}/{}.rs", out_dir, language.name()); std::fs::write(out_file, ast)?; diff --git a/codegen-sdk-ast-generator/src/query.rs b/codegen-sdk-ast-generator/src/query.rs new file mode 100644 index 0000000..6170d5e --- /dev/null +++ b/codegen-sdk-ast-generator/src/query.rs @@ -0,0 +1,426 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use codegen_sdk_common::{CSTNode, HasChildren, Language, naming::normalize_type_name}; +use codegen_sdk_cst::{CSTLanguage, ts_query}; +use codegen_sdk_cst_generator::{Field, State}; +use derive_more::Debug; +use log::{debug, info, warn}; +use proc_macro2::{Ident, TokenStream}; +use quote::{format_ident, quote}; +fn captures_for_field_definition( + node: &ts_query::FieldDefinition, +) -> impl Iterator { + let mut captures = Vec::new(); + for child in node.children() { + match child { + ts_query::FieldDefinitionChildren::NamedNode(named) => { + captures.extend(captures_for_named_node(&named)); + } + ts_query::FieldDefinitionChildren::FieldDefinition(field) => { + captures.extend(captures_for_field_definition(&field)); + } + _ => {} + } + } + captures.into_iter() +} +fn captures_for_named_node(node: &ts_query::NamedNode) -> impl Iterator { + let mut captures = Vec::new(); + for child in node.children() { + match child { + ts_query::NamedNodeChildren::Capture(capture) => captures.push(capture), + ts_query::NamedNodeChildren::NamedNode(named) => { + captures.extend(captures_for_named_node(&named)); + } + ts_query::NamedNodeChildren::FieldDefinition(field) => { + captures.extend(captures_for_field_definition(&field)); + } + _ => {} + } + } + captures.into_iter() +} +#[derive(Debug)] +pub struct Query<'a> { + node: ts_query::NamedNode, + language: &'a Language, + state: Arc>, +} +impl<'a> Query<'a> { + pub fn from_queries(source: &str, language: &'a Language) -> BTreeMap { + let parsed = ts_query::Query::parse(source).unwrap(); + let state = Arc::new(State::new(language)); + let mut queries = BTreeMap::new(); + for node in parsed.children() { + match node { + ts_query::ProgramChildren::NamedNode(named) => { + let query = Self::from_named_node(&named, language, state.clone()); + queries.insert(query.name(), query); + } + node => { + println!("Unhandled query: {:#?}", node); + } + } + } + + // let root_node: Node<'a> = tree.root_node(); + // for child in root_node.children(&mut root_node.walk()) { + // if child.kind() == "grouping" { + // continue; + // // TODO: Handle grouping + // } + // let query = Self { + // query: child, + // source: source.to_string(), + // }; + // queries.insert(query.name(), query); + // } + queries + } + fn from_named_node( + named: &ts_query::NamedNode, + language: &'a Language, + state: Arc>, + ) -> Self { + Query { + node: named.clone(), + language: language, + state: state, + } + } + /// Get the kind of the query (the node to be matched) + pub fn kind(&self) -> String { + if let ts_query::NamedNodeName::Identifier(identifier) = &(*self.node.name) { + return identifier.source(); + } + panic!("No kind found for query. {:#?}", self.node); + } + pub fn struct_name(&self) -> String { + normalize_type_name(&self.kind(), true) + } + pub fn struct_variants(&self) -> Vec { + if self + .state + .get_node_for_struct_name(&self.struct_name()) + .is_some() + { + return vec![self.struct_name()]; + } + self.state + .get_variants(&self.struct_name()) + .into_iter() + .map(|v| v.normalize()) + .filter(|v| v != "Comment") + .collect() + } + + fn captures(&self) -> Vec { + captures_for_named_node(&self.node).collect() + } + /// Get the name of the query (IE @reference.class) + pub fn name(&self) -> String { + let mut result = self.captures().last().unwrap().source(); + result.replace_range(0..1, ""); + result + } + + // for node in self.query.named_children(&mut self.query.walk()) { + // for node in node.named_children(&mut self.query.walk()) { + // if node.kind() == "capture" { + // return get_text_from_node(&node, &self.source); + // } + // } + // } + + // panic!( + // "No name found for query. {:?}\n{}", + // self.query, + // self.source() + // ); + // } + + // fn execute(&self, node: &T) -> Vec> { + // let mut result = Vec::new(); + + // for child in node.children() { + // if self + // .captures() + // .iter() + // .any(|capture| capture.source() == child.kind()) + // { + // result.push(child); + // } + // } + // result + // } + pub fn node(&self) -> &ts_query::NamedNode { + &self.node + } + pub fn executor_id(&self) -> Ident { + let raw_name = self.name(); + let name = raw_name.split(".").last().unwrap(); + if name.ends_with("s") { + format_ident!("{}es", name) + } else { + format_ident!("{}s", name) + } + } + fn get_field_for_field_name(&self, field_name: &str, struct_name: &str) -> Option<&Field> { + debug!( + "Getting field for: {:#?} on node: {:#?}", + field_name, struct_name + ); + let node = self.state.get_node_for_struct_name(struct_name); + if let Some(node) = node { + return node.get_field_for_field_name(field_name); + } + warn!( + "No node found for: {:#?}. In language: {:#?}", + struct_name, + self.language.name() + ); + None + } + fn get_matcher_for_field( + &self, + field: &ts_query::FieldDefinition, + struct_name: &str, + current_node: &Ident, + ) -> TokenStream { + let other_child: ts_query::NodeTypes = + field.children().into_iter().skip(2).next().unwrap().into(); + for name in &field.name { + if let ts_query::FieldDefinitionName::Identifier(identifier) = name { + let name = identifier.source(); + if let Some(field) = self.get_field_for_field_name(&name, struct_name) { + let field_name = format_ident!("{}", name); + let new_identifier = format_ident!("field"); + let normalized_struct_name = field.type_name(); + let wrapped = self.get_matcher_for_definition( + &normalized_struct_name, + other_child.clone(), + &new_identifier, + ); + assert!( + wrapped.to_string().len() > 0, + "Wrapped is empty, {} {} {}", + normalized_struct_name, + other_child.source(), + other_child.kind() + ); + if !field.is_optional() { + return quote! { + let #new_identifier = &#current_node.#field_name; + #wrapped + }; + } else { + return quote! { + if let Some(field) = &#current_node.#field_name { + #wrapped + } + }; + } + } else { + panic!( + "No field found for: {:#?} on node: {:#?}. In language: {:#?}. Field source: {:#?}", + name, + struct_name, + self.language.name(), + field.source() + ) + } + } + } + panic!( + "No field found for: {:#?}. In language: {:#?}", + field.source(), + self.language.name() + ) + } + fn get_matchers_for_grouping( + &self, + node: &ts_query::Grouping, + struct_name: &str, + current_node: &Ident, + ) -> TokenStream { + let mut matchers = TokenStream::new(); + for group in node.children() { + let result = self.get_matcher_for_definition(struct_name, group.into(), current_node); + matchers.extend_one(result); + } + matchers + } + + fn get_matcher_for_named_node( + &self, + node: &ts_query::NamedNode, + struct_name: &str, + current_node: &Ident, + ) -> TokenStream { + let mut matchers = TokenStream::new(); + let first_node = node.children().into_iter().next().unwrap(); + let remaining_nodes = node + .children() + .into_iter() + .skip(1) + .filter(|child| child.kind() != "capture") + .collect::>(); + if remaining_nodes.len() == 0 { + log::info!("single node, {}", first_node.source()); + return self.get_matcher_for_definition(struct_name, first_node.into(), current_node); + } + + let name_node = self.state.get_node_for_raw_name(&first_node.source()); + if let Some(name_node) = name_node { + for child in remaining_nodes { + let result = self.get_matcher_for_definition( + &name_node.normalize_name(), + child.into(), + current_node, + ); + matchers.extend_one(result); + } + } else { + let subenum = self.state.get_subenum_variants(&first_node.source()); + log::info!( + "subenum {} with {} variants", + first_node.source(), + subenum.len() + ); + for variant in subenum { + if variant.normalize_name() == "Comment" { + continue; + } + for child in remaining_nodes.clone() { + let result = self.get_matcher_for_definition( + &variant.normalize_name(), + child.into(), + current_node, + ); + matchers.extend_one(result); + } + } + } + matchers + } + fn get_default_matcher(&self) -> TokenStream { + let to_append = self.executor_id(); + quote! { + self.#to_append.push(node.clone()); + } + } + fn get_matcher_for_definition( + &self, + struct_name: &str, + node: ts_query::NodeTypes, + current_node: &Ident, + ) -> TokenStream { + if !node.is_named() { + return self.get_default_matcher(); + } + match node { + ts_query::NodeTypes::FieldDefinition(field) => { + self.get_matcher_for_field(&field, struct_name, current_node) + } + ts_query::NodeTypes::Capture(named) => { + info!("Capture: {:#?}", named.source()); + quote! {} + } + ts_query::NodeTypes::NamedNode(named) => { + self.get_matcher_for_named_node(&named, struct_name, current_node) + } + ts_query::NodeTypes::Comment(_) => { + quote! {} + } + ts_query::NodeTypes::List(subenum) => { + for child in subenum.children() { + let result = + self.get_matcher_for_definition(struct_name, child.into(), current_node); + // Currently just returns the first child + return result; // TODO: properly handle list + } + quote! {} + } + ts_query::NodeTypes::Grouping(grouping) => { + self.get_matchers_for_grouping(&grouping, struct_name, current_node) + } + ts_query::NodeTypes::Identifier(identifier) => { + let to_append = self.get_default_matcher(); + let language = format_ident!("{}", self.language.name()); + let children; + if let Some(node) = self.state.get_node_for_struct_name(struct_name) { + children = format_ident!("{}Children", struct_name); + // When there is only 1 possible child, we can use the default matcher + if node.children_struct_name() != children.to_string() { + return self.get_default_matcher(); + } + } else { + // If this is a field, we may be dealing with multiple types and can't operate over all of them + return self.get_default_matcher(); // TODO: Handle this case + } + let struct_name = + format_ident!("{}", normalize_type_name(&identifier.source(), true)); + quote! { + if #current_node.children().into_iter().any(|child| { + if let #language::#children::#struct_name(_) = child { + true + } else { + false + } + }) { + #to_append + } + } + } + unhandled => todo!( + "Unhandled definition in language {}: {:#?}, {:#?}", + self.language.name(), + unhandled.kind(), + unhandled.source() + ), + } + } + + pub fn matcher(&self, struct_name: &str) -> TokenStream { + info!( + "Generating matcher for: {:#?}. Has children: {:#?}", + self.node().source(), + self.node().children() + ); + let mut matchers = TokenStream::new(); + for child in self.node().children().into_iter().skip(1) { + let result = + self.get_matcher_for_definition(struct_name, child.into(), &format_ident!("node")); + matchers.extend_one(result); + } + matchers + } +} + +pub trait HasQuery { + fn queries(&self) -> BTreeMap; + fn queries_with_prefix(&self, prefix: &str) -> BTreeMap>> { + let mut queries = BTreeMap::new(); + for (name, query) in self.queries().into_iter() { + if name.starts_with(prefix) { + let new_name = name.split(".").last().unwrap(); + queries + .entry(new_name.to_string()) + .or_insert(Vec::new()) + .push(query); + } + } + queries + } + fn definitions(&self) -> BTreeMap>> { + self.queries_with_prefix("definition") + } + // fn references(&self) -> BTreeMap>> { + // self.queries_with_prefix("reference") + // } +} +impl HasQuery for Language { + fn queries(&self) -> BTreeMap> { + Query::from_queries(&self.tag_query, self) + } +} diff --git a/codegen-sdk-ast-generator/src/snapshots/codegen_sdk_ast_generator__visitor__tests__generate_visitor.snap b/codegen-sdk-ast-generator/src/snapshots/codegen_sdk_ast_generator__visitor__tests__generate_visitor.snap new file mode 100644 index 0000000..51a7481 --- /dev/null +++ b/codegen-sdk-ast-generator/src/snapshots/codegen_sdk_ast_generator__visitor__tests__generate_visitor.snap @@ -0,0 +1,53 @@ +--- +source: codegen-sdk-ast-generator/src/visitor.rs +expression: "codegen_sdk_common::generator::format_code(&visitor.to_string()).unwrap()" +--- +#[derive(Visitor, Default, Debug, Clone)] +#[visitor( + typescript::AbstractClassDeclaration(enter), + typescript::FunctionSignature(enter), + typescript::InterfaceDeclaration(enter), + typescript::AbstractMethodSignature(enter), + typescript::Module(enter) +)] +pub struct QueryExecutor { + pub classes: Vec, + pub functions: Vec, + pub interfaces: Vec, + pub methods: Vec, + pub modules: Vec, +} +impl QueryExecutor { + fn enter_abstract_class_declaration( + &mut self, + node: &codegen_sdk_cst::typescript::AbstractClassDeclaration, + ) { + let field = &node.name; + self.classes.push(node.clone()); + } + fn enter_abstract_method_signature( + &mut self, + node: &codegen_sdk_cst::typescript::AbstractMethodSignature, + ) { + let field = &node.name; + self.methods.push(node.clone()); + } + fn enter_function_signature( + &mut self, + node: &codegen_sdk_cst::typescript::FunctionSignature, + ) { + let field = &node.name; + self.functions.push(node.clone()); + } + fn enter_interface_declaration( + &mut self, + node: &codegen_sdk_cst::typescript::InterfaceDeclaration, + ) { + let field = &node.name; + self.interfaces.push(node.clone()); + } + fn enter_module(&mut self, node: &codegen_sdk_cst::typescript::Module) { + let field = &node.name; + self.modules.push(node.clone()); + } +} diff --git a/codegen-sdk-ast-generator/src/visitor.rs b/codegen-sdk-ast-generator/src/visitor.rs new file mode 100644 index 0000000..9b65c53 --- /dev/null +++ b/codegen-sdk-ast-generator/src/visitor.rs @@ -0,0 +1,87 @@ +use std::collections::BTreeMap; + +use codegen_sdk_common::{CSTNode, HasChildren, Language}; +use codegen_sdk_cst::ts_query; +use convert_case::{Case, Casing}; +use log::info; +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; + +use super::query::Query; +pub fn generate_visitor(queries: &Vec<&Query>, language: &Language) -> TokenStream { + log::info!("Generating visitor for language: {}", language.name()); + let language_name = format_ident!("{}", language.name()); + let mut names = Vec::new(); + let mut types = Vec::new(); + let mut variants = Vec::new(); + let mut enter_methods = BTreeMap::new(); + for query in queries { + names.push(query.executor_id()); + types.push(format_ident!("{}", query.struct_name())); + for variant in query.struct_variants() { + variants.push(format_ident!("{}", variant)); + enter_methods + .entry(variant) + .or_insert(Vec::new()) + .push(query); + } + } + let mut methods = Vec::new(); + for (variant, queries) in enter_methods { + let mut matchers = TokenStream::new(); + let enter = format_ident!("enter_{}", variant.to_case(Case::Snake)); + let struct_name = format_ident!("{}", variant); + for query in queries { + matchers.extend_one(query.matcher(&variant)); + let node = query.node(); + for child in node.children() { + info!("child kind:{} source:{}", child.kind(), child.source()); + if let ts_query::NamedNodeChildren::FieldDefinition(field_definition) = child { + let field_name = &field_definition.name; + let source = field_definition.source(); + let children = &field_definition.children(); + info!("source: {:?}", source); + info!("field_name: {:?}", field_name); + info!("children: {:?}", children); + } + } + } + methods.push(quote! { + fn #enter(&mut self, node: &codegen_sdk_cst::#language_name::#struct_name) { + #matchers + } + }); + } + let name = format_ident!("QueryExecutor"); + quote! { + #[derive(Visitor, Default, Debug, Clone)] + #[visitor( + #(#language_name::#variants(enter)),* + )] + pub struct #name { + #(pub #names: Vec<#language_name::#types>),* + } + impl #name { + #(#methods)* + } + } +} + +#[cfg(test)] +mod tests { + use codegen_sdk_common::language::typescript::Typescript; + + use super::*; + use crate::query::HasQuery; + + #[test_log::test] + fn test_generate_visitor() { + let language = &Typescript; + let queries = language.definitions(); + log::info!("Gathered {} queries", queries.len()); + let visitor = generate_visitor(&queries.values().into_iter().flatten().collect(), language); + insta::assert_snapshot!( + codegen_sdk_common::generator::format_code(&visitor.to_string()).unwrap() + ); + } +} diff --git a/codegen-sdk-ast/Cargo.toml b/codegen-sdk-ast/Cargo.toml index ef580ae..132f1fd 100644 --- a/codegen-sdk-ast/Cargo.toml +++ b/codegen-sdk-ast/Cargo.toml @@ -4,14 +4,19 @@ version = "0.1.0" edition = "2024" [dependencies] -codegen-sdk-cst = { workspace = true, features = ["json"]} +codegen-sdk-cst = { workspace = true} codegen-sdk-common = { workspace = true } - +derive-visitor = { workspace = true } +codegen-sdk-macros = { path = "../codegen-sdk-macros"} +log = { workspace = true } [build-dependencies] codegen-sdk-common = { workspace = true } env_logger = { workspace = true } rayon = { workspace = true } codegen-sdk-ast-generator = { path = "../codegen-sdk-ast-generator"} +[dev-dependencies] +test-log = { workspace = true } +tempfile = { workspace = true } [features] python = [ "codegen-sdk-cst/python"] typescript = [ "codegen-sdk-cst/typescript"] @@ -27,4 +32,4 @@ yaml = [ "codegen-sdk-cst/yaml"] toml = [ "codegen-sdk-cst/toml"] markdown = [ "codegen-sdk-cst/markdown"] ts_query = [] -default = ["json", "ts_query"] +default = ["json", "ts_query", "typescript"] diff --git a/codegen-sdk-ast/src/lib.rs b/codegen-sdk-ast/src/lib.rs index 006cdd1..6ad7881 100644 --- a/codegen-sdk-ast/src/lib.rs +++ b/codegen-sdk-ast/src/lib.rs @@ -1,6 +1,8 @@ #![recursion_limit = "512"] -use codegen_sdk_common::{File, HasNode}; +#![allow(unused)] +use codegen_sdk_common::File; pub use codegen_sdk_cst::*; +use codegen_sdk_macros::include_languages_ast; pub trait Named { fn name(&self) -> &str; } @@ -9,5 +11,4 @@ impl Named for T { self.path().file_name().unwrap().to_str().unwrap() } } -use std::path::PathBuf; -include!(concat!(env!("OUT_DIR"), "/json.rs")); +include_languages_ast!(); diff --git a/codegen-sdk-ast/tests/test_typescript.rs b/codegen-sdk-ast/tests/test_typescript.rs new file mode 100644 index 0000000..179d822 --- /dev/null +++ b/codegen-sdk-ast/tests/test_typescript.rs @@ -0,0 +1,35 @@ +#![recursion_limit = "512"] +use std::path::PathBuf; + +use codegen_sdk_ast::typescript::TypescriptFile; +use codegen_sdk_common::File; +fn write_to_temp_file(content: &str, temp_dir: &tempfile::TempDir) -> PathBuf { + let file_path = temp_dir.path().join("test.ts"); + std::fs::write(&file_path, content).unwrap(); + file_path +} +// TODO: Fix queries for classes and functions +// #[test_log::test] +// fn test_typescript_ast_class() { +// let temp_dir = tempfile::tempdir().unwrap(); +// let content = "class Test { }"; +// let file_path = write_to_temp_file(content, &temp_dir); +// let file = TypescriptFile::parse(&file_path).unwrap(); +// assert_eq!(file.visitor.classes.len(), 1); +// } +// #[test_log::test] +// fn test_typescript_ast_function() { +// let temp_dir = tempfile::tempdir().unwrap(); +// let content = "function test() { }"; +// let file_path = write_to_temp_file(content, &temp_dir); +// let file = TypescriptFile::parse(&file_path).unwrap(); +// assert_eq!(file.visitor.functions.len(), 1); +// } +#[test_log::test] +fn test_typescript_ast_interface() { + let temp_dir = tempfile::tempdir().unwrap(); + let content = "interface Test { }"; + let file_path = write_to_temp_file(content, &temp_dir); + let file = TypescriptFile::parse(&file_path).unwrap(); + assert_eq!(file.visitor.interfaces.len(), 1); +} diff --git a/codegen-sdk-common/Cargo.toml b/codegen-sdk-common/Cargo.toml index 0edbff4..5763001 100644 --- a/codegen-sdk-common/Cargo.toml +++ b/codegen-sdk-common/Cargo.toml @@ -34,6 +34,8 @@ sha2 = "0.10.8" zstd = { version = "0.13.2", features = ["zstdmt"] } enum_delegate = { workspace = true } mockall = { workspace = true } +syn = { workspace = true } +prettyplease = { workspace = true } [dev-dependencies] test-log = { workspace = true } [features] diff --git a/codegen-sdk-common/src/file.rs b/codegen-sdk-common/src/file.rs index ce8328b..e5efb8c 100644 --- a/codegen-sdk-common/src/file.rs +++ b/codegen-sdk-common/src/file.rs @@ -1,8 +1,13 @@ use std::path::PathBuf; +use crate::ParseError; + pub trait File { fn path(&self) -> &PathBuf; fn content(&self) -> String { std::fs::read_to_string(self.path()).unwrap() } + fn parse(path: &PathBuf) -> Result + where + Self: Sized; } diff --git a/codegen-sdk-common/src/generator.rs b/codegen-sdk-common/src/generator.rs new file mode 100644 index 0000000..819f982 --- /dev/null +++ b/codegen-sdk-common/src/generator.rs @@ -0,0 +1,2 @@ +mod format; +pub use format::*; diff --git a/codegen-sdk-cst-generator/src/generator/format.rs b/codegen-sdk-common/src/generator/format.rs similarity index 61% rename from codegen-sdk-cst-generator/src/generator/format.rs rename to codegen-sdk-common/src/generator/format.rs index 20bc99f..1877ea9 100644 --- a/codegen-sdk-cst-generator/src/generator/format.rs +++ b/codegen-sdk-common/src/generator/format.rs @@ -1,4 +1,4 @@ -pub fn format_cst(cst: &str) -> anyhow::Result { +pub fn format_code(cst: &str) -> anyhow::Result { let parsed = syn::parse_str::(cst)?; Ok(prettyplease::unparse(&parsed)) } diff --git a/codegen-sdk-common/src/language.rs b/codegen-sdk-common/src/language.rs index a9f0e13..2d03b6c 100644 --- a/codegen-sdk-common/src/language.rs +++ b/codegen-sdk-common/src/language.rs @@ -6,6 +6,7 @@ use tree_sitter::Parser; use crate::{ errors::ParseError, + naming::normalize_type_name, parser::{Node, parse_node_types}, }; #[derive(Debug)] @@ -70,6 +71,12 @@ impl Language { pub fn name(&self) -> &'static str { self.name } + pub fn node_for_struct_name(&self, struct_name: &str) -> Option { + self.nodes + .iter() + .find(|node| normalize_type_name(&node.type_name, node.named) == struct_name) + .cloned() + } } #[cfg(feature = "go")] pub mod go; diff --git a/codegen-sdk-common/src/language/ts_query.rs b/codegen-sdk-common/src/language/ts_query.rs index 1648978..1205c5f 100644 --- a/codegen-sdk-common/src/language/ts_query.rs +++ b/codegen-sdk-common/src/language/ts_query.rs @@ -4,7 +4,7 @@ lazy_static! { "ts_query", "Query", tree_sitter_query::NODE_TYPES, - &[], + &["scm"], tree_sitter_query::LANGUAGE.into(), "", ) diff --git a/codegen-sdk-common/src/language/typescript.rs b/codegen-sdk-common/src/language/typescript.rs index f371b84..5e59929 100644 --- a/codegen-sdk-common/src/language/typescript.rs +++ b/codegen-sdk-common/src/language/typescript.rs @@ -1,13 +1,23 @@ use super::Language; - +// const ADDITIONAL_QUERIES: &str = " +// (class_declaration +// name: (type_identifier) @name) @definition.class +// "; +const ADDITIONAL_QUERIES: &str = " +"; lazy_static! { + static ref QUERIES: String = format!( + "{}{}", + ADDITIONAL_QUERIES, + tree_sitter_typescript::TAGS_QUERY + ); pub static ref Typescript: Language = Language::new( "typescript", "Typescript", tree_sitter_typescript::TYPESCRIPT_NODE_TYPES, &["ts"], tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), - tree_sitter_typescript::TAGS_QUERY, + &QUERIES, ) .unwrap(); } diff --git a/codegen-sdk-common/src/lib.rs b/codegen-sdk-common/src/lib.rs index f16f9aa..2b43992 100644 --- a/codegen-sdk-common/src/lib.rs +++ b/codegen-sdk-common/src/lib.rs @@ -16,3 +16,4 @@ pub mod naming; pub mod serialize; pub mod tree; pub use tree::{Point, Range}; +pub mod generator; diff --git a/codegen-sdk-common/src/parser.rs b/codegen-sdk-common/src/parser.rs index e68bd06..499fab8 100644 --- a/codegen-sdk-common/src/parser.rs +++ b/codegen-sdk-common/src/parser.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::naming::normalize_type_name; + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct Node { #[serde(rename = "type")] @@ -35,7 +37,11 @@ pub struct TypeDefinition { pub type_name: String, pub named: bool, } - +impl TypeDefinition { + pub fn normalize(&self) -> String { + normalize_type_name(&self.type_name, true) + } +} #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] pub struct Children { pub multiple: bool, diff --git a/codegen-sdk-cst-generator/Cargo.toml b/codegen-sdk-cst-generator/Cargo.toml index b40dcdd..a501021 100644 --- a/codegen-sdk-cst-generator/Cargo.toml +++ b/codegen-sdk-cst-generator/Cargo.toml @@ -5,18 +5,18 @@ edition = "2024" [dependencies] convert_case = { workspace = true } -prettyplease = "0.2.29" -syn = { version = "2.0.98", features = ["proc-macro"] } +prettyplease = {workspace = true} +syn = { workspace = true } tree-sitter = { workspace = true } log = { workspace = true } codegen-sdk-common = { workspace = true } anyhow = { workspace = true } -quote = "1.0.38" -proc-macro2 = "1.0.93" -tempfile = "3.8.1" +quote = { workspace = true } +proc-macro2 = { workspace = true } +tempfile = { workspace = true } mockall_double = "0.3.1" [dev-dependencies] tree-sitter-python = { workspace = true } test-log = { workspace = true } -insta = "1.42.1" +insta = { workspace = true } diff --git a/codegen-sdk-cst-generator/src/generator.rs b/codegen-sdk-cst-generator/src/generator.rs index ec6639b..248f17f 100644 --- a/codegen-sdk-cst-generator/src/generator.rs +++ b/codegen-sdk-cst-generator/src/generator.rs @@ -1,10 +1,11 @@ #[double] use codegen_sdk_common::language::Language; +pub use field::Field; use mockall_double::double; -use state::State; +pub use node::Node; +pub use state::State; mod constants; mod field; -pub(crate) mod format; mod node; mod state; mod utils; @@ -34,7 +35,7 @@ pub fn generate_cst(language: &Language) -> anyhow::Result { let structs = state.get_structs(); result.extend_one(enums); result.extend_one(structs); - let formatted = format::format_cst(&result.to_string()); + let formatted = codegen_sdk_common::generator::format_code(&result.to_string()); match formatted { Ok(formatted) => return Ok(formatted), Err(e) => { diff --git a/codegen-sdk-cst-generator/src/generator/field.rs b/codegen-sdk-cst-generator/src/generator/field.rs index 390bb06..898c1b5 100644 --- a/codegen-sdk-cst-generator/src/generator/field.rs +++ b/codegen-sdk-cst-generator/src/generator/field.rs @@ -173,6 +173,12 @@ impl<'a> Field<'a> { } } } + pub fn is_optional(&self) -> bool { + !self.raw.required + } + pub fn is_multiple(&self) -> bool { + self.raw.multiple + } } #[cfg(test)] diff --git a/codegen-sdk-cst-generator/src/generator/node.rs b/codegen-sdk-cst-generator/src/generator/node.rs index 1bcca05..08d531f 100644 --- a/codegen-sdk-cst-generator/src/generator/node.rs +++ b/codegen-sdk-cst-generator/src/generator/node.rs @@ -265,11 +265,6 @@ impl<'a> Node<'a> { children_fields.push(field.get_children_field(convert_children)); } - let m = if children_fields.is_empty() { - quote! {} - } else { - quote! {mut} - }; let children_init = if self.has_children() { quote! { self.children.iter().cloned().collect() @@ -281,8 +276,9 @@ impl<'a> Node<'a> { }; quote! { fn children(&self) -> Vec { - let #m children: Vec<_> = #children_init; + let mut children: Vec<_> = #children_init; #(#children_fields;)* + children.sort_by_key(|c| c.start_byte()); children } } @@ -321,6 +317,9 @@ impl<'a> Node<'a> { } } } + pub fn get_field_for_field_name(&self, field_name: &str) -> Option<&Field<'a>> { + self.fields.iter().find(|f| f.name() == field_name) + } } #[cfg(test)] mod tests { diff --git a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_children_field_impl.snap b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_children_field_impl.snap index cb83d68..e7e01ee 100644 --- a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_children_field_impl.snap +++ b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_children_field_impl.snap @@ -9,5 +9,6 @@ fn children(&self) -> Vec { Self::Child::try_from(NodeTypes::from(self.test_field.as_ref().clone())) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } diff --git a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_complex.snap b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_complex.snap index 642c054..ccb5051 100644 --- a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_complex.snap +++ b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_complex.snap @@ -136,6 +136,7 @@ impl HasChildren for TestNode { ) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_children.snap b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_children.snap index 1eb6135..cd0d621 100644 --- a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_children.snap +++ b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_children.snap @@ -101,7 +101,8 @@ impl CSTNode for TestNode { impl HasChildren for TestNode { type Child = TestNodeChildren; fn children(&self) -> Vec { - let children: Vec<_> = self.children.iter().cloned().collect(); + let mut children: Vec<_> = self.children.iter().cloned().collect(); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_fields.snap b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_fields.snap index 3514124..0beed02 100644 --- a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_fields.snap +++ b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_fields.snap @@ -107,6 +107,7 @@ impl HasChildren for TestNode { Self::Child::try_from(NodeTypes::from(self.test_field.as_ref().clone())) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_single_child_type.snap b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_single_child_type.snap index 1eb6135..cd0d621 100644 --- a/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_single_child_type.snap +++ b/codegen-sdk-cst-generator/src/generator/snapshots/codegen_sdk_cst_generator__generator__node__tests__get_struct_tokens_with_single_child_type.snap @@ -101,7 +101,8 @@ impl CSTNode for TestNode { impl HasChildren for TestNode { type Child = TestNodeChildren; fn children(&self) -> Vec { - let children: Vec<_> = self.children.iter().cloned().collect(); + let mut children: Vec<_> = self.children.iter().cloned().collect(); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/generator/state.rs b/codegen-sdk-cst-generator/src/generator/state.rs index 9cb0027..0ff9aad 100644 --- a/codegen-sdk-cst-generator/src/generator/state.rs +++ b/codegen-sdk-cst-generator/src/generator/state.rs @@ -106,7 +106,7 @@ impl<'a> State<'a> { } } } - fn get_variants(&self, subenum: &str) -> Vec { + pub fn get_variants(&self, subenum: &str) -> Vec { let comment = get_comment_type(); let mut variants = vec![comment]; for node in self.nodes.values() { @@ -183,6 +183,27 @@ impl<'a> State<'a> { } struct_tokens } + pub fn get_node_for_struct_name(&self, name: &str) -> Option<&Node<'a>> { + self.nodes.get(name) + } + pub fn get_node_for_raw_name(&self, name: &str) -> Option<&Node<'a>> { + for node in self.nodes.values() { + if node.kind() == name { + return Some(node); + } + } + None + } + pub fn get_subenum_variants(&self, name: &str) -> Vec<&Node<'a>> { + let variants = self.get_variants(name); + let mut nodes = Vec::new(); + for variant in variants { + if let Some(node) = self.get_node_for_struct_name(&variant.normalize()) { + nodes.push(node); + } + } + nodes + } } #[cfg(test)] mod tests { diff --git a/codegen-sdk-cst-generator/src/lib.rs b/codegen-sdk-cst-generator/src/lib.rs index e32d564..4d563d2 100644 --- a/codegen-sdk-cst-generator/src/lib.rs +++ b/codegen-sdk-cst-generator/src/lib.rs @@ -2,7 +2,7 @@ mod generator; #[double] use codegen_sdk_common::language::Language; -pub use generator::generate_cst; +pub use generator::{Field, Node, State, generate_cst}; use mockall_double::double; pub fn generate_cst_to_file(language: &Language) -> anyhow::Result<()> { let cst = generator::generate_cst(language)?; @@ -18,7 +18,6 @@ mod test_util { use codegen_sdk_common::{language::MockLanguage, parser::Node}; use proc_macro2::TokenStream; - use super::generator::format::format_cst; // Removes quotes from the string when using insta::assert_debug_snapshot! pub struct StringDebug { pub string: String, @@ -50,7 +49,8 @@ mod test_util { language } pub fn snapshot_string(string: &str) -> StringDebug { - let formatted = format_cst(string).unwrap_or_else(|_| string.to_string()); + let formatted = codegen_sdk_common::generator::format_code(string) + .unwrap_or_else(|_| string.to_string()); StringDebug { string: formatted } } pub fn snapshot_tokens(tokens: &TokenStream) -> StringDebug { diff --git a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes__subtypes_with_fields.snap b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes__subtypes_with_fields.snap index d147608..63c037a 100644 --- a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes__subtypes_with_fields.snap +++ b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes__subtypes_with_fields.snap @@ -168,6 +168,7 @@ impl HasChildren for BinaryExpression { Self::Child::try_from(NodeTypes::from(self.right.as_ref().clone())) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_children__subtypes_with_children.snap b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_children__subtypes_with_children.snap index ec28b25..de597a6 100644 --- a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_children__subtypes_with_children.snap +++ b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_children__subtypes_with_children.snap @@ -173,7 +173,8 @@ impl CSTNode for Block { impl HasChildren for Block { type Child = BlockChildren; fn children(&self) -> Vec { - let children: Vec<_> = self.children.iter().cloned().collect(); + let mut children: Vec<_> = self.children.iter().cloned().collect(); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { @@ -286,7 +287,8 @@ impl CSTNode for IfStatement { impl HasChildren for IfStatement { type Child = IfStatementChildren; fn children(&self) -> Vec { - let children: Vec<_> = self.children.iter().cloned().collect(); + let mut children: Vec<_> = self.children.iter().cloned().collect(); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_recursive__recursive_subtypes.snap b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_recursive__recursive_subtypes.snap index 05cbe4a..51b6edb 100644 --- a/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_recursive__recursive_subtypes.snap +++ b/codegen-sdk-cst-generator/src/tests/snapshots/codegen_sdk_cst_generator__tests__test_subtypes_recursive__recursive_subtypes.snap @@ -185,6 +185,7 @@ impl HasChildren for BinaryExpression { Self::Child::try_from(NodeTypes::from(self.right.as_ref().clone())) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { @@ -330,6 +331,7 @@ impl HasChildren for CallExpression { Self::Child::try_from(NodeTypes::from(self.callee.as_ref().clone())) .unwrap(), ); + children.sort_by_key(|c| c.start_byte()); children } fn children_by_field_name(&self, field_name: &str) -> Vec { diff --git a/codegen-sdk-cst/Cargo.toml b/codegen-sdk-cst/Cargo.toml index 2a91f16..0351483 100644 --- a/codegen-sdk-cst/Cargo.toml +++ b/codegen-sdk-cst/Cargo.toml @@ -8,13 +8,13 @@ tree-sitter = { workspace = true } bytes = { workspace = true } codegen-sdk-common = { workspace = true } codegen-sdk-macros = { path = "../codegen-sdk-macros" } -derive_more = { version = "2.0.1", features = ["debug", "display"] } +derive_more = { workspace = true } convert_case = { workspace = true } rkyv = { workspace = true } subenum = "1.1.2" log = { workspace = true } enum_delegate = { workspace = true } -derive-visitor = "0.4.0" +derive-visitor = { workspace = true } [build-dependencies] codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator"} codegen-sdk-common = { workspace = true } @@ -22,8 +22,9 @@ rayon = { workspace = true } env_logger = { workspace = true } log = { workspace = true } [dev-dependencies] -tempfile = "3.16.0" +tempfile = { workspace = true } test-log = { workspace = true } + [features] python = [ "codegen-sdk-common/python"] typescript = [ "codegen-sdk-common/typescript"] @@ -38,5 +39,5 @@ ruby = [ "codegen-sdk-common/ruby"] yaml = [ "codegen-sdk-common/yaml"] toml = [ "codegen-sdk-common/toml"] markdown = [ "codegen-sdk-common/markdown"] -ts_query = [] -default = ["json", "ts_query", "toml", "typescript"] +ts_query = [ "codegen-sdk-common/ts_query"] +default = ["json", "ts_query", "toml"] diff --git a/codegen-sdk-cst/build.rs b/codegen-sdk-cst/build.rs index 1e514da..3fd0669 100644 --- a/codegen-sdk-cst/build.rs +++ b/codegen-sdk-cst/build.rs @@ -3,7 +3,7 @@ use codegen_sdk_cst_generator::generate_cst_to_file; use rayon::prelude::*; fn main() { env_logger::init(); - // println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=build.rs"); LANGUAGES.par_iter().for_each(|language| { generate_cst_to_file(language).unwrap_or_else(|e| { log::error!("Error generating CST for {}: {}", language.name(), e); diff --git a/codegen-sdk-cst/src/lib.rs b/codegen-sdk-cst/src/lib.rs index 1077757..3f9b313 100644 --- a/codegen-sdk-cst/src/lib.rs +++ b/codegen-sdk-cst/src/lib.rs @@ -1,12 +1,12 @@ #![recursion_limit = "512"] -#![feature(trivial_bounds)] +#![feature(trivial_bounds, extend_one)] use std::path::PathBuf; use codegen_sdk_common::{ParseError, serialize::Cache, traits::CSTNode}; use codegen_sdk_macros::{include_languages, parse_languages}; use rkyv::{api::high::to_bytes_in, from_bytes}; mod language; -use language::CSTLanguage; +pub use language::CSTLanguage; include_languages!(); pub fn parse_file( cache: &Cache, @@ -15,16 +15,36 @@ pub fn parse_file( parse_languages!(); Err(ParseError::UnknownLanguage) } -pub mod query; #[cfg(test)] mod tests { - use codegen_sdk_common::traits::HasChildren; + use derive_visitor::{Drive, Visitor}; use super::*; + use crate::typescript::ClassDeclaration; #[test_log::test] fn test_snazzy_items() { + let content = " + { + \"name\": \"SnazzyItems\" + } + "; + let module = json::JSON::parse(&content).unwrap(); + assert!(module.children().len() > 0); + } + #[derive(Visitor, Default)] + #[visitor(ClassDeclaration(enter))] + struct ClassVisitor { + pub items: Vec, + } + impl ClassVisitor { + fn enter_class_declaration(&mut self, node: &typescript::ClassDeclaration) { + self.items.push(node.name.source()); + } + } + #[test_log::test] + fn test_visitor() { let content = " class SnazzyItems { constructor() { @@ -33,6 +53,8 @@ mod tests { } "; let module = typescript::Typescript::parse(&content).unwrap(); - assert!(module.children().len() > 0); + let mut visitor = ClassVisitor::default(); + module.drive(&mut visitor); + assert_eq!(visitor.items, vec!["SnazzyItems"]); } } diff --git a/codegen-sdk-cst/src/main.rs b/codegen-sdk-cst/src/main.rs deleted file mode 100644 index a9a4d84..0000000 --- a/codegen-sdk-cst/src/main.rs +++ /dev/null @@ -1,10 +0,0 @@ -use codegen_sdk_common::language::LANGUAGES; -use codegen_sdk_cst::query::HasQuery; -fn main() { - for language in LANGUAGES.iter() { - println!("{}", language.name()); - println!("{:#?}", language.queries()); - println!("{:#?}", language.definitions()); - println!("{:#?}", language.references()); - } -} diff --git a/codegen-sdk-cst/src/query.rs b/codegen-sdk-cst/src/query.rs deleted file mode 100644 index 71a7e77..0000000 --- a/codegen-sdk-cst/src/query.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::collections::HashMap; - -use codegen_sdk_common::{CSTNode, HasChildren, Language, naming::normalize_type_name}; -use derive_more::Debug; - -use crate::{CSTLanguage, ts_query}; -fn captures_for_field_definition( - node: &ts_query::FieldDefinition, -) -> impl Iterator { - let mut captures = Vec::new(); - for child in node.children() { - match child { - ts_query::FieldDefinitionChildren::NamedNode(named) => { - captures.extend(captures_for_named_node(&named)); - } - ts_query::FieldDefinitionChildren::FieldDefinition(field) => { - captures.extend(captures_for_field_definition(&field)); - } - _ => {} - } - } - captures.into_iter() -} -fn captures_for_named_node(node: &ts_query::NamedNode) -> impl Iterator { - let mut captures = Vec::new(); - for child in node.children() { - match child { - ts_query::NamedNodeChildren::Capture(capture) => captures.push(capture), - ts_query::NamedNodeChildren::NamedNode(named) => { - captures.extend(captures_for_named_node(&named)); - } - ts_query::NamedNodeChildren::FieldDefinition(field) => { - captures.extend(captures_for_field_definition(&field)); - } - _ => {} - } - } - captures.into_iter() -} -#[derive(Debug)] -#[debug("{}", self.source())] -pub struct Query { - node: ts_query::NamedNode, -} -impl Query { - pub fn from_queries(source: &str) -> HashMap { - let parsed = ts_query::Query::parse(source).unwrap(); - let mut queries = HashMap::new(); - for node in parsed.children() { - match node { - ts_query::ProgramChildren::NamedNode(named) => { - let query = Self::from_named_node(&named); - queries.insert(query.name(), query); - } - node => { - println!("Unhandled query: {:#?}", node); - } - } - } - - // let root_node: Node<'a> = tree.root_node(); - // for child in root_node.children(&mut root_node.walk()) { - // if child.kind() == "grouping" { - // continue; - // // TODO: Handle grouping - // } - // let query = Self { - // query: child, - // source: source.to_string(), - // }; - // queries.insert(query.name(), query); - // } - queries - } - fn from_named_node(named: &ts_query::NamedNode) -> Self { - Query { - node: named.clone(), - } - } - /// Get the kind of the query (the node to be matched) - pub fn kind(&self) -> String { - if let ts_query::NamedNodeName::Identifier(identifier) = &(*self.node.name) { - return identifier.source(); - } - panic!("No kind found for query. {:#?}", self.node); - } - pub fn struct_name(&self) -> String { - normalize_type_name(&self.kind(), true) - } - - fn captures(&self) -> Vec { - captures_for_named_node(&self.node).collect() - } - /// Get the name of the query (IE @reference.class) - pub fn name(&self) -> String { - let mut result = self.captures().last().unwrap().source(); - result.replace_range(0..1, ""); - result - } - - // for node in self.query.named_children(&mut self.query.walk()) { - // for node in node.named_children(&mut self.query.walk()) { - // if node.kind() == "capture" { - // return get_text_from_node(&node, &self.source); - // } - // } - // } - - // panic!( - // "No name found for query. {:?}\n{}", - // self.query, - // self.source() - // ); - // } - - pub fn source(&self) -> String { - self.node.source() - } - // fn execute(&self, node: &T) -> Vec> { - // let mut result = Vec::new(); - - // for child in node.children() { - // if self - // .captures() - // .iter() - // .any(|capture| capture.source() == child.kind()) - // { - // result.push(child); - // } - // } - // result - // } -} - -pub trait HasQuery { - fn queries(&self) -> HashMap; - fn queries_with_prefix(&self, prefix: &str) -> HashMap { - let mut queries = HashMap::new(); - for (name, query) in self.queries().into_iter() { - if name.starts_with(prefix) { - let new_name = name.split(".").last().unwrap(); - queries.insert(new_name.to_string(), query); - } - } - queries - } - fn definitions(&self) -> HashMap { - self.queries_with_prefix("definition") - } - fn references(&self) -> HashMap { - self.queries_with_prefix("reference") - } -} -impl HasQuery for Language { - fn queries(&self) -> HashMap { - Query::from_queries(&self.tag_query) - } -} diff --git a/codegen-sdk-macros/src/lib.rs b/codegen-sdk-macros/src/lib.rs index d4eef2f..ef60224 100644 --- a/codegen-sdk-macros/src/lib.rs +++ b/codegen-sdk-macros/src/lib.rs @@ -9,6 +9,23 @@ fn get_language(language: &str) -> &Language { } panic!("Language not found"); } +#[proc_macro] +pub fn include_language_ast(_item: TokenStream) -> TokenStream { + let target_language = _item.to_string(); + let language = get_language(&target_language); + + format!( + "#[cfg(feature = \"{name}\")] +pub mod {name} {{ + use codegen_sdk_cst::{name}; + include!(concat!(env!(\"OUT_DIR\"), \"/{name}.rs\")); +}}", + name = language.name() + ) + .parse() + .unwrap() +} + #[proc_macro] pub fn include_language(_item: TokenStream) -> TokenStream { let target_language = _item.to_string(); @@ -83,3 +100,12 @@ pub fn include_languages(_item: TokenStream) -> TokenStream { } output.parse().unwrap() } +#[proc_macro] +pub fn include_languages_ast(_item: TokenStream) -> TokenStream { + let mut output = String::new(); + output.push_str("use codegen_sdk_macros::include_language_ast;"); + for language in LANGUAGES.iter() { + output.push_str(&format!("include_language_ast!({});", language.name())); + } + output.parse().unwrap() +}