diff --git a/Cargo.lock b/Cargo.lock index 62de602a..313f75d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,8 +166,12 @@ name = "codegen-sdk-common" version = "0.1.0" dependencies = [ "bytes", + "lazy_static", "ouroboros", "tree-sitter", + "tree-sitter-javascript", + "tree-sitter-python", + "tree-sitter-typescript", ] [[package]] @@ -176,6 +180,7 @@ version = "0.1.0" dependencies = [ "clap", "codegen-sdk-analyzer", + "codegen-sdk-common", "codegen-sdk-cst", "crossbeam", "env_logger", @@ -192,12 +197,11 @@ dependencies = [ "bytes", "codegen-sdk-common", "codegen-sdk-cst-generator", + "codegen-sdk-macros", "convert_case", "ouroboros", "tempfile", "tree-sitter", - "tree-sitter-python", - "tree-sitter-typescript", ] [[package]] @@ -213,7 +217,13 @@ dependencies = [ "serde_json", "syn 2.0.98", "tree-sitter", - "tree-sitter-python", +] + +[[package]] +name = "codegen-sdk-macros" +version = "0.1.0" +dependencies = [ + "codegen-sdk-common", ] [[package]] @@ -442,6 +452,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.169" @@ -895,6 +911,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-javascript" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf40bf599e0416c16c125c3cec10ee5ddc7d1bb8b0c60fa5c4de249ad34dc1b1" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-language" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index 77ebaa6d..e3f8299d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,8 @@ edition = "2021" [dependencies] clap = { version = "4.5.28", features = ["derive"] } codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" } -codegen-sdk-cst = { path = "codegen-sdk-cst" } +codegen-sdk-cst = { path = "codegen-sdk-cst" , features = ["typescript", "javascript", "tsx", "jsx"]} +codegen-sdk-common = { path = "codegen-sdk-common" } crossbeam = "0.8.4" env_logger = "0.11.6" glob = "0.3.2" @@ -19,7 +20,7 @@ members = [ "codegen-sdk-ast", "codegen-sdk-common", "codegen-sdk-cst", - "codegen-sdk-cst-generator", + "codegen-sdk-cst-generator", "codegen-sdk-macros", ] [workspace.dependencies] log = "0.4.25" @@ -27,6 +28,7 @@ ouroboros = "0.18.5" tree-sitter = "0.25.1" tree-sitter-python = "0.23.6" tree-sitter-typescript = "0.23.2" +tree-sitter-javascript = "0.23.1" bytes = "1.10.0" convert_case = "0.7.1" serde = { version = "1.0.217", features = ["derive"] } diff --git a/codegen-sdk-common/Cargo.toml b/codegen-sdk-common/Cargo.toml index 301f3fd5..6e64a6f0 100644 --- a/codegen-sdk-common/Cargo.toml +++ b/codegen-sdk-common/Cargo.toml @@ -7,3 +7,10 @@ edition = "2021" bytes = { workspace = true } tree-sitter = { workspace = true } ouroboros = { workspace = true } +tree-sitter-python = { workspace = true, optional = true } +tree-sitter-typescript = { workspace = true, optional = true } +tree-sitter-javascript = { workspace = true, optional = true } +lazy_static = "1.5.0" +[features] +python = ["dep:tree-sitter-python"] +typescript = ["dep:tree-sitter-typescript", "dep:tree-sitter-javascript"] diff --git a/codegen-sdk-common/src/language.rs b/codegen-sdk-common/src/language.rs new file mode 100644 index 00000000..d652dcb2 --- /dev/null +++ b/codegen-sdk-common/src/language.rs @@ -0,0 +1,41 @@ +use tree_sitter::{LanguageError, Parser}; + +pub struct Language { + pub name: &'static str, + pub struct_name: &'static str, + pub node_types: &'static str, + pub file_extensions: &'static [&'static str], + pub tree_sitter_language: tree_sitter::Language, +} +impl Language { + pub fn parse_tree_sitter(&self, content: &str) -> Result { + let mut parser = Parser::new(); + parser.set_language(&self.tree_sitter_language)?; + let tree = parser.parse(content, None).unwrap(); + Ok(tree) + } +} +#[cfg(feature = "typescript")] +pub mod javascript; +#[cfg(feature = "typescript")] +pub mod jsx; +#[cfg(feature = "python")] +pub mod python; +#[cfg(feature = "typescript")] +pub mod tsx; +#[cfg(feature = "typescript")] +pub mod typescript; +lazy_static! { + pub static ref LANGUAGES: Vec<&'static Language> = vec![ + #[cfg(feature = "python")] + &python::Python, + #[cfg(feature = "typescript")] + &typescript::Typescript, + #[cfg(feature = "typescript")] + &tsx::TSX, + #[cfg(feature = "typescript")] + &jsx::JSX, + #[cfg(feature = "typescript")] + &javascript::Javascript, + ]; +} diff --git a/codegen-sdk-common/src/language/javascript.rs b/codegen-sdk-common/src/language/javascript.rs new file mode 100644 index 00000000..1d63aa84 --- /dev/null +++ b/codegen-sdk-common/src/language/javascript.rs @@ -0,0 +1,11 @@ +use super::Language; + +lazy_static! { + pub static ref Javascript: Language = Language { + name: "javascript", + struct_name: "Javascript", + node_types: tree_sitter_javascript::NODE_TYPES, + file_extensions: &["js"], + tree_sitter_language: tree_sitter_javascript::LANGUAGE.into(), + }; +} diff --git a/codegen-sdk-common/src/language/jsx.rs b/codegen-sdk-common/src/language/jsx.rs new file mode 100644 index 00000000..b84466c4 --- /dev/null +++ b/codegen-sdk-common/src/language/jsx.rs @@ -0,0 +1,10 @@ +use super::Language; +lazy_static! { + pub static ref JSX: Language = Language { + name: "jsx", + struct_name: "JSX", + node_types: tree_sitter_typescript::TSX_NODE_TYPES, + file_extensions: &["jsx"], + tree_sitter_language: tree_sitter_typescript::LANGUAGE_TSX.into(), + }; +} diff --git a/codegen-sdk-common/src/language/python.rs b/codegen-sdk-common/src/language/python.rs new file mode 100644 index 00000000..d672b7d7 --- /dev/null +++ b/codegen-sdk-common/src/language/python.rs @@ -0,0 +1,10 @@ +use super::Language; +lazy_static! { + pub static ref Python: Language = Language { + name: "python", + struct_name: "Python", + node_types: tree_sitter_python::NODE_TYPES, + file_extensions: &["py"], + tree_sitter_language: tree_sitter_python::LANGUAGE.into(), + }; +} diff --git a/codegen-sdk-common/src/language/tsx.rs b/codegen-sdk-common/src/language/tsx.rs new file mode 100644 index 00000000..1d044049 --- /dev/null +++ b/codegen-sdk-common/src/language/tsx.rs @@ -0,0 +1,11 @@ +use super::Language; + +lazy_static! { + pub static ref TSX: Language = Language { + name: "tsx", + struct_name: "TSX", + node_types: tree_sitter_typescript::TSX_NODE_TYPES, + file_extensions: &["tsx"], + tree_sitter_language: tree_sitter_typescript::LANGUAGE_TSX.into(), + }; +} diff --git a/codegen-sdk-common/src/language/typescript.rs b/codegen-sdk-common/src/language/typescript.rs new file mode 100644 index 00000000..813e76ed --- /dev/null +++ b/codegen-sdk-common/src/language/typescript.rs @@ -0,0 +1,11 @@ +use super::Language; + +lazy_static! { + pub static ref Typescript: Language = Language { + name: "typescript", + struct_name: "Typescript", + node_types: tree_sitter_typescript::TYPESCRIPT_NODE_TYPES, + file_extensions: &["ts"], + tree_sitter_language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + }; +} diff --git a/codegen-sdk-common/src/lib.rs b/codegen-sdk-common/src/lib.rs index 1ba490a6..55581970 100644 --- a/codegen-sdk-common/src/lib.rs +++ b/codegen-sdk-common/src/lib.rs @@ -1,2 +1,5 @@ +pub mod language; pub mod traits; pub mod utils; +#[macro_use] +extern crate lazy_static; diff --git a/codegen-sdk-common/src/traits.rs b/codegen-sdk-common/src/traits.rs index c68c8408..2c5476cf 100644 --- a/codegen-sdk-common/src/traits.rs +++ b/codegen-sdk-common/src/traits.rs @@ -3,7 +3,7 @@ use tree_sitter::{self, Point}; pub trait FromNode { fn from_node(node: tree_sitter::Node) -> Self; } -pub trait CSTNode { +pub trait CSTNode: Send { fn start_byte(&self) -> usize; fn end_byte(&self) -> usize; fn start_position(&self) -> Point; @@ -13,3 +13,7 @@ pub trait CSTNode { String::from_utf8(self.text().to_vec()).unwrap() } } +pub trait HasChildren { + type Child: Send; + fn children(&self) -> &Vec; +} diff --git a/codegen-sdk-cst-generator/Cargo.toml b/codegen-sdk-cst-generator/Cargo.toml index 8dbbb213..5f42fdc6 100644 --- a/codegen-sdk-cst-generator/Cargo.toml +++ b/codegen-sdk-cst-generator/Cargo.toml @@ -14,4 +14,4 @@ tree-sitter = { workspace = true } log = { workspace = true } codegen-sdk-common = { path = "../codegen-sdk-common" } [dev-dependencies] -tree-sitter-python = { workspace = true } +codegen-sdk-common = { path = "../codegen-sdk-common" , features = ["python"] } diff --git a/codegen-sdk-cst-generator/src/generator.rs b/codegen-sdk-cst-generator/src/generator.rs index 2c83fd7a..afbdce99 100644 --- a/codegen-sdk-cst-generator/src/generator.rs +++ b/codegen-sdk-cst-generator/src/generator.rs @@ -57,9 +57,10 @@ mod tests { use crate::parser::parse_node_types; use super::*; + use codegen_sdk_common::language::python::Python; #[test] fn test_generate_cst() { - let node_types = parse_node_types(tree_sitter_python::NODE_TYPES).unwrap(); + let node_types = parse_node_types(&Python).unwrap(); let cst = generate_cst(&node_types).unwrap(); log::info!("{}", cst); } diff --git a/codegen-sdk-cst-generator/src/generator/struct_generator.rs b/codegen-sdk-cst-generator/src/generator/struct_generator.rs index 0f85fcc1..8cd085ca 100644 --- a/codegen-sdk-cst-generator/src/generator/struct_generator.rs +++ b/codegen-sdk-cst-generator/src/generator/struct_generator.rs @@ -36,6 +36,12 @@ impl CSTNode for {{name}} { &self.text } } +impl HasChildren for {{name}} { + type Child = {{children}}; + fn children(&self) -> &Vec { + self.children.as_ref() + } +} impl FromNode for {{name}} { fn from_node(node: tree_sitter::Node) -> Self { Self { @@ -154,14 +160,11 @@ fn generate_children( state: &mut State, node_name: &str, constructor_fields: &mut Vec, -) { +) -> String { let converted_type_name = convert_type_definition(&children.types, state, node_name, "children"); - state.structs.push_str(&format!( - " pub children: Vec<{}>,\n", - converted_type_name - )); constructor_fields.push(format!(" children: named_children_without_field_names(node).into_iter().map(|node| {converted_type_name}::from_node(node)).collect()", converted_type_name = converted_type_name)); + converted_type_name } pub fn generate_struct(node: &Node, state: &mut State, name: &str) { state @@ -171,13 +174,21 @@ pub fn generate_struct(node: &Node, state: &mut State, name: &str) { if let Some(fields) = &node.fields { generate_fields(fields, state, node, &mut constructor_fields); } + let mut children_type_name = "Self".to_string(); if let Some(children) = &node.children { - generate_children(children, state, &node.type_name, &mut constructor_fields); + children_type_name = + generate_children(children, state, &node.type_name, &mut constructor_fields); + } else { + constructor_fields.push(" children: vec![]".to_string()); } + state + .structs + .push_str(&format!(" pub children: Vec<{}>,\n", children_type_name)); state.structs.push_str(FOOTER_TEMPLATE); state.structs.push_str( &CONSTRUCTOR_TEMPLATE .replace("{{fields}}", &constructor_fields.join(",\n ")) - .replace("{{name}}", name), + .replace("{{name}}", name) + .replace("{{children}}", &children_type_name), ); } diff --git a/codegen-sdk-cst-generator/src/lib.rs b/codegen-sdk-cst-generator/src/lib.rs index 395f1f0e..19bbc6eb 100644 --- a/codegen-sdk-cst-generator/src/lib.rs +++ b/codegen-sdk-cst-generator/src/lib.rs @@ -1,12 +1,13 @@ mod generator; pub(crate) mod parser; +use codegen_sdk_common::language::Language; use std::error::Error; -pub fn generate_cst(source: &str, language: &str) -> Result<(), Box> { - let node_types = parser::parse_node_types(source)?; +pub fn generate_cst(language: &Language) -> Result<(), Box> { + let node_types = parser::parse_node_types(language)?; let cst = generator::generate_cst(&node_types)?; let out_dir = std::env::var("OUT_DIR").unwrap(); - let out_file = format!("{}/{}.rs", out_dir, language); + let out_file = format!("{}/{}.rs", out_dir, language.name); std::fs::write(out_file, cst)?; Ok(()) } diff --git a/codegen-sdk-cst-generator/src/parser.rs b/codegen-sdk-cst-generator/src/parser.rs index 8b86a1d8..c6aabf24 100644 --- a/codegen-sdk-cst-generator/src/parser.rs +++ b/codegen-sdk-cst-generator/src/parser.rs @@ -1,5 +1,6 @@ use std::error::Error; +use codegen_sdk_common::language::Language; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] @@ -44,18 +45,17 @@ pub struct Children { pub types: Vec, } -pub fn parse_node_types(source: &str) -> Result, Box> { - let parsed: Vec = serde_json::from_str(source)?; +pub fn parse_node_types(language: &Language) -> Result, Box> { + let parsed: Vec = serde_json::from_str(language.node_types)?; Ok(parsed) } #[cfg(test)] mod tests { use super::*; - + use codegen_sdk_common::language::python::Python; #[test] fn test_parse_node_types() { - let source = tree_sitter_python::NODE_TYPES; - let cst = parse_node_types(source).unwrap(); + let cst = parse_node_types(&Python).unwrap(); assert!(!cst.is_empty()); } } diff --git a/codegen-sdk-cst/Cargo.toml b/codegen-sdk-cst/Cargo.toml index 93b25219..b92a6020 100644 --- a/codegen-sdk-cst/Cargo.toml +++ b/codegen-sdk-cst/Cargo.toml @@ -5,17 +5,20 @@ edition = "2021" [dependencies] tree-sitter = { workspace = true } -tree-sitter-python = { workspace = true } ouroboros = { workspace = true } -tree-sitter-typescript = { workspace = true } bytes = { workspace = true } codegen-sdk-common = { path = "../codegen-sdk-common" } +codegen-sdk-macros = { path = "../codegen-sdk-macros" } convert_case = { workspace = true } [build-dependencies] -codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator" } -codegen-sdk-common = { path = "../codegen-sdk-common" } -tree-sitter-python = { workspace = true } -tree-sitter-typescript = { workspace = true } +codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator"} +codegen-sdk-common = { path = "../codegen-sdk-common", features = ["python", "typescript"] } [dev-dependencies] tempfile = "3.16.0" +[features] +python = [ "codegen-sdk-common/python"] +typescript = [ "codegen-sdk-common/typescript"] +tsx = [ "codegen-sdk-common/typescript"] +jsx = [ "codegen-sdk-common/typescript"] +javascript = [ "codegen-sdk-common/typescript"] diff --git a/codegen-sdk-cst/build.rs b/codegen-sdk-cst/build.rs index 9d2b75bb..46a6802a 100644 --- a/codegen-sdk-cst/build.rs +++ b/codegen-sdk-cst/build.rs @@ -1,24 +1,8 @@ -use std::collections::HashMap; - +use codegen_sdk_common::language::LANGUAGES; use codegen_sdk_cst_generator::generate_cst; -fn get_node_types() -> HashMap { - HashMap::from([ - ( - "python".to_string(), - tree_sitter_python::NODE_TYPES.to_string(), - ), - ( - "typescript".to_string(), - tree_sitter_typescript::TYPESCRIPT_NODE_TYPES.to_string(), - ), - ( - "tsx".to_string(), - tree_sitter_typescript::TSX_NODE_TYPES.to_string(), - ), - ]) -} + fn main() { - for (language, node_types) in get_node_types() { - generate_cst(&node_types, &language).unwrap(); + for language in LANGUAGES.iter() { + generate_cst(language).unwrap(); } } diff --git a/codegen-sdk-cst/src/lib.rs b/codegen-sdk-cst/src/lib.rs index c1dce276..e45b31b1 100644 --- a/codegen-sdk-cst/src/lib.rs +++ b/codegen-sdk-cst/src/lib.rs @@ -1,33 +1,38 @@ use std::{ error::Error, fmt::{self, Display}, - fs::File, - io::{BufReader, Read}, - panic::catch_unwind, + path::PathBuf, }; -use codegen_sdk_common::traits::FromNode; -use tree_sitter::{Language, Parser}; -fn parse_file(file_path: &str, language: Language) -> Result> { - let file = File::open(file_path)?; - let mut reader = BufReader::new(file); - let mut buffer = String::new(); - let mut parser = Parser::new(); - parser.set_language(&language)?; - reader.read_to_string(&mut buffer)?; - let tree = parser.parse(&buffer, None).unwrap(); - Ok(tree) -} -// mod python { -// include!(concat!(env!("OUT_DIR"), "/python.rs")); -// } -pub mod typescript { - include!(concat!(env!("OUT_DIR"), "/typescript.rs")); -} -pub mod tsx { - include!(concat!(env!("OUT_DIR"), "/tsx.rs")); -} +use codegen_sdk_common::{ + language::Language, + traits::{CSTNode, FromNode}, +}; +use codegen_sdk_macros::{include_language, parse_language}; + +pub trait CSTLanguage { + type Program: CSTNode + FromNode + Send; + fn language() -> &'static Language; + fn parse(content: &str) -> Result> { + let tree = Self::language().parse_tree_sitter(content)?; + Ok(Self::Program::from_node(tree.root_node())) + } + fn parse_file(file_path: &PathBuf) -> Result> { + let content = std::fs::read_to_string(file_path)?; + Self::parse(&content) + } + fn should_parse(file_path: &PathBuf) -> bool { + Self::language() + .file_extensions + .contains(&file_path.extension().unwrap().to_str().unwrap()) + } +} +include_language!(python); +include_language!(typescript); +include_language!(tsx); +include_language!(jsx); +include_language!(javascript); #[derive(Debug)] struct ParseError {} impl Error for ParseError {} @@ -36,18 +41,19 @@ impl Display for ParseError { write!(f, "ParseError") } } -pub fn parse_file_typescript(file_path: &str) -> Result, Box> { - let tree = parse_file(file_path, tree_sitter_typescript::LANGUAGE_TSX.into())?; - Ok( - catch_unwind(|| Box::new(tsx::Program::from_node(tree.root_node()))) - .map_err(|_e| ParseError {})?, - ) +pub fn parse_file(file_path: &PathBuf) -> Result, Box> { + parse_language!(python); + parse_language!(typescript); + parse_language!(tsx); + parse_language!(jsx); + parse_language!(javascript); + Err(Box::new(ParseError {})) } + #[cfg(test)] mod tests { - use std::io::Write; - use tempfile::tempdir; + use codegen_sdk_common::traits::HasChildren; use super::*; #[test] @@ -59,11 +65,7 @@ mod tests { } } "; - let dir = tempdir().unwrap(); - let path = dir.into_path().join("snazzy_items.ts"); - let mut file = File::create(&path).unwrap(); - file.write_all(content.as_bytes()).unwrap(); - let module: Box = parse_file_typescript(&path.to_str().unwrap()).unwrap(); - assert!(module.children.len() > 0); + let module = typescript::Typescript::parse(&content).unwrap(); + assert!(module.children().len() > 0); } } diff --git a/codegen-sdk-macros/Cargo.toml b/codegen-sdk-macros/Cargo.toml new file mode 100644 index 00000000..a8525b7d --- /dev/null +++ b/codegen-sdk-macros/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "codegen-sdk-macros" +version = "0.1.0" +edition = "2021" + +[dependencies] +codegen-sdk-common = { path = "../codegen-sdk-common", features = ["typescript", "python"] } +[lib] +proc-macro = true diff --git a/codegen-sdk-macros/src/lib.rs b/codegen-sdk-macros/src/lib.rs new file mode 100644 index 00000000..bb923e91 --- /dev/null +++ b/codegen-sdk-macros/src/lib.rs @@ -0,0 +1,54 @@ +extern crate proc_macro; +use codegen_sdk_common::language::{Language, LANGUAGES}; +use proc_macro::TokenStream; +fn get_language(language: &str) -> &Language { + for lang in LANGUAGES.iter() { + if lang.name.to_lowercase() == language.to_lowercase() { + return lang; + } + } + panic!("Language not found"); +} +#[proc_macro] +pub fn include_language(_item: TokenStream) -> TokenStream { + let target_language = _item.to_string(); + let language = get_language(&target_language); + + format!( + "#[cfg(feature = \"{name}\")] +pub mod {name} {{ + use crate::CSTLanguage; + use codegen_sdk_common::language::Language; + include!(concat!(env!(\"OUT_DIR\"), \"/{name}.rs\")); + pub struct {struct_name}; + impl CSTLanguage for {struct_name} {{ + type Program = Program; + fn language() -> &'static Language {{ + &codegen_sdk_common::language::{name}::{struct_name} + }} + }} +}}", + name = language.name, + struct_name = language.struct_name + ) + .parse() + .unwrap() +} + +#[proc_macro] +pub fn parse_language(_item: TokenStream) -> TokenStream { + let target_language = _item.to_string(); + let language = get_language(&target_language); + format!( + "#[cfg(feature = \"{name}\")] + if {name}::{struct_name}::should_parse(file_path) {{ + let parsed = {name}::{struct_name}::parse_file(file_path)?; + return Ok(Box::new(parsed)); + }} + ", + name = language.name, + struct_name = language.struct_name + ) + .parse() + .unwrap() +} diff --git a/src/main.rs b/src/main.rs index 65f24770..b46c5d1a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,8 @@ use clap::Parser; -use codegen_sdk_cst::{parse_file_typescript, tsx}; +use codegen_sdk_common::{language::LANGUAGES, traits::CSTNode}; use glob::glob; use rayon::prelude::*; -use std::{path, time::Instant}; +use std::{panic::catch_unwind, path, time::Instant}; use sysinfo::System; #[derive(Debug, Parser)] struct Args { @@ -22,19 +22,34 @@ fn collect_files(dir: String) -> Vec { fn parse_file( file: &path::PathBuf, tx: &crossbeam::channel::Sender, -) -> Option> { +) -> Option> { if file.is_dir() { return None; } - return match parse_file_typescript(file.to_str().unwrap()) { - Ok(program) => Some(program), - Err(e) => { + let result = catch_unwind(|| codegen_sdk_cst::parse_file(file)); + + return match result { + Ok(Ok(program)) => Some(program), + Ok(Err(e)) => { tx.send(e.to_string()).unwrap(); None } + Err(_) => { + tx.send("".to_string()).unwrap(); + None + } }; } -fn parse_files(dir: String) -> (Vec>, Vec) { +fn log_languages() { + for language in LANGUAGES.iter() { + log::info!( + "Supported language: {} with extensions: {:?}", + language.name, + language.file_extensions + ); + } +} +fn parse_files(dir: String) -> (Vec>, Vec) { rayon::ThreadPoolBuilder::new() .stack_size(1024 * 1024 * 1024 * 10) .build_global() @@ -43,7 +58,8 @@ fn parse_files(dir: String) -> (Vec>, Vec) { let mut errors = Vec::new(); let files_to_parse = collect_files(dir); log::info!("Parsing {} files", files_to_parse.len()); - let files: Vec> = files_to_parse + log_languages(); + let files: Vec> = files_to_parse .par_iter() .filter_map(|file| parse_file(file, &tx)) .collect();