Add logging (#5)

* Clean up deps and add logs

* Refactor and add env logger

* Refactor

* reformat

Author: bagel897

Cargo.lock

@@ -8,8 +8,9 @@ clap = { version = "4.5.28", features = ["derive"] }
 codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" }
 codegen-sdk-cst = { path = "codegen-sdk-cst" }
 crossbeam = "0.8.4"
+env_logger = "0.11.6"
 glob = "0.3.2"
-phf = { version = "0.11.3", features = ["macros"] }
+log = { workspace = true }
 rayon = "1.10.0"
 sysinfo = "0.33.1"
 [workspace]
@@ -21,4 +22,12 @@ members = [
   "codegen-sdk-cst-generator",
 ]
 [workspace.dependencies]
+log = "0.4.25"
 ouroboros = "0.18.5"
+tree-sitter = "0.25.1"
+tree-sitter-python = "0.23.6"
+tree-sitter-typescript = "0.23.2"
+bytes = "1.10.0"
+convert_case = "0.7.1"
+serde = { version = "1.0.217", features = ["derive"] }
+serde_json = "1.0.138"

Cargo.toml

@@ -6,4 +6,3 @@ edition = "2021"
 [dependencies]
 salsa = "0.16.1"
 codegen-sdk-ast = { path = "../codegen-sdk-ast" }
-convert_case = "0.7.1"

codegen-sdk-analyzer/Cargo.toml

@@ -4,6 +4,6 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-bytes = "1.10.0"
-tree-sitter = "0.25.1"
-ouroboros = "0.18.5"
+bytes = { workspace = true }
+tree-sitter = { workspace = true }
+ouroboros = { workspace = true }

codegen-sdk-common/Cargo.toml

@@ -4,14 +4,14 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-convert_case = "0.7.1"
+convert_case = { workspace = true }
 phf = { version = "0.11.3", features = ["macros"] }
 prettyplease = "0.2.29"
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = "1.0.138"
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
 syn = "2.0.98"
-tree-sitter = "0.25.1"
+tree-sitter = { workspace = true }
+log = { workspace = true }
 codegen-sdk-common = { path = "../codegen-sdk-common" }
 [dev-dependencies]
-serde_json = "1.0.138"
-tree-sitter-python = "0.23.6"
+tree-sitter-python = { workspace = true }

codegen-sdk-cst-generator/Cargo.toml

@@ -61,6 +61,6 @@ mod tests {
     fn test_generate_cst() {
         let node_types = parse_node_types(tree_sitter_python::NODE_TYPES).unwrap();
         let cst = generate_cst(&node_types).unwrap();
-        println!("{}", cst);
+        log::info!("{}", cst);
     }
 }

codegen-sdk-cst-generator/src/generator.rs

@@ -1,7 +1,7 @@
 pub fn format_cst(cst: &str) -> String {
     let parsed = syn::parse_str::<syn::File>(cst)
         .map_err(|e| {
-            println!("{:#?}", e);
+            log::error!("{:#?}", e);
             e
         })
         .unwrap();

codegen-sdk-cst-generator/src/generator/format.rs

@@ -4,17 +4,18 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-tree-sitter = "0.25.1"
-tree-sitter-python = "0.23.6"
-ouroboros = "0.18.5"
-tree-sitter-typescript = "0.23.2"
-bytes = "1.10.0"
+tree-sitter = { workspace = true }
+tree-sitter-python = { workspace = true }
+ouroboros = { workspace = true }
+tree-sitter-typescript = { workspace = true }
+bytes = { workspace = true }
 codegen-sdk-common = { path = "../codegen-sdk-common" }
+convert_case = { workspace = true }
 [build-dependencies]
 codegen-sdk-cst-generator = { path = "../codegen-sdk-cst-generator" }
 codegen-sdk-common = { path = "../codegen-sdk-common" }
-tree-sitter-python = "0.23.6"
-tree-sitter-typescript = "0.23.2"
+tree-sitter-python = { workspace = true }
+tree-sitter-typescript = { workspace = true }
 
 [dev-dependencies]
 tempfile = "3.16.0"

codegen-sdk-cst/Cargo.toml

@@ -1,6 +1,4 @@
 {
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended"
-  ]
+	"$schema": "https://docs.renovatebot.com/renovate-schema.json",
+	"extends": ["config:recommended"]
 }

renovate.json

@@ -8,47 +8,61 @@ use sysinfo::System;
 struct Args {
     input: String,
 }
-fn main() {
+fn get_memory() -> u64 {
+    let s = System::new_all();
+    let current = s.process(sysinfo::get_current_pid().unwrap()).unwrap();
+    current.memory()
+}
+fn collect_files(dir: String) -> Vec<path::PathBuf> {
+    glob(&format!("{}/**/*.ts*", dir))
+        .unwrap()
+        .filter_map(|file| file.ok())
+        .collect()
+}
+fn parse_file(
+    file: &path::PathBuf,
+    tx: &crossbeam::channel::Sender<String>,
+) -> Option<Box<tsx::Program>> {
+    if file.is_dir() {
+        return None;
+    }
+    return match parse_file_typescript(file.to_str().unwrap()) {
+        Ok(program) => Some(program),
+        Err(e) => {
+            tx.send(e.to_string()).unwrap();
+            None
+        }
+    };
+}
+fn parse_files(dir: String) -> (Vec<Box<tsx::Program>>, Vec<String>) {
     rayon::ThreadPoolBuilder::new()
         .stack_size(1024 * 1024 * 1024 * 10)
         .build_global()
         .unwrap();
-    let args = Args::parse();
-    let dir = args.input;
-    let mut errors = Vec::new();
     let (tx, rx) = crossbeam::channel::unbounded();
-    let start = Instant::now();
-    let files_to_parse: Vec<Result<path::PathBuf, glob::GlobError>> =
-        glob(&format!("{}/**/*.ts*", dir)).unwrap().collect();
-
+    let mut errors = Vec::new();
+    let files_to_parse = collect_files(dir);
+    log::info!("Parsing {} files", files_to_parse.len());
     let files: Vec<Box<tsx::Program>> = files_to_parse
         .par_iter()
-        .filter_map(|file| {
-            if let Ok(file) = file {
-                if file.is_dir() {
-                    return None;
-                }
-                return match parse_file_typescript(file.to_str().unwrap()) {
-                    Ok(program) => Some(program),
-                    Err(e) => {
-                        tx.send(e.to_string()).unwrap();
-                        None
-                    }
-                };
-            }
-            None
-        })
+        .filter_map(|file| parse_file(file, &tx))
         .collect();
-    let end = Instant::now();
-    let duration: std::time::Duration = end.duration_since(start);
     drop(tx);
     for e in rx.iter() {
         errors.push(e);
     }
-    let s = System::new_all();
-    let current = s.process(sysinfo::get_current_pid().unwrap()).unwrap();
-    let memory = current.memory();
-    println!(
+    (files, errors)
+}
+fn main() {
+    env_logger::init();
+    let args = Args::parse();
+    let dir = args.input;
+    let start = Instant::now();
+    let (files, errors) = parse_files(dir);
+    let end = Instant::now();
+    let duration: std::time::Duration = end.duration_since(start);
+    let memory = get_memory();
+    log::info!(
         "{} files parsed in {:?}.{} seconds with {} errors. Using {} MB of memory",
         files.len(),
         duration.as_secs(),