feat: add support for proper parsing to dracula (#15)

* feat: add support for multiple new languages to be parsed

* feat: use tree-sitter for parsing grammars

to allow for slower but more accurate results

* feat: add support for meaningful lines to v2

also add support for Typescript and TSX

* add: support for new languages to pydracula

* fix: improve range match handling

* add tests

* add js, jsx and ts tests

* chore: improve pydracula bindings

* fix: as per suggestion

Signed-off-by: Swarnim Arun <[email protected]>

* add: indicies matching tests

---------

Signed-off-by: Swarnim Arun <[email protected]>

Author: swarnim-deepsource

Cargo.lock

@@ -1,8 +1,5 @@
 [workspace]
-members = [
-    "cdracula",
-    "pydracula"
-]
+members = ["cdracula", "pydracula"]
 
 [workspace.package]
 edition = "2021"
@@ -30,10 +27,25 @@ ra_ap_syntax = "0.0.149"
 pretty_assertions = "1.3.0"
 letr = "0.2.1"
 
+[dependencies]
+tree-sitter = "0.20.10"
+tree-sitter-rust = "0.20.3"
+tree-sitter-java = "0.20.0"
+tree-sitter-javascript = "0.20.0"
+tree-sitter-typescript = "0.20.2"
+tree-sitter-python = "0.20.2"
+tree-sitter-ruby = "0.20.0"
+tree-sitter-c-sharp = "0.20.0"
+tree-sitter-c = "0.20.2"
+tree-sitter-cpp = "0.20.0"
+tree-sitter-go = "0.19.1"
+tree-sitter-scala = { git = "https://github.com/tree-sitter/tree-sitter-scala", rev = "7d348f51e442563f4ab2b6c3e136dac658649f93" }
+tree-sitter-kotlin = { git = "https://github.com/swarnimarun/tree-sitter-kotlin", branch = "add-field-name" }
+
 [dependencies.log]
 version = "0.4"
 optional = true
 
 [features]
 log = ["dep:log"]
-cli = ["log"]
+cli = ["log"]

Cargo.toml

@@ -9,6 +9,23 @@ enum Lang {
     Java,
 }
 
+#[pyclass]
+#[derive(Debug, Clone, Copy)]
+enum Language {
+    Python,
+    Rust,
+    C,
+    Cpp,
+    Java,
+    Typescript,
+    Tsx,
+    Javascript,
+    Jsx,
+    Scala,
+    CSharp,
+    Ruby,
+}
+
 impl Lang {
     fn get_meaningful_line_indices(&self, src: &str) -> Vec<usize> {
         use dracula::count::*;
@@ -62,6 +79,35 @@ fn get_meaningful_line_indices(lang: Lang, src: &str) -> Vec<usize> {
     lang.get_meaningful_line_indices(src)
 }
 
+#[pyfunction]
+/// This function gets the list of lines that can be assumed to be meaningful/executable
+/// from a test-coverage or similar standpoint.
+///
+/// Further returns the list of line indexes starting from index 1.
+///
+/// If the parsing fails, then this returns None
+fn get_lines_with_executable_code(lang: Language, src: &str) -> Option<Vec<usize>> {
+    use dracula::parse::v2::*;
+    let treesitter_lang = match lang {
+        Language::Python => TreeSitterLanguage::Python,
+        Language::Rust => TreeSitterLanguage::Rust,
+        Language::C => TreeSitterLanguage::C,
+        Language::Cpp => TreeSitterLanguage::Cpp,
+        Language::Java => TreeSitterLanguage::Java,
+        Language::Typescript => TreeSitterLanguage::Typescript,
+        Language::Tsx => TreeSitterLanguage::TSX,
+        Language::Javascript | Language::Jsx => TreeSitterLanguage::Javascript,
+        // currently untested consider waiting for proper tests to be merged before using
+        Language::Scala => TreeSitterLanguage::Scala,
+        Language::CSharp => TreeSitterLanguage::CSharp,
+        Language::Ruby => TreeSitterLanguage::Ruby,
+        Language::Go => TreeSitterLanguage::Go,
+    };
+    Parser::new(treesitter_lang)
+        .and_then(|parser| parser.non_executable_src_spans(src))
+        .map(|spans| get_lines_without_ranges(src, &spans))
+}
+
 #[pyfunction]
 fn get_cleaned_source_code(lang: Lang, src: &str) -> String {
     lang.get_cleaned_source_code(src)
@@ -78,6 +124,7 @@ fn pydracula(_py: Python<'_>, m: &types::PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(get_meaningful_line_indices, m)?)?;
     m.add_function(wrap_pyfunction!(get_cleaned_source_code, m)?)?;
     m.add_function(wrap_pyfunction!(get_count_of_meaningful_lines, m)?)?;
+    m.add_function(wrap_pyfunction!(get_lines_with_executable_code, m)?)?;
     m.add_class::<Lang>()?;
     Ok(())
 }

pydracula/src/lib.rs

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly"