Improve using libclang and fix memory issue

Author: AmrDeveloper

src/clang_parser.rs

@@ -0,0 +1,49 @@
+use std::ffi::CString;
+use std::ptr;
+
+use clang_sys::clang_createIndex;
+use clang_sys::clang_parseTranslationUnit;
+use clang_sys::CXIndex;
+use clang_sys::CXTranslationUnit;
+
+pub struct CompilationUnit {
+    #[allow(dead_code)]
+    pub path: String,
+
+    pub index: CXIndex,
+    pub translation_unit: CXTranslationUnit,
+}
+
+pub fn parse_files(files: &[String]) -> Vec<CompilationUnit> {
+    let mut translation_units: Vec<CompilationUnit> = vec![];
+
+    for file in files {
+        unsafe {
+            let fname: CString = CString::new(file.as_str()).unwrap();
+            let index: CXIndex = clang_createIndex(0, 0);
+            let translation_unit: CXTranslationUnit = clang_parseTranslationUnit(
+                index,
+                fname.as_ptr(),
+                ptr::null_mut(),
+                0,
+                ptr::null_mut(),
+                0,
+                0,
+            );
+
+            if translation_unit.is_null() {
+                continue;
+            }
+
+            let compilation_unit = CompilationUnit {
+                path: file.to_string(),
+                index,
+                translation_unit,
+            };
+
+            translation_units.push(compilation_unit);
+        }
+    }
+
+    translation_units
+}

src/data_provider.rs

@@ -1,5 +1,7 @@
 use std::vec;
 
+use clang_sys::clang_disposeIndex;
+use clang_sys::clang_disposeTranslationUnit;
 use gitql_core::object::Row;
 use gitql_core::values::base::Value;
 use gitql_core::values::boolean::BoolValue;
@@ -8,54 +10,71 @@ use gitql_core::values::null::NullValue;
 use gitql_core::values::text::TextValue;
 use gitql_engine::data_provider::DataProvider;
 
+use crate::clang_parser::CompilationUnit;
 use crate::visitor::class;
 use crate::visitor::enumeration;
 use crate::visitor::function;
 use crate::visitor::global;
 use crate::visitor::unions;
 
-pub struct ClangAstDataProvider {
-    pub paths: Vec<String>,
+pub struct ClangDataProvider {
+    pub compilation_units: Vec<CompilationUnit>,
 }
 
-impl ClangAstDataProvider {
-    pub fn new(paths: Vec<String>) -> Self {
-        Self { paths }
+impl ClangDataProvider {
+    pub fn new(compilation_units: Vec<CompilationUnit>) -> Self {
+        Self { compilation_units }
     }
 }
 
-impl DataProvider for ClangAstDataProvider {
+impl DataProvider for ClangDataProvider {
     fn provide(&self, table: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
         let mut rows: Vec<Row> = vec![];
-
-        for path in &self.paths {
-            let mut selected_rows = select_clang_ast_objects(path, table, selected_columns)?;
+        for compilation_unit in &self.compilation_units {
+            let mut selected_rows =
+                select_clang_ast_objects(compilation_unit, table, selected_columns)?;
             rows.append(&mut selected_rows);
         }
-
         Ok(rows)
     }
 }
 
+impl Drop for ClangDataProvider {
+    fn drop(&mut self) {
+        for compilation_unit in self.compilation_units.iter() {
+            unsafe {
+                // Dispose the translation unit
+                clang_disposeTranslationUnit(compilation_unit.translation_unit);
+
+                // Dispose the index
+                clang_disposeIndex(compilation_unit.index);
+            }
+        }
+    }
+}
+
 fn select_clang_ast_objects(
-    path: &str,
+    compilation_unit: &CompilationUnit,
     table: &str,
     selected_columns: &[String],
 ) -> Result<Vec<Row>, String> {
     let rows = match table {
-        "classes" => select_classes(path, selected_columns)?,
-        "enums" => select_enums(path, selected_columns)?,
-        "unions" => select_unions(path, selected_columns)?,
-        "functions" => select_functions(path, selected_columns)?,
-        "globals" => select_variables(path, selected_columns)?,
+        "classes" => select_classes(compilation_unit, selected_columns)?,
+        "enums" => select_enums(compilation_unit, selected_columns)?,
+        "unions" => select_unions(compilation_unit, selected_columns)?,
+        "functions" => select_functions(compilation_unit, selected_columns)?,
+        "globals" => select_variables(compilation_unit, selected_columns)?,
         _ => vec![Row { values: vec![] }],
     };
     Ok(rows)
 }
 
-fn select_classes(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
+fn select_classes(
+    compilation_unit: &CompilationUnit,
+    selected_columns: &[String],
+) -> Result<Vec<Row>, String> {
     let mut rows: Vec<Row> = vec![];
-    let ast_classes = class::select_clang_classes(path);
+    let ast_classes = class::select_clang_classes(compilation_unit);
     for class in ast_classes.iter() {
         let mut values: Vec<Box<dyn Value>> = Vec::with_capacity(selected_columns.len());
 
@@ -129,9 +148,12 @@ fn select_classes(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, S
     Ok(rows)
 }
 
-fn select_enums(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
+fn select_enums(
+    compilation_unit: &CompilationUnit,
+    selected_columns: &[String],
+) -> Result<Vec<Row>, String> {
     let mut rows: Vec<Row> = vec![];
-    let ast_enums = enumeration::select_clang_enums(path);
+    let ast_enums = enumeration::select_clang_enums(compilation_unit);
     for enumeration in ast_enums.iter() {
         let mut values: Vec<Box<dyn Value>> = Vec::with_capacity(selected_columns.len());
 
@@ -184,9 +206,12 @@ fn select_enums(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, Str
     Ok(rows)
 }
 
-fn select_unions(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
+fn select_unions(
+    compilation_unit: &CompilationUnit,
+    selected_columns: &[String],
+) -> Result<Vec<Row>, String> {
     let mut rows: Vec<Row> = vec![];
-    let ast_unions = unions::select_clang_unions(path);
+    let ast_unions = unions::select_clang_unions(compilation_unit);
     for union_node in ast_unions.iter() {
         let mut values: Vec<Box<dyn Value>> = Vec::with_capacity(selected_columns.len());
 
@@ -238,9 +263,12 @@ fn select_unions(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, St
     Ok(rows)
 }
 
-fn select_functions(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
+fn select_functions(
+    compilation_unit: &CompilationUnit,
+    selected_columns: &[String],
+) -> Result<Vec<Row>, String> {
     let mut rows: Vec<Row> = vec![];
-    let ast_functions = function::select_clang_functions(path);
+    let ast_functions = function::select_clang_functions(compilation_unit);
     for function in ast_functions.iter() {
         let mut values: Vec<Box<dyn Value>> = Vec::with_capacity(selected_columns.len());
 
@@ -340,9 +368,12 @@ fn select_functions(path: &str, selected_columns: &[String]) -> Result<Vec<Row>,
     Ok(rows)
 }
 
-fn select_variables(path: &str, selected_columns: &[String]) -> Result<Vec<Row>, String> {
+fn select_variables(
+    compilation_unit: &CompilationUnit,
+    selected_columns: &[String],
+) -> Result<Vec<Row>, String> {
     let mut rows: Vec<Row> = vec![];
-    let ast_variables = global::select_clang_variables(path);
+    let ast_variables = global::select_clang_variables(compilation_unit);
     for variable in ast_variables.iter() {
         let mut values: Vec<Box<dyn Value>> = Vec::with_capacity(selected_columns.len());
         for field_name in selected_columns {

src/main.rs

@@ -4,7 +4,8 @@ use std::path::Path;
 use crate::engine::EvaluationResult::SelectedGroups;
 use arguments::Arguments;
 use arguments::Command;
-use data_provider::ClangAstDataProvider;
+use clang_parser::parse_files;
+use data_provider::ClangDataProvider;
 use gitql_cli::arguments::OutputFormat;
 use gitql_cli::diagnostic_reporter;
 use gitql_cli::diagnostic_reporter::DiagnosticReporter;
@@ -29,6 +30,7 @@ use schema::tables_fields_names;
 use schema::tables_fields_types;
 
 mod arguments;
+mod clang_parser;
 mod data_provider;
 mod schema;
 mod visitor;
@@ -70,7 +72,10 @@ fn main() {
             env.with_aggregation_functions(&aggregation_signatures, aggregation_functions);
             env.with_window_functions(&window_signatures, window_function);
 
-            execute_clangql_query(query, &arguments, files, &mut env, &mut reporter);
+            let compilation_units = parse_files(files);
+            let provider: Box<dyn DataProvider> =
+                Box::new(ClangDataProvider::new(compilation_units));
+            execute_clangql_query(query, &arguments, &mut env, &provider, &mut reporter);
         }
         Command::Help => {
             arguments::print_help_list();
@@ -111,6 +116,9 @@ fn launch_clangql_repl(arguments: Arguments) {
     global_env.with_aggregation_functions(&aggregation_signatures, aggregation_functions);
     global_env.with_window_functions(&window_signatures, window_function);
 
+    let compilation_units = parse_files(files);
+    let provider: Box<dyn DataProvider> = Box::new(ClangDataProvider::new(compilation_units));
+
     let mut input = String::new();
 
     loop {
@@ -145,8 +153,8 @@ fn launch_clangql_repl(arguments: Arguments) {
         execute_clangql_query(
             stdin_input.to_owned(),
             &arguments,
-            files,
             &mut global_env,
+            &provider,
             &mut reporter,
         );
 
@@ -158,8 +166,8 @@ fn launch_clangql_repl(arguments: Arguments) {
 fn execute_clangql_query(
     query: String,
     arguments: &Arguments,
-    files: &[String],
     env: &mut Environment,
+    provider: &Box<dyn DataProvider>,
     reporter: &mut DiagnosticReporter,
 ) {
     let front_start = std::time::Instant::now();
@@ -186,9 +194,6 @@ fn execute_clangql_query(
     let front_duration = front_start.elapsed();
 
     let engine_start = std::time::Instant::now();
-    let files = files.to_vec();
-    let file_provider = ClangAstDataProvider::new(files);
-    let provider: Box<dyn DataProvider> = Box::new(file_provider);
     let evaluation_result = engine::evaluate(env, &provider, query_node);
 
     // Report Runtime exceptions if they exists

src/visitor/class.rs

@@ -1,11 +1,10 @@
 extern crate clang_sys;
 
 use clang_sys::*;
-use std::ffi::c_char;
 use std::ffi::c_void;
 use std::ffi::CStr;
-use std::ptr;
 
+use crate::clang_parser::CompilationUnit;
 use crate::visitor::location;
 
 pub struct ClassNode {
@@ -24,30 +23,13 @@ pub struct ClassAttributes {
     pub fields_count: u32,
 }
 
-pub fn select_clang_classes(path: &str) -> Vec<ClassNode> {
+pub fn select_clang_classes(compilation_unit: &CompilationUnit) -> Vec<ClassNode> {
     let mut classes: Vec<ClassNode> = Vec::new();
     let data = &mut classes as *mut Vec<ClassNode> as *mut c_void;
 
     unsafe {
-        let index = clang_createIndex(0, 0);
-        let translation_unit: CXTranslationUnit = clang_parseTranslationUnit(
-            index,
-            path.as_ptr() as *const c_char,
-            ptr::null_mut(),
-            0,
-            ptr::null_mut(),
-            0,
-            0,
-        );
-
-        let cursor = clang_getTranslationUnitCursor(translation_unit);
+        let cursor = clang_getTranslationUnitCursor(compilation_unit.translation_unit);
         clang_visitChildren(cursor, visit_class_or_struct_declaration, data);
-
-        // Dispose the translation unit
-        clang_disposeTranslationUnit(translation_unit);
-
-        // Dispose the index
-        clang_disposeIndex(index);
     }
 
     classes

src/visitor/enumeration.rs

@@ -1,11 +1,10 @@
 extern crate clang_sys;
 
 use clang_sys::*;
-use std::ffi::c_char;
 use std::ffi::c_void;
 use std::ffi::CStr;
-use std::ptr;
 
+use crate::clang_parser::CompilationUnit;
 use crate::visitor::location;
 
 pub struct EnumNode {
@@ -20,30 +19,13 @@ pub struct EnumAttributes {
     pub constants_count: u32,
 }
 
-pub fn select_clang_enums(path: &str) -> Vec<EnumNode> {
+pub fn select_clang_enums(compilation_unit: &CompilationUnit) -> Vec<EnumNode> {
     let mut enums: Vec<EnumNode> = Vec::new();
     let data = &mut enums as *mut Vec<EnumNode> as *mut c_void;
 
     unsafe {
-        let index = clang_createIndex(0, 0);
-        let translation_unit: CXTranslationUnit = clang_parseTranslationUnit(
-            index,
-            path.as_ptr() as *const c_char,
-            ptr::null_mut(),
-            0,
-            ptr::null_mut(),
-            0,
-            0,
-        );
-
-        let cursor = clang_getTranslationUnitCursor(translation_unit);
+        let cursor = clang_getTranslationUnitCursor(compilation_unit.translation_unit);
         clang_visitChildren(cursor, visit_enum_declaration, data);
-
-        // Dispose the translation unit
-        clang_disposeTranslationUnit(translation_unit);
-
-        // Dispose the index
-        clang_disposeIndex(index);
     }
 
     enums