Rust: macro expansion

Author: aibaars

rust/extractor/src/generated/.generated.list

@@ -1,5 +1,5 @@
 use anyhow::Context;
-use ra_ap_ide_db::line_index::LineIndex;
+use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 mod archive;
 mod config;
 pub mod generated;
@@ -17,10 +17,21 @@ fn extract(
     let display_path = file.to_string_lossy();
     let mut trap = traps.create("source", &file);
     let label = trap.emit_file(&file);
-    let mut translator = translate::Translator::new(trap, label, line_index, semi);
+    let mut translator =
+        translate::Translator::new(trap, display_path.as_ref(), label, line_index, semi);
 
     for err in parse_errors {
-        translator.emit_parse_error(display_path.as_ref(), err);
+        translator.emit_parse_error(&err);
+    }
+    let no_location = (LineCol { line: 0, col: 0 }, LineCol { line: 0, col: 0 });
+    if translator.semi.is_none() {
+        translator.emit_diagnostic(
+            trap::DiagnosticSeverity::Warning,
+            "semantics".to_owned(),
+            "semantic analyzer unavailable".to_owned(),
+            "semantic analyzer unavailable: macro expansion, call graph, and type inference will be skipped.".to_owned(),
+            no_location,
+        );
     }
     translator.emit_source_file(ast);
     translator.trap.commit()?;

rust/extractor/src/generated/top.rs

@@ -85,11 +85,21 @@ impl RustAnalyzer {
                 }
             }
         }
-        let input = std::fs::read(&path).unwrap();
+        let mut errors = Vec::new();
+        let input = match std::fs::read(path) {
+            Ok(data) => data,
+            Err(e) => {
+                errors.push(SyntaxError::new(
+                    format!("Could not read {}: {}", path.to_string_lossy(), e),
+                    TextRange::empty(TextSize::default()),
+                ));
+                vec![]
+            }
+        };
         let (input, err) = from_utf8_lossy(&input);
         let parse = ra_ap_syntax::ast::SourceFile::parse(&input, Edition::CURRENT);
-        let mut errors = parse.errors();
-        errors.extend(err.into_iter());
+        errors.extend(parse.errors());
+        errors.extend(err);
         (parse.tree(), input.as_ref().into(), errors, None)
     }
 }

rust/extractor/src/main.rs

@@ -1,13 +1,18 @@
-use crate::generated::{self, AstNode};
+use crate::generated::MacroCall;
+use crate::generated::{self};
 use crate::trap::{DiagnosticSeverity, TrapFile, TrapId};
 use crate::trap::{Label, TrapClass};
 use codeql_extractor::trap::{self};
+use log::Level;
+use ra_ap_hir::db::ExpandDatabase;
 use ra_ap_hir::Semantics;
 use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 use ra_ap_ide_db::RootDatabase;
 use ra_ap_parser::SyntaxKind;
 use ra_ap_syntax::ast::RangeItem;
-use ra_ap_syntax::{ast, NodeOrToken, SyntaxElementChildren, SyntaxError, SyntaxToken, TextRange};
+use ra_ap_syntax::{
+    ast, AstNode, NodeOrToken, SyntaxElementChildren, SyntaxError, SyntaxToken, TextRange,
+};
 pub trait TextValue {
     fn try_get_text(&self) -> Option<String>;
 }
@@ -60,32 +65,35 @@ impl TextValue for ast::RangePat {
 }
 pub struct Translator<'a> {
     pub trap: TrapFile,
+    path: &'a str,
     label: trap::Label,
     line_index: LineIndex,
-    semi: Option<Semantics<'a, RootDatabase>>,
+    pub semi: Option<Semantics<'a, RootDatabase>>,
 }
 
-impl Translator<'_> {
+impl<'a> Translator<'a> {
     pub fn new(
         trap: TrapFile,
+        path: &'a str,
         label: trap::Label,
         line_index: LineIndex,
-        semi: Option<Semantics<'_, RootDatabase>>,
-    ) -> Translator {
+        semi: Option<Semantics<'a, RootDatabase>>,
+    ) -> Translator<'a> {
         Translator {
             trap,
+            path,
             label,
             line_index,
             semi,
         }
     }
-    pub fn location(&self, range: TextRange) -> (LineCol, LineCol) {
+    fn location(&self, range: TextRange) -> (LineCol, LineCol) {
         let start = self.line_index.line_col(range.start());
         let range_end = range.end();
         // QL end positions are inclusive, while TextRange offsets are exclusive and point at the position
         // right after the last character of the range. We need to shift the end offset one character to the left to
         // get the right inclusive QL position. Unfortunately, simply subtracting `1` from the end-offset may cause
-        // the offset to point in the middle of a mult-byte character, resulting in a `panic`. Therefore we use `try_line_col`
+        // the offset to point in the middle of a multi-byte character, resulting in a `panic`. Therefore we use `try_line_col`
         // with decreasing offsets to find the start of the last character included in the range.
         for i in 1..4 {
             if let Some(end) = range_end
@@ -98,28 +106,72 @@ impl Translator<'_> {
         let end = self.line_index.line_col(range_end);
         (start, end)
     }
+
+    pub fn text_range_for_node(&mut self, node: &impl ast::AstNode) -> TextRange {
+        if let Some(semi) = self.semi.as_ref() {
+            let file_range = semi.original_range(node.syntax());
+            file_range.range
+        } else {
+            node.syntax().text_range()
+        }
+    }
     pub fn emit_location<T: TrapClass>(&mut self, label: Label<T>, node: &impl ast::AstNode) {
-        let (start, end) = self.location(node.syntax().text_range());
+        let range = self.text_range_for_node(node);
+        let (start, end) = self.location(range);
         self.trap.emit_location(self.label, label, start, end)
     }
     pub fn emit_location_token(&mut self, label: Label<generated::Token>, token: &SyntaxToken) {
         let (start, end) = self.location(token.text_range());
         self.trap.emit_location(self.label, label, start, end)
     }
-    pub fn emit_parse_error(&mut self, path: &str, err: SyntaxError) {
-        let (start, end) = self.location(err.range());
-        log::warn!("{}:{}:{}: {}", path, start.line + 1, start.col + 1, err);
-        let message = err.to_string();
+    pub fn emit_diagnostic(
+        &mut self,
+        severity: DiagnosticSeverity,
+        error_tag: String,
+        error_message: String,
+        full_error_message: String,
+        location: (LineCol, LineCol),
+    ) {
+        let (start, end) = location;
+        let level = match severity {
+            DiagnosticSeverity::Debug => Level::Debug,
+            DiagnosticSeverity::Info => Level::Info,
+            DiagnosticSeverity::Warning => Level::Warn,
+            DiagnosticSeverity::Error => Level::Error,
+        };
+        log::log!(
+            level,
+            "{}:{}:{}: {}",
+            self.path,
+            start.line + 1,
+            start.col + 1,
+            &error_message
+        );
         let location = self.trap.emit_location_label(self.label, start, end);
         self.trap.emit_diagnostic(
+            severity,
+            error_tag,
+            error_message,
+            full_error_message,
+            location,
+        );
+    }
+    pub fn emit_parse_error(&mut self, err: &SyntaxError) {
+        let location = self.location(err.range());
+        let message = err.to_string();
+        self.emit_diagnostic(
             DiagnosticSeverity::Warning,
             "parse_error".to_owned(),
             message.clone(),
             message,
             location,
         );
     }
-    pub fn emit_tokens(&mut self, parent: Label<AstNode>, children: SyntaxElementChildren) {
+    pub fn emit_tokens(
+        &mut self,
+        parent: Label<generated::AstNode>,
+        children: SyntaxElementChildren,
+    ) {
         for child in children {
             if let NodeOrToken::Token(token) = child {
                 if token.kind() == SyntaxKind::COMMENT {
@@ -133,4 +185,83 @@ impl Translator<'_> {
             }
         }
     }
+    pub(crate) fn extract_macro_call_expanded(
+        &mut self,
+        mcall: &ast::MacroCall,
+        label: Label<generated::MacroCall>,
+    ) {
+        if let Some(semi) = &self.semi {
+            if let Some(expanded) = semi.expand(mcall) {
+                if let Some(value) =
+                    semi.hir_file_for(&expanded)
+                        .macro_file()
+                        .and_then(|macro_file| {
+                            semi.db
+                                .parse_macro_expansion_error(macro_file.macro_call_id)
+                        })
+                {
+                    if let Some(err) = &value.err {
+                        let (message, _error) = err.render_to_string(semi.db);
+
+                        if err.span().anchor.file_id == semi.hir_file_for(mcall.syntax()) {
+                            let location = err.span().range
+                                + semi
+                                    .db
+                                    .ast_id_map(err.span().anchor.file_id.into())
+                                    .get_erased(err.span().anchor.ast_id)
+                                    .text_range()
+                                    .start();
+                            self.emit_parse_error(&SyntaxError::new(message, location));
+                        };
+                    }
+                    for err in value.value.iter() {
+                        self.emit_parse_error(err);
+                    }
+                }
+                let expand_to = ra_ap_hir_expand::ExpandTo::from_call_site(mcall);
+                let kind = expanded.kind();
+                let value: Option<Label<crate::generated::AstNode>> = match expand_to {
+                    ra_ap_hir_expand::ExpandTo::Statements => {
+                        ast::MacroStmts::cast(expanded).map(|x| self.emit_macro_stmts(x).into())
+                    }
+                    ra_ap_hir_expand::ExpandTo::Items => {
+                        ast::MacroItems::cast(expanded).map(|x| self.emit_macro_items(x).into())
+                    }
+
+                    ra_ap_hir_expand::ExpandTo::Pattern => {
+                        ast::Pat::cast(expanded).map(|x| self.emit_pat(x).into())
+                    }
+                    ra_ap_hir_expand::ExpandTo::Type => {
+                        ast::Type::cast(expanded).map(|x| self.emit_type(x).into())
+                    }
+                    ra_ap_hir_expand::ExpandTo::Expr => {
+                        ast::Expr::cast(expanded).map(|x| self.emit_expr(x).into())
+                    }
+                };
+                if let Some(value) = value {
+                    MacroCall::emit_expanded(label, value, &mut self.trap.writer);
+                } else {
+                    let range = self.text_range_for_node(mcall);
+                    self.emit_parse_error(&SyntaxError::new(
+                        format!(
+                            "macro expansion failed: the macro '{}' expands to {:?} but a {:?} was expected",
+                            mcall.path().map(|p| p.to_string()).unwrap_or_default(),
+                            kind, expand_to
+                        ),
+                        range,
+                    ));
+                }
+            } else {
+                let range = self.text_range_for_node(mcall);
+
+                self.emit_parse_error(&SyntaxError::new(
+                    format!(
+                        "macro expansion failed: could not resolve macro '{}'",
+                        mcall.path().map(|p| p.to_string()).unwrap_or_default()
+                    ),
+                    range,
+                ));
+            }
+        }
+    }
 }

rust/extractor/src/rust_analyzer.rs

@@ -124,7 +124,7 @@ impl<T: TrapClass> From<Label<T>> for trap::Arg {
 
 pub struct TrapFile {
     path: PathBuf,
-    writer: Writer,
+    pub writer: Writer,
     compression: Compression,
 }