Add description parsers for Markdown/RST

See more info in PR or merge commit.

Author: taminomara

Cargo.lock

@@ -9,6 +9,7 @@ members = [
 # local
 emmylua_code_analysis = { path = "crates/emmylua_code_analysis", version = "0.10.0" }
 emmylua_parser = { path = "crates/emmylua_parser", version = "0.12.0" }
+emmylua_parser_desc = { path = "crates/emmylua_parser_desc", version = "0.12.0" }
 emmylua_diagnostic_macro = { path = "crates/emmylua_diagnostic_macro", version = "0.5.0" }
 
 # external
@@ -51,3 +52,4 @@ ansi_term = "0.12.1"
 num-traits = { version = "0.2", features = ["std"] }
 mimalloc = "0.1.47"
 googletest = "0.14.2"
+unicode-general-category = "1.0.0"

Cargo.toml

@@ -79,13 +79,14 @@
 
 Our project is meticulously organized into specialized crates, each serving a critical role in the Lua analysis ecosystem:
 
-| Crate | Badge | Description |
-| ----- | ----- | ----------- |
-| [🔍 **emmylua_parser**](./crates/emmylua_parser) | [![emmylua_parser](https://img.shields.io/crates/v/emmylua_parser.svg?style=flat-square)](https://crates.io/crates/emmylua_parser) | The foundational Rust-based Lua parser engineered for maximum efficiency and accuracy. Powers all downstream analysis tools. |
+| Crate                                                          | Badge                                                                                                                                                   | Description |
+|----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- |
+| [🔍 **emmylua_parser**](./crates/emmylua_parser)               | [![emmylua_parser](https://img.shields.io/crates/v/emmylua_parser.svg?style=flat-square)](https://crates.io/crates/emmylua_parser)                      | The foundational Rust-based Lua parser engineered for maximum efficiency and accuracy. Powers all downstream analysis tools. |
+| [📑 **emmylua_parser_desc**](./crates/emmylua_parser_desc)     | [![emmylua_parser_desc](https://img.shields.io/crates/v/emmylua_parser_desc.svg?style=flat-square)](https://crates.io/crates/emmylua_parser_desc)       | Extension for EmmyLua-Parser that handles Markdown/RST highlighting in comments. |
 | [🧠 **emmylua_code_analysis**](./crates/emmylua_code_analysis) | [![emmylua_code_analysis](https://img.shields.io/crates/v/emmylua_code_analysis.svg?style=flat-square)](https://crates.io/crates/emmylua_code_analysis) | Advanced semantic analysis engine providing deep code understanding, type inference, and cross-reference resolution. |
-| [🖥️ **emmylua_ls**](./crates/emmylua_ls) | [![emmylua_ls](https://img.shields.io/crates/v/emmylua_ls.svg?style=flat-square)](https://crates.io/crates/emmylua_ls) | The complete Language Server Protocol implementation offering rich IDE features across all major editors. |
-| [📚 **emmylua_doc_cli**](./crates/emmylua_doc_cli/) | [![emmylua_doc_cli](https://img.shields.io/crates/v/emmylua_doc_cli.svg?style=flat-square)](https://crates.io/crates/emmylua_doc_cli) | Professional documentation generator creating beautiful, searchable API docs from your Lua code and annotations. |
-| [✅ **emmylua_check**](./crates/emmylua_check) | [![emmylua_check](https://img.shields.io/crates/v/emmylua_check.svg?style=flat-square)](https://crates.io/crates/emmylua_check) | Comprehensive static analysis tool for code quality assurance, catching bugs before they reach production. |
+| [🖥️ **emmylua_ls**](./crates/emmylua_ls)                       | [![emmylua_ls](https://img.shields.io/crates/v/emmylua_ls.svg?style=flat-square)](https://crates.io/crates/emmylua_ls)                                  | The complete Language Server Protocol implementation offering rich IDE features across all major editors. |
+| [📚 **emmylua_doc_cli**](./crates/emmylua_doc_cli/)            | [![emmylua_doc_cli](https://img.shields.io/crates/v/emmylua_doc_cli.svg?style=flat-square)](https://crates.io/crates/emmylua_doc_cli)                   | Professional documentation generator creating beautiful, searchable API docs from your Lua code and annotations. |
+| [✅ **emmylua_check**](./crates/emmylua_check)                 | [![emmylua_check](https://img.shields.io/crates/v/emmylua_check.svg?style=flat-square)](https://crates.io/crates/emmylua_check)                         | Comprehensive static analysis tool for code quality assurance, catching bugs before they reach production. |
 
 
 ---

README.md

@@ -14,6 +14,7 @@ categories = ["development-tools"]
 # local
 emmylua_code_analysis.workspace = true
 emmylua_parser.workspace = true
+emmylua_parser_desc.workspace = true
 
 # external
 lsp-server.workspace = true

crates/emmylua_ls/Cargo.toml

@@ -693,7 +693,7 @@ impl<'a> LuaLexer<'a> {
 
         if self.reader.current_char().is_alphabetic() {
             let ch = self.reader.current_char();
-            self.error(|| format!("unexpected character '{ch}' after number literal"));
+            self.error(|| t!("unexpected character '%{ch}' after number literal", ch = ch));
         }
 
         match state {

crates/emmylua_parser/src/lexer/lua_lexer.rs

@@ -7,10 +7,12 @@ mod syntax;
 mod text;
 
 pub use kind::*;
+pub use lexer::{LexerConfig, LuaLexer, LuaLexerState, LuaTokenData};
 pub use parser::{LuaParser, ParserConfig, SpecialFunction};
 pub use parser_error::{LuaParseError, LuaParseErrorKind};
 pub use syntax::*;
 pub use text::LineIndex;
+pub use text::{Reader, SourceRange};
 
 #[macro_use]
 extern crate rust_i18n;

crates/emmylua_parser/src/lib.rs

@@ -5,4 +5,4 @@ mod text_range;
 
 pub use line_index::LineIndex;
 pub use reader::Reader;
-pub(crate) use text_range::SourceRange;
+pub use text_range::SourceRange;

crates/emmylua_parser/src/text/mod.rs

@@ -0,0 +1,19 @@
+[package]
+name = "emmylua_parser_desc"
+version = "0.12.0"
+edition = "2024"
+authors = ["CppCXY", "taminomara"]
+description = "A parser for markup within Lua comments"
+license = "MIT"
+repository = "https://github.com/CppCXY/emmylua-analyzer-rust"
+readme = "README.md"
+keywords = ["emmylua", "luals", "parser", "lua"]
+categories = ["development-tools", "parsing"]
+
+[dependencies]
+# local
+emmylua_parser.workspace = true
+
+# external
+rowan.workspace = true
+unicode-general-category.workspace = true

crates/emmylua_parser_desc/Cargo.toml

@@ -0,0 +1,35 @@
+## EmmyLua-Parser-Desc
+
+EmmyLua-Parser-Desc is an extension for EmmyLua-Parser that uses its internal machinery to provide lexic information about markup of documentation comments. It supports parsing Markdown, MyST and RST.
+
+
+### Features
+
+- Ability to parse description blocks provided by EmmyLua-Parser and report ranges of interest: highlighted keywords,  code blocks, cross-references and so on
+- Supports Markdown, MyST and RST
+- Ability to parse possible broken or unterminated MyST and RST cross-references in order to facilitate Autocompletion and Go To Definition functionality
+
+### Usage
+
+```rust
+let code = r#"
+    --- Description in **markdown format**, with example code:
+    ---
+    --- ```lua
+    --- print(a)
+    --- ```
+    local a = 1
+"#;
+let tree = LuaParser::parse(code, ParserConfig::default());
+
+let chunk = tree.get_chunk_node();
+for desc in chunk.descendants::<LuaDocDescription>() {
+    let doc_items = emmylua_parser_desc::parse(
+        DescParserType::Md,
+        code,
+        desc,
+        None
+    );
+    println!("{:?}", doc_items);
+}
+```

crates/emmylua_parser_desc/README.md

@@ -0,0 +1,101 @@
+use emmylua_parser::{LuaDocDescription, LuaTokenKind};
+use rowan::TextRange;
+
+mod md;
+mod ref_target;
+mod rst;
+mod util;
+
+pub use ref_target::*;
+use util::sort_result;
+
+#[cfg(test)]
+mod testlib;
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum DescItemKind {
+    /// Generic block of documentation.
+    Scope,
+
+    /// Cross-reference to a Lua object.
+    Ref,
+
+    /// Emphasis.
+    Em,
+
+    /// Strong emphasis.
+    Strong,
+
+    /// Code markup.
+    Code,
+
+    /// Hyperlink.
+    Link,
+
+    /// Inline markup, like stars around emphasized text.
+    Markup,
+
+    /// Directive name, code-block syntax name, role name,
+    /// or some other form of argument.
+    Arg,
+
+    /// Line of code in a code block.
+    CodeBlock,
+
+    /// Line of code in a code block highlighted by Lua lexer.
+    CodeBlockHl(LuaTokenKind),
+}
+
+#[derive(Debug, Clone)]
+pub struct DescItem {
+    pub range: TextRange,
+    pub kind: DescItemKind,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub enum DescParserType {
+    None,
+    Md,
+    MySt {
+        primary_domain: Option<String>,
+    },
+    Rst {
+        primary_domain: Option<String>,
+        default_role: Option<String>,
+    },
+}
+
+impl Default for DescParserType {
+    fn default() -> Self {
+        DescParserType::None
+    }
+}
+
+/// Parses markup in comments.
+pub trait LuaDescParser {
+    /// Process a description node and yield found documentation ranges.
+    fn parse(&mut self, text: &str, desc: LuaDocDescription) -> Vec<DescItem>;
+}
+
+pub fn parse(
+    kind: DescParserType,
+    text: &str,
+    desc: LuaDocDescription,
+    cursor_position: Option<usize>,
+) -> Vec<DescItem> {
+    let mut items = match kind {
+        DescParserType::None => Vec::new(),
+        DescParserType::Md => md::MdParser::new(cursor_position).parse(text, desc),
+        DescParserType::MySt { primary_domain } => {
+            md::MdParser::new_myst(primary_domain, cursor_position).parse(text, desc)
+        }
+        DescParserType::Rst {
+            primary_domain,
+            default_role,
+        } => rst::RstParser::new(primary_domain, default_role, cursor_position).parse(text, desc),
+    };
+
+    sort_result(&mut items);
+
+    items
+}