Document

Author: GearsDatapacks

compiler-core/src/language_server.rs

@@ -53,8 +53,8 @@ pub fn src_span_to_lsp_range(location: SrcSpan, line_numbers: &LineNumbers) -> R
 
 pub fn lsp_range_to_src_span(range: Range, line_numbers: &LineNumbers) -> SrcSpan {
     let Range { start, end } = range;
-    let start = line_numbers.byte_index(start.line, start.character);
-    let end = line_numbers.byte_index(end.line, end.character);
+    let start = line_numbers.byte_index(start.line + 1, start.character + 1);
+    let end = line_numbers.byte_index(end.line + 1, end.character + 1);
     SrcSpan { start, end }
 }
 

compiler-core/src/language_server/completer.rs

@@ -130,9 +130,10 @@ where
         &'a self,
         valid_phrase_char: &impl Fn(char) -> bool,
     ) -> (Range, String) {
-        let cursor = self
-            .src_line_numbers
-            .byte_index(self.cursor_position.line, self.cursor_position.character);
+        let cursor = self.src_line_numbers.byte_index(
+            self.cursor_position.line + 1,
+            self.cursor_position.character + 1,
+        );
 
         // Get part of phrase prior to cursor
         let before = self
@@ -195,10 +196,10 @@ where
     pub fn import_completions(&'a self) -> Option<Result<Option<Vec<CompletionItem>>>> {
         let start_of_line = self
             .src_line_numbers
-            .byte_index(self.cursor_position.line, 0);
+            .byte_index(self.cursor_position.line + 1, 1);
         let end_of_line = self
             .src_line_numbers
-            .byte_index(self.cursor_position.line + 1, 0);
+            .byte_index(self.cursor_position.line + 2, 1);
 
         // Drop all lines except the line the cursor is on
         let src = self.src.get(start_of_line as usize..end_of_line as usize)?;
@@ -561,9 +562,10 @@ where
         // e.x. when the user has typed mymodule.| we know local module and prelude values are no longer
         // relevant.
         if module_select.is_none() {
-            let cursor = self
-                .src_line_numbers
-                .byte_index(self.cursor_position.line, self.cursor_position.character);
+            let cursor = self.src_line_numbers.byte_index(
+                self.cursor_position.line + 1,
+                self.cursor_position.character + 1,
+            );
 
             // Find the function that the cursor is in and push completions for
             // its arguments and local variables.

compiler-core/src/language_server/edits.rs

@@ -47,7 +47,7 @@ pub fn add_newlines_after_import(
     src: &str,
 ) -> Newlines {
     let import_start_cursor =
-        line_numbers.byte_index(import_location.line, import_location.character);
+        line_numbers.byte_index(import_location.line + 1, import_location.character + 1);
     let is_new_line = src
         .chars()
         .nth(import_start_cursor as usize)

compiler-core/src/language_server/engine.rs

@@ -266,7 +266,7 @@ where
             let completer = Completer::new(&src, &params, &this.compiler, module);
             let byte_index = completer
                 .module_line_numbers
-                .byte_index(params.position.line, params.position.character);
+                .byte_index(params.position.line + 1, params.position.character + 1);
 
             // If in comment context, do not provide completions
             if module.extra.is_within_comment(byte_index) {
@@ -629,7 +629,8 @@ where
                 )))
             };
 
-            let byte_index = lines.byte_index(params.position.line, params.position.character);
+            let byte_index =
+                lines.byte_index(params.position.line + 1, params.position.character + 1);
 
             Ok(match reference_for_ast_node(found, &current_module.name) {
                 Some(Referenced::LocalVariable {
@@ -762,7 +763,8 @@ where
                 return Ok(None);
             };
 
-            let byte_index = lines.byte_index(position.position.line, position.position.character);
+            let byte_index =
+                lines.byte_index(position.position.line + 1, position.position.character + 1);
 
             Ok(match reference_for_ast_node(found, &module.name) {
                 Some(Referenced::LocalVariable {
@@ -996,7 +998,8 @@ Unused labelled fields:
         module: &'a Module,
     ) -> Option<(LineNumbers, Located<'a>)> {
         let line_numbers = LineNumbers::new(&module.code);
-        let byte_index = line_numbers.byte_index(params.position.line, params.position.character);
+        let byte_index =
+            line_numbers.byte_index(params.position.line + 1, params.position.character + 1);
         let node = module.find_node(byte_index);
         let node = node?;
         Some((line_numbers, node))

compiler-core/src/language_server/tests.rs

@@ -831,11 +831,13 @@ pub fn apply_code_edit(src: &str, mut change: Vec<lsp_types::TextEdit>) -> Strin
 
     change.sort_by_key(|edit| (edit.range.start.line, edit.range.start.character));
     for edit in change {
-        let start = line_numbers.byte_index(edit.range.start.line, edit.range.start.character)
+        let start = line_numbers
+            .byte_index(edit.range.start.line + 1, edit.range.start.character + 1)
+            as i32
+            - offset;
+        let end = line_numbers.byte_index(edit.range.end.line + 1, edit.range.end.character + 1)
             as i32
             - offset;
-        let end =
-            line_numbers.byte_index(edit.range.end.line, edit.range.end.character) as i32 - offset;
         let range = (start as usize)..(end as usize);
         offset += end - start;
         offset -= edit.new_text.len() as i32;

compiler-core/src/line_numbers.rs

@@ -1,15 +1,38 @@
 use std::collections::HashMap;
 
+/// A struct which contains information about line numbers of a source file,
+/// and can convert between byte offsets that are used in the compiler and
+/// line-column pairs used in LSP.
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
 pub struct LineNumbers {
+    /// The byte offsets of the start of each line of the source file
     pub line_starts: Vec<u32>,
+    /// The total length of the source file
     pub length: u32,
+    /// A mapping of byte offsets to character length information. This is used
+    /// when converting between byte indices and line-column numbers, because
+    /// LSP uses UTF-16, while Rust encodes strings as UTF-8.
+    ///
+    /// This only contains characters which are more than one byte in UTF-8,
+    /// because one byte UTF-8 characters are one UTF-16 segment also, so no
+    /// translation is needed.
+    ///
+    /// We could store the whole source file here instead, however that would
+    /// be quite wasteful. Most Gleam programs use only ASCII characters, meaning
+    /// UTF-8 offsets are the same as UTF-16 ones. With this representation, we
+    /// only need to store a few characters.
+    ///
+    /// In most programs this will be empty because they will only be using
+    /// ASCII characters.
     pub mapping: HashMap<usize, Character>,
 }
 
+/// Information about how a character is encoded in UTF-8 and UTF-16.
 #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
 pub struct Character {
+    /// The number of bytes needed to encode this in UTF-8.
     pub length_utf8: u8,
+    /// The number of 16-bit segments needed to encode this in UTF-16.
     pub length_utf16: u8,
 }
 
@@ -43,14 +66,16 @@ impl LineNumbers {
         map
     }
 
-    /// Get the line number for a byte index
+    /// Returns the 1-indexed line number of a given byte index
     pub fn line_number(&self, byte_index: u32) -> u32 {
         self.line_starts
             .binary_search(&byte_index)
             .unwrap_or_else(|next_line| next_line - 1) as u32
             + 1
     }
 
+    /// Returns the 1-indexed line and column number of a given byte index,
+    /// using a UTF-16 character offset.
     pub fn line_and_column_number(&self, byte_index: u32) -> LineColumn {
         let line = self.line_number(byte_index);
         let line_start = self
@@ -82,9 +107,10 @@ impl LineNumbers {
         }
     }
 
-    /// 0 indexed line and character to byte index
+    /// Returns the byte index of the corresponding 1-indexed line and column
+    /// numbers, translating from a UTF-16 character index to a UTF-8 byte index.
     pub fn byte_index(&self, line: u32, character: u32) -> u32 {
-        let line_start = match self.line_starts.get(line as usize) {
+        let line_start = match self.line_starts.get(line as usize - 1) {
             Some(&line_start) => line_start,
             None => return self.length,
         };
@@ -93,7 +119,7 @@ impl LineNumbers {
         let mut u16_offset = 0;
 
         loop {
-            if u16_offset >= character {
+            if u16_offset >= character - 1 {
                 break;
             }
 
@@ -120,10 +146,10 @@ pub fn main() {
 "#;
     let line_numbers = LineNumbers::new(src);
 
-    assert_eq!(line_numbers.byte_index(0, 0), 0);
-    assert_eq!(line_numbers.byte_index(0, 4), 4);
+    assert_eq!(line_numbers.byte_index(1, 1), 0);
+    assert_eq!(line_numbers.byte_index(1, 5), 4);
     assert_eq!(line_numbers.byte_index(100, 1), src.len() as u32);
-    assert_eq!(line_numbers.byte_index(2, 1), 18);
+    assert_eq!(line_numbers.byte_index(3, 2), 18);
 }
 
 // https://github.com/gleam-lang/gleam/issues/3628
@@ -139,10 +165,10 @@ pub fn main() {
 "#;
     let line_numbers = LineNumbers::new(src);
 
-    assert_eq!(line_numbers.byte_index(1, 6), 30);
-    assert_eq!(line_numbers.byte_index(5, 2), 52);
-    assert_eq!(line_numbers.byte_index(5, 17), 75);
-    assert_eq!(line_numbers.byte_index(6, 1), 91);
+    assert_eq!(line_numbers.byte_index(2, 7), 30);
+    assert_eq!(line_numbers.byte_index(6, 3), 52);
+    assert_eq!(line_numbers.byte_index(6, 18), 75);
+    assert_eq!(line_numbers.byte_index(7, 2), 91);
 }
 
 // https://github.com/gleam-lang/gleam/issues/3628