Multibyte-character support (Fix #425) (#490)

* Multibyte character support for lsp#get_position()

* Multibyte character support for "textDocument/documentHighlight"

* Fix `lsp#utils#byteindex()` would fails if a file is not yet loaded

* Add tests

* Rename lsp#utils#byteindex() and lsp#utils#charindex()

lsp#utils#byteindex() -> lsp#utils#to_col()
lsp#utils#charindex() -> lsp#utils#to_char()

* Multibyte character support for textEdit

* Remove unused line

* Multibyte character support for  "textDocument/rangeFormatting"

* Multibyte character support for  "textDocument/publishDiagnostics"

Author: machakann

autoload/lsp.vim

@@ -747,7 +747,9 @@ function! lsp#get_text_document_identifier(...) abort
 endfunction
 
 function! lsp#get_position(...) abort
-    return { 'line': line('.') - 1, 'character': col('.') -1 }
+    let l:line = line('.')
+    let l:char = lsp#utils#to_char('%', l:line, col('.'))
+    return { 'line': l:line - 1, 'character': l:char }
 endfunction
 
 function! s:get_text_document_identifier(buf) abort

autoload/lsp/omni.vim

@@ -318,7 +318,7 @@ function! s:apply_text_edits() abort
     " expand textEdit range, for omni complet inserted text.
     let l:text_edit = get(l:user_data, s:user_data_key, {})
     if !empty(l:text_edit)
-        let l:expanded_text_edit = s:expand_range(l:text_edit, len(v:completed_item['word']))
+        let l:expanded_text_edit = s:expand_range(l:text_edit, strchars(v:completed_item['word']))
         call add(l:all_text_edits, l:expanded_text_edit)
     endif
 

autoload/lsp/ui/vim.vim

@@ -252,14 +252,16 @@ function! s:document_format_range(sync) abort
     let l:server = l:servers[0]
 
     let [l:start_lnum, l:start_col, l:end_lnum, l:end_col] = s:get_visual_selection_pos()
+    let l:start_char = lsp#utils#to_char('%', l:start_lnum, l:start_col)
+    let l:end_char = lsp#utils#to_char('%', l:end_lnum, l:end_col)
     redraw | echo 'Formatting document range ...'
     call lsp#send_request(l:server, {
         \ 'method': 'textDocument/rangeFormatting',
         \ 'params': {
         \   'textDocument': lsp#get_text_document_identifier(),
         \   'range': {
-        \       'start': { 'line': l:start_lnum - 1, 'character': l:start_col - 1 },
-        \       'end': { 'line': l:end_lnum - 1, 'character': l:end_col - 1 },
+        \       'start': { 'line': l:start_lnum - 1, 'character': l:start_char },
+        \       'end': { 'line': l:end_lnum - 1, 'character': l:end_char },
         \   },
         \   'options': {
         \       'tabSize': getbufvar(bufnr('%'), '&shiftwidth'),
@@ -347,9 +349,11 @@ function! s:get_visual_selection_range() abort
     if l:column_end - 1 > len(getline(l:line_end))
       let l:column_end = len(getline(l:line_end)) + 1
     endif
+    let l:char_start = lsp#utils#to_char('%', l:line_start, l:column_start)
+    let l:char_end = lsp#utils#to_char('%', l:line_end, l:column_end)
     return {
-          \ 'start': { 'line': l:line_start - 1, 'character': l:column_start - 1 },
-          \ 'end': { 'line': l:line_end - 1, 'character': l:column_end - 1 },
+          \ 'start': { 'line': l:line_start - 1, 'character': l:char_start },
+          \ 'end': { 'line': l:line_end - 1, 'character': l:char_end },
           \}
 endfunction
 
@@ -506,14 +510,24 @@ function! s:handle_rename_prepare(server, last_req_id, type, data) abort
 
     let l:range = a:data['response']['result']
     let l:lines = getline(1, '$')
-    if l:range['start']['line'] ==# l:range['end']['line']
-        let l:name = l:lines[l:range['start']['line']][l:range['start']['character'] : l:range['end']['character']-1]
+    let l:start_line = l:range['start']['line'] + 1
+    let l:start_char = l:range['start']['character']
+    let l:start_col = lsp#utils#to_col('%', l:start_line, l:start_char)
+    let l:end_line = l:range['end']['line'] + 1
+    let l:end_char = l:range['end']['character']
+    let l:end_col = lsp#utils#to_col('%', l:end_line, l:end_char)
+    if l:start_line ==# l:end_line
+        let l:name = l:lines[l:start_line - 1][l:start_col - 1 : l:end_col - 2]
     else
-        let l:name = l:lines[l:range['start']['line']][l:range['start']['character'] :]
-        for l:i in range(l:range['start']['line']+1, l:range['end']['line']-1)
+        let l:name = l:lines[l:start_line - 1][l:start_col - 1 :]
+        for l:i in range(l:start_line, l:end_line - 2)
             let l:name .= "\n" . l:lines[l:i]
         endfor
-        let l:name .= l:lines[l:range['end']['line']][: l:range['end']['character']-1]
+        if l:end_col - 2 < 0
+            let l:name .= "\n"
+        else
+            let l:name .= l:lines[l:end_line - 1][: l:end_col - 2]
+        endif
     endif
 
     call timer_start(1, {x->s:rename(a:server, input('new name: ', l:name), l:range['start'])})

autoload/lsp/ui/vim/highlights.vim

@@ -93,15 +93,17 @@ function! s:place_highlights(server_name, path, diagnostics) abort
     let l:ns = s:get_highlight_group(a:server_name)
     let l:bufnr = bufnr(a:path)
 
-    if !empty(a:diagnostics) && bufnr(a:path) >= 0
+    if !empty(a:diagnostics) && l:bufnr >= 0
         for l:item in a:diagnostics
-            let l:line = l:item['range']['start']['line']
-            let l:start = l:item['range']['start']['character']
-            let l:end = l:item['range']['end']['character']
+            let l:line = l:item['range']['start']['line'] + 1
+            let l:start_char = l:item['range']['start']['character']
+            let l:start_col = lsp#utils#to_col(l:bufnr, l:line, l:start_char)
+            let l:end_char = l:item['range']['end']['character']
+            let l:end_col = lsp#utils#to_col(l:bufnr, l:line, l:end_char)
 
             let l:name = get(s:severity_sign_names_mapping, l:item['severity'], 'LspError')
             let l:hl_name = l:name . 'Highlight'
-            call nvim_buf_add_highlight(l:bufnr, l:ns, l:hl_name, l:line, l:start, l:end)
+            call nvim_buf_add_highlight(l:bufnr, l:ns, l:hl_name, l:line, l:start_col, l:end_col)
         endfor
     endif
 endfunction

autoload/lsp/ui/vim/references.vim

@@ -11,35 +11,41 @@ endif
 " If the range spans over multiple lines, break it down to multiple
 " positions, one for each line.
 " Return a list of positions.
-function! s:range_to_position(range) abort
+function! s:range_to_position(bufnr, range) abort
     let l:start = a:range['start']
     let l:end = a:range['end']
     let l:position = []
 
-    if l:end['line'] == l:start['line']
+    let l:start_line = l:start['line'] + 1
+    let l:start_char = l:start['character']
+    let l:start_col = lsp#utils#to_col(a:bufnr, l:start_line, l:start_char)
+    let l:end_line = l:end['line'] + 1
+    let l:end_char = l:end['character']
+    let l:end_col = lsp#utils#to_col(a:bufnr, l:end_line, l:end_char)
+    if l:end_line == l:start_line
         let l:position = [[
-        \ l:start['line'] + 1,
-        \ l:start['character'] + 1,
-        \ l:end['character'] - l:start['character']
+        \ l:start_line,
+        \ l:start_col,
+        \ l:end_col - l:start_col
         \ ]]
     else
         " First line
         let l:position = [[
-        \ l:start['line'] + 1,
-        \ l:start['character'] + 1,
+        \ l:start_line,
+        \ l:start_col,
         \ 999
         \ ]]
 
         " Last line
         call add(l:position, [
-        \ l:end['line'] + 1,
+        \ l:end_line,
         \ 1,
-        \ l:end['character']
+        \ l:end_col
         \ ])
 
         " Lines in the middle
         let l:middle_lines = map(
-        \ range(l:start['line'] + 2, end['line']),
+        \ range(l:start_line + 1, l:end_line - 1),
         \ {_, l -> [l, 0, 999]}
         \ )
 
@@ -105,7 +111,7 @@ function! s:handle_references(ctx, data) abort
     " Convert references to vim positions
     let l:position_list = []
     for l:reference in l:reference_list
-        call extend(l:position_list, s:range_to_position(l:reference['range']))
+        call extend(l:position_list, s:range_to_position(a:ctx['bufnr'], l:reference['range']))
     endfor
     call sort(l:position_list, function('s:compare_positions'))
 

autoload/lsp/ui/vim/signs.vim

@@ -124,7 +124,6 @@ function! s:place_signs(server_name, path, diagnostics) abort
     if !empty(a:diagnostics) && bufnr(a:path) >= 0
         for l:item in a:diagnostics
             let l:line = l:item['range']['start']['line'] + 1
-            let l:character = l:item['range']['start']['character'] + 1
 
             if has_key(l:item, 'severity') && !empty(l:item['severity'])
                 let l:sign_name = get(s:severity_sign_names_mapping, l:item['severity'], 'LspError')

autoload/lsp/utils.vim

@@ -202,3 +202,20 @@ function! lsp#utils#to_col(expr, lnum, char) abort
     let l:linestr = l:lines[-1]
     return strlen(strcharpart(l:linestr, 0, a:char)) + 1
 endfunction
+
+" Convert a byte-index (1-based) to a character-index (0-based)
+" This function requires a buffer specifier (expr, see :help bufname()),
+" a line number (lnum, 1-based), and a byte-index (char, 1-based).
+function! lsp#utils#to_char(expr, lnum, col) abort
+    let l:lines = getbufline(a:expr, a:lnum)
+    if l:lines == []
+        if type(a:expr) != v:t_string || !filereadable(a:expr)
+            " invalid a:expr
+            return a:col - 1
+        endif
+        " a:expr is a file that is not yet loaded as a buffer
+        let l:lines = readfile(a:expr, '', a:lnum)
+    endif
+    let l:linestr = l:lines[-1]
+    return strchars(strpart(l:linestr, 0, a:col - 1))
+endfunction

autoload/lsp/utils/text_edit.vim

@@ -167,7 +167,7 @@ function! s:generate_sub_cmd_insert(text_edit) abort
     let l:sub_cmd = s:preprocess_cmd(a:text_edit['range'])
     let l:sub_cmd .= s:generate_move_start_cmd(l:start_line, l:start_character)
 
-    if l:start_character >= len(getline(l:start_line))
+    if l:start_character >= strchars(getline(l:start_line))
         let l:sub_cmd .= "\"=l:merged_text_edit['merged']['newText']\<CR>p"
     else
         let l:sub_cmd .= "\"=l:merged_text_edit['merged']['newText']\<CR>P"

test/lsp/utils.vimspec

@@ -109,4 +109,32 @@ Describe lsp#utils
             Assert lsp#utils#to_col('./test/testfiles/multibyte.txt', 3, 0) == 1
         End
     End
+
+    Describe lsp#utils#to_char
+        It should return the character-index from the given byte-index on a buffer
+            call setline(1, ['a β c', 'δ', ''])
+            Assert lsp#utils#to_char('%', 1, 1) == 0
+            Assert lsp#utils#to_char('%', 1, 2) == 1
+            Assert lsp#utils#to_char('%', 1, 3) == 2
+            Assert lsp#utils#to_char('%', 1, 5) == 3
+            Assert lsp#utils#to_char('%', 1, 6) == 4
+            Assert lsp#utils#to_char('%', 1, 7) == 5
+            Assert lsp#utils#to_char('%', 2, 1) == 0
+            Assert lsp#utils#to_char('%', 2, 3) == 1
+            Assert lsp#utils#to_char('%', 3, 1) == 0
+            %delete
+        End
+
+        It should return the character-index from the given byte-index in an unloaded file
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 1) == 0
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 2) == 1
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 3) == 2
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 5) == 3
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 6) == 4
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 1, 7) == 5
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 2, 1) == 0
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 2, 3) == 1
+            Assert lsp#utils#to_char('./test/testfiles/multibyte.txt', 3, 1) == 0
+        End
+    End
 End