fix(parser): support free-form line continuation in lexical tokens

The support is more permissive than the standard dictates
and obviously does not comform with all the LSP responses
e.g. completions under certain conditions.

Fixes #235

Author: gnikit

CHANGELOG.md

@@ -8,6 +8,8 @@
   ([#398](https://github.com/fortran-lang/fortls/issues/398))
 - Fixed bug with string quotes not being escaped when looking for parenthesis
   ([#250](https://github.com/fortran-lang/fortls/issues/250))
+- Fixed bug with line continuations in lexical tokens
+  ([#235](https://github.com/fortran-lang/fortls/issues/235))
 
 ## 3.0.0
 

fortls/parsers/internal/parser.py

@@ -1121,10 +1121,7 @@ def get_code_line(
                         continue
                     opt_cont_match = FRegex.FREE_CONT.match(next_line)
                     if opt_cont_match:
-                        next_line = (
-                            " " * opt_cont_match.end(0)
-                            + next_line[opt_cont_match.end(0) :]
-                        )
+                        next_line = next_line[opt_cont_match.end(0) :]
                     post_lines.append(next_line)
                     line_stripped = strip_strings(next_line, maintain_len=True)
                     iAmper = line_stripped.find("&")

test/test_server_hover.py

@@ -680,3 +680,17 @@ def test_complicated_kind_spec():
         '```fortran90\nREAL(int(sin(0.5))+8+len("ab))c")-3) :: z\n```',
     ]
     validate_hover(results, ref_results)
+
+
+def test_multiline_lexical_token():
+    string = write_rpc_request(1, "initialize", {"rootPath": str(test_dir / "hover")})
+    file_path = test_dir / "hover" / "multiline_lexical_tokens.f90"
+    string += hover_req(file_path, 4, 8)
+    string += hover_req(file_path, 8, 16)
+    errcode, results = run_request(string, fortls_args=["-n", "1"])
+    assert errcode == 0
+    ref_results = [
+        "```fortran90\nINTEGER :: i\n```",
+        '```fortran90\nREAL(int(sin(0.5))+8+len("ab))c")-3) :: Z\n```',
+    ]
+    validate_hover(results, ref_results)

test/test_source/hover/multiline_lexical_tokens.f90

@@ -0,0 +1,10 @@
+program multiline_lexical_token
+implicit none
+inte&
+    &ger &
+    :: i
+RE&
+&AL(int(sin(0.5))&
+ &+8+len("ab)&
+    &)c")-3) :: Z
+end program