SONARPY-2035: Update TokenLocation to make use of the correct location for reporting (#1886)

Author: joke1196

python-frontend/src/main/java/org/sonar/plugins/python/api/tree/Token.java

@@ -40,4 +40,6 @@ public interface Token extends Tree {
 
   int includedEscapeChars();
 
+  int valueLength();
+
 }

python-frontend/src/main/java/org/sonar/python/TokenLocation.java

@@ -41,7 +41,7 @@ public TokenLocation(Token token) {
 
     } else {
       this.endLine = this.startLine;
-      this.endLineOffset = this.startLineOffset + token.value().length();
+      this.endLineOffset = this.startLineOffset + token.valueLength();
     }
   }
 

python-frontend/src/main/java/org/sonar/python/tree/TokenImpl.java

@@ -112,4 +112,9 @@ public Token firstToken() {
   public Token lastToken() {
     return this;
   }
+
+  @Override
+  public int valueLength() {
+    return value().length() + includedEscapeChars();
+  }
 }

python-frontend/src/test/java/org/sonar/python/TokenLocationTest.java

@@ -65,13 +65,40 @@ void test_comment() {
     assertOffsets(commentLocation, 1, 0, 1, 8);
   }
 
+
+  @Test
+  void test_escaped_chars_ipython_lexer() {
+    var token = new TokenImpl(iPythonLex("\"1\\n3\"").get(0), 3, 10, 3);
+    TokenLocation tokenLocation = new TokenLocation(token); 
+    assertOffsets(tokenLocation, 3, 10, 3, 19);
+
+    token = new TokenImpl(iPythonLex("foo").get(0), 10 , 20, 0);
+    tokenLocation = new TokenLocation(token);
+    assertOffsets(tokenLocation, 10, 20, 10, 23);
+  }
+
+  @Test
+  void test_multiline_ipython_lexer() {
+    var tokens = iPythonLex("'''first line\nsecond\\t'''");
+    var token = new TokenImpl(tokens.get(0), 3, 10, 1);
+    TokenLocation tokenLocation = new TokenLocation(token); 
+    assertOffsets(tokenLocation, 3, 10, 4, 11);
+  }
+
   private static void assertOffsets(TokenLocation tokenLocation, int startLine, int startLineOffset, int endLine, int endLineOffset) {
     assertThat(tokenLocation.startLine()).as("start line").isEqualTo(startLine);
     assertThat(tokenLocation.startLineOffset()).as("start line offset").isEqualTo(startLineOffset);
     assertThat(tokenLocation.endLine()).as("end line").isEqualTo(endLine);
     assertThat(tokenLocation.endLineOffset()).as("end line offset").isEqualTo(endLineOffset);
   }
 
+  private List<com.sonar.sslr.api.Token> iPythonLex(String toLex) {
+    LexerState lexerState = new LexerState();
+    lexerState.reset();
+    Lexer lexer = PythonLexer.create(lexerState);
+    return lexer.lex(toLex);
+  }
+
   private List<Token> lex(String toLex) {
     LexerState lexerState = new LexerState();
     lexerState.reset();