SONARPY-2046: Fix missing enrichment of tokens (#1898)

Author: joke1196

python-frontend/src/main/java/org/sonar/python/tree/PythonTreeMaker.java

@@ -135,8 +135,8 @@ protected Token toPyToken(@Nullable com.sonar.sslr.api.Token token) {
     return new TokenImpl(token);
   }
 
-  private static List<Token> toPyToken(List<com.sonar.sslr.api.Token> tokens) {
-    return tokens.stream().map(TokenImpl::new).collect(Collectors.toList());
+  protected List<Token> toPyToken(List<com.sonar.sslr.api.Token> tokens) {
+    return tokens.stream().map(this::toPyToken).toList();
   }
 
   public void setParents(Tree root) {
@@ -1319,7 +1319,7 @@ private Expression repr(AstNode astNode) {
     return new ReprExpressionImpl(openingBacktick, expressionListTree, closingBacktick);
   }
 
-  private static List<Token> punctuators(AstNode astNode, PythonPunctuator punctuator) {
+  private List<Token> punctuators(AstNode astNode, PythonPunctuator punctuator) {
     return toPyToken(astNode.getChildren(punctuator).stream().map(AstNode::getToken).toList());
   }
 

python-frontend/src/test/java/org/sonar/python/tree/IPythonTreeMakerTest.java

@@ -291,8 +291,8 @@ void assignmentRhs() {
 
   @Test
   void enrichTokens() {
-    var offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(4, 15, 8, 20)));
-    var statementList = parseIPython(
+    var offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(4, 15, 8, 20, -1, 2)));
+    var statementList = parseIPython( 
       "a = \"123\"", new IPythonTreeMaker(offsetMap)::fileInput).statements();
     assertThat(statementList).isNotNull();
 
@@ -301,13 +301,16 @@ void enrichTokens() {
     assertThat(stringLiteral).hasSize(1);
     assertThat(stringLiteral.get(0).firstToken().line()).isEqualTo(7);
     assertThat(stringLiteral.get(0).firstToken().column()).isEqualTo(14);
-  }
 
-  @Test 
-  void toPyTokenShouldReturnNull(){
-    //when the token is null
-    var token = treeMaker.toPyToken(null);
-    assertThat(token).isNull();
+    offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(-1,0)), 2, new IPythonLocation(8, 10, Map.of(-1, 0)));
+    statementList = parseIPython( 
+      "def foo(): # comment \n    pass", new IPythonTreeMaker(offsetMap)::fileInput).statements();
+    assertThat(statementList).isNotNull();
+    var passStatement = findChildrenWithKind(statementList, Tree.Kind.PASS_STMT)
+      .stream().map(PassStatementImpl.class::cast).toList();
+    assertThat(passStatement).hasSize(1);
+    assertThat(passStatement.get(0).firstToken().line()).isEqualTo(8);
+    assertThat(passStatement.get(0).firstToken().column()).isEqualTo(14);
   }
 
   private static void assertLineMagicStatement(Statement statement) {