Skip to content

Commit 657a505

Browse files
joke1196ghislainpiot
joke1196
authored and
ghislainpiot
committed
SONARPY-2047: Enrich trivia tokens (#1902)
1 parent a1bad88 commit 657a505

File tree

10 files changed

+117
-42
lines changed

10 files changed

+117
-42
lines changed

‎python-frontend/src/main/java/org/sonar/python/tree/PythonTreeMaker.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ private List<Statement> getStatementsFromSuite(AstNode astNode) {
273273
return Collections.emptyList();
274274
}
275275

276-
static List<StatementWithSeparator> getStatements(AstNode astNode) {
276+
List<StatementWithSeparator> getStatements(AstNode astNode) {
277277
List<AstNode> statements = astNode.getChildren(PythonGrammar.STATEMENT);
278278
List<StatementWithSeparator> statementsWithSeparators = new ArrayList<>();
279279
for (AstNode stmt : statements) {
@@ -288,19 +288,19 @@ static List<StatementWithSeparator> getStatements(AstNode astNode) {
288288
return statementsWithSeparators;
289289
}
290290

291-
private static List<StatementWithSeparator> getStatementsWithSeparators(AstNode stmt) {
291+
protected List<StatementWithSeparator> getStatementsWithSeparators(AstNode stmt) {
292292
List<StatementWithSeparator> statementsWithSeparators = new ArrayList<>();
293293
AstNode stmtListNode = stmt.getFirstChild(PythonGrammar.STMT_LIST);
294294
AstNode newLine = stmt.getFirstChild(PythonTokenType.NEWLINE);
295295
List<AstNode> children = stmtListNode.getChildren();
296296
int nbChildren = children.size();
297297
for (int i = 0; i < nbChildren; i += 2) {
298298
AstNode current = children.get(i);
299-
AstNode separator = current.getNextSibling();
300-
AstNode newLineForSeparator = null;
299+
Token separator = current.getNextSibling() == null ? null : toPyToken(current.getNextSibling().getToken());
300+
Token newLineForSeparator = null;
301301
boolean isLastStmt = nbChildren - i <= 2;
302302
if (isLastStmt) {
303-
newLineForSeparator = newLine;
303+
newLineForSeparator = newLine == null ? null : toPyToken(newLine.getToken());
304304
}
305305
statementsWithSeparators.add(new StatementWithSeparator(current.getFirstChild(), new Separators(separator, newLineForSeparator)));
306306
}

‎python-frontend/src/main/java/org/sonar/python/tree/Separators.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package org.sonar.python.tree;
2121

22-
import com.sonar.sslr.api.AstNode;
2322
import java.util.List;
2423
import java.util.Objects;
2524
import java.util.stream.Stream;
@@ -35,9 +34,9 @@ public class Separators {
3534
private final Token newline;
3635
private final List<Token> elements;
3736

38-
Separators(@Nullable AstNode separator, @Nullable AstNode newline){
39-
this.separator = separator == null ? null : new TokenImpl(separator.getToken());
40-
this.newline = newline == null ? null : new TokenImpl(newline.getToken());
37+
Separators(@Nullable Token separator, @Nullable Token newline) {
38+
this.separator = separator;
39+
this.newline = newline;
4140
this.elements = Stream.of(this.separator, this.newline).filter(Objects::nonNull).toList();
4241
}
4342

‎python-frontend/src/main/java/org/sonar/python/tree/TokenEnricher.java

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.List;
2424
import java.util.Map;
2525
import java.util.Set;
26+
import org.sonar.plugins.python.api.tree.Trivia;
2627
import org.sonar.python.IPythonLocation;
2728

2829
public class TokenEnricher {
@@ -44,17 +45,41 @@ public static TokenImpl enrichToken(Token token, Map<Integer, IPythonLocation> o
4445
}
4546
Map<Integer, Integer> escapeCharsMap = location.colOffset();
4647
int startCol = computeColWithEscapes(token.getColumn(), escapeCharsMap, location.column());
47-
int escapedCharInToken = 0;
48-
for (int i = 0; i < token.getValue().length(); i++) {
49-
if (ESCAPED_CHARS.contains(token.getValue().charAt(i))) {
50-
escapedCharInToken++;
51-
}
52-
}
53-
return new TokenImpl(token, location.line(), startCol, escapedCharInToken);
48+
int escapedCharInToken = computeEscapeCharsInToken(token.getValue());
49+
List<Trivia> trivia = token.getTrivia().stream()
50+
.map(t -> computeTriviaLocation(t, location.line(), startCol, token.getLine(), offsetMap))
51+
.toList();
52+
53+
return new TokenImpl(token, location.line(), startCol, escapedCharInToken, trivia);
5454
}
5555
return new TokenImpl(token);
5656
}
5757

58+
private static Trivia computeTriviaLocation(com.sonar.sslr.api.Trivia trivia, int parentLine, int parentCol, int parentPythonLine, Map<Integer, IPythonLocation> offsetMap) {
59+
int escapedCharInToken = computeEscapeCharsInToken(trivia.getToken().getValue());
60+
var line = parentLine;
61+
var col = parentCol - escapedCharInToken - trivia.getToken().getValue().length();
62+
if (parentPythonLine != trivia.getToken().getLine()) {
63+
IPythonLocation location = offsetMap.get(trivia.getToken().getLine());
64+
line = location.line();
65+
Map<Integer, Integer> escapeCharsMap = location.colOffset();
66+
col = computeColWithEscapes(trivia.getToken().getColumn(), escapeCharsMap, location.column());
67+
}
68+
return new TriviaImpl(new TokenImpl(trivia.getToken(), line, col,
69+
escapedCharInToken, List.of()));
70+
}
71+
72+
private static int computeEscapeCharsInToken(String tokenValue) {
73+
int escapedCharInToken = 0;
74+
for (int i = 0; i < tokenValue.length(); i++) {
75+
if (ESCAPED_CHARS.contains(tokenValue.charAt(i))) {
76+
escapedCharInToken++;
77+
}
78+
}
79+
return escapedCharInToken;
80+
81+
}
82+
5883
private static int computeColWithEscapes(int currentCol, Map<Integer, Integer> escapes, int offsetColumn) {
5984
return (int) escapes.keySet().stream().filter(k -> k > 0 && k < currentCol).count() + offsetColumn + currentCol;
6085
}

‎python-frontend/src/main/java/org/sonar/python/tree/TokenImpl.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,12 @@ public TokenImpl(com.sonar.sslr.api.Token token) {
4242
this.trivia = token.getTrivia().stream().map(tr -> new TriviaImpl(new TokenImpl(tr.getToken()))).collect(Collectors.toList());
4343
}
4444

45-
public TokenImpl(com.sonar.sslr.api.Token token, int line, int column, int includedEscapeChars) {
46-
this(token);
45+
public TokenImpl(com.sonar.sslr.api.Token token, int line, int column, int includedEscapeChars, List<Trivia> trivia) {
46+
this.token = token;
4747
this.line = line;
4848
this.column = column;
4949
this.includedEscapeChars = includedEscapeChars;
50+
this.trivia = trivia;
5051
}
5152

5253
@Override

‎python-frontend/src/test/java/org/sonar/python/TokenLocationTest.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,25 +63,25 @@ void test_one_line() {
6363
void test_comment() {
6464
TokenLocation commentLocation = new TokenLocation(lex("#comment\n").get(0).trivia().get(0).token());
6565
assertOffsets(commentLocation, 1, 0, 1, 8);
66-
}
6766

67+
}
6868

6969
@Test
7070
void test_escaped_chars_ipython_lexer() {
71-
var token = new TokenImpl(iPythonLex("\"1\\n3\"").get(0), 3, 10, 3);
72-
TokenLocation tokenLocation = new TokenLocation(token);
71+
var token = new TokenImpl(iPythonLex("\"1\\n3\"").get(0), 3, 10, 3, List.of());
72+
TokenLocation tokenLocation = new TokenLocation(token);
7373
assertOffsets(tokenLocation, 3, 10, 3, 19);
7474

75-
token = new TokenImpl(iPythonLex("foo").get(0), 10 , 20, 0);
75+
token = new TokenImpl(iPythonLex("foo").get(0), 10, 20, 0, List.of());
7676
tokenLocation = new TokenLocation(token);
7777
assertOffsets(tokenLocation, 10, 20, 10, 23);
7878
}
7979

8080
@Test
8181
void test_multiline_ipython_lexer() {
8282
var tokens = iPythonLex("'''first line\nsecond\\t'''");
83-
var token = new TokenImpl(tokens.get(0), 3, 10, 1);
84-
TokenLocation tokenLocation = new TokenLocation(token);
83+
var token = new TokenImpl(tokens.get(0), 3, 10, 1, List.of());
84+
TokenLocation tokenLocation = new TokenLocation(token);
8585
assertOffsets(tokenLocation, 3, 10, 4, 11);
8686
}
8787

‎python-frontend/src/test/java/org/sonar/python/tree/IPythonTreeMakerTest.java

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ void assignmentRhs() {
292292
@Test
293293
void enrichTokens() {
294294
var offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(4, 15, 8, 20, -1, 2)));
295-
var statementList = parseIPython(
295+
var statementList = parseIPython(
296296
"a = \"123\"", new IPythonTreeMaker(offsetMap)::fileInput).statements();
297297
assertThat(statementList).isNotNull();
298298

@@ -302,15 +302,27 @@ void enrichTokens() {
302302
assertThat(stringLiteral.get(0).firstToken().line()).isEqualTo(7);
303303
assertThat(stringLiteral.get(0).firstToken().column()).isEqualTo(14);
304304

305-
offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(-1,0)), 2, new IPythonLocation(8, 10, Map.of(-1, 0)));
306-
statementList = parseIPython(
305+
offsetMap = Map.of(1, new IPythonLocation(7, 10, Map.of(-1, 0)), 2, new IPythonLocation(8, 10, Map.of(-1, 0)));
306+
statementList = parseIPython(
307307
"def foo(): # comment \n pass", new IPythonTreeMaker(offsetMap)::fileInput).statements();
308308
assertThat(statementList).isNotNull();
309309
var passStatement = findChildrenWithKind(statementList, Tree.Kind.PASS_STMT)
310310
.stream().map(PassStatementImpl.class::cast).toList();
311311
assertThat(passStatement).hasSize(1);
312312
assertThat(passStatement.get(0).firstToken().line()).isEqualTo(8);
313313
assertThat(passStatement.get(0).firstToken().column()).isEqualTo(14);
314+
315+
List<TriviaImpl> comments = findChildrenWithKind(statementList, Tree.Kind.TOKEN)
316+
.stream()
317+
.map(TokenImpl.class::cast)
318+
.map(TokenImpl::trivia)
319+
.flatMap(List::stream)
320+
.map(TriviaImpl.class::cast)
321+
.toList();
322+
assertThat(comments).hasSize(1);
323+
assertThat(comments.get(0).token().line()).isEqualTo(7);
324+
assertThat(comments.get(0).token().column()).isEqualTo(21);
325+
314326
}
315327

316328
private static void assertLineMagicStatement(Statement statement) {

‎python-frontend/src/test/java/org/sonar/python/tree/TokenEnricherTest.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,4 +120,41 @@ void shouldComputeColCorrectly() {
120120
assertThat(eofToken.column()).isEqualTo(329);
121121
assertThat(eofToken.includedEscapeChars()).isZero();
122122
}
123+
124+
@Test
125+
void shouldComputeColCorrectlyForTrivia() {
126+
var code = "a = 3 # comment";
127+
var expectedTokens = lexer.lex(code);
128+
var tokens = TokenEnricher.enrichTokens(expectedTokens, Map.of(1, new IPythonLocation(100, 300, Map.of(-1, 0))));
129+
var trivias = tokens.get(tokens.size() - 1).trivia();
130+
assertThat(trivias).hasSize(1);
131+
assertThat(trivias.get(0).token().line()).isEqualTo(100);
132+
assertThat(trivias.get(0).token().column()).isEqualTo(306);
133+
assertThat(trivias.get(0).token().includedEscapeChars()).isZero();
134+
}
135+
136+
@Test
137+
void shouldComputeColCorrectlyForTriviaWithEscapeChar() {
138+
var code = "a = 3 # test\\n";
139+
var expectedTokens = lexer.lex(code);
140+
var tokens = TokenEnricher.enrichTokens(expectedTokens, Map.of(1, new IPythonLocation(100, 300, Map.of(-1, 1, 12, 13))));
141+
var trivias = tokens.get(tokens.size() - 1).trivia();
142+
assertThat(trivias).hasSize(1);
143+
assertThat(trivias.get(0).token().line()).isEqualTo(100);
144+
assertThat(trivias.get(0).token().column()).isEqualTo(306);
145+
assertThat(trivias.get(0).token().includedEscapeChars()).isEqualTo(1);
146+
}
147+
148+
@Test
149+
void shouldComputeColCorrectlyForTriviaOnDifferentLine() {
150+
var code = "# comment\na = 3";
151+
var expectedTokens = lexer.lex(code);
152+
var tokens = TokenEnricher.enrichTokens(expectedTokens, Map.of(1, new IPythonLocation(100, 300, Map.of(-1, 0)), 2, new IPythonLocation(101, 300, Map.of(-1, 0))));
153+
assertThat(tokens.get(0).line()).isEqualTo(101);
154+
var trivias = tokens.get(0).trivia();
155+
assertThat(trivias).hasSize(1);
156+
assertThat(trivias.get(0).token().line()).isEqualTo(100);
157+
assertThat(trivias.get(0).token().column()).isEqualTo(300);
158+
assertThat(trivias.get(0).token().includedEscapeChars()).isZero();
159+
}
123160
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonHighlighter.java

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,17 @@
2828
import org.sonar.api.batch.sensor.highlighting.TypeOfText;
2929
import org.sonar.plugins.python.api.PythonSubscriptionCheck;
3030
import org.sonar.plugins.python.api.PythonVisitorContext;
31-
import org.sonar.python.SubscriptionVisitor;
32-
import org.sonar.python.TokenLocation;
33-
import org.sonar.python.api.PythonKeyword;
34-
import org.sonar.python.api.PythonTokenType;
3531
import org.sonar.plugins.python.api.tree.ClassDef;
3632
import org.sonar.plugins.python.api.tree.FileInput;
3733
import org.sonar.plugins.python.api.tree.FunctionDef;
3834
import org.sonar.plugins.python.api.tree.StringLiteral;
3935
import org.sonar.plugins.python.api.tree.Token;
4036
import org.sonar.plugins.python.api.tree.Tree;
4137
import org.sonar.plugins.python.api.tree.Trivia;
38+
import org.sonar.python.SubscriptionVisitor;
39+
import org.sonar.python.TokenLocation;
40+
import org.sonar.python.api.PythonKeyword;
41+
import org.sonar.python.api.PythonTokenType;
4242

4343
import static com.sonar.sslr.api.GenericTokenType.IDENTIFIER;
4444

@@ -90,13 +90,11 @@ public class PythonHighlighter extends PythonSubscriptionCheck {
9090

9191
private NewHighlighting newHighlighting;
9292
private Set<Token> docStringTokens;
93-
private PythonInputFile inputFile;
9493

9594
public PythonHighlighter(SensorContext context, PythonInputFile inputFile) {
9695
docStringTokens = new HashSet<>();
9796
newHighlighting = context.newHighlighting();
9897
newHighlighting.onFile(inputFile.wrappedFile());
99-
this.inputFile = inputFile;
10098
}
10199

102100
@Override
@@ -138,10 +136,8 @@ private void visitToken(Token token) {
138136

139137
}
140138

141-
if (inputFile.kind() == PythonInputFile.Kind.PYTHON) {
142-
for (Trivia trivia : token.trivia()) {
143-
highlight(trivia.token(), TypeOfText.COMMENT);
144-
}
139+
for (Trivia trivia : token.trivia()) {
140+
highlight(trivia.token(), TypeOfText.COMMENT);
145141
}
146142
}
147143

‎sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonHighlighterTest.java

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,13 +202,14 @@ void number() {
202202

203203
@Test
204204
void highlightingNotebooks() {
205-
String pythonContent = "def foo():\n pass\na = \"test\" # comment\nb = 3J\n#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER";
205+
String pythonContent = "def foo():\n pass\na = \"test\" # comment\n# test \\n \\\\n test\nb = 3J\n#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER";
206206
var locations = Map.of(
207207
1, new IPythonLocation(9, 5, Map.of(-1, 0)),
208208
2, new IPythonLocation(10, 5, Map.of(-1, 0)),
209209
3, new IPythonLocation(11, 5, Map.of(-1, 2, 9, 10, 14, 16)),
210-
4, new IPythonLocation(12, 5, Map.of(-1, 0)),
211-
5, new IPythonLocation(12, 0, Map.of(-1, 0))); //EOF Token
210+
4, new IPythonLocation(12, 5, Map.of(-1, 3, 7, 12, 10, 16, 11, 18)),
211+
5, new IPythonLocation(13, 5, Map.of(-1, 0)),
212+
6, new IPythonLocation(13, 5, Map.of(-1, 0))); //EOF Token
212213
PythonHighlighter pythonHighlighter = new PythonHighlighter(context, new GeneratedIPythonFile(notebookInputFile, pythonContent, locations));
213214
TestPythonVisitorRunner.scanNotebookFile(notebookFile, locations, pythonContent, pythonHighlighter);
214215
// def
@@ -217,8 +218,12 @@ void highlightingNotebooks() {
217218
checkOnRange(10, 9, 4, notebookFile, TypeOfText.KEYWORD);
218219
// \"test\"
219220
checkOnRange(11, 9, 8, notebookFile, TypeOfText.STRING);
221+
// # comment
222+
checkOnRange(11, 18, 9, notebookFile, TypeOfText.COMMENT);
223+
// # test \\n \\\\n test
224+
checkOnRange(12, 5, 21, notebookFile, TypeOfText.COMMENT);
220225
// 3J
221-
checkOnRange(12, 9, 2, notebookFile, TypeOfText.CONSTANT);
226+
checkOnRange(13, 9, 2, notebookFile, TypeOfText.CONSTANT);
222227
}
223228

224229
/**
@@ -240,7 +245,6 @@ private void checkOnRange(int line, int firstColumn, int length, File file, Type
240245
checkInternal(line, firstColumn + length, " (= after the token)", file, null);
241246
}
242247

243-
244248
/**
245249
* Checks the highlighting of one column. The first column of a line has index 0.
246250
*/

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebookHighlighter.ipynb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"def foo():\n",
1010
" pass\n",
1111
"a = \"test\" # comment\n",
12+
"# test \\n \\\\n test\n",
1213
"b = 3J"
1314
]
1415
}

0 commit comments

Comments
 (0)