SONARPY-2014 SONARPY-2377 Repair rules reading PythonVisitorContext.pythonFile.contents() on Jupyter notebooks (#2192)

Author: ghislainpiot

its/ruling/src/test/resources/expected/ipython-LineLength.json

@@ -1,247 +1,22 @@
 {
 "project:Keras-Tutorials-master/01_linear_regression.ipynb": [
-36,
-337
-],
-"project:Keras-Tutorials-master/03_net.ipynb": [
-324
-],
-"project:Keras-Tutorials-master/04_modern_net.ipynb": [
-340
-],
-"project:Keras-Tutorials-master/05_convolutional_net.ipynb": [
-168,
-425
-],
-"project:Keras-Tutorials-master/06_autoencoder.ipynb": [
-390
-],
-"project:Keras-Tutorials-master/10_save_restore_net.ipynb": [
-121,
-123,
-125,
-127,
-129,
-131,
-133,
-135,
-137,
-139,
-146
+36
 ],
 "project:pecos/tutorials/kdd22/Session 2 Extreme Multi-label Classification with PECOS.ipynb": [
-10,
-16,
-23,
-86,
-88,
-102,
-105,
-107,
-112,
-157,
-231,
-234,
-235,
-280,
-286,
-288,
-309,
-323,
-325,
-358,
-379,
-411,
-444,
-466,
-470,
-474,
-475,
-476,
-477,
-478,
-479,
-486,
-496,
-498,
-531,
-533,
-537,
-570,
-583,
-584,
-585,
-586,
-599,
-602,
-612,
-614,
-617,
-620,
-675,
-690,
-691,
-692,
-693,
-732,
-799,
-810,
-824,
-855,
-961,
-991,
-1025,
-1057,
-1061,
-1067,
-1069,
-1163,
-1213,
-1300,
-1302
+855
 ],
 "project:pecos/tutorials/kdd22/Session 3 Approximate Nearest Neighbor Search in PECOS.ipynb": [
-21,
-23,
-28,
-29,
-40,
-58,
-59,
-72,
-77,
-145,
-147,
-148,
-149,
-150,
-153,
-155,
-216,
-219,
-228,
-257,
-258,
-348,
-443,
-474,
-481,
-495,
-701,
-751,
-755,
-766
+443
 ],
 "project:pecos/tutorials/kdd22/Session 4 Utilities in PECOS.ipynb": [
-10,
-25,
-35,
-183,
-261,
-263,
-273,
-274,
-409,
-437,
-462,
-531,
 581,
 582,
-592,
-724,
-791,
-798,
-801,
-803,
-808,
-818,
-895,
-1003,
-1014
+1003
 ],
 "project:pecos/tutorials/kdd22/Session 5 eXtreme Multi-label Classification with XR-Transformer.ipynb": [
-12,
-31,
-103,
-113,
-117,
-178,
 195,
-276,
-277,
-278,
-320,
-322,
-323,
 352,
-428,
 450,
-472,
-482,
-483,
-484,
-485,
-486,
-576,
-577,
-579,
-594,
-595,
-596,
-597,
-598,
-599,
-600,
-601,
-602,
-647,
-650,
-671,
-672,
-673,
-674,
-675,
-676,
-706,
-719,
-770,
-786,
-787,
-788,
-789,
-829,
-892,
-893,
-894,
-895,
-967,
 974
-],
-"project:tensorflow/python/autograph/g3doc/pyct_tutorial.ipynb": [
-16,
-18,
-45,
-55,
-131,
-158,
-221,
-290,
-325,
-370,
-372,
-413,
-444
-],
-"project:tensorflow/python/autograph/g3doc/reference/_control_flow_tutorial.ipynb": [
-10,
-73,
-75,
-131,
-198,
-268,
-299,
-377,
-524,
-704
 ]
 }

its/ruling/src/test/resources/expected/ipython-S113.json

@@ -13,5 +13,35 @@
 ],
 "project:Keras-Tutorials-master/05_convolutional_net.ipynb": [
 0
+],
+"project:Keras-Tutorials-master/06_autoencoder.ipynb": [
+0
+],
+"project:Keras-Tutorials-master/07_lstm.ipynb": [
+0
+],
+"project:Keras-Tutorials-master/09_tensorboard.ipynb": [
+0
+],
+"project:Keras-Tutorials-master/10_save_restore_net.ipynb": [
+0
+],
+"project:pecos/tutorials/kdd22/Session 2 Extreme Multi-label Classification with PECOS.ipynb": [
+0
+],
+"project:pecos/tutorials/kdd22/Session 3 Approximate Nearest Neighbor Search in PECOS.ipynb": [
+0
+],
+"project:pecos/tutorials/kdd22/Session 4 Utilities in PECOS.ipynb": [
+0
+],
+"project:pecos/tutorials/kdd22/Session 5 eXtreme Multi-label Classification with XR-Transformer.ipynb": [
+0
+],
+"project:tensorflow/python/autograph/g3doc/pyct_tutorial.ipynb": [
+0
+],
+"project:tensorflow/python/autograph/g3doc/reference/_control_flow_tutorial.ipynb": [
+0
 ]
 }

its/ruling/src/test/resources/expected/ipython-S1131.json

@@ -0,0 +1,48 @@
+{
+"project:Keras-Tutorials-master/06_autoencoder.ipynb": [
+403
+],
+"project:pecos/tutorials/kdd22/Session 2 Extreme Multi-label Classification with PECOS.ipynb": [
+75,
+644,
+665,
+774,
+847,
+851,
+852,
+853,
+856,
+866,
+867,
+868,
+873,
+877,
+879,
+882,
+1316,
+1354
+],
+"project:pecos/tutorials/kdd22/Session 3 Approximate Nearest Neighbor Search in PECOS.ipynb": [
+342,
+346,
+350
+],
+"project:pecos/tutorials/kdd22/Session 4 Utilities in PECOS.ipynb": [
+108,
+116,
+122,
+375,
+628,
+635,
+672,
+712,
+1074
+],
+"project:pecos/tutorials/kdd22/Session 5 eXtreme Multi-label Classification with XR-Transformer.ipynb": [
+70,
+197
+],
+"project:tensorflow/python/autograph/g3doc/reference/_control_flow_tutorial.ipynb": [
+677
+]
+}

python-checks/src/main/java/org/sonar/python/checks/ParsingErrorCheck.java

@@ -29,7 +29,7 @@ public class ParsingErrorCheck implements PythonCheck {
   public void scanFile(PythonVisitorContext context) {
     RecognitionException parsingException = context.parsingException();
     if (parsingException != null) {
-      context.addIssue(new PreciseIssue(this, IssueLocation.atLineLevel(parsingException.getMessage(), parsingException.getLine())));
+      context.addIssue(new PreciseIssue(this, IssueLocation.atLineLevel(parsingException.getMessage(), parsingException.getLine(), context.pythonFile())));
     }
   }
 

python-checks/src/main/java/org/sonar/python/checks/TrailingCommentCheck.java

@@ -61,7 +61,7 @@ public void initialize(Context context) {
           String comment = commentToken.value();
           if (!pattern.matcher(comment).matches()) {
             var issue = ctx.addIssue(commentToken, MESSAGE);
-            String line = getLines(ctx).get(commentToken.line() - 1);
+            String line = getLines(ctx).get(commentToken.pythonLine() - 1);
             addQuickFix(issue, commentToken, line);
           }
         }
@@ -72,10 +72,10 @@ public void initialize(Context context) {
 
   private static void addQuickFix(PreciseIssue issue, Token commentToken, String line) {
     String indent = calculateIndent(line);
-    PythonTextEdit insertComment = TextEditUtils.insertAtPosition(commentToken.line(), 0, indent + commentToken.value() + "\n");
+    PythonTextEdit insertComment = TextEditUtils.insertAtPosition(commentToken.pythonLine(), 0, indent + commentToken.value() + "\n");
 
     int startColumnRemove = calculateStartColumnToRemove(commentToken, line);
-    PythonTextEdit removeTrailingComment = TextEditUtils.removeRange(commentToken.line(), startColumnRemove, commentToken.line(), line.length());
+    PythonTextEdit removeTrailingComment = TextEditUtils.removeRange(commentToken.pythonLine(), startColumnRemove, commentToken.pythonLine(), line.length());
 
     PythonQuickFix fix = PythonQuickFix.newQuickFix(MESSAGE, removeTrailingComment, insertComment);
     issue.addQuickFix(fix);
@@ -95,7 +95,7 @@ private List<String> getLines(SubscriptionContext ctx) {
   }
 
   private static int calculateStartColumnToRemove(Token commentToken, String line) {
-    return line.substring(0, commentToken.column()).stripTrailing().length();
+    return line.substring(0, commentToken.pythonColumn()).stripTrailing().length();
   }
 }
 

python-checks/src/main/java/org/sonar/python/checks/TrailingWhitespaceCheck.java

@@ -38,10 +38,10 @@ public void scanFile(PythonVisitorContext ctx) {
       Matcher matcher = TRAILING_WS.matcher(lines[i]);
       if (matcher.find()) {
         int lineNumber = i + 1;
-        PreciseIssue issue = new PreciseIssue(this, IssueLocation.atLineLevel(MESSAGE, lineNumber));
+        PreciseIssue issue = new PreciseIssue(this, IssueLocation.atLineLevel(MESSAGE, lineNumber, ctx.pythonFile()));
 
         issue.addQuickFix(PythonQuickFix.newQuickFix("Remove trailing whitespaces")
-            .addTextEdit(TextEditUtils.removeRange(lineNumber, matcher.start(), lineNumber, matcher.end()))
+          .addTextEdit(TextEditUtils.removeRange(lineNumber, matcher.start(), lineNumber, matcher.end()))
           .build());
 
         ctx.addIssue(issue);