SONARPY-2004 Track escaped characters correctly in notebook parser (#1869)

Author: ghislainpiot

sonar-python-plugin/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java

@@ -25,6 +25,7 @@
 import com.fasterxml.jackson.core.JsonToken;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 public class IpynbNotebookParser {
@@ -91,7 +92,9 @@ private boolean parseSourceArray(JsonParser jParser, JsonToken jsonToken) throws
     }
     while (jParser.nextToken() != JsonToken.END_ARRAY) {
       String sourceLine = jParser.getValueAsString();
-      addLineToSource(sourceLine,  jParser.currentTokenLocation());
+      var tokenLocation = jParser.currentTokenLocation();
+      var countEscapedChar = countEscapeCharacters(sourceLine, new LinkedHashMap<>(), tokenLocation.getColumnNr());
+      addLineToSource(sourceLine, tokenLocation, countEscapedChar);
     }
     // Account for the last cell delimiter
     addDelimiterToSource();
@@ -104,19 +107,29 @@ private boolean parseSourceMultilineString(JsonParser jParser, JsonToken jsonTok
     }
     String sourceLine = jParser.getValueAsString();
     JsonLocation tokenLocation = jParser.currentTokenLocation();
+    var previousLen = 0;
+    var previousExtraChars = 0;
 
     for (String line : sourceLine.lines().toList()) {
-      aggregatedSource.append(line);
-      addLineToSource("\n", tokenLocation);
+      var countEscapedChar = countEscapeCharacters(line, new LinkedHashMap<>(), tokenLocation.getColumnNr());
+      var currentCount = countEscapedChar.get(-1);
+      addLineToSource(line, new IPythonLocation(tokenLocation.getLineNr(),
+        tokenLocation.getColumnNr() + previousLen + previousExtraChars + 1, countEscapedChar));
+      previousLen = line.length() + 2;
+      previousExtraChars = currentCount;
     }
     // Account for the last cell delimiter
     addDelimiterToSource();
     return true;
   }
 
-  private void addLineToSource(String sourceLine, JsonLocation tokenLocation) {
+  private void addLineToSource(String sourceLine, JsonLocation tokenLocation, Map<Integer, Integer> colOffset) {
+    addLineToSource(sourceLine, new IPythonLocation(tokenLocation.getLineNr(), tokenLocation.getColumnNr(), colOffset));
+  }
+
+  private void addLineToSource(String sourceLine, IPythonLocation location) {
     aggregatedSource.append(sourceLine);
-    locationMap.put(aggregatedSourceLine, new IPythonLocation(tokenLocation.getLineNr(), tokenLocation.getColumnNr()));
+    locationMap.put(aggregatedSourceLine, location);
     aggregatedSourceLine++;
   }
 
@@ -125,10 +138,36 @@ private void addDelimiterToSource() {
     aggregatedSourceLine++;
   }
 
+  private static Map<Integer, Integer> countEscapeCharacters(String sourceLine, Map<Integer, Integer> colMap, int colOffSet) {
+    int count = 0;
+    var numberOfExtraChars = 0;
+    var arr = sourceLine.toCharArray();
+    for (int i = 1; i < sourceLine.length(); ++i) {
+      char c = arr[i];
+      switch (c) {
+        case '"', '\'', '/':
+          // + 1 as we do have to count the open quote.
+          colMap.put(i, i + colOffSet + count + 1);
+          if (c != '/') {
+            numberOfExtraChars++;
+          }
+          break;
+        case '\\', '\b', '\f', '\n', '\r', '\t':
+          count += 2;
+          numberOfExtraChars++;
+          break;
+        default:
+          break;
+      }
+    }
+    colMap.put(-1, numberOfExtraChars);
+    return colMap;
+  }
+
   public record ParseResult(PythonInputFile inputFile, String aggregatedSource, Map<Integer, IPythonLocation> locationMap) {
   }
 
-  public record IPythonLocation(int line, int column) {
+  public record IPythonLocation(int line, int column, Map<Integer, Integer> colOffset) {
   }
 
 }

sonar-python-plugin/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java

@@ -20,6 +20,7 @@
 package org.sonar.plugins.python;
 
 import java.io.File;
+import java.util.Map;
 import org.junit.jupiter.api.Test;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.internal.apachecommons.lang.StringUtils;
@@ -35,11 +36,20 @@ class IpynbNotebookParserTest {
   void testParseNotebook() {
     var inputFile = createInputFile(baseDir, "notebook.ipynb", InputFile.Status.CHANGED, InputFile.Type.MAIN);
 
-    IpynbNotebookParser.ParseResult result = IpynbNotebookParser.parseNotebook(inputFile);
+    var result = IpynbNotebookParser.parseNotebook(inputFile);
 
-    assertThat(result.locationMap().keySet()).hasSize(18);
+    assertThat(result.locationMap().keySet()).hasSize(20);
     assertThat(StringUtils.countMatches(result.aggregatedSource(), IpynbNotebookParser.SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER))
-      .isEqualTo(6);
+      .isEqualTo(7);
+    assertThat(result.locationMap()).extracting(map -> map.get(17)).isEqualTo(new IpynbNotebookParser.IPythonLocation(64, 27, Map.of(6, 21, 20, 37, -1, 3)));
+
+    // The wrapped file changes the lines of the notebook
+    assertThat(result.locationMap()).extracting(map -> map.get(22)).isEqualTo(new IpynbNotebookParser.IPythonLocation(84, 15, Map.of(6, 21, 15, 32, -1, 3)));
+    assertThat(result.locationMap()).extracting(map -> map.get(23)).isEqualTo(new IpynbNotebookParser.IPythonLocation(84, 37, Map.of(6, 21, 23, 40, -1, 3)));
+
+    assertThat(result.locationMap()).extracting(map -> map.get(25))
+      .isEqualTo(new IpynbNotebookParser.IPythonLocation(91, 15, Map.of(4, 19, 39, 62, 41, 64, 42, 65, 46, 71, -1, 7)));
+    assertThat(result.locationMap()).extracting(map -> map.get(26)).isEqualTo(new IpynbNotebookParser.IPythonLocation(91, 71, Map.of(-1, 0)));
   }
 
   @Test

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook.ipynb

@@ -82,6 +82,13 @@
    "metadata": {},
    "outputs": [],
    "source": "print(\"My\\ntext\")\nprint(\"Something else\\n\")"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": "a = \"A bunch of characters \\n \\t \\f \\r / // \\ \"\nb = None"
   }
  ],
  "metadata": {