SONARPY-2045: Fix incorrect count of escaped chars (#1897)

Author: joke1196

python-frontend/src/main/java/org/sonar/python/tree/TokenEnricher.java

@@ -56,7 +56,7 @@ public static TokenImpl enrichToken(Token token, Map<Integer, IPythonLocation> o
   }
 
   private static int computeColWithEscapes(int currentCol, Map<Integer, Integer> escapes, int offsetColumn) {
-    return (int) escapes.keySet().stream().filter(k -> k < currentCol).count() + offsetColumn + currentCol;
+    return (int) escapes.keySet().stream().filter(k -> k > 0 && k < currentCol).count() + offsetColumn + currentCol;
   }
 
 }

sonar-python-plugin/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java

@@ -159,10 +159,10 @@ private boolean parseSourceMultilineString(JsonParser jParser, JsonToken jsonTok
     var previousExtraChars = 0;
 
     for (String line : sourceLine.lines().toList()) {
-      var countEscapedChar = countEscapeCharacters(line, new LinkedHashMap<>(), tokenLocation.getColumnNr());
+      var countEscapedChar = countEscapeCharacters(line, new LinkedHashMap<>(), previousLen + previousExtraChars + tokenLocation.getColumnNr());
       var currentCount = countEscapedChar.get(-1);
       addLineToSource(line, new IPythonLocation(tokenLocation.getLineNr(),
-        tokenLocation.getColumnNr() + previousLen + previousExtraChars + 1, countEscapedChar));
+        tokenLocation.getColumnNr() + previousLen + previousExtraChars, countEscapedChar));
       aggregatedSource.append("\n");
       previousLen = line.length() + 2;
       previousExtraChars = currentCount;
@@ -200,16 +200,14 @@ private static Map<Integer, Integer> countEscapeCharacters(String sourceLine, Ma
     for (int i = 1; i < sourceLine.length(); ++i) {
       char c = arr[i];
       switch (c) {
-        case '"', '\'', '/':
-          // + 1 as we do have to count the open quote.
-          colMap.put(i, i + colOffSet + count + 1);
-          if (c != '/') {
-            numberOfExtraChars++;
-          }
-          break;
-        case '\\', '\b', '\f', '\n', '\r', '\t':
-          count += 2;
+        case '"', '\'', '\\':
           numberOfExtraChars++;
+          colMap.put(i, i + colOffSet + count + numberOfExtraChars);
+          break;
+        // we never encounter \n or \r as the lines are split at these characters 
+        case '\b', '\f', '\t':
+          // we increase the count of one char as we count the \ but not the t or b
+          count += 1;
           break;
         default:
           break;

sonar-python-plugin/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java

@@ -48,14 +48,21 @@ void testParseNotebook() throws IOException {
     assertThat(result.contents()).hasLineCount(27);
     assertThat(StringUtils.countMatches(result.contents(), IpynbNotebookParser.SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER))
       .isEqualTo(7);
-    assertThat(result.locationMap()).extracting(map -> map.get(17)).isEqualTo(new IPythonLocation(64, 27, Map.of(6, 21, 20, 37, -1, 3)));
-
-    assertThat(result.locationMap()).extracting(map -> map.get(22)).isEqualTo(new IPythonLocation(83, 15, Map.of(6, 21, 15, 32, -1, 3)));
-    assertThat(result.locationMap()).extracting(map -> map.get(23)).isEqualTo(new IPythonLocation(83, 37, Map.of(6, 21, 23, 40, -1, 3)));
-
+    assertThat(result.locationMap()).extracting(map -> map.get(1)).isEqualTo(new IPythonLocation(17, 5, Map.of(-1, 0)));
+    //"    print \"not none\"\n"
+    assertThat(result.locationMap()).extracting(map -> map.get(3)).isEqualTo(new IPythonLocation(19, 5, Map.of(10, 16, 19, 26, -1, 2)));
+    //"source": "#Some code\nprint(\"hello world\\n\")",
+    assertThat(result.locationMap()).extracting(map -> map.get(16)).isEqualTo(new IPythonLocation(64, 14, Map.of(-1, 0)));
+    assertThat(result.locationMap()).extracting(map -> map.get(17)).isEqualTo(new IPythonLocation(64, 26, Map.of(6, 33, 18, 46, 20, 49, -1, 3)));
+
+    //"source": "print(\"My\\ntext\")\nprint(\"Something else\\n\")"
+    assertThat(result.locationMap()).extracting(map -> map.get(22)).isEqualTo(new IPythonLocation(83, 14, Map.of(6, 21, 9, 25, 15, 32, -1, 3)));
+    assertThat(result.locationMap()).extracting(map -> map.get(23)).isEqualTo(new IPythonLocation(83, 36, Map.of(6, 43, 21, 59, 23, 62, -1, 3)));
+
+    //"source": "a = \"A bunch of characters \\n \\f \\r \\ \t \"\nb = None"
     assertThat(result.locationMap()).extracting(map -> map.get(25))
-      .isEqualTo(new IPythonLocation(90, 15, Map.of(4, 19, 39, 62, 41, 64, 42, 65, 46, 71, -1, 7)));
-    assertThat(result.locationMap()).extracting(map -> map.get(26)).isEqualTo(new IPythonLocation(90, 71, Map.of(-1, 0)));
+      .isEqualTo(new IPythonLocation(90, 14, Map.of(4, 19, 27, 43, 30, 47, 33, 51, 36, 55, 40, 61, -1, 6)));
+    assertThat(result.locationMap()).extracting(map -> map.get(26)).isEqualTo(new IPythonLocation(90, 63, Map.of(-1, 0)));
     // last line with the cell delimiter which contains the EOF token 
     assertThat(result.locationMap()).extracting(map -> map.get(27)).isEqualTo(new IPythonLocation(90, 14, Map.of(-1, 0)));
   }

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook.ipynb

@@ -87,7 +87,7 @@
    "execution_count": 1,
    "metadata": {},
    "outputs": [],
-   "source": "a = \"A bunch of characters \\n \\t \\f \\r / // \\ \"\nb = None"
+   "source": "a = \"A bunch of characters \\n \\f \\r \\ \t \"\nb = None"
   }
  ],
  "metadata": {