SONARPY-2067 First line of the first cell of notebooks should end at the correct location (#1920)

Author: ghislainpiot

sonar-python-plugin/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java

@@ -198,6 +198,7 @@ private static NotebookParsingData parseSourceMultilineString(int startLine, Jso
     JsonLocation tokenLocation = jParser.currentTokenLocation();
     var previousLen = 0;
     var previousExtraChars = 0;
+    var isFirstLine = true;
 
     for (String line : sourceLine.lines().toList()) {
       var countEscapedChar = countEscapeCharacters(line, previousLen + previousExtraChars + tokenLocation.getColumnNr());
@@ -207,6 +208,10 @@ private static NotebookParsingData parseSourceMultilineString(int startLine, Jso
       cellData.appendToSource("\n");
       previousLen = previousLen + line.length() + 2;
       previousExtraChars = previousExtraChars + currentCount;
+      if (isFirstLine) {
+        isFirstLine = false;
+        previousLen += 1;
+      }
     }
     // Account for the last cell delimiter
     cellData.addDelimiterToSource(SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER + "\n", tokenLocation.getLineNr(), tokenLocation.getColumnNr());
@@ -218,7 +223,7 @@ private static Map<Integer, Integer> countEscapeCharacters(String sourceLine, in
     int count = 0;
     var numberOfExtraChars = 0;
     var arr = sourceLine.toCharArray();
-    for (int i = 1; i < sourceLine.length(); ++i) {
+    for (int i = 0; i < sourceLine.length(); ++i) {
       char c = arr[i];
       switch (c) {
         case '"', '\\':

sonar-python-plugin/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java

@@ -53,14 +53,14 @@ void testParseNotebook() throws IOException {
     assertThat(result.locationMap()).extracting(map -> map.get(3)).isEqualTo(new IPythonLocation(19, 5, Map.of(10, 16, 19, 26, -1, 2)));
     //"source": "#Some code\nprint(\"hello world\\n\")",
     assertThat(result.locationMap()).extracting(map -> map.get(16)).isEqualTo(new IPythonLocation(64, 14, Map.of(-1, 0)));
-    assertThat(result.locationMap()).extracting(map -> map.get(17)).isEqualTo(new IPythonLocation(64, 26, Map.of(6, 33, 18, 46, 20, 49, -1, 3)));
+    assertThat(result.locationMap()).extracting(map -> map.get(17)).isEqualTo(new IPythonLocation(64, 27, Map.of(6, 34, 18, 47, 20, 50, -1, 3)));
     //"source": "print(\"My\\ntext\")\nprint(\"Something else\\n\")"
     assertThat(result.locationMap()).extracting(map -> map.get(22)).isEqualTo(new IPythonLocation(83, 14, Map.of(6, 21, 9, 25, 15, 32, -1, 3)));
-    assertThat(result.locationMap()).extracting(map -> map.get(23)).isEqualTo(new IPythonLocation(83, 36, Map.of(6, 43, 21, 59, 23, 62, -1, 3)));
+    assertThat(result.locationMap()).extracting(map -> map.get(23)).isEqualTo(new IPythonLocation(83, 37, Map.of(6, 44, 21, 60, 23, 63, -1, 3)));
 
-    //"source": "a = \"A bunch of characters \\n \\f \\r \\ \t \"\nb = None"
+    //"source": "a = \"A bunch of characters \\n \\f \\r \\  \"\nb = None"
     assertThat(result.locationMap()).extracting(map -> map.get(25))
-      .isEqualTo(new IPythonLocation(90, 14, Map.of(4, 19, 27, 43, 30, 47, 33, 51, 36, 55, 40, 61, -1, 6)));
+      .isEqualTo(new IPythonLocation(90, 14, Map.of(4,19, 27, 43, 30, 47, 33, 51, 36, 55, 39, 59, -1, 6)));
     assertThat(result.locationMap()).extracting(map -> map.get(26)).isEqualTo(new IPythonLocation(90, 63, Map.of(-1, 0)));
     // last line with the cell delimiter which contains the EOF token 
     assertThat(result.locationMap()).extracting(map -> map.get(27)).isEqualTo(new IPythonLocation(90, 14, Map.of(-1, 0)));
@@ -82,7 +82,7 @@ void testParseNotebookWithEmptyLines() throws IOException {
       .isEqualTo(1);
     assertThat(result.locationMap()).extracting(map -> map.get(3)).isEqualTo(new IPythonLocation(11, 5, Map.of(-1, 0)));
 
-    // last line with the cell delimiter which contains the EOF token 
+    // last line with the cell delimiter which contains the EOF token
     assertThat(result.locationMap()).extracting(map -> map.get(4)).isEqualTo(new IPythonLocation(11, 5, Map.of(-1, 0)));
   }
 
@@ -135,10 +135,18 @@ void testParseNotebookSingleLine() throws IOException {
     assertThat(resultOptional).isPresent();
 
     var result = resultOptional.get();
-    assertThat(result.locationMap()).hasSize(5);
-    assertThat(result.contents()).hasLineCount(5);
+    assertThat(result.locationMap()).hasSize(9);
+    assertThat(result.contents()).hasLineCount(9);
     // position of variable t
-    assertThat(result.locationMap().get(4).column()).isEqualTo(451);
+    assertThat(result.locationMap().get(4).column()).isEqualTo(452);
+
+    // First and second line
+    assertThat(result.locationMap()).containsEntry(1, new IPythonLocation(1, 382, Map.of(-1, 0)));
+    assertThat(result.locationMap()).containsEntry(2, new IPythonLocation(1, 429, Map.of(-1, 0)));
+
+    assertThat(result.locationMap()).containsEntry(6, new IPythonLocation(1, 559, Map.of(-1, 3, 0, 560, 1, 562, 2, 564)));
+    assertThat(result.locationMap()).containsEntry(7, new IPythonLocation(1, 610, Map.of(-1, 0)));
+    assertThat(result.locationMap()).containsEntry(8, new IPythonLocation(1, 637, Map.of(-1, 3, 1, 640, 2, 642, 0, 638)));
   }
 
   @Test

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook.ipynb

@@ -87,7 +87,7 @@
    "execution_count": 1,
    "metadata": {},
    "outputs": [],
-   "source": "a = \"A bunch of characters \\n \\f \\r \\ \t \"\nb = None"
+   "source": "a = \"A bunch of characters \\n \\f \\r \\  \"\nb = None"
   }
  ],
  "metadata": {

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook_single_line.ipynb

@@ -1,2 +1,2 @@
-{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-04-18T13:57:08.798932Z","iopub.status.busy":"2022-04-18T13:57:08.798479Z","iopub.status.idle":"2022-04-18T13:57:10.082993Z","shell.execute_reply":"2022-04-18T13:57:10.081937Z","shell.execute_reply.started":"2022-04-18T13:57:08.798842Z"},"trusted":true},"outputs":[],"source":"# Code you have previously used to load data\nimport pandas as pd\n\nt = '../input/home-data-for-ml-course/train.csv'"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":4}
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-04-18T13:57:08.798932Z","iopub.status.busy":"2022-04-18T13:57:08.798479Z","iopub.status.idle":"2022-04-18T13:57:10.082993Z","shell.execute_reply":"2022-04-18T13:57:10.081937Z","shell.execute_reply.started":"2022-04-18T13:57:08.798842Z"},"trusted":true},"outputs":[],"source":"# Code you have previously used to load data\nimport pandas as pd\n\nt = '../input/home-data-for-ml-course/train.csv'"},{"cell_type":"code","metadata":{},"outputs":[],"source":"\"\"\" A docstring that is going to be multiline\nHere is the second line !\n\"\"\""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":4}