SONARPY-2003 Implement basic notebook parser with support for multiline string code cells (#1861)

Author: ghislainpiot

sonar-python-plugin/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java

@@ -44,7 +44,7 @@ private IpynbNotebookParser(PythonInputFile inputFile) {
   }
 
   private final PythonInputFile inputFile;
-  private StringBuilder aggregatedSource = new StringBuilder();
+  private final StringBuilder aggregatedSource = new StringBuilder();
 
   // Keys are the aggregated source line number
   private final Map<Integer, IPythonLocation> locationMap = new HashMap<>();
@@ -76,7 +76,7 @@ private void processCodeCell(JsonParser jParser) throws IOException {
       JsonToken jsonToken = jParser.nextToken();
       if (JsonToken.FIELD_NAME.equals(jsonToken) && "source".equals(jParser.currentName())) {
         jsonToken = jParser.nextToken();
-        if (parseSourceArray(jParser, jsonToken)) {
+        if (parseSourceArray(jParser, jsonToken) || parseSourceMultilineString(jParser, jsonToken)) {
           break;
         } else {
           throw new IllegalStateException("Unexpected token: " + jsonToken);
@@ -91,16 +91,38 @@ private boolean parseSourceArray(JsonParser jParser, JsonToken jsonToken) throws
     }
     while (jParser.nextToken() != JsonToken.END_ARRAY) {
       String sourceLine = jParser.getValueAsString();
-      JsonLocation tokenLocation = jParser.currentTokenLocation();
+      addLineToSource(sourceLine,  jParser.currentTokenLocation());
+    }
+    // Account for the last cell delimiter
+    addDelimiterToSource();
+    return true;
+  }
+
+  private boolean parseSourceMultilineString(JsonParser jParser, JsonToken jsonToken) throws IOException {
+    if (jsonToken != JsonToken.VALUE_STRING) {
+      return false;
+    }
+    String sourceLine = jParser.getValueAsString();
+    JsonLocation tokenLocation = jParser.currentTokenLocation();
 
-      aggregatedSource.append(sourceLine);
-      locationMap.put(aggregatedSourceLine, new IPythonLocation(tokenLocation.getLineNr(), tokenLocation.getColumnNr()));
-      aggregatedSourceLine++;
+    for (String line : sourceLine.lines().toList()) {
+      aggregatedSource.append(line);
+      addLineToSource("\n", tokenLocation);
     }
     // Account for the last cell delimiter
+    addDelimiterToSource();
+    return true;
+  }
+
+  private void addLineToSource(String sourceLine, JsonLocation tokenLocation) {
+    aggregatedSource.append(sourceLine);
+    locationMap.put(aggregatedSourceLine, new IPythonLocation(tokenLocation.getLineNr(), tokenLocation.getColumnNr()));
+    aggregatedSourceLine++;
+  }
+
+  private void addDelimiterToSource() {
     aggregatedSource.append(SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER);
     aggregatedSourceLine++;
-    return true;
   }
 
   public record ParseResult(PythonInputFile inputFile, String aggregatedSource, Map<Integer, IPythonLocation> locationMap) {

sonar-python-plugin/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java

@@ -37,9 +37,9 @@ void testParseNotebook() {
 
     IpynbNotebookParser.ParseResult result = IpynbNotebookParser.parseNotebook(inputFile);
 
-    assertThat(result.locationMap().keySet()).hasSize(12);
+    assertThat(result.locationMap().keySet()).hasSize(18);
     assertThat(StringUtils.countMatches(result.aggregatedSource(), IpynbNotebookParser.SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER))
-      .isEqualTo(3);
+      .isEqualTo(6);
   }
 
   @Test

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook.ipynb

@@ -58,6 +58,30 @@
    "source": [
     "x = 42"
    ]
+  },
+  {
+   "cell_type": "code",
+   "source": "#Some code\nprint(\"hello world\\n\")",
+   "execution_count": 0,
+   "outputs": [],
+   "metadata": {}
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"My\\ntext\")\n",
+    "print(\"Something else\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": "print(\"My\\ntext\")\nprint(\"Something else\\n\")"
   }
  ],
  "metadata": {