SONARPY-2066: IPython help command should be parsed correctly (#1926)

Author: joke1196

python-frontend/src/main/java/org/sonar/python/api/IPythonGrammarBuilder.java

@@ -89,7 +89,8 @@ protected void iPythonRules(LexerfulGrammarBuilder b) {
         TEST,
         QUESTION_MARK,
         b.optional(QUESTION_MARK)
-      )
+      ),
+      QUESTION_MARK
     ));
   }
 

python-frontend/src/test/java/org/sonar/python/parser/PythonParserTest.java

@@ -26,23 +26,36 @@
 import org.junit.jupiter.api.Test;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 
 class PythonParserTest {
 
   private final PythonParser parser = PythonParser.create();
 
+  private final PythonParser notebookParser = PythonParser.createIPythonParser();
+
   @Test
-  void test() throws Exception {
-    Collection<File> files = listFiles();
+  void test_python() throws Exception {
+    Collection<File> files = listFiles("python");
     for (File file : files) {
       String fileContent = new String(Files.readAllBytes(file.toPath()), UTF_8);
-      parser.parse(fileContent);
+      assertDoesNotThrow(() -> parser.parse(fileContent));
     }
   }
 
-  private static Collection<File> listFiles() {
-    File dir = new File("src/test/resources/parser/");
+  private static Collection<File> listFiles(String folderName) {
+    File dir = new File(String.format("src/test/resources/parser/%s/", folderName));
     return FileUtils.listFiles(dir, new String[]{"py"}, true);
   }
 
+
+  @Test
+  void test_notebook() throws Exception {
+    Collection<File> files = listFiles("notebooks");
+    for (File file : files) {
+      String fileContent = new String(Files.readAllBytes(file.toPath()), UTF_8);
+      assertDoesNotThrow(() -> notebookParser.parse(fileContent));
+    }
+  }
+
 }

python-frontend/src/test/java/org/sonar/python/tree/IPythonTreeMakerTest.java

@@ -134,17 +134,19 @@ void cellMagicUntilEndOfFile() {
 
   @Test
   void dynamicObjectInfo() {
-    var file = parseIPython("a = A()\n" +
-      "??a.foo\n" +
-      "?a.foo\n" +
-      "?a.foo?\n" +
-      "a.foo?\n" +
-      "a.foo??\n" +
-      "??a.foo()??\n" +
-      "b = a.foo()", treeMaker::fileInput);
+    var file = parseIPython("""
+      a = A()
+      ??a.foo
+      ?a.foo
+      ?a.foo?
+      a.foo?
+      a.foo??
+      ??a.foo()??
+      ?
+      b = a.foo()""", treeMaker::fileInput);
     var statementList = findFirstChildWithKind(file, Tree.Kind.STATEMENT_LIST);
     var statements = statementList.children();
-    assertThat(statements).hasSize(8);
+    assertThat(statements).hasSize(9);
 
     assertThat(statements.get(0).getKind()).isEqualTo(Tree.Kind.ASSIGNMENT_STMT);
     checkDynamicObjectInfo(statements.get(1), 2, 0);
@@ -153,7 +155,8 @@ void dynamicObjectInfo() {
     checkDynamicObjectInfo(statements.get(4), 0, 1);
     checkDynamicObjectInfo(statements.get(5), 0, 2);
     checkDynamicObjectInfo(statements.get(6), 2, 2);
-    assertThat(statements.get(7).getKind()).isEqualTo(Tree.Kind.ASSIGNMENT_STMT);
+    checkDynamicObjectInfo(statements.get(7), 0, 0);
+    assertThat(statements.get(8).getKind()).isEqualTo(Tree.Kind.ASSIGNMENT_STMT);
   }
 
   private void checkDynamicObjectInfo(Tree dynamicObjectInfo, int questionMarksBefore, int questionMarksAfter) {

python-frontend/src/test/resources/parser/notebooks/help_command.py

@@ -0,0 +1,5 @@
+?
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+import collections
+collections.Counter??
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER