SONARPY-1995: Support Jupyter Delimiter in Notebooks lexer (#2047)

Author: joke1196

python-frontend/src/main/java/org/sonar/python/lexer/IPynbCellDelimiterChannel.java

@@ -30,7 +30,8 @@
 public class IPynbCellDelimiterChannel extends Channel<Lexer> {
 
   private final StringBuilder tmpBuilder = new StringBuilder();
-  private final Matcher matcher = Pattern.compile("#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER").matcher("");
+  private final Matcher sonarLintVSCodeCellDelimiter = Pattern.compile("#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER").matcher("");
+  private final Matcher cellDelimiter = Pattern.compile("#\\h?%%(\\h+\\w+)?").matcher("");
   private final Token.Builder tokenBuilder = Token.builder();
   private final LexerState lexerState;
 
@@ -44,7 +45,7 @@ public boolean consume(CodeReader code, Lexer lexer) {
       // Cell delimiters must be at the beginning of a line
       return false;
     }
-    if (code.popTo(matcher, tmpBuilder) > 0) {
+    if (code.popTo(sonarLintVSCodeCellDelimiter, tmpBuilder) > 0 || code.popTo(cellDelimiter, tmpBuilder) > 0) {
       resetIndentationLevel(code, lexer);
       String value = tmpBuilder.toString();
 

python-frontend/src/test/java/org/sonar/python/lexer/IPythonLexerTest.java

@@ -30,7 +30,6 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.not;
 
-
 class IPythonLexerTest {
   private static TestLexer lexer;
 
@@ -50,9 +49,19 @@ List<Token> lex(String code) {
   }
 
   @Test
-  void cellDelimiterTest() {
+  void sonarLintVSCodeCellDelimiterTest() {
     assertThat(lexer.lex("foo\n#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER"), hasToken(PythonTokenType.IPYNB_CELL_DELIMITER));
     assertThat(lexer.lex("foo #SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER"), not(hasToken(PythonTokenType.IPYNB_CELL_DELIMITER)));
     assertThat(lexer.lex("if cond:\n  foo()\n#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER"), hasToken(PythonTokenType.DEDENT));
   }
+
+  @Test
+  void cellDelimiterTest() {
+    assertThat(lexer.lex("#%%\ndef foo():\n    pass\n#%%"), hasToken(PythonTokenType.IPYNB_CELL_DELIMITER));
+    assertThat(lexer.lex("#%% md\nthis is text ="), hasToken(PythonTokenType.IPYNB_CELL_DELIMITER));
+    assertThat(lexer.lex("foo\n#%%"), hasToken(PythonTokenType.IPYNB_CELL_DELIMITER));
+    assertThat(lexer.lex("foo\n# %%"), hasToken(PythonTokenType.IPYNB_CELL_DELIMITER));
+    assertThat(lexer.lex("foo #%% md"), not(hasToken(PythonTokenType.IPYNB_CELL_DELIMITER)));
+    assertThat(lexer.lex("if cond:\n  foo()\n# %% [markdown]"), hasToken(PythonTokenType.DEDENT));
+  }
 }

python-frontend/src/test/java/org/sonar/python/parser/PythonParserTest.java

@@ -48,7 +48,6 @@ private static Collection<File> listFiles(String folderName) {
     return FileUtils.listFiles(dir, new String[]{"py"}, true);
   }
 
-
   @Test
   void test_notebook() throws Exception {
     Collection<File> files = listFiles("notebooks");

python-frontend/src/test/resources/parser/notebooks/cell_delimiter.py

@@ -0,0 +1,8 @@
+#%%
+import collections
+collections.Counter??
+#%%
+def foo():
+    pass
+# %%
+a = 3