SONARPY-2056 IPython line magics starting with numbers or special characters should be parsed correctly (#1910)

Author: ghislainpiot

python-frontend/src/main/java/org/sonar/python/api/IPythonGrammarBuilder.java

@@ -24,12 +24,12 @@
 import static com.sonar.sslr.api.GenericTokenType.EOF;
 import static org.sonar.python.api.IPythonGrammar.CELL;
 import static org.sonar.python.api.IPythonGrammar.CELL_MAGIC_STATEMENT;
-import static org.sonar.python.api.IPythonGrammar.LINE_MAGIC_STATEMENT;
 import static org.sonar.python.api.IPythonGrammar.DYNAMIC_OBJECT_INFO_STATEMENT;
 import static org.sonar.python.api.IPythonGrammar.LINE_MAGIC;
+import static org.sonar.python.api.IPythonGrammar.LINE_MAGIC_STATEMENT;
 import static org.sonar.python.api.IPythonGrammar.MAGIC_CELL;
-import static org.sonar.python.api.PythonGrammar.ASSERT_STMT;
 import static org.sonar.python.api.PythonGrammar.ANNOTATED_RHS;
+import static org.sonar.python.api.PythonGrammar.ASSERT_STMT;
 import static org.sonar.python.api.PythonGrammar.BREAK_STMT;
 import static org.sonar.python.api.PythonGrammar.CONTINUE_STMT;
 import static org.sonar.python.api.PythonGrammar.DEL_STMT;
@@ -38,7 +38,6 @@
 import static org.sonar.python.api.PythonGrammar.FILE_INPUT;
 import static org.sonar.python.api.PythonGrammar.GLOBAL_STMT;
 import static org.sonar.python.api.PythonGrammar.IMPORT_STMT;
-import static org.sonar.python.api.PythonGrammar.NAME;
 import static org.sonar.python.api.PythonGrammar.NONLOCAL_STMT;
 import static org.sonar.python.api.PythonGrammar.PASS_STMT;
 import static org.sonar.python.api.PythonGrammar.PRINT_STMT;
@@ -75,7 +74,7 @@ protected void iPythonRules(LexerfulGrammarBuilder b) {
     b.rule(LINE_MAGIC_STATEMENT).is(LINE_MAGIC);
     b.rule(LINE_MAGIC).is(
       b.firstOf(PythonPunctuator.MOD, "!", PythonPunctuator.DIV),
-      NAME,
+      b.anyTokenButNot(PythonPunctuator.MOD),
       b.zeroOrMore(b.anyTokenButNot(b.firstOf(NEWLINE, b.next(EOF))))
     );
     b.rule(DYNAMIC_OBJECT_INFO_STATEMENT).is(b.firstOf(

python-frontend/src/main/java/org/sonar/python/tree/IPythonTreeMaker.java

@@ -41,7 +41,6 @@
 import org.sonar.python.DocstringExtractor;
 import org.sonar.python.IPythonLocation;
 import org.sonar.python.api.IPythonGrammar;
-import org.sonar.python.api.PythonGrammar;
 
 public class IPythonTreeMaker extends PythonTreeMaker {
 
@@ -153,15 +152,14 @@ protected DynamicObjectInfoStatement dynamicObjectInfoStatement(AstNode astNode)
 
   protected LineMagic lineMagic(AstNode astNode) {
     var percent = toPyToken(astNode.getFirstChild().getToken());
-    var name = toPyToken(astNode.getFirstChild(PythonGrammar.NAME).getToken());
 
     var tokens = astNode.getChildren()
       .stream()
-      .skip(2)
+      .skip(1)
       .map(AstNode::getTokens)
       .flatMap(Collection::stream)
       .map(this::toPyToken)
       .toList();
-    return new LineMagicImpl(percent, name, tokens);
+    return new LineMagicImpl(percent, tokens);
   }
 }

python-frontend/src/main/java/org/sonar/python/tree/LineMagicImpl.java

@@ -29,12 +29,10 @@
 public class LineMagicImpl extends PyTree implements LineMagic {
 
   private final Token percent;
-  private final Tree name;
   private final List<Token> tokens;
 
-  public LineMagicImpl(Token percent, Tree name, List<Token> tokens) {
+  public LineMagicImpl(Token percent, List<Token> tokens) {
     this.percent = percent;
-    this.name = name;
     this.tokens = tokens;
   }
 
@@ -52,7 +50,6 @@ public Kind getKind() {
   List<Tree> computeChildren() {
     var children = new ArrayList<Tree>();
     children.add(percent);
-    children.add(name);
     children.addAll(tokens);
     return children;
   }

python-frontend/src/test/java/org/sonar/python/tree/IPythonTreeMakerTest.java

@@ -212,6 +212,20 @@ void lineMagic() {
     assertThat(lineMagic)
       .isNotNull()
       .isInstanceOf(LineMagic.class);
+
+    statements = parseIPython("""
+      !7z --help
+      !./script.sh
+      !
+      """, treeMaker::fileInput).statements().statements();
+    assertThat(statements).hasSize(3);
+    assertThat(statements.get(0).getKind()).isEqualTo(Tree.Kind.EXPRESSION_STMT);
+    assertThat(((ExpressionStatement) statements.get(0)).expressions()).hasSize(1).allMatch(e -> e.is(Tree.Kind.LINE_MAGIC));
+    assertThat(statements.get(1).getKind()).isEqualTo(Tree.Kind.EXPRESSION_STMT);
+    assertThat(((ExpressionStatement) statements.get(1)).expressions()).hasSize(1).allMatch(e -> e.is(Tree.Kind.LINE_MAGIC));
+    assertThat(statements.get(2).getKind()).isEqualTo(Tree.Kind.EXPRESSION_STMT);
+    assertThat(((ExpressionStatement) statements.get(2)).expressions()).hasSize(1).allMatch(e -> e.is(Tree.Kind.LINE_MAGIC));
+
   }
 
   @Test