[GR-26178] Scope finder is not able to find all scopes.

PullRequest: graalpython/1284

Author: ppisl

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/parser/sst/SSTCheckOffsetVisitor.java

@@ -167,6 +167,13 @@ public void decorator02() throws Exception {
                         "class foo():pass");
     }
 
+    @Test
+    public void decorator03() throws Exception {
+        checkScopeAndTree("class C(object):\n" +
+                        "    @staticmethod\n" +
+                        "    def foo(): return 42\n");
+    }
+
     @Test
     public void classDoc01() throws Exception {
         checkTreeResult(

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ClassDefTests.java

@@ -43,6 +43,7 @@
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.parser.PythonParserImpl;
 import com.oracle.graal.python.parser.ScopeInfo;
+import com.oracle.graal.python.parser.sst.SSTCheckOffsetVisitor;
 import com.oracle.graal.python.parser.sst.SSTNode;
 import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonParser;
@@ -97,6 +98,7 @@ public Node parse(String src, String moduleName, PythonParser.ParserMode mode, F
         PythonParser parser = context.getCore().getParser();
         Node result = ((PythonParserImpl) parser).parseN(mode, context.getCore(), source, fd, null);
         lastGlobalScope = ((PythonParserImpl) parser).getLastGlobaScope();
+        lastSST = ((PythonParserImpl) parser).getLastSST();
         return result;
     }
 
@@ -260,6 +262,12 @@ public void checkScopeResult(String source, PythonParser.ParserMode mode) throws
         assertDescriptionMatches(scopes.toString(), goldenScopeFile);
     }
 
+    public void checkSSTNodeOffsets(SSTNode node) {
+        SSTCheckOffsetVisitor checker = new SSTCheckOffsetVisitor(node);
+        boolean result = node.accept(checker);
+        assertTrue(checker.getMessage(), result);
+    }
+
     protected String printTreeToString(Node node) {
         ParserTreePrinter visitor = new ParserTreePrinter();
         visitor.printFormatStringLiteralDetail = printFormatStringLiteralValues;
@@ -428,6 +436,7 @@ private static String lineSeparator(int number) {
 
     public void checkScopeAndTree(String source, PythonParser.ParserMode mode) throws Exception {
         checkScopeResult(source, mode);
+        checkSSTNodeOffsets(lastSST);
         checkTreeResult(source, mode);
     }
 
@@ -437,10 +446,12 @@ public void checkScopeAndTree(String source) throws Exception {
 
     public void checkTreeResult(String source) throws Exception {
         checkTreeResult(source, PythonParser.ParserMode.File);
+        checkSSTNodeOffsets(lastSST);
     }
 
     public void checkScopeAndTreeFromFile(File testFile) throws Exception {
         checkScopeFromFile(testFile, true);
+        checkSSTNodeOffsets(lastSST);
         checkTreeFromFile(testFile, true);
     }
 }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParserTestBase.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -165,5 +165,6 @@ private void checkScopeAndTree() throws Exception {
         File testFile = getTestFileFromTestAndTestMethod();
         checkScopeFromFile(testFile, true);
         checkTreeFromFile(testFile, true);
+        checkSSTNodeOffsets(getLastSST());
     }
 }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/RuntimeFileTests.java

@@ -98,6 +98,47 @@ public void ifTest() throws Exception {
         checkFinder(code, code.indexOf("def fn3"), code.indexOf("30") + 3, false);
     }
 
+    @Test
+    public void decoratedTest() throws Exception {
+        String code = "@mydecorator(lambda : 10)\n" +
+                        "def fn():\n" +
+                        "    pass\n";
+        checkFinder(code, code.indexOf("lambda"), code.indexOf("10") + 2, true);
+        checkFinder(code, code.indexOf("def fn"), code.indexOf("pass") + 5, false);
+    }
+
+    @Test
+    public void lambdaInLambda() throws Exception {
+        String code = "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()";
+        checkFinder(code, code.indexOf("lambda x"), code.indexOf("x()") + 3, true);
+        checkFinder(code, code.indexOf("lambda y"), code.indexOf("y()") + 3, true);
+        checkFinder(code, code.indexOf("lambda z"), code.indexOf("z :") + 1, true);
+    }
+
+    @Test
+    public void lambdaInCall01() throws Exception {
+        String code = "fn(lambda: 20)";
+        checkFinder(code, code.indexOf("lambda"), code.indexOf("20") + 2, true);
+    }
+
+    @Test
+    public void lambdaInCall02() throws Exception {
+        String code = "fn(a = lambda: 20)";
+        checkFinder(code, code.indexOf("lambda"), code.indexOf("20") + 2, true);
+    }
+
+    @Test
+    public void funcDefTest01() throws Exception {
+        String code = "def fn (filter = lambda x: x > 4): pass";
+        checkFinder(code, code.indexOf("lambda"), code.indexOf("4") + 1, true);
+    }
+
+    @Test
+    public void with01() throws Exception {
+        String code = "with lambda: fn() as michael: pass";
+        checkFinder(code, code.indexOf("lambda"), code.indexOf("fn()") + 4, true);
+    }
+
     private void checkFinder(String code, int startOffset, int endOffset, boolean isLambda) {
         SSTNode result = findNodeWithScope(code, startOffset, endOffset);
         Assert.assertNotNull("No node with scope was found ", result);

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/SSTNodeWithScopeFinderTest.java

@@ -35,7 +35,7 @@ ModuleRootNode Name: <module 'asyncWith02'> SourceSection: [0,47]`async def f():
                     BlockNode SourceSection: None
                     FunctionBodyNode SourceSection: [22,47]`with a as b, c as d:...`
                       WithNode SourceSection: [22,47]`with a as b, c as d:...`
-                        WithNode SourceSection: None
+                        WithNode SourceSection: [35,47]`c as d: pass`
                           ExpressionStatementNode SourceSection: [43,47]`pass`
                             EmptyNode SourceSection: [43,47]`pass`
                           WriteLocalVariableNodeGen SourceSection: None

graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/asyncWith02.tast

@@ -10,7 +10,7 @@ ModuleRootNode Name: <module 'with02'> SourceSection: [0,31]`with A() as a, B()
         EmptyNode SourceSection: None
       SideEffect:
         WithNode SourceSection: [0,31]`with A() as a, B() a...`
-          WithNode SourceSection: None
+          WithNode SourceSection: [15,31]`B() as b:↵  pass`
             ExpressionStatementNode SourceSection: [27,31]`pass`
               EmptyNode SourceSection: [27,31]`pass`
             WriteNameNodeGen SourceSection: None

graalpython/com.oracle.graal.python.test/testData/goldenFiles/BasicTests/with02.tast

@@ -35,7 +35,7 @@ ModuleRootNode Name: <module 'with04'> SourceSection: [0,45]`def fn():↵  with
                     BlockNode SourceSection: None
                     FunctionBodyNode SourceSection: [12,45]`with A() as a, B() a...`
                       WithNode SourceSection: [12,45]`with A() as a, B() a...`
-                        WithNode SourceSection: None
+                        WithNode SourceSection: [27,45]`B() as b:↵    pass`
                           ExpressionStatementNode SourceSection: [41,45]`pass`
                             EmptyNode SourceSection: [41,45]`pass`
                           WriteLocalVariableNodeGen SourceSection: None

graalpython/com.oracle.graal.python.test/testData/goldenFiles/BasicTests/with04.tast

@@ -0,0 +1,15 @@
+Scope: []
+    Kind: Module
+    FrameDescriptor: Empty
+    CellVars: Empty
+    FreeVars: Empty
+    Scope: C
+        Kind: Class
+        FrameDescriptor: [foo]
+        CellVars: Empty
+        FreeVars: Empty
+        Scope: foo
+            Kind: Function
+            FrameDescriptor: [<return_val>]
+            CellVars: Empty
+            FreeVars: Empty

graalpython/com.oracle.graal.python.test/testData/goldenFiles/ClassDefTests/decorator03.scope

@@ -0,0 +1,104 @@
+ModuleRootNode Name: <module 'decorator03'> SourceSection: [0,60]`class C(object):↵   ...`
+  Signature: varArgs=False, varKeywordArgs=False, noArguments=True, positionalOnly=True, requiresKeywordArgs=False
+  FreeVars: None
+  NeedsCellFrame: False
+  FrameDescriptor:  Empty
+  Documentation: None
+  InnerRootNode SourceSection: [0,60]`class C(object):↵   ...`
+    ExpressionWithSideEffect SourceSection: [0,60]`class C(object):↵   ...`
+      Expression:
+        EmptyNode SourceSection: None
+      SideEffect:
+        WriteNameNodeGen SourceSection: [0,60]`class C(object):↵   ...`
+          Identifier: C
+          PythonCallTernary SourceSection: [0,60]`class C(object):↵   ...`
+            CallTernaryMethodNodeGen SourceSection: None
+            GetCallAttributeNodeGen SourceSection: None
+              BuiltinsLiteralNode SourceSection: None
+            FunctionDefinitionNode Name: C SourceSection: None
+              Arguments: None
+              KwArguments: None
+              Documentation: None
+              FreeVarSlots: None
+              ExecutionSlots:
+                FreeVarsSlots: None
+                CellVarsSlots: None
+              ClassBodyRootNode SourceSection: [0,60]`class C(object):↵   ...`
+                Name: C
+                Signature: varArgs=False, varKeywordArgs=False, noArguments=False, positionalOnly=True, requiresKeywordArgs=False
+                  Param Names: namespace
+                CelVars: None
+                FreeVars: None
+                NeedsCellFrame: False
+                FrameDescriptor: 1 slots [foo]
+                ExecutionSlots:
+                  FreeVarsSlots: None
+                  CellVarsSlots: None
+                InnerRootNode SourceSection: [0,60]`class C(object):↵   ...`
+                  ReturnTargetNode SourceSection: [0,60]`class C(object):↵   ...`
+                    Body: BlockNode SourceSection: None
+                      ClassDefinitionPrologueNode SourceSection: None
+                        QualName: C
+                        ReadGlobalOrBuiltinNodeGen SourceSection: None
+                          Identifier: __name__
+                          ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
+                        ReadIndexedArgumentNodeGen SourceSection: None
+                          Index: 0
+                        SetItemIfNotPresentNodeGen SourceSection: None
+                          SetItemNodeGen SourceSection: None
+                      FunctionBodyNode SourceSection: [21,60]`@staticmethod↵    de...`
+                        SetItemNodeGen SourceSection: [21,60]`@staticmethod↵    de...`
+                          ArgumentExpressionNode SourceSection: None
+                            ReadIndexedArgumentNodeGen SourceSection: None
+                              Index: 0
+                          StringLiteralNode SourceSection: None
+                          PythonCallUnary SourceSection: [21,34]`@staticmethod`
+                            CallUnaryMethodNodeGen SourceSection: None
+                            ReadClassAttributeNodeGen SourceSection: [21,34]`@staticmethod`
+                              GetItemNodeGen SourceSection: None
+                                ArgumentExpressionNode SourceSection: None
+                                  ReadIndexedArgumentNodeGen SourceSection: None
+                                    Index: 0
+                                StringLiteralNode SourceSection: None
+                              ReadGlobalOrBuiltinNodeGen SourceSection: None
+                                Identifier: staticmethod
+                                ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
+                            FunctionDefinitionNode Name: foo SourceSection: None
+                              Arguments: None
+                              KwArguments: None
+                              Documentation: None
+                              FreeVarSlots: None
+                              ExecutionSlots:
+                                FreeVarsSlots: None
+                                CellVarsSlots: None
+                              FunctionRootNode SourceSection: [39,60]`def foo(): return 42...`
+                                Name: foo
+                                Signature: varArgs=False, varKeywordArgs=False, noArguments=True, positionalOnly=True, requiresKeywordArgs=False
+                                CelVars: None
+                                FreeVars: None
+                                NeedsCellFrame: False
+                                FrameDescriptor: 1 slots [<return_val>]
+                                ExecutionSlots:
+                                  FreeVarsSlots: None
+                                  CellVarsSlots: None
+                                InnerRootNode SourceSection: [39,60]`def foo(): return 42...`
+                                  ReturnTargetNode SourceSection: [39,60]`def foo(): return 42...`
+                                    Body: BlockNode SourceSection: None
+                                      BlockNode SourceSection: None
+                                      FunctionBodyNode SourceSection: [50,59]`return 42`
+                                        FrameReturnNode SourceSection: [50,59]`return 42`
+                                          WriteLocalVariableNodeGen SourceSection: None
+                                            Identifier: <return_val>
+                                            WriteLocalFrameSlotNodeGen SourceSection: None
+                                              Frame: [0,<return_val>,Illegal]
+                                            IntegerLiteralNode SourceSection: [57,59]`42`
+                                              Value: 42
+                                    Return Expresssion: ReadLocalVariableNode SourceSection: None
+                                      Frame: [0,<return_val>,Illegal]
+                                      ReadVariableFromFrameNodeGen SourceSection: None
+                    Return Expresssion: ObjectLiteralNode SourceSection: None
+            StringLiteralNode SourceSection: None
+            ReadNameNodeGen SourceSection: [8,14]`object`
+              Identifier: object
+              IsBuiltinClassProfile SourceSection: None
+                CachedDispatchFirst SourceSection: None