[GR-40094] Fix codeobject inspection and creation under bytecode interpreter

PullRequest: graalpython/2364

Author: msimacek

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "31fa7a976122ea476d023bc2304fbdb61c97b47e" }
+{ "overlay": "2f16c4826b16c64e35f81862b2676a2ad596f0a7" }

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ErrorTests.java

@@ -40,10 +40,10 @@
  */
 package com.oracle.graal.python.pegparser;
 
-import org.junit.Test;
-
 import java.util.EnumSet;
 
+import org.junit.Test;
+
 /**
  * Testing invalid rules. If the test method has the same names, but different number, then it tests
  * different alts in the rule.
@@ -334,4 +334,9 @@ public void canNotBeUsedWithinAnnotation() {
         checkSyntaxErrorMessage("i: (yield from f) = 3\n", "'yield expression' can not be used within an annotation", EnumSet.of(FutureFeature.ANNOTATIONS));
         checkSyntaxErrorMessage("i: (x:=42) = 3\n", "'named expression' can not be used within an annotation", EnumSet.of(FutureFeature.ANNOTATIONS));
     }
+
+    @Test
+    public void testDeprecationInvalidEscape() {
+        checkDeprecationWarning("'\\z'", "invalid escape sequence '\\z'");
+    }
 }

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ParserTestBase.java

@@ -151,6 +151,15 @@ public void checkSyntaxErrorMessage(String source, String expectedMessage, Input
         assertEquals("The expected message:\n\"" + expectedMessage + "\"\n was not found. The message is: \n\"" + error.getMessage() + "\"", error.getMessage(), expectedMessage);
     }
 
+    public void checkDeprecationWarning(String source, String expectedMessage) {
+        ModTy node = parse(source, getFileName(), InputType.FILE);
+        if (node != null) {
+            ScopeEnvironment.analyze(node, errorCallback, EMPTY_FUTURE);
+        }
+        assertTrue("Expected a DeprecationWarning.", errorCallback.hasWarnings());
+        assertEquals(expectedMessage, errorCallback.getWarnings().get(0).getMessage());
+    }
+
     public void checkIndentationError(String source) {
         ModTy node = parse(source, getFileName(), InputType.FILE);
         if (node != null) {
@@ -225,6 +234,11 @@ public void onError(ErrorCallback.ErrorType type, SourceRange sourceRange, Strin
                 errors.add(String.format("%s[%d:%d]:%s", type.name(), sourceRange.startOffset, sourceRange.endOffset, message));
             }
 
+            @Override
+            public void warnDeprecation(SourceRange sourceRange, String message) {
+                fail("Unexpected deprecation warning");
+            }
+
             @Override
             public void reportIncompleteSource(int line) {
                 fail("Unexpected call to reportIncompleteSource");

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/TestErrorCallbackImpl.java

@@ -48,20 +48,34 @@
 public class TestErrorCallbackImpl implements ErrorCallback {
 
     private final List<Error> errors = new ArrayList<>(1);
+    private final List<Error> warnings = new ArrayList<>(0);
 
     public List<Error> getErrors() {
         return errors;
     }
 
+    public List<Error> getWarnings() {
+        return warnings;
+    }
+
     public boolean hasErrors() {
         return !errors.isEmpty();
     }
 
+    public boolean hasWarnings() {
+        return !warnings.isEmpty();
+    }
+
     @Override
     public void onError(ErrorType errorType, SourceRange sourceRange, String message) {
         errors.add(new Error(errorType, sourceRange, message));
     }
 
+    @Override
+    public void warnDeprecation(SourceRange sourceRange, String message) {
+        warnings.add(new Error(ErrorType.Syntax, sourceRange, message));
+    }
+
     @Override
     public void reportIncompleteSource(int line) {
         throw new IncompleteSourceException(line);

graalpython/com.oracle.graal.python.pegparser.test/testData/parser/goldenFiles/AssignmentTests/assignmentWithTypeComment01.tast

@@ -1,5 +1,4 @@
-Module[0, 17]
-    Assign[0, 17]
+Module[0, 5]
+    Assign[0, 5]
         LHS: Name[0, 1] Value: "a" Store
-        RHS: LONG[4, 5] Value: 1
-        TypeComment: int
+        RHS: LONG[4, 5] Value: 1

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/ErrorCallback.java

@@ -59,11 +59,17 @@ public enum ErrorType {
 
     void onError(ErrorType errorType, SourceRange sourceRange, String message);
 
+    void warnDeprecation(SourceRange sourceRange, String message);
+
     default void onError(ErrorType errorType, SourceRange sourceRange, String message, Object... arguments) {
         onError(errorType, sourceRange, String.format(message, arguments));
     }
 
     default void onError(SourceRange sourceRange, String message, Object... arguments) {
         onError(ErrorType.Generic, sourceRange, String.format(message, arguments));
     }
+
+    default void warnDeprecation(SourceRange sourceRange, String message, Object... arguments) {
+        warnDeprecation(sourceRange, String.format(message, arguments));
+    }
 }

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/ParserTokenizer.java

@@ -66,7 +66,7 @@ public ParserTokenizer(ErrorCallback errorCallback, byte[] code, InputType type,
     }
 
     private static EnumSet<Tokenizer.Flag> getTokenizerFlags(InputType type, boolean interactiveTerminal) {
-        EnumSet<Tokenizer.Flag> flags = EnumSet.of(Tokenizer.Flag.TYPE_COMMENT);
+        EnumSet<Tokenizer.Flag> flags = EnumSet.noneOf(Tokenizer.Flag.class);
         if (type == InputType.FILE) {
             flags.add(Tokenizer.Flag.EXEC_INPUT);
         } else if (type == InputType.SINGLE && interactiveTerminal) {

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/sst/StringLiteralUtils.java

@@ -127,6 +127,11 @@ public static <T> ExprTy createStringLiteral(String[] values, SourceRange[] sour
 
             text = text.substring(strStartIndex, strEndIndex);
             if (isBytes) {
+                for (int i = 0; i < text.length(); i++) {
+                    if (text.charAt(i) >= 0x80) {
+                        errorCallback.onError(ErrorCallback.ErrorType.Syntax, sourceRange, "bytes can only contain ASCII literal characters");
+                    }
+                }
                 if (sb != null || isFormatString) {
                     errorCallback.onError(sourceRange, CANNOT_MIX_MESSAGE);
                     if (sb != null) {
@@ -427,7 +432,7 @@ public static int createTokens(ArrayList<Token> tokens, ErrorCallback errorCallb
                                     // treated as \N escape sequence
                                     index++;
                                 } else if (lookahead(text, index, len, '{')) {
-                                    warnInvalidEscapeSequence(errorCallback, text.charAt(index + 1));
+                                    warnInvalidEscapeSequence(errorCallback, textSourceRange, text.charAt(index + 1));
                                 } else if (lookahead(text, index, len, 'N', '{')) {
                                     // skip escape sequence \N{...}, it should not be treated as an
                                     // expression inside f-string, but \\N{...} should be left
@@ -938,7 +943,7 @@ public static StringBuilder decodeEscapes(ErrorCallback errors, String string, S
                         charList.append(chr);
                         if (!wasDeprecationWarning) {
                             wasDeprecationWarning = true;
-                            warnInvalidEscapeSequence(errors, chr);
+                            warnInvalidEscapeSequence(errors, sourceRange, chr);
                         }
                     }
             }
@@ -1052,7 +1057,7 @@ private static <T> T unescapeString(SourceRange sourceRange, ErrorCallback error
                     default:
                         if (!wasDeprecationWarning) {
                             wasDeprecationWarning = true;
-                            warnInvalidEscapeSequence(errorCallback, nextChar);
+                            warnInvalidEscapeSequence(errorCallback, sourceRange, nextChar);
                         }
                         sb.appendCodePoint(ch);
                         sb.appendCodePoint(nextChar);
@@ -1193,10 +1198,8 @@ public static int getCodePoint(String charName) {
         return -1;
     }
 
-    // TODO
-    @SuppressWarnings("unused")
-    public static void warnInvalidEscapeSequence(ErrorCallback errorCallback, char nextChar) {
-        // errorCallback.warn("invalid escape sequence '\\%c'", nextChar);
+    public static void warnInvalidEscapeSequence(ErrorCallback errorCallback, SourceRange sourceRange, char nextChar) {
+        errorCallback.warnDeprecation(sourceRange, "invalid escape sequence '\\%c'", nextChar);
     }
 
     private static final String UNICODE_ERROR = "'unicodeescape' codec can't decode bytes in position %d-%d:";

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/CompilerTests.java

@@ -40,13 +40,14 @@
  */
 package com.oracle.graal.python.test.compiler;
 
+import static org.junit.Assert.fail;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.EnumSet;
 
-import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -60,10 +61,9 @@
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.Parser;
 import com.oracle.graal.python.pegparser.sst.ModTy;
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import com.oracle.graal.python.test.PythonTests;
 
-import static org.junit.Assert.fail;
-
 public class CompilerTests extends PythonTests {
     public CompilerTests() {
     }
@@ -80,6 +80,11 @@ public void testComplexNumber() {
         doTest("-2 + 3j");
     }
 
+    @Test
+    public void testMinusFolding() {
+        doTest("-1 * -7.0");
+    }
+
     @Test
     public void testAssignment() {
         doTest("a = 12");
@@ -528,10 +533,40 @@ public void testEmptyDict() {
     }
 
     @Test
-    public void testTupleLiteral() {
+    public void testTupleLiteralInts() {
         doTest("(1, 2, 3)");
     }
 
+    @Test
+    public void testTupleLiteralDoubles() {
+        doTest("(1.0, 2.0, 3.0)");
+    }
+
+    @Test
+    public void testTupleLiteralBooleans() {
+        doTest("(False, True)");
+    }
+
+    @Test
+    public void testTupleLiteralObjects() {
+        doTest("('a', 1, None)");
+    }
+
+    @Test
+    public void testTupleLiteralMixed() {
+        doTest("(1, 2, 3.0)");
+    }
+
+    @Test
+    public void testTupleLiteralNonConstant() {
+        doTest("(1, 2, [3])");
+    }
+
+    @Test
+    public void testTupleLiteralMixedIntegers() {
+        doTest("(1, 17179869184, 3)");
+    }
+
     @Test
     public void testTupleLiteralExpand() {
         doTest("(1, 2, 3, *a, 5)");
@@ -826,7 +861,7 @@ private static CodeUnit assemble(String src, InputType type) {
         ModTy result = (ModTy) parser.parse();
         Compiler compiler = new Compiler(errorCallback);
         CompilationUnit cu = compiler.compile(result, EnumSet.noneOf(Compiler.Flags.class), 2);
-        return cu.assemble(0);
+        return cu.assemble();
     }
 
     private void checkCodeUnit(CodeUnit co) {
@@ -857,6 +892,11 @@ public void reportIncompleteSource(int line) {
         public void onError(ErrorType errorType, SourceRange sourceRange, String message) {
             throw new SyntaxError(errorType, message);
         }
+
+        @Override
+        public void warnDeprecation(SourceRange sourceRange, String message) {
+            throw new AssertionError("Unexpected warning: " + message);
+        }
     }
 
     private static final class SyntaxError extends RuntimeException {

graalpython/com.oracle.graal.python.test/src/tests/test_code.py

@@ -178,8 +178,9 @@ def test_module_code():
         assert set(code.co_varnames) == set()
         assert code.co_filename.endswith("__init__.py")
         assert code.co_name.startswith("<module")
-        if sys.implementation.name == 'graalpy':
-            assert code.co_firstlineno == 1
+        # AST interpreter doesn't pass this
+        if not sys.implementation.name == 'graalpy' or __graalpython__.uses_bytecode_interpreter:
+            assert code.co_firstlineno == 40
         # assert code.co_lnotab  == b''
         assert code.co_freevars == tuple()
         assert code.co_cellvars == tuple()