Interpret encoding comments when passing bytes to compile/eval/exec

Author: msimacek

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonFileDetector.java

@@ -41,14 +41,17 @@
 package com.oracle.graal.python;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.StringReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.oracle.graal.python.util.CharsetMapping;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.TruffleFile;
 
 public final class PythonFileDetector implements TruffleFile.FileTypeDetector {
@@ -79,7 +82,7 @@ public String getEncodingName() {
         }
     }
 
-    public static Charset tryGetCharsetFromLine(String line) {
+    private static Charset tryGetCharsetFromLine(String line) {
         if (line == null) {
             return null;
         }
@@ -94,6 +97,7 @@ public static Charset tryGetCharsetFromLine(String line) {
         return null;
     }
 
+    @TruffleBoundary
     public static Charset findEncodingStrict(BufferedReader reader) throws IOException {
         Charset charset;
         if ((charset = tryGetCharsetFromLine(reader.readLine())) != null) {
@@ -105,13 +109,15 @@ public static Charset findEncodingStrict(BufferedReader reader) throws IOExcepti
         return StandardCharsets.UTF_8;
     }
 
+    @TruffleBoundary
     public static Charset findEncodingStrict(TruffleFile file) throws IOException {
         // Using Latin-1 to read the header avoids exceptions on non-ascii characters
         try (BufferedReader reader = file.newBufferedReader(StandardCharsets.ISO_8859_1)) {
             return findEncodingStrict(reader);
         }
     }
 
+    @TruffleBoundary
     public static Charset findEncodingStrict(String source) {
         try (BufferedReader reader = new BufferedReader(new StringReader(source))) {
             return findEncodingStrict(reader);
@@ -121,6 +127,17 @@ public static Charset findEncodingStrict(String source) {
         }
     }
 
+    @TruffleBoundary
+    public static Charset findEncodingStrict(byte[] source) {
+        // Using Latin-1 to read the header avoids exceptions on non-ascii characters
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(source), StandardCharsets.ISO_8859_1))) {
+            return findEncodingStrict(reader);
+        } catch (IOException e) {
+            // Shouldn't happen on a string
+            throw new RuntimeException(e);
+        }
+    }
+
     @Override
     public Charset findEncoding(TruffleFile file) throws IOException {
         try {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

@@ -71,8 +71,10 @@
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
 
 import java.math.BigInteger;
+import java.nio.charset.Charset;
 import java.util.List;
 
+import com.oracle.graal.python.PythonFileDetector;
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
@@ -719,9 +721,23 @@ public CompileNode() {
         public abstract PCode execute(VirtualFrame frame, Object source, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize);
 
         @Specialization
-        PCode compile(VirtualFrame frame, PBytes source, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
+        PCode compile(VirtualFrame frame, PBytes pBytes, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
                         @Cached("create()") BytesNodes.ToBytesNode toBytesNode) {
-            return compile(createString(toBytesNode.execute(frame, source)), filename, mode, kwFlags, kwDontInherit, kwOptimize);
+            try {
+                byte[] bytes = toBytesNode.execute(frame, pBytes);
+                Charset charset = PythonFileDetector.findEncodingStrict(bytes);
+                return compile(createString(bytes, charset), filename, mode, kwFlags, kwDontInherit, kwOptimize);
+            } catch (PythonFileDetector.InvalidEncodingException e) {
+                throw handleInvalidEncoding(filename, e);
+            }
+        }
+
+        @TruffleBoundary
+        private RuntimeException handleInvalidEncoding(String filename, PythonFileDetector.InvalidEncodingException e) {
+            PythonContext context = getContext();
+            // Create non-empty source to avoid overwriting the message with "unexpected EOF"
+            Source source = PythonLanguage.newSource(context, " ", filename, mayBeFromFile);
+            throw getCore().raiseInvalidSyntax(source, source.createUnavailableSection(), "encoding problem: %s", e.getEncodingName());
         }
 
         @SuppressWarnings("unused")
@@ -766,8 +782,8 @@ PCode compile(PCode code, String filename, String mode, Object flags, Object don
         }
 
         @TruffleBoundary
-        private static String createString(byte[] bytes) {
-            return new String(bytes);
+        private static String createString(byte[] bytes, Charset charset) {
+            return new String(bytes, charset);
 
         }