Always have a TruffleString in RubySource

* This simplifies callers significantly and avoids duplicating that logic.
* It is also more efficient as it creates the TruffleString immediately in the right encoding.

Author: eregon

spec/truffle/interop/polyglot/polyglot_spec.rb

@@ -42,10 +42,8 @@
       Polyglot.eval("ruby", "# encoding: us-ascii\n__ENCODING__.name").should == "US-ASCII"
     end
 
-    it "will not allow code in Ruby to have a magic comment to change the encoding to something not a subset of UTF-8" do
-      -> {
-        Polyglot.eval("ruby", "# encoding: big5\n__ENCODING__.name")
-      }.should raise_error(ArgumentError, /big5 cannot be used as an encoding for a Polyglot API Source/)
+    it "will allow code in Ruby to have a magic comment to change the encoding to something not a subset of UTF-8" do
+      Polyglot.eval("ruby", "# encoding: big5\n__ENCODING__.name").should == "Big5"
     end
   end
 
@@ -87,10 +85,8 @@
       Polyglot.eval_file("ruby", fixture(__FILE__, "usascii_magic.rb")).should == "US-ASCII"
     end
 
-    it "will not allow code in Ruby to have a magic comment to change the encoding" do
-      -> {
-        Polyglot.eval_file("ruby", fixture(__FILE__, "big5_magic.rb"))
-      }.should raise_error(ArgumentError, /big5 cannot be used as an encoding for a Polyglot API Source/)
+    it "will allow code in Ruby to have a magic comment to change the encoding" do
+      Polyglot.eval_file("ruby", fixture(__FILE__, "big5_magic.rb")).should == "Big5"
     end
   end
 

src/main/java/org/truffleruby/RubyFileTypeDetector.java

@@ -65,34 +65,48 @@ public String findMimeType(TruffleFile file) throws IOException {
     }
 
     @Override
-    public Charset findEncoding(TruffleFile file) throws IOException {
+    public Charset findEncoding(TruffleFile file) {
         try (BufferedReader fileContent = file.newBufferedReader(StandardCharsets.UTF_8)) {
-            final String firstLine = fileContent.readLine();
+            return findEncoding(fileContent).getCharset();
+        } catch (IOException | SecurityException e) {
+            // Reading random files as UTF-8 could cause all sorts of errors
+            return Encodings.UTF_8.jcoding.getCharset();
+        }
+    }
+
+    public static Encoding findEncoding(BufferedReader reader) {
+        try {
+            final String firstLine = reader.readLine();
             if (firstLine != null) {
-                String encodingCommentLine;
+                final String encodingCommentLine;
                 if (SHEBANG_REGEXP.matcher(firstLine).matches()) {
-                    encodingCommentLine = fileContent.readLine();
+                    encodingCommentLine = reader.readLine();
                 } else {
                     encodingCommentLine = firstLine;
                 }
+
                 if (encodingCommentLine != null) {
                     var encodingComment = new TStringWithEncoding(TStringUtils.utf8TString(encodingCommentLine),
                             Encodings.UTF_8);
-                    Charset[] encodingHolder = new Charset[1];
+                    Encoding[] encodingHolder = new Encoding[1];
                     RubyLexer.parseMagicComment(encodingComment, (name, value) -> {
                         if (RubyLexer.isMagicEncodingComment(name)) {
                             Encoding encoding = EncodingManager.getEncoding(value);
                             if (encoding != null) {
-                                encodingHolder[0] = encoding.getCharset();
+                                encodingHolder[0] = encoding;
                             }
                         }
                     });
-                    return encodingHolder[0];
+                    if (encodingHolder[0] != null) {
+                        return encodingHolder[0];
+                    }
                 }
             }
-        } catch (IOException | SecurityException e) {
-            // Reading random files as UTF-8 could cause all sorts of errors
+        } catch (IOException e) {
+            // Use the default encoding if reading failed somehow
         }
-        return null;
+
+        // The default encoding
+        return Encodings.UTF_8.jcoding;
     }
 }

src/main/java/org/truffleruby/parser/RubySource.java

@@ -9,15 +9,19 @@
  */
 package org.truffleruby.parser;
 
+import java.io.BufferedReader;
+import java.io.StringReader;
 import java.util.Objects;
 
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.source.SourceSection;
 import com.oracle.truffle.api.strings.TruffleString;
 import org.truffleruby.RubyContext;
+import org.truffleruby.RubyFileTypeDetector;
 import org.truffleruby.RubyLanguage;
 
 import com.oracle.truffle.api.source.Source;
+import org.truffleruby.core.encoding.Encodings;
 import org.truffleruby.core.encoding.RubyEncoding;
 import org.truffleruby.core.encoding.TStringUtils;
 import org.truffleruby.core.string.TStringWithEncoding;
@@ -43,7 +47,7 @@ public RubySource(Source source, String sourcePath, TStringWithEncoding code) {
         this(source, sourcePath, code, false);
     }
 
-    public RubySource(Source source, String sourcePath, TStringWithEncoding code, boolean isEval) {
+    private RubySource(Source source, String sourcePath, TStringWithEncoding code, boolean isEval) {
         this(source, sourcePath, code, isEval, 0);
     }
 
@@ -52,8 +56,19 @@ public RubySource(Source source, String sourcePath, TStringWithEncoding code, bo
         this.source = Objects.requireNonNull(source);
         //intern() to improve footprint
         this.sourcePath = Objects.requireNonNull(sourcePath).intern();
-        this.code = code != null ? code.tstring : null;
-        this.encoding = code != null ? code.encoding : null;
+
+        if (code == null) {
+            // We only have the Source, which only contains a java.lang.String.
+            // The sourcePath might not exist, so we cannot reread from the filesystem.
+            // So we look for the magic encoding comment and if not found use UTF-8.
+            var sourceString = source.getCharacters().toString();
+            var jcoding = RubyFileTypeDetector.findEncoding(new BufferedReader(new StringReader(sourceString)));
+            var encoding = Encodings.getBuiltInEncoding(jcoding);
+            code = new TStringWithEncoding(TStringUtils.fromJavaString(sourceString, encoding), encoding);
+        }
+
+        this.code = code.tstring;
+        this.encoding = code.encoding;
         this.isEval = isEval;
         this.lineOffset = lineOffset;
     }