store sources in serialized format (again)

Author: lukasstadler

graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/Deserializing.java

@@ -44,6 +44,8 @@
 import com.oracle.graal.python.parser.PythonSSTNodeFactory;
 import com.oracle.graal.python.runtime.PythonParser;
 import com.oracle.truffle.api.source.Source;
+import com.oracle.truffle.api.source.SourceSection;
+
 import java.util.ArrayList;
 import java.util.List;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -86,7 +88,8 @@ private List<Item> createSerializations() {
             try {
                 PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(core, source, parser);
                 PythonParserImpl.CacheItem item = parser.parseWithANTLR(PythonParser.ParserMode.File, 0, core, sstFactory, source, null, null);
-                result.add(new Item(PythonParserImpl.serialize(source, item.getAntlrResult(), item.getGlobalScope(), true)));
+                SourceSection section = source.createSection(0, source.getLength());
+                result.add(new Item(PythonParserImpl.serialize(section, item.getAntlrResult(), item.getGlobalScope(), true)));
             } catch (RuntimeException e) {
                 // do nothing
             }

graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/Serializing.java

@@ -41,6 +41,8 @@
 package com.oracle.graal.python.benchmarks.parser;
 
 import com.oracle.graal.python.parser.PythonParserImpl;
+import com.oracle.truffle.api.source.SourceSection;
+
 import java.util.List;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Setup;
@@ -62,7 +64,8 @@ public void setup() {
     public void execute(Blackhole bh) {
         for (int n = 0; n < parsingCycles; n++) {
             for (PythonParserImpl.CacheItem item : ssts) {
-                bh.consume(PythonParserImpl.serialize(item.getSource(), item.getAntlrResult(), item.getGlobalScope(), true));
+                SourceSection section = item.getSource().createSection(0, item.getSource().getLength());
+                bh.consume(PythonParserImpl.serialize(section, item.getAntlrResult(), item.getGlobalScope(), true));
             }
         }
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonParserImpl.java

@@ -115,13 +115,17 @@ public SSTNode parseExpression(String text, PythonSSTNodeFactory nodeFactory) {
                         null).antlrResult;
     }
 
-    public static byte[] serialize(Source source, SSTNode node, ScopeInfo scope, boolean isModule) {
+    public static byte[] serialize(SourceSection section, SSTNode node, ScopeInfo scope, boolean isModule) {
+        Source source = section.getSource();
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(baos);
         try {
             dos.writeByte(SerializationUtils.VERSION);
             dos.writeUTF(source.getName() == null ? "" : encodeHome(source.getName()));
             dos.writeUTF(source.getPath() == null ? "" : encodeHome(source.getPath()));
+            byte[] bytes = section.getCharacters().toString().getBytes(StandardCharsets.UTF_8);
+            dos.writeInt(bytes.length);
+            dos.write(bytes);
             ScopeInfo.write(dos, scope);
             dos.writeInt(isModule ? 0 : node.getStartOffset());
             node.accept(new SSTSerializerVisitor(dos));
@@ -155,20 +159,20 @@ public byte[] serialize(RootNode rootNode) {
         Source source = rootNode.getSourceSection().getSource();
         assert source != null;
         CacheItem lastParserResult = cachedLastAntlrResult;
-        if (!source.equals(lastParserResult.source)) {
+        if (source != lastParserResult.source && !source.equals(lastParserResult.source)) {
             // we need to parse the source again
             PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(PythonLanguage.getCore(), source, this);
             lastParserResult = parseWithANTLR(ParserMode.File, 0, PythonLanguage.getCore(), sstFactory, source, null, null);
         }
         if (rootNode instanceof ModuleRootNode) {
             // serialize whole module
-            return serialize(source, lastParserResult.antlrResult, lastParserResult.globalScope, true);
+            return serialize(rootNode.getSourceSection(), lastParserResult.antlrResult, lastParserResult.globalScope, true);
         } else {
             // serialize just the part
             SSTNodeWithScopeFinder finder = new SSTNodeWithScopeFinder(rootNode.getSourceSection().getCharIndex(), rootNode.getSourceSection().getCharEndIndex());
             SSTNodeWithScope rootSST = lastParserResult.antlrResult.accept(finder);
             // store with parent scope
-            return serialize(source, rootSST, rootSST.getScope().getParent(), false);
+            return serialize(rootNode.getSourceSection(), rootSST, rootSST.getScope().getParent(), false);
         }
     }
 
@@ -195,18 +199,17 @@ public RootNode deserialize(byte[] data, String[] cellvars, String[] freevars) {
             }
             String name = decodeHome(dis.readUTF()).intern();
             String path = decodeHome(dis.readUTF()).intern();
-            if (path.isEmpty()) {
-                source = Source.newBuilder(PythonLanguage.ID, "", name).build();
-            } else {
-                try {
-                    source = Source.newBuilder(PythonLanguage.ID, PythonLanguage.getContext().getEnv().getPublicTruffleFile(path)).name(name).build();
-                } catch (IOException e) {
-                    // error accessing source file, ignore and continue with an empty source
-                    source = Source.newBuilder(PythonLanguage.ID, "", name).build();
-                }
-            }
+            byte[] bytes = new byte[dis.readInt()];
+            dis.readFully(bytes);
+            String contents = new String(bytes, StandardCharsets.UTF_8);
             globalScope = ScopeInfo.read(dis, null);
             int offset = dis.readInt();
+
+            if (path.isEmpty() || offset != 0) {
+                source = Source.newBuilder(PythonLanguage.ID, contents, name).build();
+            } else {
+                source = Source.newBuilder(PythonLanguage.ID, PythonLanguage.getContext().getEnv().getPublicTruffleFile(path)).content(contents).name(name).build();
+            }
             sstNode = new SSTDeserializer(dis, globalScope, offset).readNode();
         } catch (IOException e) {
             throw PythonLanguage.getCore().raise(PythonBuiltinClassType.ValueError, "Is not possible get correct bytecode data %s, %s", e.getClass().getSimpleName(), e.getMessage());