Gzip the result of CompilerOptions::setTypedAstOutputFile

PiperOrigin-RevId: 515467692

Author: lauraharker

src/com/google/javascript/jscomp/AbstractCommandLineRunner.java

@@ -83,6 +83,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import org.jspecify.nullness.Nullable;
@@ -142,6 +143,8 @@ public abstract class AbstractCommandLineRunner<A extends Compiler, B extends Co
               + "%outname% in the value.");
 
   static final String WAITING_FOR_INPUT_WARNING = "The compiler is waiting for input via stdin.";
+  // Use an 8MiB buffer since the concatenated TypedAst file can be very large.
+  private static final int GZIPPED_TYPEDAST_BUFFER_SIZE = 8 * 1024 * 1024;
 
   @GwtIncompatible("Unnecessary")
   private final CommandLineConfig config;
@@ -1417,8 +1420,8 @@ private void initWithTypedAstFilesystem(
       List<SourceFile> sources,
       CompilerOptions options,
       String filename) {
-    try (BufferedInputStream typedAstListStream =
-        new BufferedInputStream(new FileInputStream(filename))) {
+    try (GZIPInputStream typedAstListStream =
+        new GZIPInputStream(new FileInputStream(filename), GZIPPED_TYPEDAST_BUFFER_SIZE)) {
       compiler.initWithTypedAstFilesystem(externs, sources, options, typedAstListStream);
     } catch (IOException e) {
       compiler.report(JSError.make(COULD_NOT_DESERIALIZE_AST, filename));
@@ -1428,8 +1431,8 @@ private void initWithTypedAstFilesystem(
   @GwtIncompatible("Unnecessary")
   private void initModulesWithTypedAstFilesystem(
       List<SourceFile> externs, List<JSChunk> chunks, CompilerOptions options, String filename) {
-    try (BufferedInputStream typedAstListStream =
-        new BufferedInputStream(new FileInputStream(filename))) {
+    try (GZIPInputStream typedAstListStream =
+        new GZIPInputStream(new FileInputStream(filename), GZIPPED_TYPEDAST_BUFFER_SIZE)) {
       compiler.initModulesWithTypedAstFilesystem(externs, chunks, options, typedAstListStream);
     } catch (IOException e) {
       compiler.report(JSError.make(COULD_NOT_DESERIALIZE_AST, filename));

src/com/google/javascript/jscomp/CompilerOptions.java

@@ -210,7 +210,11 @@ public boolean getInstrumentForCoverageOnly() {
 
   private @Nullable Path typedAstOutputFile = null;
 
-  /** Sets file to output in-progress TypedAST format to. DO NOT USE! */
+  /**
+   * Sets file to output in-progress TypedAST format to. DO NOT USE!
+   *
+   * <p>The "TypedAST format" is currently a gzipped TypedAst proto but this is not stable.
+   */
   public void setTypedAstOutputFile(@Nullable Path file) {
     this.typedAstOutputFile = file;
   }

src/com/google/javascript/jscomp/serialization/SerializeTypedAstPass.java

@@ -25,6 +25,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.function.Consumer;
+import java.util.zip.GZIPOutputStream;
 
 /**
  * A compiler pass intended to serialize the types in the AST.
@@ -41,6 +42,12 @@ public final class SerializeTypedAstPass implements CompilerPass {
     this.consumer = astConsumer;
   }
 
+  /**
+   * Serializes a TypedAst to the given output stream.
+   *
+   * <p>Unlike {@link #createFromPath(AbstractCompiler, Path)}, this method does not automatically
+   * gzip the TypedAST. The "out" parameter may or may not already be a GZIPOutputStream.
+   */
   public static SerializeTypedAstPass createFromOutputStream(AbstractCompiler c, OutputStream out) {
     Consumer<TypedAst> toOutputStream =
         ast -> {
@@ -53,10 +60,11 @@ public static SerializeTypedAstPass createFromOutputStream(AbstractCompiler c, O
     return new SerializeTypedAstPass(c, toOutputStream);
   }
 
+  /** Serializes a gzipped TypedAst to the specified outputPath */
   public static SerializeTypedAstPass createFromPath(AbstractCompiler compiler, Path outputPath) {
     Consumer<TypedAst> toPath =
         ast -> {
-          try (OutputStream out = Files.newOutputStream(outputPath)) {
+          try (OutputStream out = new GZIPOutputStream(Files.newOutputStream(outputPath))) {
             TypedAst.List.newBuilder().addTypedAsts(ast).build().writeTo(out);
           } catch (IOException e) {
             throw new IllegalArgumentException("Cannot create TypedAst output file", e);

src/com/google/javascript/jscomp/serialization/TypedAstDeserializer.java

@@ -401,7 +401,11 @@ private static void deserializeTypedAsts(
         int tag = codedInput.readTag();
         if (WireFormat.getTagFieldNumber(tag) != TypedAst.List.TYPED_ASTS_FIELD_NUMBER
             || WireFormat.getTagWireType(tag) != WireFormat.WIRETYPE_LENGTH_DELIMITED) {
-          throw new InvalidProtocolBufferException("Unexpected tag " + tag);
+          throw new InvalidProtocolBufferException(
+              "Unexpected field number "
+                  + WireFormat.getTagFieldNumber(tag)
+                  + " or wire type "
+                  + WireFormat.getTagWireType(tag));
         }
         codedInput.readMessage(typedAstBuilder, ExtensionRegistry.getEmptyRegistry());
         TypedAst typedAst = typedAstBuilder.build();

test/com/google/javascript/jscomp/integration/TypedAstIntegrationTest.java

@@ -43,13 +43,14 @@
 import com.google.javascript.rhino.Node;
 import com.google.javascript.rhino.StaticSourceFile.SourceKind;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.SequenceInputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.zip.GZIPInputStream;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -739,22 +740,16 @@ private void precompileLibrary(SourceFile... files) throws IOException {
   }
 
   /** Converts the list of paths into an input stream sequentially reading all the given files */
-  private static InputStream toInputStream(ArrayList<Path> typedAsts) {
-    InputStream inputStream = null;
+  private static InputStream toInputStream(ArrayList<Path> typedAsts) throws IOException {
+    ArrayList<InputStream> inputShards = new ArrayList<>();
     for (Path typedAst : typedAsts) {
-      FileInputStream inputShard;
-      try {
-        inputShard = new FileInputStream(typedAst.toFile());
-      } catch (FileNotFoundException ex) {
-        throw new AssertionError(ex);
-      }
-      if (inputStream == null) {
-        inputStream = inputShard;
-      } else {
-        inputStream = new SequenceInputStream(inputStream, inputShard);
-      }
+      // Each shard is a gzipped TypedAst proto.
+      // NOTE: while it's generally possible to concatenate individually gzipped files
+      // (https://stackoverflow.com/questions/8005114/fast-concatenation-of-multiple-gzip-files)
+      // it doesn't seem to work to wrap a SequenceInputStream in a GZIPInputStream directly.
+      inputShards.add(new GZIPInputStream(new FileInputStream(typedAst.toFile())));
     }
-    return inputStream;
+    return new SequenceInputStream(Collections.enumeration(inputShards));
   }
 
   private Node parseExpectedCode(String... files) {