Remove and Restore Hashbangs (#320)

* Restore hashbangs
* Missing import
* Tests for using banner option with Rollup

Author: kristoferbaxter

src/index.ts

@@ -32,10 +32,12 @@ import {
 } from './transformers/source/transforms';
 import { preCompilation, create as createChunkTransforms } from './transformers/chunk/transforms';
 import { Mangle } from './transformers/mangle';
+import { Ebbinghaus } from './transformers/ebbinghaus';
 import { SourceTransform } from './transform';
 
 export default function closureCompiler(requestedCompileOptions: CompileOptions = {}): Plugin {
   const mangler: Mangle = new Mangle();
+  const memory: Ebbinghaus = new Ebbinghaus();
   let inputOptions: InputOptions;
   let context: PluginContext;
   let sourceTransforms: Array<SourceTransform>;
@@ -49,6 +51,7 @@ export default function closureCompiler(requestedCompileOptions: CompileOptions
         context,
         requestedCompileOptions,
         mangler,
+        memory,
         inputOptions,
         {},
       );
@@ -76,6 +79,7 @@ export default function closureCompiler(requestedCompileOptions: CompileOptions
         context,
         requestedCompileOptions,
         mangler,
+        memory,
         inputOptions,
         outputOptions,
       );

src/transform.ts

@@ -18,6 +18,7 @@ import { logTransformChain } from './debug';
 import { TransformInterface, PluginOptions } from './types';
 import { PluginContext, InputOptions, OutputOptions, TransformSourceDescription } from 'rollup';
 import { Mangle } from './transformers/mangle';
+import { Ebbinghaus } from './transformers/ebbinghaus';
 import * as path from 'path';
 import MagicString from 'magic-string';
 import { DecodedSourceMap as RemappingDecodedSourceMap } from '@ampproject/remapping/dist/types/types';
@@ -27,6 +28,7 @@ class Transform implements TransformInterface {
   protected context: PluginContext;
   protected pluginOptions: PluginOptions;
   protected mangler: Mangle;
+  protected memory: Ebbinghaus;
   protected inputOptions: InputOptions;
   protected outputOptions: OutputOptions;
   public name: string = 'Transform';
@@ -35,12 +37,14 @@ class Transform implements TransformInterface {
     context: PluginContext,
     pluginOptions: PluginOptions,
     mangler: Mangle,
+    memory: Ebbinghaus,
     inputOptions: InputOptions,
     outputOptions: OutputOptions,
   ) {
     this.context = context;
     this.pluginOptions = pluginOptions;
     this.mangler = mangler;
+    this.memory = memory;
     this.inputOptions = inputOptions;
     this.outputOptions = outputOptions;
   }

src/transformers/chunk/hashbang-apply.ts

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ChunkTransform } from '../../transform';
+import { TransformInterface } from '../../types';
+import MagicString from 'magic-string';
+
+/**
+ * Closure Compiler will not compile code that is prefixed with a hashbang (common to rollup output for CLIs).
+ *
+ * This transform will restore the hashbang if Ebbinghaus knows it exists.
+ */
+export default class HashbangApplyTransform extends ChunkTransform implements TransformInterface {
+  public name = 'HashbangApplyTransform';
+
+  /**
+   * @param source MagicString of source to process post Closure Compilation.
+   */
+  public async post(source: MagicString): Promise<MagicString> {
+    if (this.memory.hashbang === null) {
+      return source;
+    }
+
+    source.prepend(this.memory.hashbang + '\n');
+    return source;
+  }
+}

src/transformers/chunk/hashbang-remove.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ChunkTransform } from '../../transform';
+import { TransformInterface } from '../../types';
+import MagicString from 'magic-string';
+
+/**
+ * Closure Compiler will not compile code that is prefixed with a hashbang (common to rollup output for CLIs).
+ *
+ * This transform will remove the hashbang (if present) and ask Ebbinghaus to remember if for after compilation.
+ */
+export default class HashbangRemoveTransform extends ChunkTransform implements TransformInterface {
+  public name = 'HashbangRemoveTransform';
+
+  /**
+   * @param source MagicString of source to process post Closure Compilation.
+   */
+  public async pre(source: MagicString): Promise<MagicString> {
+    const stringified = source.trim().toString();
+    const match = /^#!.*/.exec(stringified);
+
+    if (!match) {
+      return source;
+    }
+
+    this.memory.hashbang = match[0];
+    source.remove(0, match[0].length);
+    return source;
+  }
+}

src/transformers/chunk/transforms.ts

@@ -21,6 +21,8 @@ import {
   RenderedChunk,
   TransformSourceDescription,
 } from 'rollup';
+import HashbangRemoveTransform from './hashbang-remove';
+import HashbangApplyTransform from './hashbang-apply';
 import IifeTransform from './iife';
 import CJSTransform from './cjs';
 import LiteralComputedKeys from './literal-computed-keys';
@@ -30,24 +32,30 @@ import StrictTransform from './strict';
 import ConstTransform from './const';
 import { ChunkTransform, chunkLifecycle } from '../../transform';
 import { Mangle } from '../mangle';
+import { Ebbinghaus } from '../ebbinghaus';
 import { CompileOptions } from 'google-closure-compiler';
 import { pluckPluginOptions } from '../../options';
 
 const TRANSFORMS: Array<typeof ChunkTransform> = [
+  HashbangRemoveTransform,
+  // Acorn can parse content starting here
   ConstTransform,
   IifeTransform,
   CJSTransform,
   LiteralComputedKeys,
   StrictTransform,
   ExportTransform,
   ImportTransform,
+  // Acorn cannot parse content starting here.
+  HashbangApplyTransform,
 ];
 
 /**
  * Instantiate transform class instances for the plugin invocation.
  * @param context Plugin context to bind for each transform instance.
  * @param requestedCompileOptions Originally requested compile options from configuration.
  * @param mangler Mangle instance used for this transform instance.
+ * @param memory Ebbinghaus instance used to store information that could be lost from source.
  * @param inputOptions Rollup input options
  * @param outputOptions Rollup output options
  * @return Instantiated transform class instances for the given entry point.
@@ -56,12 +64,14 @@ export function create(
   context: PluginContext,
   requestedCompileOptions: CompileOptions,
   mangler: Mangle,
+  memory: Ebbinghaus,
   inputOptions: InputOptions,
   outputOptions: OutputOptions,
 ): Array<ChunkTransform> {
   const pluginOptions = pluckPluginOptions(requestedCompileOptions);
   return TRANSFORMS.map(
-    transform => new transform(context, pluginOptions, mangler, inputOptions, outputOptions),
+    transform =>
+      new transform(context, pluginOptions, mangler, memory, inputOptions, outputOptions),
   );
 }
 

src/transformers/ebbinghaus.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Hermann Ebbinghaus is credited with discovering the forgetting curve.
+ *
+ * This class stores data the compiler would typically loose to the forgetting curve.
+ * For instance:
+ *  - original source contained a `hashbang`
+ *  - original source used external imports
+ *
+ * This data can be used later to inform transforms following Closure Compiler.
+ *
+ * For more information, visit: https://en.wikipedia.org/wiki/Hermann_Ebbinghaus
+ */
+
+export class Ebbinghaus {
+  public hashbang: string | null = null;
+}

src/transformers/source/hashbang.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SourceTransform } from '../../transform';
+import { TransformInterface } from '../../types';
+import MagicString from 'magic-string';
+
+/**
+ * Closure Compiler will not compile code that is prefixed with a hashbang (common to rollup output for CLIs).
+ *
+ * This transform will remove the hashbang (if present) and ask Ebbinghaus to remember if for after compilation.
+ */
+export default class HashbangTransform extends SourceTransform implements TransformInterface {
+  public name = 'HashbangTransform';
+
+  public transform = async (id: string, source: MagicString): Promise<MagicString> => {
+    const stringified = source.trim().toString();
+    const match = /^#!.*/.exec(stringified);
+
+    if (!match) {
+      return source;
+    }
+
+    this.memory.hashbang = match[0];
+    source.remove(0, match[0].length);
+    return source;
+  };
+}

src/transformers/source/transforms.ts

@@ -15,21 +15,22 @@
  */
 
 import { SourceTransform, sourceLifecycle } from '../../transform';
-// import { ImportTransform } from './imports';
-// import { ExportTransform } from './exports';
 import { Mangle } from '../mangle';
 import { PluginContext, InputOptions, OutputOptions, TransformSourceDescription } from 'rollup';
 import { CompileOptions } from 'google-closure-compiler';
+import HashbangTransform from './hashbang';
+import { Ebbinghaus } from '../ebbinghaus';
 
-const TRANSFORMS: Array<typeof SourceTransform> = [];
-// Temporarily disabling SourceTransforms, aligning for future release.
+const TRANSFORMS: Array<typeof SourceTransform> = [HashbangTransform];
+// Temporarily disabling many SourceTransforms, aligning for future release.
 // ImportTransform, ExportTransform
 
 /**
  * Instantiate transform class instances for the plugin invocation.
  * @param context Plugin context to bind for each transform instance.
  * @param requestedCompileOptions Originally requested compile options from configuration.
  * @param mangler Mangle instance used for this transform instance.
+ * @param memory Ebbinghaus instance used to store information that could be lost from source.
  * @param inputOptions Rollup input options
  * @param outputOptions Rollup output options
  * @return Instantiated transform class instances for the given entry point.
@@ -38,10 +39,13 @@ export const create = (
   context: PluginContext,
   requestedCompileOptions: CompileOptions,
   mangler: Mangle,
+  memory: Ebbinghaus,
   inputOptions: InputOptions,
   outputOptions: OutputOptions,
 ): Array<SourceTransform> =>
-  TRANSFORMS.map(transform => new transform(context, {}, mangler, inputOptions, outputOptions));
+  TRANSFORMS.map(
+    transform => new transform(context, {}, mangler, memory, inputOptions, outputOptions),
+  );
 
 /**
  * Run each transform's `transform` lifecycle.

test/generator.js

@@ -56,7 +56,7 @@ const fixtureLocation = (category, name, format, optionsKey, minified = false) =
       : `${name}.js`
   }`;
 
-async function compile(category, name, codeSplit, closureFlags, optionKey, format, wrapper) {
+async function compile(category, name, codeSplit, closureFlags, optionKey, format, wrapper, banner) {
   const bundle = await rollup.rollup({
     input: fixtureLocation(category, name, format, optionKey, false),
     plugins: [compiler(closureFlags[optionKey])],
@@ -69,6 +69,7 @@ async function compile(category, name, codeSplit, closureFlags, optionKey, forma
     format,
     name: wrapper,
     sourcemap: true,
+    banner,
   });
 
   const output = [];
@@ -107,7 +108,7 @@ async function compile(category, name, codeSplit, closureFlags, optionKey, forma
   return output;
 }
 
-function generate(shouldFail, category, name, codeSplit, formats, closureFlags, wrapper) {
+function generate(shouldFail, category, name, codeSplit, formats, closureFlags, wrapper, banner) {
   const targetLength = longest(formats);
   const optionLength = longest(Object.keys(closureFlags));
 
@@ -125,6 +126,7 @@ function generate(shouldFail, category, name, codeSplit, formats, closureFlags,
             optionKey,
             format,
             wrapper,
+            banner,
           );
 
           t.plan(output.length);
@@ -144,8 +146,9 @@ function failureGenerator(
   formats = [ESM_OUTPUT],
   closureFlags = defaultClosureFlags,
   wrapper = null,
+  banner = null,
 ) {
-  generate(true, category, name, codeSplit, formats, closureFlags, wrapper);
+  generate(true, category, name, codeSplit, formats, closureFlags, wrapper, banner);
 }
 
 function generator(
@@ -155,8 +158,9 @@ function generator(
   formats = [ESM_OUTPUT],
   closureFlags = defaultClosureFlags,
   wrapper = null,
+  banner = null,
 ) {
-  generate(false, category, name, codeSplit, formats, closureFlags, wrapper);
+  generate(false, category, name, codeSplit, formats, closureFlags, wrapper, banner);
 }
 
 module.exports = {

test/hashbang/fixtures/hashbang-banner.esm.advanced.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export function foo(){return"hello world"};