JS: Extractor: More robust ES2015 checking

Created shared AbstractDetector to not duplicate all the tedious logic
;)

I took inspiration from the tests in  `javascript/extractor/tests/esnext/input/dynamic-import.js`

Author: RasmusWL

javascript/extractor/src/com/semmle/js/extractor/ES2015Detector.java

@@ -0,0 +1,34 @@
+package com.semmle.js.extractor;
+
+import com.semmle.js.ast.DynamicImport;
+import com.semmle.js.ast.ExportDeclaration;
+import com.semmle.js.ast.Expression;
+import com.semmle.js.ast.ImportDeclaration;
+import com.semmle.js.ast.Node;
+import com.semmle.js.ast.Statement;
+
+/** A utility class for detecting Node.js code. */
+public class ES2015Detector extends AbstractDetector {
+  /**
+   * Is {@code ast} a program that uses ES2015 import/export code?
+   */
+  public static boolean looksLikeES2015(Node ast) {
+    return new ES2015Detector().programDetection(ast);
+  }
+
+  @Override
+  protected boolean visitStatement(Statement stmt) {
+    if (stmt instanceof ImportDeclaration || stmt instanceof ExportDeclaration) {
+      return true;
+    }
+    return super.visitStatement(stmt);
+  }
+
+  @Override
+  protected boolean visitExpression(Expression e) {
+    if (e instanceof DynamicImport) {
+      return true;
+    }
+    return super.visitExpression(e);
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -58,6 +58,26 @@ public Pair<Label, ParseResultInfo> extract(
 
     JSParser.Result parserRes =
         JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+
+    // Check if we guessed wrong with the regex in `establishSourceType`, (which could
+    // happen due to a block-comment line starting with '  import').
+    if (config.getSourceType() == SourceType.AUTO && sourceType != SourceType.SCRIPT) {
+      boolean wrongGuess = false;
+
+      if (sourceType == SourceType.MODULE) {
+        // check that we did see an import/export declaration
+        wrongGuess = ES2015Detector.looksLikeES2015(parserRes.getAST()) == false;
+      } else if (sourceType == SourceType.CLOSURE_MODULE ) {
+        // TODO
+      }
+
+      if (wrongGuess) {
+        sourceType = SourceType.SCRIPT;
+        parserRes =
+          JSParser.parse(config, sourceType, source, textualExtractor.getMetrics());
+      }
+    }
+
     return extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, parserRes);
   }
 

javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -18,6 +18,7 @@
 @SuiteClasses({
   JSXTests.class,
   NodeJSDetectorTests.class,
+  ES2015DetectorTests.class,
   TrapTests.class,
   ObjectRestSpreadTests.class,
   ClassPropertiesTests.class,

javascript/extractor/test/com/semmle/js/extractor/test/ES2015DetectorTests.java

@@ -0,0 +1,58 @@
+package com.semmle.js.extractor.test;
+
+import com.semmle.js.ast.Node;
+import com.semmle.js.extractor.ES2015Detector;
+import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.ExtractorConfig;
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+import com.semmle.js.parser.JSParser;
+import com.semmle.js.parser.JSParser.Result;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ES2015DetectorTests {
+  // using `experimental: true` as we do in real extractor, see `extractSource` method
+  // in `AutoBuild.java`
+  private static final ExtractorConfig CONFIG = new ExtractorConfig(true);
+
+  private void isES2015(String src, boolean expected) {
+    Result res = JSParser.parse(CONFIG, SourceType.MODULE, src, new ExtractionMetrics());
+    Node ast = res.getAST();
+    Assert.assertNotNull(ast);
+    Assert.assertTrue(ES2015Detector.looksLikeES2015(ast) == expected);
+  }
+
+  @Test
+  public void testImport() {
+    isES2015("import * as fs from 'fs';", true);
+  }
+
+  @Test
+  public void testExport() {
+    isES2015("export function foo() { };", true);
+  }
+
+  @Test
+  public void testDynamicImport() {
+    isES2015("import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportAssign() {
+    isES2015("var fs = import('fs');", true);
+  }
+
+  @Test
+  public void testDynamicImportThen() {
+    isES2015("import('o').then((o) => {});", true);
+  }
+
+  @Test
+  public void importInBlockComment() {
+    isES2015("/*\n"
+      + "  import * from 'fs';\n"
+      + "*/\n"
+      + "const fs = require('fs');",
+      false);
+  }
+}

javascript/extractor/tests/node/output/trap/detection.js.trap

@@ -137,7 +137,6 @@ variables(#20046,"__dirname",#20041)
 #20047=@"var;{arguments};{#20041}"
 variables(#20047,"arguments",#20041)
 is_module(#20001)
-is_es2015_module(#20001)
 #20048=@"var;{fs};{#20041}"
 variables(#20048,"fs",#20041)
 #20049=*
@@ -199,5 +198,6 @@ successor(#20053,#20050)
 successor(#20052,#20055)
 successor(#20050,#20061)
 successor(#20059,#20049)
+is_nodejs(#20001)
 numlines(#10000,6,1,5)
 filetype(#10000,"javascript")