Merge pull request github#3832 from asger-semmle/js/typescript-in-html-files3

Approved by erik-krogh

Author: semmle-qlci

change-notes/1.25/analysis-javascript.md

@@ -29,6 +29,8 @@
 
 * TypeScript 3.9 is now supported.
 
+* TypeScript code embedded in HTML and Vue files is now extracted and analyzed.
+
 * The analysis of sanitizers has improved, leading to more accurate
   results from the security queries.
 

javascript/extractor/lib/typescript/src/common.ts

@@ -48,7 +48,7 @@ export class Project {
   public load(): void {
     const { config, host } = this;
     this.program = ts.createProgram(config.fileNames, config.options, host);
-    this.typeTable.setProgram(this.program);
+    this.typeTable.setProgram(this.program, this.virtualSourceRoot);
   }
 
   /**
@@ -71,10 +71,19 @@ export class Project {
         redirectedReference: ts.ResolvedProjectReference,
         options: ts.CompilerOptions) {
 
+    let oppositePath =
+      this.virtualSourceRoot.toVirtualPath(containingFile) ||
+      this.virtualSourceRoot.fromVirtualPath(containingFile);
+
     const { host, resolutionCache } = this;
     return moduleNames.map((moduleName) => {
       let redirected = this.redirectModuleName(moduleName, containingFile, options);
       if (redirected != null) return redirected;
+      if (oppositePath != null) {
+        // If the containing file is in the virtual source root, try resolving from the real source root, and vice versa.
+        redirected = ts.resolveModuleName(moduleName, oppositePath, options,  host, resolutionCache).resolvedModule;
+        if (redirected != null) return redirected;
+      }
       return ts.resolveModuleName(moduleName, containingFile, options,  host, resolutionCache).resolvedModule;
     });
   }
@@ -90,15 +99,7 @@ export class Project {
 
     // Get the overridden location of this package, if one exists.
     let packageEntryPoint = this.packageEntryPoints.get(packageName);
-    if (packageEntryPoint == null) {
-      // The package is not overridden, but we have established that it begins with a valid package name.
-      // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
-      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
-      if (virtualContainingFile != null) {
-        return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
-      }
-      return null;
-    }
+    if (packageEntryPoint == null) return null;
 
     // If the requested module name is exactly the overridden package name,
     // return the entry point file (it is not necessarily called `index.ts`).

javascript/extractor/lib/typescript/src/main.ts

@@ -414,7 +414,25 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
      */
     let parseConfigHost: ts.ParseConfigHost = {
         useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
+            // Perform the glob matching in both real and virtual source roots.
+            let exclusions = excludes == null ? [] : [...excludes];
+            if (virtualSourceRoot.virtualSourceRoot != null) {
+                // qltest puts the virtual source root inside the real source root (.testproj).
+                // Make sure we don't find files inside the virtual source root in this pass.
+                exclusions.push(virtualSourceRoot.virtualSourceRoot);
+            }
+            let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
+            let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
+            if (virtualDir == null) {
+                return originalResults;
+            }
+            // Make sure glob matching does not to discover anything in node_modules.
+            let virtualExclusions = excludes == null ? [] : [...excludes];
+            virtualExclusions.push('**/node_modules/**/*');
+            let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
+            return [ ...originalResults, ...virtualResults ];
+        },
         fileExists: (path: string) => {
             return ts.sys.fileExists(path)
                 || virtualSourceRoot.toVirtualPathIfFileExists(path) != null

javascript/extractor/lib/typescript/src/type_table.ts

@@ -1,4 +1,5 @@
 import * as ts from "./typescript";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface AugmentedSymbol extends ts.Symbol {
   parent?: AugmentedSymbol;
@@ -379,12 +380,15 @@ export class TypeTable {
    */
   public restrictedExpansion = false;
 
+  private virtualSourceRoot: VirtualSourceRoot;
+
   /**
    * Called when a new compiler instance has started.
    */
-  public setProgram(program: ts.Program) {
+  public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) {
     this.typeChecker = program.getTypeChecker();
     this.arbitraryAstNode = program.getSourceFiles()[0];
+    this.virtualSourceRoot = virtualSourceRoot;
   }
 
   /**
@@ -703,14 +707,21 @@ export class TypeTable {
   private getSymbolString(symbol: AugmentedSymbol): string {
     let parent = symbol.parent;
     if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
-      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
+      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol);
     } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
-      return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     } else {
-      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     }
   }
 
+  private rewriteSymbolName(symbol: AugmentedSymbol) {
+    let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot;
+    let { name } = symbol;
+    if (virtualSourceRoot == null || sourceRoot == null) return name;
+    return name.replace(virtualSourceRoot, sourceRoot);
+  }
+
   /**
    * Gets a string that distinguishes the given symbol from symbols with different
    * lexical roots, or an empty string if the symbol is not a lexical root.

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -7,23 +7,42 @@ import * as ts from "./typescript";
  */
 export class VirtualSourceRoot {
   constructor(
-    private sourceRoot: string | null,
+    public sourceRoot: string | null,
 
     /**
      * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
      * or undefined if no virtual source root exists.
      */
-    private virtualSourceRoot: string | null,
+    public virtualSourceRoot: string | null,
   ) {}
 
+  private static translate(oldRoot: string, newRoot: string, path: string) {
+    if (!oldRoot || !newRoot) return null;
+    let relative = pathlib.relative(oldRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(newRoot, relative);
+  }
+
   /**
    * Maps a path under the real source root to the corresponding path in the virtual source root.
+   *
+   * Returns `null` for paths already in the virtual source root.
    */
   public toVirtualPath(path: string) {
-    if (!this.virtualSourceRoot || !this.sourceRoot) return null;
-    let relative = pathlib.relative(this.sourceRoot, path);
-    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
-    return pathlib.join(this.virtualSourceRoot, relative);
+    let { virtualSourceRoot } = this;
+    if (path.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path);
+  }
+
+  /**
+   * Maps a path under the virtual source root to the corresponding path in the real source root.
+   */
+  public fromVirtualPath(path: string) {
+    return VirtualSourceRoot.translate(this.virtualSourceRoot, this.sourceRoot, path);
   }
 
   /**