Merge pull request #16684 from amcasey/Vsts434619

Watch for the creation of missing files

Author: amcasey

Jakefile.js

@@ -138,6 +138,7 @@ var harnessSources = harnessCoreSources.concat([
     "telemetry.ts",
     "transform.ts",
     "customTransforms.ts",
+    "programMissingFiles.ts",
 ].map(function (f) {
     return path.join(unittestsDirectory, f);
 })).concat([

src/compiler/core.ts

@@ -1098,6 +1098,15 @@ namespace ts {
         return result;
     }
 
+    /**
+     * Creates a set from the elements of an array.
+     *
+     * @param array the array of input elements.
+     */
+    export function arrayToSet<T>(array: T[], makeKey: (value: T) => string): Map<true> {
+        return arrayToMap<T, true>(array, makeKey, () => true);
+    }
+
     export function cloneMap<T>(map: Map<T>) {
         const clone = createMap<T>();
         copyEntries(map, clone);

src/compiler/program.ts

@@ -472,7 +472,7 @@ namespace ts {
             resolveTypeReferenceDirectiveNamesWorker = (typeReferenceDirectiveNames, containingFile) => loadWithLocalCache(typeReferenceDirectiveNames, containingFile, loader);
         }
 
-        const filesByName = createMap<SourceFile>();
+        const filesByName = createMap<SourceFile | undefined>();
         // stores 'filename -> file association' ignoring case
         // used to track cases when two file names differ only in casing
         const filesByNameIgnoreCase = host.useCaseSensitiveFileNames() ? createFileMap<SourceFile>(fileName => fileName.toLowerCase()) : undefined;
@@ -513,6 +513,8 @@ namespace ts {
             }
         }
 
+        const missingFilePaths = arrayFrom(filesByName.keys(), p => <Path>p).filter(p => !filesByName.get(p));
+
         // unconditionally set moduleResolutionCache to undefined to avoid unnecessary leaks
         moduleResolutionCache = undefined;
 
@@ -524,6 +526,7 @@ namespace ts {
             getSourceFile,
             getSourceFileByPath,
             getSourceFiles: () => files,
+            getMissingFilePaths: () => missingFilePaths,
             getCompilerOptions: () => options,
             getSyntacticDiagnostics,
             getOptionsDiagnostics,
@@ -862,6 +865,21 @@ namespace ts {
                 return oldProgram.structureIsReused;
             }
 
+            // If a file has ceased to be missing, then we need to discard some of the old
+            // structure in order to pick it up.
+            // Caution: if the file has created and then deleted between since it was discovered to
+            // be missing, then the corresponding file watcher will have been closed and no new one
+            // will be created until we encounter a change that prevents complete structure reuse.
+            // During this interval, creation of the file will go unnoticed.  We expect this to be
+            // both rare and low-impact.
+            if (oldProgram.getMissingFilePaths().some(missingFilePath => host.fileExists(missingFilePath))) {
+                return oldProgram.structureIsReused = StructureIsReused.SafeModules;
+            }
+
+            for (const p of oldProgram.getMissingFilePaths()) {
+                filesByName.set(p, undefined);
+            }
+
             // update fileName -> file mapping
             for (let i = 0; i < newSourceFiles.length; i++) {
                 filesByName.set(filePaths[i], newSourceFiles[i]);

src/compiler/sys.ts

@@ -4,7 +4,13 @@ declare function setTimeout(handler: (...args: any[]) => void, timeout: number):
 declare function clearTimeout(handle: any): void;
 
 namespace ts {
-    export type FileWatcherCallback = (fileName: string, removed?: boolean) => void;
+    export enum FileWatcherEventKind {
+        Created,
+        Changed,
+        Deleted
+    }
+
+    export type FileWatcherCallback = (fileName: string, eventKind: FileWatcherEventKind) => void;
     export type DirectoryWatcherCallback = (fileName: string) => void;
     export interface WatchedFile {
         fileName: string;
@@ -174,7 +180,7 @@ namespace ts {
                         const callbacks = fileWatcherCallbacks.get(fileName);
                         if (callbacks) {
                             for (const fileCallback of callbacks) {
-                                fileCallback(fileName);
+                                fileCallback(fileName, FileWatcherEventKind.Changed);
                             }
                         }
                     }
@@ -340,11 +346,22 @@ namespace ts {
                     }
 
                     function fileChanged(curr: any, prev: any) {
-                        if (+curr.mtime <= +prev.mtime) {
+                        const isCurrZero = +curr.mtime === 0;
+                        const isPrevZero = +prev.mtime === 0;
+                        const created = !isCurrZero && isPrevZero;
+                        const deleted = isCurrZero && !isPrevZero;
+
+                        const eventKind = created
+                            ? FileWatcherEventKind.Created
+                            : deleted
+                                ? FileWatcherEventKind.Deleted
+                                : FileWatcherEventKind.Changed;
+
+                        if (eventKind === FileWatcherEventKind.Changed && +curr.mtime <= +prev.mtime) {
                             return;
                         }
 
-                        callback(fileName);
+                        callback(fileName, eventKind);
                     }
                 },
                 watchDirectory: (directoryName, callback, recursive) => {

src/compiler/tsc.ts

@@ -285,6 +285,16 @@ namespace ts {
 
             setCachedProgram(compileResult.program);
             reportWatchDiagnostic(createCompilerDiagnostic(Diagnostics.Compilation_complete_Watching_for_file_changes));
+
+            const missingPaths = compileResult.program.getMissingFilePaths();
+            missingPaths.forEach(path => {
+                const fileWatcher = sys.watchFile(path, (_fileName, eventKind) => {
+                    if (eventKind === FileWatcherEventKind.Created) {
+                        fileWatcher.close();
+                        startTimerForRecompilation();
+                    }
+                });
+            });
         }
 
         function cachedFileExists(fileName: string): boolean {
@@ -308,7 +318,7 @@ namespace ts {
             const sourceFile = hostGetSourceFile(fileName, languageVersion, onError);
             if (sourceFile && isWatchSet(compilerOptions) && sys.watchFile) {
                 // Attach a file watcher
-                sourceFile.fileWatcher = sys.watchFile(sourceFile.fileName, (_fileName: string, removed?: boolean) => sourceFileChanged(sourceFile, removed));
+                sourceFile.fileWatcher = sys.watchFile(sourceFile.fileName, (_fileName, eventKind) => sourceFileChanged(sourceFile, eventKind));
             }
             return sourceFile;
         }
@@ -330,10 +340,10 @@ namespace ts {
         }
 
         // If a source file changes, mark it as unwatched and start the recompilation timer
-        function sourceFileChanged(sourceFile: SourceFile, removed?: boolean) {
+        function sourceFileChanged(sourceFile: SourceFile, eventKind: FileWatcherEventKind) {
             sourceFile.fileWatcher.close();
             sourceFile.fileWatcher = undefined;
-            if (removed) {
+            if (eventKind === FileWatcherEventKind.Deleted) {
                 unorderedRemoveItem(rootFileNames, sourceFile.fileName);
             }
             startTimerForRecompilation();

src/compiler/types.ts

@@ -2426,6 +2426,13 @@ namespace ts {
          */
         getSourceFiles(): SourceFile[];
 
+        /**
+         * Get a list of file names that were passed to 'createProgram' or referenced in a
+         * program source file but could not be located.
+         */
+        /* @internal */
+        getMissingFilePaths(): Path[];
+
         /**
          * Emits the JavaScript and declaration files.  If targetSourceFile is not specified, then
          * the JavaScript and declaration files will be produced for all the files in this program.

src/harness/tsconfig.json

@@ -129,6 +129,7 @@
         "./unittests/transform.ts",
         "./unittests/customTransforms.ts",
         "./unittests/textChanges.ts",
-        "./unittests/telemetry.ts"
+        "./unittests/telemetry.ts",
+        "./unittests/programMissingFiles.ts"
     ]
 }

src/harness/unittests/cachingInServerLSHost.ts

@@ -75,7 +75,7 @@ namespace ts {
         const rootScriptInfo = projectService.getOrCreateScriptInfo(rootFile, /* openedByClient */ true, /*containingProject*/ undefined);
 
         const project = projectService.createInferredProjectWithRootFileIfNecessary(rootScriptInfo);
-        project.setCompilerOptions({ module: ts.ModuleKind.AMD } );
+        project.setCompilerOptions({ module: ts.ModuleKind.AMD, noLib: true } );
         return {
             project,
             rootScriptInfo

src/harness/unittests/compileOnSave.ts

@@ -208,7 +208,7 @@ namespace ts.projectSystem {
 
                 file1Consumer1.content = `let y = 10;`;
                 host.reloadFS([moduleFile1, file1Consumer1, file1Consumer2, configFile, libFile]);
-                host.triggerFileWatcherCallback(file1Consumer1.path, /*removed*/ false);
+                host.triggerFileWatcherCallback(file1Consumer1.path, FileWatcherEventKind.Changed);
 
                 session.executeCommand(changeModuleFile1ShapeRequest1);
                 sendAffectedFileRequestAndCheckResult(session, moduleFile1FileListRequest, [{ projectFileName: configFile.path, files: [moduleFile1, file1Consumer2] }]);
@@ -225,7 +225,7 @@ namespace ts.projectSystem {
                 session.executeCommand(changeModuleFile1ShapeRequest1);
                 // Delete file1Consumer2
                 host.reloadFS([moduleFile1, file1Consumer1, configFile, libFile]);
-                host.triggerFileWatcherCallback(file1Consumer2.path, /*removed*/ true);
+                host.triggerFileWatcherCallback(file1Consumer2.path, FileWatcherEventKind.Deleted);
                 sendAffectedFileRequestAndCheckResult(session, moduleFile1FileListRequest, [{ projectFileName: configFile.path, files: [moduleFile1, file1Consumer1] }]);
             });
 
@@ -475,7 +475,7 @@ namespace ts.projectSystem {
 
                 openFilesForSession([referenceFile1], session);
                 host.reloadFS([referenceFile1, configFile]);
-                host.triggerFileWatcherCallback(moduleFile1.path, /*removed*/ true);
+                host.triggerFileWatcherCallback(moduleFile1.path, FileWatcherEventKind.Deleted);
 
                 const request = makeSessionRequest<server.protocol.FileRequestArgs>(CommandNames.CompileOnSaveAffectedFileList, { file: referenceFile1.path });
                 sendAffectedFileRequestAndCheckResult(session, request, [

src/harness/unittests/programMissingFiles.ts

@@ -0,0 +1,103 @@
+/// <reference path="..\harness.ts" />
+
+namespace ts {
+    describe("Program.getMissingFilePaths", () => {
+
+        const options: CompilerOptions = {
+            noLib: true,
+        };
+
+        const emptyFileName = "empty.ts";
+        const emptyFileRelativePath = "./" + emptyFileName;
+
+        const emptyFile: Harness.Compiler.TestFile = {
+            unitName: emptyFileName,
+            content: ""
+        };
+
+        const referenceFileName = "reference.ts";
+        const referenceFileRelativePath = "./" + referenceFileName;
+
+        const referenceFile: Harness.Compiler.TestFile = {
+            unitName: referenceFileName,
+            content:
+                "/// <reference path=\"d:/imaginary/nonexistent1.ts\"/>\n" + // Absolute
+                "/// <reference path=\"./nonexistent2.ts\"/>\n" + // Relative
+                "/// <reference path=\"nonexistent3.ts\"/>\n" + // Unqualified
+                "/// <reference path=\"nonexistent4\"/>\n"   // No extension
+        };
+
+        const testCompilerHost = Harness.Compiler.createCompilerHost(
+            /*inputFiles*/ [emptyFile, referenceFile],
+            /*writeFile*/ undefined,
+            /*scriptTarget*/ undefined,
+            /*useCaseSensitiveFileNames*/ false,
+            /*currentDirectory*/ "d:\\pretend\\",
+            /*newLineKind*/ NewLineKind.LineFeed,
+            /*libFiles*/ undefined
+        );
+
+        it("handles no missing root files", () => {
+            const program = createProgram([emptyFileRelativePath], options, testCompilerHost);
+            const missing = program.getMissingFilePaths();
+            assert.isDefined(missing);
+            assert.deepEqual(missing, []);
+        });
+
+        it("handles missing root file", () => {
+            const program = createProgram(["./nonexistent.ts"], options, testCompilerHost);
+            const missing = program.getMissingFilePaths();
+            assert.isDefined(missing);
+            assert.deepEqual(missing, ["d:/pretend/nonexistent.ts"]); // Absolute path
+        });
+
+        it("handles multiple missing root files", () => {
+            const program = createProgram(["./nonexistent0.ts", "./nonexistent1.ts"], options, testCompilerHost);
+            const missing = program.getMissingFilePaths().sort();
+            assert.deepEqual(missing, ["d:/pretend/nonexistent0.ts", "d:/pretend/nonexistent1.ts"]);
+        });
+
+        it("handles a mix of present and missing root files", () => {
+            const program = createProgram(["./nonexistent0.ts", emptyFileRelativePath, "./nonexistent1.ts"], options, testCompilerHost);
+            const missing = program.getMissingFilePaths().sort();
+            assert.deepEqual(missing, ["d:/pretend/nonexistent0.ts", "d:/pretend/nonexistent1.ts"]);
+        });
+
+        it("handles repeatedly specified root files", () => {
+            const program = createProgram(["./nonexistent.ts", "./nonexistent.ts"], options, testCompilerHost);
+            const missing = program.getMissingFilePaths();
+            assert.isDefined(missing);
+            assert.deepEqual(missing, ["d:/pretend/nonexistent.ts"]);
+        });
+
+        it("normalizes file paths", () => {
+            const program0 = createProgram(["./nonexistent.ts", "./NONEXISTENT.ts"], options, testCompilerHost);
+            const program1 = createProgram(["./NONEXISTENT.ts", "./nonexistent.ts"], options, testCompilerHost);
+            const missing0 = program0.getMissingFilePaths();
+            const missing1 = program1.getMissingFilePaths();
+            assert.equal(missing0.length, 1);
+            assert.deepEqual(missing0, missing1);
+        });
+
+        it("handles missing triple slash references", () => {
+            const program = createProgram([referenceFileRelativePath], options, testCompilerHost);
+            const missing = program.getMissingFilePaths().sort();
+            assert.isDefined(missing);
+            assert.deepEqual(missing, [
+                // From absolute reference
+                "d:/imaginary/nonexistent1.ts",
+
+                // From relative reference
+                "d:/pretend/nonexistent2.ts",
+
+                // From unqualified reference
+                "d:/pretend/nonexistent3.ts",
+
+                // From no-extension reference
+                "d:/pretend/nonexistent4.d.ts",
+                "d:/pretend/nonexistent4.ts",
+                "d:/pretend/nonexistent4.tsx"
+            ]);
+        });
+    });
+}