Merge branch 'master' into transforms

Author: rbuckton

.travis.yml

@@ -6,3 +6,25 @@ node_js:
   - '0.10'
 
 sudo: false
+
+matrix:
+  fast_finish: true
+  include:
+  - os: osx
+    node_js: stable
+    osx_image: xcode7.3
+
+branches:
+  only:
+  - master
+  - transforms
+
+install:
+  - npm uninstall typescript
+  - npm uninstall tslint
+  - npm install
+  - npm update
+
+cache:
+  directories:
+    - node_modules

Gulpfile.ts

@@ -11,8 +11,11 @@ import newer = require("gulp-newer");
 import tsc = require("gulp-typescript");
 declare module "gulp-typescript" {
     interface Settings {
-        stripInternal?: boolean;
+        pretty?: boolean;
         newLine?: string;
+        noImplicitThis?: boolean;
+        stripInternal?: boolean;
+        types?: string[];
     }
     interface CompileStream extends NodeJS.ReadWriteStream {} // Either gulp or gulp-typescript has some odd typings which don't reflect reality, making this required
 }
@@ -56,7 +59,6 @@ const cmdLineOptions = minimist(process.argv.slice(2), {
         browser: process.env.browser || process.env.b || "IE",
         tests: process.env.test || process.env.tests || process.env.t,
         light: process.env.light || false,
-        port: process.env.port || process.env.p || "8888",
         reporter: process.env.reporter || process.env.r,
         lint: process.env.lint || true,
         files: process.env.f || process.env.file || process.env.files || "",
@@ -82,12 +84,9 @@ let host = cmdLineOptions["host"];
 
 // Constants
 const compilerDirectory = "src/compiler/";
-const servicesDirectory = "src/services/";
-const serverDirectory = "src/server/";
 const harnessDirectory = "src/harness/";
 const libraryDirectory = "src/lib/";
 const scriptsDirectory = "scripts/";
-const unittestsDirectory = "tests/cases/unittests/";
 const docDirectory = "doc/";
 
 const builtDirectory = "built/";
@@ -104,69 +103,6 @@ const nodeModulesPathPrefix = path.resolve("./node_modules/.bin/");
 const isWin = /^win/.test(process.platform);
 const mocha = path.join(nodeModulesPathPrefix, "mocha") + (isWin ? ".cmd" : "");
 
-const compilerSources = require("./src/compiler/tsconfig.json").files.map((file) => path.join(compilerDirectory, file));
-
-const servicesSources = require("./src/services/tsconfig.json").files.map((file) => path.join(servicesDirectory, file));
-
-const serverCoreSources = require("./src/server/tsconfig.json").files.map((file) => path.join(serverDirectory, file));
-
-const languageServiceLibrarySources = [
-    "editorServices.ts",
-    "protocol.d.ts",
-    "session.ts"
-].map(function (f) {
-    return path.join(serverDirectory, f);
-}).concat(servicesSources);
-
-const harnessCoreSources = [
-    "harness.ts",
-    "sourceMapRecorder.ts",
-    "harnessLanguageService.ts",
-    "fourslash.ts",
-    "runnerbase.ts",
-    "compilerRunner.ts",
-    "typeWriter.ts",
-    "fourslashRunner.ts",
-    "projectsRunner.ts",
-    "loggedIO.ts",
-    "rwcRunner.ts",
-    "test262Runner.ts",
-    "runner.ts"
-].map(function (f) {
-    return path.join(harnessDirectory, f);
-});
-
-const harnessSources = harnessCoreSources.concat([
-    "incrementalParser.ts",
-    "jsDocParsing.ts",
-    "services/colorization.ts",
-    "services/documentRegistry.ts",
-    "services/preProcessFile.ts",
-    "services/patternMatcher.ts",
-    "session.ts",
-    "versionCache.ts",
-    "convertToBase64.ts",
-    "transpile.ts",
-    "reuseProgramStructure.ts",
-    "cachingInServerLSHost.ts",
-    "moduleResolution.ts",
-    "tsconfigParsing.ts",
-    "commandLineParsing.ts",
-    "convertCompilerOptionsFromJson.ts",
-    "convertTypingOptionsFromJson.ts",
-    "tsserverProjectSystem.ts",
-    "matchFiles.ts",
-].map(function (f) {
-    return path.join(unittestsDirectory, f);
-})).concat([
-    "protocol.d.ts",
-    "session.ts",
-    "client.ts",
-    "editorServices.ts"
-].map(function (f) {
-    return path.join(serverDirectory, f);
-}));
-
 const es2015LibrarySources = [
     "es2015.core.d.ts",
     "es2015.collection.d.ts",
@@ -306,6 +242,11 @@ function needsUpdate(source: string | string[], dest: string | string[]): boolea
 
 function getCompilerSettings(base: tsc.Settings, useBuiltCompiler?: boolean): tsc.Settings {
     const copy: tsc.Settings = {};
+    copy.noEmitOnError = true;
+    copy.noImplicitAny = true;
+    copy.noImplicitThis = true;
+    copy.pretty = true;
+    copy.types = [];
     for (const key in base) {
         copy[key] = base[key];
     }
@@ -492,21 +433,18 @@ const tsserverLibraryFile = path.join(builtLocalDirectory, "tsserverlibrary.js")
 const tsserverLibraryDefinitionFile = path.join(builtLocalDirectory, "tsserverlibrary.d.ts");
 
 gulp.task(tsserverLibraryFile, false, [servicesFile], (done) => {
-    const settings: tsc.Settings = getCompilerSettings({
-        declaration: true,
-        outFile: tsserverLibraryFile
-    }, /*useBuiltCompiler*/ true);
-    const {js, dts}: {js: NodeJS.ReadableStream, dts: NodeJS.ReadableStream} = gulp.src(languageServiceLibrarySources)
+    const serverLibraryProject = tsc.createProject("src/server/tsconfig.library.json", getCompilerSettings({}, /*useBuiltCompiler*/ true));
+    const {js, dts}: {js: NodeJS.ReadableStream, dts: NodeJS.ReadableStream} = serverLibraryProject.src()
         .pipe(sourcemaps.init())
         .pipe(newer(tsserverLibraryFile))
-        .pipe(tsc(settings));
+        .pipe(tsc(serverLibraryProject));
 
     return merge2([
         js.pipe(prependCopyright())
           .pipe(sourcemaps.write("."))
-          .pipe(gulp.dest(".")),
+          .pipe(gulp.dest(builtLocalDirectory)),
         dts.pipe(prependCopyright())
-        .pipe(gulp.dest("."))
+          .pipe(gulp.dest(builtLocalDirectory))
     ]);
 });
 
@@ -575,15 +513,13 @@ gulp.task("LKG", "Makes a new LKG out of the built js files", ["clean", "dontUse
 // Task to build the tests infrastructure using the built compiler
 const run = path.join(builtLocalDirectory, "run.js");
 gulp.task(run, false, [servicesFile], () => {
-    const settings: tsc.Settings = getCompilerSettings({
-        outFile: run
-    }, /*useBuiltCompiler*/ true);
-    return gulp.src(harnessSources)
+    const testProject = tsc.createProject("src/harness/tsconfig.json", getCompilerSettings({}, /*useBuiltCompiler*/true));
+    return testProject.src()
         .pipe(newer(run))
         .pipe(sourcemaps.init())
-        .pipe(tsc(settings))
+        .pipe(tsc(testProject))
         .pipe(sourcemaps.write(".", { includeContent: false, sourceRoot: "../../" }))
-        .pipe(gulp.dest("."));
+        .pipe(gulp.dest(builtLocalDirectory));
 });
 
 const internalTests = "internal/";
@@ -757,23 +693,50 @@ gulp.task(nodeServerOutFile, false, [servicesFile], () => {
         .pipe(gulp.dest(path.dirname(nodeServerOutFile)));
 });
 
+import convertMap = require("convert-source-map");
+import sorcery = require("sorcery");
+declare module "convert-source-map" {
+    export function fromSource(source: string, largeSource?: boolean): SourceMapConverter;
+}
+
 gulp.task("browserify", "Runs browserify on run.js to produce a file suitable for running tests in the browser", [servicesFile], (done) => {
-    const settings: tsc.Settings = getCompilerSettings({
-        outFile: "built/local/bundle.js"
-    }, /*useBuiltCompiler*/ true);
-    return gulp.src(harnessSources)
+    const testProject = tsc.createProject("src/harness/tsconfig.json", getCompilerSettings({ outFile: "built/local/bundle.js" }, /*useBuiltCompiler*/ true));
+    return testProject.src()
         .pipe(newer("built/local/bundle.js"))
         .pipe(sourcemaps.init())
-        .pipe(tsc(settings))
+        .pipe(tsc(testProject))
         .pipe(through2.obj((file, enc, next) => {
-            browserify(intoStream(file.contents))
+            const originalMap = file.sourceMap;
+            const prebundledContent = file.contents.toString();
+            // Make paths absolute to help sorcery deal with all the terrible paths being thrown around
+            originalMap.sources = originalMap.sources.map(s => path.resolve("src", s));
+            // intoStream (below) makes browserify think the input file is named this, so this is what it puts in the sourcemap
+            originalMap.file = "built/local/_stream_0.js";
+
+            browserify(intoStream(file.contents), { debug: true })
                 .bundle((err, res) => {
                     // assumes file.contents is a Buffer
-                    file.contents = res;
+                    const maps = JSON.parse(convertMap.fromSource(res.toString(), /*largeSource*/true).toJSON());
+                    delete maps.sourceRoot;
+                    maps.sources = maps.sources.map(s => path.resolve(s === "_stream_0.js" ? "built/local/_stream_0.js" : s));
+                    // Strip browserify's inline comments away (could probably just let sorcery do this, but then we couldn't fix the paths)
+                    file.contents = new Buffer(convertMap.removeComments(res.toString()));
+                    const chain = sorcery.loadSync("built/local/bundle.js", {
+                        content: {
+                            "built/local/_stream_0.js": prebundledContent,
+                            "built/local/bundle.js": file.contents.toString()
+                        },
+                        sourcemaps: {
+                            "built/local/_stream_0.js": originalMap,
+                            "built/local/bundle.js": maps,
+                        }
+                    });
+                    const finalMap = chain.apply();
+                    file.sourceMap = finalMap;
                     next(undefined, file);
                 });
         }))
-        .pipe(sourcemaps.write(".", { includeContent: false, sourceRoot: "../../" }))
+        .pipe(sourcemaps.write(".", { includeContent: false }))
         .pipe(gulp.dest("."));
 });
 
@@ -802,7 +765,7 @@ function writeTestConfigFile(tests: string, light: boolean, taskConfigsFolder?:
 }
 
 
-gulp.task("runtests-browser", "Runs the tests using the built run.js file like 'gulp runtests'. Syntax is gulp runtests-browser. Additional optional parameters --tests=[regex], --port=, --browser=[chrome|IE]", ["browserify", nodeServerOutFile], (done) => {
+gulp.task("runtests-browser", "Runs the tests using the built run.js file like 'gulp runtests'. Syntax is gulp runtests-browser. Additional optional parameters --tests=[regex], --browser=[chrome|IE]", ["browserify", nodeServerOutFile], (done) => {
     cleanTestDirs((err) => {
         if (err) { console.error(err); done(err); process.exit(1); }
         host = "node";
@@ -817,9 +780,6 @@ gulp.task("runtests-browser", "Runs the tests using the built run.js file like '
         }
 
         const args = [nodeServerOutFile];
-        if (cmdLineOptions["port"]) {
-            args.push(cmdLineOptions["port"]);
-        }
         if (cmdLineOptions["browser"]) {
             args.push(cmdLineOptions["browser"]);
         }
@@ -954,37 +914,20 @@ gulp.task("update-sublime", "Updates the sublime plugin's tsserver", ["local", s
     return gulp.src([serverFile, serverFile + ".map"]).pipe(gulp.dest("../TypeScript-Sublime-Plugin/tsserver/"));
 });
 
-
-const tslintRuleDir = "scripts/tslint";
-const tslintRules = [
-    "nextLineRule",
-    "preferConstRule",
-    "booleanTriviaRule",
-    "typeOperatorSpacingRule",
-    "noInOperatorRule",
-    "noIncrementDecrementRule",
-    "objectLiteralSurroundingSpaceRule",
-];
-const tslintRulesFiles = tslintRules.map(function(p) {
-    return path.join(tslintRuleDir, p + ".ts");
-});
-const tslintRulesOutFiles = tslintRules.map(function(p, i) {
-    const pathname = path.join(builtLocalDirectory, "tslint", p + ".js");
-    gulp.task(pathname, false, [], () => {
-        const settings: tsc.Settings = getCompilerSettings({ module: "commonjs" }, /*useBuiltCompiler*/ false);
-        return gulp.src(tslintRulesFiles[i])
-            .pipe(newer(pathname))
-            .pipe(sourcemaps.init())
-            .pipe(tsc(settings))
-            .pipe(sourcemaps.write("."))
-            .pipe(gulp.dest(path.join(builtLocalDirectory, "tslint")));
-    });
-    return pathname;
+gulp.task("build-rules", "Compiles tslint rules to js", () => {
+    const settings: tsc.Settings = getCompilerSettings({ module: "commonjs" }, /*useBuiltCompiler*/ false);
+    const dest = path.join(builtLocalDirectory, "tslint");
+    return gulp.src("scripts/tslint/**/*.ts")
+        .pipe(newer({
+            dest,
+            ext: ".js"
+        }))
+        .pipe(sourcemaps.init())
+        .pipe(tsc(settings))
+        .pipe(sourcemaps.write("."))
+        .pipe(gulp.dest(dest));
 });
 
-gulp.task("build-rules", "Compiles tslint rules to js", tslintRulesOutFiles);
-
-
 function getLinterOptions() {
     return {
         configuration: require("./tslint.json"),
@@ -1005,36 +948,38 @@ function lintFile(options, path) {
     return lintFileContents(options, path, contents);
 }
 
-const lintTargets = ["Gulpfile.ts"]
-    .concat(compilerSources)
-    .concat(harnessSources)
-    // Other harness sources
-    .concat(["instrumenter.ts"].map(function(f) { return path.join(harnessDirectory, f); }))
-    .concat(serverCoreSources)
-    .concat(tslintRulesFiles)
-    .concat(servicesSources);
+const lintTargets = [
+    "Gulpfile.ts",
+    "src/compiler/**/*.ts",
+    "src/harness/**/*.ts",
+    "!src/harness/unittests/services/formatting/**/*.ts",
+    "src/server/**/*.ts",
+    "scripts/tslint/**/*.ts",
+    "src/services/**/*.ts",
+    "tests/*.ts", "tests/webhost/*.ts" // Note: does *not* descend recursively
+];
 
 
 gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are: --f[iles]=regex", ["build-rules"], () => {
+    const fileMatcher = RegExp(cmdLineOptions["files"]);
     const lintOptions = getLinterOptions();
     let failed = 0;
-    const fileMatcher = RegExp(cmdLineOptions["files"]);
-    const done = {};
-    for (const i in lintTargets) {
-        const target = lintTargets[i];
-        if (!done[target] && fileMatcher.test(target)) {
-            const result = lintFile(lintOptions, target);
+    return gulp.src(lintTargets)
+        .pipe(insert.transform((contents, file) => {
+            if (!fileMatcher.test(file.path)) return contents;
+            const result = lintFile(lintOptions, file.path);
             if (result.failureCount > 0) {
                 console.log(result.output);
                 failed += result.failureCount;
             }
-            done[target] = true;
-        }
-    }
-    if (failed > 0) {
-        console.error("Linter errors.");
-        process.exit(1);
-    }
+            return contents; // TODO (weswig): Automatically apply fixes? :3
+        }))
+        .on("end", () => {
+            if (failed > 0) {
+                console.error("Linter errors.");
+                process.exit(1);
+            }
+        });
 });