From e6c38bf67b451794471245b4a9f4909d02f66320 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Mon, 6 Nov 2017 15:16:33 -0800 Subject: [PATCH 1/9] Add DefinitelyTyped test runner Assumes that ../DefinitelyTyped holds the DefinitelyTyped repo. --- src/harness/definitelyRunner.ts | 51 +++++++++++++++++++++++++++++++++ src/harness/runner.ts | 6 ++++ src/harness/runnerbase.ts | 2 +- src/harness/tsconfig.json | 1 + src/harness/userRunner.ts | 2 +- 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 src/harness/definitelyRunner.ts diff --git a/src/harness/definitelyRunner.ts b/src/harness/definitelyRunner.ts new file mode 100644 index 0000000000000..afd39b7242465 --- /dev/null +++ b/src/harness/definitelyRunner.ts @@ -0,0 +1,51 @@ +/// +/// +class DefinitelyTypedRunner extends RunnerBase { + private static readonly testDir = "../DefinitelyTyped/types/"; + public enumerateTestFiles() { + return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir).map(dir => DefinitelyTypedRunner.testDir + dir); + } + + public kind(): TestRunnerKind { + return "definitely"; + } + + /** Setup the runner's tests so that they are ready to be executed by the harness + * The first test should be a describe/it block that sets up the harness's compiler instance appropriately + */ + public initializeTests(): void { + // Read in and evaluate the test list + const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); + + describe(`${this.kind()} code samples`, () => { + for (const test of testList) { + this.runTest(test); + } + }); + } + + private runTest(directoryName: string) { + describe(directoryName, () => { + const cp = require("child_process"); + const path = require("path"); + + it("should build successfully", () => { + const cwd = path.join(__dirname, "../../", directoryName); + const timeout = 600000; // 600s = 10 minutes + const stdio = isWorker ? "pipe" : "inherit"; + const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); + if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); + Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { + const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js"), "--lib dom,es6", "--strict"], { cwd, timeout, shell: true }); + return `Exit Code: ${result.status} +Standard output: +${result.stdout.toString().replace(/\r\n/g, "\n")} + + +Standard error: +${result.stderr.toString().replace(/\r\n/g, "\n")}`; + }); + }); + }); + } +} diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 70954e9e853d6..fb66e74b95859 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -19,6 +19,7 @@ /// /// /// +/// /// /// @@ -62,6 +63,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase { return new Test262BaselineRunner(); case "user": return new UserCodeRunner(); + case "definitely": + return new DefinitelyTypedRunner(); } ts.Debug.fail(`Unknown runner kind ${kind}`); } @@ -183,6 +186,9 @@ function handleTestConfig() { case "user": runners.push(new UserCodeRunner()); break; + case "definitely": + runners.push(new DefinitelyTypedRunner()); + break; } } } diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index 2fef2264b73ea..42e625a897de4 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -1,7 +1,7 @@ /// -type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user"; +type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "definitely"; type CompilerTestKind = "conformance" | "compiler"; type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server"; diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json index 6e61b7690bc79..96f1999e9e856 100644 --- a/src/harness/tsconfig.json +++ b/src/harness/tsconfig.json @@ -93,6 +93,7 @@ "loggedIO.ts", "rwcRunner.ts", "userRunner.ts", + "definitelyRunner.ts", "test262Runner.ts", "./parallel/shared.ts", "./parallel/host.ts", diff --git a/src/harness/userRunner.ts b/src/harness/userRunner.ts index 9be652aebf8ae..61a46d7e84f84 100644 --- a/src/harness/userRunner.ts +++ b/src/harness/userRunner.ts @@ -36,7 +36,7 @@ class UserCodeRunner extends RunnerBase { const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true }); + const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); return `Exit Code: ${result.status} Standard output: ${result.stdout.toString().replace(/\r\n/g, "\n")} From f2d4b36a49df7e7fb7a828b68d88a86f41f58717 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Mon, 6 Nov 2017 15:17:21 -0800 Subject: [PATCH 2/9] Update Jakefile with definitelyRunner.ts --- Jakefile.js | 1 + 1 file changed, 1 insertion(+) diff --git a/Jakefile.js b/Jakefile.js index 7f0915ad7e990..40520e79a4a24 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -106,6 +106,7 @@ var harnessCoreSources = [ "loggedIO.ts", "rwcRunner.ts", "userRunner.ts", + "definitelyRunner.ts", "test262Runner.ts", "./parallel/shared.ts", "./parallel/host.ts", From 9a415a2b23fa26e1f997580718ea41948b51f7ce Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Tue, 7 Nov 2017 09:50:17 -0800 Subject: [PATCH 3/9] DefinitelyRunner cleanup and speedup 1. Only `npm install` packages with a package.json 2. Add `workingDirectory` to runnerBase to differentiate input directory from output directory (which should be different for definitelyRunner). 3. Don't output anything on success. --- src/harness/definitelyRunner.ts | 20 +++++++++++++------- src/harness/parallel/host.ts | 8 ++++---- src/harness/runnerbase.ts | 3 +++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/harness/definitelyRunner.ts b/src/harness/definitelyRunner.ts index afd39b7242465..c2f7a04d3e7ff 100644 --- a/src/harness/definitelyRunner.ts +++ b/src/harness/definitelyRunner.ts @@ -2,8 +2,11 @@ /// class DefinitelyTypedRunner extends RunnerBase { private static readonly testDir = "../DefinitelyTyped/types/"; + + public workingDirectory = DefinitelyTypedRunner.testDir; + public enumerateTestFiles() { - return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir).map(dir => DefinitelyTypedRunner.testDir + dir); + return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir); } public kind(): TestRunnerKind { @@ -28,16 +31,19 @@ class DefinitelyTypedRunner extends RunnerBase { describe(directoryName, () => { const cp = require("child_process"); const path = require("path"); + const fs = require("fs"); it("should build successfully", () => { - const cwd = path.join(__dirname, "../../", directoryName); + const cwd = path.join(__dirname, "../../", DefinitelyTypedRunner.testDir, directoryName); const timeout = 600000; // 600s = 10 minutes - const stdio = isWorker ? "pipe" : "inherit"; - const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); - if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); + if (fs.existsSync(path.join(cwd, 'package.json'))) { + const stdio = isWorker ? "pipe" : "inherit"; + const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); + if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); + } Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js"), "--lib dom,es6", "--strict"], { cwd, timeout, shell: true }); - return `Exit Code: ${result.status} + const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); + return result.status === 0 ? null : `Exit Code: ${result.status} Standard output: ${result.stdout.toString().replace(/\r\n/g, "\n")} diff --git a/src/harness/parallel/host.ts b/src/harness/parallel/host.ts index f37a3b1099e70..e278d267e1ff6 100644 --- a/src/harness/parallel/host.ts +++ b/src/harness/parallel/host.ts @@ -77,18 +77,18 @@ namespace Harness.Parallel.Host { console.log("Discovering runner-based tests..."); const discoverStart = +(new Date()); const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); + const path: { join: (...args: string[]) => string } = require("path"); for (const runner of runners) { - const files = runner.enumerateTestFiles(); - for (const file of files) { + for (const file of runner.enumerateTestFiles()) { let size: number; if (!perfData) { try { - size = statSync(file).size; + size = statSync(path.join(runner.workingDirectory, file)).size; } catch { // May be a directory try { - size = Harness.IO.listFiles(file, /.*/g, { recursive: true }).reduce((acc, elem) => acc + statSync(elem).size, 0); + size = Harness.IO.listFiles(path.join(runner.workingDirectory, file), /.*/g, { recursive: true }).reduce((acc, elem) => acc + statSync(elem).size, 0); } catch { // Unknown test kind, just return 0 and let the historical analysis take over after one run diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index 42e625a897de4..1f5b31db2d696 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -24,6 +24,9 @@ abstract class RunnerBase { abstract enumerateTestFiles(): string[]; + /** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */ + public workingDirectory = ""; + /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ From 2378ff32b1164388ec8c7d74872f3f8dee8d764a Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Tue, 7 Nov 2017 10:45:42 -0800 Subject: [PATCH 4/9] Fix lint and allow null keyword --- src/harness/definitelyRunner.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/harness/definitelyRunner.ts b/src/harness/definitelyRunner.ts index c2f7a04d3e7ff..ca9f9137ca4b2 100644 --- a/src/harness/definitelyRunner.ts +++ b/src/harness/definitelyRunner.ts @@ -36,13 +36,14 @@ class DefinitelyTypedRunner extends RunnerBase { it("should build successfully", () => { const cwd = path.join(__dirname, "../../", DefinitelyTypedRunner.testDir, directoryName); const timeout = 600000; // 600s = 10 minutes - if (fs.existsSync(path.join(cwd, 'package.json'))) { + if (fs.existsSync(path.join(cwd, "package.json"))) { const stdio = isWorker ? "pipe" : "inherit"; const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); } Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); + // tslint:disable:no-null-keyword return result.status === 0 ? null : `Exit Code: ${result.status} Standard output: ${result.stdout.toString().replace(/\r\n/g, "\n")} @@ -50,6 +51,7 @@ ${result.stdout.toString().replace(/\r\n/g, "\n")} Standard error: ${result.stderr.toString().replace(/\r\n/g, "\n")}`; + // tslint:enable:no-null-keyword }); }); }); From 88a31d60967d86fd8107f630e4575b5d75048661 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Tue, 7 Nov 2017 10:57:54 -0800 Subject: [PATCH 5/9] Change runner name from 'definitely' to 'dt' --- Jakefile.js | 2 +- src/harness/{definitelyRunner.ts => dtRunner.ts} | 2 +- src/harness/runner.ts | 6 +++--- src/harness/runnerbase.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename src/harness/{definitelyRunner.ts => dtRunner.ts} (96%) diff --git a/Jakefile.js b/Jakefile.js index 40520e79a4a24..a133d0dba32ca 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -106,7 +106,7 @@ var harnessCoreSources = [ "loggedIO.ts", "rwcRunner.ts", "userRunner.ts", - "definitelyRunner.ts", + "dtRunner.ts", "test262Runner.ts", "./parallel/shared.ts", "./parallel/host.ts", diff --git a/src/harness/definitelyRunner.ts b/src/harness/dtRunner.ts similarity index 96% rename from src/harness/definitelyRunner.ts rename to src/harness/dtRunner.ts index ca9f9137ca4b2..1da07b92d86c1 100644 --- a/src/harness/definitelyRunner.ts +++ b/src/harness/dtRunner.ts @@ -10,7 +10,7 @@ class DefinitelyTypedRunner extends RunnerBase { } public kind(): TestRunnerKind { - return "definitely"; + return "dt"; } /** Setup the runner's tests so that they are ready to be executed by the harness diff --git a/src/harness/runner.ts b/src/harness/runner.ts index fb66e74b95859..0a9a7d3428ddf 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -19,7 +19,7 @@ /// /// /// -/// +/// /// /// @@ -63,7 +63,7 @@ function createRunner(kind: TestRunnerKind): RunnerBase { return new Test262BaselineRunner(); case "user": return new UserCodeRunner(); - case "definitely": + case "dt": return new DefinitelyTypedRunner(); } ts.Debug.fail(`Unknown runner kind ${kind}`); @@ -186,7 +186,7 @@ function handleTestConfig() { case "user": runners.push(new UserCodeRunner()); break; - case "definitely": + case "dt": runners.push(new DefinitelyTypedRunner()); break; } diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index 1f5b31db2d696..2858738f4b56a 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -1,7 +1,7 @@ /// -type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "definitely"; +type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "dt"; type CompilerTestKind = "conformance" | "compiler"; type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server"; From 5e5b5652ed8437eac908f40a1ac91e2f55a04761 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Tue, 7 Nov 2017 11:10:24 -0800 Subject: [PATCH 6/9] Remove package-lock.json before `npm install` --- src/harness/dtRunner.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/harness/dtRunner.ts b/src/harness/dtRunner.ts index 1da07b92d86c1..3b739b28f1cb9 100644 --- a/src/harness/dtRunner.ts +++ b/src/harness/dtRunner.ts @@ -37,6 +37,9 @@ class DefinitelyTypedRunner extends RunnerBase { const cwd = path.join(__dirname, "../../", DefinitelyTypedRunner.testDir, directoryName); const timeout = 600000; // 600s = 10 minutes if (fs.existsSync(path.join(cwd, "package.json"))) { + if (fs.existsSync(path.join(cwd, "package-lock.json"))) { + fs.unlinkSync(path.join(cwd, "package-lock.json")); + } const stdio = isWorker ? "pipe" : "inherit"; const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); From d64a8f62f28e0f77fb8b5b681494e76cb35b3373 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Wed, 8 Nov 2017 13:28:35 -0800 Subject: [PATCH 7/9] Refactor user+dt runners into externalCompilerRunner --- Jakefile.js | 3 +- .../{dtRunner.ts => externalCompileRunner.ts} | 129 +++++++++--------- src/harness/runner.ts | 3 +- src/harness/tsconfig.json | 3 +- src/harness/userRunner.ts | 51 ------- 5 files changed, 70 insertions(+), 119 deletions(-) rename src/harness/{dtRunner.ts => externalCompileRunner.ts} (76%) delete mode 100644 src/harness/userRunner.ts diff --git a/Jakefile.js b/Jakefile.js index a133d0dba32ca..147270e0a5924 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -105,8 +105,7 @@ var harnessCoreSources = [ "projectsRunner.ts", "loggedIO.ts", "rwcRunner.ts", - "userRunner.ts", - "dtRunner.ts", + "externalCompileRunner.ts", "test262Runner.ts", "./parallel/shared.ts", "./parallel/host.ts", diff --git a/src/harness/dtRunner.ts b/src/harness/externalCompileRunner.ts similarity index 76% rename from src/harness/dtRunner.ts rename to src/harness/externalCompileRunner.ts index 3b739b28f1cb9..8ca0be807bfbf 100644 --- a/src/harness/dtRunner.ts +++ b/src/harness/externalCompileRunner.ts @@ -1,62 +1,67 @@ -/// -/// -class DefinitelyTypedRunner extends RunnerBase { - private static readonly testDir = "../DefinitelyTyped/types/"; - - public workingDirectory = DefinitelyTypedRunner.testDir; - - public enumerateTestFiles() { - return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir); - } - - public kind(): TestRunnerKind { - return "dt"; - } - - /** Setup the runner's tests so that they are ready to be executed by the harness - * The first test should be a describe/it block that sets up the harness's compiler instance appropriately - */ - public initializeTests(): void { - // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); - - describe(`${this.kind()} code samples`, () => { - for (const test of testList) { - this.runTest(test); - } - }); - } - - private runTest(directoryName: string) { - describe(directoryName, () => { - const cp = require("child_process"); - const path = require("path"); - const fs = require("fs"); - - it("should build successfully", () => { - const cwd = path.join(__dirname, "../../", DefinitelyTypedRunner.testDir, directoryName); - const timeout = 600000; // 600s = 10 minutes - if (fs.existsSync(path.join(cwd, "package.json"))) { - if (fs.existsSync(path.join(cwd, "package-lock.json"))) { - fs.unlinkSync(path.join(cwd, "package-lock.json")); - } - const stdio = isWorker ? "pipe" : "inherit"; - const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); - if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); - } - Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); - // tslint:disable:no-null-keyword - return result.status === 0 ? null : `Exit Code: ${result.status} -Standard output: -${result.stdout.toString().replace(/\r\n/g, "\n")} - - -Standard error: -${result.stderr.toString().replace(/\r\n/g, "\n")}`; - // tslint:enable:no-null-keyword - }); - }); - }); - } -} +/// +/// +abstract class ExternalCompileRunnerBase extends RunnerBase { + abstract testDir: string; + public enumerateTestFiles() { + return Harness.IO.getDirectories(this.testDir); + } + /** Setup the runner's tests so that they are ready to be executed by the harness + * The first test should be a describe/it block that sets up the harness's compiler instance appropriately + */ + public initializeTests(): void { + // Read in and evaluate the test list + const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); + + describe(`${this.kind()} code samples`, () => { + for (const test of testList) { + this.runTest(test); + } + }); + } + private runTest(directoryName: string) { + describe(directoryName, () => { + const cp = require("child_process"); + const path = require("path"); + const fs = require("fs"); + + it("should build successfully", () => { + const cwd = path.join(__dirname, "../../", this.testDir, directoryName); + const timeout = 600000; // 600s = 10 minutes + if (fs.existsSync(path.join(cwd, "package.json"))) { + if (fs.existsSync(path.join(cwd, "package-lock.json"))) { + fs.unlinkSync(path.join(cwd, "package-lock.json")); + } + const stdio = isWorker ? "pipe" : "inherit"; + const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); + if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); + } + Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { + const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); + // tslint:disable-next-line:no-null-keyword + return result.status === 0 ? null : `Exit Code: ${result.status} +Standard output: +${result.stdout.toString().replace(/\r\n/g, "\n")} + + +Standard error: +${result.stderr.toString().replace(/\r\n/g, "\n")}`; + }); + }); + }); + } +} + +class UserCodeRunner extends ExternalCompileRunnerBase { + public readonly testDir = "tests/cases/user/"; + public kind(): TestRunnerKind { + return "user"; + } +} + +class DefinitelyTypedRunner extends ExternalCompileRunnerBase { + public readonly testDir = "../DefinitelyTyped/types/"; + public workingDirectory = this.testDir; + public kind(): TestRunnerKind { + return "dt"; + } +} diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 0a9a7d3428ddf..a121059109010 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -18,8 +18,7 @@ /// /// /// -/// -/// +/// /// /// diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json index 96f1999e9e856..1ab2cb955c8c3 100644 --- a/src/harness/tsconfig.json +++ b/src/harness/tsconfig.json @@ -92,8 +92,7 @@ "projectsRunner.ts", "loggedIO.ts", "rwcRunner.ts", - "userRunner.ts", - "definitelyRunner.ts", + "externalCompileRunner.ts", "test262Runner.ts", "./parallel/shared.ts", "./parallel/host.ts", diff --git a/src/harness/userRunner.ts b/src/harness/userRunner.ts deleted file mode 100644 index 61a46d7e84f84..0000000000000 --- a/src/harness/userRunner.ts +++ /dev/null @@ -1,51 +0,0 @@ -/// -/// -class UserCodeRunner extends RunnerBase { - private static readonly testDir = "tests/cases/user/"; - public enumerateTestFiles() { - return Harness.IO.getDirectories(UserCodeRunner.testDir); - } - - public kind(): TestRunnerKind { - return "user"; - } - - /** Setup the runner's tests so that they are ready to be executed by the harness - * The first test should be a describe/it block that sets up the harness's compiler instance appropriately - */ - public initializeTests(): void { - // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); - - describe(`${this.kind()} code samples`, () => { - for (const test of testList) { - this.runTest(test); - } - }); - } - - private runTest(directoryName: string) { - describe(directoryName, () => { - const cp = require("child_process"); - const path = require("path"); - - it("should build successfully", () => { - const cwd = path.join(__dirname, "../../", UserCodeRunner.testDir, directoryName); - const timeout = 600000; // 10 minutes - const stdio = isWorker ? "pipe" : "inherit"; - const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); - if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); - Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); - return `Exit Code: ${result.status} -Standard output: -${result.stdout.toString().replace(/\r\n/g, "\n")} - - -Standard error: -${result.stderr.toString().replace(/\r\n/g, "\n")}`; - }); - }); - }); - } -} From 397b5497a33ce30be6d86ecd86576287e42073a3 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Wed, 8 Nov 2017 13:29:28 -0800 Subject: [PATCH 8/9] Remove positive baselines for user tests --- tests/baselines/reference/user/ajv.log | 6 ------ tests/baselines/reference/user/antd.log | 6 ------ tests/baselines/reference/user/axios.log | 6 ------ tests/baselines/reference/user/bignumber.js.log | 6 ------ tests/baselines/reference/user/discord.js.log | 6 ------ tests/baselines/reference/user/eventemitter2.log | 6 ------ tests/baselines/reference/user/eventemitter3.log | 6 ------ tests/baselines/reference/user/firebase.log | 6 ------ tests/baselines/reference/user/github.log | 6 ------ tests/baselines/reference/user/immutable.log | 6 ------ tests/baselines/reference/user/isobject.log | 6 ------ tests/baselines/reference/user/jimp.log | 6 ------ tests/baselines/reference/user/jsonschema.log | 6 ------ tests/baselines/reference/user/keycode.log | 6 ------ tests/baselines/reference/user/localforage.log | 6 ------ tests/baselines/reference/user/log4js.log | 6 ------ tests/baselines/reference/user/mobx.log | 6 ------ tests/baselines/reference/user/moment.log | 6 ------ tests/baselines/reference/user/mqtt.log | 6 ------ tests/baselines/reference/user/parse5.log | 6 ------ tests/baselines/reference/user/portfinder.log | 6 ------ tests/baselines/reference/user/postcss.log | 6 ------ tests/baselines/reference/user/protobufjs.log | 6 ------ tests/baselines/reference/user/redux.log | 6 ------ tests/baselines/reference/user/reselect.log | 6 ------ tests/baselines/reference/user/should.log | 6 ------ tests/baselines/reference/user/sift.log | 6 ------ tests/baselines/reference/user/soap.log | 6 ------ tests/baselines/reference/user/sugar.log | 6 ------ tests/baselines/reference/user/tslint.log | 6 ------ tests/baselines/reference/user/vue.log | 6 ------ tests/baselines/reference/user/vuex.log | 6 ------ tests/baselines/reference/user/xlsx.log | 6 ------ tests/baselines/reference/user/xpath.log | 6 ------ tests/baselines/reference/user/zone.js.log | 6 ------ 35 files changed, 210 deletions(-) delete mode 100644 tests/baselines/reference/user/ajv.log delete mode 100644 tests/baselines/reference/user/antd.log delete mode 100644 tests/baselines/reference/user/axios.log delete mode 100644 tests/baselines/reference/user/bignumber.js.log delete mode 100644 tests/baselines/reference/user/discord.js.log delete mode 100644 tests/baselines/reference/user/eventemitter2.log delete mode 100644 tests/baselines/reference/user/eventemitter3.log delete mode 100644 tests/baselines/reference/user/firebase.log delete mode 100644 tests/baselines/reference/user/github.log delete mode 100644 tests/baselines/reference/user/immutable.log delete mode 100644 tests/baselines/reference/user/isobject.log delete mode 100644 tests/baselines/reference/user/jimp.log delete mode 100644 tests/baselines/reference/user/jsonschema.log delete mode 100644 tests/baselines/reference/user/keycode.log delete mode 100644 tests/baselines/reference/user/localforage.log delete mode 100644 tests/baselines/reference/user/log4js.log delete mode 100644 tests/baselines/reference/user/mobx.log delete mode 100644 tests/baselines/reference/user/moment.log delete mode 100644 tests/baselines/reference/user/mqtt.log delete mode 100644 tests/baselines/reference/user/parse5.log delete mode 100644 tests/baselines/reference/user/portfinder.log delete mode 100644 tests/baselines/reference/user/postcss.log delete mode 100644 tests/baselines/reference/user/protobufjs.log delete mode 100644 tests/baselines/reference/user/redux.log delete mode 100644 tests/baselines/reference/user/reselect.log delete mode 100644 tests/baselines/reference/user/should.log delete mode 100644 tests/baselines/reference/user/sift.log delete mode 100644 tests/baselines/reference/user/soap.log delete mode 100644 tests/baselines/reference/user/sugar.log delete mode 100644 tests/baselines/reference/user/tslint.log delete mode 100644 tests/baselines/reference/user/vue.log delete mode 100644 tests/baselines/reference/user/vuex.log delete mode 100644 tests/baselines/reference/user/xlsx.log delete mode 100644 tests/baselines/reference/user/xpath.log delete mode 100644 tests/baselines/reference/user/zone.js.log diff --git a/tests/baselines/reference/user/ajv.log b/tests/baselines/reference/user/ajv.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/ajv.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/antd.log b/tests/baselines/reference/user/antd.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/antd.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/axios.log b/tests/baselines/reference/user/axios.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/axios.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/bignumber.js.log b/tests/baselines/reference/user/bignumber.js.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/bignumber.js.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/discord.js.log b/tests/baselines/reference/user/discord.js.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/discord.js.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/eventemitter2.log b/tests/baselines/reference/user/eventemitter2.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/eventemitter2.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/eventemitter3.log b/tests/baselines/reference/user/eventemitter3.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/eventemitter3.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/firebase.log b/tests/baselines/reference/user/firebase.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/firebase.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/github.log b/tests/baselines/reference/user/github.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/github.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/immutable.log b/tests/baselines/reference/user/immutable.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/immutable.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/isobject.log b/tests/baselines/reference/user/isobject.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/isobject.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/jimp.log b/tests/baselines/reference/user/jimp.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/jimp.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/jsonschema.log b/tests/baselines/reference/user/jsonschema.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/jsonschema.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/keycode.log b/tests/baselines/reference/user/keycode.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/keycode.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/localforage.log b/tests/baselines/reference/user/localforage.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/localforage.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/log4js.log b/tests/baselines/reference/user/log4js.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/log4js.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/mobx.log b/tests/baselines/reference/user/mobx.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/mobx.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/moment.log b/tests/baselines/reference/user/moment.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/moment.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/mqtt.log b/tests/baselines/reference/user/mqtt.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/mqtt.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/parse5.log b/tests/baselines/reference/user/parse5.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/parse5.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/portfinder.log b/tests/baselines/reference/user/portfinder.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/portfinder.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/postcss.log b/tests/baselines/reference/user/postcss.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/postcss.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/protobufjs.log b/tests/baselines/reference/user/protobufjs.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/protobufjs.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/redux.log b/tests/baselines/reference/user/redux.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/redux.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/reselect.log b/tests/baselines/reference/user/reselect.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/reselect.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/should.log b/tests/baselines/reference/user/should.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/should.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/sift.log b/tests/baselines/reference/user/sift.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/sift.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/soap.log b/tests/baselines/reference/user/soap.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/soap.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/sugar.log b/tests/baselines/reference/user/sugar.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/sugar.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/tslint.log b/tests/baselines/reference/user/tslint.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/tslint.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/vue.log b/tests/baselines/reference/user/vue.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/vue.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/vuex.log b/tests/baselines/reference/user/vuex.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/vuex.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/xlsx.log b/tests/baselines/reference/user/xlsx.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/xlsx.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/xpath.log b/tests/baselines/reference/user/xpath.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/xpath.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: diff --git a/tests/baselines/reference/user/zone.js.log b/tests/baselines/reference/user/zone.js.log deleted file mode 100644 index 15b10503c1f77..0000000000000 --- a/tests/baselines/reference/user/zone.js.log +++ /dev/null @@ -1,6 +0,0 @@ -Exit Code: 0 -Standard output: - - - -Standard error: From bb79308a2446266a3bff299d569620613cbabec7 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Wed, 8 Nov 2017 15:26:05 -0800 Subject: [PATCH 9/9] Use CRLF and emit test output for stdout/stderr I've got git problems and I'm not even on Windows! --- src/harness/externalCompileRunner.ts | 134 +++++++++++++-------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts index 8ca0be807bfbf..9b8fc8c4fc59e 100644 --- a/src/harness/externalCompileRunner.ts +++ b/src/harness/externalCompileRunner.ts @@ -1,67 +1,67 @@ -/// -/// -abstract class ExternalCompileRunnerBase extends RunnerBase { - abstract testDir: string; - public enumerateTestFiles() { - return Harness.IO.getDirectories(this.testDir); - } - /** Setup the runner's tests so that they are ready to be executed by the harness - * The first test should be a describe/it block that sets up the harness's compiler instance appropriately - */ - public initializeTests(): void { - // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); - - describe(`${this.kind()} code samples`, () => { - for (const test of testList) { - this.runTest(test); - } - }); - } - private runTest(directoryName: string) { - describe(directoryName, () => { - const cp = require("child_process"); - const path = require("path"); - const fs = require("fs"); - - it("should build successfully", () => { - const cwd = path.join(__dirname, "../../", this.testDir, directoryName); - const timeout = 600000; // 600s = 10 minutes - if (fs.existsSync(path.join(cwd, "package.json"))) { - if (fs.existsSync(path.join(cwd, "package-lock.json"))) { - fs.unlinkSync(path.join(cwd, "package-lock.json")); - } - const stdio = isWorker ? "pipe" : "inherit"; - const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); - if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); - } - Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); - // tslint:disable-next-line:no-null-keyword - return result.status === 0 ? null : `Exit Code: ${result.status} -Standard output: -${result.stdout.toString().replace(/\r\n/g, "\n")} - - -Standard error: -${result.stderr.toString().replace(/\r\n/g, "\n")}`; - }); - }); - }); - } -} - -class UserCodeRunner extends ExternalCompileRunnerBase { - public readonly testDir = "tests/cases/user/"; - public kind(): TestRunnerKind { - return "user"; - } -} - -class DefinitelyTypedRunner extends ExternalCompileRunnerBase { - public readonly testDir = "../DefinitelyTyped/types/"; - public workingDirectory = this.testDir; - public kind(): TestRunnerKind { - return "dt"; - } -} +/// +/// +abstract class ExternalCompileRunnerBase extends RunnerBase { + abstract testDir: string; + public enumerateTestFiles() { + return Harness.IO.getDirectories(this.testDir); + } + /** Setup the runner's tests so that they are ready to be executed by the harness + * The first test should be a describe/it block that sets up the harness's compiler instance appropriately + */ + public initializeTests(): void { + // Read in and evaluate the test list + const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); + + describe(`${this.kind()} code samples`, () => { + for (const test of testList) { + this.runTest(test); + } + }); + } + private runTest(directoryName: string) { + describe(directoryName, () => { + const cp = require("child_process"); + const path = require("path"); + const fs = require("fs"); + + it("should build successfully", () => { + const cwd = path.join(__dirname, "../../", this.testDir, directoryName); + const timeout = 600000; // 600s = 10 minutes + if (fs.existsSync(path.join(cwd, "package.json"))) { + if (fs.existsSync(path.join(cwd, "package-lock.json"))) { + fs.unlinkSync(path.join(cwd, "package-lock.json")); + } + const stdio = isWorker ? "pipe" : "inherit"; + const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio }); + if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); + } + Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { + const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); + // tslint:disable-next-line:no-null-keyword + return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} +Standard output: +${result.stdout.toString().replace(/\r\n/g, "\n")} + + +Standard error: +${result.stderr.toString().replace(/\r\n/g, "\n")}`; + }); + }); + }); + } +} + +class UserCodeRunner extends ExternalCompileRunnerBase { + public readonly testDir = "tests/cases/user/"; + public kind(): TestRunnerKind { + return "user"; + } +} + +class DefinitelyTypedRunner extends ExternalCompileRunnerBase { + public readonly testDir = "../DefinitelyTyped/types/"; + public workingDirectory = this.testDir; + public kind(): TestRunnerKind { + return "dt"; + } +}