Skip to content

Commit 0944c29

Browse files
rbucktonrbuckton
committed
Adds failed test tracking
1 parent 6c8ecc7 commit 0944c29

File tree

8 files changed

+305
-25
lines changed

8 files changed

+305
-25
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts
7373
!tests/cases/user/zone.js/
7474
!tests/cases/user/bignumber.js/
7575
!tests/cases/user/discord.js/
76-
tests/baselines/reference/dt
76+
tests/baselines/reference/dt
77+
.failed-tests

‎Gulpfile.js

+17-9
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ const constEnumCaptureRegexp = /^(\s*)(export )?const enum (\S+) {(\s*)$/gm;
3838
const constEnumReplacement = "$1$2enum $3 {$4";
3939

4040
const cmdLineOptions = minimist(process.argv.slice(2), {
41-
boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"],
41+
boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"],
4242
string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
4343
alias: {
4444
"b": "browser",
@@ -598,7 +598,6 @@ gulp.task("LKG", "Makes a new LKG out of the built js files", ["clean", "dontUse
598598
return seq;
599599
});
600600

601-
602601
// Task to build the tests infrastructure using the built compiler
603602
const run = path.join(builtLocalDirectory, "run.js");
604603
gulp.task(run, /*help*/ false, [servicesFile, tsserverLibraryFile], () => {
@@ -653,10 +652,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
653652
let testTimeout = cmdLineOptions.timeout;
654653
const debug = cmdLineOptions.debug;
655654
const inspect = cmdLineOptions.inspect;
656-
const tests = cmdLineOptions.tests;
655+
let tests = cmdLineOptions.tests;
657656
const runners = cmdLineOptions.runners;
658657
const light = cmdLineOptions.light;
659658
const stackTraceLimit = cmdLineOptions.stackTraceLimit;
659+
const failed = cmdLineOptions.failed;
660+
const keepFailed = cmdLineOptions.keepFailed || failed;
660661
const testConfigFile = "test.config";
661662
if (fs.existsSync(testConfigFile)) {
662663
fs.unlinkSync(testConfigFile);
@@ -679,8 +680,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
679680
testTimeout = 400000;
680681
}
681682

682-
if (tests || runners || light || testTimeout || taskConfigsFolder) {
683-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
683+
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
684+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
684685
}
685686

686687
const colors = cmdLineOptions.colors;
@@ -690,7 +691,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
690691
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
691692
if (!runInParallel) {
692693
const args = [];
693-
args.push("-R", reporter);
694+
args.push("-R", "scripts/failed-tests");
695+
args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
694696
if (tests) {
695697
args.push("-g", `"${tests}"`);
696698
}
@@ -711,8 +713,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) {
711713
}
712714
args.push(run);
713715
setNodeEnvToDevelopment();
714-
exec(mocha, args, lintThenFinish, finish);
715-
716+
if (failed) {
717+
exec(host, ["scripts/run-failed-tests.js"].concat(args), lintThenFinish, finish);
718+
}
719+
else {
720+
exec(mocha, args, lintThenFinish, finish);
721+
}
716722
}
717723
else {
718724
// run task to load all tests and partition them between workers
@@ -888,8 +894,9 @@ function cleanTestDirs(done) {
888894
* @param {number=} workerCount
889895
* @param {string=} stackTraceLimit
890896
* @param {number=} timeout
897+
* @param {boolean=} keepFailed
891898
*/
892-
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) {
899+
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
893900
const testConfigContents = JSON.stringify({
894901
test: tests ? [tests] : undefined,
895902
runner: runners ? runners.split(",") : undefined,
@@ -899,6 +906,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
899906
taskConfigsFolder,
900907
noColor: !cmdLineOptions.colors,
901908
timeout,
909+
keepFailed
902910
});
903911
console.log("Running tests with config: " + testConfigContents);
904912
fs.writeFileSync("test.config", testConfigContents);

‎Jakefile.js

+17-6
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
388388
const runners = process.env.runners || process.env.runner || process.env.ru;
389389
const tests = process.env.test || process.env.tests || process.env.t;
390390
const light = process.env.light === undefined || process.env.light !== "false";
391+
const failed = process.env.failed;
392+
const keepFailed = process.env.keepFailed || failed;
391393
const stackTraceLimit = process.env.stackTraceLimit;
392394
const colorsFlag = process.env.color || process.env.colors;
393395
const colors = colorsFlag !== "false" && colorsFlag !== "0";
@@ -418,16 +420,17 @@ function runConsoleTests(defaultReporter, runInParallel) {
418420
testTimeout = 800000;
419421
}
420422

421-
if (tests || runners || light || testTimeout || taskConfigsFolder) {
422-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout);
423+
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
424+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed);
423425
}
424426

425427
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
426428
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
427429
if (!runInParallel) {
428430
var startTime = Travis.mark();
429431
var args = [];
430-
args.push("-R", reporter);
432+
args.push("-R", "scripts/failed-tests");
433+
args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
431434
if (tests) args.push("-g", `"${tests}"`);
432435
args.push(colors ? "--colors" : "--no-colors");
433436
if (bail) args.push("--bail");
@@ -438,7 +441,14 @@ function runConsoleTests(defaultReporter, runInParallel) {
438441
}
439442
args.push(Paths.builtLocalRun);
440443

441-
var cmd = "mocha " + args.join(" ");
444+
var cmd;
445+
if (failed) {
446+
args.unshift("scripts/run-failed-tests.js");
447+
cmd = host + " " + args.join(" ");
448+
}
449+
else {
450+
cmd = "mocha " + args.join(" ");
451+
}
442452
var savedNodeEnv = process.env.NODE_ENV;
443453
process.env.NODE_ENV = "development";
444454
exec(cmd, function () {
@@ -499,7 +509,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
499509
}
500510

501511
// used to pass data from jake command line directly to run.js
502-
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) {
512+
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) {
503513
var testConfigContents = JSON.stringify({
504514
runners: runners ? runners.split(",") : undefined,
505515
test: tests ? [tests] : undefined,
@@ -508,7 +518,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
508518
taskConfigsFolder: taskConfigsFolder,
509519
stackTraceLimit: stackTraceLimit,
510520
noColor: !colors,
511-
timeout: testTimeout
521+
timeout: testTimeout,
522+
keepFailed: keepFailed
512523
});
513524
fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" });
514525
}

‎scripts/failed-tests.d.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import Mocha = require("mocha");
2+
3+
export = FailedTestsReporter;
4+
5+
declare class FailedTestsReporter extends Mocha.reporters.Base {
6+
passes: Mocha.Test[];
7+
failures: Mocha.Test[];
8+
reporterOptions: FailedTestsReporter.ReporterOptions;
9+
reporter?: Mocha.reporters.Base;
10+
constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions });
11+
static writeFailures(file: string, passes: ReadonlyArray<Mocha.Test>, failures: ReadonlyArray<Mocha.Test>, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void;
12+
done(failures: number, fn?: (failures: number) => void): void;
13+
}
14+
15+
declare namespace FailedTestsReporter {
16+
interface ReporterOptions {
17+
file?: string;
18+
keepFailed?: boolean;
19+
reporter?: string | Mocha.ReporterConstructor;
20+
reporterOptions?: any;
21+
}
22+
}

‎scripts/failed-tests.js

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// @ts-check
2+
const Mocha = require("mocha");
3+
const path = require("path");
4+
const fs = require("fs");
5+
const os = require("os");
6+
7+
/**
8+
* .failed-tests reporter
9+
*
10+
* @typedef {Object} ReporterOptions
11+
* @property {string} [file]
12+
* @property {boolean} [keepFailed]
13+
* @property {string|Mocha.ReporterConstructor} [reporter]
14+
* @property {*} [reporterOptions]
15+
*/
16+
class FailedTestsReporter extends Mocha.reporters.Base {
17+
/**
18+
* @param {Mocha.Runner} runner
19+
* @param {{ reporterOptions?: ReporterOptions }} [options]
20+
*/
21+
constructor(runner, options) {
22+
super(runner, options);
23+
if (!runner) return;
24+
25+
const reporterOptions = this.reporterOptions = options.reporterOptions || {};
26+
if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
27+
if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
28+
if (reporterOptions.reporter) {
29+
/** @type {Mocha.ReporterConstructor} */
30+
let reporter;
31+
if (typeof reporterOptions.reporter === "function") {
32+
reporter = reporterOptions.reporter;
33+
}
34+
else if (Mocha.reporters[reporterOptions.reporter]) {
35+
reporter = Mocha.reporters[reporterOptions.reporter];
36+
}
37+
else {
38+
try {
39+
reporter = require(reporterOptions.reporter);
40+
}
41+
catch (_) {
42+
reporter = require(path.resolve(process.cwd(), reporterOptions.reporter));
43+
}
44+
}
45+
46+
const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} });
47+
this.reporter = new reporter(runner, newOptions);
48+
}
49+
50+
/** @type {Mocha.Test[]} */
51+
this.passes = [];
52+
53+
/** @type {Mocha.Test[]} */
54+
this.failures = [];
55+
56+
runner.on("pass", test => this.passes.push(test));
57+
runner.on("fail", test => this.failures.push(test));
58+
}
59+
60+
/**
61+
* @param {string} file
62+
* @param {ReadonlyArray<Mocha.Test>} passes
63+
* @param {ReadonlyArray<Mocha.Test>} failures
64+
* @param {boolean} keepFailed
65+
* @param {(err?: NodeJS.ErrnoException) => void} done
66+
*/
67+
static writeFailures(file, passes, failures, keepFailed, done) {
68+
const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
69+
if (failingTests.size > 0) {
70+
for (const test of passes) {
71+
const title = test.fullTitle().trim();
72+
if (title) failingTests.delete(title);
73+
}
74+
}
75+
for (const test of failures) {
76+
const title = test.fullTitle().trim();
77+
if (title) failingTests.add(title);
78+
}
79+
if (failingTests.size > 0) {
80+
const failed = Array.from(failingTests).join(os.EOL);
81+
fs.writeFile(file, failed, "utf8", done);
82+
}
83+
else if (!keepFailed) {
84+
fs.unlink(file, done);
85+
}
86+
else {
87+
done();
88+
}
89+
90+
function readTests() {
91+
return fs.readFileSync(file, "utf8")
92+
.split(/\r?\n/g)
93+
.map(line => line.trim())
94+
.filter(line => line.length > 0);
95+
}
96+
}
97+
98+
/**
99+
* @param {number} failures
100+
* @param {(failures: number) => void} [fn]
101+
*/
102+
done(failures, fn) {
103+
FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => {
104+
const reporter = this.reporter;
105+
if (reporter && reporter.done) {
106+
reporter.done(failures, fn);
107+
}
108+
else if (fn) {
109+
fn(failures);
110+
}
111+
112+
if (err) console.error(err);
113+
});
114+
}
115+
}
116+
117+
module.exports = FailedTestsReporter;

‎scripts/run-failed-tests.js

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
const spawn = require('child_process').spawn;
2+
const os = require("os");
3+
const fs = require("fs");
4+
const path = require("path");
5+
6+
let grep;
7+
try {
8+
const failedTests = fs.readFileSync(".failed-tests", "utf8");
9+
grep = failedTests
10+
.split(/\r?\n/g)
11+
.map(test => test.trim())
12+
.filter(test => test.length > 0)
13+
.map(escapeRegExp);
14+
}
15+
catch (e) {
16+
grep = [];
17+
}
18+
19+
let args = [];
20+
let waitForGrepValue = false;
21+
let grepIndex = -1;
22+
process.argv.slice(2).forEach((arg, index) => {
23+
const [flag, value] = arg.split('=');
24+
if (flag === "g" || flag === "grep") {
25+
grepIndex = index - 1;
26+
waitForGrepValue = arg !== flag;
27+
if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, ""));
28+
return;
29+
}
30+
if (waitForGrepValue) {
31+
grep.push(arg.replace(/^"|"$/g, ""));
32+
waitForGrepValue = false;
33+
return;
34+
}
35+
args.push(arg);
36+
});
37+
38+
let mocha = "./node_modules/mocha/bin/mocha";
39+
let grepOption;
40+
let grepOptionValue;
41+
let grepFile;
42+
if (grep.length) {
43+
grepOption = "--grep";
44+
grepOptionValue = grep.join("|");
45+
if (grepOptionValue.length > 20) {
46+
grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts");
47+
fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8");
48+
grepOption = "--opts";
49+
grepOptionValue = grepFile;
50+
mocha = "./node_modules/mocha/bin/_mocha";
51+
}
52+
}
53+
54+
if (grepOption) {
55+
if (grepIndex >= 0) {
56+
args.splice(grepIndex, 0, grepOption, grepOptionValue);
57+
}
58+
else {
59+
args.push(grepOption, grepOptionValue);
60+
}
61+
}
62+
63+
args.unshift(path.resolve(mocha));
64+
65+
console.log(args.join(" "));
66+
const proc = spawn(process.execPath, args, {
67+
stdio: 'inherit'
68+
});
69+
proc.on('exit', (code, signal) => {
70+
process.on('exit', () => {
71+
if (grepFile) {
72+
fs.unlinkSync(grepFile);
73+
}
74+
75+
if (signal) {
76+
process.kill(process.pid, signal);
77+
} else {
78+
process.exit(code);
79+
}
80+
});
81+
});
82+
83+
// terminate children.
84+
process.on('SIGINT', () => {
85+
proc.kill('SIGINT'); // calls runner.abort()
86+
proc.kill('SIGTERM'); // if that didn't work, we're probably in an infinite loop, so make it die.
87+
});
88+
89+
function escapeRegExp(pattern) {
90+
return pattern
91+
.replace(/[^-\w\d\s]/g, match => "\\" + match)
92+
.replace(/\s/g, "\\s");
93+
}

0 commit comments

Comments
 (0)