Skip to content

Commit a41836b

Browse files
dgozmandgozman
authored
chore(testrunner): introduce TestRun (#1582)
1 parent 5499b18 commit a41836b

File tree

4 files changed

+383
-390
lines changed

4 files changed

+383
-390
lines changed

‎test/test.js

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,17 @@ const testRunner = new TestRunner({
4343
breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1,
4444
installCommonHelpers: false
4545
});
46-
testRunner.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
47-
testRunner.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip));
46+
testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true));
47+
testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
4848
testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
4949
testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
5050
testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
5151
testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
5252
testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
53-
testRunner.testAttribute('focus', t => t.setMode(t.Modes.Focus));
54-
testRunner.suiteAttribute('focus', s => s.setMode(s.Modes.Focus));
53+
testRunner.testAttribute('focus', t => t.setFocused(true));
54+
testRunner.suiteAttribute('focus', s => s.setFocused(true));
5555
testRunner.testAttribute('debug', t => {
56-
t.setMode(t.Modes.Focus);
56+
t.setFocused(true);
5757
t.setTimeout(100000000);
5858

5959
let session;

‎utils/testrunner/Reporter.js

Lines changed: 84 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -39,34 +39,23 @@ class Reporter {
3939
runner.on('testfinished', this._onTestFinished.bind(this));
4040
}
4141

42-
_onStarted(runnableTests) {
42+
_onStarted(testRuns) {
4343
this._testCounter = 0;
4444
this._timestamp = Date.now();
4545
const allTests = this._runner.tests();
46-
if (allTests.length === runnableTests.length) {
47-
console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
46+
if (!this._runner.hasFocusedTestsOrSuites()) {
47+
console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
4848
} else {
49-
console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
49+
console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
5050
console.log('');
51-
const focusedSuites = this._runner.focusedSuites().map(suite => ({
52-
id: suite.location().filePath + ':' + suite.location().lineNumber + ':' + suite.location().columnNumber,
53-
fullName: suite.fullName(),
54-
location: suite.location(),
55-
}));
56-
const focusedTests = this._runner.focusedTests().map(test => ({
57-
id: test.location().filePath + ':' + test.location().lineNumber + ':' + test.location().columnNumber,
58-
fullName: test.fullName(),
59-
location: test.location(),
60-
}));
61-
const focusedEntities = new Map([
62-
...focusedSuites.map(suite => ([suite.id, suite])),
63-
...focusedTests.map(test => ([test.id, test])),
64-
]);
65-
if (focusedEntities.size) {
51+
const focusedEntities = [
52+
...this._runner.suites().filter(suite => suite.focused()),
53+
...this._runner.tests().filter(test => test.focused()),
54+
];
55+
if (focusedEntities.length) {
6656
console.log('Focused Suites and Tests:');
67-
const entities = [...focusedEntities.values()];
68-
for (let i = 0; i < entities.length; ++i)
69-
console.log(` ${i + 1}) ${entities[i].fullName} (${formatLocation(entities[i].location)})`);
57+
for (let i = 0; i < focusedEntities.length; ++i)
58+
console.log(` ${i + 1}) ${focusedEntities[i].fullName()} (${formatLocation(focusedEntities[i].location())})`);
7059
console.log('');
7160
}
7261
}
@@ -80,168 +69,164 @@ class Reporter {
8069
}
8170

8271
for (let i = 0; i < result.errors.length; i++) {
83-
const { message, error, workerId, tests } = result.errors[i];
72+
const { message, error, runs } = result.errors[i];
8473
console.log(`\n${colors.magenta('NON-TEST ERROR #' + i)}: ${message}`);
8574
if (error && error.stack)
8675
console.log(padLines(error.stack, 2));
87-
const lastTests = tests.slice(tests.length - Math.min(10, tests.length));
88-
if (lastTests.length)
76+
const lastRuns = runs.slice(runs.length - Math.min(10, runs.length));
77+
if (lastRuns.length)
8978
console.log(`WORKER STATE`);
90-
for (let j = 0; j < lastTests.length; j++)
91-
this._printVerboseTestResult(j, lastTests[j], workerId);
79+
for (let j = 0; j < lastRuns.length; j++)
80+
this._printVerboseTestRunResult(j, lastRuns[j]);
9281
}
9382
console.log('');
9483
console.log('');
9584
}
9685

9786
_onFinished(result) {
98-
this._printTestResults();
87+
this._printTestResults(result);
9988
if (!result.ok())
10089
this._printFailedResult(result);
10190
process.exitCode = result.exitCode;
10291
}
10392

104-
_printTestResults() {
93+
_printTestResults(result) {
10594
// 2 newlines after completing all tests.
10695
console.log('\n');
10796

108-
const failedTests = this._runner.failedTests();
109-
if (this._summary && failedTests.length > 0) {
97+
const runs = result.runs;
98+
const failedRuns = runs.filter(run => run.isFailure());
99+
const executedRuns = runs.filter(run => run.result());
100+
const okRuns = runs.filter(run => run.ok());
101+
const skippedRuns = runs.filter(run => run.result() === 'skipped');
102+
const markedAsFailingRuns = runs.filter(run => run.result() === 'markedAsFailing');
103+
104+
if (this._summary && failedRuns.length > 0) {
110105
console.log('\nFailures:');
111-
for (let i = 0; i < failedTests.length; ++i) {
112-
const test = failedTests[i];
113-
this._printVerboseTestResult(i + 1, test);
106+
for (let i = 0; i < failedRuns.length; ++i) {
107+
this._printVerboseTestRunResult(i + 1, failedRuns[i]);
114108
console.log('');
115109
}
116110
}
117111

118-
const skippedTests = this._runner.skippedTests();
119-
const markedAsFailingTests = this._runner.markedAsFailingTests();
120-
if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) {
121-
if (markedAsFailingTests.length > 0) {
112+
if (this._showMarkedAsFailingTests && this._summary && markedAsFailingRuns.length) {
113+
if (markedAsFailingRuns.length > 0) {
122114
console.log('\nMarked as failing:');
123-
markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => {
124-
console.log(`${index + 1}) ${test.fullName()} (${formatLocation(test.location())})`);
115+
markedAsFailingRuns.slice(0, this._showMarkedAsFailingTests).forEach((testRun, index) => {
116+
console.log(`${index + 1}) ${testRun.test().fullName()} (${formatLocation(testRun.test().location())})`);
125117
});
126118
}
127-
if (this._showMarkedAsFailingTests < markedAsFailingTests.length) {
119+
if (this._showMarkedAsFailingTests < markedAsFailingRuns.length) {
128120
console.log('');
129-
console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
121+
console.log(`... and ${colors.yellow(markedAsFailingRuns.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
130122
}
131123
}
132124

133125
if (this._showSlowTests) {
134-
const slowTests = this._runner.passedTests().sort((a, b) => {
135-
const aDuration = a.endTimestamp - a.startTimestamp;
136-
const bDuration = b.endTimestamp - b.startTimestamp;
137-
return bDuration - aDuration;
138-
}).slice(0, this._showSlowTests);
126+
const slowRuns = okRuns.sort((a, b) => b.duration() - a.duration()).slice(0, this._showSlowTests);
139127
console.log(`\nSlowest tests:`);
140-
for (let i = 0; i < slowTests.length; ++i) {
141-
const test = slowTests[i];
142-
const duration = test.endTimestamp - test.startTimestamp;
143-
console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName()} (${formatLocation(test.location())})`);
128+
for (let i = 0; i < slowRuns.length; ++i) {
129+
const run = slowRuns[i];
130+
console.log(` (${i + 1}) ${colors.yellow((run.duration() / 1000) + 's')} - ${run.test().fullName()} (${formatLocation(run.test().location())})`);
144131
}
145132
}
146133

147-
const tests = this._runner.tests();
148-
const executedTests = tests.filter(test => test.result);
149-
const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length;
150134
let summaryText = '';
151-
if (failedTests.length || markedAsFailingTests.length) {
152-
const summary = [`ok - ${colors.green(okTestsLength)}`];
153-
if (failedTests.length)
154-
summary.push(`failed - ${colors.red(failedTests.length)}`);
155-
if (markedAsFailingTests.length)
156-
summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`);
157-
if (skippedTests.length)
158-
summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
135+
if (failedRuns.length || markedAsFailingRuns.length) {
136+
const summary = [`ok - ${colors.green(okRuns.length)}`];
137+
if (failedRuns.length)
138+
summary.push(`failed - ${colors.red(failedRuns.length)}`);
139+
if (markedAsFailingRuns.length)
140+
summary.push(`marked as failing - ${colors.yellow(markedAsFailingRuns.length)}`);
141+
if (skippedRuns.length)
142+
summary.push(`skipped - ${colors.yellow(skippedRuns.length)}`);
159143
summaryText = ` (${summary.join(', ')})`;
160144
}
161145

162-
console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`);
146+
console.log(`\nRan ${executedRuns.length}${summaryText} of ${runs.length} test${runs.length > 1 ? 's' : ''}`);
163147
const milliseconds = Date.now() - this._timestamp;
164148
const seconds = milliseconds / 1000;
165149
console.log(`Finished in ${colors.yellow(seconds)} seconds`);
166150
}
167151

168-
_onTestStarted(test, workerId) {
152+
_onTestStarted(testRun) {
169153
}
170154

171-
_onTestFinished(test, workerId) {
155+
_onTestFinished(testRun) {
172156
if (this._verbose) {
173157
++this._testCounter;
174-
this._printVerboseTestResult(this._testCounter, test, workerId);
158+
this._printVerboseTestRunResult(this._testCounter, testRun);
175159
} else {
176-
if (test.result === 'ok')
160+
if (testRun.result() === 'ok')
177161
process.stdout.write(colors.green('\u00B7'));
178-
else if (test.result === 'skipped')
162+
else if (testRun.result() === 'skipped')
179163
process.stdout.write(colors.yellow('\u00B7'));
180-
else if (test.result === 'markedAsFailing')
164+
else if (testRun.result() === 'markedAsFailing')
181165
process.stdout.write(colors.yellow('\u00D7'));
182-
else if (test.result === 'failed')
166+
else if (testRun.result() === 'failed')
183167
process.stdout.write(colors.red('F'));
184-
else if (test.result === 'crashed')
168+
else if (testRun.result() === 'crashed')
185169
process.stdout.write(colors.red('C'));
186-
else if (test.result === 'terminated')
170+
else if (testRun.result() === 'terminated')
187171
process.stdout.write(colors.magenta('.'));
188-
else if (test.result === 'timedout')
172+
else if (testRun.result() === 'timedout')
189173
process.stdout.write(colors.red('T'));
190174
}
191175
}
192176

193-
_printVerboseTestResult(resultIndex, test, workerId = undefined) {
177+
_printVerboseTestRunResult(resultIndex, testRun) {
178+
const test = testRun.test();
194179
let prefix = `${resultIndex})`;
195-
if (this._runner.parallel() > 1 && workerId !== undefined)
196-
prefix += ' ' + colors.gray(`[worker = ${workerId}]`);
197-
if (test.result === 'ok') {
180+
if (this._runner.parallel() > 1)
181+
prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`);
182+
if (testRun.result() === 'ok') {
198183
console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`);
199-
} else if (test.result === 'terminated') {
184+
} else if (testRun.result() === 'terminated') {
200185
console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`);
201-
} else if (test.result === 'crashed') {
186+
} else if (testRun.result() === 'crashed') {
202187
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`);
203-
} else if (test.result === 'skipped') {
204-
} else if (test.result === 'markedAsFailing') {
188+
} else if (testRun.result() === 'skipped') {
189+
} else if (testRun.result() === 'markedAsFailing') {
205190
console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`);
206-
} else if (test.result === 'timedout') {
191+
} else if (testRun.result() === 'timedout') {
207192
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`);
208-
if (test.output) {
193+
if (testRun.output) {
209194
console.log(' Output:');
210-
for (const line of test.output)
195+
for (const line of testRun.output)
211196
console.log(' ' + line);
212197
}
213-
} else if (test.result === 'failed') {
198+
} else if (testRun.result() === 'failed') {
214199
console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`);
215-
if (test.error instanceof MatchError) {
216-
let lines = this._filePathToLines.get(test.error.location.filePath);
200+
if (testRun.error() instanceof MatchError) {
201+
let lines = this._filePathToLines.get(testRun.error().location.filePath);
217202
if (!lines) {
218203
try {
219-
lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n');
204+
lines = fs.readFileSync(testRun.error().location.filePath, 'utf8').split('\n');
220205
} catch (e) {
221206
lines = [];
222207
}
223-
this._filePathToLines.set(test.error.location.filePath, lines);
208+
this._filePathToLines.set(testRun.error().location.filePath, lines);
224209
}
225-
const lineNumber = test.error.location.lineNumber;
210+
const lineNumber = testRun.error().location.lineNumber;
226211
if (lineNumber < lines.length) {
227212
const lineNumberLength = (lineNumber + 1 + '').length;
228213
const FROM = Math.max(test.location().lineNumber - 1, lineNumber - 5);
229214
const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n');
230-
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^';
215+
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(testRun.error().location.columnNumber - 1) + '^';
231216
console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n');
232217
}
233-
console.log(padLines(test.error.formatter(), 4));
218+
console.log(padLines(testRun.error().formatter(), 4));
234219
console.log('');
235220
} else {
236221
console.log(' Message:');
237-
let message = '' + (test.error.message || test.error);
238-
if (test.error.stack && message.includes(test.error.stack))
239-
message = message.substring(0, message.indexOf(test.error.stack));
222+
let message = '' + (testRun.error().message || testRun.error());
223+
if (testRun.error().stack && message.includes(testRun.error().stack))
224+
message = message.substring(0, message.indexOf(testRun.error().stack));
240225
if (message)
241226
console.log(` ${colors.red(message)}`);
242-
if (test.error.stack) {
227+
if (testRun.error().stack) {
243228
console.log(' Stack:');
244-
let stack = test.error.stack;
229+
let stack = testRun.error().stack;
245230
// Highlight first test location, if any.
246231
const match = stack.match(new RegExp(test.location().filePath + ':(\\d+):(\\d+)'));
247232
if (match) {
@@ -252,9 +237,9 @@ class Reporter {
252237
console.log(padLines(stack, 4));
253238
}
254239
}
255-
if (test.output) {
240+
if (testRun.output) {
256241
console.log(' Output:');
257-
for (const line of test.output)
242+
for (const line of testRun.output)
258243
console.log(' ' + line);
259244
}
260245
}

0 commit comments

Comments
 (0)