Skip to content

Commit 6877242

Browse files
author
Alexey Rybakov
committed
feat: save stacktraces
1 parent 1203e5e commit 6877242

File tree

11 files changed

+93
-90
lines changed

11 files changed

+93
-90
lines changed

‎hermione.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ module.exports = (hermione, opts) => {
1616
hermione.on(hermione.events.TEST_PASS, (data) => collector.addSuccess(data));
1717

1818
hermione.on(hermione.events.TEST_FAIL, (data) => collector.addFail(data));
19-
hermione.on(hermione.events.SUITE_FAIL, (data) => collector.addFail(data));
2019

2120
hermione.on(hermione.events.TEST_PENDING, (data) => collector.addSkipped(data));
2221

‎lib/collector/data-collector/gemini.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,6 @@ module.exports = class GeminiDataCollector extends DataCollector {
2323
test.duration = Date.now() - this._startTime[id];
2424
delete this._startTime[id];
2525

26-
this._data[id] = test;
26+
super.append(test);
2727
}
2828
};

‎lib/collector/data-collector/index.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22

33
const _ = require('lodash');
44

5+
function customMerger(objValue, srcValue) {
6+
if (_.isArray(objValue)) {
7+
return objValue.concat(srcValue);
8+
}
9+
}
10+
511
module.exports = class DataCollector {
612
static create() {
713
return new DataCollector();
@@ -14,6 +20,11 @@ module.exports = class DataCollector {
1420
append(test) {
1521
const id = this._generateId(test);
1622

23+
if (this._data[id]) {
24+
this._data[id] = _.mergeWith(this._data[id], test, customMerger);
25+
return;
26+
}
27+
1728
this._data[id] = test;
1829
}
1930

‎lib/collector/gemini.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,10 @@ module.exports = class GeminiCollector extends Collector {
1717
const test = this._toolCollector.configureTestResult(result);
1818
this._dataCollector.saveStartTime(test);
1919
}
20+
21+
addRetry(result) {
22+
this._toolCollector.isFailedTest(result)
23+
? this.addFail(result)
24+
: this.addError(result);
25+
}
2026
};

‎lib/collector/index.js

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,13 @@ module.exports = class Collector {
2222
}
2323

2424
addFail(result) {
25-
this._addTestResult(result, {status: 'fail'});
25+
const error = result.err.stack;
26+
27+
this._addTestResult(result, {
28+
status: 'fail',
29+
errorReason: error,
30+
retries: [{error}]
31+
});
2632
}
2733

2834
addSkipped(result) {
@@ -33,15 +39,16 @@ module.exports = class Collector {
3339
}
3440

3541
addRetry(result) {
36-
this._toolCollector.isFailedTest(result)
37-
? this.addFail(result)
38-
: this.addError(result);
42+
this.addFail(result);
3943
}
4044

4145
addError(result) {
46+
const error = result.stack || result.message || '';
47+
4248
this._addTestResult(result, {
4349
status: 'error',
44-
errorReason: result.stack || result.message || ''
50+
errorReason: error,
51+
retries: [{error}]
4552
});
4653
}
4754

‎lib/collector/tool/hermione.js

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@ exports.configureTestResult = (result) => {
2424
return testResult;
2525
};
2626

27-
exports.isFailedTest = (result) => {
28-
return result.state === 'failed' || result.hook && result.hook.state === 'failed';
29-
};
30-
3127
exports.getSkipReason = (result) => {
3228
const findSkipReason = (test) => test.parent && findSkipReason(test.parent) || test.skipReason;
3329

‎test/hermione.js

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -90,15 +90,13 @@ describe('json-reporter/hermione', () => {
9090
assert.calledOnceWith(Collector.prototype.addSuccess, data);
9191
});
9292

93-
['TEST_FAIL', 'SUITE_FAIL'].forEach((eventName) => {
94-
it('should call appropriate method for failed test', () => {
95-
const data = {foo: 'bar'};
96-
sandbox.stub(Collector.prototype, 'addFail');
93+
it('should call appropriate method for failed test', () => {
94+
const data = {foo: 'bar'};
95+
sandbox.stub(Collector.prototype, 'addFail');
9796

98-
hermione.emit(hermione.events[eventName], data);
97+
hermione.emit(hermione.events.TEST_FAIL, data);
9998

100-
assert.calledOnceWith(Collector.prototype.addFail, data);
101-
});
99+
assert.calledOnceWith(Collector.prototype.addFail, data);
102100
});
103101

104102
it('should call appropriate method for skipped test', () => {

‎test/lib/collector/data-collector/index.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,14 @@ describe('collector/data-collector/index', () => {
4949
});
5050
});
5151

52-
it('should replace the existing test in data collector', () => {
52+
it('should merge the existing test in data collector with new one', () => {
5353
const dataCollector = DataCollector.create({});
5454

55-
dataCollector.append({foo: 'bar'});
56-
dataCollector.append({foo: 'baz'});
55+
dataCollector.append({foo: 'bar', arr: [1]});
56+
dataCollector.append({foo: 'baz', arr: [2]});
5757

5858
assert.deepEqual(dataCollector.getData(), {
59-
'': {foo: 'baz'}
59+
'': {foo: 'baz', arr: [1, 2]}
6060
});
6161
});
6262
});

‎test/lib/collector/gemini.js

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,14 @@ describe('collector/gemini', () => {
2828
return GeminiCollector.create(toolCollector, config);
2929
};
3030

31+
const saveReport_ = (collector) => {
32+
return collector.saveFile()
33+
.then(() => fs.outputJsonAsync.firstCall.args[1]);
34+
};
35+
3136
beforeEach(() => {
3237
clock = sinon.useFakeTimers();
38+
sandbox.stub(fs, 'outputJsonAsync').returns(Promise.resolve());
3339
});
3440

3541
afterEach(() => {
@@ -65,15 +71,6 @@ describe('collector/gemini', () => {
6571
});
6672

6773
describe('should add "duration" time to the', () => {
68-
const saveReport_ = (collector) => {
69-
return collector.saveFile()
70-
.then(() => fs.outputJsonAsync.firstCall.args[1]);
71-
};
72-
73-
beforeEach(() => {
74-
sandbox.stub(fs, 'outputJsonAsync').returns(Promise.resolve());
75-
});
76-
7774
it('succesfully passed test', () => {
7875
const data = {fullName: 'some name', browserId: 'bro'};
7976
const collector = mkGeminiCollector_();
@@ -99,7 +96,7 @@ describe('collector/gemini', () => {
9996
});
10097

10198
it('failed test', () => {
102-
const data = {fullName: 'some name', browserId: 'bro'};
99+
const data = {fullName: 'some name', browserId: 'bro', err: new Error('test')};
103100
const collector = mkGeminiCollector_();
104101

105102
collector.markTestStart(data);
@@ -110,43 +107,64 @@ describe('collector/gemini', () => {
110107
});
111108
});
112109

113-
it('failed test if the retry fails', () => {
110+
it('errored test', () => {
114111
const data = {fullName: 'some name', browserId: 'bro'};
115-
const collector = mkGeminiCollector_({
116-
isFailedTest: sandbox.stub().returns(true)
117-
});
112+
const collector = mkGeminiCollector_();
118113

119114
collector.markTestStart(data);
120-
collector.addRetry(data);
115+
collector.addError(data);
121116

122117
return saveReport_(collector).then((result) => {
123118
assert.propertyVal(result['some name.bro'], 'duration', 0);
124119
});
125120
});
126121

127-
it('errored test', () => {
122+
it('errored test if the retry does not fail', () => {
128123
const data = {fullName: 'some name', browserId: 'bro'};
129-
const collector = mkGeminiCollector_();
124+
const collector = mkGeminiCollector_({
125+
isFailedTest: sandbox.stub().returns(false)
126+
});
130127

131128
collector.markTestStart(data);
132-
collector.addError(data);
129+
collector.addRetry(data);
133130

134131
return saveReport_(collector).then((result) => {
135132
assert.propertyVal(result['some name.bro'], 'duration', 0);
136133
});
137134
});
135+
});
136+
137+
describe('should add on retry', () => {
138+
it('failed test if the retry fails', () => {
139+
const testError = new Error('test');
140+
const data = {fullName: 'some name', browserId: 'bro', err: testError};
141+
const collector = mkGeminiCollector_({
142+
isFailedTest: sandbox.stub().returns(true)
143+
});
138144

139-
it('errored test if the retry does not fails', () => {
145+
collector.addRetry(data);
146+
147+
return saveReport_(collector).then((result) => {
148+
const bro = result['some name.bro'];
149+
assert.propertyVal(bro, 'status', 'fail');
150+
assert.propertyVal(bro, 'errorReason', testError.stack);
151+
assert.deepEqual(bro.retries, [{error: testError.stack}]);
152+
});
153+
});
154+
155+
it('errored test if the retry does not fail', () => {
140156
const data = {fullName: 'some name', browserId: 'bro'};
141157
const collector = mkGeminiCollector_({
142158
isFailedTest: sandbox.stub().returns(false)
143159
});
144160

145-
collector.markTestStart(data);
146161
collector.addRetry(data);
147162

148163
return saveReport_(collector).then((result) => {
149-
assert.propertyVal(result['some name.bro'], 'duration', 0);
164+
const bro = result['some name.bro'];
165+
assert.propertyVal(bro, 'status', 'error');
166+
assert.propertyVal(bro, 'errorReason', '');
167+
assert.deepEqual(bro.retries, [{error: ''}]);
150168
});
151169
});
152170
});

‎test/lib/collector/index.js

Lines changed: 15 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@ describe('collector/index', () => {
1212

1313
const mkToolCollector_ = (opts) => {
1414
return _.defaults(opts || {}, {
15-
configureTestResult: (data) => data,
16-
isFailedTest: sandbox.stub(),
15+
configureTestResult: (data) => {
16+
delete data.err;
17+
return data;
18+
},
1719
getSkipReason: sandbox.stub()
1820
});
1921
};
@@ -70,7 +72,8 @@ describe('collector/index', () => {
7072
});
7173

7274
it('should add failed test', () => {
73-
const data = {fullName: 'some name', browserId: 'bro'};
75+
const testError = new Promise.OperationalError('test');
76+
const data = {fullName: 'some name', browserId: 'bro', err: testError};
7477
const collector = mkCollector_();
7578

7679
collector.addFail(data);
@@ -79,7 +82,9 @@ describe('collector/index', () => {
7982
assert.deepEqual(result, {'some name.bro': {
8083
fullName: 'some name',
8184
browserId: 'bro',
82-
status: 'fail'
85+
status: 'fail',
86+
errorReason: testError.stack,
87+
retries: [{error: testError.stack}]
8388
}});
8489
});
8590
});
@@ -103,36 +108,19 @@ describe('collector/index', () => {
103108
});
104109

105110
it('should add failed test if the retry fails', () => {
106-
const data = {fullName: 'some name', browserId: 'bro'};
107-
const collector = mkCollector_({
108-
isFailedTest: sandbox.stub().returns(true)
109-
});
110-
111-
collector.addRetry(data);
112-
113-
return saveReport_(collector).then((result) => {
114-
assert.deepEqual(result, {'some name.bro': {
115-
fullName: 'some name',
116-
browserId: 'bro',
117-
status: 'fail'
118-
}});
119-
});
120-
});
121-
122-
it('should add errored test if the retry does not fails', () => {
123-
const data = {fullName: 'some name', browserId: 'bro'};
124-
const collector = mkCollector_({
125-
isFailedTest: sandbox.stub().returns(false)
126-
});
111+
const testError = new Promise.OperationalError('test');
112+
const data = {fullName: 'some name', browserId: 'bro', err: testError};
113+
const collector = mkCollector_();
127114

128115
collector.addRetry(data);
129116

130117
return saveReport_(collector).then((result) => {
131118
assert.deepEqual(result, {'some name.bro': {
132119
fullName: 'some name',
133120
browserId: 'bro',
134-
status: 'error',
135-
errorReason: ''
121+
status: 'fail',
122+
errorReason: testError.stack,
123+
retries: [{error: testError.stack}]
136124
}});
137125
});
138126
});

‎test/lib/collector/tool/hermione.js

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -106,26 +106,6 @@ describe('collector/tool/hermione', () => {
106106
});
107107
});
108108

109-
describe('isFailedTest', () => {
110-
it('should return "true" if test is failed', () => {
111-
assert.isTrue(hermioneToolCollector.isFailedTest({state: 'failed'}));
112-
});
113-
114-
it('should return "true" if hook is failed', () => {
115-
const result = {hook: {
116-
state: 'failed'
117-
}};
118-
119-
assert.isTrue(hermioneToolCollector.isFailedTest(result));
120-
});
121-
122-
it('should return "false" if test and hook are not failed', () => {
123-
const result = {hook: {}};
124-
125-
assert.isFalse(hermioneToolCollector.isFailedTest(result));
126-
});
127-
});
128-
129109
describe('getSkipReason', () => {
130110
it('should return default skip reason if "skipReason" is not specified', () => {
131111
assert.strictEqual(hermioneToolCollector.getSkipReason({}), 'No skip reason');

0 commit comments

Comments
 (0)