Skip to content

Commit 7713242

Browse files
shreyas-pandurangashreyas-panduranga
committed
Adds unit and integration tests for the option inroduced
1 parent fcecf5a commit 7713242

File tree

2 files changed

+404
-0
lines changed

2 files changed

+404
-0
lines changed

‎test/integration/runner-spec/stop-on-assertion-failure.test.js‎

Lines changed: 360 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,360 @@
1+
var expect = require('chai').expect;
2+
3+
describe('Stop on assertion failure', function () {
4+
describe('with stopOnAssertionFailure: true', function () {
5+
describe('should stop on assertion failures', function () {
6+
var testrun;
7+
8+
before(function (done) {
9+
this.run({
10+
stopOnAssertionFailure: true,
11+
collection: {
12+
item: [{
13+
name: 'First Request',
14+
event: [{
15+
listen: 'test',
16+
script: {
17+
exec: [
18+
'pm.test("This test will pass", function () {',
19+
' pm.expect(pm.response.code).to.equal(200);',
20+
'});',
21+
'pm.test("This test will fail", function () {',
22+
' pm.expect(pm.response.code).to.equal(404);',
23+
'});'
24+
]
25+
}
26+
}],
27+
request: {
28+
url: 'https://postman-echo.com/get',
29+
method: 'GET'
30+
}
31+
}, {
32+
name: 'Second Request',
33+
event: [{
34+
listen: 'test',
35+
script: {
36+
exec: [
37+
'pm.test("This should not run", function () {',
38+
' pm.expect(pm.response.code).to.equal(200);',
39+
'});'
40+
]
41+
}
42+
}],
43+
request: {
44+
url: 'https://postman-echo.com/get',
45+
method: 'GET'
46+
}
47+
}]
48+
}
49+
}, function (err, results) {
50+
testrun = results;
51+
done(err);
52+
});
53+
});
54+
55+
it('should have completed the run', function () {
56+
expect(testrun).to.be.ok;
57+
expect(testrun.done.getCall(0).args[0]).to.be.null;
58+
expect(testrun).to.nested.include({
59+
'done.calledOnce': true,
60+
'start.calledOnce': true
61+
});
62+
});
63+
64+
it('should gracefully stop the run on assertion failures', function () {
65+
expect(testrun).to.nested.include({
66+
'item.calledOnce': true,
67+
'request.calledOnce': true,
68+
'test.calledOnce': true,
69+
'iteration.calledOnce': true
70+
});
71+
72+
// Second request should not have been executed
73+
expect(testrun.request.callCount).to.equal(1);
74+
});
75+
76+
it('should have assertion failures', function () {
77+
expect(testrun.assertion.callCount).to.be.at.least(2);
78+
79+
// Check if any assertion call has a failed assertion
80+
var hasFailedAssertion = testrun.assertion.args.some(function (args) {
81+
return args[1] && args[1].some(function (assertion) {
82+
return assertion && assertion.passed === false;
83+
});
84+
});
85+
86+
expect(hasFailedAssertion).to.be.true;
87+
});
88+
});
89+
90+
describe('should stop on legacy tests assertion failures', function () {
91+
var testrun;
92+
93+
before(function (done) {
94+
this.run({
95+
stopOnAssertionFailure: true,
96+
collection: {
97+
item: [{
98+
name: 'First Request',
99+
event: [{
100+
listen: 'test',
101+
script: {
102+
exec: [
103+
'tests["Body contains headers"] = responseBody.has("headers");',
104+
'tests["Body contains args"] = responseBody.has("args");',
105+
'tests["fail"] = false;' // This will cause an assertion failure
106+
]
107+
}
108+
}],
109+
request: {
110+
url: 'https://postman-echo.com/get',
111+
method: 'GET'
112+
}
113+
}, {
114+
name: 'Second Request',
115+
event: [{
116+
listen: 'test',
117+
script: {
118+
exec: [
119+
'tests["should not run"] = true;'
120+
]
121+
}
122+
}],
123+
request: {
124+
url: 'https://postman-echo.com/get',
125+
method: 'GET'
126+
}
127+
}]
128+
}
129+
}, function (err, results) {
130+
testrun = results;
131+
done(err);
132+
});
133+
});
134+
135+
it('should have completed the run', function () {
136+
expect(testrun).to.be.ok;
137+
expect(testrun.done.getCall(0).args[0]).to.be.null;
138+
expect(testrun).to.nested.include({
139+
'done.calledOnce': true,
140+
'start.calledOnce': true
141+
});
142+
});
143+
144+
it('should gracefully stop the run on assertion failures', function () {
145+
expect(testrun).to.nested.include({
146+
'item.calledOnce': true,
147+
'request.calledOnce': true,
148+
'test.calledOnce': true,
149+
'iteration.calledOnce': true
150+
});
151+
152+
// Second request should not have been executed
153+
expect(testrun.request.callCount).to.equal(1);
154+
});
155+
});
156+
157+
describe('should NOT stop on script execution errors', function () {
158+
var testrun;
159+
160+
before(function (done) {
161+
this.run({
162+
stopOnAssertionFailure: true,
163+
collection: {
164+
item: [{
165+
name: 'First Request',
166+
event: [{
167+
listen: 'test',
168+
script: {
169+
exec: [
170+
'console.log(foo);' // deliberate reference error
171+
]
172+
}
173+
}],
174+
request: {
175+
url: 'https://postman-echo.com/get',
176+
method: 'GET'
177+
}
178+
}, {
179+
name: 'Second Request',
180+
event: [{
181+
listen: 'test',
182+
script: {
183+
exec: [
184+
'pm.test("This should run", function () {',
185+
' pm.expect(pm.response.code).to.equal(200);',
186+
'});'
187+
]
188+
}
189+
}],
190+
request: {
191+
url: 'https://postman-echo.com/get',
192+
method: 'GET'
193+
}
194+
}]
195+
}
196+
}, function (err, results) {
197+
testrun = results;
198+
done(err);
199+
});
200+
});
201+
202+
it('should have completed the run', function () {
203+
expect(testrun).to.be.ok;
204+
expect(testrun.done.getCall(0).args[0]).to.be.null;
205+
expect(testrun).to.nested.include({
206+
'done.calledOnce': true,
207+
'start.calledOnce': true
208+
});
209+
});
210+
211+
it('should NOT stop the run on script execution errors', function () {
212+
// Both requests should have been executed
213+
expect(testrun.request.callCount).to.equal(2);
214+
expect(testrun.item.callCount).to.equal(2);
215+
});
216+
217+
it('should have script error in first request', function () {
218+
expect(testrun.script.getCall(0).args[0]).to.be.ok;
219+
expect(testrun.script.getCall(0).args[0]).to.have.property('message');
220+
});
221+
});
222+
223+
describe('should stop on assertion failures with multiple iterations', function () {
224+
var testrun;
225+
226+
before(function (done) {
227+
this.run({
228+
stopOnAssertionFailure: true,
229+
iterationCount: 3,
230+
collection: {
231+
item: [{
232+
name: 'First Request',
233+
event: [{
234+
listen: 'test',
235+
script: {
236+
exec: [
237+
'pm.test("This test will pass", function () {',
238+
' pm.expect(pm.response.code).to.equal(200);',
239+
'});',
240+
'if (iteration === 1) {',
241+
' pm.test("This test will fail", function () {',
242+
' pm.expect(pm.response.code).to.equal(404);',
243+
' });',
244+
'}'
245+
]
246+
}
247+
}],
248+
request: {
249+
url: 'https://postman-echo.com/get',
250+
method: 'GET'
251+
}
252+
}, {
253+
name: 'Second Request',
254+
event: [{
255+
listen: 'test',
256+
script: {
257+
exec: [
258+
'pm.test("This should not run in iteration 1", function () {',
259+
' pm.expect(pm.response.code).to.equal(200);',
260+
'});'
261+
]
262+
}
263+
}],
264+
request: {
265+
url: 'https://postman-echo.com/get',
266+
method: 'GET'
267+
}
268+
}]
269+
}
270+
}, function (err, results) {
271+
testrun = results;
272+
done(err);
273+
});
274+
});
275+
276+
it('should have completed the run', function () {
277+
expect(testrun).to.be.ok;
278+
expect(testrun.done.getCall(0).args[0]).to.be.null;
279+
expect(testrun).to.nested.include({
280+
'done.calledOnce': true,
281+
'start.calledOnce': true
282+
});
283+
});
284+
285+
it('should stop on assertion failure in iteration 1', function () {
286+
// First iteration: both requests should run
287+
// Second iteration: first request fails, second should not run
288+
// Third iteration: should not run at all
289+
expect(testrun.request.callCount).to.be.at.least(2);
290+
expect(testrun.request.callCount).to.be.lessThan(6); // Less than 3 iterations * 2 requests
291+
});
292+
});
293+
});
294+
295+
describe('with stopOnAssertionFailure: false', function () {
296+
describe('should NOT stop on assertion failures', function () {
297+
var testrun;
298+
299+
before(function (done) {
300+
this.run({
301+
stopOnAssertionFailure: false,
302+
collection: {
303+
item: [{
304+
name: 'First Request',
305+
event: [{
306+
listen: 'test',
307+
script: {
308+
exec: [
309+
'pm.test("This test will fail", function () {',
310+
' pm.expect(pm.response.code).to.equal(404);',
311+
'});'
312+
]
313+
}
314+
}],
315+
request: {
316+
url: 'https://postman-echo.com/get',
317+
method: 'GET'
318+
}
319+
}, {
320+
name: 'Second Request',
321+
event: [{
322+
listen: 'test',
323+
script: {
324+
exec: [
325+
'pm.test("This should run", function () {',
326+
' pm.expect(pm.response.code).to.equal(200);',
327+
'});'
328+
]
329+
}
330+
}],
331+
request: {
332+
url: 'https://postman-echo.com/get',
333+
method: 'GET'
334+
}
335+
}]
336+
}
337+
}, function (err, results) {
338+
testrun = results;
339+
done(err);
340+
});
341+
});
342+
343+
it('should have completed the run', function () {
344+
expect(testrun).to.be.ok;
345+
expect(testrun.done.getCall(0).args[0]).to.be.null;
346+
expect(testrun).to.nested.include({
347+
'done.calledOnce': true,
348+
'start.calledOnce': true
349+
});
350+
});
351+
352+
it('should NOT stop the run on assertion failures', function () {
353+
// Both requests should have been executed
354+
expect(testrun.request.callCount).to.equal(2);
355+
expect(testrun.item.callCount).to.equal(2);
356+
});
357+
});
358+
});
359+
});
360+

0 commit comments

Comments
 (0)