Skip to content

Commit 0b7dab9

Browse files
SagarGanigaSagarGaniga
authored
Merge pull request #127 from browserstack/spec_limit
Add warning around spec count
2 parents ea96c7f + b71b533 commit 0b7dab9

File tree

5 files changed

+143
-3
lines changed

5 files changed

+143
-3
lines changed

‎bin/commands/runs.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ module.exports = function run(args) {
6666
// accept the number of parallels
6767
utils.setParallels(bsConfig, args, specFiles.length);
6868

69+
// warn if specFiles cross our limit
70+
utils.warnSpecLimit(bsConfig, args, specFiles);
71+
6972
// Archive the spec files
7073
return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude).then(function (data) {
7174

‎bin/helpers/constants.js

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ const userMessages = {
4141
CYPRESS_VERSION_CHANGED: "Your build will run using Cypress <actualVersion> instead of Cypress <preferredVersion>. Read more about supported versions here: http://browserstack.com/docs/automate/cypress/supported-versions",
4242
LOCAL_START_FAILED: "Local Testing setup failed.",
4343
LOCAL_STOP_FAILED: "Local Binary stop failed.",
44-
INVALID_LOCAL_MODE_WARNING: "Invalid value specified for local_mode. local_mode: (\"always-on\" | \"on-demand\"). For more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference"
44+
INVALID_LOCAL_MODE_WARNING: "Invalid value specified for local_mode. local_mode: (\"always-on\" | \"on-demand\"). For more info, check out https://www.browserstack.com/docs/automate/cypress/cli-reference",
45+
SPEC_LIMIT_WARNING: "You might not see all your results on the dashboard because of high spec count, please consider reducing the number of spec files in this folder."
4546
};
4647

4748
const validationMessages = {
@@ -136,6 +137,9 @@ const specFileTypes = ['js', 'ts', 'feature', 'jsx', 'coffee', 'cjsx'];
136137

137138
const DEFAULT_CYPRESS_SPEC_PATH = "cypress/integration"
138139

140+
const SPEC_TOTAL_CHAR_LIMIT = 32243;
141+
const METADATA_CHAR_BUFFER_PER_SPEC = 175;
142+
139143
module.exports = Object.freeze({
140144
syncCLI,
141145
userMessages,
@@ -145,5 +149,7 @@ module.exports = Object.freeze({
145149
allowedFileTypes,
146150
filesToIgnoreWhileUploading,
147151
specFileTypes,
148-
DEFAULT_CYPRESS_SPEC_PATH
152+
DEFAULT_CYPRESS_SPEC_PATH,
153+
SPEC_TOTAL_CHAR_LIMIT,
154+
METADATA_CHAR_BUFFER_PER_SPEC
149155
});

‎bin/helpers/utils.js

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,24 @@ exports.setParallels = (bsConfig, args, numOfSpecs) => {
148148
}
149149
};
150150

151+
exports.warnSpecLimit = (bsConfig, args, specFiles) => {
152+
let expectedCharLength = specFiles.join("").length + Constants.METADATA_CHAR_BUFFER_PER_SPEC * specFiles.length;
153+
let parallels = bsConfig.run_settings.parallels;
154+
let combinations = this.getBrowserCombinations(bsConfig).length;
155+
let parallelPerCombination = parallels > combinations ? Math.floor(parallels / combinations) : 1;
156+
let expectedCharLengthPerParallel = Math.floor(expectedCharLength / parallelPerCombination);
157+
if (expectedCharLengthPerParallel > Constants.SPEC_TOTAL_CHAR_LIMIT) {
158+
logger.warn(Constants.userMessages.SPEC_LIMIT_WARNING);
159+
this.sendUsageReport(
160+
bsConfig,
161+
args,
162+
Constants.userMessages.SPEC_LIMIT_WARNING,
163+
Constants.messageTypes.WARNING,
164+
null
165+
);
166+
}
167+
}
168+
151169
exports.setDefaults = (bsConfig, args) => {
152170
// setting setDefaultAuthHash to {} if not present and set via env variables or via args.
153171
if (this.isUndefined(bsConfig['auth']) && (!this.isUndefined(args.username) || !this.isUndefined(process.env.BROWSERSTACK_USERNAME))) {

‎test/unit/bin/commands/runs.js

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ describe("runs", () => {
194194
beforeEach(() => {
195195
sandbox = sinon.createSandbox();
196196
setParallelsStub = sandbox.stub();
197+
warnSpecLimitStub = sandbox.stub();
197198
setUsernameStub = sandbox.stub();
198199
setAccessKeyStub = sandbox.stub();
199200
getConfigPathStub = sandbox.stub();
@@ -236,6 +237,7 @@ describe("runs", () => {
236237
validateBstackJson: validateBstackJsonStub,
237238
sendUsageReport: sendUsageReportStub,
238239
setParallels: setParallelsStub,
240+
warnSpecLimit: warnSpecLimitStub,
239241
setUsername: setUsernameStub,
240242
setAccessKey: setAccessKeyStub,
241243
setBuildName: setBuildNameStub,
@@ -286,6 +288,7 @@ describe("runs", () => {
286288
sinon.assert.calledOnce(setCypressConfigFilenameStub);
287289
sinon.assert.calledOnce(getNumberOfSpecFilesStub);
288290
sinon.assert.calledOnce(setParallelsStub);
291+
sinon.assert.calledOnce(warnSpecLimitStub);
289292
sinon.assert.calledOnce(setLocalStub);
290293
sinon.assert.calledOnce(setLocalIdentifierStub);
291294
sinon.assert.calledOnce(setHeadedStub);
@@ -316,6 +319,7 @@ describe("runs", () => {
316319
sandbox = sinon.createSandbox();
317320
validateBstackJsonStub = sandbox.stub();
318321
setParallelsStub = sandbox.stub();
322+
warnSpecLimitStub = sandbox.stub();
319323
setUsernameStub = sandbox.stub();
320324
setAccessKeyStub = sandbox.stub();
321325
setBuildNameStub = sandbox.stub();
@@ -358,6 +362,7 @@ describe("runs", () => {
358362
validateBstackJson: validateBstackJsonStub,
359363
sendUsageReport: sendUsageReportStub,
360364
setParallels: setParallelsStub,
365+
warnSpecLimit: warnSpecLimitStub,
361366
setUsername: setUsernameStub,
362367
setAccessKey: setAccessKeyStub,
363368
setBuildName: setBuildNameStub,
@@ -408,6 +413,7 @@ describe("runs", () => {
408413
sinon.assert.calledOnce(setLocalConfigFileStub);
409414
sinon.assert.calledOnce(getNumberOfSpecFilesStub);
410415
sinon.assert.calledOnce(setParallelsStub);
416+
sinon.assert.calledOnce(warnSpecLimitStub);
411417
sinon.assert.calledOnce(setLocalStub);
412418
sinon.assert.calledOnce(setLocalIdentifierStub);
413419
sinon.assert.calledOnce(setHeadedStub);
@@ -441,6 +447,7 @@ describe("runs", () => {
441447
sandbox = sinon.createSandbox();
442448
validateBstackJsonStub = sandbox.stub();
443449
setParallelsStub = sandbox.stub();
450+
warnSpecLimitStub = sandbox.stub();
444451
setUsernameStub = sandbox.stub();
445452
setAccessKeyStub = sandbox.stub();
446453
setBuildNameStub = sandbox.stub();
@@ -485,6 +492,7 @@ describe("runs", () => {
485492
validateBstackJson: validateBstackJsonStub,
486493
sendUsageReport: sendUsageReportStub,
487494
setParallels: setParallelsStub,
495+
warnSpecLimit: warnSpecLimitStub,
488496
setUsername: setUsernameStub,
489497
setAccessKey: setAccessKeyStub,
490498
setBuildName: setBuildNameStub,
@@ -546,6 +554,7 @@ describe("runs", () => {
546554
sinon.assert.calledOnce(capabilityValidatorStub);
547555
sinon.assert.calledOnce(getNumberOfSpecFilesStub);
548556
sinon.assert.calledOnce(setParallelsStub);
557+
sinon.assert.calledOnce(warnSpecLimitStub);
549558
sinon.assert.calledOnce(setLocalStub);
550559
sinon.assert.calledOnce(setLocalIdentifierStub);
551560
sinon.assert.calledOnce(setHeadedStub);
@@ -579,6 +588,7 @@ describe("runs", () => {
579588
sandbox = sinon.createSandbox();
580589
validateBstackJsonStub = sandbox.stub();
581590
setParallelsStub = sandbox.stub();
591+
warnSpecLimitStub = sandbox.stub()
582592
setUsernameStub = sandbox.stub();
583593
setAccessKeyStub = sandbox.stub();
584594
setBuildNameStub = sandbox.stub();
@@ -633,6 +643,7 @@ describe("runs", () => {
633643
setTestEnvs: setTestEnvsStub,
634644
setUsageReportingFlag: setUsageReportingFlagStub,
635645
setParallels: setParallelsStub,
646+
warnSpecLimit: warnSpecLimitStub,
636647
getConfigPath: getConfigPathStub,
637648
setLocal: setLocalStub,
638649
setLocalMode: setLocalModeStub,
@@ -688,6 +699,7 @@ describe("runs", () => {
688699
sinon.assert.calledOnce(capabilityValidatorStub);
689700
sinon.assert.calledOnce(getNumberOfSpecFilesStub);
690701
sinon.assert.calledOnce(setParallelsStub);
702+
sinon.assert.calledOnce(warnSpecLimitStub);
691703
sinon.assert.calledOnce(setLocalStub);
692704
sinon.assert.calledOnce(setLocalModeStub);
693705
sinon.assert.calledOnce(setupLocalTestingStub);

‎test/unit/bin/helpers/utils.js

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1496,7 +1496,6 @@ describe('utils', () => {
14961496
});
14971497

14981498
describe('getNumberOfSpecFiles', () => {
1499-
15001499
it('glob search pattern should be equal to bsConfig.run_settings.specs', () => {
15011500
let getNumberOfSpecFilesStub = sinon.stub(glob, 'sync');
15021501
let bsConfig = {
@@ -1560,6 +1559,108 @@ describe('utils', () => {
15601559
});
15611560
});
15621561

1562+
describe('warnSpecLimit', () => {
1563+
let sendUsageReportStub, loggerStub;
1564+
let bsConfig = {run_settings: {}}, args = {};
1565+
beforeEach(() => {
1566+
sendUsageReportStub = sandbox
1567+
.stub(utils, 'sendUsageReport')
1568+
.callsFake(function () {
1569+
return 'end';
1570+
});
1571+
loggerStub = sinon.stub(logger, 'warn');
1572+
});
1573+
1574+
afterEach(() => {
1575+
sandbox.restore();
1576+
sinon.restore();
1577+
});
1578+
1579+
context('limit crossing', () => {
1580+
it('should log and send to eds for one combination, one parallel', () => {
1581+
let specFiles = {
1582+
length: 1,
1583+
join: function() {
1584+
return {
1585+
length: constant.SPEC_TOTAL_CHAR_LIMIT
1586+
}
1587+
}
1588+
};
1589+
sinon.stub(utils, "getBrowserCombinations").returns(1);
1590+
bsConfig.run_settings.parallels = 1;
1591+
utils.warnSpecLimit(bsConfig, args, specFiles);
1592+
sinon.assert.calledOnce(sendUsageReportStub);
1593+
sinon.assert.calledOnce(loggerStub);
1594+
});
1595+
1596+
it('should log and send to eds for one combination, two parallel', () => {
1597+
let specFiles = {
1598+
length: 1,
1599+
join: function() {
1600+
return {
1601+
length: constant.SPEC_TOTAL_CHAR_LIMIT
1602+
}
1603+
}
1604+
};
1605+
sinon.stub(utils, "getBrowserCombinations").returns(1);
1606+
bsConfig.run_settings.parallels = 2;
1607+
utils.warnSpecLimit(bsConfig, args, specFiles);
1608+
sinon.assert.calledOnce(sendUsageReportStub);
1609+
sinon.assert.calledOnce(loggerStub);
1610+
});
1611+
1612+
it('should log and send to eds for multiple combination, multiple parallel', () => {
1613+
let specFiles = {
1614+
length: 1,
1615+
join: function() {
1616+
return {
1617+
length: constant.SPEC_TOTAL_CHAR_LIMIT
1618+
}
1619+
}
1620+
};
1621+
sinon.stub(utils, "getBrowserCombinations").returns(3);
1622+
bsConfig.run_settings.parallels = 4;
1623+
utils.warnSpecLimit(bsConfig, args, specFiles);
1624+
sinon.assert.calledOnce(sendUsageReportStub);
1625+
sinon.assert.calledOnce(loggerStub);
1626+
});
1627+
});
1628+
1629+
context('within limit', () => {
1630+
it('should not log for one combination, one parallel', () => {
1631+
let specFiles = {
1632+
length: 1,
1633+
join: function() {
1634+
return {
1635+
length: 1
1636+
}
1637+
}
1638+
};
1639+
sinon.stub(utils, "getBrowserCombinations").returns(1);
1640+
bsConfig.run_settings.parallels = 1;
1641+
utils.warnSpecLimit(bsConfig, args, specFiles);
1642+
sinon.assert.notCalled(sendUsageReportStub);
1643+
sinon.assert.notCalled(loggerStub);
1644+
});
1645+
1646+
it('should not log for one combination, multiple parallel', () => {
1647+
let specFiles = {
1648+
length: 1,
1649+
join: function() {
1650+
return {
1651+
length: 1
1652+
}
1653+
}
1654+
};
1655+
sinon.stub(utils, "getBrowserCombinations").returns(1);
1656+
bsConfig.run_settings.parallels = 2;
1657+
utils.warnSpecLimit(bsConfig, args, specFiles);
1658+
sinon.assert.notCalled(sendUsageReportStub);
1659+
sinon.assert.notCalled(loggerStub);
1660+
});
1661+
});
1662+
});
1663+
15631664
describe('capitalizeFirstLetter', () => {
15641665

15651666
it('should capitalize First Letter ', () => {

0 commit comments

Comments
 (0)