Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
149854d
feat: Implement Nightwatch Test Orchestration Module
harshit-bstack Oct 15, 2025
fd1a87e
feat: Enhance test file collection and orchestration logic with impro…
harshit-bstack Oct 15, 2025
7e7615e
feat: Refactor test orchestration logic to apply specific feature pat…
harshit-bstack Oct 15, 2025
3633265
feat: Update cucumber feature path for API tests in Nightwatch globals
harshit-bstack Oct 15, 2025
3b1ceac
feat: Update cucumber feature path handling to use ordered files
harshit-bstack Oct 15, 2025
37873fa
feat: Add framework name to build data payload in OrchestrationUtils
harshit-bstack Oct 16, 2025
7eab4c8
feat: Remove unused helper functions and debug logs from orchestratio…
harshit-bstack Oct 23, 2025
1b98318
feat: Update framework name comment in build data payload and enhance…
harshit-bstack Oct 23, 2025
b8f963a
feat: Remove unused orchestration files and refactor helper functions…
harshit-bstack Oct 23, 2025
df2b1c8
style: eslint update
harshit-bstack Oct 28, 2025
c69da11
feat: Remove framework name from build data payload in OrchestrationU…
harshit-bstack Oct 28, 2025
093a7b4
Merge branch 'main' into to_integration
harshit-bstack Oct 28, 2025
ee4fbc1
feat: Enhance build details extraction by adding observability options
harshit-bstack Oct 28, 2025
e235cf7
feat: Add test observability options to project and build name retrieval
harshit-bstack Oct 28, 2025
f56a9f4
feat: Enhance test orchestration by adding observability session chec…
harshit-bstack Oct 29, 2025
7b0784d
feat: Add test observability session check to build data collection i…
harshit-bstack Oct 29, 2025
32eac85
feat: Refactor test orchestration logic to improve readability and ma…
harshit-bstack Oct 29, 2025
7c16d67
feat: Validate smart selection mode in run smart selection settings
harshit-bstack Oct 30, 2025
37627bf
fix: Corrected base branch retrieval by updating the string replaceme…
harshit-bstack Oct 30, 2025
b84700f
refactor: Change log level from info to debug for test orchestration …
harshit-bstack Oct 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 74 additions & 73 deletions nightwatch/globals.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ const path = require('path');
const AccessibilityAutomation = require('../src/accessibilityAutomation');
const eventHelper = require('../src/utils/eventHelper');
const OrchestrationUtils = require('../src/testorchestration/orchestrationUtils');
const {type} = require('os');
const localTunnel = new LocalTunnel();
const testObservability = new TestObservability();
const accessibilityAutomation = new AccessibilityAutomation();
Expand Down Expand Up @@ -327,83 +326,82 @@ module.exports = {

// Initialize and configure test orchestration
try {
const orchestrationUtils = OrchestrationUtils.getInstance(settings);
if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()) {
// Apply test orchestration to reorder test files before execution
const TestOrchestrationIntegration = require('../src/testorchestration/testOrchestrationIntegration');
const orchestrationIntegration = TestOrchestrationIntegration.getInstance();
orchestrationIntegration.configure(settings);

// Check if we have test files to reorder from various sources
let allTestFiles = [];

// Checking either for Feature Path or src_folders, feature path take priority
if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
Logger.debug('Getting test files from feature_path configuration...');
if (Array.isArray(settings.test_runner.options.feature_path)){
settings.test_runner.options.feature_path.forEach(featurePath => {
const files = helper.collectTestFiles(featurePath, 'feature_path config');
if (helper.isTestObservabilitySession()) {
const orchestrationUtils = OrchestrationUtils.getInstance(settings);
if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()){
// Apply test orchestration to reorder test files before execution
const TestOrchestrationIntegration = require('../src/testorchestration/testOrchestrationIntegration');
const orchestrationIntegration = TestOrchestrationIntegration.getInstance();
orchestrationIntegration.configure(settings);

// Check if we have test files to reorder from various sources
let allTestFiles = [];

// Checking either for Feature Path or src_folders, feature path take priority
if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
Logger.debug('Getting test files from feature_path configuration...');
if (Array.isArray(settings.test_runner.options.feature_path)){
settings.test_runner.options.feature_path.forEach(featurePath => {
const files = helper.collectTestFiles(featurePath, 'feature_path config');
allTestFiles = allTestFiles.concat(files);
});
} else if (typeof settings.test_runner.options.feature_path === 'string'){
const files = helper.collectTestFiles(settings.test_runner.options.feature_path, 'feature_path config');
allTestFiles = allTestFiles.concat(files);
}
} else if (settings.src_folders && Array.isArray(settings.src_folders) && settings.src_folders.length > 0) {
Logger.debug('Getting test files from src_folders configuration...');
settings.src_folders.forEach(folder => {
const files = helper.collectTestFiles(folder, 'src_folders config');
allTestFiles = allTestFiles.concat(files);
});
} else if (typeof settings.test_runner.options.feature_path === 'string'){
const files = helper.collectTestFiles(settings.test_runner.options.feature_path, 'feature_path config');
allTestFiles = allTestFiles.concat(files);
}
} else if (settings.src_folders && Array.isArray(settings.src_folders) && settings.src_folders.length > 0) {
Logger.debug('Getting test files from src_folders configuration...');
settings.src_folders.forEach(folder => {
const files = helper.collectTestFiles(folder, 'src_folders config');
allTestFiles = allTestFiles.concat(files);
});
}

// Remove duplicates and ensure all paths are relative to cwd
allTestFiles = [...new Set(allTestFiles)].map(file => {
return path.isAbsolute(file) ? path.relative(process.cwd(), file) : file;
});
// Remove duplicates and ensure all paths are relative to cwd
allTestFiles = [...new Set(allTestFiles)].map(file => {
return path.isAbsolute(file) ? path.relative(process.cwd(), file) : file;
});


if (allTestFiles.length > 0) {
Logger.info(`Applying test orchestration to reorder test files... Found ${allTestFiles.length} test files`);
Logger.debug(`Test files: ${JSON.stringify(allTestFiles)}`);

// Apply orchestration to get ordered test files (synchronously)
try {
const orderedFiles = await orchestrationIntegration.applyOrchestration(allTestFiles, settings);
if (orderedFiles && orderedFiles.length > 0) {
Logger.info(`Test files reordered by orchestration: ${orderedFiles.length} files`);

Logger.info('Test orchestration recommended order change:');
Logger.info(`Original: ${allTestFiles.join(', ')}`);
Logger.info(`Optimized: ${orderedFiles.join(', ')}`);
try {
if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
// For cucumber, we override the feature_path option with ordered files
settings.test_runner.options['feature_path'] = orderedFiles;
} else {
settings.src_folders = orderedFiles;
for (const envName in testEnvSettings) {
testEnvSettings[envName].src_folders = orderedFiles;
testEnvSettings[envName].test_runner.src_folders = orderedFiles;
if (allTestFiles.length > 0) {
Logger.debug(`Applying test orchestration to reorder test files... Found ${allTestFiles.length} test files`);
Logger.debug(`Test files: ${JSON.stringify(allTestFiles)}`);
// Apply orchestration to get ordered test files (synchronously)
try {
const orderedFiles = await orchestrationIntegration.applyOrchestration(allTestFiles, settings);
if (orderedFiles && orderedFiles.length > 0) {
Logger.info(`Test files reordered by orchestration: ${orderedFiles.length} files`);
try {
if (helper.isCucumberTestSuite(settings) && settings?.test_runner?.options?.feature_path){
// For cucumber, we override the feature_path option with ordered files
settings.test_runner.options['feature_path'] = orderedFiles;
} else {
settings.src_folders = orderedFiles;
for (const envName in testEnvSettings) {
testEnvSettings[envName].src_folders = orderedFiles;
testEnvSettings[envName].test_runner.src_folders = orderedFiles;
}
if (settings.test_runner && typeof settings.test_runner === 'object' && !Array.isArray(settings.test_runner)) {
settings.test_runner.src_folders = orderedFiles;
}
}
if (settings.test_runner && typeof settings.test_runner === 'object' && !Array.isArray(settings.test_runner)) {
settings.test_runner.src_folders = orderedFiles;
}
}

} catch (reorderError) {
Logger.error(`Runtime reordering failed: ${reorderError.message}`);
Logger.info('Falling back to original order for current execution.');
}
} else {
Logger.info('Split test API called - no reordering available');

} catch (reorderError) {
Logger.error(`Runtime reordering failed: ${reorderError.message}`);
Logger.info('Falling back to original order for current execution.');
}
} else {
Logger.info('Split test API called - no reordering available');
}
} catch (error) {
Logger.error(`Error applying test orchestration: ${error}`);
}
} catch (error) {
Logger.error(`Error applying test orchestration: ${error}`);

} else {
Logger.debug('No test files found for orchestration - skipping split test API call');
}
} else {
Logger.debug('No test files found for orchestration - skipping split test API call');
}
}
} catch (error) {
Expand Down Expand Up @@ -434,10 +432,13 @@ module.exports = {

// Collect build data for test orchestration if enabled
try {
const orchestrationUtils = OrchestrationUtils.getInstance();
if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()) {
Logger.info('Collecting build data for test orchestration...');
await orchestrationUtils.collectBuildData(this.settings || {});
if (helper.isTestObservabilitySession()) {
const orchestrationUtils = OrchestrationUtils.getInstance();
if (orchestrationUtils && orchestrationUtils.testOrderingEnabled()){

Logger.info('Collecting build data for test orchestration...');
await orchestrationUtils.collectBuildData(this.settings || {});
}
}
} catch (error) {
Logger.error(`Error collecting build data for test orchestration: ${error}`);
Expand Down
6 changes: 1 addition & 5 deletions src/testorchestration/applyOrchestration.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ const TestOrchestrationHandler = require('./testOrchestrationHandler');
async function applyOrchestrationIfEnabled(specs, config) {
// Initialize orchestration handler
const orchestrationHandler = TestOrchestrationHandler.getInstance(config);
Logger.info('Orchestration handler is initialized');

if (!orchestrationHandler) {
Logger.warn('Orchestration handler is not initialized. Skipping orchestration.');
Expand All @@ -32,13 +31,10 @@ async function applyOrchestrationIfEnabled(specs, config) {
orchestrationHandler.addToOrderingInstrumentationData('enabled', orchestrationHandler.testOrderingEnabled());

const startTime = performance.now();

Logger.info('Test orchestration is enabled. Attempting to reorder test files.');


// Get the test files from the specs
const testFiles = specs;
testOrderingApplied = true;
Logger.info(`Test files to be reordered: ${testFiles.join(', ')}`);

// Reorder the test files
const orderedFiles = await orchestrationHandler.reorderTestFiles(testFiles);
Expand Down
16 changes: 12 additions & 4 deletions src/testorchestration/orchestrationUtils.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class OrchestrationUtils {
this._setRunSmartSelection(
runSmartSelectionOpts.enabled || false,
runSmartSelectionOpts.mode || 'relevantFirst',
runSmartSelectionOpts.source || null
runSmartSelectionOpts.source
);

// Extract build details
Expand Down Expand Up @@ -98,9 +98,12 @@ class OrchestrationUtils {
*/
_extractBuildDetails() {
try {
this.buildName = helper.getBuildName(this._settings, this._bstackOptions) || '';
const fromProduct = {
test_observability: true
};
this.buildName = helper.getBuildName(this._settings, this._bstackOptions, fromProduct) || '';

this.projectName = helper.getProjectName(this._settings, this._bstackOptions) || '';
this.projectName = helper.getProjectName(this._settings, this._bstackOptions, fromProduct) || '';

this.buildIdentifier = process.env.BROWSERSTACK_BUILD_RUN_IDENTIFIER || '';

Expand Down Expand Up @@ -217,8 +220,13 @@ class OrchestrationUtils {
_setRunSmartSelection(enabled, mode, source = null) {
try {
this.runSmartSelection = Boolean(enabled);
this.smartSelectionMode = mode;
if (['relevantFirst', 'relevantOnly'].includes(mode)) {
this.smartSelectionMode = mode;
} else {
this.smartSelectionMode = 'relevantFirst';
}

this.smartSelectionSource = [];
// Log the configuration for debugging
this.logger.debug(`Setting runSmartSelection: enabled=${this.runSmartSelection}, mode=${this.smartSelectionMode}`);

Expand Down
8 changes: 6 additions & 2 deletions src/testorchestration/testOrderingServer.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ class TestOrderingServer {
if (config && config.desiredCapabilities && config.desiredCapabilities['bstack:options']) {
this._bstackOptions = config.desiredCapabilities['bstack:options'];
}
this.fromProduct = {
test_observability: true
};
}


/**
* Initiates the split tests request and stores the response data for polling.
Expand All @@ -46,8 +50,8 @@ class TestOrderingServer {
orchestrationMetadata,
nodeIndex: parseInt(process.env.BROWSERSTACK_NODE_INDEX || '0'),
totalNodes: parseInt(process.env.BROWSERSTACK_TOTAL_NODE_COUNT || '1'),
projectName: getProjectName(this._settings, this._bstackOptions),
buildName: getBuildName(this._settings, this._bstackOptions),
projectName: getProjectName(this._settings, this._bstackOptions, this.fromProduct),
buildName: getBuildName(this._settings, this._bstackOptions, this.fromProduct),
buildRunIdentifier: process.env.BROWSERSTACK_BUILD_RUN_IDENTIFIER || '',
hostInfo: getHostInfo(),
prDetails
Expand Down
6 changes: 5 additions & 1 deletion src/utils/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -1057,7 +1057,7 @@ function getBaseBranch() {
try {
const originHeadOutput = execSync('git symbolic-ref refs/remotes/origin/HEAD').toString().trim();
if (originHeadOutput.startsWith('refs/remotes/origin/')) {
return originHeadOutput.replace('refs/remotes/', '');
return originHeadOutput.replace('refs/remotes/origin/', '');
}
} catch (e) {
// Symbolic ref might not exist
Expand Down Expand Up @@ -1134,7 +1134,11 @@ function getChangedFilesFromCommits(commitHashes) {
* @param multiRepoSource Array of repository paths for multi-repo setup
*/
exports.getGitMetadataForAiSelection = (folders = []) => {

if (folders && folders.length === 0) {
return [];
}
if (folders === null){
folders = [process.cwd()];
}

Expand Down