From b1bedffcc118165d22f2d5025b08872036930460 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 28 Oct 2025 11:13:20 -0400 Subject: [PATCH] MLE-24733 Enabling multipleWorker test again Added Copilot-suggested fix for the "before()" function to not finish before the first test, which seems to be the problem? --- Jenkinsfile | 17 +- .../nodejs-ds-multipleWorker.js | 498 +++++++++++------- 2 files changed, 311 insertions(+), 204 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 43e44c1c..98f75156 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,20 +92,15 @@ def runE2ETests() { gulp loadToModulesDB gulp generateFnClasses gulp copyFnClasses - # Adding sleep for the gulp commands to complete. - sleep 30 cp *.js ../test-complete/ cp -R ml-modules/ ../test-complete cd ../test-complete - ../node_modules/.bin/mocha -R xunit --timeout 60000 nodejs-ds-setup-docs.js - ../node_modules/.bin/mocha -R xunit --timeout 60000 "nodejs-ds-required-params.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-required-params-results.xml || true - ../node_modules/.bin/mocha -R xunit --timeout 60000 "nodejs-ds-error-map.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true - - # Disabling this for now, failing with the new all-mlDeploy setup for unknown reasons. - # ../node_modules/.bin/mocha -R xunit --timeout 60000 -R xunit "nodejs-ds-multipleWorker.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true - - ../node_modules/.bin/mocha -R xunit --timeout 60000 -R xunit "nodejs-ds-transactions.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-transactions-results.js.xml || true - ../node_modules/.bin/mocha -R xunit --timeout 60000 -R xunit "nodejs-ds-dynamic.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-dynamic-results.xml || true + ../node_modules/.bin/mocha -R xunit --timeout 20000 nodejs-ds-setup-docs.js + ../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-required-params.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-required-params-results.xml || true + ../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-error-map.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true + ../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-multipleWorker.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true + ../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-transactions.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-transactions-results.js.xml || true + ../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-dynamic.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-dynamic-results.xml || true ''' junit '**/*.xml' } diff --git a/test-complete-proxy/nodejs-ds-multipleWorker.js b/test-complete-proxy/nodejs-ds-multipleWorker.js index 859e8a33..949521bc 100755 --- a/test-complete-proxy/nodejs-ds-multipleWorker.js +++ b/test-complete-proxy/nodejs-ds-multipleWorker.js @@ -1,193 +1,305 @@ -/* -* Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. -*/ - -const fs = require('fs'); - -const expect = require('chai').expect; -const should = require('should'); - -const {Worker, isMainThread, parentPort, workerData} = require('worker_threads'); - -var testconfig = require('../etc/test-config-qa.js'); - -var marklogic = require('../'); -var q = marklogic.queryBuilder; - -var db = marklogic.createDatabaseClient(testconfig.restEvaluatorConnection); - // Run tests from test-complete-proxy folder. Else adjust paths of worker's js files. - describe('Multiple-Worker-Test', function(){ - before(function(done) { - // runs once before the first test in this block - var s11 = 'Vannevar Bush wrote an article for The Atlantic Monthly 1'; - var s12 = 'Lisa wrote an article for The Strait Times 1'; - - var s21 = 'Vannevar Bush wrote an article for The Atlantic Monthly 2'; - var s22 = 'Lisa wrote an article for The Strait Times 2'; - - var inputFiles1 = [s11, s12]; - var uris1 = ['Test1stream11', 'Test1stream12']; - - var inputFiles2 = [s21, s22]; - var uris2 = ['Test1stream21', 'Test1stream22']; - - if (isMainThread) { - const workerOneInsert = new Worker('./insertFromMultipleStreams.js', {workerData: {files: inputFiles1, uris:uris1}}); - workerOneInsert.on('done', (result) => { - //console.log('workerOneInsert message is ' + result ); - }); - - workerOneInsert.on('exit', (code) => { - if (code !== 0) - reject(new Error('Worker workerOneInsert stopped with exit code ${code}')); - //else console.debug('workerOneInsert exits normally'); - }); - - const workerTwoInsert = new Worker('./insertFromMultipleStreams.js', {workerData: {files: inputFiles2, uris:uris2}}); - workerTwoInsert.on('message', (result) => { - //console.log('workerTwoInsert message is ' + result ); - }); - - workerTwoInsert.on('exit', (code) => { - if (code !== 0) - reject(new Error('Worker workerTwoInsert stopped with exit code ${code}')); - //else console.debug('workerTwoInsert exits normally'); - }); - - } - - done(); - }); - - it('Verify inserts using client API query', function(done){ - - var res1; - db.documents.query(q.where(q.parsedFrom('Bush'))).result( function(results) { - res1 = JSON.stringify(results); - //console.log(res1); - expect(res1).to.include('Monthly 1'); - expect(res1).to.include('Monthly 2'); - }); - - var res2; - db.documents.query(q.where(q.parsedFrom('Lisa'))).result( function(results) { - res2 = JSON.stringify(results); - //console.log(res2); - expect(res2).to.include('Times 1'); - expect(res2).to.include('Times 2'); - - done(); - - }); - }); - - it('One worker', function(done){ - try { - // Get results from all workers - var searchResults1 = []; - var searchResults2 = []; - - if (isMainThread) { - const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); - workerOneSearch.on('done', (result) => { - searchResults1.push(result); - //console.log('Results 1 from search is :', searchResults1); - expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); - }); - workerOneSearch.on('exit', (code) => { - if (code !== 0) - reject(new Error('Worker workerOneSearch stopped with exit code ${code}')); - //else console.debug('workerOneSearch exits normally'); - }); - done(); - } - } - catch(err) { - //console.debug(err); - done(); - } - }); - - it('Multiple workers', function(done){ - try { - // Get results from all workers - var searchResults1 = []; - var searchResults2 = []; - - if (isMainThread) { - const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); - workerOneSearch.on('done', (result) => { - searchResults1.push(result); - //console.log('Results 1 from search is :', searchResults1); - expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); - }); - - const workerTwoSearch = new Worker('./searchMultiple.js', {workerData: {search:'Lisa'}}); - workerTwoSearch.on('done', (result) => { - searchResults2.push(result); - //console.log('Results 2 from search is :', searchResults2); - expect(searchResults2[0]).to.have.members(["/Test1stream12.json", "/Test1stream22.json"]); - }); - done(); - } - } - catch(err) { - //console.debug(err); - done(); - } - }); - - it('Multiple workers-One result back', function(done){ - try { - // Get results from all workers - var searchResults1 = []; - var searchResults2 = []; - - if (isMainThread) { - const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); - workerOneSearch.on('done', (result) => { - searchResults1.push(result); - //console.log('Results 1 from search is :', searchResults1); - expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); - }); - - const workerTwoSearch = new Worker('./searchMultiple.js', {workerData: {search:'100'}}); - workerTwoSearch.on('done', (result) => { - searchResults2.push(result); - //console.log('Results 2 from search is :', searchResults2); - expect(searchResults2[0]).to.eql([]); - }); - done(); - } - } - catch(err) { - //console.debug(err); - done(); - } - }); - - it('Multiple workers- incorrect data', function(done){ - try { - // Get results from all workers - var searchResults1 = []; - var searchResults2 = []; - - if (isMainThread) { - const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); - workerOneSearch.on('done', (result) => { - searchResults1.push(result); - //console.log('Results 1 from search is :', searchResults1); - expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); - }); - - expect( - () => new Worker('./searchMultiple.js', {workerData: {find:100}}).to.throw('null value not allowed for parameter')); - done(); - } - } - catch(err) { - //console.debug(err); - done(); - } - }); -}); +/* +* Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +*/ + +const fs = require('fs'); + +const expect = require('chai').expect; +const should = require('should'); + +const {Worker, isMainThread, parentPort, workerData} = require('worker_threads'); + +var testconfig = require('../etc/test-config-qa.js'); + +var marklogic = require('../'); +var q = marklogic.queryBuilder; + +var db = marklogic.createDatabaseClient(testconfig.restEvaluatorConnection); + // Run tests from test-complete-proxy folder. Else adjust paths of worker's js files. + describe('Multiple-Worker-Test', function(){ + before(function(done) { + // runs once before the first test in this block + var s11 = 'Vannevar Bush wrote an article for The Atlantic Monthly 1'; + var s12 = 'Lisa wrote an article for The Strait Times 1'; + + var s21 = 'Vannevar Bush wrote an article for The Atlantic Monthly 2'; + var s22 = 'Lisa wrote an article for The Strait Times 2'; + + var inputFiles1 = [s11, s12]; + var uris1 = ['Test1stream11', 'Test1stream12']; + + var inputFiles2 = [s21, s22]; + var uris2 = ['Test1stream21', 'Test1stream22']; + + if (isMainThread) { + let workersCompleted = 0; + const totalWorkers = 2; + let hasError = false; + + const handleWorkerComplete = (err) => { + if (hasError) return; + + if (err) { + hasError = true; + done(err); + return; + } + + workersCompleted++; + if (workersCompleted === totalWorkers) { + done(); + } + }; + + const workerOneInsert = new Worker('./insertFromMultipleStreams.js', {workerData: {files: inputFiles1, uris:uris1}}); + workerOneInsert.on('done', (result) => { + //console.log('workerOneInsert message is ' + result ); + }); + + workerOneInsert.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerOneInsert stopped with exit code ${code}`)); + } else { + //console.debug('workerOneInsert exits normally'); + handleWorkerComplete(); + } + }); + + const workerTwoInsert = new Worker('./insertFromMultipleStreams.js', {workerData: {files: inputFiles2, uris:uris2}}); + workerTwoInsert.on('message', (result) => { + //console.log('workerTwoInsert message is ' + result ); + }); + + workerTwoInsert.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerTwoInsert stopped with exit code ${code}`)); + } else { + //console.debug('workerTwoInsert exits normally'); + handleWorkerComplete(); + } + }); + } else { + done(); + } + }); + + it('Verify inserts using client API query', function(done){ + Promise.all([ + db.documents.query(q.where(q.parsedFrom('Bush'))).result(), + db.documents.query(q.where(q.parsedFrom('Lisa'))).result() + ]) + .then(([bushResults, lisaResults]) => { + const res1 = JSON.stringify(bushResults); + console.log(res1); + expect(res1).to.include('Monthly 1'); + expect(res1).to.include('Monthly 2'); + + const res2 = JSON.stringify(lisaResults); + console.log(res2); + expect(res2).to.include('Times 1'); + expect(res2).to.include('Times 2'); + + done(); + }) + .catch(err => { + done(err); + }); + }); + + it('One worker', function(done){ + // Get results from all workers + var searchResults1 = []; + + if (isMainThread) { + const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); + workerOneSearch.on('done', (result) => { + searchResults1.push(result); + //console.log('Results 1 from search is :', searchResults1); + expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); + }); + workerOneSearch.on('exit', (code) => { + if (code !== 0) { + done(new Error(`Worker workerOneSearch stopped with exit code ${code}`)); + } else { + //console.debug('workerOneSearch exits normally'); + done(); + } + }); + } else { + done(); + } + }); + + it('Multiple workers', function(done){ + // Get results from all workers + var searchResults1 = []; + var searchResults2 = []; + + if (isMainThread) { + let workersCompleted = 0; + const totalWorkers = 2; + let hasError = false; + + const handleWorkerComplete = (err) => { + if (hasError) return; + + if (err) { + hasError = true; + done(err); + return; + } + + workersCompleted++; + if (workersCompleted === totalWorkers) { + done(); + } + }; + + const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); + workerOneSearch.on('done', (result) => { + searchResults1.push(result); + //console.log('Results 1 from search is :', searchResults1); + expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); + }); + workerOneSearch.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerOneSearch stopped with exit code ${code}`)); + } else { + handleWorkerComplete(); + } + }); + + const workerTwoSearch = new Worker('./searchMultiple.js', {workerData: {search:'Lisa'}}); + workerTwoSearch.on('done', (result) => { + searchResults2.push(result); + //console.log('Results 2 from search is :', searchResults2); + expect(searchResults2[0]).to.have.members(["/Test1stream12.json", "/Test1stream22.json"]); + }); + workerTwoSearch.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerTwoSearch stopped with exit code ${code}`)); + } else { + handleWorkerComplete(); + } + }); + } else { + done(); + } + }); + + it('Multiple workers-One result back', function(done){ + // Get results from all workers + var searchResults1 = []; + var searchResults2 = []; + + if (isMainThread) { + let workersCompleted = 0; + const totalWorkers = 2; + let hasError = false; + + const handleWorkerComplete = (err) => { + if (hasError) return; + + if (err) { + hasError = true; + done(err); + return; + } + + workersCompleted++; + if (workersCompleted === totalWorkers) { + done(); + } + }; + + const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); + workerOneSearch.on('done', (result) => { + searchResults1.push(result); + //console.log('Results 1 from search is :', searchResults1); + expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); + }); + workerOneSearch.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerOneSearch stopped with exit code ${code}`)); + } else { + handleWorkerComplete(); + } + }); + + const workerTwoSearch = new Worker('./searchMultiple.js', {workerData: {search:'100'}}); + workerTwoSearch.on('done', (result) => { + searchResults2.push(result); + //console.log('Results 2 from search is :', searchResults2); + expect(searchResults2[0]).to.eql([]); + }); + workerTwoSearch.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerTwoSearch stopped with exit code ${code}`)); + } else { + handleWorkerComplete(); + } + }); + } else { + done(); + } + }); + + it('Multiple workers- incorrect data', function(done){ + // Get results from all workers + var searchResults1 = []; + + if (isMainThread) { + let workersCompleted = 0; + const totalWorkers = 2; + let hasError = false; + + const handleWorkerComplete = (err) => { + if (hasError) return; + + if (err) { + hasError = true; + done(err); + return; + } + + workersCompleted++; + if (workersCompleted === totalWorkers) { + done(); + } + }; + + const workerOneSearch = new Worker('./searchMultiple.js', {workerData: {search:'Bush'}}); + workerOneSearch.on('done', (result) => { + searchResults1.push(result); + //console.log('Results 1 from search is :', searchResults1); + expect(searchResults1[0]).to.have.members(["/Test1stream11.json", "/Test1stream21.json"]); + }); + workerOneSearch.on('exit', (code) => { + if (code !== 0) { + handleWorkerComplete(new Error(`Worker workerOneSearch stopped with exit code ${code}`)); + } else { + handleWorkerComplete(); + } + }); + + // Test worker with incorrect data - should fail/exit with error + const workerBadData = new Worker('./searchMultiple.js', {workerData: {find:100}}); + workerBadData.on('error', (err) => { + // Expected error path - worker received bad data + //console.log('Expected error from workerBadData:', err.message); + handleWorkerComplete(); + }); + workerBadData.on('exit', (code) => { + if (code !== 0) { + // This is also acceptable - worker exited with error code + handleWorkerComplete(); + } else { + // Worker shouldn't succeed with bad data + handleWorkerComplete(new Error('Worker with incorrect data should have failed')); + } + }); + } else { + done(); + } + }); +});