Improve regex in TRSS to handle long CSV output

Signed-off-by: Lan Xia <Lan_Xia@ca.ibm.com>

Author: llxia

TestResultSummaryService/DataManager.js

@@ -22,21 +22,26 @@ class DataManager {
     }
 
     async parseOutput(buildName, output) {
+        logger.verbose('parseOutput 0');
         const parserTypes = await Promise.all(
             Object.keys(Parsers).map(async (type) => {
+                logger.verbose('parseOutput type', type);
                 if (Parsers[type].canParse(buildName, output)) {
                     const parser = new Parsers[type](buildName);
+                    logger.verbose('parseOutput parser', parser);
                     return await parser.parse(output, buildName);
                 }
             })
         );
+        logger.verbose('parseOutput 1');
         let results = parserTypes.filter((element) => {
             return element !== undefined;
         });
         if (results.length === 0) {
             const parser = new DefaultParser();
             results = await parser.parse(output);
         }
+        logger.verbose('parseOutput 2');
         return Object.assign.apply({}, results);
     }
 
@@ -133,6 +138,7 @@ class DataManager {
             buildName,
             output
         );
+        logger.verbose('updateBuildWithOutput 1');
         const testResults = new TestResultsDB();
         const outputDB = new OutputDB();
         let update = {
@@ -141,6 +147,7 @@ class DataManager {
         };
         if (!rootBuildId) {
             const rootBuildId = await testResults.getRootBuildId(_id);
+            logger.verbose('updateBuildWithOutput 2');
             update.rootBuildId = new ObjectID(rootBuildId);
         }
         if (builds && builds.length > 0) {
@@ -171,6 +178,7 @@ class DataManager {
                     });
                 })
             );
+            logger.verbose('updateBuildWithOutput 3');
 
             const outputData = {
                 id: data.buildOutputId ? data.buildOutputId : null,
@@ -219,7 +227,8 @@ class DataManager {
             update.buildOutputId = buildOutputId;
             update.hasChildren = false;
         }
-        const result = await testResults.update(criteria, { $set: update });
+        await testResults.update(criteria, { $set: update });
+        logger.verbose('done updateBuildWithOutput');
     }
 
     // create build only if the build does not exist in database

TestResultSummaryService/EventHandler.js

@@ -40,7 +40,7 @@ class EventHandler {
                     logger.debug(
                         'EventHandler: processBuild() is waiting for 1 secs before processing the next build'
                     );
-                    await Promise.delay(1 * 1000);
+                    await Promise.delay(0.5 * 1000);
                 }
             } catch (e) {
                 logger.error('Exception in database query: ', e);

TestResultSummaryService/JenkinsInfo.js

@@ -69,7 +69,7 @@ class JenkinsInfo {
         });
         const size = await logStream.getSize();
         logger.debug(
-            'JenkinsInfo: getBuildOutput() is waiting for 5 secs after getSize()'
+            'JenkinsInfo: getBuildOutput() is waiting for 1 secs after getSize()'
         );
         await Promise.delay(1 * 1000);
 

TestResultSummaryService/parsers/Parser.js

@@ -14,7 +14,7 @@ class Parser {
             /=JAVA VERSION OUTPUT BEGIN=[\r\n]+([\s\S]*?)[\r\n]+.*=JAVA VERSION OUTPUT END=/;
         const javaBuildDateRegex =
             /\s([0-9]{4})-?(0[1-9]|1[012])-?(0[1-9]|[12][0-9]|3[01])/;
-        const sdkResourceRegex = /.*?SDK_RESOURCE\=(.*)[\r\n]+/;
+        const sdkResourceRegex = /SDK_RESOURCE\=(.*)[\r\n]+/;
         let curRegexResult = null;
         let javaVersion, jdkDate, sdkResource;
         if ((curRegexResult = javaVersionRegex.exec(output)) !== null) {
@@ -142,7 +142,7 @@ class Parser {
         let disabled = 0;
         // An example of test result summary: "TOTAL: 69   EXECUTED: 64   PASSED: 64   FAILED: 0   DISABLED: 0   SKIPPED: 5\n"
         const summaryRegex =
-            /\S*\s*?TOTAL:\s*([0-9]*)\s*EXECUTED:\s*([0-9]*)\s*PASSED:\s*([0-9]*)\s*FAILED:\s*([0-9]*)\s*DISABLED:\s*([0-9]*)\s*SKIPPED:\s*([0-9]*)\s*(\r\n|\r|\n)/;
+            /TOTAL:\s*([0-9]*)\s*EXECUTED:\s*([0-9]*)\s*PASSED:\s*([0-9]*)\s*FAILED:\s*([0-9]*)\s*DISABLED:\s*([0-9]*)\s*SKIPPED:\s*([0-9]*)\s*(\r\n|\r|\n)/;
         if ((m = summaryRegex.exec(output)) !== null) {
             total = parseInt(m[1], 10);
             executed = parseInt(m[2], 10);

TestResultSummaryService/parsers/Test.js

@@ -1,8 +1,8 @@
 const Parser = require('./Parser');
-const regexRunningTest = /.*?Running test (.*?) \.\.\.\r?/;
-const testSeparator = /.*?===============================================\r?/;
-const regexFinishTime = /(.*?) Finish Time\: .* Epoch Time \(ms\)\: (\d+).*/;
-const regexStartTime = /(.*?) Start Time\: .* Epoch Time \(ms\)\: (\d+).*/;
+const regexRunningTest = /Running test (.*?) \.\.\.\r?/;
+const testSeparator = /===============================================\r?/;
+const regexFinishTime = /^(.*?) Finish Time\: .* Epoch Time \(ms\)\: (\d+).*/;
+const regexStartTime = /^(.*?) Start Time\: .* Epoch Time \(ms\)\: (\d+).*/;
 const TestBenchmarkParser = require(`./TestBenchmarkParser`);
 const ExternalTestParser = require(`./ExternalTestParser`);
 
@@ -44,6 +44,7 @@ class Test extends Parser {
     }
 
     async extract(str) {
+        console.log('extract');
         const preTest = 'Pre Test';
         const postTest = 'Post Test';
         let m,
@@ -121,6 +122,7 @@ class Test extends Parser {
                                 ? finishTime - startTime
                                 : null,
                         startTime: parseInt(startTime),
+                        finishTime: parseInt(finishTime),
                     });
                     if (testResult == 'FAILED') {
                         executed++;
@@ -187,13 +189,16 @@ class Test extends Parser {
         );
         if (isPerf) {
             results = TestBenchmarkParser.parsePerf(results);
+            console.log('TestBenchmarkParser.parsePerf done');
             buildResult = Utils.perfBuildResult(results);
+            console.log('Utils.perfBuildResult done');
         }
 
         const isExternal = ExternalTestParser.canParse(this.buildName);
         if (isExternal) {
             results = new ExternalTestParser().parseExternal(results);
         }
+        console.log('isExternal done');
 
         return {
             tests: results,

TestResultSummaryService/parsers/TestBenchmarkParser.js

@@ -41,13 +41,16 @@ class TestBenchmarkParser extends Parser {
                         this.getBenchmarkInfo(result.testName);
                     result.benchmarkName = benchmarkName;
                     result.benchmarkVariant = benchmarkVariant;
+                    console.log('parsePerf', benchmarkName);
                     result.testData = Utils.parseOutput(
                         benchmarkParserKey,
                         result.testOutput
                     );
+                    console.log('benchmarkParserKey', benchmarkParserKey);
                 }
             }
         }
+        console.log('parsePerf done');
         return results;
     }
 }

TestResultSummaryService/parsers/Utils.js

@@ -34,6 +34,10 @@ class Utils {
             // if outerRegex is undefined, all runs should be measured. Parse metrics in every run
             // if outerRegex is defined, any runs before outerRegex will be ignored. Parse metrics in warm runs only
             if (curBenchVariant.outerRegex !== undefined) {
+                console.log(
+                    'curBenchVariant.outerRegex',
+                    curBenchVariant.outerRegex
+                );
                 if (
                     (curRegexResult =
                         curBenchVariant.outerRegex.exec(testOutput)) !== null