Coverage check (#17)

* Added coverage increase check for head commit on pull request events

* Bug fixes

* Logs archived #15 & log returned on GET id

Author: k1o0

CHANGELOG.md

@@ -4,11 +4,19 @@
 
 ### Added
 
+### Modified
+
+- Fix'd description field for pending status response
+
+## [2.0.0]
+### Added
+
 - changelog
 - status and coverage endpoints for shields
+- coverage increase check on pull request events
 
 ### Modified
 
 - changed from using Smee client to Serveo for exposing ZTEST
 - fixes for test reports endpoint
-- tests now performed only on head commit
+- tests now performed only on head commit

index.js

@@ -74,24 +74,99 @@ srv.post('/github', async (req, res, next) => {
 
 /**
  * Load MATLAB test results from db.json file.
- * @param {string} id - Function to call with job and done callback when.
+ * @param {string, array} id - Function to call with job and done callback when.
  */
 function loadTestRecords(id) {
   let obj = JSON.parse(fs.readFileSync('./db.json', 'utf8'));
   if (!Array.isArray(obj)) obj = [obj]; // Ensure array
-  return obj.find(o => o.commit === id);
+  let records = obj.filter(o => id.includes(o.commit));
+  // If single arg return as object, otherwise keep as array
+  return (!Array.isArray(id) && records.length === 1 ? records[0] : records)
+};
+
+/**
+ * Compare coverage of two commits and post a failed status if coverage of head commit <= base commit.
+ * @param {object} data - job data object with coverage field holding head and base commit ids.
+ */
+function compareCoverage(data) {
+  let ids = data.coverage;
+  let status, description;
+  let records = loadTestRecords(Object.values(ids));
+  // Filter duplicates just in case
+  records = records.filter((set => o => !set.has(o.commit) && set.add(o.commit))(new Set));
+  has_coverage = records.every(o => (typeof o.coverage !== 'undefined' && o.coverage > 0));
+  // Check if any errored or failed to update coverage
+  if (records.filter(o => o.status === 'error').length > 0) {
+    status = 'failure';
+    description = 'Failed to determine coverage as tests incomplete due to errors';
+  } else if (records.length === 2 && has_coverage) {
+    // Ensure first record is for head commit
+    if (records[0].commit === ids.base) { records.reverse() };
+    // Calculate coverage change
+    let coverage = records[0].coverage - records[1].coverage;
+    status = (coverage > 0 ? 'success' : 'failure');
+    description = 'Coverage ' + (coverage > 0 ? 'increased' : 'decreased')
+                              + ' from ' + Math.round(records[1].coverage*100)/100 + '%'
+                              + ' to ' + Math.round(records[0].coverage*100)/100 + '%';
+  } else {
+    for (let commit in ids) {
+       // Check test isn't already on the pile
+       let job = queue.pile.filter(o => o.data.sha === ids[commit]);
+       if (job.length > 0) { // Already on pile
+          // Add coverage key to job data structure
+          if (typeof job[0].data.coverage === 'undefined') { job[0].data.coverage = ids; };
+       } else { // Add test to queue
+          queue.add({
+             skipPost: true,
+             sha: ids[commit],
+             owner: 'cortex-lab', // @todo Generalize repo owner
+             repo: data.repo,
+             status: '',
+             context: '',
+             coverage: ids // Note cf commit
+          });
+       }
+    }
+    return;
+  };
+  // Post a our coverage status
+  request('POST /repos/:owner/:repo/statuses/:sha', {
+          owner: 'cortex-lab',
+          repo: data.repo,
+          headers: {
+              authorization: `token ${installationAccessToken}`,
+              accept: 'application/vnd.github.machine-man-preview+json'
+          },
+          sha: ids.head,
+          state: status,
+          target_url: `${process.env.WEBHOOK_PROXY_URL}/events/${ids.head}`, // fail
+          description: description,
+          context: 'coverage/ZTEST'
+  });
 };
 
 // Serve the test results for requested commit id
 srv.get('/github/:id', function (req, res) {
-  console.log('Request for test results for commit ' + req.params.id.substring(0,6))
+  console.log('Request for test log for commit ' + req.params.id.substring(0,6))
+  let log = `.\\src\\matlab_tests-${req.params.id}.log`;
+  fs.readFile(log, 'utf8', (err, data) => {
+    if (err) {
+    	res.statusCode = 404;
+    	res.send(`Record for commit ${req.params.id} not found`);
+    } else {
+    	res.statusCode = 200;
+      res.send(data);
+    }
+  });
+  /*
   const record = loadTestRecords(req.params.id);
   if (typeof record == 'undefined') {
   	res.statusCode = 404;
   	res.send(`Record for commit ${req.params.id} not found`);
   } else {
   	res.send(record['results']);
   }
+  */
 });
 
 // Serve the coverage results
@@ -214,7 +289,7 @@ queue.process(async (job, done) => {
            runTests.kill();
       	  done(new Error('Job stalled')) }, 5*60000);
      let args = ['-r', `runAllTests (""${job.data.sha}"",""${job.data.repo}"")`,
-       '-wait', '-log', '-nosplash', '-logfile', 'matlab_tests.log'];
+       '-wait', '-log', '-nosplash', '-logfile', `.\\src\\matlab_tests-${job.data.sha}.log`];
      runTests = cp.execFile('matlab', args, (error, stdout, stderr) => {
        clearTimeout(timer);
        if (error) { // Send error status
@@ -240,6 +315,11 @@ queue.process(async (job, done) => {
  */
 queue.on('finish', job => { // On job end post result to API
   console.log(`Job ${job.id} complete`)
+  // If job was part of coverage test and error'd, call compare function 
+  // (otherwise this is done by the on complete callback after writing coverage to file)
+  if (typeof job.data.coverage !== 'undefined' && job.data['status'] == 'error') { 
+    compareCoverage(job.data); 
+  };
   if (job.data.skipPost === true) { return; }
   request("POST /repos/:owner/:repo/statuses/:sha", {
     owner: job.data['owner'],
@@ -275,7 +355,9 @@ queue.on('complete', job => { // On job end post result to API
     for (let o of records) { if (o.commit === job.data.sha) {o.coverage = hits / (hits + misses) * 100; break; }} // Add percentage
     // Save object
     fs.writeFile('./db.json', JSON.stringify(records), function(err) {
-    if (err) { console.log(err); }
+    if (err) { console.log(err); return; }
+    // If this test was to ascertain coverage, call comparison function
+    if (typeof job.data.coverage !== 'undefined') { compareCoverage(job.data); };
     });
   });
 });
@@ -293,7 +375,7 @@ handler.on('push', async function (event) {
   console.log('Received a push event for %s to %s',
     event.payload.repository.name,
     event.payload.ref)
-  // Ignore documentaion branches
+  // Ignore documentation branches
   if (event.payload.ref.endsWith('documentation')) { return; }
   try { // Run tests for head commit only
     let head_commit = event.payload.head_commit.id;
@@ -308,7 +390,7 @@ handler.on('push', async function (event) {
             sha: head_commit,
             state: 'pending',
             target_url: `${process.env.WEBHOOK_PROXY_URL}/events/${head_commit}`, // fail
-            description: 'Tests error',
+            description: 'Tests running',
             context: 'continuous-integration/ZTEST'
     });
     // Add a new test job to the queue
@@ -322,6 +404,57 @@ handler.on('push', async function (event) {
   } catch (error) {console.log(error)}
 });
 
+// Handle pull request events
+// Here we'll update coverage
+handler.on('pull_request', async function (event) {
+  // Log the event
+  console.log('Received a pull_request event for %s to %s',
+    event.payload.pull_request.head.repo.name,
+    event.payload.pull_request.head.ref)
+  if (!event.payload.action.endsWith('opened') && event.payload.action !== 'synchronize') { return; }
+  try { // Compare test coverage
+    let head_commit = event.payload.pull_request.head.sha;
+    let base_commit = event.payload.pull_request.base.sha;
+    if (false) { // TODO for alyx only
+      // Post a 'pending' status while we do our tests
+      await request('POST /repos/:owner/:repo/statuses/:sha', {
+          owner: 'cortex-lab',
+          repo: event.payload.repository.name,
+          headers: {
+              authorization: `token ${installationAccessToken}`,
+              accept: 'application/vnd.github.machine-man-preview+json'
+          },
+          sha: head_commit,
+          state: 'pending',
+          target_url: `${process.env.WEBHOOK_PROXY_URL}/events/${head_commit}`, // fail
+          description: 'Tests running',
+          context: 'continuous-integration/ZTEST'
+      });
+    };
+
+    // Post a 'pending' status while we do our tests
+    request('POST /repos/:owner/:repo/statuses/:sha', {
+          owner: 'cortex-lab',
+          repo: event.payload.repository.name,
+          headers: {
+              authorization: `token ${installationAccessToken}`,
+              accept: 'application/vnd.github.machine-man-preview+json'
+          },
+          sha: head_commit,
+          state: 'pending',
+          target_url: `${process.env.WEBHOOK_PROXY_URL}/events/${head_commit}`, // fail
+          description: 'Checking coverage',
+          context: 'coverage/ZTEST'
+    });
+    // Check coverage exists
+    let data = {
+      repo: event.payload.repository.name, 
+      coverage: {head: head_commit, base: base_commit}
+    };
+    compareCoverage(data);
+  } catch (error) {console.log(error)}
+});
+
 // Start the server in the port 3000
 var server = srv.listen(3000, function () {
    var host = server.address().address