Harmony 1963 - data size change information in job status page

services/harmony/app/frontends/jobs.ts

@@ -6,6 +6,7 @@ import HarmonyRequest from '../models/harmony-request';
 import { getRelatedLinks, Job, JobForDisplay, JobQuery, JobStatus } from '../models/job';
 import JobLink from '../models/job-link';
 import JobMessage, { getMessagesForJob, JobMessageLevel } from '../models/job-message';
+import { getTotalWorkItemSizesForJobID } from '../models/work-item';
 import db from '../util/db';
 import { isAdminUser } from '../util/edl-api';
 import env from '../util/env';
@@ -171,6 +172,58 @@ export async function getJobsListing(
   }
 }
 
+/**
+ * Get a message explaining the change in size from the input to the output
+ *
+ * @param sizes - original and output sizes of the input in MiB (1024 x 1024 bytes)
+ * @param precision - the number of decimal places to allow in the output
+ * @returns a message explaining the size change as a percentage
+ */
+export function sizeChangeMessage(
+  sizes: { originalSize: number; outputSize: number; },
+  precision: number = 2): string {
+  if (sizes.originalSize === 0) {
+    return 'Original size is 0 - percent size change N/A';
+  }
+  if (sizes.outputSize === 0) {
+    return 'Output size is 0 - percent size change N/A';
+  }
+  let result: string;
+  const diff = sizes.originalSize - sizes.outputSize;
+  if (diff < 0) {
+    const percent = (-diff / sizes.originalSize * 100.0).toFixed(precision);
+    result = `${percent}% increase`;
+  } else if (diff > 0) {
+    let percent = (diff / sizes.originalSize * 100.0).toFixed(precision);
+    // due to JS precision issues, big changes will appear to be 100% reduction, which is impossible
+    if (percent === '100.00') percent = '99.99';
+
+    result = `${percent}% reduction`;
+  } else {
+    result = 'no change';
+  }
+
+  return result;
+}
+
+/**
+ * Format a data size number as a string for human presentation
+ * @param mibSize - the float size in MiB (1024x1024 bytes)
+ * @param precision - the number of decimal places to allow in the output
+ * @returns a string representing the size using B, KiB, MiB, etc., notation
+ */
+export function formatDataSize(mibSize: number, precision: number = 2): string {
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
+  let size = mibSize * 1024 * 1024;
+  let unitIndex = 0;
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex++;
+  }
+
+  return `${size.toFixed(precision)} ${units[unitIndex]}`;
+}
+
 /**
  * Express.js handler that returns job status for a single job `(/jobs/{jobID})`
  *
@@ -189,17 +242,18 @@ export async function getJobStatus(
   try {
     validateJobId(jobID);
     const { page, limit } = getPagingParams(req, env.defaultResultPageSize);
-    let job: Job;
-    let pagination;
-    let messages: JobMessage[];
+    let workItemsSizes: { originalSize: number; outputSize: number; };
 
-    await db.transaction(async (tx) => {
-      ({ job, pagination } = await Job.byJobID(tx, jobID, true, true, false, page, limit));
-      messages = await getMessagesForJob(tx, jobID);
-    });
+    const { job, pagination } = await Job.byJobID(db, jobID, true, true, false, page, limit);
     if (!job) {
       throw new NotFoundError(`Unable to find job ${jobID}`);
     }
+    const messages: JobMessage[] = await getMessagesForJob(db, jobID);
+    // only get data reduction numbers when the job is complete and at least partially successful
+    if ([JobStatus.SUCCESSFUL, JobStatus.COMPLETE_WITH_ERRORS].includes(job.status)) {
+      workItemsSizes = await getTotalWorkItemSizesForJobID(db, jobID);
+    }
+
     const isAdmin = await isAdminUser(req);
     const isAdminOrOwner = job.belongsToOrIsAdmin(req.user, isAdmin);
     const isJobShareable = await job.isShareable(req.accessToken);
@@ -210,6 +264,11 @@ export async function getJobStatus(
     const pagingLinks = getPagingLinks(req, pagination).map((link) => new JobLink(link));
     job.links = job.links.concat(pagingLinks);
     const jobForDisplay = getJobForDisplay(job, urlRoot, linkType, messages);
+    if (workItemsSizes) {
+      jobForDisplay.originalDataSize = formatDataSize(workItemsSizes.originalSize);
+      jobForDisplay.outputDataSize = formatDataSize(workItemsSizes.outputSize);
+      jobForDisplay.dataSizePercentChange = sizeChangeMessage(workItemsSizes);
+    }
     res.send(jobForDisplay);
   } catch (e) {
     req.context.logger.error(e);

services/harmony/app/models/job.ts

@@ -114,6 +114,12 @@ export class JobForDisplay {
 
   numInputGranules: number;
 
+  originalDataSize?: string;
+
+  outputDataSize?: string;
+
+  dataSizePercentChange?: string;
+
   errors?: JobMessage[];
 
   warnings?: JobMessage[];

services/harmony/app/models/work-item.ts

@@ -2,16 +2,19 @@
 import { subMinutes } from 'date-fns';
 import { ILengthAwarePagination } from 'knex-paginate';
 import _ from 'lodash';
-import logger from '../util/log';
+
+import { getWorkSchedulerQueue } from '../../app/util/queue/queue-factory';
+import { eventEmitter } from '../events';
 import db, { Transaction } from '../util/db';
-import DataOperation from './data-operation';
 import env from '../util/env';
+import logger from '../util/log';
+import DataOperation from './data-operation';
 import { Job, JobStatus } from './job';
 import Record from './record';
+import {
+  getStacLocation, WorkItemQuery, WorkItemRecord, WorkItemStatus,
+} from './work-item-interface';
 import WorkflowStep from './workflow-steps';
-import { WorkItemRecord, WorkItemStatus, getStacLocation, WorkItemQuery } from './work-item-interface';
-import { eventEmitter } from '../events';
-import { getWorkSchedulerQueue } from '../../app/util/queue/queue-factory';
 
 // The step index for the query-cmr task. Right now query-cmr only runs as the first step -
 // if this changes we will have to revisit this

services/harmony/test/helpers/work-items.ts

@@ -1,15 +1,20 @@
 import { Application } from 'express';
+import _ from 'lodash';
 import { afterEach, beforeEach } from 'mocha';
 import request, { Test } from 'supertest';
-import _ from 'lodash';
+
+import { RecordConstructor } from '../../app/models/record';
 import WorkItem from '../../app/models/work-item';
+import {
+  getStacLocation, WorkItemRecord, WorkItemStatus,
+} from '../../app/models/work-item-interface';
 import db, { Transaction } from '../../app/util/db';
+import { objectStoreForProtocol } from '../../app/util/object-store';
 import { truncateAll } from './db';
 import { hookBackendRequest } from './hooks';
-import { buildWorkflowStep, hookWorkflowStepCreation, hookWorkflowStepCreationEach } from './workflow-steps';
-import { RecordConstructor } from '../../app/models/record';
-import { WorkItemStatus, WorkItemRecord, getStacLocation } from '../../app/models/work-item-interface';
-import { objectStoreForProtocol } from '../../app/util/object-store';
+import {
+  buildWorkflowStep, hookWorkflowStepCreation, hookWorkflowStepCreationEach,
+} from './workflow-steps';
 
 export const exampleWorkItemProps = {
   jobID: '1',
@@ -45,7 +50,7 @@ export function buildWorkItem(fields: Partial<WorkItemRecord> = {}): WorkItem {
 
 /**
  * Save a work item without validating or updating createdAt/updatedAt
- * @param tx - The transaction to use for saving the job
+ * @param tx - The transaction to use for saving the work item
  * @param fields - The fields to save to the database, defaults to example values
  * @returns The saved work item
  * @throws Error - if the save to the database fails

services/harmony/test/jobs/jobs-status.ts

@@ -4,9 +4,11 @@ import { stub } from 'sinon';
 import request from 'supertest';
 import { v4 as uuid } from 'uuid';
 
+import { formatDataSize, sizeChangeMessage } from '../../app/frontends/jobs';
 import { EXPIRATION_DAYS, Job, JobStatus } from '../../app/models/job';
 import JobMessage, { JobMessageLevel } from '../../app/models/job-message';
 import { setLabelsForJob } from '../../app/models/label';
+import { WorkItemStatus } from '../../app/models/work-item-interface';
 import env from '../../app/util/env';
 import { hookDatabaseFailure, hookTransaction } from '../helpers/db';
 import { hookRedirect, hookUrl } from '../helpers/hooks';
@@ -19,6 +21,8 @@ import {
 import { hookRangesetRequest } from '../helpers/ogc-api-coverages';
 import hookServersStartStop from '../helpers/servers';
 import StubService from '../helpers/stub-service';
+import { buildWorkItem } from '../helpers/work-items';
+import { buildWorkflowStep } from '../helpers/workflow-steps';
 
 const aJob = buildJob({ username: 'joe' });
 const pausedJob = buildJob({ username: 'joe' });
@@ -32,6 +36,7 @@ const warningMessage = new JobMessage({
   message_category: 'nodata',
 });
 const warningJob = buildJob({ username: 'joe', status: JobStatus.SUCCESSFUL, message: 'Service could not get data', messages: [warningMessage] });
+const successfulJob = buildJob({ username: 'joe', status: JobStatus.SUCCESSFUL, message: 'Success' });
 
 const timeStampRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z/;
 
@@ -74,16 +79,39 @@ describe('Individual job status route', function () {
   hookTransaction();
   before(async function () {
     await aJob.save(this.trx);
+    console.log(`JOB ID: ${aJob.jobID}`);
     await setLabelsForJob(this.trx, aJob.jobID, aJob.username, aJobLabels);
     await pausedJob.save(this.trx);
     await previewingJob.save(this.trx);
     await warningJob.save(this.trx);
+    await successfulJob.save(this.trx);
+    const initialSuccessJobWorkflowStep = buildWorkflowStep({ jobID: successfulJob.jobID, stepIndex: 1 });
+    await initialSuccessJobWorkflowStep.save(this.trx);
+    const finalSuccessJobWorkflowStep = buildWorkflowStep({ jobID: successfulJob.jobID, stepIndex: 2 });
+    await finalSuccessJobWorkflowStep.save(this.trx);
+    const initialWarningJobWorkflowStep = buildWorkflowStep({ jobID: warningJob.jobID, stepIndex: 1 });
+    await initialWarningJobWorkflowStep.save(this.trx);
+    const finalWarningJobWorkflowStep = buildWorkflowStep({ jobID: warningJob.jobID, stepIndex: 2 });
+    await finalWarningJobWorkflowStep.save(this.trx);
+    const initialSuccessJobWorkItem1 = buildWorkItem({ jobID: successfulJob.jobID, workflowStepIndex: 1, status: WorkItemStatus.SUCCESSFUL, totalItemsSize: 1.3701868057250977 });
+    await initialSuccessJobWorkItem1.save(this.trx);
+    const finalSuccessJobWorkItem1 = buildWorkItem({ jobID: successfulJob.jobID, workflowStepIndex: 2, status: WorkItemStatus.WARNING, totalItemsSize: 0.09036064147949219 });
+    await finalSuccessJobWorkItem1.save(this.trx);
+    const initialSuccessJobWorkItem2 = buildWorkItem({ jobID: successfulJob.jobID, workflowStepIndex: 1, status: WorkItemStatus.SUCCESSFUL, totalItemsSize: 1.7018680517250977 });
+    await initialSuccessJobWorkItem2.save(this.trx);
+    const finalSuccessJobWorkItem2 = buildWorkItem({ jobID: successfulJob.jobID, workflowStepIndex: 2, status: WorkItemStatus.WARNING, totalItemsSize: 0.09361064147949219 });
+    await finalSuccessJobWorkItem2.save(this.trx);
+    const initialWarningJobWorkItem = buildWorkItem({ jobID: warningJob.jobID, workflowStepIndex: 1, status: WorkItemStatus.SUCCESSFUL, totalItemsSize: 1.3701868057250977 });
+    await initialWarningJobWorkItem.save(this.trx);
+    const finalWarningJobWorkItem = buildWorkItem({ jobID: warningJob.jobID, workflowStepIndex: 2, status: WorkItemStatus.WARNING, totalItemsSize: 0.09036064147949219 });
+    await finalWarningJobWorkItem.save(this.trx);
     this.trx.commit();
   });
   const jobID = aJob.requestId;
   const pausedjobID = pausedJob.requestId;
   const previewingJobID = previewingJob.requestId;
   const warningJobID = warningJob.requestId;
+  const successJobID = successfulJob.requestId;
 
   describe('For a user who is not logged in', function () {
     before(async function () {
@@ -150,6 +178,13 @@ describe('Individual job status route', function () {
         expect(job.labels).deep.equal(['000', 'bar', 'foo', 'z-label']);
       });
 
+      it('does not include data size reduction information', function () {
+        const job = JSON.parse(this.res.text);
+        expect(job.originalDataSize).to.be.undefined;
+        expect(job.outputDataSize).to.be.undefined;
+        expect(job.dataSizePercentChange).to.be.undefined;
+      });
+
       itIncludesADataExpirationField();
     });
   }
@@ -173,6 +208,13 @@ describe('Individual job status route', function () {
       expect(resumeLinks.length).to.equal(0);
     });
 
+    it('does not include data size reduction information', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.originalDataSize).to.be.undefined;
+      expect(job.outputDataSize).to.be.undefined;
+      expect(job.dataSizePercentChange).to.be.undefined;
+    });
+
     itIncludesADataExpirationField();
   });
 
@@ -205,6 +247,52 @@ describe('Individual job status route', function () {
       expect(previewSkipLinks.length).to.equal(0);
     });
 
+    it('does not include data size reduction information', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.originalDataSize).to.be.undefined;
+      expect(job.outputDataSize).to.be.undefined;
+      expect(job.dataSizePercentChange).to.be.undefined;
+    });
+
+    itIncludesADataExpirationField();
+  });
+
+  describe('when the job is successful', function () {
+    hookJobStatus({ jobID: successJobID, username: 'joe' });
+
+    it('returns a status field of "successful"', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.status).to.eql('successful');
+    });
+
+    it('returns a human-readable message field corresponding to its state', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.message).to.include('Success');
+    });
+
+    it('does not include links for canceling and resuming the job', function () {
+      const job = new Job(JSON.parse(this.res.text));
+      const resumeLinks = job.getRelatedLinks('resumer');
+      expect(resumeLinks.length).to.equal(0);
+      const cancelLinks = job.getRelatedLinks('canceler');
+      expect(cancelLinks.length).to.equal(0);
+    });
+
+    it('does not include irrelevant state change links', function () {
+      const job = new Job(JSON.parse(this.res.text));
+      const pauseLinks = job.getRelatedLinks('pauser');
+      expect(pauseLinks.length).to.equal(0);
+      const previewSkipLinks = job.getRelatedLinks('preview-skipper');
+      expect(previewSkipLinks.length).to.equal(0);
+    });
+
+    it('includes data size reduction information', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.originalDataSize).to.eql('3.07 MiB');
+      expect(job.outputDataSize).to.eql('188.39 KiB');
+      expect(job.dataSizePercentChange).to.eql('94.01% reduction');
+    });
+
     itIncludesADataExpirationField();
   });
 
@@ -245,6 +333,13 @@ describe('Individual job status route', function () {
       expect(warnings[0].message).to.eql(warningMessage.message);
     });
 
+    it('includes data size reduction information', function () {
+      const job = JSON.parse(this.res.text);
+      expect(job.originalDataSize).to.eql('1.37 MiB');
+      expect(job.outputDataSize).to.eql('92.53 KiB');
+      expect(job.dataSizePercentChange).to.eql('93.41% reduction');
+    });
+
     itIncludesADataExpirationField();
   });
 
@@ -968,3 +1063,44 @@ describe('Individual job status route', function () {
     });
   });
 });
+
+describe('unit tests for size reduction calcuation functions', function () {
+  // loop over some test cases for formatDataSize and sizeChangeMessage
+  const testCases = [
+    [1.3701868057250977, 0.09036064147949219, '1.37 MiB', '92.53 KiB', '93.41% reduction'],
+    [100.29018617250945, 10.0149421, '100.29 MiB', '10.01 MiB', '90.01% reduction'],
+    [1234567890, 9876543, '1.15 PiB', '9.42 TiB', '99.20% reduction'],
+    [10241024, 10024.5, '9.77 TiB', '9.79 GiB', '99.90% reduction'],
+    [1024, 10024, '1.00 GiB', '9.79 GiB', '878.91% increase'],
+  ];
+
+  for (const testCase of testCases) {
+    const originalSize = +testCase[0];
+    const outputSize = +testCase[1];
+    it('formats MiB as a human readable string', function () {
+      expect(formatDataSize(originalSize)).to.eql(testCase[2]);
+      expect(formatDataSize(outputSize)).to.eql(testCase[3]);
+    });
+    it('provides a human-readable message about the size change of the data', function () {
+      expect(sizeChangeMessage({ originalSize, outputSize })).to.eql(testCase[4]);
+    });
+  }
+
+  describe('when the size reduction is very large', function () {
+    it('does not claim 100% size reduction', function () {
+      expect(sizeChangeMessage({ originalSize: 1234567890, outputSize: 1 })).to.eql('99.99% reduction');
+    });
+  });
+
+  describe('when the original size is zero', function () {
+    it('warns that the percent change does not apply', function () {
+      expect(sizeChangeMessage({ originalSize: 0, outputSize: 0 })).to.eql('Original size is 0 - percent size change N/A');
+    });
+  });
+
+  describe('when the output size is zero', function () {
+    it('warns that the percent change does not apply', function () {
+      expect(sizeChangeMessage({ originalSize: 100, outputSize: 0 })).to.eql('Output size is 0 - percent size change N/A');
+    });
+  });
+});