Merge branch 'harmony-1978' of https://github.com/nasa/harmony into harmony-1978

Author: vinnyinverso

db/db.sql

@@ -41,6 +41,13 @@ CREATE TABLE `job_errors` (
   `jobID` char(36) not null,
   `url` varchar(4096) not null,
   `message` varchar(4096) not null,
+  `level` varchar(255) check (
+  `level` in (
+    'error',
+    'warning'
+  )
+  ) not null default 'error',
+  `category` varchar(255),
   `createdAt` datetime not null,
   `updatedAt` datetime not null,
   FOREIGN KEY(jobID) REFERENCES jobs(jobID)
@@ -80,7 +87,8 @@ CREATE TABLE `work_items` (
   `workflowStepIndex` integer not null,
   `scrollID` varchar(4096),
   `serviceID` varchar(255) not null,
-  `status` text check (`status` in ('ready', 'queued', 'running', 'successful', 'failed', 'canceled')) not null,
+  `status` varchar(255) check (`status` in ('ready', 'queued', 'running', 'successful', 'failed', 'canceled', 'warning')) not null,
+  `sub_status` varchar(255),
   `stacCatalogLocation` varchar(255),
   `totalItemsSize` double precision not null default 0,
   `outputItemSizesJson` text,
@@ -183,9 +191,12 @@ CREATE INDEX jobs_username_idx ON jobs(jobID, username);
 CREATE INDEX job_links_jobID_idx ON job_links(jobID);
 CREATE INDEX job_links_jobID_id_idx ON job_links(jobID, id);
 CREATE INDEX job_errors_jobID_idx ON job_errors(jobID);
+CREATE INDEX job_errors_level_idx ON job_errors(level);
+CREATE INDEX job_errors_category_idx ON job_errors(category);
 CREATE INDEX work_items_jobID_idx ON work_items(jobID);
 CREATE INDEX work_items_serviceID_idx ON work_items(serviceID);
 CREATE INDEX work_items_status_idx ON work_items(status);
+CREATE INDEX work_items_sub_status_idx ON work_items(sub_status);
 CREATE INDEX workflow_steps_jobID_idx ON workflow_steps(jobID);
 CREATE INDEX workflow_steps_jobID_StepIndex_idx ON workflow_steps(jobID, stepIndex);
 CREATE INDEX workflow_steps_serviceID_idx ON workflow_steps(serviceID);

db/migrations/20250122190038_add_warning_status_and_sub_status_to_work_items.js

@@ -0,0 +1,30 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function (knex, Promise) {
+  return knex.schema.raw(`
+    ALTER TABLE "work_items"
+    DROP CONSTRAINT "work_items_status_check",
+    ADD CONSTRAINT "work_items_status_check"
+    CHECK (status IN ('ready', 'queued', 'running', 'successful', 'failed', 'canceled', 'warning')),
+    ADD COLUMN "sub_status" VARCHAR(255);
+
+    CREATE INDEX work_items_sub_status_index ON work_items (sub_status);
+  `);
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function (knex) {
+  return knex.schema.raw(`
+    DROP INDEX work_items_sub_status_index;
+    ALTER TABLE "work_items"
+    DROP COLUMN "sub_status",
+    DROP CONSTRAINT "work_items_status_check",
+    ADD CONSTRAINT "work_items_status_check"
+    CHECK (status IN ('ready', 'queued', 'running', 'successful', 'failed', 'canceled'));
+  `);
+};

db/migrations/20250124155544_add_level_and_category_to_job_errors.js

@@ -0,0 +1,31 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function (knex, Promise) {
+  return knex.schema.raw(`
+    ALTER TABLE "job_errors"
+    ADD COLUMN "level" VARCHAR(255) DEFAULT 'error' NOT NULL,
+    ADD CONSTRAINT "job_errors_level_check"
+    CHECK (level IN ('error', 'warning')),
+    ADD COLUMN "category" VARCHAR(255);
+
+    CREATE INDEX job_errors_level_index ON job_errors (level);
+    CREATE INDEX job_errors_category_index ON job_errors (category);
+  `);
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function (knex) {
+  return knex.schema.raw(`
+    DROP INDEX job_errors_category_index;
+    DROP INDEX job_errors_level_index;
+    ALTER TABLE "job_errors"
+    DROP COLUMN "category",
+    DROP CONSTRAINT "job_errors_level_check",
+    DROP COLUMN "level";
+  `);
+};

docs/guides/adapting-new-services.md

@@ -187,6 +187,7 @@ The structure of an entry in the [services.yml](../../config/services.yml) file
       - image/png
       - image/gif
     reprojection: true            # The service supports reprojection
+  validate_variables: true        # Whether to validate the requested variables exist in the CMR. Defaults to true.
   steps:
       - image: !Env ${QUERY_CMR_IMAGE} # The image to use for the first step in the chain
       - image: !Env ${HARMONY_EXAMPLE_IMAGE}     # The image to use for the second step in the chain

packages/util/tsconfig.build.json

@@ -10,7 +10,10 @@
       "allowJs": true,
       "noImplicitAny": false,
       "sourceMap": true,
-      "outDir": "built"
+      "outDir": "built",
+      "typeRoots": [
+        "node_modules/@types"
+      ]
   },
   "include": ["."],
   "exclude": ["./node_modules", "coverage"]

packages/util/tsconfig.json

@@ -10,7 +10,10 @@
       "allowJs": true,
       "noImplicitAny": false,
       "sourceMap": true,
-      "outDir": "built"
+      "outDir": "built",
+      "typeRoots": [
+        "node_modules/@types"
+      ],
   },
   "include": ["."],
   "exclude": ["./node_modules", "coverage"]

services/harmony/app/backends/workflow-orchestration/work-item-updates.ts

@@ -552,7 +552,8 @@ export async function preprocessWorkItem(
   let outputItemSizes;
   let catalogItems;
   try {
-    if (status === WorkItemStatus.SUCCESSFUL && !nextWorkflowStep) {
+    // TODO fix this in HARMONY-1995
+    if ([WorkItemStatus.SUCCESSFUL, WorkItemStatus.WARNING].includes(status) && !nextWorkflowStep) {
       // if we are CREATING STAC CATALOGS for the last step in the chain we should read the catalog items
       // since they are needed for generating the output links we will save
       catalogItems = await readCatalogsItems(results);
@@ -608,13 +609,13 @@ export async function processWorkItem(
 ): Promise<void> {
   const { jobID } = job;
   const { status, errorMessage, catalogItems, outputItemSizes } = preprocessResult;
-  const { workItemID, hits, results, scrollID } = update;
+  const { workItemID, hits, results, scrollID, sub_status } = update;
   const startTime = new Date().getTime();
   let durationMs;
   let jobSaveStartTime;
   let didCreateWorkItem = false;
-  if (status === WorkItemStatus.SUCCESSFUL) {
-    logger.info(`Updating work item ${workItemID} to ${status}`);
+  if (status === WorkItemStatus.SUCCESSFUL || status === WorkItemStatus.WARNING) {
+    logger.info(`Updating work item ${workItemID} to ${status} | ${sub_status}`);
   }
 
   try {
@@ -700,6 +701,7 @@ export async function processWorkItem(
       tx,
       workItemID,
       status,
+      sub_status,
       duration,
       totalItemsSize,
       outputItemSizes);
@@ -711,6 +713,7 @@ export async function processWorkItem(
     logger.info(`Updated work item. Duration (ms) was: ${duration}`);
 
     workItem.status = status;
+    workItem.sub_status = sub_status;
 
     let allWorkItemsForStepComplete = false;
 

services/harmony/app/backends/workflow-orchestration/workflow-orchestration.ts

@@ -96,6 +96,7 @@ export async function updateWorkItem(req: HarmonyRequest, res: Response): Promis
   const { id } = req.params;
   const {
     status,
+    sub_status,
     hits,
     results,
     scrollID,
@@ -116,6 +117,7 @@ export async function updateWorkItem(req: HarmonyRequest, res: Response): Promis
   const update = {
     workItemID: parseInt(id),
     status,
+    sub_status,
     hits,
     results,
     scrollID,

services/harmony/app/frontends/capabilities.ts

@@ -85,7 +85,7 @@ async function loadCollectionInfo(req: HarmonyRequest): Promise<CmrCollection> {
     collections = await getCollectionsByShortName(req.context, shortname, req.accessToken);
     if (collections.length === 0) {
       const message = `Unable to find collection short name ${shortname} in the CMR. Please `
-        + ' make sure the short name is correct and that you have access to the collection.';
+        + 'make sure the short name is correct and that you have access to the collection.';
       throw new NotFoundError(message);
     }
     pickedCollection = collections[0];

services/harmony/app/frontends/ogc-coverages/describe-collections.ts

@@ -4,7 +4,7 @@ import HarmonyRequest from '../../models/harmony-request';
 import { RequestValidationError } from '../../util/errors';
 import { keysToLowerCase } from '../../util/object';
 import { getSanitizedRequestUrl } from '../../util/url';
-import { parseVariables, fullPath } from '../../util/variables';
+import { parseVariables, fullPath, getVariableInfo } from '../../util/variables';
 import { getServiceConfigs } from '../../models/services';
 
 const WGS84 = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84';
@@ -72,7 +72,7 @@ function buildCollectionInfo(
  * Get a list of variables that are defined in the service configs as being available for
  * processing for the given collection. If the returned set is empty this means there are no
  * limits set as to which variables a service will process.
- * @param collection - the CMR collection 
+ * @param collection - the CMR collection
  * @returns A Set of variable IDs obtained from service configs
  */
 function getSupportedVariablesForCollection(
@@ -111,7 +111,7 @@ export function describeCollections(req: HarmonyRequest, res: Response): void {
   const ogcCollections = [];
   const requestUrl = getSanitizedRequestUrl(req, false);
   const ogcApiRoot = requestUrl.replace(/\/collections$/, '');
-  for (const collection of req.collections) {
+  for (const collection of req.context.collections) {
     const collectionShortLabel = `${collection.short_name} v${collection.version_id}`;
     const rootLink = {
       title: `OGC coverages API root for ${collectionShortLabel}`,
@@ -136,7 +136,7 @@ export function describeCollections(req: HarmonyRequest, res: Response): void {
     }
     for (const variable of collection.variables) {
       // if a service has limited the variables for the collection, only allow variables in that
-      // set 
+      // set
       if (supportedVariables.has(variable.meta['concept-id']) || supportedVariables.size === 0) {
         const collectionInfo = buildCollectionInfo(
           collection, variable, `${requestUrl}/${encodeURIComponent(fullPath(variable))}`, extent,
@@ -165,11 +165,12 @@ export function describeCollection(req: HarmonyRequest, res: Response): void {
   if (query.f && query.f !== 'json') {
     throw new RequestValidationError(`Unsupported format "${query.f}". Currently only the json format is supported.`);
   }
-  const collection = req.collections[0];
+  const collection = req.context.collections[0];
   const requestUrl = getSanitizedRequestUrl(req, false);
   const extent = generateExtent(collection);
-  const variableInfo = parseVariables([collection], req.params.collectionId);
-  const variable = variableInfo[0].variables[0];
+  const requestedVariableId = parseVariables(req.params.collectionId, []);
+  const variableInfos = getVariableInfo([collection], requestedVariableId, true);
+  const variable = variableInfos[0].variables[0];
   const collectionInfo = buildCollectionInfo(collection, variable, requestUrl, extent);
   res.send(collectionInfo);
 }