Job output transform

.github/workflows/tests.yml

@@ -124,7 +124,7 @@ jobs:
         if: ${{ matrix.test-case != 'test-docker' }}
         # install package and dependencies directly,
         # skip sys/conda setup to use active python
-        run: make install-sys install-pkg install-pip install-raw install-dev install-dev-npm
+        run: SUDO=sudo make install-sys install-pkg install-pip install-raw install-dev install-dev-npm
       - name: Display Packages
         # skip python setup if running with docker
         if: ${{ matrix.test-case != 'test-docker' }}

.readthedocs.yml

@@ -13,3 +13,4 @@ python:
   install:
     - requirements: requirements-sys.txt
     - requirements: requirements-doc.txt
+    - requirements: requirements-transform.txt

CHANGES.rst

@@ -12,11 +12,23 @@ Changes
 
 Changes:
 --------
-- No change.
-
-Fixes:
-------
-- No change.
+- Add support for various GeoTIFF formats, allowing flexible handling and representation of GeoTIFFs in outputs
+  (fixes `#100 <https://github.com/crim-ca/weaver/issues/100>`_).
+- Add support for ``GET /jobs/{jobId}/results/{id}`` and ``GET /jobs/{jobId}/outputs/{id}`` routes to enable direct access to
+  individual `Job` result items by ID. This enhancement includes support alternate representations based on the ``Accept`` header.
+  If an alternate format (e.g., YAML for a JSON source) is requested it will be automatically generated and returned.
+- Return ``Link`` headers with ``rel="alternate"`` containing all possible output formats, allowing retrieval via query parameters
+  (e.g., ``/jobs/{jobId}/outputs/{id}?f=application/x-yaml``) (fixes `#18 <https://github.com/crim-ca/weaver/issues/18>`_).
+- Add ``weaver.formats.OutputFormat.CSV`` format support and extend ``OutputFormat.convert()`` method to handle CSV conversions
+  from JSON data structures.
+- Add ``weaver.transform`` module providing format conversion handlers for `Job` outputs. Output formats are dynamically
+  extended from the original `Process` definition to provide alternate result representations without modifying the deployed
+  `Process` metadata.
+
+Fixes:
+------
+- Fix ``OutputFormat.convert()`` returning ``bytes`` instead of ``str`` for XML/HTML conversions, causing write errors when
+  saving transformation results to files.
 
 .. _changes_6.9.1:
 

Makefile

@@ -15,6 +15,7 @@ APP_INI     ?= $(APP_ROOT)/config/$(APP_NAME).ini
 # guess OS (Linux, Darwin,...)
 OS_NAME := $(shell uname -s 2>/dev/null || echo "unknown")
 CPU_ARCH := $(shell uname -m 2>/dev/null || uname -p 2>/dev/null || echo "unknown")
+SUDO ?=
 
 # conda
 CONDA_CMD      ?= __EMPTY__
@@ -227,10 +228,10 @@ conda-env-export:		## export the conda environment
 install: install-all    ## alias for 'install-all' target
 
 .PHONY: install-run
-install-run: conda-install install-sys install-pkg install-raw 	## install requirements and application to run locally
+install-run: conda-install install-sys install-pkg install-raw install-dev ## install requirements and application to run locally
 
 .PHONY: install-all
-install-all: conda-install install-sys install-pkg install-pip install-dev  ## install application with all dependencies
+install-all: conda-install install-sys install-pkg install-pip install-dev ## install application with all dependencies
 
 .PHONY: install-doc
 install-doc: install-pip	## install documentation dependencies
@@ -239,7 +240,7 @@ install-doc: install-pip	## install documentation dependencies
 	@echo "Install with pip complete. Run documentation generation with 'make docs' target."
 
 .PHONY: install-dev
-install-dev: install-pip	## install development and test dependencies
+install-dev: install-pip install-transform 	## install development and test dependencies
 	@echo "Installing development packages with pip..."
 	@bash -c '$(CONDA_CMD) pip install $(PIP_XARGS) -r "$(APP_ROOT)/requirements-dev.txt"'
 	@echo "Install with pip complete. Test service with 'make test*' variations."
@@ -273,7 +274,7 @@ install-raw:	## install without any requirements or dependencies (suppose everyt
 install-npm:	## install npm package manager and dependencies if they cannot be found
 	@[ -f "$(shell which npm)" ] || ( \
 		echo "Binary package manager npm not found. Attempting to install it."; \
-		apt-get install npm \
+		$(SUDO) apt-get install npm \
 	)
 
 .PHONY: install-npm-stylelint
@@ -290,6 +291,19 @@ install-npm-remarklint: install-npm		## install remark-lint dependency for 'chec
 		npm install --save-dev \
 	)
 
+.PHONY: install-transform
+install-transform: install-cairo-dependencies       # install-transform dependencies
+	@echo "Installing transformation dependencies..."
+	@bash -c '$(CONDA_CMD) pip install $(PIP_XARGS) -r "$(APP_ROOT)/requirements-transform.txt" --no-cache-dir'
+	@echo "Install with pip complete."
+
+.PHONY: install-cairo-dependencies
+install-cairo-dependencies:   ## install required dependencies for Transformer
+	@[ -f "$(shell which cairo)" ] || ( \
+		echo "Binary package manager cairo not found. Attempting to install it."; \
+		$(SUDO) apt-get install libpangocairo-1.0-0 \
+	)
+
 .PHONY: install-dev-npm
 install-dev-npm: install-npm install-npm-remarklint install-npm-remarklint	## install all npm development dependencies
 

SECURITY.md

@@ -8,9 +8,9 @@ Security and vulnerability reports should be submitted as an issue of type [Secu
 
 Patches for security vulnerabilities will be released for the following versions:
 
-| Version | Supported          |
-|---------|--------------------|
-| 4.x     | :white_check_mark: |
-| < 4.x   | :x:                |
+| Version | Supported |
+| ------- | --------- |
+| 4.x     | ✅         |
+| < 4.x   | ❌         |
 
 [svr]: https://github.com/crim-ca/weaver/issues/new?assignees=fmigneault&labels=triage%2Fsecurity&template=security.md

docker/Dockerfile-base

@@ -23,8 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         g++ \
         git \
         nodejs \
+        libpangocairo-1.0-0 \
     && pip install --no-cache-dir --upgrade -r requirements-sys.txt \
-    && pip install --no-cache-dir -r requirements.txt \
+    && pip install --no-cache-dir -r requirements.txt -r requirements-transform.txt \
     && pip install --no-cache-dir -e ${APP_DIR} \
     && apt-get remove -y \
         gcc \

requirements-transform.txt

@@ -0,0 +1,7 @@
+cairosvg
+fpdf
+multipagetiff
+pandas
+Pillow
+python-magic
+rasterio

tests/functional/test_cli.py

@@ -488,7 +488,11 @@ def test_describe(self):
         for out_fmt in output_formats:
             out_fmt.pop("$schema", None)
             out_fmt.pop("$id", None)
-        assert output_formats == [{"default": True, "mediaType": ContentType.TEXT_PLAIN}]
+        assert output_formats == [
+            {"default": True, "mediaType": ContentType.TEXT_PLAIN},
+            {"mediaType": ContentType.TEXT_HTML},
+            {"mediaType": ContentType.APP_PDF}
+        ]
         assert "undefined" not in result.message, "CLI should not have confused process description as response detail."
         assert result.body["description"] == (
             "Dummy process that simply echo's back the input message for testing purposes."
@@ -1564,6 +1568,13 @@ def test_deploy_payload_process_info_merged(self):
             out_cwl_fmt = {"default": False, "mediaType": io_fmt, "encoding": "base64"}
             out_oas_fmt = {"default": True, "mediaType": ContentType.APP_JSON}
             out_any_fmt = [out_cwl_fmt, out_oas_fmt]
+            # Alternative format added in process description
+            out_alt_fmt = [
+                {"mediaType": ContentType.TEXT_CSV},
+                {"mediaType": ContentType.APP_XML},
+                {"mediaType": ContentType.APP_YAML},
+            ]
+            out_any_fmt.extend(out_alt_fmt)
             # ignore schema specifications for comparison only of contents
             for field in ["$id", "$schema"]:
                 in_schema.pop(field, None)