Issue #391/#651 add docs, changelog, review tweaks

Author: soxofaan

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `MultiResult` helper class to build process graphs with multiple result nodes ([#391](https://github.com/Open-EO/openeo-python-client/issues/391))
+
 ### Changed
 
 ### Removed

docs/api.rst

@@ -47,6 +47,15 @@ openeo.rest.mlmodel
    :inherited-members:
 
 
+openeo.rest.multiresult
+-----------------------
+
+.. automodule:: openeo.rest.multiresult
+   :members: MultiResult
+   :inherited-members:
+   :special-members: __init__
+
+
 openeo.metadata
 ----------------
 

docs/datacube_construction.rst

@@ -196,3 +196,53 @@ Re-parameterization
 ```````````````````
 
 TODO
+
+
+
+.. _multi-result-process-graphs:
+Building process graphs with multiple result nodes
+===================================================
+
+.. note::
+    Multi-result support is added in version 0.35.0
+
+Most openEO use cases are just about building a single result data cube,
+which is readily covered in the openEO Python client library through classes like
+:py:class:`~openeo.rest.datacube.DataCube` and :py:class:`~openeo.rest.vectorcube.VectorCube`.
+It is straightforward to create a batch job from these, or execute/download them synchronously.
+
+The openEO API also allows multiple result nodes in a single process graph,
+for example to persist intermediate results or produce results in different output formats.
+To support this, the openEO Python client library provides the :py:class:`~openeo.rest.multiresult.MultiResult` class,
+which allows to group multiple :py:class:`~openeo.rest.datacube.DataCube` and :py:class:`~openeo.rest.vectorcube.VectorCube` objects
+in a single entity that can be used to create or run batch jobs. For example:
+
+
+.. code-block:: python
+
+    cube1 = ...
+    cube2 = ...
+    multi_result = MultiResult([cube1, cube2])
+    job = multi_result.create_job()
+
+
+Moreover, it is not necessary to explicitly create such a
+:py:class:`~openeo.rest.multiresult.MultiResult` object,
+as the :py:meth:`Connection.create_job() <openeo.rest.connection.Connection.create_job>` method
+directly supports passing multiple data cube objects in a list,
+which will be automatically grouped as a multi-result:
+
+.. code-block:: python
+
+    cube1 = ...
+    cube2 = ...
+    job = connection.create_job([cube1, cube2])
+
+
+.. important::
+
+    Only a single :py:class:`~openeo.rest.connection.Connection` can be in play
+    when grouping multiple results like this.
+    As everything is to be merged in a single process graph
+    to be sent to a single backend,
+    it is not possible to mix cubes created from different connections.

openeo/rest/_testing.py

@@ -3,7 +3,7 @@
 import collections
 import json
 import re
-from typing import Callable, Iterable, Iterator, Optional, Sequence, Union
+from typing import Callable, Iterable, Iterator, Optional, Sequence, Tuple, Union
 
 from openeo import Connection, DataCube
 from openeo.rest.vectorcube import VectorCube
@@ -82,7 +82,7 @@ def __init__(
         requests_mock.post(connection.build_url("/validation"), json=self._handle_post_validation)
 
     @classmethod
-    def at(cls, root_url: str, *, requests_mock, capabilities: Optional[dict] = None) -> DummyBackend:
+    def at_url(cls, root_url: str, *, requests_mock, capabilities: Optional[dict] = None) -> DummyBackend:
         """
         Factory to build dummy backend from given root URL
         including creation of connection and mocking of capabilities doc
@@ -92,12 +92,36 @@ def at(cls, root_url: str, *, requests_mock, capabilities: Optional[dict] = None
         connection = Connection(root_url)
         return cls(requests_mock=requests_mock, connection=connection)
 
-    def setup_collection(self, collection_id: str):
+    def setup_collection(
+        self,
+        collection_id: str,
+        *,
+        temporal: Union[bool, Tuple[str, str]] = True,
+        bands: Sequence[str] = ("B1", "B2", "B3"),
+    ):
         # TODO: also mock `/collections` overview
+        # TODO: option to override cube_dimensions as a whole, or override dimension names
+        cube_dimensions = {
+            "x": {"type": "spatial"},
+            "y": {"type": "spatial"},
+        }
+
+        if temporal:
+            cube_dimensions["t"] = {
+                "type": "temporal",
+                "extent": temporal if isinstance(temporal, tuple) else [None, None],
+            }
+        if bands:
+            cube_dimensions["bands"] = {"type": "bands", "values": list(bands)}
+
         self._requests_mock.get(
             self.connection.build_url(f"/collections/{collection_id}"),
             # TODO: add more metadata?
-            json={"id": collection_id},
+            json={
+                "id": collection_id,
+                # define temporal  and band dim
+                "cube:dimensions": {"t": {"type": "temporal"}, "bands": {"type": "bands"}},
+            },
         )
         return self
 
@@ -201,7 +225,6 @@ def get_batch_pg(self) -> dict:
     def get_validation_pg(self) -> dict:
         """
         Get process graph of the one and only validation request.
-        :return:
         """
         assert len(self.validation_requests) == 1
         return self.validation_requests[0]

openeo/rest/connection.py

@@ -1135,7 +1135,13 @@ def validate_process_graph(
         """
         Validate a process graph without executing it.
 
-        :param process_graph: (flat) dict representing process graph
+        :param process_graph: openEO-style (flat) process graph representation,
+            or an object that can be converted to such a representation:
+            a dictionary, a :py:class:`~openeo.rest.datacube.DataCube` object,
+            a string with a JSON representation,
+            a local file path or URL to a JSON representation,
+            a :py:class:`~openeo.rest.multiresult.MultiResult` object, ...
+
         :return: list of errors (dictionaries with "code" and "message" fields)
         """
         pg_with_metadata = self._build_request_with_process_graph(process_graph)["process"]
@@ -1754,8 +1760,12 @@ def create_job(
         """
         Create a new job from given process graph on the back-end.
 
-        :param process_graph: (flat) dict representing a process graph, or process graph as raw JSON string,
-            or as local file path or URL
+        :param process_graph: openEO-style (flat) process graph representation,
+            or an object that can be converted to such a representation:
+            a dictionary, a :py:class:`~openeo.rest.datacube.DataCube` object,
+            a string with a JSON representation,
+            a local file path or URL to a JSON representation,
+            a :py:class:`~openeo.rest.multiresult.MultiResult` object, ...
         :param title: job title
         :param description: job description
         :param plan: The billing plan to process and charge the job with
@@ -1765,6 +1775,9 @@ def create_job(
         :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
         :return: Created job
+
+        .. versionchanged:: 0.35.0
+            Add :ref:`multi-result support <multi-result-process-graphs>`.
         """
         # TODO move all this (BatchJob factory) logic to BatchJob?
 

openeo/rest/multiresult.py

@@ -10,9 +10,15 @@
 
 class MultiResult(FlatGraphableMixin):
     """
-    Adapter to create/run batch jobs from process graphs with multiple end/result/leaf nodes.
+    Helper to create and run batch jobs with process graphs
+    that contain multiple result nodes
+    or, more generally speaking, multiple process graph "leaf" nodes.
 
-    Usage example:
+    Provide multiple
+    :py:class:`~openeo.rest.datacube.DataCube`/:py:class:`~openeo.rest.vectorcube.VectorCube`
+    instances to the constructor,
+    and start a batch job from that,
+    for example as follows:
 
     .. code-block:: python
 
@@ -21,10 +27,25 @@ class MultiResult(FlatGraphableMixin):
         multi_result = MultiResult([cube1, cube2])
         job = multi_result.create_job()
 
+    .. seealso::
 
+        :ref:`multi-result-process-graphs`
+
+    .. versionadded:: 0.35.0
     """
 
     def __init__(self, leaves: List[FlatGraphableMixin], connection: Optional[Connection] = None):
+        """
+        Build a :py:class:`MultiResult` instance from multiple leaf nodes
+
+        :param leaves: list of objects that can be
+            converted to an openEO-style (flat) process graph representation,
+            typically :py:class:`~openeo.rest.datacube.DataCube`
+            or :py:class:`~openeo.rest.vectorcube.VectorCube` instances.
+        :param connection: Optional connection to use for creating/starting batch jobs,
+            for special use cases where the provided leaf instances
+            are not already associated with a connection.
+        """
         self._multi_leaf_graph = MultiLeafGraph(leaves=leaves)
         self._connection = self._common_connection(leaves=leaves, connection=connection)
 

tests/rest/conftest.py

@@ -112,7 +112,7 @@ def dummy_backend(requests_mock, con120) -> DummyBackend:
 @pytest.fixture
 def another_dummy_backend(requests_mock) -> DummyBackend:
     root_url = "https://openeo.other.test/"
-    another_dummy_backend = DummyBackend.at(
+    another_dummy_backend = DummyBackend.at_url(
         root_url, requests_mock=requests_mock, capabilities={"api_version": "1.2.0"}
     )
     another_dummy_backend.setup_collection("S2")

tests/rest/test_connection.py

@@ -53,7 +53,6 @@
 refresh_token_store = refresh_token_store
 
 
-
 @pytest.mark.parametrize(
     ["base", "paths", "expected_path"],
     [
@@ -3803,3 +3802,41 @@ def test_create_job_with_mixed_connections(self, con120, dummy_backend, another_
 
         with pytest.raises(OpenEoClientException, match="Mixing different connections"):
             other_connection.create_job([save1, save2])
+
+    def test_create_job_intermediate_resultst(self, con120, dummy_backend):
+        cube = con120.load_collection("S2")
+        save1 = cube.save_result(format="GTiff")
+        reduced = cube.reduce_temporal("mean")
+        save2 = reduced.save_result(format="GTiff")
+        con120.create_job([save1, save2])
+        assert dummy_backend.get_batch_pg() == {
+            "loadcollection1": {
+                "process_id": "load_collection",
+                "arguments": {"id": "S2", "spatial_extent": None, "temporal_extent": None},
+            },
+            "saveresult1": {
+                "process_id": "save_result",
+                "arguments": {"data": {"from_node": "loadcollection1"}, "format": "GTiff", "options": {}},
+            },
+            "reducedimension1": {
+                "process_id": "reduce_dimension",
+                "arguments": {
+                    "data": {"from_node": "loadcollection1"},
+                    "dimension": "t",
+                    "reducer": {
+                        "process_graph": {
+                            "mean1": {
+                                "arguments": {"data": {"from_parameter": "data"}},
+                                "process_id": "mean",
+                                "result": True,
+                            }
+                        }
+                    },
+                },
+            },
+            "saveresult2": {
+                "process_id": "save_result",
+                "arguments": {"data": {"from_node": "reducedimension1"}, "format": "GTiff", "options": {}},
+                "result": True,
+            },
+        }