Merge branch 'issue720-synchronize-docs'

Author: soxofaan

openeo/internal/documentation.py

@@ -4,9 +4,10 @@
 
 import collections
 import inspect
+import re
 import textwrap
 from functools import partial
-from typing import Callable, Optional, Tuple, TypeVar
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union
 
 # TODO: give this a proper public API?
 _process_registry = collections.defaultdict(list)
@@ -58,3 +59,85 @@ def decorate(f: Callable) -> Callable:
         return f
 
     return decorate
+
+
+def get_docstring(obj: Union[str, Callable]) -> str:
+    """
+    Get docstring of a method or function.
+    """
+    if isinstance(obj, str):
+        doc = obj
+    else:
+        doc = obj.__doc__
+    return textwrap.dedent(doc)
+
+
+def extract_params(doc: Union[str, Callable]) -> Dict[str, str]:
+    """
+    Extract parameters (``:param name:`` format) from a docstring.
+    """
+    doc = get_docstring(doc)
+    params_regex = re.compile(r"^:param\s+(?P<param>\w+)\s*:(?P<doc>.*(\n +.*)*)", re.MULTILINE)
+    return {m.group("param"): m.group("doc").strip() for m in params_regex.finditer(doc)}
+
+
+def extract_return(doc: Union[str, Callable]) -> Union[str, None]:
+    """
+    Extract return value description (``:return:`` format) from a docstring.
+    """
+    doc = get_docstring(doc)
+    return_regex = re.compile(r"^:return\s*:(?P<doc>.*(\n +.*)*)", re.MULTILINE)
+    matches = [m.group("doc").strip() for m in return_regex.finditer(doc)]
+    assert 0 <= len(matches) <= 1
+    return matches[0] if matches else None
+
+
+def extract_main_description(doc: Union[str, Callable]) -> List[str]:
+    """
+    Extract main description from a docstring:
+    paragraphs before the params/returns description.
+    """
+    paragraphs = []
+    for part in re.split(r"\s*\n(?:\s*\n)+", get_docstring(doc)):
+        if re.match(r"\s*:", part):
+            break
+        paragraphs.append(part.strip("\n"))
+    assert len(paragraphs) > 0
+    return paragraphs
+
+
+def assert_same_param_docs(doc_a: Union[str, Callable], doc_b: Union[str, Callable], only_intersection: bool = False):
+    """
+    Compare parameters (``:param name:`` format) from two docstrings.
+    """
+    # TODO: option to also check order?
+    params_a = extract_params(doc_a)
+    params_b = extract_params(doc_b)
+
+    if only_intersection:
+        intersection = set(params_a.keys()).intersection(params_b.keys())
+        params_a = {k: v for k, v in params_a.items() if k in intersection}
+        params_b = {k: v for k, v in params_b.items() if k in intersection}
+
+    assert params_a == params_b
+
+
+def assert_same_return_docs(doc_a: Union[str, Callable], doc_b: Union[str, Callable]):
+    """
+    Compare return value descriptions from two docstrings.
+    """
+    assert extract_return(doc_a) == extract_return(doc_b)
+
+
+def assert_same_main_description(doc_a: Union[str, Callable], doc_b: Union[str, Callable], ignore: Sequence[str] = ()):
+    """
+    Compare main description from two docstrings.
+    """
+    description_a = extract_main_description(doc_a)
+    description_b = extract_main_description(doc_b)
+
+    for s in ignore:
+        description_a = [p.replace(s, "<IGNORED>") for p in description_a]
+        description_b = [p.replace(s, "<IGNORED>") for p in description_b]
+
+    assert description_a == description_b

openeo/rest/connection.py

@@ -1559,21 +1559,27 @@ def download(
         job_options: Optional[dict] = None,
     ) -> Union[None, bytes]:
         """
-        Downloads the result of a process graph synchronously,
-        and save the result to the given file or return bytes object if no outputfile is specified.
-        This method is useful to export binary content such as images. For json content, the execute method is recommended.
+        Send the underlying process graph to the backend
+        for synchronous processing and directly download the result.
+
+        If ``outputfile`` is provided, the result is downloaded to that path.
+        Otherwise a :py:class:`bytes` object is returned with the raw data.
 
         :param graph: (flat) dict representing a process graph, or process graph as raw JSON string,
             or as local file path or URL
-        :param outputfile: output file
+        :param outputfile: (optional) output path to download to.
         :param timeout: timeout to wait for response
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
         :param chunk_size: chunk size for streaming response.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
 
+        :return: if ``outputfile`` was not specified:
+            a :py:class:`bytes` object containing the raw data.
+            Otherwise, ``None`` is returned.
+
         .. versionadded:: 0.36.0
             Added arguments ``additional`` and ``job_options``.
         """
@@ -1595,6 +1601,7 @@ def download(
             with target.open(mode="wb") as f:
                 for chunk in response.iter_content(chunk_size=chunk_size):
                     f.write(chunk)
+            # TODO: return target path instead of None? Or return a generic result wrapper?
         else:
             return response.content
 
@@ -1659,27 +1666,35 @@ def create_job(
         log_level: Optional[str] = None,
     ) -> BatchJob:
         """
-        Create a new job from given process graph on the back-end.
+        Send the underlying process graph to the backend
+        to create an openEO batch job
+        and return a corresponding :py:class:`~openeo.rest.job.BatchJob` instance.
+
+        Note that this method only *creates* the openEO batch job at the backend,
+        but it does not *start* it.
+        Use :py:meth:`execute_batch` instead to let the openEO Python client
+        take care of the full job life cycle: create, start and track its progress until completion.
 
         :param process_graph: openEO-style (flat) process graph representation,
             or an object that can be converted to such a representation:
             a dictionary, a :py:class:`~openeo.rest.datacube.DataCube` object,
             a string with a JSON representation,
             a local file path or URL to a JSON representation,
             a :py:class:`~openeo.rest.multiresult.MultiResult` object, ...
-        :param title: job title
-        :param description: job description
-        :param plan: The billing plan to process and charge the job with
-        :param budget: Maximum budget to be spent on executing the job.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
-        :return: Created job
+
+        :return: Handle to the job created at the backend.
 
         .. versionchanged:: 0.35.0
             Add :ref:`multi-result support <multi-result-process-graphs>`.

openeo/rest/datacube.py

@@ -2338,7 +2338,7 @@ def save_result(
         Materialize the processed data to the given file format.
 
         :param format: an output format supported by the backend.
-        :param options: file format options
+        :param options: (optional) file format options
 
         .. versionchanged:: 0.39.0
             returns a :py:class:`~openeo.rest.result.SaveResult` instance instead
@@ -2384,22 +2384,25 @@ def download(
         job_options: Optional[dict] = None,
     ) -> Union[None, bytes]:
         """
-        Execute synchronously and download the raster data cube, e.g. as GeoTIFF.
+        Send the underlying process graph to the backend
+        for synchronous processing and directly download the result.
 
-        If outputfile is provided, the result is stored on disk locally, otherwise, a bytes object is returned.
-        The bytes object can be passed on to a suitable decoder for decoding.
+        If ``outputfile`` is provided, the result is downloaded to that path.
+        Otherwise a :py:class:`bytes` object is returned with the raw data.
 
-        :param outputfile: Optional, output path to download to.
-        :param format: Optional, an output format supported by the backend.
-        :param options: Optional, file format options
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param outputfile: (optional) output path to download to.
+        :param format: (optional) an output format supported by the backend.
+        :param options: (optional) file format options
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
 
-        :return: None if the result is stored to disk, or a bytes object returned by the backend.
+        :return: if ``outputfile`` was not specified:
+            a :py:class:`bytes` object containing the raw data.
+            Otherwise, ``None`` is returned.
 
         .. versionchanged:: 0.32.0
             Added ``auto_add_save_result`` option
@@ -2544,24 +2547,27 @@ def execute_batch(
             for batch jobs that are expected to complete
             in a time that is reasonable for your use case.
 
-        :param outputfile: Optional, output path to download to.
-        :param out_format: (optional) File format to use for the job result.
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with
-        :param budget: Maximum budget to be spent on executing the job.
+        :param outputfile: (optional) output path to download to.
+        :param out_format: (optional) file format to use for the job result.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
         :param show_error_logs: whether to automatically print error logs when the batch job failed.
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
         :param max_poll_interval: maximum number of seconds to sleep between job status polls
         :param connection_retry_interval: how long to wait when status poll failed due to connection issue
+        :param print: print/logging function to show progress/status
+
+        :return: Handle to the job created at the backend.
 
         .. versionchanged:: 0.32.0
             Added ``auto_add_save_result`` option
@@ -2632,22 +2638,22 @@ def create_job(
         Use :py:meth:`execute_batch` instead to let the openEO Python client
         take care of the full job life cycle: create, start and track its progress until completion.
 
-        :param out_format: output file format.
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with.
-        :param budget: Maximum budget to be spent on executing the job.
+        :param out_format: (optional) file format to use for the job result.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
 
-        :return: Created job.
+        :return: Handle to the job created at the backend.
 
         .. versionchanged:: 0.32.0
             Added ``auto_add_save_result`` option

openeo/rest/job.py

@@ -233,12 +233,14 @@ def run_synchronous(
         """
         Start the job, wait for it to finish and download result
 
-        :param outputfile: The path of a file to which a result can be written
+        :param outputfile: (optional) output path to download to.
         :param print: print/logging function to show progress/status
         :param max_poll_interval: maximum number of seconds to sleep between job status polls
         :param connection_retry_interval: how long to wait when status poll failed due to connection issue
         :param show_error_logs: whether to automatically print error logs when the batch job failed.
 
+        :return: Handle to the job created at the backend.
+
         .. versionchanged:: 0.37.0
             Added argument ``show_error_logs``.
         """