Synchronize docs between DataCube, VectorCube, ...

use tests to enforce keeping docs in sync

related to #402/#720/#725

Author: soxofaan

openeo/internal/documentation.py

@@ -4,9 +4,10 @@
 
 import collections
 import inspect
+import re
 import textwrap
 from functools import partial
-from typing import Callable, Optional, Tuple, TypeVar
+from typing import Callable, Dict, Optional, Tuple, TypeVar, Union
 
 # TODO: give this a proper public API?
 _process_registry = collections.defaultdict(list)
@@ -58,3 +59,39 @@ def decorate(f: Callable) -> Callable:
         return f
 
     return decorate
+
+
+def _get_doc(obj: Union[str, Callable]) -> str:
+    """
+    Get docstring of a method or function.
+    """
+    if isinstance(obj, str):
+        doc = obj
+    else:
+        doc = obj.__doc__
+    return textwrap.dedent(doc)
+
+
+def extract_params(doc: Union[str, Callable]) -> Dict[str, str]:
+    """
+    Extract parameters (``:param name:`` format) from a docstring.
+    """
+    doc = _get_doc(doc)
+    params_regex = re.compile(r"^:param\s+(?P<param>\w+)\s*:(?P<doc>.*(\n +.*)*)", re.MULTILINE)
+    return {m.group("param"): m.group("doc").strip() for m in params_regex.finditer(doc)}
+
+
+def assert_same_param_docs(doc_a: Union[str, Callable], doc_b: Union[str, Callable], only_intersection: bool = False):
+    """
+    Compare parameters (``:param name:`` format) from two docstrings.
+    """
+    # TODO: option to also check order?
+    params_a = extract_params(doc_a)
+    params_b = extract_params(doc_b)
+
+    if only_intersection:
+        intersection = set(params_a.keys()).intersection(params_b.keys())
+        params_a = {k: v for k, v in params_a.items() if k in intersection}
+        params_b = {k: v for k, v in params_b.items() if k in intersection}
+
+    assert params_a == params_b

openeo/rest/datacube.py

@@ -2338,7 +2338,7 @@ def save_result(
         Materialize the processed data to the given file format.
 
         :param format: an output format supported by the backend.
-        :param options: file format options
+        :param options: (optional) file format options
 
         .. versionchanged:: 0.39.0
             returns a :py:class:`~openeo.rest.result.SaveResult` instance instead
@@ -2389,14 +2389,14 @@ def download(
         If outputfile is provided, the result is stored on disk locally, otherwise, a bytes object is returned.
         The bytes object can be passed on to a suitable decoder for decoding.
 
-        :param outputfile: Optional, output path to download to.
-        :param format: Optional, an output format supported by the backend.
-        :param options: Optional, file format options
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param outputfile: (optional) output path to download to.
+        :param format: (optional) an output format supported by the backend.
+        :param options: (optional) file format options
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
 
         :return: None if the result is stored to disk, or a bytes object returned by the backend.
@@ -2544,24 +2544,25 @@ def execute_batch(
             for batch jobs that are expected to complete
             in a time that is reasonable for your use case.
 
-        :param outputfile: Optional, output path to download to.
-        :param out_format: (optional) File format to use for the job result.
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with
-        :param budget: Maximum budget to be spent on executing the job.
+        :param outputfile: (optional) output path to download to.
+        :param out_format: (optional) file format to use for the job result.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
         :param show_error_logs: whether to automatically print error logs when the batch job failed.
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
         :param max_poll_interval: maximum number of seconds to sleep between job status polls
         :param connection_retry_interval: how long to wait when status poll failed due to connection issue
+        :param print: print/logging function to show progress/status
 
         .. versionchanged:: 0.32.0
             Added ``auto_add_save_result`` option
@@ -2632,22 +2633,22 @@ def create_job(
         Use :py:meth:`execute_batch` instead to let the openEO Python client
         take care of the full job life cycle: create, start and track its progress until completion.
 
-        :param out_format: output file format.
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with.
-        :param budget: Maximum budget to be spent on executing the job.
+        :param out_format: (optional) file format to use for the job result.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param auto_add_save_result: Automatically add a ``save_result`` node to the process graph.
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param auto_add_save_result: whether to automatically add a ``save_result`` node to the process graph.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
 
-        :return: Created job.
+        :return: Handle for the job created at the backend.
 
         .. versionchanged:: 0.32.0
             Added ``auto_add_save_result`` option

openeo/rest/mlmodel.py

@@ -79,23 +79,37 @@ def execute_batch(
         log_level: Optional[str] = None,
     ) -> BatchJob:
         """
-        Evaluate the process graph by creating a batch job, and retrieving the results when it is finished.
-        This method is mostly recommended if the batch job is expected to run in a reasonable amount of time.
-
-        For very long-running jobs, you probably do not want to keep the client running.
-
-        :param job_options:
-        :param outputfile: The path of a file to which a result can be written
-        :param out_format: (optional) Format of the job result.
-        :param format_options: String Parameters for the job result format
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        Execute the underlying process graph at the backend in batch job mode:
+
+        - create the job (like :py:meth:`create_job`)
+        - start the job (like :py:meth:`BatchJob.start() <openeo.rest.job.BatchJob.start>`)
+        - track the job's progress with an active polling loop
+          (like :py:meth:`BatchJob.run_synchronous() <openeo.rest.job.BatchJob.run_synchronous>`)
+        - optionally (if ``outputfile`` is specified) download the job's results
+          when the job finished successfully
+
+        .. note::
+            Because of the active polling loop,
+            which blocks any further progress of your script or application,
+            this :py:meth:`execute_batch` method is mainly recommended
+            for batch jobs that are expected to complete
+            in a time that is reasonable for your use case.
+
+        :param outputfile: (optional) output path to download to.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
+            Note that some backends do not honor this limit.
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
         :param show_error_logs: whether to automatically print error logs when the batch job failed.
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
         :param max_poll_interval: maximum number of seconds to sleep between job status polls
         :param connection_retry_interval: how long to wait when status poll failed due to connection issue
+        :param print: print/logging function to show progress/status
 
         .. versionchanged:: 0.36.0
             Added argument ``additional``.
@@ -136,18 +150,25 @@ def create_job(
         log_level: Optional[str] = None,
     ) -> BatchJob:
         """
-        Sends a job to the backend and returns a ClientJob instance.
+        Send the underlying process graph to the backend
+        to create an openEO batch job
+        and return a corresponding :py:class:`~openeo.rest.job.BatchJob` instance.
+
+        Note that this method only *creates* the openEO batch job at the backend,
+        but it does not *start* it.
+        Use :py:meth:`execute_batch` instead to let the openEO Python client
+        take care of the full job life cycle: create, start and track its progress until completion.
+
 
-        :param title: job title
-        :param description: job description
-        :param plan: The billing plan to process and charge the job with
-        :param budget: Maximum budget to be spent on executing the job.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param format_options: String Parameters for the job result format
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
         :return: Created job.
 

openeo/rest/result.py

@@ -10,7 +10,7 @@ class SaveResult(StacResource):
     and :py:meth:`VectorCube.save_result() <openeo.rest.vectorcube.VectorCube.save_result>`.
 
     .. note ::
-        This class is practically a just direct alias for
+        This class is practically just a direct alias for
         :py:class:`~openeo.rest.stac_resource.StacResource`,
         but with a more self-explanatory name.
 

openeo/rest/stac_resource.py

@@ -74,11 +74,11 @@ def download(
         If outputfile is provided, the result is stored on disk locally, otherwise, a bytes object is returned.
         The bytes object can be passed on to a suitable decoder for decoding.
 
-        :param outputfile: Optional, output path to download to.
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param outputfile: (optional) output path to download to.
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
 
         :return: None if the result is stored to disk, or a bytes object returned by the backend.
@@ -113,17 +113,17 @@ def create_job(
         Use :py:meth:`execute_batch` instead to let the openEO Python client
         take care of the full job life cycle: create, start and track its progress until completion.
 
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with.
-        :param budget: Maximum budget to be spent on executing the job.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
 
         :return: Handle for the job created at the backend.
@@ -174,18 +174,18 @@ def execute_batch(
             for batch jobs that are expected to complete
             in a time that is reasonable for your use case.
 
-        :param outputfile: Optional, output path to download to.
-        :param title: job title.
-        :param description: job description.
-        :param plan: The billing plan to process and charge the job with
-        :param budget: Maximum budget to be spent on executing the job.
+        :param outputfile: (optional) output path to download to.
+        :param title: (optional) job title.
+        :param description: (optional) job description.
+        :param plan: (optional) the billing plan to process and charge the job with.
+        :param budget: (optional) maximum budget to be spent on executing the job.
             Note that some backends do not honor this limit.
-        :param additional: additional (top-level) properties to set in the request body
-        :param job_options: dictionary of job options to pass to the backend
+        :param additional: (optional) additional (top-level) properties to set in the request body
+        :param job_options: (optional) dictionary of job options to pass to the backend
             (under top-level property "job_options")
-        :param validate: Optional toggle to enable/prevent validation of the process graphs before execution
+        :param validate: (optional) toggle to enable/prevent validation of the process graphs before execution
             (overruling the connection's ``auto_validate`` setting).
-        :param log_level: Optional minimum severity level for log entries that the back-end should keep track of.
+        :param log_level: (optional) minimum severity level for log entries that the back-end should keep track of.
             One of "error" (highest severity), "warning", "info", and "debug" (lowest severity).
         :param print: print/logging function to show progress/status
         :param max_poll_interval: maximum number of seconds to sleep between job status polls