Documentation: Improvements to SageMaker pipeline documentation (#2510)

* fix errors in SageMaker pipelines docstrings
* add more code snippets

Co-authored-by: Zhankui Lu <[email protected]>
Co-authored-by: Ahsan Khan <[email protected]>
Co-authored-by: Shreya Pandit <[email protected]>

Author: 4 people

doc/workflows/pipelines/index.rst

@@ -1,8 +1,8 @@
-################
-Amazon Pipelines
-################
+###################
+SageMaker Pipelines
+###################
 
-SageMaker APIs for creating and managing Amazon Pipelines.
+SageMaker APIs for creating and managing SageMaker Pipelines.
 
 .. toctree::
     :maxdepth: 2

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -33,8 +33,6 @@ Conditions
 
 .. autoclass:: sagemaker.workflow.conditions.ConditionOr
 
-.. autofunction:: sagemaker.workflow.conditions.primitive_or_expr
-
 Entities
 --------
 
@@ -44,7 +42,7 @@ Entities
 
 .. autoclass:: sagemaker.workflow.entities.Expression
 
-Execution_variables
+Execution Variables
 -------------------
 
 .. autoclass:: sagemaker.workflow.execution_variables.ExecutionVariable
@@ -73,12 +71,17 @@ Pipeline
 --------
 
 .. autoclass:: sagemaker.workflow.pipeline.Pipeline
+    :members:
+
+.. autoclass:: sagemaker.workflow.pipeline._PipelineExecution
+    :members:
 
-.. autofunction:: sagemaker.workflow.pipeline.format_start_parameters
+Pipeline Experiment Config
+--------------------------
 
-.. autofunction:: sagemaker.workflow.pipeline.interpolate
+.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfig
 
-.. autofunction:: sagemaker.workflow.pipeline.update_args
+.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfigProperty
 
 Properties
 ----------
@@ -121,7 +124,4 @@ Steps
 
 .. autoclass:: sagemaker.workflow.callback_step.CallbackStep
 
-Utilities
----------
-
-.. autofunction:: sagemaker.workflow.utilities.list_to_request
+.. autoclass:: sagemaker.workflow.steps.CacheConfig

src/sagemaker/workflow/condition_step.py

@@ -55,10 +55,10 @@ def __init__(
             conditions (List[Condition]): A list of `sagemaker.workflow.conditions.Condition`
                 instances.
             if_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
-                and `sagemaker.workflow.step_collections.StepCollection` instances that are
+                or `sagemaker.workflow.step_collections.StepCollection` instances that are
                 marked as ready for execution if the list of conditions evaluates to True.
             else_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
-                and `sagemaker.workflow.step_collections.StepCollection` instances that are
+                or `sagemaker.workflow.step_collections.StepCollection` instances that are
                 marked as ready for execution if the list of conditions evaluates to False.
         """
         super(ConditionStep, self).__init__(name, StepTypeEnum.CONDITION, depends_on)

src/sagemaker/workflow/conditions.py

@@ -66,13 +66,13 @@ class ConditionComparison(Condition):
     """Generic comparison condition that can be used to derive specific condition comparisons.
 
     Attributes:
-        left (ConditionValueType): The execution variable, parameter, or
-            property to use in the comparison.
+        left (Union[ConditionValueType, PrimitiveType]): The execution variable, parameter,
+            property, or Python primitive value to use in the comparison.
         right (Union[ConditionValueType, PrimitiveType]): The execution variable,
             parameter, property, or Python primitive value to compare to.
     """
 
-    left: ConditionValueType = attr.ib(default=None)
+    left: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
     right: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
 
     def to_request(self) -> RequestType:
@@ -87,12 +87,16 @@ def to_request(self) -> RequestType:
 class ConditionEquals(ConditionComparison):
     """A condition for equality comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct A condition for equality comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -103,12 +107,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionGreaterThan(ConditionComparison):
     """A condition for greater than comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct an instance of ConditionGreaterThan for greater than comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -119,12 +127,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionGreaterThanOrEqualTo(ConditionComparison):
     """A condition for greater than or equal to comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct of ConditionGreaterThanOrEqualTo for greater than or equal to comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -135,12 +147,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionLessThan(ConditionComparison):
     """A condition for less than comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct an instance of ConditionLessThan for less than comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -151,12 +167,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionLessThanOrEqualTo(ConditionComparison):
     """A condition for less than or equal to comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct ConditionLessThanOrEqualTo for less than or equal to comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -168,13 +188,15 @@ class ConditionIn(Condition):
     """A condition to check membership."""
 
     def __init__(
-        self, value: ConditionValueType, in_values: List[Union[ConditionValueType, PrimitiveType]]
+        self,
+        value: Union[ConditionValueType, PrimitiveType],
+        in_values: List[Union[ConditionValueType, PrimitiveType]],
     ):
         """Construct a `ConditionIn` condition to check membership.
 
         Args:
-            value (ConditionValueType): The execution variable,
-                parameter, or property to use for the in comparison.
+            value (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property or primitive value to check for membership.
             in_values (List[Union[ConditionValueType, PrimitiveType]]): The list
                 of values to check for membership in.
         """

src/sagemaker/workflow/execution_variables.py

@@ -37,10 +37,7 @@ def expr(self) -> RequestType:
 
 
 class ExecutionVariables:
-    """Enum-like class for all ExecutionVariable instances.
-
-    Considerations to move these as module-level constants should be made.
-    """
+    """All available ExecutionVariable."""
 
     START_DATETIME = ExecutionVariable("StartDateTime")
     CURRENT_DATETIME = ExecutionVariable("CurrentDateTime")

src/sagemaker/workflow/functions.py

@@ -24,9 +24,19 @@
 class Join(Expression):
     """Join together properties.
 
+    Examples:
+    Build a Amazon S3 Uri with bucket name parameter and pipeline execution Id and use it
+    as training input::
+
+        bucket = ParameterString('bucket', default_value='my-bucket')
+
+        TrainingInput(
+            s3_data=Join(on='/', ['s3:/', bucket, ExecutionVariables.PIPELINE_EXECUTION_ID]),
+            content_type="text/csv")
+
     Attributes:
-        values (List[Union[PrimitiveType, Parameter]]): The primitive types
-            and parameters to join.
+        values (List[Union[PrimitiveType, Parameter, Expression]]):
+            The primitive type values, parameters, step properties, expressions to join.
         on_str (str): The string to join the values on (Defaults to "").
     """
 

src/sagemaker/workflow/parameters.py

@@ -54,7 +54,7 @@ class Parameter(Entity):
     Attributes:
         name (str): The name of the parameter.
         parameter_type (ParameterTypeEnum): The type of the parameter.
-        default_value (PrimitiveType): The default Python value of the parameter.
+        default_value (PrimitiveType): The default value of the parameter.
     """
 
     name: str = attr.ib(factory=str)
@@ -143,7 +143,7 @@ def _check_default_value_type(cls, value, python_type):
 
 
 class ParameterString(Parameter, str):
-    """Pipeline string parameter for workflow."""
+    """String parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):  # pylint: disable=unused-argument
         """Subclass str"""
@@ -155,7 +155,11 @@ def __init__(self, name: str, default_value: str = None, enum_values: List[str]
 
         Args:
             name (str): The name of the parameter.
-            default_value (str): The default Python value of the parameter. Defaults to None.
+            default_value (str): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
+            enum_values (List[str]): Enum values for this parameter.
         """
         super(ParameterString, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.STRING, default_value=default_value
@@ -175,7 +179,7 @@ def to_request(self) -> RequestType:
 
 
 class ParameterInteger(Parameter, int):
-    """Pipeline string parameter for workflow."""
+    """Integer parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):
         """Subclass int"""
@@ -187,15 +191,18 @@ def __init__(self, name: str, default_value: int = None):
 
         Args:
             name (str): The name of the parameter.
-            default_value (int): The default Python value of the parameter.
+            default_value (int): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
         """
         super(ParameterInteger, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.INTEGER, default_value=default_value
         )
 
 
 class ParameterFloat(Parameter, float):
-    """Pipeline float parameter for workflow."""
+    """Float parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):
         """Subclass float"""
@@ -207,7 +214,10 @@ def __init__(self, name: str, default_value: float = None):
 
         Args:
             name (str): The name of the parameter.
-            default_value (float): The default Python value of the parameter.
+            default_value (float): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
         """
         super(ParameterFloat, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.FLOAT, default_value=default_value

src/sagemaker/workflow/pipeline.py

@@ -45,15 +45,15 @@ class Pipeline(Entity):
 
     Attributes:
         name (str): The name of the pipeline.
-        parameters (Sequence[Parameters]): The list of the parameters.
+        parameters (Sequence[Parameter]): The list of the parameters.
         pipeline_experiment_config (Optional[PipelineExperimentConfig]): If set,
             the workflow will attempt to create an experiment and trial before
             executing the steps. Creation will be skipped if an experiment or a trial with
             the same name already exists. By default, pipeline name is used as
             experiment name and execution id is used as the trial name.
             If set to None, no experiment or trial will be created automatically.
-        steps (Sequence[Steps]): The list of the non-conditional steps associated with the pipeline.
-            Any steps that are within the
+        steps (Sequence[Union[Step, StepCollection]]): The list of the non-conditional steps
+            associated with the pipeline. Any steps that are within the
             `if_steps` or `else_steps` of a `ConditionStep` cannot be listed in the steps of a
             pipeline. Of particular note, the workflow service rejects any pipeline definitions that
             specify a step in the list of steps of a pipeline and that step in the `if_steps` or
@@ -139,7 +139,9 @@ def describe(self) -> Dict[str, Any]:
         """Describes a Pipeline in the Workflow service.
 
         Returns:
-            Response dict from the service.
+            Response dict from the service. See `boto3 client documentation
+            <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
+sagemaker.html#SageMaker.Client.describe_pipeline>`_
         """
         return self.sagemaker_session.sagemaker_client.describe_pipeline(PipelineName=self.name)
 
@@ -209,15 +211,15 @@ def delete(self) -> Dict[str, Any]:
 
     def start(
         self,
-        parameters: Dict[str, Any] = None,
+        parameters: Dict[str, Union[str, bool, int, float]] = None,
         execution_display_name: str = None,
         execution_description: str = None,
     ):
         """Starts a Pipeline execution in the Workflow service.
 
         Args:
-            parameters (List[Dict[str, str]]): A list of parameter dicts of the form
-                {"Name": "string", "Value": "string"}.
+            parameters (Dict[str, Union[str, bool, int, float]]): values to override
+                pipeline parameters.
             execution_display_name (str): The display name of the pipeline execution.
             execution_description (str): A description of the execution.
 
@@ -370,13 +372,27 @@ def stop(self):
         )
 
     def describe(self):
-        """Describes a pipeline execution."""
+        """Describes a pipeline execution.
+
+        Returns:
+             Information about the pipeline execution. See
+             `boto3 client describe_pipeline_execution
+             <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
+sagemaker.html#SageMaker.Client.describe_pipeline_execution>`_.
+        """
         return self.sagemaker_session.sagemaker_client.describe_pipeline_execution(
             PipelineExecutionArn=self.arn
         )
 
     def list_steps(self):
-        """Describes a pipeline execution's steps."""
+        """Describes a pipeline execution's steps.
+
+        Returns:
+             Information about the steps of the pipeline execution. See
+             `boto3 client list_pipeline_execution_steps
+             <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
+sagemaker.html#SageMaker.Client.list_pipeline_execution_steps>`_.
+        """
         response = self.sagemaker_session.sagemaker_client.list_pipeline_execution_steps(
             PipelineExecutionArn=self.arn
         )