[PNE-7018] Deprecate PEWs.

PEWs no longer need to be a customer facing asset type. Their only
function is to allow reusing the same set of evaluators with different
predictors, which A) doesn't often work due to mismatched responses, and
B) isn't particularly helpful. As such, we can move users on to a
cleaner, easier to use workflow, where predictor evaluation is kicked
off directly from a predictor, and you can start one using the default
evaluators with a single call.

In addition to exposing additional endpoints and adding evaluators to
the PEW executions, this also deprecates the
predictor_evaluation_workflow property, and other Predictor Evaluation
Workflow related assets. It also deprecates properties and classes named
something akin to "predictor evaluation execution", in favor of the
simpler and clearer "predictor evaluation".

Author: anoto-moniz

src/citrine/__version__.py

@@ -1 +1 @@
-__version__ = "3.22.1"
+__version__ = "3.23.0"

src/citrine/informatics/executions/__init__.py

@@ -1,4 +1,5 @@
 # flake8: noqa
-from .predictor_evaluation_execution import *
 from .design_execution import *
 from .generative_design_execution import *
+from .predictor_evaluation import *
+from .predictor_evaluation_execution import *

src/citrine/informatics/executions/predictor_evaluation.py

@@ -0,0 +1,67 @@
+from functools import lru_cache
+
+from citrine.informatics.executions.execution import Execution
+from citrine.informatics.predictor_evaluation_result import PredictorEvaluationResult
+from citrine.informatics.predictor_evaluator import PredictorEvaluator
+from citrine._rest.resource import Resource
+from citrine._serialization import properties
+from citrine._utils.functions import format_escaped_url
+
+
+class PredictorEvaluation(Resource['PredictorEvaluation'], Execution):
+    """The execution of a PredictorEvaluationWorkflow.
+
+    Possible statuses are INPROGRESS, SUCCEEDED, and FAILED.
+    Predictor evaluation executions also have a ``status_description`` field with more information.
+
+    """
+
+    evaluators = properties.List(properties.Object(PredictorEvaluator), "evaluators",
+                                 serializable=False)
+    """:List{PredictorEvaluator]:the predictor evaluators that were executed. These are used
+    when calling the ``results()`` method."""
+    workflow_id = properties.UUID('workflow_id', serializable=False)
+    """:UUID: Unique identifier of the workflow that was executed"""
+    predictor_id = properties.UUID('predictor_id', serializable=False)
+    predictor_version = properties.Integer('predictor_version', serializable=False)
+
+    def _path(self):
+        return format_escaped_url(
+            '/projects/{project_id}/predictor-evaluation-executions/{execution_id}',
+            project_id=str(self.project_id),
+            execution_id=str(self.uid)
+        )
+
+    @lru_cache()
+    def results(self, evaluator_name: str) -> PredictorEvaluationResult:
+        """
+        Get a specific evaluation result by the name of the evaluator that produced it.
+
+        Parameters
+        ----------
+        evaluator_name: str
+            Name of the evaluator for which to get the results
+
+        Returns
+        -------
+        PredictorEvaluationResult
+            The evaluation result from the evaluator with the given name
+
+        """
+        params = {"evaluator_name": evaluator_name}
+        resource = self._session.get_resource(self._path() + "/results", params=params)
+        return PredictorEvaluationResult.build(resource)
+
+    @property
+    def evaluator_names(self):
+        """Names of the predictor evaluators. Used when calling the ``results()`` method."""
+        return list(iter(self))
+
+    def __getitem__(self, item):
+        if isinstance(item, str):
+            return self.results(item)
+        else:
+            raise TypeError("Results are accessed by string names")
+
+    def __iter__(self):
+        return iter(e.name for e in self.evaluators)

src/citrine/informatics/predictor_evaluator.py

@@ -6,10 +6,11 @@
 from citrine.informatics.predictor_evaluation_metrics import PredictorEvaluationMetric
 from citrine.informatics.data_sources import DataSource
 
-__all__ = ['PredictorEvaluator',
-           'CrossValidationEvaluator',
-           'HoldoutSetEvaluator'
-           ]
+__all__ = [
+    'CrossValidationEvaluator',
+    'HoldoutSetEvaluator',
+    'PredictorEvaluator'
+]
 
 
 class PredictorEvaluator(PolymorphicSerializable["PredictorEvaluator"]):

src/citrine/jobs/waiting.py

@@ -7,7 +7,7 @@
 from citrine.informatics.executions.design_execution import DesignExecution
 from citrine.informatics.executions.generative_design_execution import GenerativeDesignExecution
 from citrine.informatics.executions.sample_design_space_execution import SampleDesignSpaceExecution
-from citrine.informatics.executions import PredictorEvaluationExecution
+from citrine.informatics.executions import PredictorEvaluation, PredictorEvaluationExecution
 
 
 class ConditionTimeoutError(RuntimeError):
@@ -130,12 +130,14 @@ def wait_while_validating(
 def wait_while_executing(
     *,
     collection: Union[
+        Collection[PredictorEvaluation],
         Collection[PredictorEvaluationExecution],
         Collection[DesignExecution],
         Collection[GenerativeDesignExecution],
         Collection[SampleDesignSpaceExecution]
     ],
     execution: Union[
+        PredictorEvaluation,
         PredictorEvaluationExecution,
         DesignExecution,
         GenerativeDesignExecution,
@@ -145,6 +147,7 @@ def wait_while_executing(
     timeout: float = 1800.0,
     interval: float = 3.0,
 ) -> Union[
+        PredictorEvaluation,
         PredictorEvaluationExecution,
         DesignExecution,
         GenerativeDesignExecution,
@@ -155,20 +158,24 @@ def wait_while_executing(
 
     Parameters
     ----------
-    execution : Union[PredictorEvaluationExecution, DesignExecution]
+    execution : Union[PredictorEvaluation, PredictorEvaluationExecution, DesignExecution,
+                      GenerativeDesignExecution, SampleDesignSpaceExecution]
         an execution to monitor
     print_status_info : bool, optional
         Whether to print status info, by default False
     timeout : float, optional
         Maximum time spent inquiring in seconds, by default 1800.0
     interval : float, optional
         Inquiry interval in seconds, by default 3.0
-    collection : Union[Collection[PredictorEvaluationExecution], Collection[DesignExecution]]
+    collection : Union[Collection[PredictorEvaluation], Collection[PredictorEvaluationExecution],
+                       Collection[DesignExecution], Collection[GenerativeDesignExecution],
+                       Collection[SampleDesignSpaceExecution]]
         Collection containing executions
 
     Returns
     -------
-    Union[PredictorEvaluationExecution, DesignExecution]
+    execution : Union[PredictorEvaluation, PredictorEvaluationExecution, DesignExecution,
+                      GenerativeDesignExecution, SampleDesignSpaceExecution]
         the updated execution after it has finished executing
 
 

src/citrine/resources/predictor.py

@@ -9,16 +9,29 @@
 from citrine._rest.resource import Resource
 from citrine._rest.paginator import Paginator
 from citrine._serialization import properties
+from citrine._serialization.serializable import Serializable
 from citrine._session import Session
 from citrine.informatics.data_sources import DataSource
 from citrine.informatics.design_candidate import HierarchicalDesignMaterial
+from citrine.informatics.executions.predictor_evaluation import PredictorEvaluation
+from citrine.informatics.predictor_evaluator import PredictorEvaluator
 from citrine.informatics.predictors import GraphPredictor
 from citrine.resources.status_detail import StatusDetail
 
 
-# Refers to the most recently edited prediction version. Could be a draft.
+# The most recently edited prediction version. Could be a draft.
 MOST_RECENT_VER = "most_recent"
-LATEST_VER = "latest"  # Refers to the highest saved predictor version.
+# The highest trained predictor version.
+LATEST_VER = "latest"
+
+
+class EvaluatorsPayload(Serializable['EvaluatorsPayload']):
+    """Container object for predictor evaluators."""
+
+    evaluators = properties.List(properties.Object(PredictorEvaluator), "evaluators")
+
+    def __init__(self, evaluators):
+        self.evaluators = evaluators
 
 
 class AsyncDefaultPredictor(Resource["AsyncDefaultPredictor"]):
@@ -196,6 +209,38 @@ def rename(self,
         entity = self.session.put_resource(path, json, version=self._api_version)
         return self.build(entity)
 
+    def default_evaluators(self,
+                           uid: Union[UUID, str],
+                           *,
+                           version: Union[int, str]) -> List[PredictorEvaluator]:
+        path = self._construct_path(uid, version, action="default-evaluators")
+        evaluators = self.session.get_resource(path)
+        return EvaluatorsPayload.build(evaluators).evaluators
+
+    def _build_predictor_evaluation(self, data):
+        evaluation = PredictorEvaluation.build(data)
+        evaluation.project_id = self.project_id
+        evaluation._session = self.session
+        return evaluation
+
+    def evaluate(self,
+                 uid: Union[UUID, str],
+                 *,
+                 version: Union[int, str],
+                 evaluators: List[PredictorEvaluator]) -> PredictorEvaluation:
+        path = self._construct_path(uid, version, "evaluate")
+        payload = EvaluatorsPayload(evaluators=evaluators).dump()
+        response = self.session.post_resource(path, payload, version=self._api_version)
+        return self._build_predictor_evaluation(response)
+
+    def evaluate_default(self,
+                         uid: Union[UUID, str],
+                         *,
+                         version: Union[int, str]) -> PredictorEvaluation:
+        path = self._construct_path(uid, version, "evaluate-default")
+        response = self.session.post_resource(path, {}, version=self._api_version)
+        return self._build_predictor_evaluation(response)
+
     def delete(self, uid: Union[UUID, str], *, version: Union[int, str] = MOST_RECENT_VER):
         """Predictor versions cannot be deleted at this time."""
         msg = "Predictor versions cannot be deleted. Use 'archive_version' instead."
@@ -578,6 +623,95 @@ def rename(self,
             uid, version=version, name=name, description=description
         )
 
+    def default_evaluators_from_config(self,
+                                       predictor: GraphPredictor) -> List[PredictorEvaluator]:
+        """Retrieve the default evaluators for an arbitrary (but valid) predictor config.
+
+        See :func:`~citrine.resources.PredictorCollection.default_evaluators` for details on the
+        evaluators.
+        """
+        path = self._get_path(action="default-evaluators-from-config")
+        payload = predictor.dump()
+        evaluators = self.session.post_resource(path, json=payload, version=self._api_version)
+        return EvaluatorsPayload.build(evaluators).evaluators
+
+    def default_evaluators(self,
+                           uid: Union[UUID, str],
+                           *,
+                           version: Union[int, str] = MOST_RECENT_VER) -> List[PredictorEvaluator]:
+        """Retrieve the default evaluators for a stored predictor.
+
+        The current default evaluators perform 5-fold, 3-trial cross-validation on all valid
+        predictor responses. Valid responses are those that are **not** produced by the
+        following predictors:
+
+        * :class:`~citrine.informatics.predictors.generalized_mean_property_predictor.GeneralizedMeanPropertyPredictor`
+        * :class:`~citrine.informatics.predictors.mean_property_predictor.MeanPropertyPredictor`
+        * :class:`~citrine.informatics.predictors.ingredient_fractions_predictor.IngredientFractionsPredictor`
+        * :class:`~citrine.informatics.predictors.ingredients_to_simple_mixture_predictor.IngredientsToSimpleMixturePredictor`
+        * :class:`~citrine.informatics.predictors.ingredients_to_formulation_predictor.IngredientsToFormulationPredictor`
+        * :class:`~citrine.informatics.predictors.label_fractions_predictor.LabelFractionsPredictor`
+        * :class:`~citrine.informatics.predictors.molecular_structure_featurizer.MolecularStructureFeaturizer`
+        * :class:`~citrine.informatics.predictors.simple_mixture_predictor.SimpleMixturePredictor`
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """  # noqa: E501,W505
+        return self._versions_collection.default_evaluators(uid, version=version)
+
+    def evaluate(self,
+                 uid: Union[UUID, str],
+                 *,
+                 version: Union[int, str] = LATEST_VER,
+                 evaluators: List[PredictorEvaluator]) -> PredictorEvaluation:
+        """Evaluate a predictor using the provided evaluators.
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate. Defaults to the latest trained version.
+        evaluators: list
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """
+        return self._versions_collection.evaluate(uid, version=version, evaluators=evaluators)
+
+    def evaluate_default(self,
+                         uid: Union[UUID, str],
+                         *,
+                         version: Union[int, str] = MOST_RECENT_VER) -> PredictorEvaluation:
+        """Evaluate a predictor using the default evaluators.
+
+        See :func:`~citrine.resources.PredictorCollection.default_evaluators` for details on the evaluators.
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """  # noqa: E501,W505
+        return self._versions_collection.evaluate_default(uid, version=version)
+
     def delete(self, uid: Union[UUID, str]):
         """Predictors cannot be deleted at this time."""
         msg = "Predictors cannot be deleted. Use 'archive_version' or 'archive_root' instead."