[PNE-7018] Add new evaluations API.

anoto-moniz · anoto-moniz · commit de7d659c0db2 · 2025-09-05T16:42:55.000-04:00
Working on adding tests now. Need to add evaluators to the factory file,
then use it for better testing.

Also need to deprecate the PEW stuff.
diff --git a/src/citrine/__version__.py b/src/citrine/__version__.py
@@ -1 +1 @@
-__version__ = "3.22.1"
+__version__ = "3.23.0"
diff --git a/src/citrine/_rest/engine_resource.py b/src/citrine/_rest/engine_resource.py
@@ -42,13 +42,14 @@ def is_archived(self):
         return self.archived_by is not None
 
     def _post_dump(self, data: dict) -> dict:
-        # Only the data portion of an entity is sent to the server.
-        data = data["data"]
-
-        if "instance" in data:
-            # Currently, name and description exists on both the data envelope and the config.
-            data["instance"]["name"] = data["name"]
-            data["instance"]["description"] = data["description"]
+        if data:
+            # Only the data portion of an entity is sent to the server.
+            data = data["data"]
+
+            if "instance" in data:
+                # Currently, name and description exists on both the data envelope and the config.
+                data["instance"]["name"] = data["name"]
+                data["instance"]["description"] = data["description"]
 
         return super()._post_dump(data)
 
diff --git a/src/citrine/informatics/executions/predictor_evaluation.py b/src/citrine/informatics/executions/predictor_evaluation.py
@@ -0,0 +1,98 @@
+from functools import lru_cache
+from typing import List, Optional, Union
+from uuid import UUID
+
+from citrine.informatics.predictor_evaluation_result import PredictorEvaluationResult
+from citrine.informatics.predictor_evaluator import PredictorEvaluator
+from citrine.resources.status_detail import StatusDetail
+from citrine._rest.engine_resource import EngineResourceWithoutStatus
+from citrine._rest.resource import PredictorRef
+from citrine._serialization import properties
+from citrine._serialization.serializable import Serializable
+from citrine._utils.functions import format_escaped_url
+
+
+class PredictorEvaluatorsResponse(Serializable['EvaluatorsPayload']):
+    """Container object for a default predictor evaluator response."""
+
+    evaluators = properties.List(properties.Object(PredictorEvaluator), "evaluators")
+
+    def __init__(self, evaluators: List[PredictorEvaluator]):
+        self.evaluators = evaluators
+
+
+class PredictorEvaluationRequest(Serializable['EvaluatorsPayload']):
+    """Container object for a predictor evaluation request."""
+
+    predictor = properties.Object(PredictorRef, "predictor")
+    evaluators = properties.List(properties.Object(PredictorEvaluator), "evaluators")
+
+    def __init__(self,
+                 *,
+                 evaluators: List[PredictorEvaluator],
+                 predictor_id: Union[UUID, str],
+                 predictor_version: Optional[Union[int, str]] = None):
+        self.evaluators = evaluators
+        self.predictor = PredictorRef(predictor_id, predictor_version)
+
+
+class PredictorEvaluation(EngineResourceWithoutStatus['PredictorEvaluation']):
+    """The evaluation of a predictor's performance."""
+
+    uid: UUID = properties.UUID('id', serializable=False)
+    """:UUID: Unique identifier of the evaluation"""
+    evaluators = properties.List(properties.Object(PredictorEvaluator), "data.evaluators",
+                                 serializable=False)
+    """:List{PredictorEvaluator]:the predictor evaluators that were executed. These are used
+    when calling the ``results()`` method."""
+    predictor_id = properties.UUID('metadata.predictor_id', serializable=False)
+    """:UUID:"""
+    predictor_version = properties.Integer('metadata.predictor_version', serializable=False)
+    status = properties.String('metadata.status.major', serializable=False)
+    """:str: short description of the evaluation's status"""
+    status_description = properties.String('metadata.status.minor', serializable=False)
+    """:str: more detailed description of the evaluation's status"""
+    status_detail = properties.List(properties.Object(StatusDetail), 'metadata.status.detail',
+                                    default=[], serializable=False)
+    """:List[StatusDetail]: a list of structured status info, containing the message and level"""
+
+    def _path(self):
+        return format_escaped_url(
+            '/projects/{project_id}/predictor-evaluations/{evaluation_id}',
+            project_id=str(self.project_id),
+            evaluation_id=str(self.uid)
+        )
+
+    @lru_cache()
+    def results(self, evaluator_name: str) -> PredictorEvaluationResult:
+        """
+        Get a specific evaluation result by the name of the evaluator that produced it.
+
+        Parameters
+        ----------
+        evaluator_name: str
+            Name of the evaluator for which to get the results
+
+        Returns
+        -------
+        PredictorEvaluationResult
+            The evaluation result from the evaluator with the given name
+
+        """
+        params = {"evaluator_name": evaluator_name}
+        resource = self._session.get_resource(self._path() + "/results", params=params)
+        return PredictorEvaluationResult.build(resource)
+
+    @property
+    def evaluator_names(self):
+        """Names of the predictor evaluators. Used when calling the ``results()`` method."""
+        return list(iter(self))
+
+    def __getitem__(self, item):
+        if isinstance(item, str):
+            return self.results(item)
+        else:
+            raise TypeError("Results are accessed by string names")
+
+    def __iter__(self):
+        return iter(e.name for e in self.evaluators)
diff --git a/src/citrine/resources/design_space.py b/src/citrine/resources/design_space.py
@@ -129,15 +129,15 @@ def _list_base(self, *, per_page: int = 100, archived: Optional[bool] = None):
                                         per_page=per_page)
 
     def list_all(self, *, per_page: int = 20) -> Iterable[DesignSpace]:
-        """List the most recent version of all design spaces."""
+        """List all design spaces."""
         return self._list_base(per_page=per_page)
 
     def list(self, *, per_page: int = 20) -> Iterable[DesignSpace]:
-        """List the most recent version of all non-archived design spaces."""
+        """List non-archived design spaces."""
         return self._list_base(per_page=per_page, archived=False)
 
     def list_archived(self, *, per_page: int = 20) -> Iterable[DesignSpace]:
-        """List the most recent version of all archived predictors."""
+        """List archived design spaces."""
         return self._list_base(per_page=per_page, archived=True)
 
     def create_default(self,
diff --git a/src/citrine/resources/predictor_evaluation.py b/src/citrine/resources/predictor_evaluation.py
@@ -0,0 +1,219 @@
+from functools import partial
+from typing import Iterable, Iterator, List, Optional, Union
+from uuid import UUID
+
+from citrine.informatics.executions.predictor_evaluation import PredictorEvaluation, \
+    PredictorEvaluationRequest, PredictorEvaluatorsResponse
+from citrine.informatics.predictor_evaluator import PredictorEvaluator
+from citrine.informatics.predictors import GraphPredictor
+from citrine.resources.predictor import LATEST_VER as LATEST_PRED_VER
+from citrine._rest.collection import Collection
+from citrine._rest.resource import PredictorRef
+from citrine._session import Session
+
+
+class PredictorEvaluationCollection(Collection[PredictorEvaluation]):
+    """Represents the collection of predictor evaluations.
+
+    Parameters
+    ----------
+    project_id: UUID
+        the UUID of the project
+
+    """
+
+    _api_version = 'v1'
+    _path_template = '/projects/{project_id}/predictor-evaluations'
+    _individual_key = None
+    _resource = PredictorEvaluation
+    _collection_key = 'response'
+
+    def __init__(self, project_id: UUID, session: Session):
+        self.project_id = project_id
+        self.session: Session = session
+
+    def build(self, data: dict) -> PredictorEvaluation:
+        """Build an individual predictor evaluation."""
+        evaluation = PredictorEvaluation.build(data)
+        evaluation._session = self.session
+        evaluation._project_id = self.project_id
+        return evaluation
+
+    def _list_base(self,
+                   *,
+                   per_page: int = 100,
+                   predictor_id: Optional[UUID] = None,
+                   predictor_version: Optional[Union[int, str]] = None,
+                   archived: Optional[bool] = None
+                   ) -> Iterator[PredictorEvaluation]:
+        params = {"archived": archived}
+        if predictor_id is not None:
+            params["predictor_id"] = str(predictor_id)
+        if predictor_version is not None:
+            params["predictor_version"] = predictor_version
+
+        fetcher = partial(self._fetch_page, additional_params=params)
+        return self._paginator.paginate(page_fetcher=fetcher,
+                                        collection_builder=self._build_collection_elements,
+                                        per_page=per_page)
+
+    def list_all(self,
+                 *,
+                 per_page: int = 100,
+                 predictor_id: Optional[UUID] = None,
+                 predictor_version: Optional[Union[int, str]] = None
+                 ) -> Iterable[PredictorEvaluation]:
+        """List all predictor evaluations."""
+        return self._list_base(per_page=per_page,
+                               predictor_id=predictor_id,
+                               predictor_version=predictor_version)
+
+    def list(self,
+             *,
+             per_page: int = 100,
+             predictor_id: Optional[UUID] = None,
+             predictor_version: Optional[Union[int, str]] = None
+             ) -> Iterable[PredictorEvaluation]:
+        """List non-archived predictor evaluations."""
+        return self._list_base(per_page=per_page,
+                               predictor_id=predictor_id,
+                               predictor_version=predictor_version,
+                               archived=False)
+
+    def list_archived(self,
+                      *,
+                      per_page: int = 100,
+                      predictor_id: Optional[UUID] = None,
+                      predictor_version: Optional[Union[int, str]] = None
+                      ) -> Iterable[PredictorEvaluation]:
+        """List archived predictor evaluations."""
+        return self._list_base(per_page=per_page,
+                               predictor_id=predictor_id,
+                               predictor_version=predictor_version,
+                               archived=True)
+
+    def archive(self, uid: Union[UUID, str]):
+        """Archive an evaluation."""
+        url = self._get_path(uid, action="archive")
+        result = self.session.put_resource(url, {}, version=self._api_version)
+        return self.build(result)
+
+    def restore(self, uid: Union[UUID, str]):
+        """Restore an archived evaluation."""
+        url = self._get_path(uid, action="restore")
+        result = self.session.put_resource(url, {}, version=self._api_version)
+        return self.build(result)
+
+    def default_from_config(self, config: GraphPredictor) -> List[PredictorEvaluator]:
+        """Retrieve the default evaluators for an arbitrary (but valid) predictor config.
+
+        See :func:`~citrine.resources.PredictorEvaluationCollection.default_evaluators` for details
+        on the resulting evaluators.
+        """
+        path = self._get_path(action="default-from-config")
+        payload = config.dump()["instance"]
+        result = self.session.post_resource(path, json=payload, version=self._api_version)
+        return PredictorEvaluatorsResponse.build(result).evaluators
+
+    def default(self,
+                *,
+                predictor_id: Union[UUID, str],
+                predictor_version: Union[int, str] = LATEST_PRED_VER
+                ) -> List[PredictorEvaluator]:
+        """Retrieve the default evaluators for a stored predictor.
+
+        The current default evaluators perform 5-fold, 3-trial cross-validation on all valid
+        predictor responses. Valid responses are those that are **not** produced by the
+        following predictors:
+
+        * :class:`~citrine.informatics.predictors.generalized_mean_property_predictor.GeneralizedMeanPropertyPredictor`
+        * :class:`~citrine.informatics.predictors.mean_property_predictor.MeanPropertyPredictor`
+        * :class:`~citrine.informatics.predictors.ingredient_fractions_predictor.IngredientFractionsPredictor`
+        * :class:`~citrine.informatics.predictors.ingredients_to_simple_mixture_predictor.IngredientsToSimpleMixturePredictor`
+        * :class:`~citrine.informatics.predictors.ingredients_to_formulation_predictor.IngredientsToFormulationPredictor`
+        * :class:`~citrine.informatics.predictors.label_fractions_predictor.LabelFractionsPredictor`
+        * :class:`~citrine.informatics.predictors.molecular_structure_featurizer.MolecularStructureFeaturizer`
+        * :class:`~citrine.informatics.predictors.simple_mixture_predictor.SimpleMixturePredictor`
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """  # noqa: E501,W505
+        path = self._get_path(action="default")
+        payload = PredictorRef(uid=predictor_id, version=predictor_version).dump()
+        result = self.session.post_resource(path, json=payload, version=self._api_version)
+        return PredictorEvaluatorsResponse.build(result).evaluators
+
+    def trigger(self,
+                *,
+                predictor_id: Union[UUID, str],
+                predictor_version: Union[int, str] = LATEST_PRED_VER,
+                evaluators: List[PredictorEvaluator]) -> PredictorEvaluation:
+        """Evaluate a predictor using the provided evaluators.
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate. Defaults to the latest trained version.
+        evaluators: List[PredictorEvaluator]
+            The evaluators to use to measure predictor performance.
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """
+        path = self._get_path("trigger")
+        payload = PredictorEvaluationRequest(evaluators=evaluators,
+                                             predictor_id=predictor_id,
+                                             predictor_version=predictor_version).dump()
+        result = self.session.post_resource(path, payload, version=self._api_version)
+        return self.build(result)
+
+    def trigger_default(self,
+                        *,
+                        predictor_id: Union[UUID, str],
+                        predictor_version: Union[int, str] = LATEST_PRED_VER
+                        ) -> PredictorEvaluation:
+        """Evaluate a predictor using the default evaluators.
+
+        See :func:`~citrine.resources.PredictorCollection.default_evaluators` for details on the evaluators.
+
+        Parameters
+        ----------
+        predictor_id: UUID
+            Unique identifier of the predictor to evaluate
+        predictor_version: Option[Union[int, str]]
+            The version of the predictor to evaluate
+
+        Returns
+        -------
+        PredictorEvaluation
+
+        """  # noqa: E501,W505
+        path = self._get_path("trigger-default")
+        payload = PredictorRef(uid=predictor_id, version=predictor_version).dump()
+        result = self.session.post_resource(path, json=payload, version=self._api_version)
+        return self.build(result)
+
+    def register(self, model: PredictorEvaluation) -> PredictorEvaluation:
+        """Cannot register an evaluation."""
+        raise NotImplementedError("Cannot register a PredictorEvaluation.")
+
+    def update(self, model: PredictorEvaluation) -> PredictorEvaluation:
+        """Cannot update an evaluation."""
+        raise NotImplementedError("Cannot update a PredictorEvaluation.")
+
+    def delete(self, uid: Union[UUID, str]):
+        """Cannot delete an evaluation."""
+        raise NotImplementedError("Cannot delete a PredictorEvaluation.")
diff --git a/src/citrine/resources/project.py b/src/citrine/resources/project.py
@@ -40,6 +40,7 @@
     PredictorEvaluationExecutionCollection
 from citrine.resources.predictor_evaluation_workflow import \
     PredictorEvaluationWorkflowCollection
+from citrine.resources.predictor_evaluation import PredictorEvaluationCollection
 from citrine.resources.generative_design_execution import \
     GenerativeDesignExecutionCollection
 from citrine.resources.project_member import ProjectMember
@@ -148,6 +149,11 @@ def predictor_evaluation_executions(self) -> PredictorEvaluationExecutionCollect
         """Return a collection representing all visible predictor evaluation executions."""
         return PredictorEvaluationExecutionCollection(project_id=self.uid, session=self.session)
 
+    @property
+    def predictor_evaluations(self) -> PredictorEvaluationCollection:
+        """Return a collection representing all visible predictor evaluations."""
+        return PredictorEvaluationCollection(project_id=self.uid, session=self.session)
+
     @property
     def design_workflows(self) -> DesignWorkflowCollection:
         """Return a collection representing all visible design workflows."""
diff --git a/tests/resources/test_project.py b/tests/resources/test_project.py
@@ -314,6 +314,10 @@ def test_pe_executions_get_project_id(project):
         project.predictor_evaluation_executions.trigger(uuid.uuid4())
 
 
+def test_predictor_evaluations_get_project_id(project):
+    assert project.uid == project.predictor_evaluations.project_id
+
+
 def test_design_workflows_get_project_id(project):
     assert project.uid == project.design_workflows.project_id
 
diff --git a/tests/utils/factories.py b/tests/utils/factories.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "3.22.1"`
	`1`	`+__version__ = "3.23.0"`