Merge pull request #768 from CitrineInformatics/PLA-9819/holdout-evaluator

Add holdout evaluator

Author: bfolie

src/citrine/__version__.py

@@ -1 +1 @@
-__version__ = '1.39.0'
+__version__ = '1.40.0'

src/citrine/informatics/predictor_evaluation_result.py

@@ -4,8 +4,8 @@
 from citrine._serialization.polymorphic_serializable import PolymorphicSerializable
 from citrine._serialization.serializable import Serializable
 from citrine.informatics.predictor_evaluation_metrics import PredictorEvaluationMetric
-from citrine.informatics.predictor_evaluator import PredictorEvaluator
-
+from citrine.informatics.predictor_evaluator import PredictorEvaluator, HoldoutSetEvaluator,\
+    CrossValidationEvaluator
 
 __all__ = ['MetricValue',
            'RealMetricValue',
@@ -166,6 +166,7 @@ def get_type(cls, data) -> Type[Serializable]:
         """Return the subtype."""
         return {
             "CrossValidationResult": CrossValidationResult,
+            "HoldoutSetResult": HoldoutSetResult
         }[data["type"]]
 
     @property
@@ -191,11 +192,11 @@ class CrossValidationResult(Serializable["CrossValidationResult"], PredictorEval
     where ``cvResult`` is a
     :class:`citrine.informatics.predictor_evaluation_result.CrossValidationResult`
     and ``'response_name'`` is a response analyzed by a
-    :class:`citrine.informatics.predictor_evaluator.PredictorEvaluator`.
+    :class:`citrine.informatics.predictor_evaluator.CrossValidationEvaluator`.
 
     """
 
-    _evaluator = properties.Object(PredictorEvaluator, "evaluator")
+    _evaluator = properties.Object(CrossValidationEvaluator, "evaluator")
     _response_results = properties.Mapping(properties.String, properties.Object(ResponseMetrics),
                                            "response_results")
     typ = properties.String('type', default='CrossValidationResult', deserializable=False)
@@ -207,7 +208,45 @@ def __iter__(self):
         return iter(self.responses)
 
     @property
-    def evaluator(self) -> PredictorEvaluator:
+    def evaluator(self) -> CrossValidationEvaluator:
+        """:PredictorEvaluator: Evaluator that produced this result."""
+        return self._evaluator
+
+    @property
+    def responses(self) -> Set[str]:
+        """Responses for which results are present."""
+        return set(self._response_results.keys())
+
+    @property
+    def metrics(self) -> Set[PredictorEvaluationMetric]:
+        """:Set[PredictorEvaluationMetric]: Metrics for which results are present."""
+        return self._evaluator.metrics
+
+
+class HoldoutSetResult(Serializable["HoldoutSetResult"], PredictorEvaluationResult):
+    """Result of performing holdout evaluation on a predictor.
+
+    Results held-out response can be accessed via ``result['response_name']``,
+    where ``result`` is a
+    :class:`citrine.informatics.predictor_evaluation_result.HoldoutSetResult`
+    and ``'response_name'`` is a response analyzed by a
+    :class:`citrine.informatics.predictor_evaluator.HoldoutSetEvaluator`.
+
+    """
+
+    _evaluator = properties.Object(HoldoutSetEvaluator, "evaluator")
+    _response_results = properties.Mapping(properties.String, properties.Object(ResponseMetrics),
+                                           "response_results")
+    typ = properties.String('type', default='HoldoutSetResult', deserializable=False)
+
+    def __getitem__(self, item):
+        return self._response_results[item]
+
+    def __iter__(self):
+        return iter(self.responses)
+
+    @property
+    def evaluator(self) -> HoldoutSetEvaluator:
         """:PredictorEvaluator: Evaluator that produced this result."""
         return self._evaluator
 

src/citrine/informatics/predictor_evaluator.py

@@ -4,9 +4,12 @@
 from citrine._serialization.polymorphic_serializable import PolymorphicSerializable
 from citrine._serialization.serializable import Serializable
 from citrine.informatics.predictor_evaluation_metrics import PredictorEvaluationMetric
+from citrine.informatics.data_sources import DataSource
 
 __all__ = ['PredictorEvaluator',
-           'CrossValidationEvaluator']
+           'CrossValidationEvaluator',
+           'HoldoutSetEvaluator'
+           ]
 
 
 class PredictorEvaluator(PolymorphicSerializable["PredictorEvaluator"]):
@@ -17,6 +20,7 @@ def get_type(cls, data) -> Type[Serializable]:
         """Return the subtype."""
         return {
             "CrossValidationEvaluator": CrossValidationEvaluator,
+            "HoldoutSetEvaluator": HoldoutSetEvaluator
         }[data["type"]]
 
     def _attrs(self) -> List[str]:
@@ -129,3 +133,57 @@ def responses(self) -> Set[str]:
     def metrics(self) -> Set[PredictorEvaluationMetric]:
         """Set of metrics computed during cross-validation."""
         return self._metrics
+
+
+class HoldoutSetEvaluator(Serializable["HoldoutSetEvaluator"], PredictorEvaluator):
+    """Evaluate a predictor using a holdout set.
+
+    For each response, the actual values are masked off and the predictor makes predictions.
+    These predictions are compared with the ground-truth values in the holdout set using
+    specified metrics.
+
+    Parameters
+    ----------
+    name: str
+        Name of the evaluator
+    responses: Set[str]
+        Set of descriptor keys to evaluate
+    data_source: DataSource
+        Source of holdout data
+    metrics: Optional[Set[PredictorEvaluationMetric]]
+        Optional set of metrics to compute for each response. Default is all metrics.
+
+    """
+
+    def _attrs(self) -> List[str]:
+        return ["typ", "name", "responses", "data_source", "metrics"]
+
+    name = properties.String("name")
+    description = properties.String("description")
+    _responses = properties.Set(properties.String, "responses")
+    data_source = properties.Object(DataSource, "data_source")
+    _metrics = properties.Optional(properties.Set(properties.Object(PredictorEvaluationMetric)),
+                                   "metrics")
+    typ = properties.String("type", default="HoldoutSetEvaluator", deserializable=False)
+
+    def __init__(self,
+                 name: str, *,
+                 description: str = "",
+                 responses: Set[str],
+                 data_source: DataSource,
+                 metrics: Optional[Set[PredictorEvaluationMetric]] = None):
+        self.name: str = name
+        self.description: str = description
+        self._responses: Set[str] = responses
+        self.data_source = data_source
+        self._metrics: Optional[Set[PredictorEvaluationMetric]] = metrics
+
+    @property
+    def responses(self) -> Set[str]:
+        """Set of responses to predict and compare against the ground-truth values."""
+        return self._responses
+
+    @property
+    def metrics(self) -> Set[PredictorEvaluationMetric]:
+        """Set of metrics computed on the predictions."""
+        return self._metrics

tests/conftest.py

@@ -571,8 +571,8 @@ def valid_simple_mixture_predictor_data():
     return PredictorEntityDataFactory(data=PredictorDataDataFactory(instance=instance))
 
 
-@pytest.fixture()
-def example_evaluator_dict():
+@pytest.fixture
+def example_cv_evaluator_dict():
     return {
         "type": "CrossValidationEvaluator",
         "name": "Example evaluator",
@@ -587,6 +587,17 @@ def example_evaluator_dict():
     }
 
 
+@pytest.fixture
+def example_holdout_evaluator_dict(valid_gem_data_source_dict):
+    return {
+        "type": "HoldoutSetEvaluator",
+        "name": "Example holdout evaluator",
+        "description": "",
+        "responses": ["sweetness"],
+        "data_source": valid_gem_data_source_dict,
+        "metrics": [{"type": "RMSE"}]
+    }
+
 @pytest.fixture()
 def example_rmse_metrics():
     return {
@@ -652,10 +663,10 @@ def example_categorical_pva_metrics():
 
 
 @pytest.fixture()
-def example_result_dict(example_evaluator_dict, example_rmse_metrics, example_categorical_pva_metrics, example_f1_metrics, example_real_pva_metrics):
+def example_cv_result_dict(example_cv_evaluator_dict, example_rmse_metrics, example_categorical_pva_metrics, example_f1_metrics, example_real_pva_metrics):
     return {
         "type": "CrossValidationResult",
-        "evaluator": example_evaluator_dict,
+        "evaluator": example_cv_evaluator_dict,
         "response_results": {
             "salt?": {
                 "metrics": {
@@ -673,6 +684,21 @@ def example_result_dict(example_evaluator_dict, example_rmse_metrics, example_ca
     }
 
 
+@pytest.fixture()
+def example_holdout_result_dict(example_holdout_evaluator_dict, example_rmse_metrics):
+    return {
+        "type": "HoldoutSetResult",
+        "evaluator": example_holdout_evaluator_dict,
+        "response_results": {
+            "sweetness": {
+                "metrics": {
+                    "rmse": example_rmse_metrics
+                }
+            }
+        }
+    }
+
+
 @pytest.fixture()
 def example_candidates():
     return {
@@ -745,12 +771,12 @@ def design_execution_dict(generic_entity):
 
 
 @pytest.fixture
-def predictor_evaluation_workflow_dict(generic_entity, example_evaluator_dict):
+def predictor_evaluation_workflow_dict(generic_entity, example_cv_evaluator_dict, example_holdout_evaluator_dict):
     ret = deepcopy(generic_entity)
     ret.update({
         "name": "Example PEW",
         "description": "Example PEW for testing",
-        "evaluators": [example_evaluator_dict]
+        "evaluators": [example_cv_evaluator_dict, example_holdout_evaluator_dict]
     })
     return ret
 

tests/informatics/test_predictor_evaluation_result.py

@@ -9,48 +9,61 @@
 
 
 @pytest.fixture
-def example_result(example_result_dict):
-    return PredictorEvaluationResult.build(example_result_dict)
+def example_cv_result(example_cv_result_dict):
+    return PredictorEvaluationResult.build(example_cv_result_dict)
 
 
-def test_indexing(example_result):
-    assert example_result.responses == {"saltiness", "salt?"}
-    assert example_result.metrics == {RMSE(), PVA(), F1()}
-    assert set(example_result["salt?"]) == {repr(F1()), repr(PVA())}
-    assert set(example_result) == {"salt?", "saltiness"}
+@pytest.fixture
+def example_holdout_result(example_holdout_result_dict):
+    return PredictorEvaluationResult.build(example_holdout_result_dict)
+
+
+def test_indexing(example_cv_result, example_holdout_result):
+    assert example_cv_result.responses == {"saltiness", "salt?"}
+    assert example_holdout_result.responses == {"sweetness"}
+    assert example_cv_result.metrics == {RMSE(), PVA(), F1()}
+    assert example_holdout_result.metrics == {RMSE()}
+    assert set(example_cv_result["salt?"]) == {repr(F1()), repr(PVA())}
+    assert set(example_cv_result) == {"salt?", "saltiness"}
+    assert set(example_holdout_result["sweetness"]) == {repr(RMSE())}
+    assert set(example_holdout_result) == {"sweetness"}
+
 
+def test_cv_serde(example_cv_result, example_cv_result_dict):
+    round_trip = PredictorEvaluationResult.build(json.loads(json.dumps(example_cv_result_dict)))
+    assert example_cv_result.evaluator == round_trip.evaluator
 
-def test_serde(example_result, example_result_dict):
-    round_trip = PredictorEvaluationResult.build(json.loads(json.dumps(example_result_dict)))
-    assert example_result.evaluator == round_trip.evaluator
 
+def test_holdout_serde(example_holdout_result, example_holdout_result_dict):
+    round_trip = PredictorEvaluationResult.build(json.loads(json.dumps(example_holdout_result_dict)))
+    assert example_holdout_result.evaluator == round_trip.evaluator
 
-def test_evaluator(example_result, example_evaluator_dict):
-    args = example_evaluator_dict
+def test_evaluator(example_cv_result, example_cv_evaluator_dict):
+    args = example_cv_evaluator_dict
     del args["type"]
     expected = CrossValidationEvaluator(**args)
-    assert example_result.evaluator == expected
-    assert example_result.evaluator != 0  # make sure eq does something for mismatched classes
+    assert example_cv_result.evaluator == expected
+    assert example_cv_result.evaluator != 0  # make sure eq does something for mismatched classes
 
 
-def test_check_rmse(example_result, example_rmse_metrics):
-    assert example_result["saltiness"]["rmse"].mean == example_rmse_metrics["mean"]
-    assert example_result["saltiness"][RMSE()].standard_error == example_rmse_metrics["standard_error"]
+def test_check_rmse(example_cv_result, example_rmse_metrics):
+    assert example_cv_result["saltiness"]["rmse"].mean == example_rmse_metrics["mean"]
+    assert example_cv_result["saltiness"][RMSE()].standard_error == example_rmse_metrics["standard_error"]
     # check eq method does something
-    assert example_result["saltiness"][RMSE()] != 0
+    assert example_cv_result["saltiness"][RMSE()] != 0
     with pytest.raises(TypeError):
-        foo = example_result["saltiness"][0]
+        foo = example_cv_result["saltiness"][0]
 
 
-def test_real_pva(example_result, example_real_pva_metrics):
+def test_real_pva(example_cv_result, example_real_pva_metrics):
     args = example_real_pva_metrics["value"][0]
     expected = PredictedVsActualRealPoint.build(args)
-    assert example_result["saltiness"]["predicted_vs_actual"][0].predicted == expected.predicted
-    assert next(iter(example_result["saltiness"]["predicted_vs_actual"])).actual == expected.actual
+    assert example_cv_result["saltiness"]["predicted_vs_actual"][0].predicted == expected.predicted
+    assert next(iter(example_cv_result["saltiness"]["predicted_vs_actual"])).actual == expected.actual
 
 
-def test_categorical_pva(example_result, example_categorical_pva_metrics):
+def test_categorical_pva(example_cv_result, example_categorical_pva_metrics):
     args = example_categorical_pva_metrics["value"][0]
     expected = PredictedVsActualCategoricalPoint.build(args)
-    assert example_result["salt?"]["predicted_vs_actual"][0].predicted == expected.predicted
-    assert next(iter(example_result["salt?"]["predicted_vs_actual"])).actual == expected.actual
+    assert example_cv_result["salt?"]["predicted_vs_actual"][0].predicted == expected.predicted
+    assert next(iter(example_cv_result["salt?"]["predicted_vs_actual"])).actual == expected.actual

tests/informatics/workflows/test_predictor_evaluation_workflow.py

@@ -1,16 +1,20 @@
 import pytest
+import uuid
 
-from citrine.informatics.predictor_evaluator import CrossValidationEvaluator, PredictorEvaluator
+from citrine.informatics.data_sources import GemTableDataSource
+from citrine.informatics.predictor_evaluator import HoldoutSetEvaluator, CrossValidationEvaluator, PredictorEvaluator
 from citrine.informatics.workflows import PredictorEvaluationWorkflow
 
 
 @pytest.fixture()
 def pew():
-    evaluator = CrossValidationEvaluator(name="test", responses={"foo"})
+    data_source = GemTableDataSource(table_id=uuid.uuid4(), table_version=3)
+    evaluator1 = CrossValidationEvaluator(name="test CV", responses={"foo"})
+    evaluator2 = HoldoutSetEvaluator(name="test holdout", responses={"foo"}, data_source=data_source)
     pew = PredictorEvaluationWorkflow(
         name="Test",
         description="TestWorkflow",
-        evaluators=[evaluator]
+        evaluators=[evaluator1, evaluator2]
     )
     return pew
 
@@ -20,6 +24,7 @@ def test_round_robin(pew):
     assert dumped["name"] == "Test"
     assert dumped["description"] == "TestWorkflow"
     assert PredictorEvaluator.build(dumped["evaluators"][0]).name == pew.evaluators[0].name
+    assert PredictorEvaluator.build(dumped["evaluators"][1]).name == pew.evaluators[1].name
 
 
 def test_print(pew):

tests/resources/test_predictor_evaluation_executions.py

@@ -61,15 +61,16 @@ def test_build_new_execution(collection, predictor_evaluation_execution_dict):
     assert execution.in_progress() and not execution.succeeded() and not execution.failed()
 
 
-def test_workflow_execution_results(workflow_execution: PredictorEvaluationExecution, session, example_result_dict):
+def test_workflow_execution_results(workflow_execution: PredictorEvaluationExecution, session,
+                                    example_cv_result_dict):
     # Given
-    session.set_response(example_result_dict)
+    session.set_response(example_cv_result_dict)
 
     # When
     results = workflow_execution["Example Evaluator"]
 
     # Then
-    assert results.evaluator == PredictorEvaluationResult.build(example_result_dict).evaluator
+    assert results.evaluator == PredictorEvaluationResult.build(example_cv_result_dict).evaluator
     expected_path = '/projects/{}/predictor-evaluation-executions/{}/results'.format(
         workflow_execution.project_id,
         workflow_execution.uid,