Merge pull request #778 from CitrineInformatics/model-selection-report

Support for model selection summary

Author: Sean Friedowitz

src/citrine/__version__.py

@@ -1 +1 @@
-__version__ = '1.36.6'
+__version__ = '1.37.0'

src/citrine/informatics/reports.py

@@ -1,5 +1,5 @@
 """Tools for working with reports."""
-from typing import Type, Dict, TypeVar, Iterable
+from typing import Type, Dict, TypeVar, Iterable, Any, Set
 from abc import abstractmethod
 from itertools import groupby
 from logging import getLogger
@@ -9,6 +9,7 @@
 from citrine._serialization.serializable import Serializable
 from citrine._rest.asynchronous_object import AsynchronousObject
 from citrine.informatics.descriptors import Descriptor
+from citrine.informatics.predictor_evaluation_result import ResponseMetrics
 
 SelfType = TypeVar('SelfType', bound='Report')
 
@@ -62,6 +63,58 @@ def __str__(self):
         return "<FeatureImportanceReport {!r}>".format(self.output_key)  # pragma: no cover
 
 
+class ModelEvaluationResult(Serializable["ModelEvaluationResult"]):
+    """[ALPHA] Settings and evaluation metrics for a single algorithm from AutoML model selection.
+
+    ModelEvaluationResult objects are included in a ModelSelectionReport
+    and should not be user-instantiated.
+    """
+
+    model_settings = properties.Raw('model_settings')
+    _response_results = properties.Mapping(
+        properties.String,
+        properties.Object(ResponseMetrics),
+        "response_results"
+    )
+
+    def __init__(self):
+        pass  # pragma: no cover
+
+    def __str__(self):
+        return '<ModelEvaluationResult>'  # pragma: no cover
+
+    def __getitem__(self, item):
+        return self._response_results[item]
+
+    def __iter__(self):
+        return iter(self.responses)
+
+    @property
+    def responses(self) -> Set[str]:
+        """Responses the model was evaluated on."""
+        return set(self._response_results.keys())
+
+
+class ModelSelectionReport(Serializable["ModelSelectionReport"]):
+    """[ALPHA] Summary of selection settings and model performance from AutoML model selection.
+
+    ModelSelectionReport objects are constructed from saved models and
+    should not be user-instantiated.
+    """
+
+    n_folds = properties.Integer('n_folds')
+    evaluation_results = properties.List(
+        properties.Object(ModelEvaluationResult),
+        "evaluation_results"
+    )
+
+    def __init__(self):
+        pass  # pragma: no cover
+
+    def __str__(self):
+        return '<ModelSelectionReport>'  # pragma: no cover
+
+
 class ModelSummary(Serializable['ModelSummary']):
     """[ALPHA] Summary of information about a single model in a predictor.
 
@@ -87,6 +140,10 @@ class ModelSummary(Serializable['ModelSummary']):
     feature_importances = properties.List(
         properties.Object(FeatureImportanceReport), 'feature_importances')
     """:List[FeatureImportanceReport]: feature importance reports for each output"""
+    selection_summary = properties.Optional(
+        properties.Object(ModelSelectionReport), "selection_summary"
+    )
+    """:Optional[ModelSelectionReport]: optional results of AutoML model selection"""
     predictor_name = properties.String('predictor_configuration_name', default='')
     """:str: the name of the predictor that created this model"""
     predictor_uid = properties.Optional(properties.UUID(), 'predictor_configuration_uid')
@@ -126,8 +183,13 @@ def __init__(self):
     def post_build(self):
         """Modify a PredictorReport object in-place after deserialization."""
         self._fill_out_descriptors()
-        for _, model in enumerate(self.model_summaries):
-            self._collapse_model_settings(model)
+        for _, summary in enumerate(self.model_summaries):
+            # Collapse settings on final trained model
+            summary.model_settings = self._collapse_model_settings(summary.model_settings)
+            if summary.selection_summary is not None:
+                # Collapse settings on any child model evaluation results
+                for result in summary.selection_summary.evaluation_results:
+                    result.model_settings = self._collapse_model_settings(result.model_settings)
 
     def _fill_out_descriptors(self):
         """Replace descriptor keys in `model_summaries` with full Descriptor objects."""
@@ -146,7 +208,7 @@ def _fill_out_descriptors(self):
                     model.outputs[j] = descriptor_map[output_key]
                 except KeyError:
                     raise RuntimeError("Model {} contains output \'{}\', but no descriptor found "
-                                       "with that key".format(model.name, input_key))
+                                       "with that key".format(model.name, output_key))
 
     @staticmethod
     def _get_sole_descriptor(it: Iterable):
@@ -170,7 +232,7 @@ def _get_sole_descriptor(it: Iterable):
         return as_list[0]
 
     @staticmethod
-    def _collapse_model_settings(model: ModelSummary):
+    def _collapse_model_settings(raw_settings: Dict[str, Any]):
         """Collapse a model's settings into a flat dictionary.
 
         Model settings are returned as a dictionary with a "name" field, a "value" field,
@@ -187,6 +249,6 @@ def _recurse_model_settings(settings: Dict[str, str], list_or_dict):
                 settings[list_or_dict['name']] = list_or_dict['value']
                 _recurse_model_settings(settings, list_or_dict['children'])
 
-        settings = dict()
-        _recurse_model_settings(settings, model.model_settings)
-        model.model_settings = settings
+        collapsed = dict()
+        _recurse_model_settings(collapsed, raw_settings)
+        return collapsed

tests/conftest.py

@@ -317,7 +317,7 @@ def valid_expression_predictor_data():
 
 
 @pytest.fixture
-def valid_predictor_report_data():
+def valid_predictor_report_data(example_categorical_pva_metrics, example_f1_metrics):
     """Produce valid data used for tests."""
     from citrine.informatics.descriptors import RealDescriptor
     x = RealDescriptor("x", lower_bound=0, upper_bound=1, units="")
@@ -355,6 +355,32 @@ def valid_predictor_report_data():
                             top_features=5
                         )
                     ],
+                    selection_summary=dict(
+                        n_folds=4,
+                        evaluation_results=[
+                            dict(
+                                model_settings=[
+                                    dict(
+                                        name='Algorithm',
+                                        value='Ensemble of non-linear estimators',
+                                        children=[
+                                            dict(name='Number of estimators', value=64, children=[]),
+                                            dict(name='Leaf model', value='Mean', children=[]),
+                                            dict(name='Use jackknife', value=True, children=[])
+                                        ]
+                                    )
+                                ],
+                                response_results=dict(
+                                    response_name=dict(
+                                        metrics=dict(
+                                            predicted_vs_actual=example_categorical_pva_metrics,
+                                            f1=example_f1_metrics
+                                        )
+                                    )
+                                )
+                            )
+                        ]
+                    ),
                     predictor_configuration_name="Predict y from x with ML"
                 ),
                 dict(

tests/informatics/test_reports.py

@@ -7,3 +7,20 @@ def test_status(valid_predictor_report_data):
     """Ensure we can check the status of report generation."""
     report = Report.build(valid_predictor_report_data)
     assert report.succeeded() and not report.in_progress() and not report.failed()
+
+
+def test_selection_summary(valid_predictor_report_data):
+    """Ensure that we can iterate selection summary results as expected."""
+    report = PredictorReport.build(valid_predictor_report_data)
+    selection_summaries = [
+        s.selection_summary for s in report.model_summaries if s.selection_summary is not None
+    ]
+
+    assert len(selection_summaries) > 0
+    for s in selection_summaries:
+        assert len(s.evaluation_results) > 0
+        for result in s.evaluation_results:
+            assert len(result.model_settings) > 0
+            for response_key in result:
+                metrics = result[response_key].metrics
+                assert len(metrics) > 0