Merge pull request #713 from openml/fix_687

Adding function to retrieve evaluation measures

Author: PGijsbers

doc/api.rst

@@ -84,6 +84,7 @@ Modules
    :template: function.rst
 
     list_evaluations
+    list_evaluation_measures
 
 :mod:`openml.flows`: Flow Functions
 -----------------------------------

doc/progress.rst

@@ -8,6 +8,8 @@ Changelog
 
 0.10.0
 ~~~~~~
+
+* ADD #687: Adds a function to retrieve the list of evaluation measures available.
 * ADD #695: A function to retrieve all the data quality measures available.
 
 0.9.0

openml/evaluations/__init__.py

@@ -1,4 +1,4 @@
 from .evaluation import OpenMLEvaluation
-from .functions import list_evaluations
+from .functions import list_evaluations, list_evaluation_measures
 
-__all__ = ['OpenMLEvaluation', 'list_evaluations']
+__all__ = ['OpenMLEvaluation', 'list_evaluations', 'list_evaluation_measures']

openml/evaluations/functions.py

@@ -200,3 +200,29 @@ def __list_evaluations(api_call, output_format='object'):
         evals = pd.DataFrame.from_dict(evals, orient='index')
 
     return evals
+
+
+def list_evaluation_measures() -> List[str]:
+    """ Return list of evaluation measures available.
+
+    The function performs an API call to retrieve the entire list of
+    evaluation measures that are available.
+
+    Returns
+    -------
+    list
+
+    """
+    api_call = "evaluationmeasure/list"
+    xml_string = openml._api_calls._perform_api_call(api_call, 'get')
+    qualities = xmltodict.parse(xml_string, force_list=('oml:measures'))
+    # Minimalistic check if the XML is useful
+    if 'oml:evaluation_measures' not in qualities:
+        raise ValueError('Error in return XML, does not contain '
+                         '"oml:evaluation_measures"')
+    if not isinstance(qualities['oml:evaluation_measures']['oml:measures'][0]['oml:measure'],
+                      list):
+        raise TypeError('Error in return XML, does not contain '
+                        '"oml:measure" as a list')
+    qualities = qualities['oml:evaluation_measures']['oml:measures'][0]['oml:measure']
+    return qualities

tests/test_evaluations/test_evaluation_functions.py

@@ -116,3 +116,8 @@ def test_evaluation_list_per_fold(self):
         for run_id in evaluations.keys():
             self.assertIsNotNone(evaluations[run_id].value)
             self.assertIsNone(evaluations[run_id].values)
+
+    def test_list_evaluation_measures(self):
+        measures = openml.evaluations.list_evaluation_measures()
+        self.assertEqual(isinstance(measures, list), True)
+        self.assertEqual(all([isinstance(s, str) for s in measures]), True)