Merge branch 'develop' into fix_608

Author: PGijsbers

doc/api.rst

@@ -72,6 +72,7 @@ Modules
     get_dataset
     get_datasets
     list_datasets
+    list_qualities
     status_update
 
 :mod:`openml.evaluations`: Evaluation Functions
@@ -83,6 +84,7 @@ Modules
    :template: function.rst
 
     list_evaluations
+    list_evaluation_measures
 
 :mod:`openml.flows`: Flow Functions
 -----------------------------------

doc/progress.rst

@@ -9,6 +9,8 @@ Changelog
 0.10.0
 ~~~~~~
 * FIX #608: Fixing dataset_id referenced before assignment error in get_run function.
+* ADD #687: Adds a function to retrieve the list of evaluation measures available.
+* ADD #695: A function to retrieve all the data quality measures available.
 
 0.9.0
 ~~~~~

openml/datasets/__init__.py

@@ -6,6 +6,7 @@
     get_datasets,
     list_datasets,
     status_update,
+    list_qualities
 )
 from .dataset import OpenMLDataset
 from .data_feature import OpenMLDataFeature
@@ -20,4 +21,5 @@
     'OpenMLDataset',
     'OpenMLDataFeature',
     'status_update',
+    'list_qualities'
 ]

openml/datasets/functions.py

@@ -165,6 +165,30 @@ def _get_cache_directory(dataset: OpenMLDataset) -> str:
     return _create_cache_directory_for_id(DATASETS_CACHE_DIR_NAME, dataset.dataset_id)
 
 
+def list_qualities() -> List[str]:
+    """ Return list of data qualities available.
+
+    The function performs an API call to retrieve the entire list of
+    data qualities that are computed on the datasets uploaded.
+
+    Returns
+    -------
+    list
+    """
+    api_call = "data/qualities/list"
+    xml_string = openml._api_calls._perform_api_call(api_call, 'get')
+    qualities = xmltodict.parse(xml_string, force_list=('oml:quality'))
+    # Minimalistic check if the XML is useful
+    if 'oml:data_qualities_list' not in qualities:
+        raise ValueError('Error in return XML, does not contain '
+                         '"oml:data_qualities_list"')
+    if not isinstance(qualities['oml:data_qualities_list']['oml:quality'], list):
+        raise TypeError('Error in return XML, does not contain '
+                        '"oml:quality" as a list')
+    qualities = qualities['oml:data_qualities_list']['oml:quality']
+    return qualities
+
+
 def list_datasets(
     offset: Optional[int] = None,
     size: Optional[int] = None,

openml/evaluations/__init__.py

@@ -1,4 +1,4 @@
 from .evaluation import OpenMLEvaluation
-from .functions import list_evaluations
+from .functions import list_evaluations, list_evaluation_measures
 
-__all__ = ['OpenMLEvaluation', 'list_evaluations']
+__all__ = ['OpenMLEvaluation', 'list_evaluations', 'list_evaluation_measures']

openml/evaluations/functions.py

@@ -200,3 +200,29 @@ def __list_evaluations(api_call, output_format='object'):
         evals = pd.DataFrame.from_dict(evals, orient='index')
 
     return evals
+
+
+def list_evaluation_measures() -> List[str]:
+    """ Return list of evaluation measures available.
+
+    The function performs an API call to retrieve the entire list of
+    evaluation measures that are available.
+
+    Returns
+    -------
+    list
+
+    """
+    api_call = "evaluationmeasure/list"
+    xml_string = openml._api_calls._perform_api_call(api_call, 'get')
+    qualities = xmltodict.parse(xml_string, force_list=('oml:measures'))
+    # Minimalistic check if the XML is useful
+    if 'oml:evaluation_measures' not in qualities:
+        raise ValueError('Error in return XML, does not contain '
+                         '"oml:evaluation_measures"')
+    if not isinstance(qualities['oml:evaluation_measures']['oml:measures'][0]['oml:measure'],
+                      list):
+        raise TypeError('Error in return XML, does not contain '
+                        '"oml:measure" as a list')
+    qualities = qualities['oml:evaluation_measures']['oml:measures'][0]['oml:measure']
+    return qualities

tests/test_datasets/test_dataset_functions.py

@@ -1190,3 +1190,8 @@ def test_create_dataset_attributes_auto_without_df(self):
                 original_data_url=original_data_url,
                 paper_url=paper_url
             )
+
+    def test_list_qualities(self):
+        qualities = openml.datasets.list_qualities()
+        self.assertEqual(isinstance(qualities, list), True)
+        self.assertEqual(all([isinstance(q, str) for q in qualities]), True)

tests/test_evaluations/test_evaluation_functions.py

@@ -116,3 +116,8 @@ def test_evaluation_list_per_fold(self):
         for run_id in evaluations.keys():
             self.assertIsNotNone(evaluations[run_id].value)
             self.assertIsNone(evaluations[run_id].values)
+
+    def test_list_evaluation_measures(self):
+        measures = openml.evaluations.list_evaluation_measures()
+        self.assertEqual(isinstance(measures, list), True)
+        self.assertEqual(all([isinstance(s, str) for s in measures]), True)