Merge pull request #715 from sahithyaravi1493/fix962

add sort to list_evaluations

Author: PGijsbers

doc/progress.rst

@@ -8,6 +8,7 @@ Changelog
 
 0.10.0
 ~~~~~~
+* ADD #715: `list_evaluations` now has an option to sort evaluations by score (value).
 * FIX #589: Fixing a bug that did not successfully upload the columns to ignore when creating and publishing a dataset.
 * DOC #639: More descriptive documention for function to convert array format.
 * ADD #687: Adds a function to retrieve the list of evaluation measures available.

openml/evaluations/functions.py

@@ -2,6 +2,7 @@
 import xmltodict
 import pandas as pd
 from typing import Union, List, Optional, Dict
+import collections
 
 import openml.utils
 import openml._api_calls
@@ -19,6 +20,7 @@ def list_evaluations(
     uploader: Optional[List] = None,
     tag: Optional[str] = None,
     per_fold: Optional[bool] = None,
+    sort_order: Optional[str] = None,
     output_format: str = 'object'
 ) -> Union[Dict, pd.DataFrame]:
     """
@@ -48,6 +50,9 @@ def list_evaluations(
 
     per_fold : bool, optional
 
+    sort_order : str, optional
+       order of sorting evaluations, ascending ("asc") or descending ("desc")
+
     output_format: str, optional (default='object')
         The parameter decides the format of the output.
         - If 'object' the output is a dict of OpenMLEvaluation objects
@@ -77,6 +82,7 @@ def list_evaluations(
                                   flow=flow,
                                   uploader=uploader,
                                   tag=tag,
+                                  sort_order=sort_order,
                                   per_fold=per_fold_str)
 
 
@@ -87,6 +93,7 @@ def _list_evaluations(
     setup: Optional[List] = None,
     flow: Optional[List] = None,
     uploader: Optional[List] = None,
+    sort_order: Optional[str] = None,
     output_format: str = 'object',
     **kwargs
 ) -> Union[Dict, pd.DataFrame]:
@@ -114,6 +121,9 @@ def _list_evaluations(
     kwargs: dict, optional
         Legal filter operators: tag, limit, offset.
 
+    sort_order : str, optional
+        order of sorting evaluations, ascending ("asc") or descending ("desc")
+
     output_format: str, optional (default='dict')
         The parameter decides the format of the output.
         - If 'dict' the output is a dict of dict
@@ -141,6 +151,8 @@ def _list_evaluations(
         api_call += "/flow/%s" % ','.join([str(int(i)) for i in flow])
     if uploader is not None:
         api_call += "/uploader/%s" % ','.join([str(int(i)) for i in uploader])
+    if sort_order is not None:
+        api_call += "/sort_order/%s" % sort_order
 
     return __list_evaluations(api_call, output_format=output_format)
 
@@ -157,7 +169,7 @@ def __list_evaluations(api_call, output_format='object'):
     assert type(evals_dict['oml:evaluations']['oml:evaluation']) == list, \
         type(evals_dict['oml:evaluations'])
 
-    evals = dict()
+    evals = collections.OrderedDict()
     for eval_ in evals_dict['oml:evaluations']['oml:evaluation']:
         run_id = int(eval_['oml:run_id'])
         value = None

openml/utils.py

@@ -5,6 +5,7 @@
 import warnings
 import pandas as pd
 from functools import wraps
+import collections
 
 import openml._api_calls
 import openml.exceptions
@@ -182,7 +183,7 @@ def _list_all(listing_call, output_format='dict', *args, **filters):
     active_filters = {key: value for key, value in filters.items()
                       if value is not None}
     page = 0
-    result = {}
+    result = collections.OrderedDict()
     if output_format == 'dataframe':
         result = pd.DataFrame()
 

tests/test_evaluations/test_evaluation_functions.py

@@ -117,6 +117,27 @@ def test_evaluation_list_per_fold(self):
             self.assertIsNotNone(evaluations[run_id].value)
             self.assertIsNone(evaluations[run_id].values)
 
+    def test_evaluation_list_sort(self):
+        size = 10
+        task_id = 115
+        # Get all evaluations of the task
+        unsorted_eval = openml.evaluations.list_evaluations(
+            "predictive_accuracy", offset=0, task=[task_id])
+        # Get top 10 evaluations of the same task
+        sorted_eval = openml.evaluations.list_evaluations(
+            "predictive_accuracy", size=size, offset=0, task=[task_id], sort_order="desc")
+        self.assertEqual(len(sorted_eval), size)
+        self.assertGreater(len(unsorted_eval), 0)
+        sorted_output = [evaluation.value for evaluation in sorted_eval.values()]
+        unsorted_output = [evaluation.value for evaluation in unsorted_eval.values()]
+
+        # Check if output from sort is sorted in the right order
+        self.assertTrue(sorted(sorted_output, reverse=True) == sorted_output)
+
+        # Compare manual sorting against sorted output
+        test_output = sorted(unsorted_output, reverse=True)
+        self.assertTrue(test_output[:size] == sorted_output)
+
     def test_list_evaluation_measures(self):
         measures = openml.evaluations.list_evaluation_measures()
         self.assertEqual(isinstance(measures, list), True)