Merge pull request #273 from openml/evaluationlist

Evaluationlist

Author: janvanrijn

openml/evaluations/functions.py

@@ -3,10 +3,64 @@
 from .._api_calls import _perform_api_call
 from ..evaluations import OpenMLEvaluation
 
-def list_evaluations(function, task_id):
+def list_evaluations(function, offset=None, size=None, id=None, task=None, setup=None,
+              flow=None, uploader=None, tag=None):
+    """List all run-evaluation pairs matching all of the given filters.
+
+        Perform API call `/evaluation/function{function}/{filters} 
+        
+        Parameters
+        ----------
+        function : str 
+            the evaluation function. e.g., predictive_accuracy
+        offset : int, optional
+            the number of runs to skip, starting from the first
+        size : int, optional
+            the maximum number of runs to show
+
+        id : list, optional
+
+        task : list, optional
+
+        setup: list, optional
+
+        flow : list, optional
+
+        uploader : list, optional
+
+        tag : str, optional
+
+        Returns
+        -------
+        list
+            List of found evaluations.
+        """
+
+    api_call = "evaluation/list/function/%s" %function
+    if offset is not None:
+        api_call += "/offset/%d" % int(offset)
+    if size is not None:
+        api_call += "/limit/%d" % int(size)
+    if id is not None:
+        api_call += "/run/%s" % ','.join([str(int(i)) for i in id])
+    if task is not None:
+        api_call += "/task/%s" % ','.join([str(int(i)) for i in task])
+    if setup is not None:
+        api_call += "/setup/%s" % ','.join([str(int(i)) for i in setup])
+    if flow is not None:
+        api_call += "/flow/%s" % ','.join([str(int(i)) for i in flow])
+    if uploader is not None:
+        api_call += "/uploader/%s" % ','.join([str(int(i)) for i in uploader])
+    if tag is not None:
+        api_call += "/tag/%s" % tag
+
+    return _list_evaluations(api_call)
+
+
+def _list_evaluations(api_call):
     """Helper function to parse API calls which are lists of runs"""
 
-    xml_string = _perform_api_call("evaluation/list/function/%s/task/%d" %(function, task_id))
+    xml_string = _perform_api_call(api_call)
 
     evals_dict = xmltodict.parse(xml_string)
     # Minimalistic check if the XML is useful
@@ -17,7 +71,7 @@ def list_evaluations(function, task_id):
     if isinstance(evals_dict['oml:evaluations']['oml:evaluation'], list):
         evals_list = evals_dict['oml:evaluations']['oml:evaluation']
     elif isinstance(evals_dict['oml:evaluations']['oml:evaluation'], dict):
-        evals_list = [evals_dict['oml:runs']['oml:run']]
+        evals_list = [evals_dict['oml:evaluations']['oml:evaluation']]
     else:
         raise TypeError()
 

tests/test_evaluations/test_evaluation_functions.py

@@ -4,12 +4,68 @@
 
 class TestEvaluationFunctions(TestBase):
 
-    def test_evaluation_list(self):
+    def test_evaluation_list_filter_task(self):
         openml.config.server = self.production_server
 
         task_id = 7312
 
-        res = openml.evaluations.list_evaluations("predictive_accuracy", task_id)
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", task=[task_id])
 
-        self.assertGreater(len(res), 100)
+        self.assertGreater(len(evaluations), 100)
+        for run_id in evaluations.keys():
+            self.assertEquals(evaluations[run_id].task_id, task_id)
 
+
+    def test_evaluation_list_filter_uploader(self):
+        openml.config.server = self.production_server
+
+        uploader_id = 16
+
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", uploader=[uploader_id])
+
+        self.assertGreater(len(evaluations), 100)
+        # for run_id in evaluations.keys():
+        #     self.assertEquals(evaluations[run_id].uploader, uploader_id)
+
+
+    def test_evaluation_list_filter_uploader(self):
+        openml.config.server = self.production_server
+
+        setup_id = 10
+
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", setup=[setup_id])
+
+        self.assertGreater(len(evaluations), 100)
+        for run_id in evaluations.keys():
+            self.assertEquals(evaluations[run_id].setup_id, setup_id)
+
+
+    def test_evaluation_list_filter_flow(self):
+        openml.config.server = self.production_server
+
+        flow_id = 100
+
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", flow=[flow_id])
+
+        self.assertGreater(len(evaluations), 2)
+        for run_id in evaluations.keys():
+            self.assertEquals(evaluations[run_id].flow_id, flow_id)
+
+
+    def test_evaluation_list_filter_run(self):
+        openml.config.server = self.production_server
+
+        run_id = 1
+
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", id=[run_id])
+
+        self.assertEquals(len(evaluations), 1)
+        for run_id in evaluations.keys():
+            self.assertEquals(evaluations[run_id].run_id, run_id)
+
+
+    def test_evaluation_list_limit(self):
+        openml.config.server = self.production_server
+
+        evaluations = openml.evaluations.list_evaluations("predictive_accuracy", size=100, offset=100)
+        self.assertEquals(len(evaluations), 100)