Merge pull request #278 from openml/setuplist

initial commit for setup list

Author: janvanrijn

openml/__init__.py

@@ -22,6 +22,8 @@
 from . import runs
 from . import flows
 from . import setups
+from . import study
+from . import evaluations
 from .runs import OpenMLRun
 from .tasks import OpenMLTask, OpenMLSplit
 from .flows import OpenMLFlow
@@ -66,5 +68,6 @@ def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
 
 
 __all__ = ['OpenMLDataset', 'OpenMLDataFeature', 'OpenMLRun',
-           'OpenMLSplit', 'datasets', 'OpenMLTask', 'OpenMLFlow',
+           'OpenMLSplit', 'OpenMLEvaluation', 'OpenMLSetup',
+           'OpenMLTask', 'OpenMLFlow', 'datasets', 'evaluations',
            'config', 'runs', 'flows', 'tasks', 'setups']

openml/setups/__init__.py

@@ -1,3 +1,4 @@
-from .functions import get_setup, setup_exists, initialize_model
+from .setup import OpenMLSetup
+from .functions import get_setup, list_setups, setup_exists, initialize_model
 
-__all__ = ['get_setup', 'setup_exists', 'initialize_model']
+__all__ = ['get_setup', 'list_setups', 'setup_exists', 'initialize_model']

openml/setups/functions.py

@@ -4,7 +4,7 @@
 import xmltodict
 
 from .setup import OpenMLSetup, OpenMLParameter
-from openml.flows import sklearn_to_flow, flow_exists
+from openml.flows import flow_exists
 
 
 def setup_exists(flow, model=None):
@@ -74,6 +74,80 @@ def get_setup(setup_id):
     return _create_setup_from_xml(result_dict)
 
 
+def list_setups(flow=None, tag=None, setup=None, offset=None, size=None):
+    """List all setups matching all of the given filters.
+
+        Perform API call `/setup/list/{filters}
+
+        Parameters
+        ----------
+        flow : int, optional
+
+        tag : str, optional
+
+        setup : list(int), optional
+
+        offset : int, optional
+
+        size : int, optional
+
+        Returns
+        -------
+        list
+            List of found setups.
+        """
+
+    api_call = "setup/list"
+    if offset is not None:
+        api_call += "/offset/%d" % int(offset)
+    if size is not None:
+        api_call += "/limit/%d" % int(size)
+    if setup is not None:
+        api_call += "/setup/%s" % ','.join([str(int(i)) for i in setup])
+    if flow is not None:
+        api_call += "/flow/%s" % flow
+    if tag is not None:
+        api_call += "/tag/%s" % tag
+
+    return _list_setups(api_call)
+
+
+def _list_setups(api_call):
+    """Helper function to parse API calls which are lists of setups"""
+
+    xml_string = openml._api_calls._perform_api_call(api_call)
+
+    setups_dict = xmltodict.parse(xml_string)
+    # Minimalistic check if the XML is useful
+    if 'oml:setups' not in setups_dict:
+        raise ValueError('Error in return XML, does not contain "oml:setups": %s'
+                         % str(setups_dict))
+    elif '@xmlns:oml' not in setups_dict['oml:setups']:
+        raise ValueError('Error in return XML, does not contain '
+                         '"oml:setups"/@xmlns:oml: %s'
+                         % str(setups_dict))
+    elif setups_dict['oml:setups']['@xmlns:oml'] != 'http://openml.org/openml':
+        raise ValueError('Error in return XML, value of  '
+                         '"oml:seyups"/@xmlns:oml is not '
+                         '"http://openml.org/openml": %s'
+                         % str(setups_dict))
+
+    if isinstance(setups_dict['oml:setups']['oml:setup'], list):
+        setups_list = setups_dict['oml:setups']['oml:setup']
+    elif isinstance(setups_dict['oml:setups']['oml:setup'], dict):
+        setups_list = [setups_dict['oml:setups']['oml:setup']]
+    else:
+        raise TypeError()
+
+    setups = dict()
+    for setup_ in setups_list:
+        # making it a dict to give it the right format
+        current = _create_setup_from_xml({'oml:setup_parameters': setup_})
+        setups[current.setup_id] = current
+
+    return setups
+
+
 def initialize_model(setup_id):
     '''
     Initialized a model based on a setup_id (i.e., using the exact
@@ -147,6 +221,7 @@ def _create_setup_from_xml(result_dict):
     '''
      Turns an API xml result into a OpenMLSetup object
     '''
+    setup_id = int(result_dict['oml:setup_parameters']['oml:setup_id'])
     flow_id = int(result_dict['oml:setup_parameters']['oml:flow_id'])
     parameters = {}
     if 'oml:parameter' not in result_dict['oml:setup_parameters']:
@@ -164,7 +239,7 @@ def _create_setup_from_xml(result_dict):
         else:
             raise ValueError('Expected None, list or dict, received someting else: %s' %str(type(xml_parameters)))
 
-    return OpenMLSetup(flow_id, parameters)
+    return OpenMLSetup(setup_id, flow_id, parameters)
 
 def _create_setup_parameter_from_xml(result_dict):
     return OpenMLParameter(int(result_dict['oml:id']),
@@ -173,4 +248,4 @@ def _create_setup_parameter_from_xml(result_dict):
                            result_dict['oml:parameter_name'],
                            result_dict['oml:data_type'],
                            result_dict['oml:default_value'],
-                           result_dict['oml:value'])
+                           result_dict['oml:value'])

openml/setups/setup.py

@@ -4,13 +4,24 @@ class OpenMLSetup(object):
 
        Parameters
        ----------
+       setup_id : int
+            The OpenML setup id
        flow_id : int
             The flow that it is build upon
         parameters : dict
             The setting of the parameters
            """
 
-    def __init__(self, flow_id, parameters):
+    def __init__(self, setup_id, flow_id, parameters):
+        if not isinstance(setup_id, int):
+            raise ValueError('setup id should be int')
+        if not isinstance(flow_id, int):
+            raise ValueError('flow id should be int')
+        if parameters is not None:
+            if not isinstance(parameters, dict):
+                raise ValueError('parameters should be dict')
+
+        self.setup_id = setup_id
         self.flow_id = flow_id
         self.parameters = parameters
 

openml/study/functions.py

@@ -27,9 +27,11 @@ def get_study(study_id):
     creator = result_dict['oml:creator']
     tags = []
     for tag in _multitag_to_list(result_dict, 'oml:tag'):
-        tags.append({'name': tag['oml:name'],
-                     'window_start': tag['oml:window_start'],
-                     'write_access': tag['oml:write_access']})
+        current_tag = {'name': tag['oml:name'],
+                       'write_access': tag['oml:write_access']}
+        if 'oml:window_start' in tag:
+            current_tag['window_start'] = tag['oml:window_start']
+        tags.append(current_tag)
 
     datasets = None
     tasks = None

tests/test_flows/test_flow.py

@@ -239,17 +239,26 @@ def get_sentinel():
     def test_existing_flow_exists(self):
         # create a flow
         nb = sklearn.naive_bayes.GaussianNB()
-        flow = openml.flows.sklearn_to_flow(nb)
-        flow, _ = self._add_sentinel_to_flow_name(flow, None)
-        #publish the flow
-        flow = flow.publish()
-        #redownload the flow
-        flow = openml.flows.get_flow(flow.flow_id)
-
-        # check if flow exists can find it
-        flow = openml.flows.get_flow(flow.flow_id)
-        downloaded_flow_id = openml.flows.flow_exists(flow.name, flow.external_version)
-        self.assertEquals(downloaded_flow_id, flow.flow_id)
+
+        steps = [('imputation', sklearn.preprocessing.Imputer(strategy='median')),
+                 ('hotencoding', sklearn.preprocessing.OneHotEncoder(sparse=False,
+                                                                     handle_unknown='ignore')),
+                 ('variencethreshold', sklearn.feature_selection.VarianceThreshold()),
+                 ('classifier', sklearn.tree.DecisionTreeClassifier())]
+        complicated = sklearn.pipeline.Pipeline(steps=steps)
+
+        for classifier in [nb, complicated]:
+            flow = openml.flows.sklearn_to_flow(classifier)
+            flow, _ = self._add_sentinel_to_flow_name(flow, None)
+            #publish the flow
+            flow = flow.publish()
+            #redownload the flow
+            flow = openml.flows.get_flow(flow.flow_id)
+
+            # check if flow exists can find it
+            flow = openml.flows.get_flow(flow.flow_id)
+            downloaded_flow_id = openml.flows.flow_exists(flow.name, flow.external_version)
+            self.assertEquals(downloaded_flow_id, flow.flow_id)
 
     def test_sklearn_to_upload_to_flow(self):
         iris = sklearn.datasets.load_iris()

tests/test_flows/test_flow_functions.py

@@ -25,7 +25,7 @@ def test_list_flows(self):
         # data from the internet...
         flows = openml.flows.list_flows()
         # 3000 as the number of flows on openml.org
-        self.assertGreaterEqual(len(flows), 3000)
+        self.assertGreaterEqual(len(flows), 1500)
         for fid in flows:
             self._check_flow(flows[fid])
 

tests/test_setups/test_setup_functions.py

@@ -47,7 +47,6 @@ def get_params(self, deep=True):
         return {}
 
 
-
 class TestRun(TestBase):
 
     def test_nonexisting_setup_exists(self):
@@ -117,3 +116,28 @@ def test_get_setup(self):
                 self.assertIsNone(current.parameters)
             else:
                 self.assertEquals(len(current.parameters), num_params[idx])
+
+    def test_setup_list_filter_flow(self):
+        openml.config.server = self.production_server
+
+        flow_id = 5873
+
+        setups = openml.setups.list_setups(flow=flow_id)
+
+        self.assertGreater(len(setups), 0) # TODO: please adjust 0
+        for setup_id in setups.keys():
+            self.assertEquals(setups[setup_id].flow_id, flow_id)
+
+    def test_setuplist_offset(self):
+        # TODO: remove after pull on live for better testing
+        # openml.config.server = self.production_server
+
+        size = 100
+        setups = openml.setups.list_setups(offset=0, size=size)
+        self.assertEquals(len(setups), size)
+        setups2 = openml.setups.list_setups(offset=size, size=size)
+        self.assertEquals(len(setups), size)
+
+        all = set(setups.keys()).union(setups2.keys())
+
+        self.assertEqual(len(all), size * 2)