support for local evaluations from url

Author: janvanrijn

openml/_api_calls.py

@@ -117,6 +117,7 @@ def _read_url(url, data=None):
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
     return response.text
 
+
 def _parse_server_exception(response):
     # OpenML has a sopisticated error system
     # where information about failures is provided. try to parse this

openml/runs/run.py

@@ -127,7 +127,9 @@ def get_metric_score(self, sklearn_fn, kwargs={}):
         if self.data_content is not None:
             predictions_arff = self._generate_arff_dict()
         elif 'predictions' in self.output_files:
-            raise ValueError('Not Implemented Yet: Function can currently only be used on locally executed runs (contributor needed!)')
+            predictions_file_url = _file_id_to_url(self.output_files['predictions'], 'predictions.arff')
+            predictions_arff = arff.loads(openml._api_calls._read_url(predictions_file_url))
+            # TODO: make this a stream reader
         else:
             raise ValueError('Run should have been locally executed.')
 

openml/setups/functions.py

@@ -102,7 +102,7 @@ def list_setups(flow=None, tag=None, setup=None, offset=None, size=None):
         api_call += "/offset/%d" % int(offset)
     if size is not None:
         api_call += "/limit/%d" % int(size)
-    if size is not None:
+    if setup is not None:
         api_call += "/setup/%s" % ','.join([str(int(i)) for i in setup])
     if flow is not None:
         api_call += "/flow/%s" % flow