Fix unit tests (#448)

* FIX fixes and improves unit tests

* ADD additional asserts to hunt down bug

* ADD debug output

* remove print statement

Author: mfeurer

.gitignore

@@ -1,3 +1,4 @@
+*~
 doc/generated
 examples/.ipynb_checkpoints
 # Byte-compiled / optimized / DLL files

openml/runs/functions.py

@@ -231,26 +231,28 @@ def _run_exists(task_id, setup_id):
     Parameters
     ----------
     task_id: int
+
     setup_id: int
 
     Returns
     -------
-        List of run ids iff these already exists on the server, False otherwise
+        Set run ids for runs where flow setup_id was run on task_id. Empty
+        set if it wasn't run yet.
     """
     if setup_id <= 0:
         # openml setups are in range 1-inf
-        return False
+        return set()
 
     try:
         result = list_runs(task=[task_id], setup=[setup_id])
         if len(result) > 0:
             return set(result.keys())
         else:
-            return False
+            return set()
     except OpenMLServerException as exception:
         # error code 512 implies no results. This means the run does not exist yet
         assert(exception.code == 512)
-        return False
+        return set()
 
 
 def _get_seeded_model(model, seed=None):

tests/test_runs/test_run_functions.py

@@ -527,10 +527,13 @@ def test_get_run_trace(self):
             flow = openml.flows.sklearn_to_flow(clf)
             flow_exists = openml.flows.flow_exists(flow.name, flow.external_version)
             self.assertIsInstance(flow_exists, int)
+            self.assertGreater(flow_exists, 0)
             downloaded_flow = openml.flows.get_flow(flow_exists)
             setup_exists = openml.setups.setup_exists(downloaded_flow)
             self.assertIsInstance(setup_exists, int)
+            self.assertGreater(setup_exists, 0)
             run_ids = _run_exists(task.task_id, setup_exists)
+            self.assertGreater(len(run_ids), 0)
             run_id = random.choice(list(run_ids))
 
         # now the actual unit test ...

tests/test_utils/__init__.py

@@ -0,0 +1,18 @@
+from openml.testing import TestBase
+import openml
+
+
+class OpenMLTaskTest(TestBase):
+    _multiprocess_can_split_ = True
+
+    def test_list_all(self):
+        list_datasets = openml.datasets.functions._list_datasets
+        datasets = openml.utils.list_all(list_datasets)
+
+        self.assertGreaterEqual(len(datasets), 100)
+        for did in datasets:
+            self._check_dataset(datasets[did])
+
+        # TODO implement these tests
+        # datasets = openml.utils.list_all(list_datasets, limit=50)
+        # self.assertEqual(len(datasets), 50)