Add debug output (#860)

* Add debug output

* try to please test server

* redirect one more test to the live server

* add commit as requested by Jan

* Removing hard coded retrievals from task example

* Improved use of pandas retrieval

Author: mfeurer

examples/30_extended/tasks_tutorial.py

@@ -196,11 +196,11 @@
     # Error code for 'task already exists'
     if e.code == 614:
         # Lookup task
-        tasks = openml.tasks.list_tasks(data_id=128, output_format='dataframe').to_numpy()
-        tasks = tasks[tasks[:, 4] == "Supervised Classification"]
-        tasks = tasks[tasks[:, 6] == "10-fold Crossvalidation"]
-        tasks = tasks[tasks[:, 19] == "predictive_accuracy"]
-        task_id = tasks[0][0]
+        tasks = openml.tasks.list_tasks(data_id=128, output_format='dataframe')
+        tasks = tasks.query('task_type == "Supervised Classification" '
+                            'and estimation_procedure == "10-fold Crossvalidation" '
+                            'and evaluation_measures == "predictive_accuracy"')
+        task_id = tasks.loc[:, "tid"].values[0]
         print("Task already exists. Task ID is", task_id)
 
 # reverting to prod server

tests/test_evaluations/test_evaluation_functions.py

@@ -149,8 +149,9 @@ def test_evaluation_list_per_fold(self):
             self.assertIsNone(evaluations[run_id].values)
 
     def test_evaluation_list_sort(self):
+        openml.config.server = self.production_server
         size = 10
-        task_id = 115
+        task_id = 6
         # Get all evaluations of the task
         unsorted_eval = openml.evaluations.list_evaluations(
             "predictive_accuracy", offset=0, task=[task_id])

tests/test_runs/test_run_functions.py

@@ -79,7 +79,7 @@ def _wait_for_processed_run(self, run_id, max_waiting_time_seconds):
             if len(run.evaluations) > 0:
                 return
             else:
-                time.sleep(10)
+                time.sleep(3)
         raise RuntimeError('Could not find any evaluations! Please check whether run {} was '
                            'evaluated correctly on the server'.format(run_id))
 
@@ -1120,8 +1120,13 @@ def test_get_run(self):
         )
 
     def _check_run(self, run):
+        # This tests that the API returns seven entries for each run
+        # Check out https://openml.org/api/v1/xml/run/list/flow/1154
+        # They are run_id, task_id, task_type_id, setup_id, flow_id, uploader, upload_time
+        # error_message and run_details exist, too, but are not used so far. We need to update
+        # this check once they are used!
         self.assertIsInstance(run, dict)
-        self.assertEqual(len(run), 7)
+        assert len(run) == 7, str(run)
 
     def test_get_runs_list(self):
         # TODO: comes from live, no such lists on test

tests/test_utils/test_utils.py

@@ -75,14 +75,14 @@ def test_list_all_for_setups(self):
         self.assertEqual(len(setups), required_size)
 
     def test_list_all_for_runs(self):
-        required_size = 48
+        required_size = 21
         runs = openml.runs.list_runs(batch_size=self._batch_size, size=required_size)
 
         # might not be on test server after reset, please rerun test at least once if fails
         self.assertEqual(len(runs), required_size)
 
     def test_list_all_for_evaluations(self):
-        required_size = 57
+        required_size = 22
         # TODO apparently list_evaluations function does not support kwargs
         evaluations = openml.evaluations.list_evaluations(function='predictive_accuracy',
                                                           size=required_size)