Increase unit test stability (#926)

* Increase unit test stability

by waiting longer for the server to process run traces, and by
querying the server less frequently for new run traces.

* Make test stricter

actually, we only wait for evaluations to ensure that the trace
is processed by the server. Therefore, we can also simply wait
for the trace being available instead of relying on the proxy
indicator of evaluations being available.

* fix stricter test

Author: mfeurer

tests/test_runs/test_run_functions.py

@@ -81,10 +81,19 @@ def _wait_for_processed_run(self, run_id, max_waiting_time_seconds):
         start_time = time.time()
         while time.time() - start_time < max_waiting_time_seconds:
             run = openml.runs.get_run(run_id, ignore_cache=True)
-            if len(run.evaluations) > 0:
-                return
-            else:
-                time.sleep(3)
+
+            try:
+                openml.runs.get_run_trace(run_id)
+            except openml.exceptions.OpenMLServerException:
+                time.sleep(10)
+                continue
+
+            if len(run.evaluations) == 0:
+                time.sleep(10)
+                continue
+
+            return
+
         raise RuntimeError(
             "Could not find any evaluations! Please check whether run {} was "
             "evaluated correctly on the server".format(run_id)
@@ -915,7 +924,7 @@ def test_get_run_trace(self):
             run = run.publish()
             TestBase._mark_entity_for_removal("run", run.run_id)
             TestBase.logger.info("collected from test_run_functions: {}".format(run.run_id))
-            self._wait_for_processed_run(run.run_id, 200)
+            self._wait_for_processed_run(run.run_id, 400)
             run_id = run.run_id
         except openml.exceptions.OpenMLRunsExistError as e:
             # The only error we expect, should fail otherwise.