Subclass all test classes from openml test helper (#609)

* Subclass all test classes from openml test helper

* FIX inheritance issues

* TST add sentinel to dataset upload

* TEST redirect a few tests to the live server again

* MAINT fix pep8

* Trying simple solution

Author: mfeurer

openml/runs/functions.py

@@ -127,6 +127,11 @@ def run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None,
             raise ValueError('flow.flow_id is not None, but the flow does not'
                              'exist on the server according to flow_exists')
         _publish_flow_if_necessary(flow)
+        # if the flow was published successfully
+        # and has an id
+        if flow.flow_id is not None:
+            flow_id = flow.flow_id
+
 
     data_content, trace, fold_evaluations, sample_evaluations = res
     if not isinstance(flow.flow_id, int):

openml/testing.py

@@ -77,16 +77,20 @@ def tearDown(self):
                 raise
         openml.config.server = self.production_server
 
-    def _add_sentinel_to_flow_name(self, flow, sentinel=None):
+    def _get_sentinel(self, sentinel=None):
         if sentinel is None:
             # Create a unique prefix for the flow. Necessary because the flow is
             # identified by its name and external version online. Having a unique
             #  name allows us to publish the same flow in each test run
             md5 = hashlib.md5()
             md5.update(str(time.time()).encode('utf-8'))
+            md5.update(str(os.getpid()).encode('utf-8'))
             sentinel = md5.hexdigest()[:10]
             sentinel = 'TEST%s' % sentinel
+        return sentinel
 
+    def _add_sentinel_to_flow_name(self, flow, sentinel=None):
+        sentinel = self._get_sentinel(sentinel=sentinel)
         flows_to_visit = list()
         flows_to_visit.append(flow)
         while len(flows_to_visit) > 0:

tests/test_datasets/test_dataset_functions.py

@@ -367,7 +367,7 @@ def test__retrieve_class_labels(self):
     def test_upload_dataset_with_url(self):
 
         dataset = OpenMLDataset(
-            "UploadTestWithURL",
+            "%s-UploadTestWithURL" % self._get_sentinel(),
             "test",
             data_format="arff",
             version=1,
@@ -378,7 +378,8 @@ def test_upload_dataset_with_url(self):
 
     def test_data_status(self):
         dataset = OpenMLDataset(
-            "UploadTestWithURL", "test", "ARFF",
+            "%s-UploadTestWithURL" % self._get_sentinel(),
+            "test", "ARFF",
             version=1,
             url="https://www.openml.org/data/download/61/dataset_61_iris.arff")
         dataset.publish()
@@ -476,7 +477,7 @@ def test_create_dataset_numpy(self):
                       for i in range(data.shape[1])]
 
         dataset = create_dataset(
-            name='NumPy_testing_dataset',
+            name='%s-NumPy_testing_dataset' % self._get_sentinel(),
             description='Synthetic dataset created from a NumPy array',
             creator='OpenML tester',
             contributor=None,
@@ -536,7 +537,7 @@ def test_create_dataset_list(self):
         ]
 
         dataset = create_dataset(
-            name="ModifiedWeather",
+            name="%s-ModifiedWeather" % self._get_sentinel(),
             description=(
                 'Testing dataset upload when the data is a list of lists'
             ),
@@ -583,7 +584,7 @@ def test_create_dataset_sparse(self):
         ]
 
         xor_dataset = create_dataset(
-            name="XOR",
+            name="%s-XOR" % self._get_sentinel(),
             description='Dataset representing the XOR operation',
             creator=None,
             contributor=None,
@@ -620,7 +621,7 @@ def test_create_dataset_sparse(self):
         ]
 
         xor_dataset = create_dataset(
-            name="XOR",
+            name="%s-XOR" % self._get_sentinel(),
             description='Dataset representing the XOR operation',
             creator=None,
             contributor=None,
@@ -732,7 +733,7 @@ def test_create_dataset_pandas(self):
         df['windy'] = df['windy'].astype('bool')
         df['play'] = df['play'].astype('category')
         # meta-information
-        name = 'Pandas_testing_dataset'
+        name = '%s-pandas_testing_dataset' % self._get_sentinel()
         description = 'Synthetic dataset created from a Pandas DataFrame'
         creator = 'OpenML tester'
         collection_date = '01-01-2018'
@@ -842,7 +843,7 @@ def test_create_dataset_pandas(self):
 
     def test_create_dataset_row_id_attribute_error(self):
         # meta-information
-        name = 'Pandas_testing_dataset'
+        name = '%s-pandas_testing_dataset' % self._get_sentinel()
         description = 'Synthetic dataset created from a Pandas DataFrame'
         creator = 'OpenML tester'
         collection_date = '01-01-2018'
@@ -884,7 +885,7 @@ def test_create_dataset_row_id_attribute_error(self):
 
     def test_create_dataset_row_id_attribute_inference(self):
         # meta-information
-        name = 'Pandas_testing_dataset'
+        name = '%s-pandas_testing_dataset' % self._get_sentinel()
         description = 'Synthetic dataset created from a Pandas DataFrame'
         creator = 'OpenML tester'
         collection_date = '01-01-2018'

tests/test_flows/test_flow_functions.py

@@ -5,9 +5,10 @@
 import six
 
 import openml
+from openml.testing import TestBase
 
 
-class TestFlowFunctions(unittest.TestCase):
+class TestFlowFunctions(TestBase):
     _multiprocess_can_split_ = True
 
     def _check_flow(self, flow):
@@ -23,6 +24,7 @@ def _check_flow(self, flow):
                         flow['external_version'] is None)
 
     def test_list_flows(self):
+        openml.config.server = self.production_server
         # We can only perform a smoke test here because we test on dynamic
         # data from the internet...
         flows = openml.flows.list_flows()
@@ -32,19 +34,22 @@ def test_list_flows(self):
             self._check_flow(flows[fid])
 
     def test_list_flows_empty(self):
+        openml.config.server = self.production_server
         flows = openml.flows.list_flows(tag='NoOneEverUsesThisTag123')
         if len(flows) > 0:
             raise ValueError('UnitTest Outdated, got somehow results (please adapt)')
 
         self.assertIsInstance(flows, dict)
 
     def test_list_flows_by_tag(self):
+        openml.config.server = self.production_server
         flows = openml.flows.list_flows(tag='weka')
         self.assertGreaterEqual(len(flows), 5)
         for did in flows:
             self._check_flow(flows[did])
 
     def test_list_flows_paginate(self):
+        openml.config.server = self.production_server
         size = 10
         max = 100
         for i in range(0, max, size):

tests/test_flows/test_sklearn.py

@@ -33,6 +33,7 @@
     from sklearn.impute import SimpleImputer as Imputer
 
 import openml
+from openml.testing import TestBase
 from openml.flows import OpenMLFlow, sklearn_to_flow, flow_to_sklearn
 from openml.flows.functions import assert_flows_equal
 from openml.flows.sklearn_converter import _format_external_version, \
@@ -56,11 +57,12 @@ def fit(self, X, y):
         pass
 
 
-class TestSklearn(unittest.TestCase):
+class TestSklearn(TestBase):
     # Splitting not helpful, these test's don't rely on the server and take less
     # than 1 seconds
 
     def setUp(self):
+        super(TestSklearn, self).setUp()
         iris = sklearn.datasets.load_iris()
         self.X = iris.data
         self.y = iris.target

tests/test_runs/test_trace.py

@@ -1,9 +1,8 @@
-import unittest
-
 from openml.runs import OpenMLRunTrace, OpenMLTraceIteration
+from openml.testing import TestBase
 
 
-class TestTrace(unittest.TestCase):
+class TestTrace(TestBase):
     def test_get_selected_iteration(self):
         trace_iterations = {}
         for i in range(5):

tests/test_tasks/test_split.py

@@ -5,9 +5,10 @@
 import numpy as np
 
 from openml import OpenMLSplit
+from openml.testing import TestBase
 
 
-class OpenMLSplitTest(unittest.TestCase):
+class OpenMLSplitTest(TestBase):
     # Splitting not helpful, these test's don't rely on the server and take less
     # than 5 seconds + rebuilding the test would potentially be costly