Cleaning files after unit test from local (#721)

* Collecting and cleaning unit test dump

* Adding session level fixture with yield to delay deletion of files

* Adding PEP8 ignore F401

* Changelog update + pytest argument fix

* Leaner implementation without additional imports

* Removing TODO

* Making changes suggested

* Editing CI script to check for files after unit testing

* Adding comments to shell script change

Author: Neeratyoy

ci_scripts/test.sh

@@ -1,5 +1,9 @@
 set -e
 
+# check status and branch before running the unit tests
+before="`git status --porcelain -b`"
+before="$before"
+
 run_tests() {
     # Get into a temp directory to run test from the installed scikit learn and
     # check if we  do not leave artifacts
@@ -32,3 +36,11 @@ fi
 if [[ "$SKIP_TESTS" != "true" ]]; then
     run_tests
 fi
+
+# check status and branch after running the unit tests
+# compares with $before to check for remaining files
+after="`git status --porcelain -b`"
+if [[ "$before" != "$after" ]]; then
+    echo "All generated files have not been deleted!"
+    exit 1
+fi

doc/progress.rst

@@ -15,6 +15,7 @@ Changelog
 * DOC #639: More descriptive documention for function to convert array format.
 * ADD #687: Adds a function to retrieve the list of evaluation measures available.
 * ADD #695: A function to retrieve all the data quality measures available.
+* FIX #447: All files created by unit tests are deleted after the completion of all unit tests.
 
 0.9.0
 ~~~~~

openml/testing.py

@@ -17,6 +17,8 @@
 import openml
 from openml.tasks import TaskTypeEnum
 
+import pytest
+
 
 class TestBase(unittest.TestCase):
     """Base class for tests
@@ -44,7 +46,6 @@ def setUp(self, n_levels: int = 1):
             Number of nested directories the test is in. Necessary to resolve the path to the
             ``files`` directory, which is located directly under the ``tests`` directory.
         """
-
         # This cache directory is checked in to git to simulate a populated
         # cache
         self.maxDiff = None
@@ -103,6 +104,48 @@ def tearDown(self):
         openml.config.server = self.production_server
         openml.config.connection_n_retries = self.connection_n_retries
 
+    @pytest.fixture(scope="session", autouse=True)
+    def _cleanup_fixture(self):
+        """Cleans up files generated by Unit tests
+
+        This function is called at the beginning of the invocation of
+        TestBase (defined below), by each of class that inherits TestBase.
+        The 'yield' creates a checkpoint and breaks away to continue running
+        the unit tests of the sub class. When all the tests end, execution
+        resumes from the checkpoint.
+        """
+
+        abspath_this_file = os.path.abspath(inspect.getfile(self.__class__))
+        static_cache_dir = os.path.dirname(abspath_this_file)
+        # Could be a risky while condition, however, going up a directory
+        # n-times will eventually end at main directory
+        while True:
+            if 'openml' in os.listdir(static_cache_dir):
+                break
+            else:
+                static_cache_dir = os.path.join(static_cache_dir, '../')
+        directory = os.path.join(static_cache_dir, 'tests/files/')
+        # directory = "{}/tests/files/".format(static_cache_dir)
+        files = os.walk(directory)
+        old_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                old_file_list.append(os.path.join(root, filename))
+        # context switches to other remaining tests
+        # pauses the code execution here till all tests in the 'session' is over
+        yield
+        # resumes from here after all collected tests are completed
+        files = os.walk(directory)
+        new_file_list = []
+        for root, _, filenames in files:
+            for filename in filenames:
+                new_file_list.append(os.path.join(root, filename))
+        # filtering the files generated during this run
+        new_file_list = list(set(new_file_list) - set(old_file_list))
+        print("Files to delete in local: {}".format(new_file_list))
+        for file in new_file_list:
+            os.remove(file)
+
     def _get_sentinel(self, sentinel=None):
         if sentinel is None:
             # Create a unique prefix for the flow. Necessary because the flow

tests/test_datasets/test_dataset_functions.py

@@ -43,14 +43,14 @@ def tearDown(self):
         super(TestOpenMLDataset, self).tearDown()
 
     def _remove_pickle_files(self):
-        cache_dir = self.static_cache_dir
+        self.lock_path = os.path.join(openml.config.get_cache_directory(), 'locks')
         for did in ['-1', '2']:
             with lockutils.external_lock(
                     name='datasets.functions.get_dataset:%s' % did,
-                    lock_path=os.path.join(openml.config.get_cache_directory(), 'locks'),
+                    lock_path=self.lock_path,
             ):
-                pickle_path = os.path.join(cache_dir, 'datasets', did,
-                                           'dataset.pkl')
+                pickle_path = os.path.join(openml.config.get_cache_directory(), 'datasets',
+                                           did, 'dataset.pkl.py3')
                 try:
                     os.remove(pickle_path)
                 except (OSError, FileNotFoundError):

tests/test_tasks/test_split.py

@@ -19,8 +19,7 @@ def setUp(self):
             self.directory, "..", "files", "org", "openml", "test",
             "tasks", "1882", "datasplits.arff"
         )
-        # TODO Needs to be adapted regarding the python version
-        self.pd_filename = self.arff_filename.replace(".arff", ".pkl")
+        self.pd_filename = self.arff_filename.replace(".arff", ".pkl.py3")
 
     def tearDown(self):
         try:

tests/test_tasks/test_task_functions.py

@@ -12,6 +12,12 @@
 class TestTask(TestBase):
     _multiprocess_can_split_ = True
 
+    def setUp(self):
+        super(TestTask, self).setUp()
+
+    def tearDown(self):
+        super(TestTask, self).tearDown()
+
     def test__get_cached_tasks(self):
         openml.config.cache_directory = self.static_cache_dir
         tasks = openml.tasks.functions._get_cached_tasks()

tests/test_tasks/test_task_methods.py

@@ -7,6 +7,12 @@
 # Common methods between tasks
 class OpenMLTaskMethodsTest(TestBase):
 
+    def setUp(self):
+        super(OpenMLTaskMethodsTest, self).setUp()
+
+    def tearDown(self):
+        super(OpenMLTaskMethodsTest, self).tearDown()
+
     def test_tagging(self):
         task = openml.tasks.get_task(1)
         tag = "testing_tag_{}_{}".format(self.id(), time())