Add test for checking the setup time

- run performance tests only if configured per env var
- times are taken from current CI tests with some margin
- safeguard against performance degradation in setup

Author: mrbean-bremen

.github/workflows/pythonpackage.yml

@@ -81,8 +81,8 @@ jobs:
       shell: bash
     - name: Run performance tests
       run: |
-        python -m pyfakefs.tests.performance_test SetupPerformanceTest
-        python -m pyfakefs.tests.performance_test SetupNoCachePerformanceTest
+        export TEST_PERFORMANCE=1
+        python -m pyfakefs.tests.performance_test
       shell: bash
 
   dockertests:

pyfakefs/tests/performance_test.py

@@ -10,31 +10,63 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Shall provide tests to check performance overhead of pyfakefs."""
-
+import os
+import time
 import unittest
 
 from pyfakefs.fake_filesystem_unittest import TestCase
+from pyfakefs.helpers import IS_PYPY
+
+if os.environ.get('TEST_PERFORMANCE'):
+
+    class SetupPerformanceTest(TestCase):
+        @classmethod
+        def setUpClass(cls) -> None:
+            cls.start_time = time.time()
+
+        @classmethod
+        def tearDownClass(cls) -> None:
+            cls.elapsed_time = time.time() - cls.start_time
+            print('Elapsed time per test for cached setup: {:.3f} ms'.format(
+                cls.elapsed_time * 10))
+
+        def setUp(self) -> None:
+            self.setUpPyfakefs()
 
+    class SetupNoCachePerformanceTest(TestCase):
+        @classmethod
+        def setUpClass(cls) -> None:
+            cls.start_time = time.time()
 
-class SetupPerformanceTest(TestCase):
-    def setUp(self) -> None:
-        self.setUpPyfakefs()
+        @classmethod
+        def tearDownClass(cls) -> None:
+            cls.elapsed_time = time.time() - cls.start_time
+            print('Elapsed time per test for uncached setup: {:.3f} ms'.format(
+                cls.elapsed_time * 10))
 
+        def setUp(self) -> None:
+            self.setUpPyfakefs(use_cache=False)
 
-class SetupNoCachePerformanceTest(TestCase):
-    def setUp(self) -> None:
-        self.setUpPyfakefs(use_cache=False)
+    @unittest.skipIf(IS_PYPY, 'PyPy times are not comparable')
+    class TimePerformanceTest(TestCase):
+        """Make sure performance degradation in setup is noticed.
+        The numbers are related to the CI builds and may fail in local builds.
+        """
 
+        def test_cached_time(self):
+            self.assertLess(SetupPerformanceTest.elapsed_time, 0.4)
 
-def test_setup(self):
-    pass
+        def test_uncached_time(self):
+            self.assertLess(SetupNoCachePerformanceTest.elapsed_time, 6)
 
+    def test_setup(self):
+        pass
 
-for n in range(100):
-    test_name = "test_" + str(n)
-    setattr(SetupPerformanceTest, test_name, test_setup)
-    test_name = "test_nocache" + str(n)
-    setattr(SetupNoCachePerformanceTest, test_name, test_setup)
+    for n in range(100):
+        test_name = "test_" + str(n)
+        setattr(SetupPerformanceTest, test_name, test_setup)
+        test_name = "test_nocache" + str(n)
+        setattr(SetupNoCachePerformanceTest, test_name, test_setup)
 
-if __name__ == "__main__":
-    unittest.main()
+    if __name__ == "__main__":
+        unittest.main()