Merge pull request #13 from fedora-python/python_upstream

Changes from Python upstream

Author: frenzymadness

README.md

@@ -84,8 +84,6 @@ ZeroDivisionError: division by zero
 
 * ✓ Test it with Python packages in COPR
 
-## ToDo
-
 * Push it to Fedora rawhide
 
   * ✓ %py_byte_compile RPM macro uses `compileall2` (done in [python-rpm-macros](https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/25))
@@ -96,17 +94,18 @@ ZeroDivisionError: division by zero
 * ✓ Prepare patches for upstream CPython
     * [Pull request](https://github.com/python/cpython/pull/16012) merged and will be part of Python 3.9
 
+* ✓ Changes from upstream CPython backported back
+
 ## Testing
 
 You can test it locally with tox or unittest directly:
 
 ```shell
-$ python3 -m unittest test_compileall_original.py
-............sss.......................sss.....................ss.................
-----------------------------------------------------------------------
-Ran 81 tests in 2.137s
+$ python3 -m unittest test_compileall2.py
+..............sss....ss.......................sss....ss.....................ss.............................----------------------------------------------------------------------
+Ran 107 tests in 3.714s
 
-OK (skipped=8)
+OK (skipped=12)
 ```
 
 but running in a Docker container might be better because the superuser has privileges to write to `sys.path` which lowers the number of skipped tests.
@@ -125,23 +124,26 @@ and run tests in it:
 
 ```shell
 $ docker run --rm -it -e TOXENV=py37 -v $PWD:/src:Z -w /src  compileall2
-py37 create: /src/.tox/py37
-py37 installdeps: pytest
-py37 installed: atomicwrites==1.3.0,attrs==19.1.0,more-itertools==6.0.0,pluggy==0.9.0,py==1.8.0,pytest==4.3.1,six==1.12.0
-py37 run-test-pre: PYTHONHASHSEED='519909491'
-py37 runtests: commands[0] | pytest -v -s
-========================================== test session starts ==========================================
-platform linux -- Python 3.7.2, pytest-4.3.1, py-1.8.0, pluggy-0.9.0 -- /src/.tox/py37/bin/python
+py37 installed: atomicwrites==1.3.0,attrs==19.3.0,compileall2==0.5.0,coverage==4.5.4,importlib-metadata==0.23,more-itertools==7.2.0,packaging==19.2,pluggy==0.13.0,py==1.8.0,pyparsing==2.4.5,pytest==5.2.3,six==1.13.0,wcwidth==0.1.7,zipp==0.6.0
+py37 run-test-pre: PYTHONHASHSEED='1615314833'
+py37 runtests: commands[0] | coverage run --append -m py.test
+==================================== test session starts =====================================
+platform linux -- Python 3.7.5, pytest-5.2.3, py-1.8.0, pluggy-0.13.0
 cachedir: .tox/py37/.pytest_cache
-rootdir: /src, inifile:
-collected 81 items
-
-test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_dir_pathlike PASSED
-test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_file_pathlike PASSED
-test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_file_pathlike_ddir PASSED
-... etc ...
-================================= 79 passed, 2 skipped in 5.15 seconds ==================================
-________________________________________________ summary ________________________________________________
+rootdir: /src
+collected 107 items
+test_compileall2.py ............ss..................................................ss [ 61%]
+..............................ss.........                                              [100%]
+
+=============================== 101 passed, 6 skipped in 7.40s ===============================
+py37 runtests: commands[1] | coverage report -i '--omit=.tox/*'
+Name                  Stmts   Miss  Cover
+-----------------------------------------
+compileall2.py          232     48    79%
+test_compileall2.py     621      8    99%
+-----------------------------------------
+TOTAL                   853     56    93%
+__________________________________________ summary ___________________________________________
   py37: commands succeeded
   congratulations :)
 ```

compileall2.py

@@ -43,8 +43,6 @@
     pyc_header_lenght = 8
     pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER)
 
-RECURSION_LIMIT = sys.getrecursionlimit()
-
 __all__ = ["compile_dir","compile_file","compile_path"]
 
 def optimization_kwarg(opt):
@@ -60,7 +58,7 @@ def optimization_kwarg(opt):
         else:
             return dict()
 
-def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
+def _walk_dir(dir, maxlevels, quiet=0):
     if PY36 and quiet < 2 and isinstance(dir, os.PathLike):
         dir = os.fspath(dir)
     else:
@@ -85,7 +83,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
             yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
                                  quiet=quiet)
 
-def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
+def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
                 invalidation_mode=None, stripdir=None,
                 prependdir=None, limit_sl_dest=None):
@@ -123,6 +121,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
                 from concurrent.futures import ProcessPoolExecutor
             except ImportError:
                 workers = 1
+    if maxlevels is None:
+        maxlevels = sys.getrecursionlimit()
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
@@ -173,6 +173,11 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     limit_sl_dest: ignore symlinks if they are pointing outside of
                    the defined path.
     """
+
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+
     success = True
     if PY36 and quiet < 2 and isinstance(fullname, os.PathLike):
         fullname = os.fspath(fullname)
@@ -327,7 +332,7 @@ def main():
     parser = argparse.ArgumentParser(
         description='Utilities to support installing Python libraries.')
     parser.add_argument('-l', action='store_const', const=0,
-                        default=RECURSION_LIMIT, dest='maxlevels',
+                        default=None, dest='maxlevels',
                         help="don't recurse into subdirectories")
     parser.add_argument('-r', type=int, dest='recursion',
                         help=('control the maximum recursion level. '
@@ -349,16 +354,16 @@ def main():
                         default=None,
                         help=('part of path to left-strip from path '
                               'to source file - for example buildroot. '
-                              'if `-d` and `-s` options are specified, '
-                              'then `-d` takes precedence.'))
+                              '`-d` and `-s` options cannot be '
+                              'specified together.'))
     parser.add_argument('-p', metavar='PREPENDDIR',  dest='prependdir',
                         default=None,
                         help=('path to add as prefix to path '
                               'to source file - for example / to make '
                               'it absolute when some part is removed '
-                              'by `-s` option'
-                              'if `-d` and `-a` options are specified, '
-                              'then `-d` takes precedence.'))
+                              'by `-s` option. '
+                              '`-d` and `-p` options cannot be '
+                              'specified together.'))
     parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
                         help=('skip files matching the regular expression; '
                               'the regexp is searched for in the full path '
@@ -408,6 +413,11 @@ def main():
     if args.opt_levels is None:
         args.opt_levels = [-1]
 
+    if args.ddir is not None and (
+        args.stripdir is not None or args.prependdir is not None
+    ):
+        parser.error("-d cannot be used in combination with -s or -p")
+
     # if flist is provided then load it
     if args.flist:
         try:

test_compileall2.py

@@ -103,16 +103,6 @@ def setUp(self):
         os.mkdir(self.subdirectory)
         self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
         shutil.copyfile(self.source_path, self.source_path3)
-        many_directories = [str(number) for number in range(1, 100)]
-        self.long_path = os.path.join(self.directory,
-                                      "long",
-                                      *many_directories)
-        os.makedirs(self.long_path)
-        self.source_path_long = os.path.join(self.long_path, '_test4.py')
-        shutil.copyfile(self.source_path, self.source_path_long)
-        self.bc_path_long = importlib.util.cache_from_source(
-            self.source_path_long
-        )
 
     def tearDown(self):
         shutil.rmtree(self.directory)
@@ -266,14 +256,22 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(compile_file_mock.called)
 
-    def text_compile_dir_maxlevels(self):
-        # Test the actual impact of maxlevels attr
-        compileall.compile_dir(os.path.join(self.directory, "long"),
-                               maxlevels=10, quiet=True)
-        self.assertFalse(os.path.isfile(self.bc_path_long))
-        compileall.compile_dir(os.path.join(self.directory, "long"),
-                               maxlevels=110, quiet=True)
-        self.assertTrue(os.path.isfile(self.bc_path_long))
+    def test_compile_dir_maxlevels(self):
+        # Test the actual impact of maxlevels parameter
+        depth = 3
+        path = self.directory
+        for i in range(1, depth + 1):
+            path = os.path.join(path, "dir_{}".format(i))
+            source = os.path.join(path, 'script.py')
+            os.mkdir(path)
+            shutil.copyfile(self.source_path, source)
+        pyc_filename = importlib.util.cache_from_source(source)
+
+        compileall.compile_dir(self.directory, quiet=True, maxlevels=depth - 1)
+        self.assertFalse(os.path.isfile(pyc_filename))
+
+        compileall.compile_dir(self.directory, quiet=True, maxlevels=depth)
+        self.assertTrue(os.path.isfile(pyc_filename))
 
     def test_strip_only(self):
         fullpath = ["test", "build", "real", "path"]
@@ -330,6 +328,15 @@ def test_strip_and_prepend(self):
             str(err, encoding=sys.getdefaultencoding())
         )
 
+    def test_strip_prepend_and_ddir(self):
+        fullpath = ["test", "build", "real", "path", "ddir"]
+        path = os.path.join(self.directory, *fullpath)
+        os.makedirs(path)
+        script_helper.make_script(path, "test", "1 / 0")
+        with self.assertRaises(ValueError):
+            compileall.compile_dir(path, quiet=True, ddir="/bar",
+                                   stripdir="/foo", prependdir="/bar")
+
     def test_multiple_optimization_levels(self):
         script = script_helper.make_script(self.directory,
                                            "test_optimization",
@@ -341,7 +348,7 @@ def test_multiple_optimization_levels(self):
         test_combinations = [[0, 1], [1, 2], [0, 2], [0, 1, 2]]
 
         for opt_combination in test_combinations:
-            compileall.compile_file(script,
+            compileall.compile_file(script, quiet=True,
                                     optimize=opt_combination)
             for opt_level in opt_combination:
                 self.assertTrue(os.path.isfile(bc[opt_level]))
@@ -372,7 +379,7 @@ def test_ignore_symlink_destination(self):
         allowed_bc = importlib.util.cache_from_source(allowed_symlink)
         prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)
 
-        compileall.compile_dir(symlinks_path, quiet=False, limit_sl_dest=allowed_path)
+        compileall.compile_dir(symlinks_path, quiet=True, limit_sl_dest=allowed_path)
 
         self.assertTrue(os.path.isfile(allowed_bc))
         self.assertFalse(os.path.isfile(prohibited_bc))
@@ -642,21 +649,19 @@ def test_recursion_limit(self):
         self.assertCompiled(spamfn)
         self.assertCompiled(eggfn)
 
-    def test_default_recursion_limit(self):
-        many_directories = [str(number) for number in range(1, 100)]
-        self.long_path = os.path.join(self.directory,
-                                      "long",
-                                      *many_directories)
-        os.makedirs(self.long_path)
-        self.source_path_long = script_helper.make_script(
-            self.long_path, "deepscript", ""
-        )
-        self.bc_path_long = importlib.util.cache_from_source(
-            self.source_path_long
-        )
-        self.assertFalse(os.path.isfile(self.bc_path_long))
-        self.assertRunOK('-q', os.path.join(self.directory, "long"))
-        self.assertTrue(os.path.isfile(self.bc_path_long))
+    @support.skip_unless_symlink
+    def test_symlink_loop(self):
+        # Currently, compileall ignores symlinks to directories.
+        # If that limitation is ever lifted, it should protect against
+        # recursion in symlink loops.
+        pkg = os.path.join(self.pkgdir, 'spam')
+        script_helper.make_pkg(pkg)
+        os.symlink('.', os.path.join(pkg, 'evil'))
+        os.symlink('.', os.path.join(pkg, 'evil2'))
+        self.assertRunOK('-q', self.pkgdir)
+        self.assertCompiled(os.path.join(
+            self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
+        ))
 
     def test_quiet(self):
         noisy = self.assertRunOK(self.pkgdir)