bpo-25711: Rewrite zipimport in pure Python. (GH-6809)

Author: serhiy-storchaka

Lib/ctypes/test/test_values.py

@@ -64,6 +64,7 @@ class struct_frozen(Structure):
         bootstrap_expected = [
                 b'_frozen_importlib',
                 b'_frozen_importlib_external',
+                b'zipimport',
                 ]
         for entry in ft:
             # This is dangerous. We *can* iterate over a pointer, but

Lib/importlib/__init__.py

@@ -48,8 +48,8 @@
     sys.modules['importlib._bootstrap_external'] = _bootstrap_external
 
 # To simplify imports in test code
-_w_long = _bootstrap_external._w_long
-_r_long = _bootstrap_external._r_long
+_pack_uint32 = _bootstrap_external._pack_uint32
+_unpack_uint32 = _bootstrap_external._unpack_uint32
 
 # Fully bootstrapped at this point, import whatever you like, circular
 # dependencies and startup overhead minimisation permitting :)

Lib/importlib/_bootstrap_external.py

@@ -43,14 +43,20 @@ def _relax_case():
     return _relax_case
 
 
-def _w_long(x):
+def _pack_uint32(x):
     """Convert a 32-bit integer to little-endian."""
     return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
 
 
-def _r_long(int_bytes):
+def _unpack_uint32(data):
     """Convert 4 bytes in little-endian to an integer."""
-    return int.from_bytes(int_bytes, 'little')
+    assert len(data) == 4
+    return int.from_bytes(data, 'little')
+
+def _unpack_uint16(data):
+    """Convert 2 bytes in little-endian to an integer."""
+    assert len(data) == 2
+    return int.from_bytes(data, 'little')
 
 
 def _path_join(*path_parts):
@@ -503,7 +509,7 @@ def _classify_pyc(data, name, exc_details):
         message = f'reached EOF while reading pyc header of {name!r}'
         _bootstrap._verbose_message('{}', message)
         raise EOFError(message)
-    flags = _r_long(data[4:8])
+    flags = _unpack_uint32(data[4:8])
     # Only the first two flags are defined.
     if flags & ~0b11:
         message = f'invalid flags {flags!r} in {name!r}'
@@ -530,12 +536,12 @@ def _validate_timestamp_pyc(data, source_mtime, source_size, name,
     An ImportError is raised if the bytecode is stale.
 
     """
-    if _r_long(data[8:12]) != (source_mtime & 0xFFFFFFFF):
+    if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF):
         message = f'bytecode is stale for {name!r}'
         _bootstrap._verbose_message('{}', message)
         raise ImportError(message, **exc_details)
     if (source_size is not None and
-        _r_long(data[12:16]) != (source_size & 0xFFFFFFFF)):
+        _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)):
         raise ImportError(f'bytecode is stale for {name!r}', **exc_details)
 
 
@@ -579,9 +585,9 @@ def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
 def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
     "Produce the data for a timestamp-based pyc."
     data = bytearray(MAGIC_NUMBER)
-    data.extend(_w_long(0))
-    data.extend(_w_long(mtime))
-    data.extend(_w_long(source_size))
+    data.extend(_pack_uint32(0))
+    data.extend(_pack_uint32(mtime))
+    data.extend(_pack_uint32(source_size))
     data.extend(marshal.dumps(code))
     return data
 
@@ -590,7 +596,7 @@ def _code_to_hash_pyc(code, source_hash, checked=True):
     "Produce the data for a hash-based pyc."
     data = bytearray(MAGIC_NUMBER)
     flags = 0b1 | checked << 1
-    data.extend(_w_long(flags))
+    data.extend(_pack_uint32(flags))
     assert len(source_hash) == 8
     data.extend(source_hash)
     data.extend(marshal.dumps(code))

Lib/test/test_bdb.py

@@ -726,7 +726,7 @@ def main():
                 ('line', 2, 'tfunc_import'), ('step', ),
                 ('line', 3, 'tfunc_import'), ('quit', ),
             ]
-            skip = ('importlib*', TEST_MODULE)
+            skip = ('importlib*', 'zipimport', TEST_MODULE)
             with TracerRun(self, skip=skip) as tracer:
                 tracer.runcall(tfunc_import)
 

Lib/test/test_importlib/source/test_file_loader.py

@@ -629,7 +629,7 @@ def test_old_timestamp(self):
                 bytecode_file.write(zeros)
             self.import_(mapping['_temp'], '_temp')
             source_mtime = os.path.getmtime(mapping['_temp'])
-            source_timestamp = self.importlib._w_long(source_mtime)
+            source_timestamp = self.importlib._pack_uint32(source_mtime)
             with open(bytecode_path, 'rb') as bytecode_file:
                 bytecode_file.seek(8)
                 self.assertEqual(bytecode_file.read(4), source_timestamp)

Lib/test/test_importlib/test_abc.py

@@ -712,9 +712,9 @@ def __init__(self, path, magic=None):
         if magic is None:
             magic = self.util.MAGIC_NUMBER
         data = bytearray(magic)
-        data.extend(self.init._w_long(0))
-        data.extend(self.init._w_long(self.source_mtime))
-        data.extend(self.init._w_long(self.source_size))
+        data.extend(self.init._pack_uint32(0))
+        data.extend(self.init._pack_uint32(self.source_mtime))
+        data.extend(self.init._pack_uint32(self.source_size))
         code_object = compile(self.source, self.path, 'exec',
                                 dont_inherit=True)
         data.extend(marshal.dumps(code_object))
@@ -876,9 +876,9 @@ def verify_code(self, code_object, *, bytecode_written=False):
         if bytecode_written:
             self.assertIn(self.cached, self.loader.written)
             data = bytearray(self.util.MAGIC_NUMBER)
-            data.extend(self.init._w_long(0))
-            data.extend(self.init._w_long(self.loader.source_mtime))
-            data.extend(self.init._w_long(self.loader.source_size))
+            data.extend(self.init._pack_uint32(0))
+            data.extend(self.init._pack_uint32(self.loader.source_mtime))
+            data.extend(self.init._pack_uint32(self.loader.source_size))
             data.extend(marshal.dumps(code_object))
             self.assertEqual(self.loader.written[self.cached], bytes(data))
 

Lib/test/test_zipimport.py

@@ -551,7 +551,12 @@ def replace(self, old, new):
             z.writestr(name, data)
             z.close()
             zi = zipimport.zipimporter(TEMP_ZIP)
-            self.assertEqual(data, zi.get_data(FunnyStr(name)))
+            try:
+                data2 = zi.get_data(FunnyStr(name))
+            except AttributeError:
+                pass
+            else:
+                self.assertEqual(data2, data)
         finally:
             z.close()
             os.remove(TEMP_ZIP)
@@ -677,24 +682,24 @@ def testBytesPath(self):
 
         zipimport.zipimporter(filename)
         zipimport.zipimporter(os.fsencode(filename))
-        with self.assertWarns(DeprecationWarning):
+        with self.assertRaises(TypeError):
             zipimport.zipimporter(bytearray(os.fsencode(filename)))
-        with self.assertWarns(DeprecationWarning):
+        with self.assertRaises(TypeError):
             zipimport.zipimporter(memoryview(os.fsencode(filename)))
 
     @support.cpython_only
     def testUninitializedZipimporter(self):
         # The interpreter shouldn't crash in case of calling methods of an
         # uninitialized zipimport.zipimporter object.
         zi = zipimport.zipimporter.__new__(zipimport.zipimporter)
-        self.assertRaises(ValueError, zi.find_module, 'foo')
-        self.assertRaises(ValueError, zi.find_loader, 'foo')
-        self.assertRaises(ValueError, zi.load_module, 'foo')
-        self.assertRaises(ValueError, zi.get_filename, 'foo')
-        self.assertRaises(ValueError, zi.is_package, 'foo')
-        self.assertRaises(ValueError, zi.get_data, 'foo')
-        self.assertRaises(ValueError, zi.get_code, 'foo')
-        self.assertRaises(ValueError, zi.get_source, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.find_module, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.find_loader, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.load_module, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.get_filename, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.is_package, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.get_data, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.get_code, 'foo')
+        self.assertRaises((ValueError, AttributeError), zi.get_source, 'foo')
 
 
 @support.requires_zlib
@@ -712,7 +717,7 @@ def bad_decompress(*args):
             zip_file.writestr('bar.py', b'print("hello world")', ZIP_DEFLATED)
         zi = zipimport.zipimporter(TEMP_ZIP)
         with support.swap_attr(zlib, 'decompress', bad_decompress):
-            self.assertRaises(TypeError, zi.get_source, 'bar')
+            self.assertRaises((TypeError, AttributeError), zi.get_source, 'bar')
 
 
 class BadFileZipImportTestCase(unittest.TestCase):