feat(python): make setup.py conditional abi3 compatible, and python2 compatible again

Author: K0lb3

python/tests/_test_utils.py

@@ -18,7 +18,7 @@
   test_dir = os.path.join(project_dir, 'tests')
 if BRO_ARGS[0] is None:
   python_exe = sys.executable or 'python'
-  bro_path = os.path.join(project_dir, 'python', 'bro.py')
+  bro_path = os.path.join(project_dir, 'python', 'brotli', '__main__.py')
   BRO_ARGS = [python_exe, bro_path]
 
 # Get the platform/version-specific build folder.

setup.py

@@ -7,6 +7,7 @@
 import platform
 import re
 import unittest
+import sys
 
 try:
     from setuptools import Extension
@@ -15,7 +16,6 @@
     from distutils.core import Extension
     from distutils.core import setup
 from distutils.command.build_ext import build_ext
-from distutils import errors
 from distutils import dep_util
 from distutils import log
 
@@ -24,7 +24,7 @@
 CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
 
 
-def bool_from_environ(key: str):
+def bool_from_environ(key):
     value = os.environ.get(key)
     if not value:
         return False
@@ -39,7 +39,7 @@ def read_define(path, macro):
     """ Return macro value from the given file. """
     with open(path, 'r') as f:
         for line in f:
-            m = re.match(rf'#define\s{macro}\s+(.+)', line)
+            m = re.match('#define\\s{macro}\\s+(.+)'.format(macro=macro), line)
             if m:
                 return m.group(1)
 
@@ -54,7 +54,7 @@ def get_version():
     patch = read_define(version_file_path, 'BROTLI_VERSION_PATCH')
     if not major or not minor or not patch:
         return ''
-    return f'{major}.{minor}.{patch}'
+    return '{major}.{minor}.{patch}'.format(major=major, minor=minor, patch=patch)
 
 
 def get_test_suite():
@@ -64,20 +64,7 @@ def get_test_suite():
 
 
 class BuildExt(build_ext):
-
-    def get_source_files(self):
-        filenames = build_ext.get_source_files(self)
-        for ext in self.extensions:
-            filenames.extend(ext.depends)
-        return filenames
-
     def build_extension(self, ext):
-        if ext.sources is None or not isinstance(ext.sources, (list, tuple)):
-            raise errors.DistutilsSetupError(
-            "in 'ext_modules' option (extension '%s'), "
-            "'sources' must be present and must be "
-            "a list of source filenames" % ext.name)
-
         ext_path = self.get_ext_fullpath(ext.name)
         depends = ext.sources + ext.depends
         if not (self.force or dep_util.newer_group(depends, ext_path, 'newer')):
@@ -86,59 +73,16 @@ def build_extension(self, ext):
         else:
             log.info("building '%s' extension", ext.name)
 
-        c_sources = []
-        for source in ext.sources:
-            if source.endswith('.c'):
-                 c_sources.append(source)
-        extra_args = ext.extra_compile_args or []
-
-        objects = []
-
-        macros = ext.define_macros[:]
         if platform.system() == 'Darwin':
-            macros.append(('OS_MACOSX', '1'))
+            ext.define_macros.append(('OS_MACOSX', '1'))
         elif self.compiler.compiler_type == 'mingw32':
             # On Windows Python 2.7, pyconfig.h defines "hypot" as "_hypot",
             # This clashes with GCC's cmath, and causes compilation errors when
             # building under MinGW: http://bugs.python.org/issue11566
-            macros.append(('_hypot', 'hypot'))
-        for undef in ext.undef_macros:
-            macros.append((undef,))
-
-        objs = self.compiler.compile(
-            c_sources,
-            output_dir=self.build_temp,
-            macros=macros,
-            include_dirs=ext.include_dirs,
-            debug=self.debug,
-            extra_postargs=extra_args,
-            depends=ext.depends)
-        objects.extend(objs)
-
-        self._built_objects = objects[:]
-        if ext.extra_objects:
-            objects.extend(ext.extra_objects)
-        extra_args = ext.extra_link_args or []
-        # when using GCC on Windows, we statically link libgcc and libstdc++,
-        # so that we don't need to package extra DLLs
-        if self.compiler.compiler_type == 'mingw32':
-            extra_args.extend(['-static-libgcc', '-static-libstdc++'])
+            ext.define_macros.append(('_hypot', 'hypot'))
+            ext.extra_link_args.extend(['-static-libgcc', '-static-libstdc++'])
 
-        ext_path = self.get_ext_fullpath(ext.name)
-        # Detect target language, if not provided
-        language = ext.language or self.compiler.detect_language(c_sources)
-
-        self.compiler.link_shared_object(
-            objects,
-            ext_path,
-            libraries=self.get_libraries(ext),
-            library_dirs=ext.library_dirs,
-            runtime_library_dirs=ext.runtime_library_dirs,
-            extra_postargs=extra_args,
-            export_symbols=self.get_export_symbols(ext),
-            debug=self.debug,
-            build_temp=self.build_temp,
-            target_lang=language)
+        build_ext.build_extension(self, ext)
 
 
 NAME = 'Brotli'
@@ -183,9 +127,20 @@ def build_extension(self, ext):
 
 PACKAGE_DIR = {'': 'python'}
 
-PY_MODULES = ['brotli']
-
 USE_SYSTEM_BROTLI = bool_from_environ('USE_SYSTEM_BROTLI')
+BROTLI_PYTHON3_LIMITED_API = bool_from_environ('BROTLI_PYTHON3_LIMITED_API')
+IS_PYTHON3 = sys.version_info[0] == 3
+
+class VersionedExtension(Extension):
+    def __init__(self, *args, **kwargs):
+        define_macros = []
+
+        if IS_PYTHON3 and BROTLI_PYTHON3_LIMITED_API:
+            kwargs['py_limited_api'] = True
+            define_macros.append(('Py_LIMITED_API', '0x03060000'))
+        
+        kwargs['define_macros'] = kwargs.get('define_macros', []) + define_macros
+        Extension.__init__(self, *args, **kwargs)
 
 if USE_SYSTEM_BROTLI:
     import pkgconfig
@@ -205,8 +160,8 @@ def build_extension(self, ext):
         libraries += package_configuration["libraries"]
         library_dirs += package_configuration["library_dirs"]
 
-    brotli_extension = Extension(
-        '_brotli',
+    brotli_extension = VersionedExtension(
+        'brotli._brotli',
         sources=[
             'python/_brotli.c'
         ],
@@ -220,8 +175,8 @@ def build_extension(self, ext):
     EXT_MODULES = [brotli_extension]
 else:
     EXT_MODULES = [
-        Extension(
-            '_brotli',
+        VersionedExtension(
+            name='brotli._brotli',
             sources=[
                 'python/_brotli.c',
                 'c/common/constants.c',
@@ -323,6 +278,23 @@ def build_extension(self, ext):
     'build_ext': BuildExt,
 }
 
+if IS_PYTHON3 and BROTLI_PYTHON3_LIMITED_API:
+    from wheel.bdist_wheel import bdist_wheel
+    # adopted from:
+    # https://github.com/joerick/python-abi3-package-sample/blob/7f05b22b9e0cfb4e60293bc85252e95278a80720/setup.py
+    class bdist_wheel_abi3(bdist_wheel):
+        def get_tag(self):
+            python, abi, plat = super().get_tag()
+
+            if python.startswith("cp"):
+                # on CPython, our wheels are abi3 and compatible back to 3.6
+                return "cp36", "abi3", plat
+
+            return python, abi, plat
+        
+    CMD_CLASS["bdist_wheel"] = bdist_wheel_abi3
+
+
 with open("README.md", "r") as f:
     README = f.read()
 
@@ -338,7 +310,7 @@ def build_extension(self, ext):
     platforms=PLATFORMS,
     classifiers=CLASSIFIERS,
     package_dir=PACKAGE_DIR,
-    py_modules=PY_MODULES,
+    packages=["brotli"],
     ext_modules=EXT_MODULES,
     test_suite=TEST_SUITE,
     cmdclass=CMD_CLASS)