cffi: Support using --system-zstd

.github/workflows/external-zstd.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Build
         run: |
-          python -m pip install --config-settings=--global-option=--system-zstd .
+          python -m pip install --config-settings=--global-option=--system-zstd --config-settings=--global-option=--system-zstd-cffi .
 
   macOS:
     runs-on: 'macos-13'

make_cffi.py

@@ -16,22 +16,6 @@
 import cffi
 import packaging.tags
 
-HERE = os.path.abspath(os.path.dirname(__file__))
-
-SOURCES = [
-    "zstd/zstd.c",
-]
-
-# Headers whose preprocessed output will be fed into cdef().
-HEADERS = [
-    os.path.join(HERE, "zstd", p)
-    for p in ("zstd_errors.h", "zstd.h", "zdict.h")
-]
-
-INCLUDE_DIRS = [
-    os.path.join(HERE, "zstd"),
-]
-
 # cffi can't parse some of the primitives in zstd.h. So we invoke the
 # preprocessor and feed its output into cffi.
 compiler = distutils.ccompiler.new_compiler()
@@ -168,21 +152,62 @@ def normalize_output(output):
     return b"\n".join(lines)
 
 
-# musl 1.1 doesn't define qsort_r. We need to force using the C90
-# variant.
-define_macros = []
-for tag in packaging.tags.platform_tags():
-    if tag.startswith("musllinux_1_1_"):
-        define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90"))
+def get_ffi(system_zstd = False):
+    zstd_sources = []
+    include_dirs = []
+    libraries = []
+
+    if not system_zstd:
+        here = os.path.abspath(os.path.dirname(__file__))
 
+        zstd_sources += [
+            "zstd/zstd.c",
+        ]
 
-ffi = cffi.FFI()
-# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
-# when cffi uses the function. Since we statically link against zstd, even
-# if we use the deprecated functions it shouldn't be a huge problem.
-ffi.set_source(
-    "zstandard._cffi",
-    """
+        # Headers whose preprocessed output will be fed into cdef().
+        headers = [
+            os.path.join(here, "zstd", p)
+            for p in ("zstd_errors.h", "zstd.h", "zdict.h")
+        ]
+
+        include_dirs += [
+            os.path.join(here, "zstd"),
+        ]
+    else:
+        libraries += ["zstd"]
+
+        # Locate headers using the preprocessor.
+        include_re = re.compile(r'^# \d+ "([^"]+/(?:zstd_errors|zstd|zdict)\.h)"')
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with open(os.path.join(temp_dir, "input.h"), "w") as f:
+                f.write("""
+#include <zstd_errors.h>
+#include <zstd.h>
+#include <zdict.h>
+""")
+            compiler.preprocess(os.path.join(temp_dir, "input.h"),
+                                os.path.join(temp_dir, "output.h"))
+            with open(os.path.join(temp_dir, "output.h"), "r") as f:
+                headers = list({
+                    m.group(1) for m in map(include_re.match, f)
+                    if m is not None
+                })
+
+    # musl 1.1 doesn't define qsort_r. We need to force using the C90
+    # variant.
+    define_macros = []
+    for tag in packaging.tags.platform_tags():
+        if tag.startswith("musllinux_1_1_"):
+            define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90"))
+
+
+    ffi = cffi.FFI()
+    # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
+    # when cffi uses the function. Since we statically link against zstd, even
+    # if we use the deprecated functions it shouldn't be a huge problem.
+    ffi.set_source(
+        "zstandard._cffi",
+        """
 #define ZSTD_STATIC_LINKING_ONLY
 #define ZSTD_DISABLE_DEPRECATE_WARNINGS
 #include <zstd_errors.h>
@@ -191,48 +216,51 @@ def normalize_output(output):
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
 #include <zdict.h>
 """,
-    sources=SOURCES,
-    include_dirs=INCLUDE_DIRS,
-    define_macros=define_macros,
-)
-
-DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)")
-
-sources = []
-
-# Feed normalized preprocessor output for headers into the cdef parser.
-for header in HEADERS:
-    preprocessed = preprocess(header)
-    sources.append(normalize_output(preprocessed))
-
-    # #define's are effectively erased as part of going through preprocessor.
-    # So perform a manual pass to re-add those to the cdef source.
-    with open(header, "rb") as fh:
-        for line in fh:
-            line = line.strip()
-            m = DEFINE.match(line)
-            if not m:
-                continue
+        sources=zstd_sources,
+        include_dirs=include_dirs,
+        define_macros=define_macros,
+        libraries=libraries,
+    )
 
-            if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
-                continue
+    define = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)")
 
-            # The parser doesn't like some constants with complex values.
-            if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
-                continue
+    sources = []
 
-            # These defines create aliases from old (camelCase) type names
-            # to the new PascalCase names, which breaks CFFI.
-            if m.group(1).lower() == m.group(2).lower():
-                continue
+    # Feed normalized preprocessor output for headers into the cdef parser.
+    for header in headers:
+        preprocessed = preprocess(header)
+        sources.append(normalize_output(preprocessed))
+
+        # #define's are effectively erased as part of going through preprocessor.
+        # So perform a manual pass to re-add those to the cdef source.
+        with open(header, "rb") as fh:
+            for line in fh:
+                line = line.strip()
+                m = define.match(line)
+                if not m:
+                    continue
+
+                if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
+                    continue
+
+                # The parser doesn't like some constants with complex values.
+                if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
+                    continue
+
+                # These defines create aliases from old (camelCase) type names
+                # to the new PascalCase names, which breaks CFFI.
+                if m.group(1).lower() == m.group(2).lower():
+                    continue
+
+                # The ... is magic syntax by the cdef parser to resolve the
+                # value at compile time.
+                sources.append(b"#define " + m.group(1) + b" ...")
 
-            # The ... is magic syntax by the cdef parser to resolve the
-            # value at compile time.
-            sources.append(b"#define " + m.group(1) + b" ...")
+    cdeflines = b"\n".join(sources).splitlines()
+    cdeflines = [line for line in cdeflines if line.strip()]
+    ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
+    return ffi
 
-cdeflines = b"\n".join(sources).splitlines()
-cdeflines = [line for line in cdeflines if line.strip()]
-ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
 
 if __name__ == "__main__":
-    ffi.compile()
+    get_ffi().compile()

setup.py

@@ -73,6 +73,7 @@
 
 SUPPORT_LEGACY = False
 SYSTEM_ZSTD = False
+SYSTEM_ZSTD_CFFI = False
 WARNINGS_AS_ERRORS = False
 C_BACKEND = True
 CFFI_BACKEND = True
@@ -103,6 +104,10 @@
     SYSTEM_ZSTD = True
     sys.argv.remove("--system-zstd")
 
+if "--system-zstd-cffi" in sys.argv:
+    SYSTEM_ZSTD_CFFI = True
+    sys.argv.remove("--system-zstd-cffi")
+
 if "--warnings-as-errors" in sys.argv:
     WARNINGS_AS_ERRORS = True
     sys.argv.remove("--warning-as-errors")
@@ -138,7 +143,7 @@
 if CFFI_BACKEND and cffi:
     import make_cffi
 
-    extensions.append(make_cffi.ffi.distutils_extension())
+    extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD_CFFI).distutils_extension())
 
 version = None