From 6ec59f14d141a1e94afa8dba36e1e5fa865ae0d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 24 Jan 2025 14:59:29 +0100 Subject: [PATCH 1/2] cffi: Move CFFI logic from global scope into `get_ffi()` function Move the CFFI-related logic from the global scope of `make_cffi.py` file to a `get_ffi()` function. This makes it possible to import the file without executing the code immediately, and it will make it possible to parametrize the invocation using explicit function parameters. Along with the change, I've renamed the variables to lowercase since they are no longer global. --- make_cffi.py | 129 ++++++++++++++++++++++++++------------------------- setup.py | 2 +- 2 files changed, 67 insertions(+), 64 deletions(-) diff --git a/make_cffi.py b/make_cffi.py index 70d0c8e2..31c9a748 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -15,19 +15,6 @@ import cffi -HERE = os.path.abspath(os.path.dirname(__file__)) - -SOURCES = [ - "zstd/zstd.c", -] - -# Headers whose preprocessed output will be fed into cdef(). -HEADERS = [os.path.join(HERE, "zstd", p) for p in ("zstd.h", "zdict.h")] - -INCLUDE_DIRS = [ - os.path.join(HERE, "zstd"), -] - # cffi can't parse some of the primitives in zstd.h. So we invoke the # preprocessor and feed its output into cffi. compiler = distutils.ccompiler.new_compiler() @@ -158,59 +145,75 @@ def normalize_output(output): return b"\n".join(lines) -ffi = cffi.FFI() -# zstd.h uses a possible undefined MIN(). Define it until -# https://github.com/facebook/zstd/issues/976 is fixed. -# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning -# when cffi uses the function. Since we statically link against zstd, even -# if we use the deprecated functions it shouldn't be a huge problem. -ffi.set_source( - "zstandard._cffi", - """ -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define ZSTD_STATIC_LINKING_ONLY -#define ZSTD_DISABLE_DEPRECATE_WARNINGS -#include -#define ZDICT_STATIC_LINKING_ONLY -#define ZDICT_DISABLE_DEPRECATE_WARNINGS -#include -""", - sources=SOURCES, - include_dirs=INCLUDE_DIRS, -) - -DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ") - -sources = [] - -# Feed normalized preprocessor output for headers into the cdef parser. -for header in HEADERS: - preprocessed = preprocess(header) - sources.append(normalize_output(preprocessed)) - - # #define's are effectively erased as part of going through preprocessor. - # So perform a manual pass to re-add those to the cdef source. - with open(header, "rb") as fh: - for line in fh: - line = line.strip() - m = DEFINE.match(line) - if not m: - continue +def get_ffi(): + here = os.path.abspath(os.path.dirname(__file__)) + + zstd_sources = [ + "zstd/zstd.c", + ] + + # Headers whose preprocessed output will be fed into cdef(). + headers = [os.path.join(here, "zstd", p) for p in ("zstd.h", "zdict.h")] + + include_dirs = [ + os.path.join(here, "zstd"), + ] + + ffi = cffi.FFI() + # zstd.h uses a possible undefined MIN(). Define it until + # https://github.com/facebook/zstd/issues/976 is fixed. + # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning + # when cffi uses the function. Since we statically link against zstd, even + # if we use the deprecated functions it shouldn't be a huge problem. + ffi.set_source( + "zstandard._cffi", + """ + #define MIN(a,b) ((a)<(b) ? (a) : (b)) + #define ZSTD_STATIC_LINKING_ONLY + #define ZSTD_DISABLE_DEPRECATE_WARNINGS + #include + #define ZDICT_STATIC_LINKING_ONLY + #define ZDICT_DISABLE_DEPRECATE_WARNINGS + #include + """, + sources=zstd_sources, + include_dirs=include_dirs, + ) - if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": - continue + DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ") - # The parser doesn't like some constants with complex values. - if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): - continue + sources = [] + + # Feed normalized preprocessor output for headers into the cdef parser. + for header in headers: + preprocessed = preprocess(header) + sources.append(normalize_output(preprocessed)) + + # #define's are effectively erased as part of going through preprocessor. + # So perform a manual pass to re-add those to the cdef source. + with open(header, "rb") as fh: + for line in fh: + line = line.strip() + m = DEFINE.match(line) + if not m: + continue + + if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": + continue + + # The parser doesn't like some constants with complex values. + if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): + continue + + # The ... is magic syntax by the cdef parser to resolve the + # value at compile time. + sources.append(m.group(0) + b" ...") - # The ... is magic syntax by the cdef parser to resolve the - # value at compile time. - sources.append(m.group(0) + b" ...") + cdeflines = b"\n".join(sources).splitlines() + cdeflines = [line for line in cdeflines if line.strip()] + ffi.cdef(b"\n".join(cdeflines).decode("latin1")) + return ffi -cdeflines = b"\n".join(sources).splitlines() -cdeflines = [line for line in cdeflines if line.strip()] -ffi.cdef(b"\n".join(cdeflines).decode("latin1")) if __name__ == "__main__": - ffi.compile() + get_ffi().compile() diff --git a/setup.py b/setup.py index 921f3839..0d3d8819 100755 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.ffi.distutils_extension()) + extensions.append(make_cffi.get_ffi().distutils_extension()) version = None From 8a9ddf7f090d46a0a0a11e196d6aaf6a97883bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 24 Jan 2025 15:54:02 +0100 Subject: [PATCH 2/2] cffi: Support --system-zstd Support using the system zstd library with the CFFI backend. Use the GCC / Clang preprocessor output to find the paths to header files for preprocessing. Link to the system library, matching the behavior for the C backend. --- make_cffi.py | 41 +++++++++++++++++++++++++++++++---------- setup.py | 2 +- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/make_cffi.py b/make_cffi.py index 31c9a748..44dc2c38 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -145,19 +145,39 @@ def normalize_output(output): return b"\n".join(lines) -def get_ffi(): - here = os.path.abspath(os.path.dirname(__file__)) +def get_ffi(system_zstd = False): + zstd_sources = [] + include_dirs = [] + libraries = [] - zstd_sources = [ - "zstd/zstd.c", - ] + if not system_zstd: + here = os.path.abspath(os.path.dirname(__file__)) - # Headers whose preprocessed output will be fed into cdef(). - headers = [os.path.join(here, "zstd", p) for p in ("zstd.h", "zdict.h")] + zstd_sources += [ + "zstd/zstd.c", + ] + + # Headers whose preprocessed output will be fed into cdef(). + headers = [os.path.join(here, "zstd", p) for p in ("zstd.h", "zdict.h")] - include_dirs = [ - os.path.join(here, "zstd"), - ] + include_dirs += [ + os.path.join(here, "zstd"), + ] + else: + libraries += ["zstd"] + + # Locate headers using the preprocessor. + include_re = re.compile(r'^# \d+ "([^"]+/(?:zstd|zdict)\.h)"') + with tempfile.TemporaryDirectory() as temp_dir: + with open(os.path.join(temp_dir, "input.h"), "w") as f: + f.write("#include \n#include \n") + compiler.preprocess(os.path.join(temp_dir, "input.h"), + os.path.join(temp_dir, "output.h")) + with open(os.path.join(temp_dir, "output.h"), "r") as f: + headers = list({ + m.group(1) for m in map(include_re.match, f) + if m is not None + }) ffi = cffi.FFI() # zstd.h uses a possible undefined MIN(). Define it until @@ -178,6 +198,7 @@ def get_ffi(): """, sources=zstd_sources, include_dirs=include_dirs, + libraries=libraries, ) DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ") diff --git a/setup.py b/setup.py index 0d3d8819..cf1e65c5 100755 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.get_ffi().distutils_extension()) + extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD).distutils_extension()) version = None