Skip to content

Commit 6f5bc87

Browse files
emmatypingemmatyping
authored
Update Rust detection and binding generation (#15)
This commit updates the build system to automatically detect cargo and enable/disable _base64 without needing to pass a flag. If cargo is unavailable, _base64 is disabled. It also updates cpython-sys to use a hand written header (which is what Linux seems to do) and splits off the parser bindings to be handled in the future (since the files are included differently).
1 parent 35cdedb commit 6f5bc87

File tree

11 files changed

+268
-137
lines changed

11 files changed

+268
-137
lines changed

‎Makefile.pre.in‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1652,10 +1652,6 @@ Makefile Modules/config.c: Makefile.pre \
16521652
@mv config.c Modules
16531653
@echo "The Makefile was updated, you may need to re-run make."
16541654

1655-
.PHONY: regen-rust-wrapper-h
1656-
regen-rust-wrapper-h: $(PYTHON_HEADERS)
1657-
PYTHON_HEADERS="$(PYTHON_HEADERS)" $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/regen-rust-wrapper-h.py
1658-
16591655
.PHONY: regen-test-frozenmain
16601656
regen-test-frozenmain: $(BUILDPYTHON)
16611657
# Regenerate Programs/test_frozenmain.h
@@ -3376,6 +3372,9 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
33763372
##########################################################################
33773373
# Module dependencies and platform-specific files
33783374

3375+
cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h
3376+
cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE)
3377+
33793378
# force rebuild when header file or module build flavor (static/shared) is changed
33803379
MODULE_DEPS_STATIC=Modules/config.c
33813380
MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@

‎Modules/cpython-sys/.gitignore‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎Modules/cpython-sys/build.rs‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,29 @@ use std::path::{Path, PathBuf};
44
fn main() {
55
let curdir = std::env::current_dir().unwrap();
66
let srcdir = curdir.parent().and_then(Path::parent).unwrap();
7+
let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
8+
generate_c_api_bindings(srcdir, &out_path.as_path());
9+
// TODO(emmatyping): generate bindings to the internal parser API
10+
// The parser includes things slightly differently, so we should generate
11+
// it's bindings independently
12+
//generate_parser_bindings(srcdir, &out_path.as_path());
13+
}
14+
15+
fn generate_c_api_bindings(srcdir: &Path, out_path: &Path) {
716
let bindings = bindgen::Builder::default()
817
.header("wrapper.h")
918
.clang_arg(format!("-I{}", srcdir.as_os_str().to_str().unwrap()))
1019
.clang_arg(format!("-I{}/Include", srcdir.as_os_str().to_str().unwrap()))
11-
.clang_arg(format!("-I{}/Include/internal", srcdir.as_os_str().to_str().unwrap()))
12-
.allowlist_function("Py.*")
13-
.allowlist_function("_Py.*")
14-
.allowlist_type("Py.*")
15-
.allowlist_type("_Py.*")
16-
.allowlist_var("Py.*")
17-
.allowlist_var("_Py.*")
20+
.allowlist_function("_?Py.*")
21+
.allowlist_type("_?Py.*")
22+
.allowlist_var("_?Py.*")
1823
.blocklist_type("^PyMethodDef$")
1924
.parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
2025
.generate()
2126
.expect("Unable to generate bindings");
2227

23-
// Write the bindings to the $OUT_DIR/bindings.rs file.
24-
let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
28+
// Write the bindings to the $OUT_DIR/c_api.rs file.
2529
bindings
26-
.write_to_file(out_path.join("bindings.rs"))
30+
.write_to_file(out_path.join("c_api.rs"))
2731
.expect("Couldn't write bindings!");
2832
}

‎Modules/cpython-sys/parser.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/* Private APIs */
2+
#define Py_BUILD_CORE
3+
4+
// Parser
5+
#include "Parser/pegen.h"
6+
#include "Parser/string_parser.h"
7+
#include "Parser/lexer/buffer.h"
8+
#include "Parser/lexer/lexer.h"
9+
#include "Parser/lexer/state.h"
10+
#include "Parser/tokenizer/tokenizer.h"
11+
#include "Parser/tokenizer/helpers.h"

‎Modules/cpython-sys/src/lib.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@
66

77
use std::ffi::{c_char, c_int, c_void};
88

9-
include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
9+
include!(concat!(env!("OUT_DIR"), "/c_api.rs"));
10+
11+
// TODO(emmatyping): include parser bindings (see build.rs)
12+
//include!(concat!(env!("OUT_DIR"), "/parser.rs"));
1013
/* Flag passed to newmethodobject */
1114
/* #define METH_OLDARGS 0x0000 -- unsupported now */
1215
pub const METH_VARARGS: c_int = 0x0001;

‎Modules/cpython-sys/wrapper.h‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/* Public APIs */
2+
#include "Python.h"
3+
4+
// Misc
5+
6+
// OS macros used in C, not necessary in Rust
7+
//#include "osdefs.h"
8+
9+
// Valgrind / analysis tools macros and functions
10+
#include "dynamic_annotations.h"
11+
// Macros for error codes
12+
// #include "errcode.h"
13+
// Macros to define symbol visibility
14+
// #include "exports.h"
15+
// Includes pyframe.h and cpython/frameobject.h
16+
#include "frameobject.h"
17+
// Includes cpython/marshal.h
18+
#include "marshal.h"
19+
// Macros defining opcodes
20+
#include "opcode.h"
21+
// More macros defining opcodes
22+
#include "opcode_ids.h"
23+
// Dtrace probes
24+
#include "pydtrace.h"
25+
//New code should use descrobject.h
26+
//#include "structmember.h"
27+
28+
// List of all stdlib names, autogenerated
29+
#include "Python/stdlib_module_names.h"
30+
31+
/* Private APIs */
32+
#define Py_BUILD_CORE
33+
34+
// Internal
35+
#include "internal/pycore_parser.h"
36+
#include "internal/pycore_mimalloc.h"
37+
#include "internal/mimalloc/mimalloc.h"
38+
#include "internal/mimalloc/mimalloc/atomic.h"
39+
#include "internal/mimalloc/mimalloc/internal.h"
40+
#include "internal/mimalloc/mimalloc/prim.h"
41+
#include "internal/mimalloc/mimalloc/track.h"
42+
#include "internal/mimalloc/mimalloc/types.h"
43+
#include "internal/pycore_abstract.h"
44+
#include "internal/pycore_asdl.h"
45+
#include "internal/pycore_ast.h"
46+
#include "internal/pycore_ast_state.h"
47+
#include "internal/pycore_atexit.h"
48+
#include "internal/pycore_audit.h"
49+
#include "internal/pycore_backoff.h"
50+
#include "internal/pycore_bitutils.h"
51+
#include "internal/pycore_blocks_output_buffer.h"
52+
#include "internal/pycore_brc.h"
53+
#include "internal/pycore_bytes_methods.h"
54+
#include "internal/pycore_bytesobject.h"
55+
#include "internal/pycore_call.h"
56+
#include "internal/pycore_capsule.h"
57+
#include "internal/pycore_cell.h"
58+
#include "internal/pycore_ceval.h"
59+
#include "internal/pycore_ceval_state.h"
60+
#include "internal/pycore_code.h"
61+
#include "internal/pycore_codecs.h"
62+
#include "internal/pycore_compile.h"
63+
#include "internal/pycore_complexobject.h"
64+
#include "internal/pycore_condvar.h"
65+
#include "internal/pycore_context.h"
66+
#include "internal/pycore_critical_section.h"
67+
#include "internal/pycore_crossinterp.h"
68+
#include "internal/pycore_debug_offsets.h"
69+
#include "internal/pycore_descrobject.h"
70+
#include "internal/pycore_dict.h"
71+
#include "internal/pycore_dict_state.h"
72+
#include "internal/pycore_dtoa.h"
73+
#include "internal/pycore_exceptions.h"
74+
#include "internal/pycore_faulthandler.h"
75+
#include "internal/pycore_fileutils.h"
76+
#include "internal/pycore_floatobject.h"
77+
#include "internal/pycore_flowgraph.h"
78+
#include "internal/pycore_format.h"
79+
#include "internal/pycore_frame.h"
80+
#include "internal/pycore_freelist.h"
81+
#include "internal/pycore_freelist_state.h"
82+
#include "internal/pycore_function.h"
83+
#include "internal/pycore_gc.h"
84+
#include "internal/pycore_genobject.h"
85+
#include "internal/pycore_getopt.h"
86+
#include "internal/pycore_gil.h"
87+
#include "internal/pycore_global_objects.h"
88+
#include "internal/pycore_global_objects_fini_generated.h"
89+
#include "internal/pycore_global_strings.h"
90+
#include "internal/pycore_hamt.h"
91+
#include "internal/pycore_hashtable.h"
92+
#include "internal/pycore_import.h"
93+
#include "internal/pycore_importdl.h"
94+
#include "internal/pycore_index_pool.h"
95+
#include "internal/pycore_initconfig.h"
96+
#include "internal/pycore_instruments.h"
97+
#include "internal/pycore_instruction_sequence.h"
98+
#include "internal/pycore_interp.h"
99+
#include "internal/pycore_interp_structs.h"
100+
#include "internal/pycore_interpframe.h"
101+
#include "internal/pycore_interpframe_structs.h"
102+
#include "internal/pycore_interpolation.h"
103+
#include "internal/pycore_intrinsics.h"
104+
#include "internal/pycore_jit.h"
105+
#include "internal/pycore_list.h"
106+
#include "internal/pycore_llist.h"
107+
#include "internal/pycore_lock.h"
108+
#include "internal/pycore_long.h"
109+
#include "internal/pycore_memoryobject.h"
110+
#include "internal/pycore_mimalloc.h"
111+
#include "internal/pycore_modsupport.h"
112+
#include "internal/pycore_moduleobject.h"
113+
#include "internal/pycore_namespace.h"
114+
#include "internal/pycore_object.h"
115+
#include "internal/pycore_object_alloc.h"
116+
#include "internal/pycore_object_deferred.h"
117+
#include "internal/pycore_object_stack.h"
118+
#include "internal/pycore_object_state.h"
119+
#include "internal/pycore_obmalloc.h"
120+
#include "internal/pycore_obmalloc_init.h"
121+
#include "internal/pycore_opcode_metadata.h"
122+
#include "internal/pycore_opcode_utils.h"
123+
#include "internal/pycore_optimizer.h"
124+
#include "internal/pycore_parking_lot.h"
125+
#include "internal/pycore_parser.h"
126+
#include "internal/pycore_pathconfig.h"
127+
#include "internal/pycore_pyarena.h"
128+
#include "internal/pycore_pyatomic_ft_wrappers.h"
129+
#include "internal/pycore_pybuffer.h"
130+
#include "internal/pycore_pyerrors.h"
131+
#include "internal/pycore_pyhash.h"
132+
#include "internal/pycore_pylifecycle.h"
133+
#include "internal/pycore_pymath.h"
134+
#include "internal/pycore_pymem.h"
135+
#include "internal/pycore_pymem_init.h"
136+
#include "internal/pycore_pystate.h"
137+
#include "internal/pycore_pystats.h"
138+
#include "internal/pycore_pythonrun.h"
139+
#include "internal/pycore_pythread.h"
140+
#include "internal/pycore_qsbr.h"
141+
#include "internal/pycore_range.h"
142+
#include "internal/pycore_runtime.h"
143+
#include "internal/pycore_runtime_init.h"
144+
#include "internal/pycore_runtime_init_generated.h"
145+
#include "internal/pycore_runtime_structs.h"
146+
#include "internal/pycore_semaphore.h"
147+
#include "internal/pycore_setobject.h"
148+
#include "internal/pycore_signal.h"
149+
#include "internal/pycore_sliceobject.h"
150+
#include "internal/pycore_stats.h"
151+
#include "internal/pycore_strhex.h"
152+
#include "internal/pycore_stackref.h"
153+
#include "internal/pycore_structs.h"
154+
#include "internal/pycore_structseq.h"
155+
#include "internal/pycore_symtable.h"
156+
#include "internal/pycore_sysmodule.h"
157+
#include "internal/pycore_template.h"
158+
#include "internal/pycore_time.h"
159+
#include "internal/pycore_token.h"
160+
#include "internal/pycore_traceback.h"
161+
#include "internal/pycore_tracemalloc.h"
162+
#include "internal/pycore_tstate.h"
163+
#include "internal/pycore_tuple.h"
164+
#include "internal/pycore_typedefs.h"
165+
#include "internal/pycore_typeobject.h"
166+
#include "internal/pycore_typevarobject.h"
167+
#include "internal/pycore_ucnhash.h"
168+
#include "internal/pycore_unicodectype.h"
169+
#include "internal/pycore_unicodeobject.h"
170+
#include "internal/pycore_unicodeobject_generated.h"
171+
#include "internal/pycore_unionobject.h"
172+
#include "internal/pycore_uniqueid.h"
173+
#include "internal/pycore_uop.h"
174+
#include "internal/pycore_uop_ids.h"
175+
#include "internal/pycore_uop_metadata.h"
176+
#include "internal/pycore_warnings.h"
177+
#include "internal/pycore_weakref.h"

‎Modules/makesetup‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
286286
done
287287
libs=
288288
# depends on the headers through cpython-sys
289-
rule="$objs: \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest Modules/cpython-sys/wrapper.h $prefixed_srcs \$(PYTHON_HEADERS)"
289+
rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)"
290290
rule="$rule; cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE)"
291291
echo "$rule" >>$rulesf
292292
for mod in $mods

‎Python/remote_debug.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ extern "C" {
2929

3030
#include "pyconfig.h"
3131
#include "internal/pycore_ceval.h"
32-
#include "internal/pycore_debug_offsets.h"
3332

3433
#ifdef __linux__
3534
# include <elf.h>

‎Tools/build/regen-rust-wrapper-h.py‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

0 commit comments

Comments
 (0)