diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 014dab1d2b58be..0e00ffb3bd2d66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,10 @@ repos: name: Run Ruff (format) on Tools/build/check_warnings.py args: [--check, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--check, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.9.0 diff --git a/Include/Python.h b/Include/Python.h index 261b4d316bdf85..78083bbf31db75 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -78,7 +78,7 @@ __pragma(warning(disable: 4201)) #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "pylock.h" +#include "cpython/pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" @@ -105,7 +105,7 @@ __pragma(warning(disable: 4201)) #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "monitoring.h" +#include "cpython/monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h new file mode 100644 index 00000000000000..6c1f7f96b6a2e8 --- /dev/null +++ b/Include/cpython/marshal.h @@ -0,0 +1,17 @@ +#ifndef _Py_CPYTHON_MARSHAL_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); + +#define Py_MARSHAL_VERSION 5 + +PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); +PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index ce92942404c9f7..5094c8c23ae32b 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -1,7 +1,13 @@ -#ifndef Py_CPYTHON_MONITORING_H -# error "this header file must not be included directly" +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif +// There is currently no limited API for monitoring + + /* Local events. * These require bytecode instrumentation */ @@ -267,3 +273,9 @@ PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike } #undef _PYMONITORING_IF_ACTIVE + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_MONITORING_H diff --git a/Include/cpython/pylock.h b/Include/cpython/pylock.h index 63886fca28eae2..460ac2c9f80e81 100644 --- a/Include/cpython/pylock.h +++ b/Include/cpython/pylock.h @@ -1,7 +1,11 @@ -#ifndef Py_CPYTHON_LOCK_H -# error "this header file must not be included directly" +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif + #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 @@ -72,3 +76,10 @@ _PyMutex_IsLocked(PyMutex *m) return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } #define PyMutex_IsLocked _PyMutex_IsLocked + + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_LOCK_H diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h new file mode 100644 index 00000000000000..4c3ea1facebc4e --- /dev/null +++ b/Include/cpython/sliceobject.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_SLICEOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Slice object interface */ + +/* +A slice object containing start, stop, and step data members (the +names are from range). After much talk with Guido, it was decided to +let these be any arbitrary python type. Py_None stands for omitted values. +*/ +typedef struct { + PyObject_HEAD + PyObject *start, *stop, *step; /* not NULL */ +} PySliceObject; + +PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); +PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, + PyObject **start_ptr, PyObject **stop_ptr, + PyObject **step_ptr); diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h new file mode 100644 index 00000000000000..328fbe86143b02 --- /dev/null +++ b/Include/cpython/structseq.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_STRUCTSEQ_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, + PyStructSequence_Desc *desc); +PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, + PyStructSequence_Desc *desc); + +typedef PyTupleObject PyStructSequence; +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0429..2ccb112b40c752 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -1,31 +1,18 @@ - /* Interface for marshal.c */ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); -PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); - -#define Py_MARSHAL_VERSION 5 - -PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); -PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_MARSHAL_H +# include "cpython/marshal.h" +# undef _Py_CPYTHON_MARSHAL_H +#endif #ifdef __cplusplus } #endif - -#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/monitoring.h b/Include/monitoring.h deleted file mode 100644 index 985f7f230e44e3..00000000000000 --- a/Include/monitoring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_MONITORING_H -#define Py_MONITORING_H -#ifdef __cplusplus -extern "C" { -#endif - -// There is currently no limited API for monitoring - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MONITORING_H -# include "cpython/monitoring.h" -# undef Py_CPYTHON_MONITORING_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_MONITORING_H */ diff --git a/Include/pylock.h b/Include/pylock.h deleted file mode 100644 index 1939ef269d3090..00000000000000 --- a/Include/pylock.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_LOCK_H -#define Py_LOCK_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_LOCK_H -# include "cpython/pylock.h" -# undef Py_CPYTHON_LOCK_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_LOCK_H */ diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 35e2ea254ca80a..00c70a6e911b41 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -16,19 +16,6 @@ PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ /* Slice object interface */ -/* - -A slice object containing start, stop, and step data members (the -names are from range). After much talk with Guido, it was decided to -let these be any arbitrary python type. Py_None stands for omitted values. -*/ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ -} PySliceObject; -#endif - PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; @@ -36,12 +23,6 @@ PyAPI_DATA(PyTypeObject) PyEllipsis_Type; PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); -#endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); Py_DEPRECATED(3.7) @@ -63,6 +44,12 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t step); #endif +#ifndef Py_LIMITED_API +# define Py_CPYTHON_SLICEOBJECT_H +# include "cpython/sliceobject.h" +# undef Py_CPYTHON_SLICEOBJECT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/structseq.h b/Include/structseq.h index 29e24fee54e613..e52d6188030af9 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -21,12 +21,6 @@ typedef struct PyStructSequence_Desc { PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, - PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); -#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); @@ -35,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -typedef PyTupleObject PyStructSequence; -#define PyStructSequence_SET_ITEM PyStructSequence_SetItem -#define PyStructSequence_GET_ITEM PyStructSequence_GetItem +# define Py_CPYTHON_STRUCTSEQ_H +# include "cpython/structseq.h" +# undef Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Makefile.pre.in b/Makefile.pre.in index 244e25c348f5a4..987d55a9bdbd3b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1191,14 +1191,12 @@ PYTHON_HEADERS= \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/pylock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1271,6 +1269,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pylock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/marshal.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ $(srcdir)/Include/cpython/modsupport.h \ @@ -1296,6 +1295,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sliceobject.h \ + $(srcdir)/Include/cpython/structseq.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index e041943492d61f..3f99836c1ebd83 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -1,8 +1,6 @@ #include "parts.h" #include "util.h" -#include "monitoring.h" - #define Py_BUILD_CORE #include "internal/pycore_instruments.h" diff --git a/PCbuild/field3.py b/PCbuild/field3.py deleted file mode 100644 index edcbe36ae083ba..00000000000000 --- a/PCbuild/field3.py +++ /dev/null @@ -1,35 +0,0 @@ -# An absurd workaround for the lack of arithmetic in MS's resource compiler. -# After building Python, run this, then paste the output into the appropriate -# part of PC\python_nt.rc. -# Example output: -# -# * For 2.3a0, -# * PY_MICRO_VERSION = 0 -# * PY_RELEASE_LEVEL = 'alpha' = 0xA -# * PY_RELEASE_SERIAL = 1 -# * -# * and 0*1000 + 10*10 + 1 = 101. -# */ -# #define FIELD3 101 - -import sys - -major, minor, micro, level, serial = sys.version_info -levelnum = {'alpha': 0xA, - 'beta': 0xB, - 'candidate': 0xC, - 'final': 0xF, - }[level] -string = sys.version.split()[0] # like '2.3a0' - -print(" * For %s," % string) -print(" * PY_MICRO_VERSION = %d" % micro) -print(" * PY_RELEASE_LEVEL = %r = %s" % (level, hex(levelnum))) -print(" * PY_RELEASE_SERIAL = %d" % serial) -print(" *") - -field3 = micro * 1000 + levelnum * 10 + serial - -print(" * and %d*1000 + %d*10 + %d = %d" % (micro, levelnum, serial, field3)) -print(" */") -print("#define FIELD3", field3) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7544d3a9fb122..71f508a7e8b5fb 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -168,9 +168,11 @@ + + @@ -190,6 +192,8 @@ + + @@ -332,7 +336,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1868b222f18534..547e9ee1abf370 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -120,9 +120,6 @@ Include - - Include - Include @@ -489,9 +486,15 @@ Include\cpython + + Include\cpython + Include\cpython + + Include\cpython + Include\cpython diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 00000000000000..aabcf8dc4f502e --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,28 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long + "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` +] diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 202dd298199b56..fdf3142c0a3b1a 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -33,7 +33,9 @@ PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode( + "utf-8" +) def updated_env(updates={}): @@ -45,7 +47,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -79,7 +83,11 @@ def wrapper(context): terminal_width = 80 print("⎯" * terminal_width) print("📁", working_dir) - if clean_ok and getattr(context, "clean", False) and working_dir.exists(): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -128,7 +136,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}" + ) return binary @@ -158,7 +168,8 @@ def make_build_python(context, working_dir): cmd = [ binary, "-c", - "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", ] version = subprocess.check_output(cmd, encoding="utf-8").strip() @@ -173,7 +184,9 @@ def check_shasum(file: str, expected_shasum: str): def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): - with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete_on_close=False) as tmp_file: + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", delete_on_close=False + ) as tmp_file: with urlopen(url) as response: shutil.copyfileobj(response, tmp_file) tmp_file.close() @@ -186,7 +199,11 @@ def make_emscripten_libffi(context, working_dir): ver = "3.4.6" libffi_dir = working_dir / f"libffi-{ver}" shutil.rmtree(libffi_dir, ignore_errors=True) - download_and_unpack(working_dir, f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e") + download_and_unpack( + working_dir, + f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", + "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + ) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], env=updated_env({"PREFIX": PREFIX_DIR}), @@ -200,7 +217,11 @@ def make_mpdec(context, working_dir): ver = "4.0.1" mpdec_dir = working_dir / f"mpdecimal-{ver}" shutil.rmtree(mpdec_dir, ignore_errors=True) - download_and_unpack(working_dir, f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8") + download_and_unpack( + working_dir, + f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", + "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", + ) call( [ "emconfigure", @@ -214,10 +235,7 @@ def make_mpdec(context, working_dir): quiet=context.quiet, ) call( - [ - "make", - "install" - ], + ["make", "install"], cwd=mpdec_dir, quiet=context.quiet, ) @@ -226,17 +244,15 @@ def make_mpdec(context, working_dir): @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" - config_site = os.fsdecode( - EMSCRIPTEN_DIR / "config.site-wasm32-emscripten" - ) + config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") emscripten_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert ( - len(lib_dirs) == 1 - ), f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -290,7 +306,9 @@ def configure_emscripten_python(context, working_dir): quiet=context.quiet, ) - shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") + shutil.copy( + EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" + ) node_entry = working_dir / "node_entry.mjs" exec_script = working_dir / "python.sh" @@ -383,13 +401,15 @@ def main(): subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the " "build Python" + "configure-build-python", help="Run `configure` for the build Python" ) make_mpdec_cmd = subcommands.add_parser( - "make-mpdec", help="Clone mpdec repo, configure and build it for emscripten" + "make-mpdec", + help="Clone mpdec repo, configure and build it for emscripten", ) make_libffi_cmd = subcommands.add_parser( - "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + "make-libffi", + help="Clone libffi repo, configure and build it for emscripten", ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" @@ -457,7 +477,11 @@ def main(): if not context.subcommand: # No command provided, display help and exit - print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) + print( + "Expected one of", + ", ".join(sorted(dispatch.keys())), + file=sys.stderr, + ) parser.print_help(sys.stderr) sys.exit(1) dispatch[context.subcommand](context) diff --git a/Tools/wasm/emscripten/prepare_external_wasm.py b/Tools/wasm/emscripten/prepare_external_wasm.py index 960e5aefd24eb5..1b0a9de4b1fe8d 100644 --- a/Tools/wasm/emscripten/prepare_external_wasm.py +++ b/Tools/wasm/emscripten/prepare_external_wasm.py @@ -19,6 +19,7 @@ }}); """ + def prepare_wasm(input_file, output_file, function_name): # Read the compiled WASM as binary and convert to hex wasm_bytes = Path(input_file).read_bytes() @@ -31,9 +32,7 @@ def prepare_wasm(input_file, output_file, function_name): ) Path(output_file).write_text(js_content) - print( - f"Successfully compiled {input_file} and generated {output_file}" - ) + print(f"Successfully compiled {input_file} and generated {output_file}") return 0 diff --git a/Tools/wasm/emscripten/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py index 78da913ff6059b..90f318f319a9f1 100755 --- a/Tools/wasm/emscripten/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -27,7 +27,9 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +WASM_STDLIB = ( + WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +) WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" diff --git a/Tools/wasm/emscripten/web_example/server.py b/Tools/wasm/emscripten/web_example/server.py index 768e6f84e07798..f2e6ed56c6bcff 100755 --- a/Tools/wasm/emscripten/web_example/server.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -6,10 +6,16 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index b49b27cbbbe66e..af55e03d10f754 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,10 +1,12 @@ -if __name__ == "__main__": +if __name__ == "__main__": import pathlib import runpy import sys - print("⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " - "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", - file=sys.stderr) + print( + "⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", + file=sys.stderr, + ) runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 973d78caa0849e..a0658cb351a86f 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -4,6 +4,7 @@ import contextlib import functools import os + try: from os import process_cpu_count as cpu_count except ImportError: @@ -17,15 +18,19 @@ CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent -assert (CHECKOUT / "configure").is_file(), "Please update the location of the file" +assert (CHECKOUT / "configure").is_file(), ( + "Please update the location of the file" +) CROSS_BUILD_DIR = CHECKOUT / "cross-build" # Build platform can also be found via `config.guess`. BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" - "# Required to statically build extension modules.").encode("utf-8") +LOCAL_SETUP_MARKER = ( + "# Generated by Tools/wasm/wasi .\n" + "# Required to statically build extension modules." +).encode("utf-8") WASI_SDK_VERSION = 24 @@ -42,7 +47,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -63,6 +70,7 @@ def updated_env(updates={}): def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" + def decorator(func): @functools.wraps(func) def wrapper(context): @@ -71,16 +79,20 @@ def wrapper(context): if callable(working_dir): working_dir = working_dir(context) try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) except subprocess.CalledProcessError: terminal_width = 80 else: terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -110,11 +122,14 @@ def call(command, *, context=None, quiet=False, logdir=None, **kwargs): stdout = None stderr = None else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - dir=logdir, - prefix="cpython-wasi-", - suffix=".log") + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) stderr = subprocess.STDOUT print(f"📝 Logging output to {stdout.name} (--quiet)...") @@ -127,8 +142,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) return binary @@ -136,9 +152,11 @@ def build_python_path(): def build_python_is_pydebug(): """Find out if the build Python is a pydebug build.""" test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run([build_python_path(), "-c", test], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + result = subprocess.run( + [build_python_path(), "-c", test], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) return bool(result.returncode) @@ -154,7 +172,7 @@ def configure_build_python(context, working_dir): print(f"📝 Creating {LOCAL_SETUP} ...") LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] if context.args: configure.extend(context.args) @@ -164,13 +182,15 @@ def configure_build_python(context, working_dir): @subdir(BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - context=context) + call(["make", "--jobs", str(cpu_count()), "all"], context=context) binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] version = subprocess.check_output(cmd, encoding="utf-8").strip() print(f"🎉 {binary} {version}") @@ -188,8 +208,11 @@ def find_wasi_sdk(): # Starting with WASI SDK 23, the tarballs went from containing a directory named # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [path for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") - if path.is_dir()] + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + if path.is_dir() + ] if len(potential_sdks) == 1: return potential_sdks[0] elif (default_path := opt_path / "wasi-sdk").is_dir(): @@ -200,8 +223,13 @@ def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) @@ -212,16 +240,20 @@ def wasi_sdk_env(context): env["PKG_CONFIG_PATH"] = "" env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) env["WASI_SYSROOT"] = os.fsdecode(sysroot) - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) return env @@ -230,54 +262,70 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") - - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi") + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi" + ) wasi_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) python_version = lib_dir.rpartition("-")[-1] - sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", - "PYTHON_WASM": working_dir / "python.wasm"} + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + } # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. if WASMTIME_HOST_RUNNER_VAR in context.host_runner: if wasmtime := shutil.which("wasmtime"): args[WASMTIME_VAR_NAME] = wasmtime else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={BUILD_DIR.name}", - f"--with-build-python={build_python}"] + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] if build_python_is_pydebug(): configure.append("--with-pydebug") if context.args: configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" @@ -291,9 +339,11 @@ def configure_wasi_python(context, working_dir): @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context) + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) exec_script = working_dir / "python.sh" call([exec_script, "--version"], quiet=False) @@ -305,11 +355,16 @@ def make_wasi_python(context, working_dir): def build_all(context): """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] + steps = [ + configure_build_python, + make_build_python, + configure_wasi_python, + make_wasi_python, + ] for step in steps: step(context) + def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -324,76 +379,113 @@ def clean_contents(context): def main(): default_host_triple = "wasm32-wasip1" default_wasi_sdk = find_wasi_sdk() - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) default_logdir = pathlib.Path(tempfile.gettempdir()) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") - subcommand.add_argument("--logdir", type=pathlib.Path, default=default_logdir, - help="Directory to store log files; " - f"defaults to {default_logdir}") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=default_wasi_sdk, - help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host; defaults to " - f"`{default_host_runner}`") + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", - default=default_host_triple, - help="The target triple for the WASI host build; " - f"defaults to {default_host_triple}") + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) context = parser.parse_args() context.init_dir = pathlib.Path().absolute() - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all, + "clean": clean_contents, + } dispatch[context.subcommand](context) -if __name__ == "__main__": +if __name__ == "__main__": main()