diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 60539fd94..be916e2d7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -46,6 +46,7 @@ body: attributes: label: PyMuPDF version options: + - 1.28.0 - 1.27.2.3 - 1.27.2.2 - 1.27.2 @@ -87,6 +88,5 @@ body: - "3.12" - "3.11" - "3.10" - - "3.9" validations: required: true diff --git a/.github/workflows/test_quick.yml b/.github/workflows/test_quick.yml index 590b62b5a..505db2f5c 100644 --- a/.github/workflows/test_quick.yml +++ b/.github/workflows/test_quick.yml @@ -43,4 +43,4 @@ jobs: env: PYMUPDF_test_args: ${{inputs.args}} run: - python scripts/test.py build test -m 'git:--branch 1.27.x https://github.com/ArtifexSoftware/mupdf.git' -a PYMUPDF_test_args + python scripts/test.py build test -m 'git:--branch 1.28.x https://github.com/ArtifexSoftware/mupdf.git' -a PYMUPDF_test_args diff --git a/changes.txt b/changes.txt index b5f1db4e7..a88ca9e0a 100644 --- a/changes.txt +++ b/changes.txt @@ -2,15 +2,18 @@ Change Log ========== +**Changes in version 1.28.0** + Fixed issues: -* **Fixed** `4423 `_: pymupdf.mupdf.FzErrorFormat: code=7: cannot find object in xref error encountered after version 1.25.3 * **Fixed** `4114 `_: ComboBox choice_values full of empty strings despite PDF having valid choices. -* **Fixed** `5001 `_: Formulae incorrectly rendered as black boxes +* **Fixed** `4423 `_: pymupdf.mupdf.FzErrorFormat: code=7: cannot find object in xref error encountered after version 1.25.3 * **Fixed** `4950 `_: remove_rotation() raises ValueError on widgets with empty/infinite rects +* **Fixed** `5001 `_: Formulae incorrectly rendered as black boxes Other: +* Use MuPDF-1.28.0-rc2. * pymupdf.Document.__init__(): new arg `archive` to support documents with archives. * pymupdf.Document.convert_to_pdf(): also generate links. * Fixed MacOS x64 platform tag - used to be macosx_10_9_x86_64, but we actually diff --git a/scripts/autovenv.py b/scripts/autovenv.py new file mode 100644 index 000000000..f3a39ab4b --- /dev/null +++ b/scripts/autovenv.py @@ -0,0 +1,196 @@ +''' +Automatic creation/use of a venv. + +Example usage: + + import autovenv + autovenv.enter(create=3, packages=['foopackage', 'barpackage']) + import foomodule + import barmodule + ... +''' + +import os +import platform +import shlex +import shutil +import subprocess +import sys +import sysconfig +import time + + +def log(text, verbose, t0): + debug = False + if debug or verbose: + text = f'autovenv [+{time.time()-t0:.1f}]: {text}' + if verbose: + print(text, flush=1) + if debug: + with open('autoenv-env-log.txt', 'a') as f: + print(text, file=f) + + +def run(command, verbose, t0, check=1, batch=True): + log(f'Running {batch=}: {command}', verbose, t0) + if batch: + def write_out(text): + for line in text.split('\n'): + log(line, verbose=True, t0=t0) + try: # pylint: disable=no-else-raise + cp = subprocess.run(command, shell=1, check=check, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except subprocess.SubprocessError as e: + text = e.stdout # pylint: disable=no-member + # Docs say that e.stdout is always bytes, but doesn't seem to be the case? + if isinstance(text, bytes): + text = text.decode('utf8', errors='replace') + write_out(text) + raise + else: + text = cp.stdout + write_out(text) + return cp + else: + cp = subprocess.run(command, shell=1, check=check) + return cp + + +def gil(): + Py_GIL_DISABLED = sysconfig.get_config_var('Py_GIL_DISABLED') + #print(f'{Py_GIL_DISABLED=}') + if Py_GIL_DISABLED==1: + gil_enabled = sys._is_gil_enabled() # pylint:disable=protected-access + #print(f'{gil_enabled=}') + if not gil_enabled: + return False + return True + + +def bits(): + return int.bit_length(sys.maxsize+1) + + +def shlex_join_windows(argv): + ''' + shlex not reliable on Windows. + Use crude quoting with "...". Seems to work. + ''' + argv2 = list() + for arg in argv: + if arg.startswith('"') and arg.endswith('"'): + assert '"' not in arg, f'Cannot quote {arg=}.' + argv2.append(arg) + else: + assert '"' not in arg, f'Cannot quote {arg=}.' + argv2.append(f'"{arg}"') + return ' '.join(argv2) + + +def enter(*, + venv_name=None, + venv_prefix='autovenv', + create=None, + packages=None, + verbose=False, + ): + ''' + Creates and re-runs inside a venv if we are not already in a venv. + + If we are already in a venv, we do nothing. + + Otherwise we do the following: + + * Create the venv if required. + * In the venv, run `pip install --upgrade` for each item in . + * Create a child process that runs `python ` inside the venv. + * Call `sys.exit()` with termination code of child process. + * We do not return. + + Args: + + venv_name: + Name of venv. If false, we use: + -- + Where is '-t' if free-thread else ''. + venv_prefix: + Used if is false. + create: + One of: + 1: Only run `python -m venv ` and install packages + if the directory does not exist. + 2: Always run `python -m venv `. + 3: Delete any existing venv and then run `python -m venv `. + If None, we use create=2. + packages: + String or list of packages, passed directly to `pip install`. + verbose: + If true we output diagnostics when running commands. + ''' + t0 = time.time() + + if create is None: + create = 2 + assert create in (1, 2, 3), f'Unrecognised {create=}, should be 1, 2 or 3.' + + if sys.prefix != sys.base_prefix: + log(f'Already in a venv, {sys.prefix=}.', verbose, t0) + return + + # We are not in a venv. + log(f'Not in a venv.', verbose, t0) + + # Set venv name. + if not venv_name: + if not venv_prefix: + venv_prefix = f'autovenv' + venv_name = f'{venv_prefix}-{platform.python_version()}{"" if gil() else "-t"}-{bits()}' + + # Create venv. + if create == 3: + # Delete any existing venv. + if os.path.isdir(venv_name): + shutil.rmtree(venv_name, ignore_errors=1) + assert not os.path.exists(venv_name) + + if create == 1 and os.path.isdir(venv_name): + # Don't recreate existing venv or install packages. + packages = list() + else: + # Create venv. + if platform.system() == 'Windows': + executable = f'"{sys.executable}"' + else: + executable = shlex.quote(sys.executable) + run(f'{executable} -m venv {venv_name}', verbose, t0) + + # Get command to enter venv. + if platform.system() == 'Windows': + # shlex not reliable on Windows. + # Use crude quoting with "...". Seems to work. + venv_enter = f'{venv_name}\\Scripts\\activate' + argv_string = '' + for arg in sys.argv: + if arg.startswith('"') and arg.endswith('"'): + argv_string += f' {arg}' + else: + assert '"' not in arg, f'Cannot handle arg containing double quote on windows: {arg=}' + argv_string += f' "{arg}"' + else: + venv_enter = f'. {venv_name}/bin/activate' + argv_string = shlex.join(sys.argv) + + # Install packages. + if isinstance(packages, str): + packages = (packages,) + for package in (packages or list()): + if package: + run(f'{venv_enter} && pip install --upgrade {shlex.quote(package)}', verbose, t0) + + # Rerun ourselves in the venv. + if platform.system() == 'Windows': + command = f'{venv_enter} && python {shlex_join_windows(sys.argv)}' + else: + command = f'{venv_enter} && python {shlex.join(sys.argv)}' + + cp = run(command, verbose, t0, check=0, batch=0) + sys.exit(cp.returncode) diff --git a/scripts/test.py b/scripts/test.py index f41a37174..46c8a6370 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -375,14 +375,13 @@ import sys import textwrap +import autovenv +autovenv.enter(packages='pipcl') + +import pipcl pymupdf_dir_abs = os.path.abspath( f'{__file__}/../..') -try: - sys.path.insert(0, f'{pymupdf_dir_abs}/src') - import pipcl -finally: - del sys.path[0] try: sys.path.insert(0, f'{pymupdf_dir_abs}/scripts') diff --git a/setup.py b/setup.py index 1dfb1fb8a..8d31cae20 100755 --- a/setup.py +++ b/setup.py @@ -720,12 +720,9 @@ def add(flavour, from_, to_): header_rel = header_abs[len(root)+1:] add('d', f'{header_abs}', f'{to_dir_d}/include/{header_rel}') - # Add a .py file containing location of MuPDF. - try: - sha, comment, diff, branch = git_info(g_root) - except Exception as e: - log(f'Failed to get git information: {e}') - sha, comment, diff, branch = (None, None, None, None) + # Add a .py file containing build-time information - location of MuPDF, + # pymupdf git info, swig version etc. + # swig = PYMUPDF_SETUP_SWIG or 'swig' swig_version_text = run(f'{swig} -version', capture=1) m = re.search('\nSWIG Version ([^\n]+)', swig_version_text) @@ -741,12 +738,10 @@ def int_or_0(text): version_p_tuple = tuple(int_or_0(i) for i in version_p.split('.')) log(f'{swig_version=}') text = '' + text += pipcl.git_info_py(g_root, check=0, prefix = 'pymupdf_git_') text += f'mupdf_location = {mupdf_location!r}\n' text += f'pymupdf_version = {version_p!r}\n' text += f'pymupdf_version_tuple = {version_p_tuple!r}\n' - text += f'pymupdf_git_sha = {sha!r}\n' - text += f'pymupdf_git_diff = {diff!r}\n' - text += f'pymupdf_git_branch = {branch!r}\n' text += f'swig_version = {swig_version!r}\n' text += f'swig_version_tuple = {swig_version_tuple!r}\n' text += f'fake_no_gil = {PYMUPDF_SETUP_FAKE_NOGIL=="1"!r}\n' @@ -830,7 +825,7 @@ def build_mupdf_windows( windows_build_tail += f'-Py_LIMITED_API_{pipcl.current_py_limited_api()}' if PYMUPDF_SETUP_FAKE_NOGIL == '1': windows_build_tail += '-nogil' - windows_build_tail += f'-x{wp.cpu.bits}-py{wp.version}' + windows_build_tail += f'-{wp.cpu.windows_name}-py{wp.version}' pipcl.log(f'{sysconfig.get_config_var("Py_GIL_DISABLED")=}') if sysconfig.get_config_var('Py_GIL_DISABLED')==1: # We are building with free-threading python. @@ -889,9 +884,8 @@ def build_mupdf_windows( def _windows_lib_directory(mupdf_local, build_type): - ret = f'{mupdf_local}/platform/win32/' - if _cpu_bits() == 64: - ret += 'x64/' + wc = pipcl.wdev.WindowsCpu() + ret = f'{mupdf_local}/platform/win32/{wc.windows_subdir}' if build_type == 'release': ret += 'Release/' elif build_type == 'debug': @@ -1253,6 +1247,7 @@ def clean(all_): shutil.rmtree(f'{path_mupdf}/platform/win32', ignore_errors=True) shutil.rmtree(f'{path_mupdf}/platform/win32/Release', ignore_errors=True) shutil.rmtree(f'{path_mupdf}/platform/win32/x64', ignore_errors=True) + shutil.rmtree(f'{path_mupdf}/platform/win32/arm64', ignore_errors=True) pipcl.log(f'Returning: {ret=}') return ret @@ -1320,9 +1315,9 @@ def sdist(): # # PyMuPDF version. -version_p = '1.27.2.3' +version_p = '1.28.0' -version_mupdf = '1.27.2' +version_mupdf = '1.28.0-rc2' # PyMuPDFb version. This is the PyMuPDF version whose PyMuPDFb wheels we will # (re)use if generating separate PyMuPDFb wheels. Though as of PyMuPDF-1.24.11 @@ -1343,7 +1338,7 @@ def get_requires_for_build_wheel(config_settings=None): return list() p = pipcl.Package( - 'PyMuPDFb', + 'pymupdfb', version_b, summary = 'Dummy PyMuPDFb wheel', description = '', @@ -1371,11 +1366,11 @@ def get_requires_for_build_wheel(config_settings=None): if 'p' in PYMUPDF_SETUP_FLAVOUR: version = version_p - name = 'PyMuPDF' + name = 'pymupdf' readme = readme_p summary = 'A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents.' if 'b' not in PYMUPDF_SETUP_FLAVOUR: - requires_dist.append(f'PyMuPDFb =={version_b}') + requires_dist.append(f'pymupdfb =={version_b}') # Create a `pymupdf` command. entry_points = textwrap.dedent(''' [console_scripts] @@ -1383,15 +1378,15 @@ def get_requires_for_build_wheel(config_settings=None): ''') elif 'b' in PYMUPDF_SETUP_FLAVOUR: version = version_b - name = 'PyMuPDFb' + name = 'pymupdfb' readme = readme_b - summary = 'MuPDF shared libraries for PyMuPDF.' + summary = 'MuPDF shared libraries for pymupdf.' tag_python = 'py3' elif 'd' in PYMUPDF_SETUP_FLAVOUR: version = version_b - name = 'PyMuPDFd' + name = 'pymupdfd' readme = readme_d - summary = 'MuPDF build-time files for PyMuPDF.' + summary = 'MuPDF build-time files for pymupdf.' tag_python = 'py3' else: assert 0, f'Unrecognised {PYMUPDF_SETUP_FLAVOUR=}.' diff --git a/src/pipcl.py b/src/pipcl.py deleted file mode 100644 index 9f3c93538..000000000 --- a/src/pipcl.py +++ /dev/null @@ -1,3598 +0,0 @@ -''' -Python packaging operations, including PEP-517 support, for use by a `setup.py` -script. - -Overview: - - The intention is to take care of as many packaging details as possible so - that setup.py contains only project-specific information, while also giving - as much flexibility as possible. - - For example we provide a function `build_extension()` that can be used - to build a SWIG extension, but we also give access to the located - compiler/linker so that a `setup.py` script can take over the details - itself. - -Doctests: - Doctest strings are provided in some comments. - - Test in the usual way with: - python -m doctest pipcl.py - - Test specific functions/classes with: - python pipcl.py --doctest run_if ... - - If no functions or classes are specified, this tests everything. - -Graal: - For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we - build for non-graal except with Graal Python's include paths and library - directory). -''' - -import base64 -import codecs -import difflib -import glob -import hashlib -import inspect -import io -import os -import pickle -import platform -import re -import shlex -import shutil -import site -import subprocess -import sys -import sysconfig -import tarfile -import textwrap -import time -import zipfile - -import wdev - - -class Package: - ''' - Our constructor takes a definition of a Python package similar to that - passed to `distutils.core.setup()` or `setuptools.setup()` (name, version, - summary etc) plus callbacks for building, getting a list of sdist - filenames, and cleaning. - - We provide methods that can be used to implement a Python package's - `setup.py` supporting PEP-517. - - We also support basic command line handling for use - with a legacy (pre-PEP-517) pip, as implemented - by legacy distutils/setuptools and described in: - https://pip.pypa.io/en/stable/reference/build-system/setup-py/ - - The file pyproject.toml must exist; this is checked if/when fn_build() is - called. - - Here is a `doctest` example of using pipcl to create a SWIG extension - module. Requires `swig`. - - Create an empty test directory: - - >>> import os - >>> import shutil - >>> shutil.rmtree('pipcl_test', ignore_errors=1) - >>> os.mkdir('pipcl_test') - - Create a `setup.py` which uses `pipcl` to define an extension module. - - >>> import textwrap - >>> with open('pipcl_test/setup.py', 'w') as f: - ... _ = f.write(textwrap.dedent(""" - ... import sys - ... import pipcl - ... - ... def build(): - ... so_leaf = pipcl.build_extension( - ... name = 'foo', - ... path_i = 'foo.i', - ... outdir = 'build', - ... source_extra = 'wibble.c', - ... ) - ... return [ - ... ('build/foo.py', 'foo/__init__.py'), - ... ('cli.py', 'foo/__main__.py'), - ... (f'build/{so_leaf}', f'foo/'), - ... ('README', '$dist-info/'), - ... (b'Hello world', 'foo/hw.txt'), - ... ] - ... - ... def sdist(): - ... return [ - ... 'pyproject.toml', - ... 'foo.i', - ... 'bar.i', - ... 'wibble.c', - ... 'setup.py', - ... 'pipcl.py', - ... 'wdev.py', - ... 'README', - ... (b'Hello word2', 'hw2.txt'), - ... ] - ... - ... p = pipcl.Package( - ... name = 'foo', - ... version = '1.2.3', - ... fn_build = build, - ... fn_sdist = sdist, - ... entry_points = ( - ... { 'console_scripts': [ - ... 'foo_cli = foo.__main__:main', - ... ], - ... }), - ... ) - ... - ... build_wheel = p.build_wheel - ... build_sdist = p.build_sdist - ... - ... # Handle old-style setup.py command-line usage: - ... if __name__ == '__main__': - ... p.handle_argv(sys.argv) - ... """)) - - Create the files required by the above `setup.py` - the SWIG `.i` input - file, the README file, and copies of `pipcl.py` and `wdev.py`. - - >>> with open('pipcl_test/foo.i', 'w') as f: - ... _ = f.write(textwrap.dedent(""" - ... %include bar.i - ... %{ - ... #include - ... #include - ... int bar(const char* text) - ... { - ... printf("bar(): text: %s\\\\n", text); - ... int len = (int) strlen(text); - ... printf("bar(): len=%i\\\\n", len); - ... fflush(stdout); - ... return len; - ... } - ... %} - ... int bar(const char* text); - ... """)) - - >>> with open('pipcl_test/bar.i', 'w') as f: - ... _ = f.write( '\\n') - - >>> with open('pipcl_test/wibble.c', 'w') as f: - ... _ = f.write( '\\n') - - >>> with open('pipcl_test/pyproject.toml', 'w') as f: - ... pass - - >>> with open('pipcl_test/README', 'w') as f: - ... _ = f.write(textwrap.dedent(""" - ... This is Foo. - ... """)) - - >>> with open('pipcl_test/cli.py', 'w') as f: - ... _ = f.write(textwrap.dedent(""" - ... def main(): - ... print('pipcl_test:main().') - ... if __name__ == '__main__': - ... main() - ... """)) - - >>> root = os.path.dirname(__file__) - >>> _ = shutil.copy2(f'{root}/pipcl.py', 'pipcl_test/pipcl.py') - >>> _ = shutil.copy2(f'{root}/wdev.py', 'pipcl_test/wdev.py') - - Use `setup.py`'s command-line interface to build and install the extension - module into root `pipcl_test/install`. - - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py --root install install', - ... shell=1, check=1) - - The actual install directory depends on `sysconfig.get_path('platlib')`: - - >>> if windows(): - ... install_dir = 'pipcl_test/install' - ... else: - ... install_dir = f'pipcl_test/install/{sysconfig.get_path("platlib").lstrip(os.sep)}' - >>> assert os.path.isfile( f'{install_dir}/foo/__init__.py') - - Create a test script which asserts that Python function call `foo.bar(s)` - returns the length of `s`, and run it with `PYTHONPATH` set to the install - directory: - - >>> with open('pipcl_test/test.py', 'w') as f: - ... _ = f.write(textwrap.dedent(""" - ... import sys - ... import foo - ... text = 'hello' - ... print(f'test.py: calling foo.bar() with text={text!r}') - ... sys.stdout.flush() - ... l = foo.bar(text) - ... print(f'test.py: foo.bar() returned: {l}') - ... assert l == len(text) - ... """)) - >>> r = subprocess.run( - ... f'{sys.executable} pipcl_test/test.py', - ... shell=1, check=1, text=1, - ... stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ... env=os.environ | dict(PYTHONPATH=install_dir), - ... ) - >>> print(r.stdout) - test.py: calling foo.bar() with text='hello' - bar(): text: hello - bar(): len=5 - test.py: foo.bar() returned: 5 - - - Check that building sdist and wheel succeeds. For now we don't attempt to - check that the sdist and wheel actually work. - - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py sdist', - ... shell=1, check=1) - - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel', - ... shell=1, check=1) - - Check that rebuild does nothing. - - >>> t0 = os.path.getmtime('pipcl_test/build/foo.py') - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel', - ... shell=1, check=1) - >>> t = os.path.getmtime('pipcl_test/build/foo.py') - >>> assert t == t0 - - Check that touching bar.i forces rebuild. - - >>> os.utime('pipcl_test/bar.i') - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel', - ... shell=1, check=1) - >>> t = os.path.getmtime('pipcl_test/build/foo.py') - >>> assert t > t0 - - Check that touching foo.i.cpp does not run swig, but does recompile/link. - - >>> t0 = time.time() - >>> os.utime('pipcl_test/build/foo.i.cpp') - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel', - ... shell=1, check=1) - >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0 - >>> so = glob.glob('pipcl_test/build/*.so') - >>> assert len(so) == 1 - >>> so = so[0] - >>> assert os.path.getmtime(so) > t0 - - Check that touching wibble.c does not run swig, but does recompile/link. - - >>> t0 = time.time() - >>> os.utime('pipcl_test/wibble.c') - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel', - ... shell=1, check=1) - >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0 - >>> so = glob.glob('pipcl_test/build/*.so') - >>> assert len(so) == 1 - >>> so = so[0] - >>> assert os.path.getmtime(so) > t0 - - Check `entry_points` causes creation of command `foo_cli` when we install - from our wheel using pip. [As of 2024-02-24 using pipcl's CLI interface - directly with `setup.py install` does not support entry points.] - - >>> print('Creating venv.', file=sys.stderr) - >>> _ = subprocess.run( - ... f'cd pipcl_test && {sys.executable} -m venv pylocal', - ... shell=1, check=1) - - >>> print('Installing from wheel into venv using pip.', file=sys.stderr) - >>> _ = subprocess.run( - ... f'. pipcl_test/pylocal/bin/activate && pip install pipcl_test/dist/*.whl', - ... shell=1, check=1) - - >>> print('Running foo_cli.', file=sys.stderr) - >>> _ = subprocess.run( - ... f'. pipcl_test/pylocal/bin/activate && foo_cli', - ... shell=1, check=1) - - Wheels and sdists - - Wheels: - We generate wheels according to: - https://packaging.python.org/specifications/binary-distribution-format/ - - * `{name}-{version}.dist-info/RECORD` uses sha256 hashes. - * We do not generate other `RECORD*` files such as - `RECORD.jws` or `RECORD.p7s`. - * `{name}-{version}.dist-info/WHEEL` has: - - * `Wheel-Version: 1.0` - * `Root-Is-Purelib: false` - * No support for signed wheels. - - Sdists: - We generate sdist's according to: - https://packaging.python.org/specifications/source-distribution-format/ - ''' - def __init__(self, - name, - version, - *, - platform = None, - supported_platform = None, - summary = None, - description = None, - description_content_type = None, - keywords = None, - home_page = None, - download_url = None, - author = None, - author_email = None, - maintainer = None, - maintainer_email = None, - license = None, - classifier = None, - requires_dist = None, - requires_python = None, - requires_external = None, - project_url = None, - provides_extra = None, - - entry_points = None, - - root = None, - fn_build = None, - fn_clean = None, - fn_sdist = None, - tag_python = None, - tag_abi = None, - tag_platform = None, - py_limited_api = None, - - wheel_compression = zipfile.ZIP_DEFLATED, - wheel_compresslevel = None, - ): - ''' - The initial args before `entry_points` define the - package metadata and closely follow the definitions in: - https://packaging.python.org/specifications/core-metadata/ - - Args: - - name: - Used for metadata `Name`. - A string, the name of the Python package. - version: - Used for metadata `Version`. - A string, the version of the Python package. Also see PEP-440 - `Version Identification and Dependency Specification`. - platform: - Used for metadata `Platform`. - A string or list of strings. - supported_platform: - Used for metadata `Supported-Platform`. - A string or list of strings. - summary: - Used for metadata `Summary`. - A string, short description of the package. - description: - Used for metadata `Description`. - A string. If contains newlines, a detailed description of the - package. Otherwise the path of a file containing the detailed - description of the package. - description_content_type: - Used for metadata `Description-Content-Type`. - A string describing markup of `description` arg. For example - `text/markdown; variant=GFM`. - keywords: - Used for metadata `Keywords`. - A string containing comma-separated keywords. - home_page: - Used for metadata `Home-page`. - URL of home page. - download_url: - Used for metadata `Download-URL`. - Where this version can be downloaded from. - author: - Used for metadata `Author`. - Author. - author_email: - Used for metadata `Author-email`. - Author email. - maintainer: - Used for metadata `Maintainer`. - Maintainer. - maintainer_email: - Used for metadata `Maintainer-email`. - Maintainer email. - license: - Used for metadata `License`. - A string containing the license text. Written into metadata - file `COPYING`. Is also written into metadata itself if not - multi-line. - classifier: - Used for metadata `Classifier`. - A string or list of strings. Also see: - - * https://pypi.org/pypi?%3Aaction=list_classifiers - * https://pypi.org/classifiers/ - - requires_dist: - Used for metadata `Requires-Dist`. - A string or list of strings, Python packages required - at runtime. None items are ignored. - requires_python: - Used for metadata `Requires-Python`. - A string or list of strings. - requires_external: - Used for metadata `Requires-External`. - A string or list of strings. - project_url: - Used for metadata `Project-URL`. - A string or list of strings, each of the form: `{name}, - {url}`. - provides_extra: - Used for metadata `Provides-Extra`. - A string or list of strings. - - entry_points: - String or dict specifying *.dist-info/entry_points.txt, for - example: - - ``` - [console_scripts] - foo_cli = foo.__main__:main - ``` - - or: - - { 'console_scripts': [ - 'foo_cli = foo.__main__:main', - ], - } - - See: https://packaging.python.org/en/latest/specifications/entry-points/ - - root: - Root of package, defaults to current directory. - - fn_build: - A function taking no args, or a single `config_settings` dict - arg (as described in PEP-517), that builds the package. - - Should return a list of items; each item should be a tuple - `(from_, to_)`, or a single string `path` which is treated as - the tuple `(path, path)`. - - `from_` can be a string or a `bytes`. If a string it should - be the path to a file; a relative path is treated as relative - to `root`. If a `bytes` it is the contents of the file to be - added. - - `to_` identifies what the file should be called within a wheel - or when installing. If `to_` is empty or `/` we set it to the - leaf of `from_` (`from_` must not be a `bytes`) - i.e. we place - the file in the root directory of the wheel; otherwise if - `to_` ends with `/` the leaf of `from_` is appended to it (and - `from_` must not be a `bytes`). - - Initial `$dist-info/` in `_to` is replaced by - `{name}-{version}.dist-info/`; this is useful for license files - etc. - - Initial `$data/` in `_to` is replaced by - `{name}-{version}.data/`. We do not enforce particular - subdirectories, instead it is up to `fn_build()` to specify - specific subdirectories such as `purelib`, `headers`, - `scripts`, `data` etc. - - If we are building a wheel (e.g. `python setup.py bdist_wheel`, - or PEP-517 pip calls `self.build_wheel()`), we add file `from_` - to the wheel archive with name `to_`. - - If we are installing (e.g. `install` command in - the argv passed to `self.handle_argv()`), then - we copy `from_` to `{sitepackages}/{to_}`, where - `sitepackages` is the installation directory, the - default being `sysconfig.get_path('platlib')` e.g. - `myvenv/lib/python3.9/site-packages/`. - - When calling this function, we assert that the file - pyproject.toml exists in the current directory. (We do this - here rather than in pipcl.Package's constructor, as otherwise - importing setup.py from non-package-related code could fail.) - - fn_clean: - A function taking a single arg `all_` that cleans generated - files. `all_` is true iff `--all` is in argv. - - For safety and convenience, can also returns a list of - files/directory paths to be deleted. Relative paths are - interpreted as relative to `root`. All paths are asserted to be - within `root`. - - fn_sdist: - A function taking no args, or a single `config_settings` dict - arg (as described in PEP517), that returns a list of items to - be copied into the sdist. The list should be in the same format - as returned by `fn_build`. - - It can be convenient to use `pipcl.git_items()`. - - The specification for sdists requires that the list contains - `pyproject.toml`; we enforce this with a Python assert. - - tag_python: - First element of wheel tag defined in PEP-425. If None we use - `cp{version}`. - - For example if code works with any Python version, one can use - 'py3'. - - tag_abi: - Second element of wheel tag defined in PEP-425. If None we use - `none`. - - tag_platform: - Third element of wheel tag defined in PEP-425. Default - is `os.environ('AUDITWHEEL_PLAT')` if set, otherwise - derived from `sysconfig.get_platform()` (was - `setuptools.distutils.util.get_platform(), before that - `distutils.util.get_platform()` as specified in the PEP), e.g. - `openbsd_7_0_amd64`. - - For pure python packages use: `tag_platform=any` - - py_limited_api: - If true we build wheels that use the Python Limited API. We use - the version of `sys.executable` to define `Py_LIMITED_API` when - compiling extensions, and use ABI tag `abi3` in the wheel name - if argument `tag_abi` is None. - - wheel_compression: - Used as `zipfile.ZipFile()`'s `compression` parameter when - creating wheels. - - wheel_compresslevel: - Used as `zipfile.ZipFile()`'s `compresslevel` parameter when - creating wheels. - - Occurrences of `None` in lists are ignored. - ''' - assert name - assert version - - def assert_str( v): - if v is not None: - assert isinstance( v, str), f'Not a string: {v!r}' - def assert_str_or_multi( v): - if v is not None: - assert isinstance( v, (str, tuple, list)), f'Not a string, tuple or list: {v!r}' - - assert_str( name) - assert_str( version) - assert_str_or_multi( platform) - assert_str_or_multi( supported_platform) - assert_str( summary) - assert_str( description) - assert_str( description_content_type) - assert_str( keywords) - assert_str( home_page) - assert_str( download_url) - assert_str( author) - assert_str( author_email) - assert_str( maintainer) - assert_str( maintainer_email) - assert_str( license) - assert_str_or_multi( classifier) - assert_str_or_multi( requires_dist) - assert_str( requires_python) - assert_str_or_multi( requires_external) - assert_str_or_multi( project_url) - assert_str_or_multi( provides_extra) - - assert re.match('^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\\Z', name, re.IGNORECASE), ( - f'Invalid package name' - f' (https://packaging.python.org/en/latest/specifications/name-normalization/)' - f': {name!r}' - ) - - # https://packaging.python.org/en/latest/specifications/core-metadata/. - assert re.match('([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', name, re.IGNORECASE), \ - f'Bad name: {name!r}' - - _assert_version_pep_440(version) - - # https://packaging.python.org/en/latest/specifications/binary-distribution-format/ - if tag_python: - assert '-' not in tag_python - if tag_abi: - assert '-' not in tag_abi - if tag_platform: - assert '-' not in tag_platform - - self.name = name - self.version = version - self.platform = platform - self.supported_platform = supported_platform - self.summary = summary - self.description = description - self.description_content_type = description_content_type - self.keywords = keywords - self.home_page = home_page - self.download_url = download_url - self.author = author - self.author_email = author_email - self.maintainer = maintainer - self.maintainer_email = maintainer_email - self.license = license - self.classifier = classifier - self.requires_dist = requires_dist - self.requires_python = requires_python - self.requires_external = requires_external - self.project_url = project_url - self.provides_extra = provides_extra - self.entry_points = entry_points - - self.root = os.path.abspath(root if root else os.getcwd()) - self.fn_build = fn_build - self.fn_clean = fn_clean - self.fn_sdist = fn_sdist - self.tag_python_ = tag_python - self.tag_abi_ = tag_abi - self.tag_platform_ = tag_platform - self.py_limited_api = py_limited_api - - self.wheel_compression = wheel_compression - self.wheel_compresslevel = wheel_compresslevel - - # If true and we are building for graal, we set PIPCL_PYTHON_CONFIG to - # a command that will print includes/libs from graal_py's sysconfig. - # - self.graal_legacy_python_config = True - - - def build_wheel(self, - wheel_directory, - config_settings=None, - metadata_directory=None, - ): - ''' - A PEP-517 `build_wheel()` function. - - Also called by `handle_argv()` to handle the `bdist_wheel` command. - - Returns leafname of generated wheel within `wheel_directory`. - ''' - log2( - f' wheel_directory={wheel_directory!r}' - f' config_settings={config_settings!r}' - f' metadata_directory={metadata_directory!r}' - ) - - if os.environ.get('CIBUILDWHEEL') == '1': - # Don't special-case graal builds when running under cibuildwheel. - pass - elif sys.implementation.name == 'graalpy': - # We build for Graal by building a native Python wheel with Graal - # Python's include paths and library directory. We then rename the - # wheel to contain graal's tag etc. - # - log0(f'### Graal build: deferring to cpython.') - python_native = os.environ.get('PIPCL_GRAAL_PYTHON') - assert python_native, f'Graal build requires that PIPCL_GRAAL_PYTHON is set.' - env_extra = dict( - PIPCL_SYSCONFIG_PATH_include = sysconfig.get_path('include'), - PIPCL_SYSCONFIG_PATH_platinclude = sysconfig.get_path('platinclude'), - PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR = sysconfig.get_config_var('LIBDIR'), - ) - # Tell native build to run pipcl.py itself to get python-config - # information about include paths etc. - if self.graal_legacy_python_config: - env_extra['PIPCL_PYTHON_CONFIG'] = f'{python_native} {os.path.abspath(__file__)} --graal-legacy-python-config' - - # Create venv. - venv_name = os.environ.get('PIPCL_GRAAL_NATIVE_VENV') - if venv_name: - log1(f'Graal using pre-existing {venv_name=}') - else: - venv_name = 'venv-pipcl-graal-native' - run(f'{shlex.quote(python_native)} -m venv {venv_name}') - log1(f'Graal using {venv_name=}') - - newfiles = NewFiles(f'{wheel_directory}/*.whl') - run( - f'. {venv_name}/bin/activate && python setup.py --dist-dir {shlex.quote(wheel_directory)} bdist_wheel', - env_extra = env_extra, - prefix = f'pipcl.py graal {python_native}: ', - ) - wheel = newfiles.get_one() - wheel_leaf = os.path.basename(wheel) - python_major_minor = run(f'{shlex.quote(python_native)} -c "import platform; import sys; sys.stdout.write(str().join(platform.python_version_tuple()[:2]))"', capture=1) - cpabi = f'cp{python_major_minor}-abi3' - assert cpabi in wheel_leaf, f'Expected wheel to be for {cpabi=}, but {wheel=}.' - graalpy_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - log1(f'{graalpy_ext_suffix=}') - m = re.match(r'\.graalpy(\d+[^\-]*)-(\d+)', graalpy_ext_suffix) - gpver = m[1] - cpver = m[2] - graalpy_wheel_tag = f'graalpy{cpver}-graalpy{gpver}_{cpver}_native' - name = wheel_leaf.replace(cpabi, graalpy_wheel_tag) - destination = f'{wheel_directory}/{name}' - log0(f'### Graal build: copying {wheel=} to {destination=}') - # Copying results in two wheels which appears to confuse pip, showing: - # Found multiple .whl files; unspecified behaviour. Will call build_wheel. - os.rename(wheel, destination) - log1(f'Returning {name=}.') - return name - - wheel_name = self.wheel_name() - path = f'{wheel_directory}/{wheel_name}' - - # Do a build and get list of files to copy into the wheel. - # - items = list() - if self.fn_build: - items = self._call_fn_build(config_settings) - - log2(f'Creating wheel: {path}') - os.makedirs(wheel_directory, exist_ok=True) - record = _Record() - with zipfile.ZipFile(path, 'w', self.wheel_compression, self.wheel_compresslevel) as z: - - def add(from_, to_): - if isinstance(from_, str): - z.write(from_, to_) - record.add_file(from_, to_) - elif isinstance(from_, bytes): - z.writestr(to_, from_) - record.add_content(from_, to_) - else: - assert 0 - - def add_str(content, to_): - add(content.encode('utf8'), to_) - - dist_info_dir = self._dist_info_dir() - - # Add the files returned by fn_build(). - # - for item in items: - from_, (to_abs, to_rel) = self._fromto(item) - add(from_, to_rel) - - # Add -.dist-info/WHEEL. - # - add_str( - f'Wheel-Version: 1.0\n' - f'Generator: pipcl\n' - f'Root-Is-Purelib: false\n' - f'Tag: {self.wheel_tag_string()}\n' - , - f'{dist_info_dir}/WHEEL', - ) - # Add -.dist-info/METADATA. - # - add_str(self._metainfo(), f'{dist_info_dir}/METADATA') - - # Add -.dist-info/COPYING. - if self.license: - add_str(self.license, f'{dist_info_dir}/COPYING') - - # Add -.dist-info/entry_points.txt. - entry_points_text = self._entry_points_text() - if entry_points_text: - add_str(entry_points_text, f'{dist_info_dir}/entry_points.txt') - - # Update -.dist-info/RECORD. This must be last. - # - z.writestr(f'{dist_info_dir}/RECORD', record.get(f'{dist_info_dir}/RECORD')) - - st = os.stat(path) - log1( f'Have created wheel size={st.st_size:,}: {path}') - if g_verbose >= 2: - with zipfile.ZipFile(path, compression=self.wheel_compression) as z: - log2(f'Contents are:') - for zi in sorted(z.infolist(), key=lambda z: z.filename): - log2(f' {zi.file_size: 10,d} {zi.filename}') - - return os.path.basename(path) - - - def build_sdist(self, - sdist_directory, - formats, - config_settings=None, - ): - ''' - A PEP-517 `build_sdist()` function. - - Also called by `handle_argv()` to handle the `sdist` command. - - Returns leafname of generated archive within `sdist_directory`. - ''' - assert self.fn_sdist, f'fn_sdist() not provided.' - log2( - f' sdist_directory={sdist_directory!r}' - f' formats={formats!r}' - f' config_settings={config_settings!r}' - ) - if formats and formats != 'gztar': - raise Exception( f'Unsupported: formats={formats}') - items = list() - if inspect.signature(self.fn_sdist).parameters: - items = self.fn_sdist(config_settings) - else: - items = self.fn_sdist() - - prefix = f'{_normalise2(self.name)}-{self.version}' - os.makedirs(sdist_directory, exist_ok=True) - tarpath = f'{sdist_directory}/{prefix}.tar.gz' - log2(f'Creating sdist: {tarpath}') - - with tarfile.open(tarpath, 'w:gz') as tar: - - names_in_tar = list() - def check_name(name): - if name in names_in_tar: - raise Exception(f'Name specified twice: {name}') - names_in_tar.append(name) - - def add(from_, name): - check_name(name) - if isinstance(from_, str): - log2( f'Adding file: {os.path.relpath(from_)} => {name}') - tar.add( from_, f'{prefix}/{name}', recursive=False) - elif isinstance(from_, bytes): - log2( f'Adding: {name}') - ti = tarfile.TarInfo(f'{prefix}/{name}') - ti.size = len(from_) - ti.mtime = time.time() - tar.addfile(ti, io.BytesIO(from_)) - else: - assert 0 - - def add_string(text, name): - textb = text.encode('utf8') - return add(textb, name) - - found_pyproject_toml = False - for item in items: - from_, (to_abs, to_rel) = self._fromto(item) - if isinstance(from_, bytes): - add(from_, to_rel) - else: - if from_.startswith(f'{os.path.abspath(sdist_directory)}/'): - # Source files should not be inside . - assert 0, f'Path is inside sdist_directory={sdist_directory}: {from_!r}' - assert os.path.exists(from_), f'Path does not exist: {from_!r}' - assert os.path.isfile(from_), f'Path is not a file: {from_!r}' - add(from_, to_rel) - if to_rel == 'pyproject.toml': - found_pyproject_toml = True - - assert found_pyproject_toml, f'Cannot create sdist because file not specified: pyproject.toml' - - # Always add a PKG-INFO file. - add_string(self._metainfo(), 'PKG-INFO') - - if self.license: - if 'COPYING' in names_in_tar: - log2(f'Not writing .license because file already in sdist: COPYING') - else: - add_string(self.license, 'COPYING') - - log1( f'Have created sdist: {tarpath}') - return os.path.basename(tarpath) - - def wheel_tag_string(self): - ''' - Returns --. - ''' - return f'{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}' - - def tag_python(self): - ''' - Get two-digit python version, e.g. 'cp3.8' for python-3.8.6. - ''' - if self.tag_python_: - ret = self.tag_python_ - else: - ret = 'cp' + ''.join(platform.python_version().split('.')[:2]) - assert '-' not in ret - return ret - - def tag_abi(self): - ''' - ABI tag. - ''' - Py_GIL_DISABLED = sysconfig.get_config_var('Py_GIL_DISABLED') - if self.tag_abi_: - return self.tag_abi_ - elif self.py_limited_api: - assert Py_GIL_DISABLED != 1, \ - f'py_limited_api and Py_GIL_DISABLED are not supported together as of 2026-02-20, e.g. see PEP 803 and PEP 809.' - return 'abi3' - elif Py_GIL_DISABLED == 1: - ret = '' - ret += 'cp' - ret += ''.join(platform.python_version().split('.')[:2]) - ret += 't' - return ret - else: - return 'none' - - def tag_platform(self): - ''' - Find platform tag used in wheel filename. - ''' - ret = self.tag_platform_ - log0(f'From self.tag_platform_: {ret=}.') - - if not ret: - # Prefer this to PEP-425. Appears to be undocumented, - # but set in manylinux docker images and appears - # to be used by cibuildwheel and auditwheel, e.g. - # https://github.com/rapidsai/shared-action-workflows/issues/80 - ret = os.environ.get( 'AUDITWHEEL_PLAT') - log0(f'From AUDITWHEEL_PLAT: {ret=}.') - - if not ret: - # Notes: - # - # PEP-425. On Linux gives `linux_x86_64` which is rejected by - # pypi.org. - # - # On local MacOS/arm64 mac-mini have seen sysconfig.get_platform() - # unhelpfully return `macosx-10.9-universal2` if `python3` is the - # system Python /usr/bin/python3; this happens if we source `. - # /etc/profile`. - # - ret = sysconfig.get_platform() - ret = ret.replace('-', '_').replace('.', '_').lower() - log0(f'From sysconfig.get_platform(): {ret=}.') - - ret = _macos_fixup_platform_tag(ret) - - log0( f'tag_platform(): returning {ret=}.') - assert '-' not in ret - return ret - - def wheel_name(self): - ret = f'{_normalise2(self.name)}-{self.version}-{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}.whl' - assert ret.count('-') == 4, f'Expected 4 dash characters in {ret=}.' - return ret - - def wheel_name_match(self, wheel): - ''' - Returns true if `wheel` matches our wheel. We basically require the - name to be the same, except that we accept platform tags that contain - extra items (see pep-0600/), for example we return true with: - - self: foo-cp38-none-manylinux2014_x86_64.whl - wheel: foo-cp38-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - ''' - log2(f'{wheel=}') - assert wheel.endswith('.whl') - wheel2 = wheel[:-len('.whl')] - name, version, tag_python, tag_abi, tag_platform = wheel2.split('-') - - py_limited_api_compatible = False - if self.py_limited_api and tag_abi == 'abi3': - # Allow lower tag_python number. - m = re.match('cp([0-9]+)', tag_python) - tag_python_int = int(m.group(1)) - m = re.match('cp([0-9]+)', self.tag_python()) - tag_python_int_self = int(m.group(1)) - if tag_python_int <= tag_python_int_self: - # This wheel uses Python stable ABI same or older than ours, so - # we can use it. - log2(f'py_limited_api; {tag_python=} compatible with {self.tag_python()=}.') - py_limited_api_compatible = True - - log2(f'{_normalise2(self.name) == name=}') - log2(f'{self.version == version=}') - log2(f'{self.tag_python() == tag_python=} {self.tag_python()=} {tag_python=}') - log2(f'{py_limited_api_compatible=}') - log2(f'{self.tag_abi() == tag_abi=}') - log2(f'{self.tag_platform() in tag_platform.split(".")=}') - log2(f'{self.tag_platform()=}') - log2(f'{tag_platform.split(".")=}') - ret = (1 - and _normalise2(self.name) == name - and self.version == version - and (self.tag_python() == tag_python or py_limited_api_compatible) - and self.tag_abi() == tag_abi - and self.tag_platform() in tag_platform.split('.') - ) - log2(f'Returning {ret=}.') - return ret - - def _entry_points_text(self): - if self.entry_points: - if isinstance(self.entry_points, str): - return self.entry_points - ret = '' - for key, values in self.entry_points.items(): - ret += f'[{key}]\n' - for value in values: - ret += f'{value}\n' - return ret - - def _call_fn_build( self, config_settings=None): - assert self.fn_build - assert os.path.isfile('pyproject.toml'), ( - 'Cannot create package because file does not exist: pyproject.toml' - ) - log2(f'calling self.fn_build={self.fn_build}') - if inspect.signature(self.fn_build).parameters: - ret = self.fn_build(config_settings) - else: - ret = self.fn_build() - assert isinstance( ret, (list, tuple)), \ - f'Expected list/tuple from {self.fn_build} but got: {ret!r}' - - # Check that any extensions that we have built, have same - # py_limited_api value. If package is marked with py_limited_api=True - # then non-py_limited_api extensions seem to fail at runtime on - # Windows. - # - # (We could possibly allow package py_limited_api=False and extensions - # py_limited_api=True, but haven't tested this, and it seems simpler to - # be strict.) - for item in ret: - from_, (to_abs, to_rel) = self._fromto(item) - from_abs = os.path.abspath(from_) - is_py_limited_api = _extensions_to_py_limited_api.get(from_abs) - if is_py_limited_api is not None: - assert bool(self.py_limited_api) == bool(is_py_limited_api), ( - f'Extension was built with' - f' py_limited_api={is_py_limited_api} but pipcl.Package' - f' name={self.name!r} has' - f' py_limited_api={self.py_limited_api}:' - f' {from_abs!r}' - ) - - return ret - - - def _argv_clean(self, all_): - ''' - Called by `handle_argv()`. - ''' - if not self.fn_clean: - return - paths = self.fn_clean(all_) - if paths: - if isinstance(paths, str): - paths = paths, - for path in paths: - if not os.path.isabs(path): - path = ps.path.join(self.root, path) - path = os.path.abspath(path) - assert path.startswith(self.root+os.sep), \ - f'path={path!r} does not start with root={self.root+os.sep!r}' - log(f'Removing: {path}') - shutil.rmtree(path, ignore_errors=True) - - - def install(self, record_path=None, root=None): - ''' - Called by `handle_argv()` to handle `install` command.. - ''' - log2( f'{record_path=} {root=}') - - # Do a build and get list of files to install. - # - items = list() - if self.fn_build: - items = self._call_fn_build( dict()) - - root2 = install_dir(root) - log2( f'{root2=}') - - log1( f'Installing into: {root2!r}') - dist_info_dir = self._dist_info_dir() - - if not record_path: - record_path = f'{root2}/{dist_info_dir}/RECORD' - record = _Record() - - def add_file(from_, to_abs, to_rel): - os.makedirs( os.path.dirname( to_abs), exist_ok=True) - if isinstance(from_, bytes): - log2(f'Copying content into {to_abs}.') - with open(to_abs, 'wb') as f: - f.write(from_) - record.add_content(from_, to_rel) - else: - log0(f'{from_=}') - log2(f'Copying from {os.path.relpath(from_, self.root)} to {to_abs}') - shutil.copy2( from_, to_abs) - record.add_file(from_, to_rel) - - def add_str(content, to_abs, to_rel): - log2( f'Writing to: {to_abs}') - os.makedirs( os.path.dirname( to_abs), exist_ok=True) - with open( to_abs, 'w') as f: - f.write( content) - record.add_content(content, to_rel) - - for item in items: - from_, (to_abs, to_rel) = self._fromto(item) - log0(f'{from_=} {to_abs=} {to_rel=}') - to_abs2 = f'{root2}/{to_rel}' - add_file( from_, to_abs2, to_rel) - - add_str( self._metainfo(), f'{root2}/{dist_info_dir}/METADATA', f'{dist_info_dir}/METADATA') - - if self.license: - add_str( self.license, f'{root2}/{dist_info_dir}/COPYING', f'{dist_info_dir}/COPYING') - - entry_points_text = self._entry_points_text() - if entry_points_text: - add_str( - entry_points_text, - f'{root2}/{dist_info_dir}/entry_points.txt', - f'{dist_info_dir}/entry_points.txt', - ) - - log2( f'Writing to: {record_path}') - with open(record_path, 'w') as f: - f.write(record.get()) - - log2(f'Finished.') - - - def _argv_dist_info(self, root): - ''' - Called by `handle_argv()`. There doesn't seem to be any documentation - for `setup.py dist_info`, but it appears to be like `egg_info` except - it writes to a slightly different directory. - ''' - if root is None: - root = f'{normalise2(self.name)}-{self.version}.dist-info' - self._write_info(f'{root}/METADATA') - if self.license: - with open( f'{root}/COPYING', 'w') as f: - f.write( self.license) - - - def _argv_egg_info(self, egg_base): - ''' - Called by `handle_argv()`. - ''' - if egg_base is None: - egg_base = '.' - self._write_info(f'{egg_base}/.egg-info') - - - def _write_info(self, dirpath=None): - ''' - Writes egg/dist info to files in directory `dirpath` or `self.root` if - `None`. - ''' - if dirpath is None: - dirpath = self.root - log2(f'Creating files in directory {dirpath}') - os.makedirs(dirpath, exist_ok=True) - with open(os.path.join(dirpath, 'PKG-INFO'), 'w') as f: - f.write(self._metainfo()) - - # These don't seem to be required? - # - #with open(os.path.join(dirpath, 'SOURCES.txt', 'w') as f: - # pass - #with open(os.path.join(dirpath, 'dependency_links.txt', 'w') as f: - # pass - #with open(os.path.join(dirpath, 'top_level.txt', 'w') as f: - # f.write(f'{self.name}\n') - #with open(os.path.join(dirpath, 'METADATA', 'w') as f: - # f.write(self._metainfo()) - - - def handle_argv(self, argv): - ''' - Attempt to handles old-style (pre PEP-517) command line passed by - old releases of pip to a `setup.py` script, and manual running of - `setup.py`. - - This is partial support at best. - ''' - global g_verbose - #log2(f'argv: {argv}') - - class ArgsRaise: - pass - - class Args: - ''' - Iterates over argv items. - ''' - def __init__( self, argv): - self.items = iter( argv) - def next( self, eof=ArgsRaise): - ''' - Returns next arg. If no more args, we return or raise an - exception if is ArgsRaise. - ''' - try: - return next( self.items) - except StopIteration: - if eof is ArgsRaise: - raise Exception('Not enough args') - return eof - - command = None - opt_all = None - opt_dist_dir = 'dist' - opt_egg_base = None - opt_formats = None - opt_install_headers = None - opt_record = None - opt_root = None - - args = Args(argv[1:]) - - while 1: - arg = args.next(None) - if arg is None: - break - - elif arg in ('-h', '--help', '--help-commands'): - log0(textwrap.dedent(''' - Usage: - [...] [...] - Commands: - bdist_wheel - Creates a wheel called - /--
.whl, where - is "dist" or as specified by --dist-dir, - and
encodes ABI and platform etc. - clean - Cleans build files. - dist_info - Creates files in -.dist-info/ or - directory specified by --egg-base. - egg_info - Creates files in .egg-info/ or directory - directory specified by --egg-base. - install - Builds and installs. Writes installation - information to if --record was - specified. - sdist - Make a source distribution: - /-.tar.gz - Options: - --all - Used by "clean". - --compile - Ignored. - --dist-dir | -d - Default is "dist". - --egg-base - Used by "egg_info". - --formats - Used by "sdist". - --install-headers - Ignored. - --python-tag - Ignored. - --record - Used by "install". - --root - Used by "install". - --single-version-externally-managed - Ignored. - --verbose -v - Extra diagnostics. - Other: - windows-vs [-y ] [-v ] [-g ] [--verbose] - Windows only; looks for matching Python. - ''')) - return - - elif arg in ('bdist_wheel', 'clean', 'dist_info', 'egg_info', 'install', 'sdist'): - assert command is None, 'Two commands specified: {command} and {arg}.' - command = arg - - elif arg in ('windows-vs', 'windows-python', 'show-sysconfig'): - assert command is None, 'Two commands specified: {command} and {arg}.' - command = arg - - elif arg == '--all': opt_all = True - elif arg == '--compile': pass - elif arg == '--dist-dir' or arg == '-d': opt_dist_dir = args.next() - elif arg == '--egg-base': opt_egg_base = args.next() - elif arg == '--formats': opt_formats = args.next() - elif arg == '--install-headers': opt_install_headers = args.next() - elif arg == '--python-tag': pass - elif arg == '--record': opt_record = args.next() - elif arg == '--root': opt_root = args.next() - elif arg == '--single-version-externally-managed': pass - elif arg == '--verbose' or arg == '-v': g_verbose += 1 - - else: - raise Exception(f'Unrecognised arg: {arg}') - - assert command, 'No command specified' - - log1(f'Handling command={command}') - if 0: pass - elif command == 'bdist_wheel': self.build_wheel(opt_dist_dir) - elif command == 'clean': self._argv_clean(opt_all) - elif command == 'dist_info': self._argv_dist_info(opt_egg_base) - elif command == 'egg_info': self._argv_egg_info(opt_egg_base) - elif command == 'install': self.install(opt_record, opt_root) - elif command == 'sdist': self.build_sdist(opt_dist_dir, opt_formats) - - elif command == 'windows-python': - version = None - while 1: - arg = args.next(None) - if arg is None: - break - elif arg == '-v': - version = args.next() - elif arg == '--verbose': - g_verbose += 1 - else: - assert 0, f'Unrecognised {arg=}' - python = wdev.WindowsPython(version=version) - print(f'Python is:\n{python.description_ml(" ")}') - - elif command == 'windows-vs': - grade = None - version = None - year = None - while 1: - arg = args.next(None) - if arg is None: - break - elif arg == '-g': - grade = args.next() - elif arg == '-v': - version = args.next() - elif arg == '-y': - year = args.next() - elif arg == '--verbose': - g_verbose += 1 - else: - assert 0, f'Unrecognised {arg=}' - vs = wdev.WindowsVS(year=year, grade=grade, version=version) - print(f'Visual Studio is:\n{vs.description_ml(" ")}') - - elif command == 'show-sysconfig': - show_sysconfig() - for mod in platform, sys: - log0(f'{mod.__name__}:') - for n in dir(mod): - if n.startswith('_'): - continue - log0(f'{mod.__name__}.{n}') - if mod is platform and n == 'uname': - continue - if mod is platform and n == 'pdb': - continue - if mod is sys and n in ('breakpointhook', 'exit'): - # We don't want to call these. - continue - v = getattr(mod, n) - if callable(v): - try: - v = v() - except Exception: - pass - else: - #print(f'{n=}', flush=1) - try: - print(f' {mod.__name__}.{n}()={v!r}') - except Exception: - print(f' Failed to print value of {mod.__name__}.{n}().') - else: - try: - print(f' {mod.__name__}.{n}={v!r}') - except Exception: - print(f' Failed to print value of {mod.__name__}.{n}.') - - else: - assert 0, f'Unrecognised command: {command}' - - log2(f'Finished handling command: {command}') - - - def __str__(self): - return ('{' - f'name={self.name!r}' - f' version={self.version!r}' - f' platform={self.platform!r}' - f' supported_platform={self.supported_platform!r}' - f' summary={self.summary!r}' - f' description={self.description!r}' - f' description_content_type={self.description_content_type!r}' - f' keywords={self.keywords!r}' - f' home_page={self.home_page!r}' - f' download_url={self.download_url!r}' - f' author={self.author!r}' - f' author_email={self.author_email!r}' - f' maintainer={self.maintainer!r}' - f' maintainer_email={self.maintainer_email!r}' - f' license={self.license!r}' - f' classifier={self.classifier!r}' - f' requires_dist={self.requires_dist!r}' - f' requires_python={self.requires_python!r}' - f' requires_external={self.requires_external!r}' - f' project_url={self.project_url!r}' - f' provides_extra={self.provides_extra!r}' - - f' root={self.root!r}' - f' fn_build={self.fn_build!r}' - f' fn_sdist={self.fn_sdist!r}' - f' fn_clean={self.fn_clean!r}' - f' tag_python={self.tag_python_!r}' - f' tag_abi={self.tag_abi_!r}' - f' tag_platform={self.tag_platform_!r}' - '}' - ) - - def _dist_info_dir( self): - return f'{_normalise2(self.name)}-{self.version}.dist-info' - - def _metainfo(self): - ''' - Returns text for `.egg-info/PKG-INFO` file, or `PKG-INFO` in an sdist - `.tar.gz` file, or `...dist-info/METADATA` in a wheel. - ''' - # 2021-04-30: Have been unable to get multiline content working on - # test.pypi.org so we currently put the description as the body after - # all the other headers. - # - ret = [''] - def add(key, value): - if value is None: - return - if isinstance( value, (tuple, list)): - for v in value: - if v is not None: - add( key, v) - return - if key == 'License' and '\n' in value: - # This is ok because we write `self.license` into - # *.dist-info/COPYING. - # - log1( f'Omitting license because contains newline(s).') - return - assert '\n' not in value, f'key={key} value contains newline: {value!r}' - if key == 'Project-URL': - assert value.count(',') == 1, f'For {key=}, should have one comma in {value!r}.' - ret[0] += f'{key}: {value}\n' - #add('Description', self.description) - add('Metadata-Version', '2.1') - - # These names are from: - # https://packaging.python.org/specifications/core-metadata/ - # - for name in ( - 'Name', - 'Version', - 'Platform', - 'Supported-Platform', - 'Summary', - 'Description-Content-Type', - 'Keywords', - 'Home-page', - 'Download-URL', - 'Author', - 'Author-email', - 'Maintainer', - 'Maintainer-email', - 'License', - 'Classifier', - 'Requires-Dist', - 'Requires-Python', - 'Requires-External', - 'Project-URL', - 'Provides-Extra', - ): - identifier = name.lower().replace( '-', '_') - add( name, getattr( self, identifier)) - - ret = ret[0] - - # Append description as the body - if self.description: - if '\n' in self.description: - description_text = self.description.strip() - else: - with open(self.description) as f: - description_text = f.read() - ret += '\n' # Empty line separates headers from body. - ret += description_text - ret += '\n' - return ret - - def _path_relative_to_root(self, path, assert_within_root=True): - ''' - Returns `(path_abs, path_rel)`, where `path_abs` is absolute path and - `path_rel` is relative to `self.root`. - - Interprets `path` as relative to `self.root` if not absolute. - - We use `os.path.realpath()` to resolve any links. - - if `assert_within_root` is true, assert-fails if `path` is not within - `self.root`. - ''' - if os.path.isabs(path): - p = path - else: - p = os.path.join(self.root, path) - p = os.path.realpath(os.path.abspath(p)) - if assert_within_root: - assert p.startswith(self.root+os.sep) or p == self.root, \ - f'Path not within root={self.root+os.sep!r}: {path=} {p=}' - p_rel = os.path.relpath(p, self.root) - return p, p_rel - - def _fromto(self, p): - ''' - Returns `(from_, (to_abs, to_rel))`. - - If `p` is a string we convert to `(p, p)`. Otherwise we assert that - `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is - str. If `from_` is a bytes it is contents of file to add, otherwise the - path of an existing file; non-absolute paths are assumed to be relative - to `self.root`. - - If `to_` is empty or `/` we set it to the leaf of `from_` (which must - be a str) - i.e. we place the file in the root directory of the wheel; - otherwise if `to_` ends with `/` we append the leaf of `from_` (which - must be a str). - - If `to_` starts with `$dist-info/`, we replace this with - `self._dist_info_dir()`. - - If `to_` starts with `$data/`, we replace this with - `{self.name}-{self.version}.data/`. - - We assert that `to_abs` is `within self.root`. - - `to_rel` is derived from the `to_abs` and is relative to self.root`. - ''' - ret = None - if isinstance(p, str): - p = p, p - assert isinstance(p, tuple) and len(p) == 2 - - from_, to_ = p - assert isinstance(from_, (str, bytes)) - assert isinstance(to_, str) - if to_ == '/' or to_ == '': - to_ = os.path.basename(from_) - elif to_.endswith('/'): - to_ += os.path.basename(from_) - prefix = '$dist-info/' - if to_.startswith( prefix): - to_ = f'{self._dist_info_dir()}/{to_[ len(prefix):]}' - prefix = '$data/' - if to_.startswith( prefix): - to_ = f'{_normalise2(self.name)}-{self.version}.data/{to_[ len(prefix):]}' - if isinstance(from_, str): - from_, _ = self._path_relative_to_root( from_, assert_within_root=False) - to_ = self._path_relative_to_root(to_) - assert isinstance(from_, (str, bytes)) - log2(f'returning {from_=} {to_=}') - return from_, to_ - -_extensions_to_py_limited_api = dict() - -def build_extension( - name, - path_i, - outdir, - *, - builddir=None, - includes=None, - defines=None, - libpaths=None, - libs=None, - optimise=True, - debug=False, - compiler_extra='', - linker_extra='', - swig=None, - cpp=True, - source_extra=None, - prerequisites_swig=None, - prerequisites_compile=None, - prerequisites_link=None, - infer_swig_includes=True, - py_limited_api=False, - ): - ''' - Builds a Python extension module using SWIG. Works on Windows, Linux, MacOS - and OpenBSD. - - On Unix, sets rpath when linking shared libraries. - - Args: - name: - Name of generated extension module. - path_i: - Path of input SWIG `.i` file. Internally we use swig to generate a - corresponding `.c` or `.cpp` file. - outdir: - Output directory for generated files: - - * `{outdir}/{name}.py` - * `{outdir}/_{name}.so` # Unix - * `{outdir}/_{name}.*.pyd` # Windows - We return the leafname of the `.so` or `.pyd` file. - builddir: - Where to put intermediate files, for example the .cpp file - generated by swig and `.d` dependency files. Default is `outdir`. - includes: - A string, or a sequence of extra include directories to be prefixed - with `-I`. - defines: - A string, or a sequence of extra preprocessor defines to be - prefixed with `-D`. - libpaths - A string, or a sequence of library paths to be prefixed with - `/LIBPATH:` on Windows or `-L` on Unix. - libs - A string, or a sequence of library names. Each item is prefixed - with `-l` on non-Windows. - optimise: - Whether to use compiler optimisations and define NDEBUG. - debug: - Whether to build with debug symbols. - compiler_extra: - Extra compiler flags. Can be None. - linker_extra: - Extra linker flags. Can be None. - swig: - Swig command; if false we use 'swig'. - cpp: - If true we tell SWIG to generate C++ code instead of C. - source_extra: - Extra source files to build into the shared library, - prerequisites_swig: - prerequisites_compile: - prerequisites_link: - - [These are mainly for use on Windows. On other systems we - automatically generate dynamic dependencies using swig/compile/link - commands' `-MD` and `-MF` args.] - - Sequences of extra input files/directories that should force - running of swig, compile or link commands if they are newer than - any existing generated SWIG `.i` file, compiled object file or - shared library file. - - If present, the first occurrence of `True` or `False` forces re-run - or no re-run. Any occurrence of None is ignored. If an item is a - directory path we look for newest file within the directory tree. - - If not a sequence, we convert into a single-item list. - - prerequisites_swig - - We use swig's -MD and -MF args to generate dynamic dependencies - automatically, so this is not usually required. - - prerequisites_compile - prerequisites_link - - On non-Windows we use cc's -MF and -MF args to generate dynamic - dependencies so this is not usually required. - infer_swig_includes: - If true, we extract `-I` and `-I ` args from - `compile_extra` (also `/I` on windows) and use them with swig so - that it can see the same header files as C/C++. This is useful - when using enviromment variables such as `CC` and `CXX` to set - `compile_extra`. - py_limited_api: - If true we build for current Python's limited API / stable ABI. - - Note that we will assert false if this extension is added to a - pipcl.Package that has a different , because - on Windows importing a non-py_limited_api extension inside a - py_limited=True package fails. - - Returns the leafname of the generated library file within `outdir`, e.g. - `_{name}.so` on Unix or `_{name}.cp311-win_amd64.pyd` on Windows. - ''' - if compiler_extra is None: - compiler_extra = '' - if linker_extra is None: - linker_extra = '' - if builddir is None: - builddir = outdir - if not swig: - swig = 'swig' - - if source_extra is None: - source_extra = list() - if isinstance(source_extra, str): - source_extra = [source_extra] - - includes_text = _flags( includes, '-I') - defines_text = _flags( defines, '-D') - libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L') - libs_text = _flags( libs, '' if windows() else '-l') - path_cpp = f'{builddir}/{os.path.basename(path_i)}' - path_cpp += '.cpp' if cpp else '.c' - os.makedirs( outdir, exist_ok=True) - - # Run SWIG. - # - if infer_swig_includes: - # Extract include flags from `compiler_extra`. - swig_includes_extra = '' - compiler_extra_items = shlex.split(compiler_extra) - i = 0 - while i < len(compiler_extra_items): - item = compiler_extra_items[i] - # Swig doesn't seem to like a space after `I`. - if item == '-I' or (windows() and item == '/I'): - swig_includes_extra += f' -I{compiler_extra_items[i+1]}' - i += 1 - elif item.startswith('-I') or (windows() and item.startswith('/I')): - swig_includes_extra += f' -I{compiler_extra_items[i][2:]}' - i += 1 - swig_includes_extra = swig_includes_extra.strip() - deps_path = f'{path_cpp}.d' - prerequisites_swig2 = _get_prerequisites( deps_path) - run_if( - f''' - {swig} - -Wall - {"-c++" if cpp else ""} - -python - -module {name} - -outdir {outdir} - -o {path_cpp} - -MD -MF {deps_path} - {includes_text} - {swig_includes_extra} - {path_i} - ''' - , - path_cpp, - path_i, - prerequisites_swig, - prerequisites_swig2, - ) - - if pyodide(): - so_suffix = '.so' - log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.') - else: - so_suffix = _so_suffix(use_so_versioning = not py_limited_api) - path_so_leaf = f'_{name}{so_suffix}' - path_so = f'{outdir}/{path_so_leaf}' - - py_limited_api2 = current_py_limited_api() if py_limited_api else None - - compiler_command, pythonflags = base_compiler(cpp=cpp) - linker_command, _ = base_linker(cpp=cpp) - # setuptools on Linux seems to use slightly different compile flags: - # - # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE - # - - general_flags = '' - if windows(): - permissive = '/permissive-' - EHsc = '/EHsc' - T = '/Tp' if cpp else '/Tc' - optimise2 = '/DNDEBUG /O2' if optimise else '/D_DEBUG' - debug2 = '/Zi' if debug else '' - py_limited_api3 = f'/DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else '' - - else: - if debug: - general_flags += '/Zi' if windows() else ' -g' - if optimise: - general_flags += ' /DNDEBUG /O2' if windows() else ' -O2 -DNDEBUG' - - py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else '' - - if windows(): - pass - elif darwin(): - # MacOS's linker does not like `-z origin`. - rpath_flag = "-Wl,-rpath,@loader_path/" - # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'. - general_flags += ' -undefined dynamic_lookup' - elif pyodide(): - # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives: - # emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags] - # wasm-ld: error: unknown -z value: origin - # - rpath_flag = "-Wl,-rpath,'$ORIGIN'" - else: - rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin" - - # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}' - # they seem to be ignored... - # - path_os = list() - - for path_source in [path_cpp] + source_extra: - path_o = f'{path_source}.obj' if windows() else f'{path_source}.o' - path_os.append(path_o) - - prerequisites_path = f'{path_o}.d' - - if windows(): - compiler_command2 = f''' - {compiler_command} - # General: - /c # Compiles without linking. - {EHsc} # Enable "Standard C++ exception handling". - - #/MD # Creates a multithreaded DLL using MSVCRT.lib. - {'/MDd' if debug else '/MD'} - - # Input/output files: - {T}{path_source} # /Tp specifies C++ source file. - /Fo{path_o} # Output file. codespell:ignore - - # Include paths: - {includes_text} - {pythonflags.includes} # Include path for Python headers. - - # Code generation: - {optimise2} - {debug2} - {permissive} # Set standard-conformance mode. - - # Diagnostics: - #/FC # Display full path of source code files passed to cl.exe in diagnostic text. - /W3 # Sets which warning level to output. /W3 is IDE default. - /diagnostics:caret # Controls the format of diagnostic messages. - /nologo # - - {defines_text} - {compiler_extra} - - {py_limited_api3} - ''' - - else: - compiler_command2 = f''' - {compiler_command} - -fPIC - {general_flags.strip()} - {pythonflags.includes} - {includes_text} - {defines_text} - -MD -MF {prerequisites_path} - -c {path_source} - -o {path_o} - {compiler_extra} - {py_limited_api3} - ''' - run_if( - compiler_command2, - path_o, - path_source, - [path_source] + _get_prerequisites(prerequisites_path), - ) - - # Link - prerequisites_path = f'{path_so}.d' - if windows(): - debug2 = '/DEBUG' if debug else '' - base, _ = os.path.splitext(path_so_leaf) - command2 = f''' - {linker_command} - /DLL # Builds a DLL. - /EXPORT:PyInit__{name} # Exports a function. - /IMPLIB:{base}.lib # Overrides the default import library name. - {libpaths_text} - {pythonflags.ldflags} - /OUT:{path_so} # Specifies the output file name. - {debug2} - /nologo - {libs_text} - {' '.join(path_os)} - {linker_extra} - ''' - elif pyodide(): - command2 = f''' - {linker_command} - -MD -MF {prerequisites_path} - -o {path_so} - {' '.join(path_os)} - {libpaths_text} - {libs_text} - {linker_extra} - {pythonflags.ldflags} - {rpath_flag} - ''' - else: - command2 = f''' - {linker_command} - -shared - {general_flags.strip()} - -MD -MF {prerequisites_path} - -o {path_so} - {' '.join(path_os)} - {libpaths_text} - {libs_text} - {linker_extra} - {pythonflags.ldflags} - {rpath_flag} - {py_limited_api3} - ''' - link_was_run = run_if( - command2, - path_so, - path_cpp, - *path_os, - *_get_prerequisites(f'{path_so}.d'), - ) - - if link_was_run and darwin(): - # We need to patch up references to shared libraries in `libs`. - sublibraries = list() - for lib in () if libs is None else libs: - for libpath in libpaths: - found = list() - for suffix in '.so', '.dylib': - path = f'{libpath}/lib{os.path.basename(lib)}{suffix}' - if os.path.exists( path): - found.append( path) - if found: - assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}' - sublibraries.append( found[0]) - break - else: - log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}') - macos_patch( path_so, *sublibraries) - - #run(f'ls -l {path_so}', check=0) - #run(f'file {path_so}', check=0) - - _extensions_to_py_limited_api[os.path.abspath(path_so)] = py_limited_api - - return path_so_leaf - - -# Functions that might be useful. -# - - -def base_compiler(vs=None, pythonflags=None, cpp=False, use_env=True): - ''' - Returns basic compiler command and PythonFlags. - - Args: - vs: - Windows only. A `wdev.WindowsVS` instance or None to use default - `wdev.WindowsVS` instance. - pythonflags: - A `pipcl.PythonFlags` instance or None to use default - `pipcl.PythonFlags` instance. - cpp: - If true we return C++ compiler command instead of C. On Windows - this has no effect - we always return `cl.exe`. - use_env: - If true we return '$CC' or '$CXX' if the corresponding - environmental variable is set (without evaluating with `getenv()` - or `os.environ`). - - Returns `(cc, pythonflags)`: - cc: - C or C++ command. On Windows this is of the form - `{vs.vcvars}&&{vs.cl}`; otherwise it is typically `cc` or `c++`. - pythonflags: - The `pythonflags` arg or a new `pipcl.PythonFlags` instance. - ''' - if not pythonflags: - pythonflags = PythonFlags() - cc = None - if use_env: - if cpp: - if os.environ.get( 'CXX'): - cc = '$CXX' - else: - if os.environ.get( 'CC'): - cc = '$CC' - if cc: - pass - elif windows(): - if not vs: - vs = wdev.WindowsVS() - cc = f'"{vs.vcvars}"&&"{vs.cl}"' - elif wasm(): - cc = 'em++' if cpp else 'emcc' - else: - cc = 'c++' if cpp else 'cc' - cc = macos_add_cross_flags( cc) - return cc, pythonflags - - -def base_linker(vs=None, pythonflags=None, cpp=False, use_env=True): - ''' - Returns basic linker command. - - Args: - vs: - Windows only. A `wdev.WindowsVS` instance or None to use default - `wdev.WindowsVS` instance. - pythonflags: - A `pipcl.PythonFlags` instance or None to use default - `pipcl.PythonFlags` instance. - cpp: - If true we return C++ linker command instead of C. On Windows this - has no effect - we always return `link.exe`. - use_env: - If true we use `os.environ['LD']` if set. - - Returns `(linker, pythonflags)`: - linker: - Linker command. On Windows this is of the form - `{vs.vcvars}&&{vs.link}`; otherwise it is typically `cc` or `c++`. - pythonflags: - The `pythonflags` arg or a new `pipcl.PythonFlags` instance. - ''' - if not pythonflags: - pythonflags = PythonFlags() - linker = None - if use_env: - if os.environ.get( 'LD'): - linker = '$LD' - if linker: - pass - elif windows(): - if not vs: - vs = wdev.WindowsVS() - linker = f'"{vs.vcvars}"&&"{vs.link}"' - elif wasm(): - linker = 'em++' if cpp else 'emcc' - else: - linker = 'c++' if cpp else 'cc' - linker = macos_add_cross_flags( linker) - return linker, pythonflags - - -def git_info( directory): - ''' - Returns `(sha, comment, diff, branch)`, all items are str or None if not - available. - - directory: - Root of git checkout. - ''' - sha, comment, diff, branch = None, None, None, None - e, out = run( - f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)', - capture=1, - check=0 - ) - if not e: - sha, _ = out.split(' ', 1) - comment, diff = _.split('\n', 1) - e, out = run( - f'cd {directory} && git rev-parse --abbrev-ref HEAD', - capture=1, - check=0 - ) - if not e: - branch = out.strip() - log1(f'git_info(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}') - return sha, comment, diff, branch - - -def git_items( directory, submodules=False): - ''' - Returns list of paths for all files known to git within a `directory`. - - Args: - directory: - Must be somewhere within a git checkout. - submodules: - If true we also include git submodules. - - Returns: - A list of paths for all files known to git within `directory`. Each - path is relative to `directory`. `directory` must be somewhere within a - git checkout. - - We run a `git ls-files` command internally. - - This function can be useful for the `fn_sdist()` callback. - ''' - command = 'cd ' + directory + ' && git ls-files' - if submodules: - command += ' --recurse-submodules' - log1(f'Running {command=}') - text = subprocess.check_output( command, shell=True) - ret = [] - for path in text.decode('utf8').strip().split( '\n'): - path2 = os.path.join(directory, path) - # Sometimes git ls-files seems to list empty/non-existent directories - # within submodules. - # - if not os.path.exists(path2): - log2(f'Ignoring git ls-files item that does not exist: {path2}') - elif os.path.isdir(path2): - log2(f'Ignoring git ls-files item that is actually a directory: {path2}') - else: - ret.append(path) - return ret - - -def git_get( - local, - *, - remote=None, - branch=None, - tag=None, - text=None, - depth=1, - env_extra=None, - update=True, - submodules=True, - ): - ''' - Creates/updates local checkout of remote repository and returns - absolute path of . - - If is set but does not start with 'git:', it is assumed to be an up - to date local checkout, and we return absolute path of without doing - any git operations. - - Args: - local: - Local directory. Created and/or updated using `git clone` and `git - fetch` etc. - remote: - Remote git repostitory, for example - 'https://github.com/ArtifexSoftware/mupdf.git'. Can be overridden - by . - branch: - Branch to use; can be overridden by . - tag: - Tag to use; can be overridden by . - text: - If None or empty: - Ignored. - - If starts with 'git:': - The remaining text should be a command-line - style string containing some or all of these args: - --branch - --tag - - These overrides , and . - Otherwise: - is assumed to be a local directory, and we simply return - it as an absolute path without doing any git operations. - - For example these all clone/update/branch master of https://foo.bar/qwerty.git to local - checkout 'foo-local': - - git_get('foo-local', remote='https://foo.bar/qwerty.git', branch='master') - git_get('foo-local', text='git:--branch master https://foo.bar/qwerty.git') - git_get('foo-local', text='git:--branch master', remote='https://foo.bar/qwerty.git') - git_get('foo-local', text='git:', branch='master', remote='https://foo.bar/qwerty.git') - depth: - Depth of local checkout when cloning and fetching, or None. - env_extra: - Dict of extra name=value environment variables to use whenever we - run git. - update: - If false we do not update existing repository. Might be useful if - testing without network access. - submodules: - If true, we clone with `--recursive --shallow-submodules` and run - `git submodule update --init --recursive` before returning. - ''' - log0(f'{remote=} {local=} {branch=} {tag=} {text=}') - - if text: - if text.startswith('git:'): - args = iter(shlex.split(text[len('git:'):])) - while 1: - try: - arg = next(args) - except StopIteration: - break - if arg == '--branch': - branch = next(args) - tag = None - elif arg == '--tag': - tag = next(args) - branch = None - else: - remote = arg - assert remote, f' unset and no remote specified in {text=}.' - assert branch or tag, f' and unset and no branch/tag specified in {text=}.' - else: - log0(f'Using local directory {text!r}.') - return os.path.abspath(text) - - assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of and ; {branch=} {tag=}.' - - depth_arg = f' --depth {depth}' if depth else '' - - def do_update(): - # This seems to pull in the entire repository. - log0(f'do_update(): attempting to update {local=}.') - # Remove any local changes. - run(f'cd {local} && git reset --hard', env_extra=env_extra) - if tag: - # `-u` avoids `fatal: Refusing to fetch into current branch`. - # Using '+' and `revs/tags/` prefix seems to avoid errors like: - # error: cannot update ref 'refs/heads/v3.16.44': - # trying to write non-commit object - # 06c4ae5fe39a03b37a25a8b95214d9f8f8a867b8 to branch - # 'refs/heads/v3.16.44' - # - run(f'cd {local} && git fetch -fuv{depth_arg} {remote} +refs/tags/{tag}:refs/tags/{tag}', env_extra=env_extra) - run(f'cd {local} && git checkout {tag}', env_extra=env_extra) - if branch: - # `-u` avoids `fatal: Refusing to fetch into current branch`. - run(f'cd {local} && git fetch -fuv{depth_arg} {remote} {branch}:{branch}', env_extra=env_extra) - run(f'cd {local} && git checkout {branch}', env_extra=env_extra) - - do_clone = True - if os.path.isdir(f'{local}/.git'): - if update: - # Try to update existing checkout. - try: - do_update() - do_clone = False - except Exception as e: - log0(f'Failed to update existing checkout {local}: {e}') - else: - do_clone = False - - if do_clone: - # No existing git checkout, so do a fresh clone. - #_fs_remove(local) - log0(f'Cloning to: {local}') - command = f'git clone --config core.longpaths=true{depth_arg}' - if submodules: - command += f' --recursive --shallow-submodules' - if branch: - command += f' -b {branch}' - if tag: - command += f' -b {tag}' - command += f' {remote} {local}' - run(command, env_extra=env_extra) - do_update() - - if submodules: - run(f'cd {local} && git submodule update --init --recursive', env_extra=env_extra) - - # Show sha of checkout. - run( f'cd {local} && git show --pretty=oneline|head -n 1', check=False) - return os.path.abspath(local) - - -def run( - command, - *, - capture=False, - check=1, - verbose=1, - env=None, - env_extra=None, - timeout=None, - caller=1, - prefix=None, - encoding=None, # System default. - errors='backslashreplace', - ): - ''' - Runs a command using `subprocess.run()`. - - Args: - command: - A string, the command to run. - - Multiple lines in `command` are treated as a single command. - - * If a line starts with `#` it is discarded. - * If a line contains ` #`, the trailing text is discarded. - - When running the command on Windows, newlines are replaced by - spaces; otherwise each line is terminated by a backslash character. - capture: - If true, we include the command's output in our return value. - check: - If true we raise an exception on error; otherwise we include the - command's returncode in our return value. - verbose: - If true we show the command. - env: - None or dict to use instead of . - env_extra: - None or dict to add to or . - timeout: - If not None, timeout in seconds; passed directly to - subprocess.run(). Note that on MacOS subprocess.run() seems to - leave processes running if timeout expires. - prefix: - String prefix for each line of output. - - If true: - * We run command with stdout=subprocess.PIPE and - stderr=subprocess.STDOUT, repetaedly reading the command's output - and writing it to stdout with . - * We do not support , which must be None. - Returns: - check capture Return - -------------------------- - false false returncode - false true (returncode, output) - true false None or raise exception - true true output or raise exception - ''' - if env is None: - env = os.environ - if env_extra: - env = env.copy() - if env_extra: - env.update(env_extra) - lines = _command_lines( command) - if verbose: - text = f'Running:' - nl = '\n ' - text += f' {nl.join(lines)}' - if env_extra: - text += f'\nwith:\n' - for k in sorted(env_extra.keys()): - text += f' {k}={shlex.quote(env_extra[k])}\n' - log1(text, caller=caller+1) - sep = ' ' if windows() else ' \\\n' - command2 = sep.join( lines) - - if prefix: - assert not timeout, f'Timeout not supported with prefix.' - child = subprocess.Popen( - command2, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding=encoding, - errors=errors, - env=env, - ) - if capture: - capture_text = '' - decoder = codecs.getincrementaldecoder(child.stdout.encoding)(errors) - line_start = True - - while 1: - raw = os.read( child.stdout.fileno(), 10000) - text = decoder.decode(raw, final=not raw) - if capture: - capture_text += text - lines = text.split('\n') - for i, line in enumerate(lines): - if line_start: - sys.stdout.write(prefix) - line_start = False - sys.stdout.write(line) - if i < len(lines) - 1: - sys.stdout.write('\n') - line_start = True - sys.stdout.flush() - if not raw: - break - if not line_start: - sys.stdout.write('\n') - e = child.wait() - if check and e: - raise subprocess.CalledProcessError(e, command2, capture_text if capture else None) - if check: - return capture_text if capture else None - else: - return (e, capture_text) if capture else e - else: - cp = subprocess.run( - command2, - shell=True, - stdout=subprocess.PIPE if capture else None, - stderr=subprocess.STDOUT if capture else None, - check=check, - encoding=encoding, - errors=errors, - env=env, - timeout=timeout, - ) - if check: - return cp.stdout if capture else None - else: - return (cp.returncode, cp.stdout) if capture else cp.returncode - - -def darwin(): - return sys.platform.startswith( 'darwin') - -def windows(): - return platform.system() == 'Windows' - -def wasm(): - return os.environ.get( 'OS') in ('wasm', 'wasm-mt') - -def pyodide(): - return os.environ.get( 'PYODIDE') == '1' - -def linux(): - return platform.system() == 'Linux' - -def openbsd(): - return platform.system() == 'OpenBSD' - - -def show_system(): - ''' - Show useful information about the system plus argv and environ. - - Omits os.environ if $PIPCL_SHOW_ENV is '0'. - ''' - def log(text): - log0(text, caller=3) - - #log(f'{__file__=}') - #log(f'{__name__=}') - log(f'{os.getcwd()=}') - log(f'{platform.machine()=}') - log(f'{platform.platform()=}') - log(f'{platform.python_implementation()=}') - log(f'{platform.python_version()=}') - log(f'{platform.system()=}') - if sys.implementation.name != 'graalpy': - log(f'{platform.uname()=}') - log(f'{sys.executable=}') - log(f'{sys.version=}') - log(f'{sys.version_info=}') - log(f'{list(sys.version_info)=}') - - log(f'{sysconfig.get_config_var("Py_GIL_DISABLED")=}') - try: - log(f'{sys._is_gil_enabled()=}') - except AttributeError: - log(f'sys._is_gil_enabled() => AttributeError') - - log(f'CPU bits: {cpu_bits()}') - - log(f'sys.argv ({len(sys.argv)}):') - for i, arg in enumerate(sys.argv): - log(f' {i}: {arg!r}') - - PIPCL_SHOW_ENV = os.environ.get('PIPCL_SHOW_ENV') - if PIPCL_SHOW_ENV == '0': - log(f'[Not showing os.environ because {PIPCL_SHOW_ENV=}.]') - else: - log(f'os.environ ({len(os.environ)}):') - for k in sorted( os.environ.keys()): - v = os.environ[ k] - if 'BEGIN OPENSSH PRIVATE KEY' in v: - # Don't show private keys. - log(f' {k} ****') - else: - log( f' {k}: {v!r}') - - -class PythonFlags: - ''' - Compile/link flags for the current python, for example the include path - needed to get `Python.h`. - - The 'PIPCL_PYTHON_CONFIG' environment variable allows to override - the location of the python-config executable. - - Members: - .includes: - String containing compiler flags for include paths. - .ldflags: - String containing linker flags for library paths. - ''' - def __init__(self): - - # Experimental detection of python flags from sysconfig.*() instead of - # python-config command. - includes_, ldflags_ = sysconfig_python_flags() - - if pyodide(): - _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR'] - _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR'] - self.includes = f'-I {_include_dir}' - self.ldflags = f'-L {_lib_dir}' - - elif 0: - - self.includes = includes_ - self.ldflags = ldflags_ - - elif windows(): - wp = wdev.WindowsPython() - self.includes = f'/I"{wp.include}"' - self.ldflags = f'/LIBPATH:"{wp.libs}"' - - elif pyodide(): - _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR'] - _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR'] - self.includes = f'-I {_include_dir}' - self.ldflags = f'-L {_lib_dir}' - - else: - python_config = os.environ.get("PIPCL_PYTHON_CONFIG") - if not python_config: - # We use python-config which appears to work better than pkg-config - # because it copes with multiple installed python's, e.g. - # manylinux_2014's /opt/python/cp*-cp*/bin/python*. - # - # But... on non-macos it seems that we should not attempt to specify - # libpython on the link command. The manylinux docker containers - # don't actually contain libpython.so, and it seems that this - # deliberate. And the link command runs ok. - # - python_exe = os.path.realpath( sys.executable) - if darwin(): - # Basic install of dev tools with `xcode-select --install` doesn't - # seem to provide a `python3-config` or similar, but there is a - # `python-config.py` accessible via sysconfig. - # - # We try different possibilities and use the last one that - # works. - # - python_config = None - for pc in ( - f'python3-config', - f'{sys.executable} {sysconfig.get_config_var("srcdir")}/python-config.py', - f'{python_exe}-config', - ): - e = subprocess.run( - f'{pc} --includes', - shell=1, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=0, - ).returncode - log2(f'{e=} from {pc!r}.') - if e == 0: - python_config = pc - assert python_config, f'Cannot find python-config' - else: - python_config = f'{python_exe}-config' - log2(f'Using {python_config=}.') - try: - self.includes = run( f'{python_config} --includes', capture=1, verbose=0).strip() - except Exception as e: - raise Exception('We require python development tools to be installed.') from e - self.ldflags = run( f'{python_config} --ldflags', capture=1, verbose=0).strip() - if linux(): - # It seems that with python-3.10 on Linux, we can get an - # incorrect -lcrypt flag that on some systems (e.g. WSL) - # causes: - # - # ImportError: libcrypt.so.2: cannot open shared object file: No such file or directory - # - ldflags2 = self.ldflags.replace(' -lcrypt ', ' ') - if ldflags2 != self.ldflags: - log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}') - self.ldflags = ldflags2 - - if 0: - log1(f'{self.includes=}') - log1(f' {includes_=}') - log1(f'{self.ldflags=}') - log1(f' {ldflags_=}') - - -def macos_add_cross_flags(command): - ''' - If running on MacOS and environment variables ARCHFLAGS is set - (indicating we are cross-building, e.g. for arm64), returns - `command` with extra flags appended. Otherwise returns unchanged - `command`. - ''' - if darwin(): - archflags = os.environ.get( 'ARCHFLAGS') - if archflags: - command = f'{command} {archflags}' - log2(f'Appending ARCHFLAGS to command: {command}') - return command - return command - - -def macos_patch( library, *sublibraries): - ''' - If running on MacOS, patches `library` so that all references to items in - `sublibraries` are changed to `@rpath/{leafname}`. Does nothing on other - platforms. - - library: - Path of shared library. - sublibraries: - List of paths of shared libraries; these have typically been - specified with `-l` when `library` was created. - ''' - log2( f'macos_patch(): library={library} sublibraries={sublibraries}') - if not darwin(): - return - if not sublibraries: - return - subprocess.run( f'otool -L {library}', shell=1, check=1) - command = 'install_name_tool' - names = [] - for sublibrary in sublibraries: - name = subprocess.run( - f'otool -D {sublibrary}', - shell=1, - check=1, - capture_output=1, - encoding='utf8', - ).stdout.strip() - name = name.split('\n') - assert len(name) == 2 and name[0] == f'{sublibrary}:', f'{name=}' - name = name[1] - # strip trailing so_name. - leaf = os.path.basename(name) - m = re.match('^(.+[.]((so)|(dylib)))[0-9.]*$', leaf) - assert m - log2(f'Changing {leaf=} to {m.group(1)}') - leaf = m.group(1) - command += f' -change {name} @rpath/{leaf}' - command += f' {library}' - log2( f'Running: {command}') - subprocess.run( command, shell=1, check=1) - subprocess.run( f'otool -L {library}', shell=1, check=1) - - -def _macos_fixup_platform_tag(tag): - ''' - Patch up platform tag on MacOS. - - E.g. `foo-1.2.3-cp311-none-macosx_13_x86_64.whl` causes `pip` to fail with: - `not a supported wheel on this platform`. We seem to need to add `_0` to - the OS version. (This is documented at - https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#macos). - - And with graal we need to replace trailing `universal2` with x86_64 - or arm64. On non-graal this causes problems because non-universal - platform tags seem more restricted than platform tags from - sysconfig.get_platform(). For example: - - pip install ...-macosx_10_13_arm64.whl - ERROR: ...-macosx_10_13_arm64.whl is not a supported wheel on this platform. - pip install ...-macosx_10_13_universal2.whl - Ok. - ''' - m = re.match( '^macosx_([0-9_]+)_([^0-9].+)$', tag) - if not m: - return tag - a = m.group(1) - if '_' not in a: - a += '_0' - b = m.group(2) - if sys.implementation.name == 'graalpy' and b == 'universal2': - # Replace 'universal2' with x86_64 or arm64. - b = platform.machine() - ret = f'macosx_{a}_{b}' - #log0(f'Changing from {tag=} to {ret=}.') - return ret - - -# Internal helpers. -# - -def _command_lines( command): - ''' - Process multiline command by running through `textwrap.dedent()`, removes - comments (lines starting with `#` or ` #` until end of line), removes - entirely blank lines. - - Returns list of lines. - ''' - command = textwrap.dedent( command) - lines = [] - for line in command.split( '\n'): - if line.startswith( '#'): - h = 0 - else: - h = line.find( ' #') - if h >= 0: - line = line[:h] - if line.strip(): - lines.append(line.rstrip()) - return lines - - -def cpu_bits(): - return int.bit_length(sys.maxsize+1) - - -def _cpu_name(): - ''' - Returns `x32` or `x64` depending on Python build. - ''' - #log(f'sys.maxsize={hex(sys.maxsize)}') - return f'x{32 if sys.maxsize == 2**31 - 1 else 64}' - - -def run_if( command, out, *prerequisites, caller=1): - ''' - Runs a command only if the output file is not up to date. - - Args: - command: - The command to run. We write this and a hash of argv[0] into a file - .cmd so that we know to run a command if the command itself - has changed. - out: - Path of the output file. - - prerequisites: - List of prerequisite paths or true/false/None items. If an item - is None it is ignored, otherwise if an item is not a string we - immediately return it cast to a bool. We recurse into directories, - effectively using the newest file in the directory. - - Returns: - True if we ran the command, otherwise None. - - - If the output file does not exist, the command is run: - - >>> verbose(1) - 1 - >>> log_line_numbers(0) - >>> out = 'run_if_test_out' - >>> if os.path.exists( out): - ... os.remove( out) - >>> if os.path.exists( f'{out}.cmd'): - ... os.remove( f'{out}.cmd') - >>> run_if( f'touch {out}', out, caller=0) - pipcl.py:run_if(): Running command because: File does not exist: 'run_if_test_out' - pipcl.py:run_if(): Running: touch run_if_test_out - True - - If we repeat, the output file will be up to date so the command is not run: - - >>> run_if( f'touch {out}', out, caller=0) - pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out' - - If we change the command, the command is run: - - >>> run_if( f'touch {out};', out, caller=0) - pipcl.py:run_if(): Running command because: Command has changed: - pipcl.py:run_if(): @@ -1,2 +1,2 @@ - pipcl.py:run_if(): touch - pipcl.py:run_if(): -run_if_test_out - pipcl.py:run_if(): +run_if_test_out; - pipcl.py:run_if(): - pipcl.py:run_if(): Running: touch run_if_test_out; - True - - If we add a prerequisite that is newer than the output, the command is run: - - >>> time.sleep(1) - >>> prerequisite = 'run_if_test_prerequisite' - >>> run( f'touch {prerequisite}', caller=0) - pipcl.py:run(): Running: touch run_if_test_prerequisite - >>> run_if( f'touch {out}', out, prerequisite, caller=0) - pipcl.py:run_if(): Running command because: Command has changed: - pipcl.py:run_if(): @@ -1,2 +1,2 @@ - pipcl.py:run_if(): touch - pipcl.py:run_if(): -run_if_test_out; - pipcl.py:run_if(): +run_if_test_out - pipcl.py:run_if(): - pipcl.py:run_if(): Running: touch run_if_test_out - True - - If we repeat, the output will be newer than the prerequisite, so the - command is not run: - - >>> run_if( f'touch {out}', out, prerequisite, caller=0) - pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out' - - We detect changes to the contents of argv[0]: - - Create a shell script and run it: - - >>> _ = subprocess.run('rm run_if_test_argv0.* 1>/dev/null 2>/dev/null || true', shell=1) - >>> with open('run_if_test_argv0.sh', 'w') as f: - ... print('#! /bin/sh', file=f) - ... print('echo hello world > run_if_test_argv0.out', file=f) - >>> _ = subprocess.run(f'chmod u+x run_if_test_argv0.sh', shell=1) - >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0) - pipcl.py:run_if(): Running command because: File does not exist: 'run_if_test_argv0.out' - pipcl.py:run_if(): Running: ./run_if_test_argv0.sh - True - - Running it a second time does nothing: - - >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0) - pipcl.py:run_if(): Not running command because up to date: 'run_if_test_argv0.out' - - Modify the script. - - >>> with open('run_if_test_argv0.sh', 'a') as f: - ... print('\\necho hello >> run_if_test_argv0.out', file=f) - - And now it is run because the hash of argv[0] has changed: - - >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0) - pipcl.py:run_if(): Running command because: arg0 hash has changed. - pipcl.py:run_if(): Running: ./run_if_test_argv0.sh - True - ''' - doit = False - - # Path of file containing pickle data for command and hash of command's - # first arg. - cmd_path = f'{out}.cmd' - - def hash_get(path): - try: - with open(path, 'rb') as f: - return hashlib.md5(f.read()).hexdigest() - except Exception as e: - #log(f'Failed to get hash of {path=}: {e}') - return None - - command_args = shlex.split(command or '') - command_arg0_path = fs_find_in_paths(command_args[0]) - command_arg0_hash = hash_get(command_arg0_path) - - cmd_args, cmd_arg0_hash = (None, None) - if os.path.isfile(cmd_path): - with open(cmd_path, 'rb') as f: - try: - cmd_args, cmd_arg0_hash = pickle.load(f) - except Exception as e: - #log(f'pickle.load() failed with {cmd_path=}: {e}') - pass - - if not doit: - # Set doit if outfile does not exist. - out_mtime = _fs_mtime( out) - if out_mtime == 0: - doit = f'File does not exist: {out!r}' - - if not doit: - # Set doit if command has changed. - if command_args != cmd_args: - if cmd_args is None: - doit = 'No previous command stored' - else: - doit = f'Command has changed' - if 0: - doit += f':\n {cmd!r}\n {command!r}' - if 0: - doit += f'\nbefore:\n' - doit += textwrap.indent(cmd, ' ') - doit += f'\nafter:\n' - doit += textwrap.indent(command, ' ') - if 1: - # Show diff based on commands split into pseudo lines by - # shlex.split(). - doit += ':\n' - lines = difflib.unified_diff( - cmd_args, - command_args, - lineterm='', - ) - # Skip initial lines. - assert next(lines) == '--- ' - assert next(lines) == '+++ ' - for line in lines: - doit += f' {line}\n' - - if not doit: - # Set doit if argv[0] hash has changed. - #print(f'{cmd_arg0_hash=} {command_arg0_hash=}', file=sys.stderr) - if command_arg0_hash != cmd_arg0_hash: - doit = f'arg0 hash has changed.' - #doit = f'arg0 hash has changed from {cmd_arg0_hash=} to {command_arg0_hash=}..' - - if not doit: - # See whether any prerequisites are newer than target. - def _make_prerequisites(p): - if isinstance( p, (list, tuple)): - return list(p) - else: - return [p] - prerequisites_all = list() - for p in prerequisites: - prerequisites_all += _make_prerequisites( p) - if 0: - log2( 'prerequisites_all:', caller=caller+1) - for i in prerequisites_all: - log2( f' {i!r}', caller=caller+1) - pre_mtime = 0 - pre_path = None - for prerequisite in prerequisites_all: - if isinstance( prerequisite, str): - mtime = _fs_mtime_newest( prerequisite) - if mtime >= pre_mtime: - pre_mtime = mtime - pre_path = prerequisite - elif prerequisite is None: - pass - elif prerequisite: - doit = str(prerequisite) - break - if not doit: - if pre_mtime > out_mtime: - doit = f'Prerequisite is new: {os.path.abspath(pre_path)!r}' - - if doit: - # Remove `cmd_path` before we run the command, so any failure - # will force rerun next time. - # - try: - os.remove( cmd_path) - except Exception: - pass - log1( f'Running command because: {doit}', caller=caller+1) - - run( command, caller=caller+1) - - # Write the command we ran, into `cmd_path`. - - with open(cmd_path, 'wb') as f: - pickle.dump((command_args, command_arg0_hash), f) - return True - else: - log1( f'Not running command because up to date: {out!r}', caller=caller+1) - - if 0: - log2( f'out_mtime={time.ctime(out_mtime)} pre_mtime={time.ctime(pre_mtime)}.' - f' pre_path={pre_path!r}: returning {ret!r}.' - ) - - -def fs_find_in_paths( name, paths=None, verbose=False): - ''' - Looks for `name` in paths and returns complete path. `paths` is list/tuple - or `os.pathsep`-separated string; if `None` we use `$PATH`. If `name` - contains `/`, we return `name` itself if it is a file or None, regardless - of . - ''' - if '/' in name: - return name if os.path.isfile( name) else None - if paths is None: - paths = os.environ.get( 'PATH', '') - if verbose: - log('From os.environ["PATH"]: {paths=}') - if isinstance( paths, str): - paths = paths.split( os.pathsep) - if verbose: - log('After split: {paths=}') - for path in paths: - p = os.path.join( path, name) - if verbose: - log('Checking {p=}') - if os.path.isfile( p): - if verbose: - log('Returning because is file: {p!r}') - return p - if verbose: - log('Returning None because not found: {name!r}') - - -def _get_prerequisites(path): - ''' - Returns list of prerequisites from Makefile-style dependency file, e.g. - created by `cc -MD -MF `. - ''' - ret = list() - if os.path.isfile(path): - with open(path) as f: - for line in f: - for item in line.split(): - if item.endswith( (':', '\\')): - continue - ret.append( item) - return ret - - -def _fs_mtime_newest( path): - ''' - path: - If a file, returns mtime of the file. If a directory, returns mtime of - newest file anywhere within directory tree. Otherwise returns 0. - ''' - ret = 0 - if os.path.isdir( path): - for dirpath, dirnames, filenames in os.walk( path): - for filename in filenames: - path = os.path.join( dirpath, filename) - ret = max( ret, _fs_mtime( path)) - else: - ret = _fs_mtime( path) - return ret - - -def _flags( items, prefix='', quote=''): - ''' - Turns sequence into string, prefixing/quoting each item. - ''' - if not items: - return '' - if isinstance( items, str): - items = items, - ret = '' - for item in items: - if ret: - ret += ' ' - ret += f'{prefix}{quote}{item}{quote}' - return ret.strip() - - -def _fs_mtime( filename, default=0): - ''' - Returns mtime of file, or `default` if error - e.g. doesn't exist. - ''' - try: - return os.path.getmtime( filename) - except OSError: - return default - - -def _normalise(name): - # https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization - return re.sub(r"[-_.]+", "-", name).lower() - - -def _normalise2(name): - # https://packaging.python.org/en/latest/specifications/binary-distribution-format/ - return _normalise(name).replace('-', '_') - - -def _assert_version_pep_440(version): - assert re.match( - r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$', - version, - ), \ - f'Bad version: {version!r}.' - - -g_verbose = int(os.environ.get('PIPCL_VERBOSE', '1')) - -def verbose(level=None): - ''' - Sets verbose level if `level` is not None. - Returns verbose level. - ''' - global g_verbose - if level is not None: - g_verbose = level - return g_verbose - -g_log_line_numbers = True - -def log_line_numbers(yes): - ''' - Sets whether to include line numbers; helps with doctest. - ''' - global g_log_line_numbers - g_log_line_numbers = bool(yes) - -def log(text='', caller=1): - _log(text, 0, caller+1) - -def log0(text='', caller=1): - _log(text, 0, caller+1) - -def log1(text='', caller=1): - _log(text, 1, caller+1) - -def log2(text='', caller=1): - _log(text, 2, caller+1) - -def _log(text, level, caller): - ''' - Logs lines with prefix, if is lower or equal to . - ''' - if level <= g_verbose: - fr = inspect.stack(context=0)[caller] - filename = relpath(fr.filename) - for line in text.split('\n'): - if g_log_line_numbers: - print(f'{filename}:{fr.lineno}:{fr.function}(): {line}', file=sys.stdout, flush=1) - else: - print(f'{filename}:{fr.function}(): {line}', file=sys.stdout, flush=1) - - -def relpath(path, start=None, allow_up=True): - ''' - A safe alternative to os.path.relpath(), avoiding an exception on Windows - if the drive needs to change - in this case we use os.path.abspath(). - - Args: - path: - Path to be processed. - start: - Start directory or current directory if None. - allow_up: - If false we return absolute path is is not within . - ''' - if windows(): - try: - ret = os.path.relpath(path, start) - except ValueError: - # os.path.relpath() fails if trying to change drives. - ret = os.path.abspath(path) - else: - ret = os.path.relpath(path, start) - if not allow_up and ret.startswith('../') or ret.startswith('..\\'): - ret = os.path.abspath(path) - return ret - - -def _so_suffix(use_so_versioning=True): - ''' - Filename suffix for shared libraries is defined in pep-3149. The - pep claims to only address posix systems, but the recommended - sysconfig.get_config_var('EXT_SUFFIX') also seems to give the - right string on Windows. - - If use_so_versioning is false, we return only the last component of - the suffix, which removes any version number, for example changing - `.cp312-win_amd64.pyd` to `.pyd`. - ''' - # Example values: - # linux: .cpython-311-x86_64-linux-gnu.so - # macos: .cpython-311-darwin.so - # openbsd: .cpython-310.so - # windows .cp311-win_amd64.pyd - # - # Only Linux and Windows seem to identify the cpu. For example shared - # libraries in numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl are called - # things like `numpy/core/_simd.cpython-311-darwin.so`. - # - ret = sysconfig.get_config_var('EXT_SUFFIX') - if not use_so_versioning: - # Use last component only. - ret = os.path.splitext(ret)[1] - return ret - - -def get_soname(path): - ''' - If we are on Linux and `path` is softlink and points to a shared library - for which `objdump -p` contains 'SONAME', return the pointee. Otherwise - return `path`. Useful if Linux shared libraries have been created with - `-Wl,-soname,...`, where we need to embed the versioned library. - ''' - if linux() and os.path.islink(path): - path2 = os.path.realpath(path) - if subprocess.run(f'objdump -p {path2}|grep SONAME', shell=1, check=0).returncode == 0: - return path2 - elif openbsd(): - # Return newest .so with version suffix. - sos = glob.glob(f'{path}.*') - log1(f'{sos=}') - sos2 = list() - for so in sos: - suffix = so[len(path):] - if not suffix or re.match('^[.][0-9.]*[0-9]$', suffix): - sos2.append(so) - sos2.sort(key=lambda p: os.path.getmtime(p)) - log1(f'{sos2=}') - return sos2[-1] - return path - - -def current_py_limited_api(): - ''' - Returns value of PyLIMITED_API to build for current Python. - ''' - a, b = map(int, platform.python_version().split('.')[:2]) - return f'0x{a:02x}{b:02x}0000' - - -def install_dir(root=None): - ''' - Returns install directory used by `install()`. - - This will be `sysconfig.get_path('platlib')`, modified by `root` if not - None. - ''' - # todo: for pure-python we should use sysconfig.get_path('purelib') ? - root2 = sysconfig.get_path('platlib') - if root: - if windows(): - # If we are in a venv, `sysconfig.get_path('platlib')` - # can be absolute, e.g. - # `C:\\...\\venv-pypackage-3.11.1-64\\Lib\\site-packages`, so it's - # not clear how to append it to `root`. So we just use `root`. - return root - else: - # E.g. if `root` is `install' and `sysconfig.get_path('platlib')` - # is `/usr/local/lib/python3.9/site-packages`, we set `root2` to - # `install/usr/local/lib/python3.9/site-packages`. - # - return os.path.join( root, root2.lstrip( os.sep)) - else: - return root2 - - -class _Record: - ''' - Internal - builds up text suitable for writing to a RECORD item, e.g. - within a wheel. - ''' - def __init__(self): - self.text = '' - - def add_content(self, content, to_, verbose=True): - if isinstance(content, str): - content = content.encode('utf8') - - # Specification for the line we write is supposed to be in - # https://packaging.python.org/en/latest/specifications/binary-distribution-format - # but it's not very clear. - # - h = hashlib.sha256(content) - digest = h.digest() - digest = base64.urlsafe_b64encode(digest) - digest = digest.rstrip(b'=') - digest = digest.decode('utf8') - - self.text += f'{to_},sha256={digest},{len(content)}\n' - if verbose: - log2(f'Adding {to_}') - - def add_file(self, from_, to_): - log1(f'Adding file: {os.path.relpath(from_)} => {to_}') - with open(from_, 'rb') as f: - content = f.read() - self.add_content(content, to_, verbose=False) - - def get(self, record_path=None): - ''' - Returns contents of the RECORD file. If `record_path` is - specified we append a final line `,,`; this can be - used to include the RECORD file itself in the contents, with - empty hash and size fields. - ''' - ret = self.text - if record_path: - ret += f'{record_path},,\n' - return ret - - -class NewFiles: - ''' - Detects new/modified/updated files matching a glob pattern. Useful for - detecting wheels created by pip or cubuildwheel etc. - ''' - def __init__(self, glob_pattern): - # Find current matches of . - self.glob_pattern = glob_pattern - self.items0 = self._items() - def get(self): - ''' - Returns list of new matches of - paths of files that - were not present previously, or have different mtimes or have different - contents. - ''' - ret = list() - items = self._items() - for path, id_ in items.items(): - id0 = self.items0.get(path) - if id0 != id_: - ret.append(path) - return ret - def get_n(self, n): - ''' - Returns new files matching , asserting that there are - exactly . - ''' - ret = self.get() - assert len(ret) == n, f'{len(ret)=}: {ret}' - return ret - def get_one(self): - ''' - Returns new match of , asserting that there is exactly - one. - ''' - return self.get_n(1)[0] - def _file_id(self, path): - mtime = os.stat(path).st_mtime - with open(path, 'rb') as f: - content = f.read() - hash_ = hashlib.md5(content).digest() - # With python >= 3.11 we can do: - #hash_ = hashlib.file_digest(f, hashlib.md5).digest() - return mtime, hash_ - def _items(self): - ret = dict() - for path in glob.glob(self.glob_pattern): - if os.path.isfile(path): - ret[path] = self._file_id(path) - return ret - - -def swig_get(swig, quick, swig_local='pipcl-swig-git'): - ''' - Returns or a new swig binary. - - If is true and starts with 'git:' (not Windows), the remaining text - is passed to git_get() and we clone/update/build swig, and return the built - binary. We default to the main swig repository, branch master, so for - example 'git:' will return the latest swig from branch master. - - Otherwise we simply return . - - Args: - swig: - If starts with 'git:', passed as arg to git_get(). - quick: - If true, we do not update/build local checkout if the binary is - already present. - swig_local: - path to use for checkout. - ''' - if swig and swig.startswith('git:'): - assert platform.system() != 'Windows', f'Cannot build swig on Windows.' - # Note that {swig_local}/install/bin/swig doesn't work on MacOS because - # {swig_local}/INSTALL is a file and the fs is case-insensitive. - swig_binary = f'{swig_local}/install-dir/bin/swig' - if quick and os.path.isfile(swig_binary): - log1(f'{quick=} and {swig_binary=} already exists, so not downloading/building.') - else: - if darwin(): - run(f'brew install automake') - run(f'brew install pcre2') - run(f'brew install bison') - # Default bison doesn't work, and Brew's bison is not added to $PATH. - # - # > bison is keg-only, which means it was not symlinked into /opt/homebrew, - # > because macOS already provides this software and installing another version in - # > parallel can cause all kinds of trouble. - # > - # > If you need to have bison first in your PATH, run: - # > echo 'export PATH="/opt/homebrew/opt/bison/bin:$PATH"' >> ~/.zshrc - # - swig_env_extra = dict() - macos_add_brew_path('bison', swig_env_extra) - run(f'which bison') - run(f'which bison', env_extra=swig_env_extra) - - # Building swig requires bison>=3.5. - bison_ok = 0 - e, text = run(f'bison --version', capture=1, check=0, env_extra=swig_env_extra) - if not e: - log(textwrap.indent(text, ' ')) - m = re.search('bison (GNU Bison) ([0-9]+)[.]([0-9]+)', text) - if m: - assert m, f'Unexpected output from `bison --version`: {text!r}' - version_tuple = int(m.group(1)), int(m.group2()) - if version_tuple >= (3, 5): - bison_ok = 1 - if not bison_ok: - if 0: - # Use git checkout. Fails to find scan-code.c. Presumably - # something wrong with ./bootstrap? - log(f'Cloning/fetching/build/installing bison.') - bison_git = git_get( - 'pipcl-bison-git', - remote='https://git.savannah.gnu.org/git/bison.git', - #branch='master', - tag='v3.5.4', - submodules=0, # recursive update fails. - ) - run(f'cd {bison_git} && git submodule update --init', prefix='bison git submodule update --init: ') - run(f'cd {bison_git} && ./bootstrap', prefix='bison bootstrap: ') - run(f'cd {bison_git} && ./configure', prefix='bison configure: ') - run(f'cd {bison_git} && make', prefix='bison make: ') - run(f'cd {bison_git} && sudo make install', prefix='bison make install: ') - else: - bison_version = 'bison-3.5.4' - if not os.path.exists(f'{bison_version}.tar.gz'): - run( - f'wget -O {bison_version}.tar.gz-0 http://www.mirrorservice.org/sites/ftp.gnu.org/gnu/bison/{bison_version}.tar.gz', - prefix='bison wget: ', - ) - os.rename(f'{bison_version}.tar.gz-0', f'{bison_version}.tar.gz') - if not os.path.exists(f'{bison_version}'): - run(f'tar -xzf {bison_version}.tar.gz', prefix='bison extract: ') - run(f'cd {bison_version} && ./configure', prefix='bison configure: ') - run(f'cd {bison_version} && make', prefix='bison make: ') - run(f'cd {bison_version} && sudo make install', prefix='bison make install: ') - - # Clone swig. - swig_env_extra = None - swig_local = git_get( - swig_local, - text=swig, - remote='https://github.com/swig/swig.git', - branch='master', - ) - # Build swig. - run(f'cd {swig_local} && ./autogen.sh', env_extra=swig_env_extra) - run(f'cd {swig_local} && ./configure --prefix={swig_local}/install-dir', env_extra=swig_env_extra) - run(f'cd {swig_local} && make', env_extra=swig_env_extra) - run(f'cd {swig_local} && make install', env_extra=swig_env_extra) - assert os.path.isfile(swig_binary) - return swig_binary - else: - return swig - - -def macos_add_brew_path(package, env=None, gnubin=True): - ''' - Adds path(s) for Brew 's binaries to env['PATH']. - - We assert-fail if the relevant directory does no exist. - - Args: - package: - Name of package. We get of installed package by - running `brew --prefix `. - env: - The environment dict to modify. If None we use os.environ. If PATH - is not in , we first copy os.environ['PATH'] into . - gnubin: - If true, we also add path to gnu binaries if it exists, - /libexe/gnubin. - ''' - if not darwin(): - return - if env is None: - env = os.environ - if 'PATH' not in env: - env['PATH'] = os.environ['PATH'] - package_root = run(f'brew --prefix {package}', capture=1).strip() - log(f'{package=} {package_root=}') - def add(path): - log(f'{path=}') - if os.path.isdir(path): - log(f'Prepending to $PATH: {path}') - PATH = env['PATH'] - env['PATH'] = f'{path}:{PATH}' - return 1 - else: - log(f'Not a directory: {path=}') - return 0 - n = 0 - n += add(f'{package_root}/bin') - if gnubin: - n += add(f'{package_root}/libexec/gnubin') - assert n, f'Failed to add to $PATH, {package=} {gnubin=}.' - - -def _show_dict(d): - ret = '' - for n in sorted(d.keys()): - v = d[n] - ret += f' {n}: {v!r}\n' - return ret - -def show_sysconfig(): - ''' - Shows contents of sysconfig.get_paths() and sysconfig.get_config_vars() dicts. - ''' - import sysconfig - paths = sysconfig.get_paths() - log0(f'show_sysconfig().') - log0(f'sysconfig.get_paths():\n{_show_dict(sysconfig.get_paths())}') - log0(f'sysconfig.get_config_vars():\n{_show_dict(sysconfig.get_config_vars())}') - - -def sysconfig_python_flags(): - ''' - Returns include paths and library directory for Python. - - Uses sysconfig.*(), overridden by environment variables - PIPCL_SYSCONFIG_PATH_include, PIPCL_SYSCONFIG_PATH_platinclude and - PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR if set. - ''' - include1_ = os.environ.get('PIPCL_SYSCONFIG_PATH_include') or sysconfig.get_path('include') - include2_ = os.environ.get('PIPCL_SYSCONFIG_PATH_platinclude') or sysconfig.get_path('platinclude') - ldflags_ = os.environ.get('PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR') or sysconfig.get_config_var('LIBDIR') - - includes_ = [include1_] - if include2_ != include1_: - includes_.append(include2_) - if windows(): - includes_ = [f'/I"{i}"' for i in includes_] - ldflags_ = f'/LIBPATH:"{ldflags_}"' - else: - includes_ = [f'-I {i}' for i in includes_] - ldflags_ = f'-L {ldflags_}' - includes_ = ' '.join(includes_) - return includes_, ldflags_ - - -def venv_in(path=None): - ''' - If path is None, returns true if we are in a venv. Otherwise returns true - only if we are in venv . - ''' - if path: - return os.path.abspath(sys.prefix) == os.path.abspath(path) - else: - return sys.prefix != sys.base_prefix - - -def venv_run(args, path, recreate=True, clean=False): - ''' - Runs Python command inside venv and returns termination code. - - Args: - args: - List of args or string command. - path: - Path of venv directory. - recreate: - If false we do not run ` -m venv ` if - already exists. This avoids a delay in the common case where - is already set up, but fails if exists but does not contain - a valid venv. - clean: - If true we first delete . - ''' - if clean: - log(f'Removing any existing venv {path}.') - assert path.startswith('venv-') - shutil.rmtree(path, ignore_errors=1) - if recreate or not os.path.isdir(path): - run(f'{sys.executable} -m venv {path}') - - if isinstance(args, str): - args_string = args - elif platform.system() == 'Windows': - # shlex not reliable on Windows so we use Use crude quoting with "...". - args_string = '' - for i, arg in enumerate(args): - assert '"' not in arg - if i: - args_string += ' ' - args_string += f'"{arg}"' - else: - args_string = shlex.join(args) - - if platform.system() == 'Windows': - command = f'{path}\\Scripts\\activate && python {args_string}' - else: - command = f'. {path}/bin/activate && python {args_string}' - e = run(command, check=0) - return e - - -if __name__ == '__main__': - # Internal-only limited command line support, used if - # graal_legacy_python_config is true. - # - includes, ldflags = sysconfig_python_flags() - if sys.argv[1] == '--doctest': - import doctest - if sys.argv[2:]: - for f in sys.argv[2:]: - ff = globals()[f] - doctest.run_docstring_examples(ff, globals()) - else: - doctest.testmod(None) - elif sys.argv[1:] == ['--graal-legacy-python-config', '--includes']: - print(includes) - elif sys.argv[1:] == ['--graal-legacy-python-config', '--ldflags']: - print(ldflags) - else: - assert 0, f'Expected `--graal-legacy-python-config --includes|--ldflags` but {sys.argv=}' diff --git a/src/wdev.py b/src/wdev.py deleted file mode 100644 index 33c348459..000000000 --- a/src/wdev.py +++ /dev/null @@ -1,424 +0,0 @@ -''' -Finds locations of Windows command-line development tools. -''' - -import os -import platform -import glob -import re -import subprocess -import sys -import sysconfig -import textwrap - -import pipcl - - -class WindowsVS: - r''' - Windows only. Finds locations of Visual Studio command-line tools. Assumes - VS2019-style paths. - - Members and example values:: - - .year: 2019 - .grade: Community - .version: 14.28.29910 - .directory: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community - .vcvars: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat - .cl: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64\cl.exe - .link: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64\link.exe - .csc: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\Roslyn\csc.exe - .msbuild: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe - .devenv: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\devenv.com - - `.csc` is C# compiler; will be None if not found. - ''' - def __init__( - self, - *, - year=None, - grade=None, - version=None, - cpu=None, - directory=None, - verbose=False, - ): - ''' - Args: - year: - None or, for example, `2019`. If None we use environment - variable WDEV_VS_YEAR if set. - grade: - None or, for example, one of: - - * `Community` - * `Professional` - * `Enterprise` - - If None we use environment variable WDEV_VS_GRADE if set. - version: - None or, for example: `14.28.29910`. If None we use environment - variable WDEV_VS_VERSION if set. - cpu: - None or a `WindowsCpu` instance. - directory: - Ignore year, grade, version and cpu and use this directory - directly. - verbose: - . - - ''' - if year is not None: - year = str(year) # Allow specification as a number. - def default(value, name): - if value is None: - name2 = f'WDEV_VS_{name.upper()}' - value = os.environ.get(name2) - if value is not None: - _log(f'Setting {name} from environment variable {name2}: {value!r}') - return value - try: - year = default(year, 'year') - grade = default(grade, 'grade') - version = default(version, 'version') - - if not cpu: - cpu = WindowsCpu() - - if not directory: - # Find `directory`. - # - pattern = _vs_pattern(year, grade) - directories = glob.glob( pattern) - if verbose: - _log( f'Matches for: {pattern=}') - _log( f'{directories=}') - assert directories, f'No match found for {pattern=}.' - directories.sort() - directory = directories[-1] - - # Find `devenv`. - # - devenv = f'{directory}\\Common7\\IDE\\devenv.com' - assert os.path.isfile( devenv), f'Does not exist: {devenv}' - - # Extract `year` and `grade` from `directory`. - # - # We use r'...' for regex strings because an extra level of escaping is - # required for backslashes. - # - regex = rf'^C:\\Program Files.*\\Microsoft Visual Studio\\([^\\]+)\\([^\\]+)' - m = re.match( regex, directory) - assert m, f'No match: {regex=} {directory=}' - year2 = m.group(1) - grade2 = m.group(2) - if year: - assert year2 == year - else: - year = year2 - if grade: - assert grade2 == grade - else: - grade = grade2 - - # Find vcvars.bat. - # - vcvars = f'{directory}\\VC\\Auxiliary\\Build\\vcvars{cpu.bits}.bat' - assert os.path.isfile( vcvars), f'No match for: {vcvars}' - - # Find cl.exe. - # - cl_pattern = f'{directory}\\VC\\Tools\\MSVC\\{version if version else "*"}\\bin\\Host{cpu.windows_name}\\{cpu.windows_name}\\cl.exe' - cl_s = glob.glob( cl_pattern) - assert cl_s, f'No match for: {cl_pattern}' - cl_s.sort() - cl = cl_s[ -1] - - # Extract `version` from cl.exe's path. - # - m = re.search( rf'\\VC\\Tools\\MSVC\\([^\\]+)\\bin\\Host{cpu.windows_name}\\{cpu.windows_name}\\cl.exe$', cl) - assert m - version2 = m.group(1) - if version: - assert version2 == version - else: - version = version2 - assert version - - # Find link.exe. - # - link_pattern = f'{directory}\\VC\\Tools\\MSVC\\{version}\\bin\\Host{cpu.windows_name}\\{cpu.windows_name}\\link.exe' - link_s = glob.glob( link_pattern) - assert link_s, f'No match for: {link_pattern}' - link_s.sort() - link = link_s[ -1] - - # Find csc.exe. - # - csc = None - for dirpath, dirnames, filenames in os.walk(directory): - for filename in filenames: - if filename == 'csc.exe': - csc = os.path.join(dirpath, filename) - #_log(f'{csc=}') - #break - - # Find MSBuild.exe. - # - msbuild = None - for dirpath, dirnames, filenames in os.walk(directory): - for filename in filenames: - if filename == 'MSBuild.exe': - msbuild = os.path.join(dirpath, filename) - #_log(f'{csc=}') - #break - - self.cl = cl - self.devenv = devenv - self.directory = directory - self.grade = grade - self.link = link - self.csc = csc - self.msbuild = msbuild - self.vcvars = vcvars - self.version = version - self.year = year - self.cpu = cpu - except Exception as e: - raise Exception( f'Unable to find Visual Studio {year=} {grade=} {version=} {cpu=} {directory=}') from e - - def description_ml( self, indent=''): - ''' - Return multiline description of `self`. - ''' - ret = textwrap.dedent(f''' - year: {self.year} - grade: {self.grade} - version: {self.version} - directory: {self.directory} - vcvars: {self.vcvars} - cl: {self.cl} - link: {self.link} - csc: {self.csc} - msbuild: {self.msbuild} - devenv: {self.devenv} - cpu: {self.cpu} - ''') - return textwrap.indent( ret, indent) - - def __repr__( self): - items = list() - for name in ( - 'year', - 'grade', - 'version', - 'directory', - 'vcvars', - 'cl', - 'link', - 'csc', - 'msbuild', - 'devenv', - 'cpu', - ): - items.append(f'{name}={getattr(self, name)!r}') - return ' '.join(items) - - -def _vs_pattern(year=None, grade=None): - return f'C:\\Program Files*\\Microsoft Visual Studio\\{year if year else "2*"}\\{grade if grade else "*"}' - - -def windows_vs_multiple(year=None, grade=None, verbose=0): - ''' - Returns list of WindowsVS instances. - ''' - ret = list() - directories = glob.glob(_vs_pattern(year, grade)) - for directory in directories: - vs = WindowsVS(directory=directory) - if verbose: - _log(vs.description_ml()) - ret.append(vs) - return ret - - -class WindowsCpu: - ''' - For Windows only. Paths and names that depend on cpu. - - Members: - .bits - 32 or 64. - .windows_subdir - Empty string or `x64/`. - .windows_name - `x86` or `x64`. - .windows_config - `x64` or `Win32`, e.g. for use in `/Build Release|x64`. - .windows_suffix - `64` or empty string. - ''' - def __init__(self, name=None): - if not name: - name = _cpu_name() - self.name = name - if name == 'x32': - self.bits = 32 - self.windows_subdir = '' - self.windows_name = 'x86' - self.windows_config = 'Win32' - self.windows_suffix = '' - elif name == 'x64': - self.bits = 64 - self.windows_subdir = 'x64/' - self.windows_name = 'x64' - self.windows_config = 'x64' - self.windows_suffix = '64' - else: - assert 0, f'Unrecognised cpu name: {name}' - - def __repr__(self): - return self.name - - -class WindowsPython: - ''' - Windows only. Information about installed Python with specific word size - and version. Defaults to the currently-running Python. - - Members: - - .path: - Path of python binary. - .version: - `{major}.{minor}`, e.g. `3.9` or `3.11`. Same as `version` passed - to `__init__()` if not None, otherwise the inferred version. - .include: - Python include path. - .cpu: - A `WindowsCpu` instance, same as `cpu` passed to `__init__()` if - not None, otherwise the inferred cpu. - - We parse the output from `py -0p` to find all available python - installations. - ''' - - def __init__( self, cpu=None, version=None, verbose=True): - ''' - Args: - - cpu: - A WindowsCpu instance. If None, we use whatever we are running - on. - version: - Two-digit Python version as a string such as `3.8`. If None we - use current Python's version. - verbose: - If true we show diagnostics. - ''' - if cpu is None: - cpu = WindowsCpu(_cpu_name()) - if version is None: - version = '.'.join(platform.python_version().split('.')[:2]) - _log(f'Looking for Python {version=} {cpu.bits=}.') - - if '.'.join(platform.python_version().split('.')[:2]) == version: - # Current python matches, so use it directly. This avoids problems - # on Github where experimental python-3.13 was not available via - # `py`, and is kept here in case a similar problems happens with - # future Python versions. - _log(f'{cpu=} {version=}: using {sys.executable=}.') - self.path = sys.executable - self.version = version - self.cpu = cpu - self.include = sysconfig.get_path('include') - - else: - command = 'py -0p' - if verbose: - _log(f'{cpu=} {version=}: Running: {command}') - text = subprocess.check_output( command, shell=True, text=True) - for line in text.split('\n'): - #_log( f' {line}') - if m := re.match( '^ *-V:([0-9.]+)(-32)? ([*])? +(.+)$', line): - version2 = m.group(1) - bits = 32 if m.group(2) else 64 - current = m.group(3) - path = m.group(4).strip() - elif m := re.match( '^ *-([0-9.]+)-((32)|(64)) +(.+)$', line): - version2 = m.group(1) - bits = int(m.group(2)) - path = m.group(5).strip() - else: - if verbose: - _log( f'No match for {line=}') - continue - if verbose: - _log( f'{version2=} {bits=} {path=} from {line=}.') - if bits != cpu.bits or version2 != version: - continue - root = os.path.dirname(path) - if not os.path.exists(path): - # Sometimes it seems that the specified .../python.exe does not exist, - # and we have to change it to .../python.exe. - # - assert path.endswith('.exe'), f'path={path!r}' - path2 = f'{path[:-4]}{version}.exe' - _log( f'Python {path!r} does not exist; changed to: {path2!r}') - assert os.path.exists( path2) - path = path2 - - self.path = path - self.version = version - self.cpu = cpu - command = f'{self.path} -c "import sysconfig; print(sysconfig.get_path(\'include\'))"' - _log(f'Finding Python include path by running {command=}.') - self.include = subprocess.check_output(command, shell=True, text=True).strip() - _log(f'Python include path is {self.include=}.') - #_log( f'pipcl.py:WindowsPython():\n{self.description_ml(" ")}') - break - else: - _log(f'Failed to find python matching cpu={cpu}.') - _log(f'Output from {command!r} was:\n{text}') - raise Exception( f'Failed to find python matching cpu={cpu} {version=}.') - - # Oddly there doesn't seem to be a - # `sysconfig.get_path('libs')`, but it seems to be next - # to `includes`: - self.libs = os.path.abspath(f'{self.include}/../libs') - - _log( f'WindowsPython:\n{self.description_ml(" ")}') - - def description_ml(self, indent=''): - ret = textwrap.dedent(f''' - path: {self.path} - version: {self.version} - cpu: {self.cpu} - include: {self.include} - libs: {self.libs} - ''') - return textwrap.indent( ret, indent) - - def __repr__(self): - return f'path={self.path!r} version={self.version!r} cpu={self.cpu!r} include={self.include!r} libs={self.libs!r}' - - -# Internal helpers. -# - -def _cpu_name(): - ''' - Returns `x32` or `x64` depending on Python build. - ''' - #log(f'sys.maxsize={hex(sys.maxsize)}') - return f'x{32 if sys.maxsize == 2**31 - 1 else 64}' - - - -def _log(text='', caller=1): - ''' - Logs lines with prefix. - ''' - pipcl.log1(text, caller+1) diff --git a/tests/test_pixmap.py b/tests/test_pixmap.py index 201fa8926..0fac14357 100644 --- a/tests/test_pixmap.py +++ b/tests/test_pixmap.py @@ -691,6 +691,57 @@ def test_5001(): wt = pymupdf.TOOLS.mupdf_warnings() assert wt + +def test_5001b(): + from PIL import ImageCms # only to fabricate an ICC profile + + icc = ImageCms.ImageCmsProfile(ImageCms.createProfile("sRGB")).tobytes() + bitmap = b"\xff" * 8 # 8x8 @ 1bpc, every bit 1 -> palette index 1 = FFFFFF (white) + rgb = b"\x00\x80\xff" * 4 # 2x2 companion image using the colorspace directly + draw = b"q 10 0 0 10 0 0 cm /Im1 Do Q q 80 0 0 30 10 10 cm /Im0 Do Q" + objs = { + 1: ("<< /Type /Catalog /Pages 2 0 R >>", None), + 2: ("<< /Type /Pages /Kids [3 0 R] /Count 1 >>", None), + 3: ("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 50] " + "/Resources << /XObject << /Im0 5 0 R /Im1 8 0 R >> >> /Contents 4 0 R >>", None), + 4: (f"<< /Length {len(draw)} >>", draw), + 5: ("<< /Type /XObject /Subtype /Image /Width 8 /Height 8 /BitsPerComponent 1 " + f"/ColorSpace [/Indexed 6 0 R 1 <000000FFFFFF>] /Length {len(bitmap)} >>", bitmap), + 6: ("[/ICCBased 7 0 R]", None), + 7: (f"<< /N 3 /Length {len(icc)} >>", icc), + 8: ("<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /BitsPerComponent 8 " + f"/ColorSpace 6 0 R /Length {len(rgb)} >>", rgb), + } + out, offsets = bytearray(b"%PDF-1.4\n"), {} + for num in sorted(objs): + head, data = objs[num] + offsets[num] = len(out) + out += f"{num} 0 obj\n{head}\n".encode() + if data is not None: + out += b"stream\n" + data + b"\nendstream\n" + out += b"endobj\n" + xref_pos = len(out) + out += f"xref\n0 {len(objs)+1}\n0000000000 65535 f \n".encode() + for num in sorted(objs): + out += f"{offsets[num]:010d} 00000 n \n".encode() + out += f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref_pos}\n%%EOF".encode() + + with pymupdf.open(stream=bytes(out), filetype="pdf") as doc: + pix = doc[0].get_pixmap(dpi=150) + p = pix.pixel(int(50 * 150 / 72), int(25 * 150 / 72)) + print("center of the indexed image:", p) + path_out = os.path.normpath(f'{__file__}/../../tests/test_5001b_out.png') + pix.save(path_out) + if pymupdf.mupdf_version_tuple >= (1, 28): + assert p == (255, 255, 255) + else: + wt = pymupdf.TOOLS.mupdf_warnings() + assert wt + # With pymupdf-1.27.2.3 we get incorrect (0, 0, 0). + # With current pymupdf and mupdf-master (as of 2026-06-15) we get + # correct (255, 255, 255). + + def test_natural(): if pymupdf.mupdf_version_tuple < (1, 28): print('test_natural(): Not running because segv fixed on mupdf master (1.28).') diff --git a/tests/test_pylint.py b/tests/test_pylint.py index e460bbe4d..b2cd215d0 100644 --- a/tests/test_pylint.py +++ b/tests/test_pylint.py @@ -122,11 +122,9 @@ def test_pylint(): 'fitz___init__.py', 'fitz_table.py', 'fitz_utils.py', - 'pipcl.py', 'pymupdf.py', 'table.py', 'utils.py', - 'wdev.py', ] leafs.sort() try: diff --git a/tests/test_release.py b/tests/test_release.py index 896ad46a7..9a763e3c0 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -1,5 +1,7 @@ import pymupdf +import pipcl + import os import re import sys @@ -8,12 +10,10 @@ g_root_abs = os.path.normpath(f'{__file__}/../../') sys.path.insert(0, g_root_abs) -sys.path.insert(0, f'{g_root_abs}/src') try: - import pipcl import setup finally: - del sys.path[0:2] + del sys.path[0] g_root = pipcl.relpath(g_root_abs) @@ -42,10 +42,10 @@ def test_release_versions(): ''' PyMuPDF and default MuPDF must have same major.minor version. ''' - version_p_tuple = [int(i) for i in setup.version_p.split('.')] - version_mupdf_tuple = [int(i) for i in setup.version_mupdf.split('.')] - assert version_p_tuple[:2] == version_mupdf_tuple[:2], \ - f'PyMuPDF and MuPDF major.minor versions do not match. {setup.version_p=} {setup.version_mupdf=}.' + version_pymupdf_tuple = pipcl.version_to_tuple(setup.version_p) + version_mupdf_tuple = pipcl.version_to_tuple(setup.version_mupdf) + assert version_pymupdf_tuple[:2] == version_mupdf_tuple[:2], \ + f'PyMuPDF and MuPDF major.minor versions do not match. {setup.version_pymupdf_tuple=} {setup.version_mupdf=}.' def test_release_bug_template(): @@ -80,7 +80,7 @@ def test_release_changelog_mupdf_version(): p = f'{g_root}/changes.txt' with open(p) as f: text = f.read() - m = re.search(f'\n[*] Use MuPDF-([0-9.]+)[.]\n', text) + m = re.search(f'\n[*] Use MuPDF-([0-9.]+(-rc[0-9])?)[.]\n', text) assert m, f'Cannot parse {p}.' assert m[1] == setup.version_mupdf, \ f'{_file_line(p, text, m)}: First mentioned MuPDF version does not match {setup.version_mupdf=}: {m[0].strip()!r}.'