Merge branch 'main' into load-from-memview

Author: JorjMcKie

.github/workflows/test_sysinstall.yml

@@ -4,6 +4,9 @@ on:
   schedule:
     - cron: '13 4 * * *'
   workflow_dispatch:
+    inputs:
+      args:
+        description: 'Extra args for scripts/sysinstall.py.'
 
 jobs:
 
@@ -31,12 +34,17 @@ jobs:
         #  # sees `--venv` and defers to a venv, so we currently have to force use of python 3.11.
         #  python-version: '3.11'
 
+      
       - name: sysinstall_venv
+        env:
+            PYMUDF_SCRIPTS_SYSINSTALL_ARGS_POST: ${{inputs.args}}
         run:
           # Use venv.
           python3 scripts/sysinstall.py --mupdf-git '--branch master https://github.com/ArtifexSoftware/mupdf.git'
 
       - name: sysinstall_sudo
+        env:
+            PYMUDF_SCRIPTS_SYSINSTALL_ARGS_POST: ${{inputs.args}}
         run:
           # Do not use a venv, instead install required packages with sudo.
           python3 scripts/sysinstall.py --mupdf-git '--branch master https://github.com/ArtifexSoftware/mupdf.git' --pip sudo --root /

changes.txt

@@ -24,6 +24,7 @@ Change Log
   * Add Widget Support to `Document.insert_pdf()`.
   * Add `bibi` to span dicts.
   * Add `synthetic' to char dict.
+  * Fixed Pyodide builds.
 
 
 **Changes in version 1.25.2 (2025-01-17)**

docs/about-feature-matrix.rst

@@ -448,7 +448,7 @@
         <tr>
             <td><cite id="transFM43">PDF Page Labels</cite></td>
             <td class="yes"></td>
-            <td class="no"></td>
+            <td class="limited">Read-only</td>
             <td class="no"></td>
             <td class="no"></td>
             <td class="no"></td>

docs/document.rst

@@ -1294,9 +1294,10 @@ For details on **embedded files** refer to Appendix 3.
      pair: links; Document.insert_pdf
      pair: annots; Document.insert_pdf
      pair: widgets; Document.insert_pdf
+     pair: join_duplicates; Document.insert_pdf
      pair: show_progress; Document.insert_pdf
 
-  .. method:: insert_pdf(docsrc, from_page=-1, to_page=-1, start_at=-1, rotate=-1, links=True, annots=True, widgets=True, show_progress=0, final=1)
+  .. method:: insert_pdf(docsrc, from_page=-1, to_page=-1, start_at=-1, rotate=-1, links=True, annots=True, widgets=True, join_duplicates=False, show_progress=0, final=1)
 
     PDF only: Copy the page range **[from_page, to_page]** (including both) of PDF document *docsrc* into the current one. Inserts will start with page number *start_at*. Value -1 indicates default values. All pages thus copied will be rotated as specified. Links, annotations and widgets can be excluded in the target, see below. All page numbers are 0-based.
 
@@ -1312,9 +1313,19 @@ For details on **embedded files** refer to Appendix 3.
     :arg int rotate: All copied pages will be rotated by the provided value (degrees, integer multiple of 90).
 
     :arg bool links: Choose whether (internal and external) links should be included in the copy. Default is `True`. *Named* links (:data:`LINK_NAMED`) and internal links to outside the copied page range are **always excluded**. 
+    
     :arg bool annots: choose whether annotations should be included in the copy.
+    
     :arg bool widgets: choose whether annotations should be included in the copy. If `True` and at least one of the source pages contains form fields, the target PDF will be turned into a Form PDF (if not already being one).
+    
+    :arg bool join_duplicates: *(New in version 1.25.5)* Choose how to handle duplicate root field names in the source pages. This parameter is ignored if `widgets=False`.
+    
+      Default is ``False`` which will add unifying strings to the name of those source root fields which have a duplicate in the target. For instance, if "name" already occurs in the target, the source widget's name will be changed to "name [text]" with a suitably chosen string "text".
+
+      If ``True``, root fields with duplicate names in source and target will be converted to so-called "Kids" of a "Parent" object (which lists all kid widgets in a PDF array). This will effectively turn those kids into instances of the "same" widget: if e.g. one of the kids is changed, then all its instances will automatically inherit this change -- no matter on which page they happen to be displayed.
+    
     :arg int show_progress: *(new in v1.17.7)* specify an interval size greater zero to see progress messages on `sys.stdout`. After each interval, a message like `Inserted 30 of 47 pages.` will be printed.
+    
     :arg int final: *(new in v1.18.0)* controls whether the list of already copied objects should be **dropped** after this method, default *True*. Set it to 0 except for the last one of multiple insertions from the same source PDF. This saves target file size and speeds up execution considerably.
 
   .. note::

scripts/sysinstall.py

@@ -118,6 +118,8 @@ def main():
         log(f'{sys.executable=}')
         log(f'{platform.python_version()=}')
         log(f'{__file__=}')
+        log(f'{os.environ.get("PYMUDF_SCRIPTS_SYSINSTALL_ARGS_PRE")=}')
+        log(f'{os.environ.get("PYMUDF_SCRIPTS_SYSINSTALL_ARGS_POST")=}')
         log(f'{sys.argv=}')
         log(f'{sysconfig.get_path("platlib")=}')
         run_command(f'python -V', check=0)
@@ -152,7 +154,9 @@ def main():
 
     # Parse command-line.
     #
-    args = iter(sys.argv[1:])
+    env_args_pre = shlex.split(os.environ.get('PYMUDF_SCRIPTS_SYSINSTALL_ARGS_PRE', ''))
+    env_args_post = shlex.split(os.environ.get('PYMUDF_SCRIPTS_SYSINSTALL_ARGS_POST', ''))
+    args = iter(env_args_pre + sys.argv[1:] + env_args_post)
     while 1:
         try:
             arg = next(args)
@@ -240,6 +244,9 @@ def run(command, env_extra=None):
     command += f' HAVE_LEPTONICA=yes'
     command += f' HAVE_TESSERACT=yes'
     command += f' USE_SYSTEM_LIBS=yes'
+    # We need latest zxingcpp so system version not ok.
+    command += f' USE_SYSTEM_ZXINGCPP=no'
+    command += f' barcode=yes'
     command += f' VENV_FLAG={"--venv" if pip == "venv" else ""}'
     if mupdf_so_mode:
         command += f' SO_INSTALL_MODE={mupdf_so_mode}'
@@ -291,7 +298,7 @@ def run(command):
         run(f'{sudo}rm -r {p}/site-packages/pymupdf.py || true')
         run(f'{sudo}rm -r {p}/site-packages/fitz || true')
         run(f'{sudo}rm -r {p}/site-packages/fitz.py || true')
-        run(f'{sudo}rm -r {p}/site-packages/PyMuPDF-*.dist-info || true')
+        run(f'{sudo}rm -r {p}/site-packages/pymupdf-*.dist-info || true')
         run(f'{sudo}rm -r {root_prefix}/bin/pymupdf || true')
         if pip == 'venv':
             run(f'{sudo}{venv_name}/bin/python -m installer --destdir {root} --prefix {prefix} {wheel}')

scripts/test.py

@@ -98,6 +98,10 @@
         Experimental, for investigating
         https://github.com/pymupdf/PyMuPDF/issues/3869. Runs run basic code
         inside C++ pybind. Requires `sudo apt install pybind11-dev` or similar.
+    --pymupdf-pypi <name>
+        Do not build PyMuPDF, instead install with `pip install <name>`. For
+        example allows testing of a specific version with `--pymupdf-pypi
+        pymupdf==1.25.0`.
     --system-site-packages 0|1
         If 1, use `--system-site-packages` when creating venv. Defaults is 0.
     --timeout <seconds>
@@ -123,6 +127,11 @@
         `pyodide build`. This runs our setup.py with CC etc set up
         to create Pyodide binaries in a wheel called, for example,
         `PyMuPDF-1.23.2-cp311-none-emscripten_3_1_32_wasm32.whl`.
+        
+        It seems that sys.version must match the Python version inside emsdk;
+        as of 2025-02-14 this is 3.12. Otherwise we get build errors such as:
+            [wasm-validator error in function 723] unexpected false: all used features should be allowed, on ...
+            
 
 Environment:
     PYMUDF_SCRIPTS_TEST_options
@@ -159,6 +168,9 @@ def main(argv):
     if len(argv) == 1:
         show_help()
         return
+    
+    log(f'{sys.executable=}')
+    log(f'{sys.version=}')
 
     build_isolation = None
     valgrind = False
@@ -180,6 +192,7 @@ def main(argv):
     system_site_packages = False
     pyodide_build_version = None
     packages = False
+    pymupdf_pypi = None
 
     options = os.environ.get('PYMUDF_SCRIPTS_TEST_options', '')
     options = shlex.split(options)
@@ -245,6 +258,8 @@ def main(argv):
             valgrind_args = next(args)
         elif arg == '--pyodide-build-version':
             pyodide_build_version = next(args)
+        elif arg == '--pymupdf-pypi':
+            pymupdf_pypi = next(args)
         else:
             assert 0, f'Unrecognised option: {arg=}.'
 
@@ -276,14 +291,17 @@ def main(argv):
         return
 
     def do_build(wheel=False):
-        build(
-                build_type=build_type,
-                build_isolation=build_isolation,
-                venv_quick=venv_quick,
-                build_mupdf=build_mupdf,
-                build_flavour=build_flavour,
-                wheel=wheel,
-                )
+        if pymupdf_pypi:
+            run(f'pip install --force-reinstall {pymupdf_pypi}')
+        else:
+            build(
+                    build_type=build_type,
+                    build_isolation=build_isolation,
+                    venv_quick=venv_quick,
+                    build_mupdf=build_mupdf,
+                    build_flavour=build_flavour,
+                    wheel=wheel,
+                    )
     def do_test():
         test(
                 implementations=implementations,

setup.py

@@ -152,6 +152,11 @@
     
     PYMUPDF_SETUP_PY_LIMITED_API
         If not '0', we build for current Python's stable ABI.
+        
+        However if unset and we are on Python-3.13 or later, we do
+        not build for the stable ABI because as of 2025-03-04 SWIG
+        generates incorrect stable ABI code with Python-3.13 - see:
+        https://github.com/swig/swig/issues/3059
     
     PYMUPDF_SETUP_URL_WHEEL
         If set, we use an existing wheel instead of building a new wheel.
@@ -208,6 +213,11 @@ def log( text):
     sys.stdout.flush()
 
 
+def run(command, check=1):
+    log(f'Running: {command}')
+    return subprocess.run( command, shell=1, check=check)
+
+
 if 1:
     # For debugging.
     log(f'### Starting.')
@@ -218,6 +228,7 @@ def log( text):
     log(f'CPU bits: {32 if sys.maxsize == 2**31 - 1 else 64} {sys.maxsize=}')
     log(f'__file__: {__file__!r}')
     log(f'os.getcwd(): {os.getcwd()!r}')
+    log(f'getconf ARG_MAX: {pipcl.run("getconf ARG_MAX", capture=1, check=0, verbose=0)!r}')
     log(f'sys.argv ({len(sys.argv)}):')
     for i, arg in enumerate(sys.argv):
         log(f'    {i}: {arg!r}')
@@ -236,10 +247,16 @@ def log( text):
 # Name of file that identifies that we are in a PyMuPDF sdist.
 g_pymupdfb_sdist_marker = 'pymupdfb_sdist'
 
+python_version_tuple = tuple(int(x) for x in platform.python_version_tuple()[:2])
+
 PYMUPDF_SETUP_PY_LIMITED_API = os.environ.get('PYMUPDF_SETUP_PY_LIMITED_API')
 assert PYMUPDF_SETUP_PY_LIMITED_API in (None, '', '0', '1'), \
         f'Should be "", "0", "1" or undefined: {PYMUPDF_SETUP_PY_LIMITED_API=}.'
-g_py_limited_api = (PYMUPDF_SETUP_PY_LIMITED_API != '0')
+if PYMUPDF_SETUP_PY_LIMITED_API is None and python_version_tuple >= (3, 13):
+    log(f'Not defaulting to Python limited api because {platform.python_version_tuple()=}.')
+    g_py_limited_api = False
+else:
+    g_py_limited_api = (PYMUPDF_SETUP_PY_LIMITED_API != '0')
 
 PYMUPDF_SETUP_URL_WHEEL =  os.environ.get('PYMUPDF_SETUP_URL_WHEEL')
 log(f'{PYMUPDF_SETUP_URL_WHEEL=}')
@@ -271,11 +288,6 @@ def error_fn(fn, path, excinfo):
     assert not os.path.exists( path)
 
 
-def run(command, check=1):
-    log(f'Running: {command}')
-    return subprocess.run( command, shell=1, check=check)
-
-
 def _git_get_branch( directory):
     command = f'cd {directory} && git branch --show-current'
     log( f'Running: {command}')
@@ -364,7 +376,7 @@ def tar_extract(path, mode='r:gz', prefix=None, exists='raise'):
     return prefix_actual
 
 
-def get_git_id( directory):
+def git_info( directory):
     '''
     Returns `(sha, comment, diff, branch)`, all items are str or None if not
     available.
@@ -390,10 +402,42 @@ def get_git_id( directory):
             )
     if cp.returncode == 0:
         branch = cp.stdout.strip()
-    log(f'get_git_id(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}')
+    log(f'git_info(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}')
     return sha, comment, diff, branch
 
 
+def git_patch(directory, patch, hard=False):
+    '''
+    Applies string <patch> with `git patch` in <directory>.
+    
+    If <hard> is true we clean the tree with `git checkout .` and then apply
+    the patch.
+
+    Otherwise we apply patch only if it is not already applied; this might fail
+    if there are conflicting changes in the tree.
+    '''
+    log(f'Applying patch in {directory}:\n{textwrap.indent(patch, "    ")}')
+    if not patch:
+        return
+    # Carriage returns break `git apply` so we use `newline='\n'` in open().
+    path = os.path.abspath(f'{directory}/pymupdf_patch.txt')
+    with open(path, 'w', newline='\n') as f:
+        f.write(patch)
+    log(f'Using patch file: {path}')
+    if hard:
+        run(f'cd {directory} && git checkout .')
+        run(f'cd {directory} && git apply {path}')
+        log(f'Have applied patch in {directory}.')
+    else:
+        e = run( f'cd {directory} && git apply --check --reverse {path}', check=0)
+        if e == 0:
+            log(f'Not patching {directory} because already patched.')
+        else:
+            run(f'cd {directory} && git apply {path}')
+            log(f'Have applied patch in {directory}.')
+    run(f'cd {directory} && git diff')
+
+
 mupdf_tgz = os.path.abspath( f'{__file__}/../mupdf.tgz')
 
 def get_mupdf_internal(out, location=None, sha=None, local_tgz=None):
@@ -444,7 +488,8 @@ def get_mupdf_internal(out, location=None, sha=None, local_tgz=None):
         if e:
             # No existing git checkout, so do a fresh clone.
             _fs_remove(local_dir)
-            run(f'git clone --recursive --depth 1 --shallow-submodules {location[4:]} {local_dir}')
+            gitargs = location[4:]
+            run(f'git clone --recursive --depth 1 --shallow-submodules {gitargs} {local_dir}')
 
         # Show sha of checkout.
         run( f'cd {local_dir} && git show --pretty=oneline|head -n 1', check=False)
@@ -856,6 +901,34 @@ def build_mupdf_unix(
 
     if openbsd or freebsd:
         env_add(env, 'CXX', 'c++', ' ')
+    
+    if darwin and os.environ.get('GITHUB_ACTIONS') == 'true':
+        if os.environ.get('ImageOS') == 'macos13':
+            # On Github macos13 we need to use Clang/LLVM (Homebrew) 15.0.7,
+            # otherwise mupdf:thirdparty/tesseract/src/api/baseapi.cpp fails to
+            # compile with:
+            #
+            #   thirdparty/tesseract/src/api/baseapi.cpp:150:25: error: 'recursive_directory_iterator' is unavailable: introduced in macOS 10.15
+            #
+            # See:
+            #   https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
+            #
+            log(f'Using llvm@15 clang and clang++')
+            cl15 = pipcl.run(f'brew --prefix llvm@15', capture=1)
+            log(f'{cl15=}')
+            cl15 = cl15.strip()
+            pipcl.run(f'ls -lL {cl15}')
+            pipcl.run(f'ls -lL {cl15}/bin')
+            cc = f'{cl15}/bin/clang'
+            cxx = f'{cl15}/bin/clang++'
+            env['CC'] = cc
+            env['CXX'] = cxx
+    
+    # Show compiler versions.
+    cc = env.get('CC', 'cc')
+    cxx = env.get('CXX', 'c++')
+    pipcl.run(f'{cc} --version')
+    pipcl.run(f'{cxx} --version')
 
     # Add extra flags for MacOS cross-compilation, where ARCHFLAGS can be
     # '-arch arm64'.
@@ -865,6 +938,8 @@ def build_mupdf_unix(
         env_add(env, 'XCFLAGS', archflags)
         env_add(env, 'XLIBS', archflags)
 
+    mupdf_version_tuple = get_mupdf_version(mupdf_local)
+    
     # We specify a build directory path containing 'pymupdf' so that we
     # coexist with non-PyMuPDF builds (because PyMuPDF builds have a
     # different config.h).
@@ -877,7 +952,16 @@ def build_mupdf_unix(
     # $_PYTHON_HOST_PLATFORM allows cross-compiled cibuildwheel builds
     # to coexist, e.g. on github.
     #
+    # Have experimented with looking at getconf_ARG_MAX to decide whether to
+    # omit `PyMuPDF-` from the build directory, to avoid command-too-long
+    # errors with mupdf-1.26. But it seems that `getconf ARG_MAX` returns
+    # a system limit, not the actual limit of the current shell, and there
+    # doesn't seem to be a way to find the current shell's limit.
+    #
     build_prefix = f'PyMuPDF-'
+    if mupdf_version_tuple >= (1, 26):
+        # Avoid link command length problems seen on musllinux.
+        build_prefix = ''
     if pyodide:
         build_prefix += 'pyodide-'
     else:
@@ -894,7 +978,6 @@ def build_mupdf_unix(
         log(f'PYMUPDF_SETUP_MUPDF_TESSERACT=0 so building mupdf without tesseract.')
     else:
         build_prefix += 'tesseract-'
-    mupdf_version_tuple = get_mupdf_version(mupdf_local)
     if (
             linux
             and os.environ.get('PYMUPDF_SETUP_MUPDF_BSYMBOLIC', '1') == '1'