Fix python packaging

kunitoki · kunitoki · commit 312b65c70e4d · 2025-12-07T15:54:55.000+01:00
diff --git a/cmake/yup_python.cmake b/cmake/yup_python.cmake
@@ -26,7 +26,7 @@ function (yup_prepare_python_stdlib target_name python_tools_path output_variabl
 
     cmake_parse_arguments (YUP_ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
 
-    set (default_ignored_library_patterns "lib2to3" "pydoc_data" "_xxtestfuzz*")
+    set (default_ignored_library_patterns "lib2to3")
 
     set (ignored_library_patterns ${default_ignored_library_patterns})
     list (APPEND ignored_library_patterns ${YUP_ARG_IGNORED_LIBRARY_PATTERNS})
diff --git a/python/tools/ArchivePythonStdlib.py b/python/tools/ArchivePythonStdlib.py
@@ -16,12 +16,68 @@ def file_hash(file):
     return h.hexdigest()
 
 
-def make_archive(file, directory, verbose=False):
+def should_ignore(path, patterns):
+    """Check if a path matches any of the ignore patterns using glob-style matching."""
+    path_obj = Path(path)
+
+    for pattern in patterns:
+        # Handle ** patterns for recursive matching
+        if pattern.startswith("**/"):
+            # Match pattern anywhere in the path
+            pattern_without_prefix = pattern[3:]
+            if any(part == pattern_without_prefix or Path(part).match(pattern_without_prefix) for part in path_obj.parts):
+                return True
+            # Also try matching the full pattern
+            if path_obj.match(pattern):
+                return True
+        elif "**" in pattern:
+            # Use pathlib match for patterns with **
+            if path_obj.match(pattern):
+                return True
+        else:
+            # For simple patterns, check if any path component matches
+            if any(Path(part).match(pattern) for part in path_obj.parts):
+                return True
+
+    return False
+
+
+def make_archive(file, directory, verbose=False, ignore_patterns=None):
     archived_files = []
-    for dirname, _, files in os.walk(directory):
+    for dirname, subdirs, files in os.walk(directory):
+        rel_dirname = os.path.relpath(dirname, directory)
+
+        # Skip directories that match ignore patterns
+        if ignore_patterns and rel_dirname != "." and should_ignore(rel_dirname, ignore_patterns):
+            if verbose:
+                print(f"Skipping directory: {rel_dirname}")
+            # Clear subdirs to prevent os.walk from descending into this directory
+            subdirs.clear()
+            continue
+
+        # Filter out subdirectories that match ignore patterns to prevent descending
+        if ignore_patterns:
+            filtered_subdirs = []
+            for subdir in subdirs:
+                rel_subdir = os.path.relpath(os.path.join(dirname, subdir), directory)
+                if should_ignore(rel_subdir, ignore_patterns):
+                    if verbose:
+                        print(f"Skipping subdirectory: {rel_subdir}")
+                else:
+                    filtered_subdirs.append(subdir)
+            subdirs[:] = filtered_subdirs
+
         for filename in files:
             path = os.path.join(dirname, filename)
-            archived_files.append((path, os.path.relpath(path, directory)))
+            archive_path = os.path.relpath(path, directory)
+
+            # Skip files that match ignore patterns
+            if ignore_patterns and should_ignore(archive_path, ignore_patterns):
+                if verbose:
+                    print(f"Skipping file: {archive_path}")
+                continue
+
+            archived_files.append((path, archive_path))
 
     with zipfile.ZipFile(file, "w") as zf:
         for path, archive_path in sorted(archived_files):
@@ -59,49 +115,181 @@ def make_archive(file, directory, verbose=False):
     final_archive = args.output_folder / f"python{version_nodot}.zip"
     temp_archive = args.output_folder / f"temp{version_nodot}.zip"
 
-    base_python: Path = args.root_folder
+    python_root: Path = args.root_folder
+    python_lib_name = f"python{version}"
+
+    # Determine the structure based on what we're given
+    # Case 1: We're given a path that ends with /lib (like Homebrew on macOS)
+    if python_root.name == "lib":
+        # Look for python{version} directly in this directory
+        base_python_lib = python_root / python_lib_name
+        if not base_python_lib.exists():
+            # Try without the dot (e.g., python313)
+            python_lib_name = f"python{version_nodot}"
+            base_python_lib = python_root / python_lib_name
+
+        if base_python_lib.exists():
+            lib_dir = python_root
+            bin_dir = python_root.parent / "bin"
+            include_dir = python_root.parent / "include"
+        else:
+            print(f"Error: Could not find Python library directory in {python_root}")
+            print(f"Expected: {python_root}/python{version}/")
+            exit(1)
+    else:
+        # Case 2: We're given a parent path, look for lib/python{version}
+        lib_dir = python_root / "lib"
+        bin_dir = python_root / "bin"
+        include_dir = python_root / "include"
+
+        base_python_lib = lib_dir / python_lib_name
+        if not base_python_lib.exists():
+            # Try without the dot (e.g., python313)
+            python_lib_name = f"python{version_nodot}"
+            base_python_lib = lib_dir / python_lib_name
+
+        if not base_python_lib.exists():
+            # Case 3: Maybe python_root itself is the python library directory
+            if (python_root / "os.py").exists():
+                base_python_lib = python_root
+                lib_dir = python_root.parent
+                bin_dir = lib_dir.parent / "bin" if lib_dir.name == "lib" else lib_dir / "bin"
+                include_dir = lib_dir.parent / "include" if lib_dir.name == "lib" else lib_dir / "include"
+            else:
+                print(f"Error: Could not find Python library directory in {python_root}")
+                print(f"Expected structure: {python_root}/lib/python{version}/")
+                exit(1)
+
+    print(f"Python root: {python_root}")
+    print(f"Python lib directory: {base_python_lib}")
+    print(f"Bin directory: {bin_dir if bin_dir.exists() else 'not found'}")
+    print(f"Include directory: {include_dir if include_dir.exists() else 'not found'}")
 
     base_patterns = [
         "**/*.pyc",
         "**/__pycache__",
         "**/__phello__",
-        "**/*config-3*",
-        "**/*tcl*",
-        "**/*tdbc*",
-        "**/*tk*",
-        "**/Tk*",
         "**/_tk*",
         "**/_test*",
-        "**/libpython*",
-        "**/pkgconfig",
+        "**/_xxtestfuzz*",
+        "**/*config-3*",
         "**/idlelib",
+        "**/pkgconfig",
+        "**/pydoc_data",
         "**/site-packages",
+        "**/*tcl*",
+        "**/*tdbc*",
+        "**/temp_*.txt",
         "**/test",
         "**/turtledemo",
-        "**/temp_*.txt",
+        "**/*tk*",
+        "**/Tk*",
         "**/.DS_Store",
         "**/EXTERNALLY-MANAGED",
         "**/LICENSE.txt",
     ]
 
+    # Add custom patterns from cmake (like "test", "lib2to3", etc.)
     if args.exclude_patterns:
-        custom_patterns = [x.strip() for x in args.exclude_patterns.replace('"', '').split(";")]
-        base_patterns += custom_patterns
+        custom_patterns = [x.strip() for x in args.exclude_patterns.replace('"', '').split(";") if x.strip()]
+        # Convert simple patterns to ** patterns for recursive matching
+        for pattern in custom_patterns:
+            if not pattern.startswith("**/") and "**" not in pattern and "/" not in pattern:
+                base_patterns.append(f"**/{pattern}")
+                base_patterns.append(f"**/{pattern}/**")
+            else:
+                base_patterns.append(pattern)
 
-    ignored_files = shutil.ignore_patterns(*base_patterns)
+    # Create a custom ignore function that uses glob matching for the lib directory
+    def ignore_lib_files(directory, files):
+        ignored = []
+        for name in files:
+            full_path = os.path.join(directory, name)
+            rel_path = os.path.relpath(full_path, base_python_lib)
+            if should_ignore(rel_path, base_patterns):
+                ignored.append(name)
+                if args.verbose:
+                    print(f"Ignoring during lib copy: {rel_path}")
+        return ignored
+
+    # Custom ignore function for bin directory (keep python binaries only)
+    def ignore_bin_files(directory, files):
+        _ = directory  # Required by shutil.copytree signature but not used
+        ignored = []
+        for name in files:
+            # Keep only python* executables
+            if not (name.startswith("python") and not name.endswith(".pyc")):
+                ignored.append(name)
+                if args.verbose:
+                    print(f"Ignoring during bin copy: {name}")
+        return ignored
 
     print(f"cleaning up {final_location}...")
     if final_location.exists():
         shutil.rmtree(final_location)
 
-    print(f"copying library from {base_python} to {final_location}...")
-    shutil.copytree(base_python, final_location, ignore=ignored_files, dirs_exist_ok=True)
+    # Create the lib/python{version} structure
+    final_lib_location = final_location / "lib" / python_lib_name
+    print(f"copying library from {base_python_lib} to {final_lib_location}...")
+    shutil.copytree(base_python_lib, final_lib_location, ignore=ignore_lib_files, dirs_exist_ok=True)
+
+    # Copy bin directory if it exists
+    if bin_dir.exists():
+        final_bin_location = final_location / "bin"
+        print(f"copying binaries from {bin_dir} to {final_bin_location}...")
+        shutil.copytree(bin_dir, final_bin_location, ignore=ignore_bin_files, dirs_exist_ok=True)
+    else:
+        print(f"Warning: bin directory not found at {bin_dir}")
+
+    # Copy include directory if it exists
+    if include_dir.exists():
+        # Look for include/python{version} subdirectory
+        include_python_subdir = include_dir / python_lib_name
+        if include_python_subdir.exists():
+            # Copy include/python{version} to final_location/include/python{version}
+            final_include_location = final_location / "include" / python_lib_name
+            print(f"copying headers from {include_python_subdir} to {final_include_location}...")
+            shutil.copytree(include_python_subdir, final_include_location, dirs_exist_ok=True)
+        else:
+            # Copy entire include directory
+            final_include_location = final_location / "include"
+            print(f"copying headers from {include_dir} to {final_include_location}...")
+            shutil.copytree(include_dir, final_include_location, dirs_exist_ok=True)
+    else:
+        print(f"Warning: include directory not found at {include_dir}")
+
+    # Update site_packages path to the new location
+    site_packages = final_lib_location / "site-packages"
     os.makedirs(site_packages, exist_ok=True)
 
+    # Move libpython library files to lib/ directory
+    # These are typically in config-3* directories but should be in lib/
+    print(f"searching for libpython library files to move to lib/...")
+    lib_extensions = [".dylib", ".dll", ".a", ".lib", ".so"]
+    final_lib_root = final_location / "lib"
+
+    moved_files = []
+    for root, dirs, files in os.walk(final_lib_location):
+        for file in files:
+            if file.startswith("libpython") and any(file.endswith(ext) for ext in lib_extensions):
+                source_path = Path(root) / file
+                # Move to lib/ directory (not lib/python{version}/)
+                dest_path = final_lib_root / file
+
+                # Only move if it's not already in the lib/ directory
+                if source_path.parent != final_lib_root:
+                    shutil.move(str(source_path), str(dest_path))
+                    moved_files.append(file)
+                    if args.verbose:
+                        print(f"Moved {file} from {source_path.parent} to {dest_path.parent}")
+
+    if moved_files:
+        print(f"Moved {len(moved_files)} libpython library file(s) to lib/")
+
     print(f"making archive {temp_archive} to {final_archive}...")
     if os.path.exists(final_archive):
-        make_archive(temp_archive, final_location, verbose=args.verbose)
+        make_archive(temp_archive, final_location, verbose=args.verbose, ignore_patterns=base_patterns)
         if file_hash(temp_archive) != file_hash(final_archive):
             shutil.copy(temp_archive, final_archive)
     else:
-        make_archive(final_archive, final_location)
+        make_archive(final_archive, final_location, ignore_patterns=base_patterns)
