diff --git a/relenv/_resources/xz/config.h b/relenv/_resources/xz/config.h new file mode 100644 index 00000000..ef921e80 --- /dev/null +++ b/relenv/_resources/xz/config.h @@ -0,0 +1,148 @@ +/* config.h for compiling liblzma (*not* the whole XZ Utils) with MSVC 2019 */ + +/* Prefix for symbols exported by tuklib_*.c files */ +#define TUKLIB_SYMBOL_PREFIX lzma_ + +/* How many MiB of RAM to assume if the real amount cannot be determined. */ +#define ASSUME_RAM 128 + +/* Define to 1 if crc32 integrity check is enabled. */ +#define HAVE_CHECK_CRC32 1 + +/* Define to 1 if crc64 integrity check is enabled. */ +#define HAVE_CHECK_CRC64 1 + +/* Define to 1 if sha256 integrity check is enabled. */ +#define HAVE_CHECK_SHA256 1 + +/* Define to 1 if any of HAVE_DECODER_foo have been defined. */ +#define HAVE_DECODERS 1 + +/* Define to 1 if arm decoder is enabled. */ +#define HAVE_DECODER_ARM 1 + +/* Define to 1 if armthumb decoder is enabled. */ +#define HAVE_DECODER_ARMTHUMB 1 + +/* Define to 1 if delta decoder is enabled. */ +#define HAVE_DECODER_DELTA 1 + +/* Define to 1 if ia64 decoder is enabled. */ +#define HAVE_DECODER_IA64 1 + +/* Define to 1 if lzma1 decoder is enabled. */ +#define HAVE_DECODER_LZMA1 1 + +/* Define to 1 if lzma2 decoder is enabled. */ +#define HAVE_DECODER_LZMA2 1 + +/* Define to 1 if powerpc decoder is enabled. */ +#define HAVE_DECODER_POWERPC 1 + +/* Define to 1 if sparc decoder is enabled. */ +#define HAVE_DECODER_SPARC 1 + +/* Define to 1 if x86 decoder is enabled. */ +#define HAVE_DECODER_X86 1 + +/* Define to 1 if any of HAVE_ENCODER_foo have been defined. */ +#define HAVE_ENCODERS 1 + +/* Define to 1 if arm encoder is enabled. */ +#define HAVE_ENCODER_ARM 1 + +/* Define to 1 if armthumb encoder is enabled. */ +#define HAVE_ENCODER_ARMTHUMB 1 + +/* Define to 1 if delta encoder is enabled. */ +#define HAVE_ENCODER_DELTA 1 + +/* Define to 1 if ia64 encoder is enabled. */ +#define HAVE_ENCODER_IA64 1 + +/* Define to 1 if lzma1 encoder is enabled. */ +#define HAVE_ENCODER_LZMA1 1 + +/* Define to 1 if lzma2 encoder is enabled. */ +#define HAVE_ENCODER_LZMA2 1 + +/* Define to 1 if powerpc encoder is enabled. */ +#define HAVE_ENCODER_POWERPC 1 + +/* Define to 1 if sparc encoder is enabled. */ +#define HAVE_ENCODER_SPARC 1 + +/* Define to 1 if x86 encoder is enabled. */ +#define HAVE_ENCODER_X86 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 to enable bt2 match finder. */ +#define HAVE_MF_BT2 1 + +/* Define to 1 to enable bt3 match finder. */ +#define HAVE_MF_BT3 1 + +/* Define to 1 to enable bt4 match finder. */ +#define HAVE_MF_BT4 1 + +/* Define to 1 to enable hc3 match finder. */ +#define HAVE_MF_HC3 1 + +/* Define to 1 to enable hc4 match finder. */ +#define HAVE_MF_HC4 1 + +/* Define to 1 if stdbool.h conforms to C99. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 or 0, depending whether the compiler supports simple visibility + declarations. */ +#define HAVE_VISIBILITY 0 + +/* Define to 1 if the system has the type `_Bool'. */ +#define HAVE__BOOL 1 + +#ifdef _M_IX86 +/* Define to 1 when using Windows 95 (and thus XP) compatible threads. This + avoids use of features that were added in Windows Vista. + This is used for 32-bit x86 builds for compatibility reasons since it + makes no measurable difference in performance compared to Vista threads. */ +#define MYTHREAD_WIN95 1 +#else +/* Define to 1 when using Windows Vista compatible threads. This uses features + that are not available on Windows XP. */ +#define MYTHREAD_VISTA 1 +#endif + +/* Define to 1 to disable debugging code. */ +#define NDEBUG 1 + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "XZ Utils" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "https://tukaani.org/xz/" + +/* The size of `size_t', as computed by sizeof. */ +#ifdef _WIN64 +#define SIZEOF_SIZE_T 8 +#else +#define SIZEOF_SIZE_T 4 +#endif + +/* Define to 1 if the system supports fast unaligned access to 16-bit and + 32-bit integers. */ +#define TUKLIB_FAST_UNALIGNED_ACCESS 1 diff --git a/relenv/_resources/xz/readme.md b/relenv/_resources/xz/readme.md new file mode 100644 index 00000000..96e13414 --- /dev/null +++ b/relenv/_resources/xz/readme.md @@ -0,0 +1,4 @@ +The config.h file was removed from XZ-Utils tarting with version 5.5.0. +XZ-Utils seems to build just fine with the config.h file from 5.4.7, so we're +including it here. This will be copied into the src/windows directory in the +extracted source for XZ-Utils. diff --git a/relenv/build/common.py b/relenv/build/common.py index 85f4221b..7083c375 100644 --- a/relenv/build/common.py +++ b/relenv/build/common.py @@ -38,6 +38,7 @@ runcmd, work_dirs, fetch_url, + Version, ) import relenv.relocate @@ -358,6 +359,99 @@ def build_sqlite(env, dirs, logfp): runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) +def update_ensurepip(directory): + """ + Update bundled dependencies for ensurepip (pip & setuptools). + """ + # ensurepip bundle location + bundle_dir = directory / "ensurepip" / "_bundled" + + # Make sure the destination directory exists + bundle_dir.mkdir(parents=True, exist_ok=True) + + # Detect existing whl. Later versions of python don't include setuptools. We + # only want to update whl files that python expects to be there + pip_version = "25.2" + setuptools_version = "80.9.0" + update_pip = False + update_setuptools = False + for file in bundle_dir.glob("*.whl"): + + log.debug("Checking whl: %s", str(file)) + if file.name.startswith("pip-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if Version(found_version) >= Version(pip_version): + log.debug("Found correct pip version or newer: %s", found_version) + else: + file.unlink() + update_pip = True + if file.name.startswith("setuptools-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if Version(found_version) >= Version(setuptools_version): + log.debug( + "Found correct setuptools version or newer: %s", found_version + ) + else: + file.unlink() + update_setuptools = True + + # Download whl files and update __init__.py + init_file = directory / "ensurepip" / "__init__.py" + if update_pip: + whl = f"pip-{pip_version}-py3-none-any.whl" + whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() + + # Update __init__.py + old = "^_PIP_VERSION.*" + new = f'_PIP_VERSION = "{pip_version}"' + patch_file(path=init_file, old=old, new=new) + + # setuptools + if update_setuptools: + whl = f"setuptools-{setuptools_version}-py3-none-any.whl" + whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() + + # setuptools + old = "^_SETUPTOOLS_VERSION.*" + new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' + patch_file(path=init_file, old=old, new=new) + + log.debug("ensurepip __init__.py contents:") + log.debug(init_file.read_text()) + + +def patch_file(path, old, new): + """ + Search a file line by line for a string to replace. + + :param path: Location of the file to search + :type path: str + :param old: The value that will be replaced + :type path: str + :param new: The value that will replace the 'old' value. + :type path: str + """ + log.debug("Patching file: %s", path) + import re + + with open(path, "r") as fp: + content = fp.read() + new_content = "" + for line in content.splitlines(): + line = re.sub(old, new, line) + new_content += line + "\n" + with open(path, "w") as fp: + fp.write(new_content) + + def tarball_version(href): if href.endswith("tar.gz"): try: @@ -1457,6 +1551,9 @@ def find_pythonlib(libdir): pymodules = libdir / find_pythonlib(libdir) + # update ensurepip + update_ensurepip(pymodules) + cwd = os.getcwd() modname = find_sysconfigdata(pymodules) path = sys.path @@ -1474,6 +1571,7 @@ def find_pythonlib(libdir): bindir = pathlib.Path(dirs.prefix) / "bin" sitepackages = pymodules / "site-packages" install_runtime(sitepackages) + # Install pip python = dirs.prefix / "bin" / "python3" if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: diff --git a/relenv/build/linux.py b/relenv/build/linux.py index c7e754e6..f1b636de 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -379,11 +379,11 @@ def build_python(env, dirs, logfp): ) if pathlib.Path("setup.py").exists(): - with tempfile.NamedTemporaryFile(mode="w", suffix="_patch") as patch_file: - patch_file.write(PATCH) - patch_file.flush() + with tempfile.NamedTemporaryFile(mode="w", suffix="_patch") as p_file: + p_file.write(PATCH) + p_file.flush() runcmd( - ["patch", "-p0", "-i", patch_file.name], + ["patch", "-p0", "-i", p_file.name], env=env, stderr=logfp, stdout=logfp, diff --git a/relenv/build/windows.py b/relenv/build/windows.py index a92425ff..8e7a4f27 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -4,13 +4,23 @@ The windows build process. """ import glob -import shutil -import sys +import logging import os import pathlib +import shutil +import sys import tarfile -import logging -from .common import runcmd, create_archive, MODULE_DIR, builds, install_runtime +from .common import ( + builds, + create_archive, + download_url, + extract_archive, + install_runtime, + MODULE_DIR, + patch_file, + runcmd, + update_ensurepip, +) from ..common import arches, WIN32 log = logging.getLogger(__name__) @@ -36,44 +46,48 @@ def populate_env(env, dirs): env["MSBUILDDISABLENODEREUSE"] = "1" -def patch_file(path, old, new): +def update_props(source, old, new): """ - Search a file line by line for a string to replace. - - :param path: Location of the file to search - :type path: str - :param old: The value that will be replaced - :type path: str - :param new: The value that will replace the 'old' value. - :type path: str - """ - import re - - with open(path, "r") as fp: - content = fp.read() - new_content = "" - for line in content.splitlines(): - re.sub(old, new, line) - new_content += line + os.linesep - with open(path, "w") as fp: - fp.write(new_content) - - -def override_dependency(source, old, new): - """ - Overwrite a dependency string for Windoes PCBuild. + Overwrite a dependency string for Windows PCBuild. :param source: Python's source directory - :type path: str + :type source: str :param old: Regular expression to search for - :type path: str + :type old: str :param new: Replacement text - :type path: str + :type new: str """ patch_file(source / "PCbuild" / "python.props", old, new) patch_file(source / "PCbuild" / "get_externals.bat", old, new) +def get_externals_source(externals_dir, url): + """ + Download external source code dependency. + + Download source code and extract to the "externals" directory in the root of + the python source. Only works with a tarball + """ + zips_dir = externals_dir / "zips" + zips_dir.mkdir(parents=True, exist_ok=True) + local_file = download_url(url=url, dest=str(zips_dir)) + extract_archive(archive=str(local_file), to_dir=str(externals_dir)) + try: + os.remove(local_file) + except OSError: + log.exception("Failed to remove temporary file") + + +def get_externals_bin(source_root, url): + """ + Download external binary dependency. + + Download binaries to the "externals" directory in the root of the python + source. + """ + pass + + def build_python(env, dirs, logfp): """ Run the commands to build Python. @@ -86,12 +100,39 @@ def build_python(env, dirs, logfp): :type logfp: file """ # Override default versions - if env["RELENV_PY_MAJOR_VERSION"] in [ - "3.10", - "3.11", - ]: - override_dependency(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", "sqlite-3.50.4.0") - override_dependency(dirs.source, r"xz-\d+.\d+.\d+", "xz-5.6.2") + + # Create externals directory + externals_dir = dirs.source / "externals" + externals_dir.mkdir(parents=True, exist_ok=True) + + # SQLITE + if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12"]: + version = "3.50.4.0" + target_dir = externals_dir / f"sqlite-{version}" + if not target_dir.exists(): + update_props(dirs.source, r"sqlite-\d+.\d+.\d+.\d+", f"sqlite-{version}") + url = "https://sqlite.org/2025/sqlite-autoconf-3500400.tar.gz" + get_externals_source(externals_dir=externals_dir, url=url) + # # we need to fix the name of the extracted directory + extracted_dir = externals_dir / "sqlite-autoconf-3500400" + shutil.move(str(extracted_dir), str(target_dir)) + + # XZ-Utils + if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13", "3.14"]: + version = "5.6.2" + target_dir = externals_dir / f"xz-{version}" + if not target_dir.exists(): + update_props(dirs.source, r"xz-\d+.\d+.\d+", f"xz-{version}") + url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz" + get_externals_source(externals_dir=externals_dir, url=url) + # Starting with version v5.5.0, XZ-Utils removed the ability to compile + # with MSBuild. We are bringing the config.h from the last version that + # had it, 5.4.7 + config_file = target_dir / "windows" / "config.h" + config_file = target_dir / "src" / "common" / "config.h" + config_file_source = dirs.root / "_resources" / "xz" / "config.h" + if not config_file.exists(): + shutil.copy(str(config_file_source), str(config_file)) arch_to_plat = { "amd64": "x64", @@ -106,6 +147,7 @@ def build_python(env, dirs, logfp): plat, "--no-tkinter", ] + log.info("Start PCbuild") runcmd(cmd, env=env, stderr=logfp, stdout=logfp) log.info("PCbuild finished") @@ -202,9 +244,11 @@ def finalize(env, dirs, logfp): """ # Lay down site customize sitepackages = dirs.prefix / "Lib" / "site-packages" - install_runtime(sitepackages) + # update ensurepip + update_ensurepip(dirs.prefix / "Lib") + # Install pip python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) @@ -243,6 +287,7 @@ def runpip(pkg): "*.pyd", "*.dll", "*.lib", + "*.whl", "/Include/*", "/Lib/site-packages/*", ] diff --git a/relenv/common.py b/relenv/common.py index 9d526ef7..2162bf8e 100644 --- a/relenv/common.py +++ b/relenv/common.py @@ -336,12 +336,19 @@ def extract_archive(to_dir, archive): :type archive: str """ if archive.endswith("tgz"): + log.debug("Found tgz archive") + read_type = "r:gz" + elif archive.endswith("tar.gz"): + log.debug("Found tar.gz archive") read_type = "r:gz" elif archive.endswith("xz"): + log.debug("Found xz archive") read_type = "r:xz" elif archive.endswith("bz2"): + log.debug("Found bz2 archive") read_type = "r:bz2" else: + log.warning("Found unknown archive type: %s", archive) read_type = "r" with tarfile.open(archive, read_type) as t: t.extractall(to_dir) @@ -411,6 +418,7 @@ def fetch_url(url, fp, backoff=3, timeout=30): n += 1 try: fin = urllib.request.urlopen(url, timeout=timeout) + break except ( urllib.error.HTTPError, urllib.error.URLError, @@ -420,6 +428,8 @@ def fetch_url(url, fp, backoff=3, timeout=30): raise RelenvException(f"Error fetching url {url} {exc}") log.debug("Unable to connect %s", url) time.sleep(n * 10) + else: + raise RelenvException(f"Error fetching url: {url}") log.info("url opened %s", url) try: total = 0 @@ -434,7 +444,6 @@ def fetch_url(url, fp, backoff=3, timeout=30): block = fin.read(10240) finally: fin.close() - # fp.close() log.info("Download complete %s", url) @@ -514,18 +523,21 @@ def download_url(url, dest, verbose=True, backoff=3, timeout=60): """ local = get_download_location(url, dest) if verbose: - print(f"Downloading {url} -> {local}") + log.debug(f"Downloading {url} -> {local}") fout = open(local, "wb") try: fetch_url(url, fout, backoff, timeout) except Exception as exc: if verbose: - print(f"Unable to download: {url} {exc}", file=sys.stderr, flush=True) + log.error(f"Unable to download: {url} {exc}", file=sys.stderr, flush=True) try: os.unlink(local) except OSError: pass raise + finally: + fout.close() + log.debug(f"Finished downloading {url} -> {local}") return local