diff --git a/common-files/audit-and-update-record-for-wheel.py b/common-files/audit-and-update-record-for-wheel.py index 5f91fda..b684b6d 100644 --- a/common-files/audit-and-update-record-for-wheel.py +++ b/common-files/audit-and-update-record-for-wheel.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# audit-and-update-record-for-wheel.py - Update RECORD file for wheels. +# audit-and-update-record-for-wheel.py - Check the lib and license for wheels, and update RECORD file for wheels. # from __future__ import annotations @@ -7,12 +7,22 @@ import logging from os.path import abspath, basename, exists, isfile import os +import json -from auditwheel.wheeltools import InWheel +from auditwheel.wheeltools import InWheelCtx logger = logging.getLogger(__name__) def configure_parser(p): + p.add_argument( + "-L", + "--lib-sdir", + dest="LIB_SDIR", + help=( + "Subdirectory in packages to store copied libraries." ' (default: ".libs")' + ), + default=".libs", + ) p.add_argument( "-w", "--wheel-dir", @@ -21,27 +31,115 @@ def configure_parser(p): help=("Directory to store delocated wheels (default:" ' "wheelhouse/")'), default="wheelhouse/", ) - p.add_argument("WHEEL_FILE", help="Path to wheel file.", nargs="+") + p.add_argument( + "WHEEL_FILE", + type=str, + help="Path to wheel file.", + ) + p.add_argument( + "-j", + "--json", + dest="LICENSE_JSON", + help="Path to lib-license json.", + ) + p.add_argument( + "-nolib", + "--ensure-no-libs", + dest="NO_LIBS", + action="store_true", + help="Ensure there is no libs.", + default=False, + ) + p.add_argument( + "-clib", + "--check-libs", + dest="CHECK_LIBS", + action="store_true", + help=( + "Enable checking libs." + ), + default=False, + ) + p.add_argument( + "-clic", + "--check-licenses", + dest="CHECK_LICENSES", + action="store_true", + help=( + "Enable checking licenses." + ), + default=False, + ) p.set_defaults(func=execute) +def parse_license_json(filepath): + libs = set() + licenses = {} + with open(filepath, "r") as fp: + for e in json.load(fp): + for lib in e["libs"]: + libs.add(lib) + for license in e["licenses"]: + licenses[license] = False + return libs, licenses + + def execute(args, p): - for wheel_file in args.WHEEL_FILE: - if not isfile(wheel_file): - p.error("cannot access %s. No such file" % wheel_file) + wheel_file = args.WHEEL_FILE + if not isfile(wheel_file): + logger.error("cannot access %s. No such file" % wheel_file) + exit(1) - if not exists(args.WHEEL_DIR): - os.makedirs(args.WHEEL_DIR) + if not exists(args.WHEEL_DIR): + os.makedirs(args.WHEEL_DIR) - out_wheel=os.path.join("wheelhouse", basename(wheel_file)) - with InWheel(in_wheel=wheel_file, out_wheel=out_wheel): + out_wheel=os.path.join(args.WHEEL_DIR, basename(wheel_file)) + with InWheelCtx(in_wheel=wheel_file, out_wheel=out_wheel) as ctx: + if not args.CHECK_LIBS and not args.CHECK_LICENSES: logger.info("Updating RECORD file of %s", basename(wheel_file)) + return + libs = set() + licenses = {} + if args.CHECK_LIBS or args.CHECK_LICENSES: + libs, licenses = parse_license_json(args.LICENSE_JSON) + real_libs = set() + for fn in ctx.iter_files(): + path_split = fn.split(os.sep) + name = path_split[-1] + if f"{args.LIB_SDIR}/" in fn and ".so" in name: + if args.NO_LIBS: + logger.error(f"found possible .so: {fn}") + exit(1) + if args.CHECK_LIBS: + # libxxx-xxxx.so(.xxx) + name = "".join(name.split(".so")[:-1]) + # libxxx-xxxx + name = "-".join(name.split("-")[:-1]) + real_libs.add(name) + if args.CHECK_LICENSES and licenses.get(name, None) is not None: + licenses[name] = True + ok = True + if args.CHECK_LIBS: + if libs != real_libs: + ok = False + logger.error(f"libs check failed. libs: {libs}, real_libs: {real_libs}") + if args.CHECK_LICENSES: + for k, v in licenses.items(): + if v != True: + ok = False + logger.error(f"{k} is not found") + if not ok: + logger.error("check is not passed.") + exit(1) + logger.info("check passed.") + logger.info("Updating RECORD file of %s", basename(wheel_file)) - logger.info("\nFixed-up wheel written to %s", out_wheel) + logger.info("\nFixed-up wheel written to %s", out_wheel) def main(): - p = argparse.ArgumentParser(description="Update RECORD for Python wheels.") + p = argparse.ArgumentParser(description="Cross-distro Python wheels.") p.add_argument( "-v", "--verbose", diff --git a/common-files/tur_build_wheel.sh b/common-files/tur_build_wheel.sh index c0ccee2..20d9777 100644 --- a/common-files/tur_build_wheel.sh +++ b/common-files/tur_build_wheel.sh @@ -4,6 +4,7 @@ : "${TUR_AUTO_ELF_CLEAN_WHEEL:=true}" : "${TUR_AUDIT_WHEEL_NO_LIBS:=false}" : "${TUR_PACKAGE_WHEEL_LICENSE:=true}" +: "${TUR_LIB_LICENSE_JSON:=""}" tur_install_wheel_license() { return @@ -37,28 +38,22 @@ tur_audit_and_repair_wheel() { mv wheelhouse/$filename $filepath } -tur_check_no_libs_after_audit_wheel() { +tur_check_libs_and_licenses_for_wheel() { local filepath="$(realpath $1)" local filename="$(basename $filepath)" - # Make a workspace and enter it - local work_dir="$(mktemp -d)" - pushd $work_dir - - # Wheel file is actually a zip file, unzip it first. - unzip -q $filepath - - local _whl_package_name="$(echo $filename | cut -d'-' -f1)" - local _lib_name= - # shopt -s nullglob - for _lib_name in $_whl_package_name-libs/*.so; do - termux_error_exit "Found lib $_lib_name packaged after auditwheel." - done - # shopt -u nullglob - - # Clean up the workspace - popd # $work_dir - rm -rf $work_dir + # Run script to check the libs and licenses + local _args="--lib-sdir=-libs" + if [ "$TUR_AUDIT_WHEEL_NO_LIBS" != "false" ]; then + _args+=" --ensure-no-libs" + fi + if [ "$TUR_LIB_LICENSE_JSON" != "" ]; then + _args+=" --check-libs --check-licenses --json $TUR_LIB_LICENSE_JSON" + elif [ "$TUR_AUDIT_WHEEL_NO_LIBS" == "false" ]; then + termux_error_exit "Must check libs and licenses after install" + fi + build-python $TERMUX_SCRIPTDIR/common-files/audit-and-update-record-for-wheel.py \ + -v $_args $filepath } tur_package_wheel_license() { @@ -142,9 +137,6 @@ tur_build_wheel() { # Audit wheel if needed if [ "$TUR_AUTO_AUDIT_WHEEL" != "false" ]; then tur_audit_and_repair_wheel $_whl - if [ "$TUR_AUDIT_WHEEL_NO_LIBS" != "false" ]; then - tur_check_no_libs_after_audit_wheel $_whl - fi fi # Package license if needed @@ -157,8 +149,8 @@ tur_build_wheel() { tur_elf_cleaner_for_wheel $_whl fi - # Finally, update RECORD file - tur_update_record_file_of_wheel $_whl + # Finally, check libs and licenses if needed, and update RECORD file + tur_check_libs_and_licenses_for_wheel $_whl done shopt -u nullglob diff --git a/tur-pypi-312/python3.12-numpy/build.sh b/tur-pypi-312/python3.12-numpy/build.sh index 59f2700..ad38fa3 100644 --- a/tur-pypi-312/python3.12-numpy/build.sh +++ b/tur-pypi-312/python3.12-numpy/build.sh @@ -23,6 +23,7 @@ TERMUX_PYTHON_CROSSENV_PREFIX=$TERMUX_PKG_BUILDDIR/python${TERMUX_PYTHON_VERSION TUR_AUTO_AUDIT_WHEEL=true TUR_AUTO_BUILD_WHEEL=false TUR_WHEEL_DIR="../src/dist" +TUR_LIB_LICENSE_JSON="$TERMUX_PKG_BUILDER_DIR/licenses.json" source $TERMUX_SCRIPTDIR/common-files/tur_build_wheel.sh @@ -78,3 +79,8 @@ termux_step_make_install() { local _whl="numpy-$TERMUX_PKG_VERSION-cp$_pyv-cp$_pyv-linux_$TERMUX_ARCH.whl" pip install --no-deps --prefix=$TERMUX_PREFIX --force-reinstall $TERMUX_PKG_SRCDIR/dist/$_whl } + +tur_install_wheel_license() { + # Install the license of libopenblas + cp $TERMUX_PREFIX/share/doc/libopenblas/copyright libopenblas-LICENSE +} diff --git a/tur-pypi-312/python3.12-numpy/licenses.json b/tur-pypi-312/python3.12-numpy/licenses.json new file mode 100644 index 0000000..c243e62 --- /dev/null +++ b/tur-pypi-312/python3.12-numpy/licenses.json @@ -0,0 +1,11 @@ +[ + { + "name": "openblas", + "libs": [ + "libopenblas" + ], + "licenses": [ + "libopenblas-LICENSE" + ] + } +] \ No newline at end of file diff --git a/tur-pypi-312/python3.12-scipy/0001-aligned-alloc.patch b/tur-pypi-312/python3.12-scipy/0001-aligned-alloc.patch new file mode 100644 index 0000000..c988141 --- /dev/null +++ b/tur-pypi-312/python3.12-scipy/0001-aligned-alloc.patch @@ -0,0 +1,11 @@ +--- a/scipy/_lib/pocketfft/pocketfft_hdronly.h ++++ b/scipy/_lib/pocketfft/pocketfft_hdronly.h +@@ -156,7 +156,7 @@ + // the standard C++ library on Windows does not provide aligned_alloc() even + // though the MinGW compiler and MSVC may advertise C++17 compliance. + // aligned_alloc is only supported from MacOS 10.15. +-#if (__cplusplus >= 201703L) && (!defined(__MINGW32__)) && (!defined(_MSC_VER)) && (__MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15) ++#if (__cplusplus >= 201703L) && (!defined(__MINGW32__)) && (!defined(_MSC_VER)) && (__MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15) && !(defined(__ANDROID__) && __ANDROID_API__ < 26) + inline void *aligned_alloc(size_t align, size_t size) + { + // aligned_alloc() requires that the requested size is a multiple of "align" diff --git a/tur-pypi-312/python3.12-scipy/0002-fix-the-logging-of-f2pymod-wrapper.patch b/tur-pypi-312/python3.12-scipy/0002-fix-the-logging-of-f2pymod-wrapper.patch new file mode 100644 index 0000000..4f362b8 --- /dev/null +++ b/tur-pypi-312/python3.12-scipy/0002-fix-the-logging-of-f2pymod-wrapper.patch @@ -0,0 +1,14 @@ +--- a/tools/generate_f2pymod.py ++++ b/tools/generate_f2pymod.py +@@ -290,9 +290,9 @@ + cwd=os.getcwd()) + out, err = p.communicate() + if not (p.returncode == 0): +- raise RuntimeError(f"Writing {args.outfile} with f2py failed!\n" ++ raise RuntimeError(f"Processing {fname_pyf} with f2py failed!\n" + f"{out}\n" +- r"{err}") ++ f"{err}") + + + if __name__ == "__main__": diff --git a/tur-pypi-312/python3.12-scipy/build.sh b/tur-pypi-312/python3.12-scipy/build.sh new file mode 100644 index 0000000..59f7cb5 --- /dev/null +++ b/tur-pypi-312/python3.12-scipy/build.sh @@ -0,0 +1,96 @@ +TERMUX_PKG_HOMEPAGE=https://scipy.org/ +TERMUX_PKG_DESCRIPTION="Fundamental algorithms for scientific computing in Python" +TERMUX_PKG_LICENSE="BSD 3-Clause" +TERMUX_PKG_MAINTAINER="@termux-user-repository" +TERMUX_PKG_VERSION="1.13.0" +TERMUX_PKG_SRCURL=git+https://github.com/scipy/scipy +TERMUX_PKG_DEPENDS="libc++, libopenblas, python, python-numpy" +TERMUX_PKG_BUILD_DEPENDS="python-numpy-static" +TERMUX_PKG_PYTHON_COMMON_DEPS="wheel, 'Cython>=3.0.4', meson-python, build" +_NUMPY_VERSION=$(. $TERMUX_SCRIPTDIR/packages/python-numpy/build.sh; echo $TERMUX_PKG_VERSION) +TERMUX_PKG_PYTHON_BUILD_DEPS="'pybind11>=2.10.4', 'numpy==$_NUMPY_VERSION'" +TERMUX_PKG_AUTO_UPDATE=true +TERMUX_PKG_UPDATE_TAG_TYPE="latest-release-tag" + +# Tests will hang on arm and will failed with `Segmentation fault` on i686. +# See https://github.com/termux-user-repository/tur/pull/21#issue-1295483266. +# +# The logs of this crash on i686 are as following. +# linalg/tests/test_basic.py: Fatal Python error: Segmentation fault +# +# Current thread 0xf7f4b580 (most recent call first): +# File "/data/data/com.termux/files/usr/lib/python3.10/site-packages/scipy-1.8.0-py3.10-linux-i686.egg/scipy/linalg/_basic.py", line 1227 in lstsq +# File "/data/data/com.termux/files/usr/lib/python3.10/site-packages/scipy-1.8.0-py3.10-linux-i686.egg/scipy/linalg/tests/test_basic.py", line 1047 in test_simple_overdet_complex +# XXX: Although it doesn't seem to work fine, I'd like to enable this package as it happens only on some functions. +# TERMUX_PKG_BLACKLISTED_ARCHES="arm, i686" + +TERMUX_PKG_RM_AFTER_INSTALL=" +bin/ +" + +TERMUX_MESON_WHEEL_CROSSFILE="$TERMUX_PKG_TMPDIR/wheel-cross-file.txt" +TERMUX_PKG_EXTRA_CONFIGURE_ARGS=" +-Dblas=openblas +-Dlapack=openblas +-Duse-pythran=false +--cross-file $TERMUX_MESON_WHEEL_CROSSFILE +" + +TERMUX_PYTHON_VERSION=3.12 +TERMUX_PYTHON_CROSSENV_PREFIX=$TERMUX_PKG_BUILDDIR/python${TERMUX_PYTHON_VERSION/./}-crossenv-prefix-$TERMUX_ARCH +TUR_AUTO_AUDIT_WHEEL=true +TUR_AUTO_BUILD_WHEEL=false +TUR_WHEEL_DIR="../src/dist" +TUR_LIB_LICENSE_JSON="$TERMUX_PKG_BUILDER_DIR/licenses.json" + +source $TERMUX_SCRIPTDIR/common-files/tur_build_wheel.sh +source $TERMUX_SCRIPTDIR/common-files/setup_toolchain_gcc.sh + +termux_step_pre_configure() { + if $TERMUX_ON_DEVICE_BUILD; then + termux_error_exit "Package '$TERMUX_PKG_NAME' is not available for on-device builds." + fi + + _setup_toolchain_ndk_gcc_11 + + LDFLAGS+=" -Wl,--no-as-needed,-lpython${TERMUX_PYTHON_VERSION},--as-needed" + + # FIXME: Don't know why NDK's libc++ should link against clang's libunwind, + # FIXME: otherwise pybind11's `register_local_exception_translator` won't + # FIXME: work properly, causing crash on `scipy.io._mmio`. + mkdir -p $TERMUX_PKG_TMPDIR/_libunwind_libdir + local _NDK_ARCH=$TERMUX_ARCH + test $_NDK_ARCH == 'i686' && _NDK_ARCH='i386' + cp $NDK/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/*/lib/linux/$_NDK_ARCH/libunwind.a \ + $TERMUX_PKG_TMPDIR/_libunwind_libdir/libunwind.a + LDFLAGS="-L$TERMUX_PKG_TMPDIR/_libunwind_libdir -l:libunwind.a ${LDFLAGS}" +} + +termux_step_configure() { + termux_setup_meson + + cp -f $TERMUX_MESON_CROSSFILE $TERMUX_MESON_WHEEL_CROSSFILE + sed -i 's|^\(\[binaries\]\)$|\1\npython = '\'$(command -v python)\''|g' \ + $TERMUX_MESON_WHEEL_CROSSFILE + sed -i 's|^\(\[properties\]\)$|\1\nnumpy-include-dir = '\'$PYTHON_SITE_PKG/numpy/_core/include\''|g' \ + $TERMUX_MESON_WHEEL_CROSSFILE + + termux_step_configure_meson +} + +termux_step_make() { + pushd $TERMUX_PKG_SRCDIR + PYTHONPATH= python -m build -w -n -x --config-setting builddir=$TERMUX_PKG_BUILDDIR . + popd +} + +termux_step_make_install() { + local _pyv="${TERMUX_PYTHON_VERSION/./}" + local _whl="scipy-${TERMUX_PKG_VERSION#*:}-cp$_pyv-cp$_pyv-linux_$TERMUX_ARCH.whl" + pip install --no-deps --prefix=$TERMUX_PREFIX $TERMUX_PKG_SRCDIR/dist/$_whl +} + +tur_install_wheel_license() { + # Install the license of libopenblas + cp $TERMUX_PREFIX/share/doc/libopenblas/copyright libopenblas-LICENSE +} diff --git a/tur-pypi-312/python3.12-scipy/licenses.json b/tur-pypi-312/python3.12-scipy/licenses.json new file mode 100644 index 0000000..c243e62 --- /dev/null +++ b/tur-pypi-312/python3.12-scipy/licenses.json @@ -0,0 +1,11 @@ +[ + { + "name": "openblas", + "libs": [ + "libopenblas" + ], + "licenses": [ + "libopenblas-LICENSE" + ] + } +] \ No newline at end of file