Skip to content

python3.11-tokenizers: rebuild #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 28, 2024
Merged

python3.11-tokenizers: rebuild #174

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions tur-pypi-311/python3.11-tokenizers/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ TERMUX_PKG_HOMEPAGE=https://github.com/huggingface/tokenizers
TERMUX_PKG_DESCRIPTION="Fast State-of-the-Art Tokenizers optimized for Research and Production"
TERMUX_PKG_LICENSE="Apache-2.0"
TERMUX_PKG_MAINTAINER="@termux-user-repository"
TERMUX_PKG_VERSION="0.21.0"
TERMUX_PKG_VERSION="0.20.4"
TERMUX_PKG_SRCURL=https://github.com/huggingface/tokenizers/archive/refs/tags/v$TERMUX_PKG_VERSION.tar.gz
TERMUX_PKG_SHA256=841279ad797d575ed3cf31fc4f30e09e37acbd35028d30c51fc0879ef7ed4094
TERMUX_PKG_SHA256=c7dd77a63d95113add5f4c85a41f117cbf41d2cbc7f6609b72650e14c7f42f31
TERMUX_PKG_AUTO_UPDATE=true
TERMUX_PKG_DEPENDS="libc++, python3.11"
TERMUX_PKG_PYTHON_COMMON_DEPS="wheel"
Expand Down Expand Up @@ -56,17 +56,22 @@ termux_step_make_install() {

build-python -m maturin build --release --skip-auditwheel --target $CARGO_BUILD_TARGET -Z build-std

# Fix wheel name for arm
local _pyver="${TERMUX_PYTHON_VERSION/./}"
# Fix wheel name, although it it built with tag `cp39-abi3`, but it is linked against `python3.x.so`
# so it will not work on other pythons.
if [ "$TERMUX_ARCH" = "arm" ]; then
mv ./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp311-cp311-linux_armv7l.whl \
./target/wheels/tokenizers-$TERMUX_PKG_VERSION-py311-none-any.whl
mv ./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp39-abi3-linux_armv7l.whl \
./target/wheels/tokenizers-$TERMUX_PKG_VERSION-py$_pyver-none-any.whl
else
mv ./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp39-abi3-linux_$TERMUX_ARCH.whl \
./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp$_pyver-cp$_pyver-linux_$TERMUX_ARCH.whl
fi

pip install --no-deps ./target/wheels/*.whl --prefix $TERMUX_PREFIX

# Fix wheel name for arm
if [ "$TERMUX_ARCH" = "arm" ]; then
mv ./target/wheels/tokenizers-$TERMUX_PKG_VERSION-py311-none-any.whl \
./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp311-cp311-linux_armv7l.whl
mv ./target/wheels/tokenizers-$TERMUX_PKG_VERSION-py$_pyver-none-any.whl \
./target/wheels/tokenizers-$TERMUX_PKG_VERSION-cp$_pyver-cp$_pyver-linux_armv7l.whl
fi
}