Skip to content

Commit 500ee61

Browse files
rajataryaseanses
rajatarya
and
seanses
authored
Adding python-version 3.13t and 3.14t to builds (#524)
Add python 3.13t and 3.14t to Release builds, they don't follow the standard abi, i.e. "abi3" so need to be build separately. Update the build and debug symbol generation and publishing workflow to allow different debug symbols for builds targeting different python version. Update the diagnosis scripts so they find the correct debug symbols. --------- Co-authored-by: di <[email protected]>
1 parent 9fde4b7 commit 500ee61

File tree

6 files changed

+168
-125
lines changed

6 files changed

+168
-125
lines changed

‎.github/workflows/release.yml‎

Lines changed: 77 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,19 @@ jobs:
2828
platform:
2929
- runner: ubuntu-22.04
3030
target: x86_64
31-
python-target: x86_64
3231
manylinux: auto
3332
- runner: ubuntu-22.04
3433
target: aarch64
35-
python-target: arm64
3634
manylinux: manylinux_2_28
35+
python-version:
36+
- 3.14
37+
- 3.13t
38+
- 3.14t
3739
steps:
3840
- uses: actions/checkout@v4
3941
- uses: actions/setup-python@v5
4042
with:
41-
python-version: 3.x
43+
python-version: ${{ matrix.python-version }}
4244
cache: 'pip'
4345
- name: Install wheel tooling
4446
run: |
@@ -58,7 +60,7 @@ jobs:
5860
uses: PyO3/maturin-action@v1
5961
with:
6062
target: ${{ matrix.platform.target }}
61-
args: --profile ${{ env.BUILD_PROFILE }} ${{ matrix.platform.features }} --out dist
63+
args: --profile ${{ env.BUILD_PROFILE }} -i ${{ matrix.python-version }} --out dist
6264
sccache: 'true'
6365
manylinux: ${{ matrix.platform.manylinux }}
6466
working-directory: hf_xet
@@ -75,26 +77,27 @@ jobs:
7577
- name: Strip debug symbols into a separate file
7678
if: env.IS_RELEASE == 'true'
7779
run: |
78-
shopt -s expand_aliases
80+
shopt -s expand_aliases
7981
if [[ ${{ matrix.platform.target }} == "aarch64" ]]
8082
then
8183
sudo apt-get install -y binutils-aarch64-linux-gnu
8284
alias objcopy=aarch64-linux-gnu-objcopy
8385
fi
84-
86+
8587
mkdir hf_xet/dbg
8688
mkdir dist
87-
pushd dist
89+
pushd dist
8890
cp ../hf_xet/dist/* .
8991
LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1)
92+
WHEEL_NAME=$(basename $LATEST_WHEEL .whl)
9093
WHEEL_VERSION=$(echo $LATEST_WHEEL | cut -d '-' -f 2)
91-
SYMBOL_FILE=hf_xet-manylinux-${{ matrix.platform.python-target}}.abi3.so.dbg
92-
wheel unpack $LATEST_WHEEL
93-
objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE}
94-
objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so
95-
objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so
94+
SYMBOL_FILE=${WHEEL_NAME}.so.dbg
95+
wheel unpack $LATEST_WHEEL
96+
objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE}
97+
objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
98+
objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
9699
mv hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} ../hf_xet/dbg/${SYMBOL_FILE}
97-
wheel pack hf_xet-${WHEEL_VERSION}
100+
wheel pack hf_xet-${WHEEL_VERSION}
98101
rm -rf hf_xet-${WHEEL_VERSION}
99102
- name: Copy unstripped wheels to upload directory
100103
if: env.IS_RELEASE != 'true'
@@ -105,12 +108,12 @@ jobs:
105108
if: env.IS_RELEASE == 'true'
106109
uses: actions/upload-artifact@v4
107110
with:
108-
name: dbg-linux-${{ matrix.platform.target }}
111+
name: dbg-linux-${{ matrix.python-version }}-${{ matrix.platform.target }}
109112
path: hf_xet/dbg
110113
- name: Upload wheels
111114
uses: actions/upload-artifact@v4
112115
with:
113-
name: wheels-linux-${{ matrix.platform.target }}
116+
name: wheels-linux-${{ matrix.python-version }}-${{ matrix.platform.target }}
114117
path: dist
115118

116119
musllinux:
@@ -120,15 +123,17 @@ jobs:
120123
platform:
121124
- runner: ubuntu-22.04
122125
target: x86_64
123-
python-target: x86_64
124126
- runner: ubuntu-22.04
125127
target: aarch64
126-
python-target: arm64
128+
python-version:
129+
- 3.14
130+
- 3.13t
131+
- 3.14t
127132
steps:
128133
- uses: actions/checkout@v4
129134
- uses: actions/setup-python@v5
130135
with:
131-
python-version: 3.x
136+
python-version: ${{ matrix.python-version }}
132137
cache: 'pip'
133138
- name: Install wheel tooling
134139
run: |
@@ -148,7 +153,7 @@ jobs:
148153
uses: PyO3/maturin-action@v1
149154
with:
150155
target: ${{ matrix.platform.target }}
151-
args: --profile ${{ env.BUILD_PROFILE }} ${{ matrix.platform.features }} --out dist
156+
args: --profile ${{ env.BUILD_PROFILE }} -i ${{ matrix.python-version }} --out dist
152157
sccache: 'true'
153158
manylinux: musllinux_1_2
154159
working-directory: hf_xet
@@ -157,7 +162,7 @@ jobs:
157162
- name: Strip debug symbols into a separate file
158163
if: env.IS_RELEASE == 'true'
159164
run: |
160-
shopt -s expand_aliases
165+
shopt -s expand_aliases
161166
if [[ ${{ matrix.platform.target }} == "aarch64" ]]
162167
then
163168
sudo apt-get install -y binutils-aarch64-linux-gnu
@@ -166,17 +171,18 @@ jobs:
166171
167172
mkdir hf_xet/dbg
168173
mkdir dist
169-
pushd dist
174+
pushd dist
170175
cp ../hf_xet/dist/* .
171176
LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1)
177+
WHEEL_NAME=$(basename $LATEST_WHEEL .whl)
172178
WHEEL_VERSION=$(echo $LATEST_WHEEL | cut -d '-' -f 2)
173-
SYMBOL_FILE=hf_xet-musllinux-${{ matrix.platform.python-target }}.abi3.so.dbg
174-
wheel unpack $LATEST_WHEEL
175-
objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE}
176-
objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so
177-
objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.abi3.so
179+
SYMBOL_FILE=${WHEEL_NAME}.so.dbg
180+
wheel unpack $LATEST_WHEEL
181+
objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE}
182+
objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
183+
objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so
178184
mv hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} ../hf_xet/dbg/${SYMBOL_FILE}
179-
wheel pack hf_xet-${WHEEL_VERSION}
185+
wheel pack hf_xet-${WHEEL_VERSION}
180186
rm -rf hf_xet-${WHEEL_VERSION}
181187
- name: Copy unstripped wheels to upload directory
182188
if: env.IS_RELEASE != 'true'
@@ -187,12 +193,12 @@ jobs:
187193
if: env.IS_RELEASE == 'true'
188194
uses: actions/upload-artifact@v4
189195
with:
190-
name: dbg-musllinux-${{ matrix.platform.target }}
196+
name: dbg-musllinux-${{ matrix.python-version }}-${{ matrix.platform.target }}
191197
path: hf_xet/dbg
192198
- name: Upload wheels with seperated debug symbols
193199
uses: actions/upload-artifact@v4
194200
with:
195-
name: wheels-musllinux-${{ matrix.platform.target }}
201+
name: wheels-musllinux-${{ matrix.python-version }}-${{ matrix.platform.target }}
196202
path: dist
197203

198204
windows:
@@ -203,12 +209,16 @@ jobs:
203209
- runner: windows-latest
204210
target: x64
205211
rust_target: x86_64-pc-windows-msvc
212+
python-version:
213+
- 3.14
214+
- 3.13t
215+
- 3.14t
206216
steps:
207217
- uses: actions/checkout@v4
208218
- uses: actions/setup-python@v5
209219
with:
210-
python-version: 3.x
211-
architecture: ${{ matrix.platform.target }}
220+
python-version: ${{ matrix.python-version }}
221+
cache: 'pip'
212222
- name: Update version in toml
213223
shell: bash
214224
run: |
@@ -221,23 +231,29 @@ jobs:
221231
uses: PyO3/maturin-action@v1
222232
with:
223233
target: ${{ matrix.platform.target }}
224-
args: --release --out dist
234+
args: --release -i python${{ matrix.python-version }} --out dist
225235
sccache: 'true'
226236
working-directory: hf_xet
227237
- name: Copy debug symbols
228238
shell: bash
229239
run: |
240+
pushd hf_xet/dist
241+
LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1)
242+
WHEEL_NAME=$(basename $LATEST_WHEEL .whl)
243+
SYMBOL_FILE=${WHEEL_NAME}.pdb
244+
popd
245+
230246
mkdir hf_xet/dbg
231-
cp hf_xet/target/${{ matrix.platform.rust_target }}/release/hf_xet.pdb hf_xet/dbg/
247+
cp hf_xet/target/${{ matrix.platform.rust_target }}/release/hf_xet.pdb hf_xet/dbg/${SYMBOL_FILE}
232248
- name: Upload debug symbols
233249
uses: actions/upload-artifact@v4
234250
with:
235-
name: dbg-windows-${{ matrix.platform.target }}
251+
name: dbg-windows-${{ matrix.python-version }}-${{ matrix.platform.target }}
236252
path: hf_xet/dbg
237253
- name: Upload wheels
238254
uses: actions/upload-artifact@v4
239255
with:
240-
name: wheels-windows-${{ matrix.platform.target }}
256+
name: wheels-windows-${{ matrix.python-version }}-${{ matrix.platform.target }}
241257
path: hf_xet/dist
242258

243259
macos:
@@ -251,11 +267,20 @@ jobs:
251267
- runner: macos-14
252268
target: aarch64
253269
rust_target: aarch64-apple-darwin
270+
python-version:
271+
- 3.14
272+
- 3.13t
273+
- 3.14t
254274
steps:
255275
- uses: actions/checkout@v4
256276
- uses: actions/setup-python@v5
257277
with:
258-
python-version: 3.x
278+
python-version: ${{ matrix.python-version }}
279+
cache: 'pip'
280+
- name: Install wheel tooling
281+
run: |
282+
python -m pip install --upgrade pip
283+
pip install wheel
259284
- name: Update version in toml
260285
run: |
261286
TAG=${{ github.event.inputs.tag }}
@@ -270,27 +295,36 @@ jobs:
270295
uses: PyO3/maturin-action@v1
271296
with:
272297
target: ${{ matrix.platform.target }}
273-
args: --profile ${{ env.BUILD_PROFILE }} --strip --out dist
298+
args: --profile ${{ env.BUILD_PROFILE }} -i ${{ matrix.python-version }} --strip --out dist
274299
sccache: 'true'
275300
working-directory: hf_xet
276301
- name: Retarget and copy debug symbols
277302
if: env.IS_RELEASE == 'true'
278303
run: |
279-
TARGET=${{ matrix.platform.target }}
304+
pushd hf_xet/dist
305+
LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1)
306+
WHEEL_NAME=$(basename $LATEST_WHEEL .whl)
307+
WHEEL_VERSION=$(echo $LATEST_WHEEL | cut -d '-' -f 2)
308+
SYMBOL_FILE=${WHEEL_NAME}.dylib.dSYM
309+
wheel unpack $LATEST_WHEEL
310+
LIB_NAME=$(basename hf_xet-${WHEEL_VERSION}/hf_xet/*.so)
311+
rm -rf hf_xet-${WHEEL_VERSION}
312+
popd
313+
280314
mkdir hf_xet/dbg
281-
pushd hf_xet/target/${{ matrix.platform.rust_target}}/release-dbgsymbols/deps
282-
sed -i '' "s/binary-path:.*/binary-path: '.\/hf_xet.abi3.so'/" libhf_xet.dylib.dSYM/Contents/Resources/Relocations/${TARGET}/libhf_xet.dylib.yml
283-
cp -r libhf_xet.dylib.dSYM ../../../../dbg/libhf_xet-macosx-${TARGET}.dylib.dSYM
315+
pushd hf_xet/target/${{ matrix.platform.rust_target}}/release-dbgsymbols
316+
sed -i '' "s/binary-path:.*/binary-path: '.\/${LIB_NAME}'/" libhf_xet.dylib.dSYM/Contents/Resources/Relocations/${{ matrix.platform.target }}/libhf_xet.dylib.yml
317+
cp -r libhf_xet.dylib.dSYM ../../../dbg/${SYMBOL_FILE}
284318
- name: Upload debug symbols
285319
if: env.IS_RELEASE == 'true'
286320
uses: actions/upload-artifact@v4
287321
with:
288-
name: dbg-macos-${{ matrix.platform.target }}
322+
name: dbg-macos-${{ matrix.python-version }}-${{ matrix.platform.target }}
289323
path: hf_xet/dbg
290324
- name: Upload wheels
291325
uses: actions/upload-artifact@v4
292326
with:
293-
name: wheels-macos-${{ matrix.platform.target }}
327+
name: wheels-macos-${{ matrix.python-version }}-${{ matrix.platform.target }}
294328
path: hf_xet/dist
295329

296330
sdist:

‎hf-xet-diag-linux.sh‎

Lines changed: 15 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ echo "Command: ${CMD[*]}"
9292
echo
9393
echo "ulimit -a:"; ulimit -a || true
9494
echo
95-
echo "python version:"; python --version || true
95+
echo "python version:"; python -VV || true
9696
echo
9797
} > "$ENV_LOG" 2>&1 || true
9898

@@ -128,7 +128,7 @@ EOF
128128
maybe_build_ptrace_helper
129129

130130
# --- download hf-xet dbg symbols ---
131-
WHEEL_VERSION=$(pip show hf-xet | grep Version | cut -d ' ' -f2)
131+
WHEEL_VERSION=$(pip show hf-xet | awk '/^Version:/{printf $2}')
132132
if [ -z "$WHEEL_VERSION" ]; then
133133
echo "Error: hf-xet package is not installed. Please install it before running this script." >&2
134134
exit 1
@@ -139,33 +139,20 @@ SYMBOL_DIR="symbols-$WHEEL_VERSION"
139139
if [ -d "$SYMBOL_DIR" ]; then
140140
echo "Existing symbols dir found, assuming previously installed."
141141
else
142-
LABEL="$(pip show hf-xet | grep Location | cut -d' ' -f2)/hf_xet"
143-
SITE_PACKAGES="$(pip show hf-xet | grep Location | cut -d' ' -f2)"
144-
DIST_NAME="$(pip show hf-xet | awk '/^Name:/{print $2}')"
145-
DIST_INFO="$(find "$SITE_PACKAGES" -maxdepth 1 -type d -iname "${DIST_NAME//-/_}-*.dist-info" | head -n1)"
146-
# Determine wheel type from WHEEL file
142+
SITE_PACKAGES="$(pip show hf-xet | awk '/^Location:/{printf $2}')"
143+
WHEEL_DIR="$SITE_PACKAGES/hf_xet"
144+
DIST_INFO="$SITE_PACKAGES/hf_xet-$WHEEL_VERSION.dist-info"
147145
WHEEL_FILE="$DIST_INFO/WHEEL"
148-
# Extract first platform tag (Tag: cp311-cp311-manylinux_2_28_x86_64)
149-
WHEEL_PLATFORM_TAG=$(grep '^Tag:' "$WHEEL_FILE" | head -n1 | awk -F'-' '{print $NF}')
150-
echo "Wheel platform tag: $WHEEL_PLATFORM_TAG"
151-
# Extract Linux type (musl vs manylinux) and arch
152-
LINUX_TYPE="$(echo "$WHEEL_PLATFORM_TAG" | grep -oE '^(musllinux|manylinux)')"
153-
ARCH=$(uname -m)
154-
# Normalize architecture names to match your debug symbols naming
155-
if [[ "$ARCH" == "aarch64" ]]; then
156-
ARCH="arm64"
157-
fi
158-
SYMBOL_FILENAME="hf_xet-${LINUX_TYPE}-${ARCH}.abi3.so.dbg"
146+
147+
# Reconstruct wheel name from wheel version and wheel tag
148+
WHEEL_TAG=$(awk '/^Tag:/{printf $2}' $WHEEL_FILE)
149+
SYMBOL_FILENAME="hf_xet-$WHEEL_VERSION-$WHEEL_TAG.so.dbg"
159150

160151
echo "Downloading debug symbols: $SYMBOL_FILENAME"
161-
# We assume the debug symbols archive is available at a predictable URL:
162-
# e.g. https://github.com/huggingface/xet-core/releases/download/latest/dbg-symbols.zip
163-
if [ -n "$WHEEL_VERSION" ]; then
164-
DOWNLOAD_URL="https://github.com/huggingface/xet-core/releases/download/v${WHEEL_VERSION}/dbg-symbols.zip"
165-
else
166-
DOWNLOAD_URL="https://github.com/huggingface/xet-core/releases/latest/download/dbg-symbols.zip"
167-
fi
168-
curl -L "$DOWNLOAD_URL" -o dbg-symbols.zip
152+
# If the version is of format "1.1.10rc0", change it to our release tag format like "1.1.10-rc0"
153+
RELEASE_TAG=$(echo -n "$WHEEL_VERSION" | sed 's/\([0-9]\)\(rc.*\)$/\1-\2/')
154+
DOWNLOAD_URL="https://github.com/huggingface/xet-core/releases/download/v${RELEASE_TAG}/dbg-symbols.zip"
155+
curl -fL "$DOWNLOAD_URL" -o dbg-symbols.zip
169156
if [ $? -ne 0 ]; then
170157
echo "Error: Failed to download debug symbols from $DOWNLOAD_URL" >&2
171158
exit 1
@@ -175,8 +162,8 @@ else
175162
unzip dbg-symbols.zip -d "$SYMBOL_DIR"
176163

177164
# Copy to package directory
178-
cp -r $SYMBOL_DIR/dbg-symbols/"$SYMBOL_FILENAME" "$LABEL/"
179-
echo "Installed dbg symbol $SYMBOL_FILENAME to $LABEL"
165+
cp -r "$SYMBOL_DIR/dbg-symbols/$SYMBOL_FILENAME" "$WHEEL_DIR/"
166+
echo "Installed dbg symbol $SYMBOL_FILENAME to $WHEEL_DIR"
180167
fi
181168

182169
# --- launch target ---

0 commit comments

Comments
 (0)