Skip to content

Commit 8c2c37e

Browse files
jordimasjordimas
authored
Merge branch 'master' into master
2 parents 7d95373 + 166b731 commit 8c2c37e

File tree

105 files changed

+4451
-726
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+4451
-726
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 103 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ jobs:
1717
CT2_VERBOSE: 1
1818
strategy:
1919
matrix:
20-
os: [ubuntu-20.04, macos-11]
20+
os: [ubuntu-22.04]
2121
backend: [mkl, dnnl]
2222

2323
steps:
24-
- uses: actions/checkout@v3
24+
- uses: actions/checkout@v4
2525
with:
2626
submodules: recursive
2727

@@ -33,17 +33,6 @@ jobs:
3333
sudo sh -c 'echo "deb https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneAPI.list'
3434
sudo apt-get update
3535
36-
- name: Store URL for downloading Intel oneAPI to environment variable
37-
if: startsWith(matrix.os, 'macos')
38-
run: |
39-
echo 'ONEAPI_INSTALLER_URL=https://registrationcenter-download.intel.com/akdlm/irc_nas/19080/m_BaseKit_p_2023.0.0.25441_offline.dmg' >> $GITHUB_ENV
40-
41-
- name: Install Intel oneAPI
42-
if: startsWith(matrix.os, 'macos')
43-
run: |
44-
wget -q $ONEAPI_INSTALLER_URL
45-
hdiutil attach -noverify -noautofsck $(basename $ONEAPI_INSTALLER_URL)
46-
4736
- name: Configure with MKL
4837
if: startsWith(matrix.os, 'ubuntu') && matrix.backend == 'mkl'
4938
env:
@@ -61,20 +50,6 @@ jobs:
6150
sudo apt-get install -y intel-oneapi-dnnl-devel=$DNNL_VERSION intel-oneapi-dnnl=$DNNL_VERSION
6251
cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=COMP -DWITH_DNNL=ON .
6352
64-
- name: Configure with MKL
65-
if: startsWith(matrix.os, 'macos') && matrix.backend == 'mkl'
66-
env:
67-
CT2_USE_MKL: 1
68-
run: |
69-
sudo /Volumes/$(basename $ONEAPI_INSTALLER_URL .dmg)/bootstrapper.app/Contents/MacOS/bootstrapper --silent --eula accept --components intel.oneapi.mac.mkl.devel
70-
cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON .
71-
72-
- name: Configure with DNNL
73-
if: startsWith(matrix.os, 'macos') && matrix.backend == 'dnnl'
74-
run: |
75-
sudo /Volumes/$(basename $ONEAPI_INSTALLER_URL .dmg)/bootstrapper.app/Contents/MacOS/bootstrapper --silent --eula accept --components intel.oneapi.mac.dnnl
76-
cmake -DCMAKE_INSTALL_PREFIX=$PWD/install -DBUILD_TESTS=ON -DWITH_MKL=OFF -DWITH_DNNL=ON .
77-
7853
- name: Build
7954
run: |
8055
make install
@@ -98,40 +73,34 @@ jobs:
9873
tests/ctranslate2_test tests/data
9974
10075
101-
build-and-test-cpp-aarch64:
102-
runs-on: ubuntu-20.04
76+
build-and-test-cpp-arm64:
77+
runs-on: ${{ matrix.os }}
10378
env:
10479
CT2_VERBOSE: 1
10580
strategy:
10681
matrix:
107-
backend: [openblas]
10882

83+
include:
84+
- os: ubuntu-22.04-arm
85+
backend: openblas
86+
- os: macos-15
87+
backend: ruy
10988
steps:
110-
- uses: actions/checkout@v3
89+
- uses: actions/checkout@v4
11190
with:
11291
submodules: recursive
11392

114-
- name: Install cross compiler and emulator
115-
run: |
116-
sudo apt-get update
117-
sudo apt-get install -y g++-aarch64-linux-gnu gfortran-aarch64-linux-gnu qemu-user-static
118-
11993
- name: Build with OpenBLAS and Ruy
12094
if: matrix.backend == 'openblas'
12195
run: |
12296
wget https://github.com/xianyi/OpenBLAS/archive/v0.3.13.tar.gz
12397
tar xzvf v0.3.13.tar.gz
12498
cd OpenBLAS-0.3.13
125-
make TARGET=ARMV8 CC=aarch64-linux-gnu-gcc FC=aarch64-linux-gnu-gfortran HOSTCC=gcc NO_LAPACK=1 -j $(nproc)
126-
sudo make PREFIX=/usr/aarch64-linux-gnu install
99+
make TARGET=ARMV8 NO_LAPACK=1 -j $(nproc)
100+
sudo make PREFIX=/usr/local install
127101
cd ..
128-
102+
export OpenBLAS_HOME=/usr/local
129103
cmake \
130-
-DCMAKE_SYSTEM_NAME=Linux \
131-
-DCMAKE_SYSTEM_PROCESSOR=aarch64 \
132-
-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
133-
-DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
134-
-DCMAKE_FIND_ROOT_PATH=/usr/aarch64-linux-gnu \
135104
-DOPENMP_RUNTIME=COMP \
136105
-DCMAKE_INSTALL_PREFIX=$PWD/install \
137106
-DWITH_MKL=OFF \
@@ -141,6 +110,18 @@ jobs:
141110
.
142111
make -j $(nproc) install
143112
113+
- name: Build Ruy
114+
if: matrix.backend == 'ruy'
115+
run: |
116+
CMAKE_EXTRA_OPTIONS='-DCMAKE_OSX_ARCHITECTURES=arm64 -DWITH_ACCELERATE=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=NONE -DWITH_RUY=ON'
117+
cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
118+
-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
119+
-DCMAKE_INSTALL_PREFIX=$PWD/install \
120+
-DBUILD_TESTS=ON \
121+
$CMAKE_EXTRA_OPTIONS \
122+
.
123+
make -j $(nproc) install
124+
144125
- name: Download test data
145126
run: |
146127
wget https://opennmt-models.s3.amazonaws.com/transliteration-aren-all.tar.gz
@@ -149,24 +130,23 @@ jobs:
149130
- name: Test
150131
run: |
151132
tests/ctranslate2_test tests/data
152-
env:
153-
QEMU_LD_PREFIX: /usr/aarch64-linux-gnu
154-
155133
156134
build-python-wheels:
157135
runs-on: ${{ matrix.os }}
158136
strategy:
159137
matrix:
160-
os: [ubuntu-20.04, macos-11, windows-2019]
138+
os: [ubuntu-22.04, windows-2022]
161139
arch: [auto64]
162140
include:
163-
- os: ubuntu-20.04
141+
- os: ubuntu-22.04
164142
arch: aarch64
165-
- os: macos-11
143+
- os: macos-15
166144
arch: arm64
145+
- os: macos-15-intel
146+
arch: x86_64
167147

168148
steps:
169-
- uses: actions/checkout@v3
149+
- uses: actions/checkout@v4
170150
with:
171151
submodules: recursive
172152

@@ -175,27 +155,27 @@ jobs:
175155
name: Set up QEMU
176156

177157
- name: Build wheels
178-
uses: pypa/cibuildwheel@v2.16.5
158+
uses: pypa/cibuildwheel@v3.2.1
179159
with:
180160
package-dir: python
181161
output-dir: python/wheelhouse
182162
env:
183163
CIBW_ENVIRONMENT_PASS_LINUX: CIBW_ARCHS
184164
CIBW_ENVIRONMENT_WINDOWS: CTRANSLATE2_ROOT='${{ github.workspace }}\install'
185-
CIBW_ENVIRONMENT_MACOS: CTRANSLATE2_ROOT='/usr/local'
165+
CIBW_ENVIRONMENT_MACOS: "CTRANSLATE2_ROOT='/usr/local' MACOSX_DEPLOYMENT_TARGET=11.00"
186166
CIBW_BEFORE_ALL_LINUX: python/tools/prepare_build_environment_linux.sh
187167
CIBW_BEFORE_ALL_MACOS: python/tools/prepare_build_environment_macos.sh
188168
CIBW_BEFORE_ALL_WINDOWS: bash python/tools/prepare_build_environment_windows.sh
189169
CIBW_BEFORE_BUILD: pip install -r python/install_requirements.txt
190170
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
191171
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
192172
CIBW_ARCHS: ${{ matrix.arch }}
193-
CIBW_SKIP: pp* *-musllinux_*
173+
CIBW_SKIP: "*-musllinux_*"
194174

195175
- name: Upload Python wheels
196-
uses: actions/upload-artifact@v3
176+
uses: actions/upload-artifact@v4
197177
with:
198-
name: python-wheels
178+
name: python-wheels-${{ runner.os }}-${{ matrix.arch }}
199179
path: python/wheelhouse
200180

201181

@@ -206,58 +186,59 @@ jobs:
206186
runs-on: ${{ matrix.os }}
207187
strategy:
208188
matrix:
209-
os: [ubuntu-20.04, macos-11, windows-2019]
189+
include:
190+
- os: ubuntu-22.04
191+
artifact_pattern: python-wheels-Linux-auto64
192+
wheel_pattern: "*cp310*manylinux*x86_64.whl"
193+
194+
- os: ubuntu-22.04-arm
195+
artifact_pattern: python-wheels-Linux-aarch64
196+
wheel_pattern: "*cp310*manylinux*_aarch64.whl"
197+
198+
- os: macos-15
199+
artifact_pattern: python-wheels-macOS-arm64
200+
wheel_pattern: "*cp310*macosx*arm64.whl"
210201

211202
steps:
212-
- name: Set up Python 3.8
213-
uses: actions/setup-python@v4
203+
- name: Set up Python 3.10.11
204+
uses: actions/setup-python@v5
214205
with:
215-
python-version: 3.8
206+
python-version: "3.10.11"
216207

217-
- uses: actions/checkout@v3
208+
- uses: actions/checkout@v4
218209

219210
- name: Prepare test environment
220211
shell: bash
221212
run: |
222213
./python/tools/prepare_test_environment.sh
223214
224215
- name: Download Python wheels
225-
uses: actions/download-artifact@v3
216+
uses: actions/download-artifact@v4
226217
with:
227-
name: python-wheels
228-
229-
- name: Install wheel
230-
if: startsWith(matrix.os, 'ubuntu')
231-
run: |
232-
pip install *cp38*manylinux*x86_64.whl
233-
234-
- name: Install wheel
235-
if: startsWith(matrix.os, 'macos')
236-
run: |
237-
pip install *cp38*macosx*x86_64.whl
218+
pattern: ${{ matrix.artifact_pattern }}
219+
merge-multiple: true
220+
path: .
238221

239222
- name: Install wheel
240-
if: startsWith(matrix.os, 'windows')
241223
shell: bash
242224
run: |
243-
pip install *cp38*win*.whl
225+
pip install ${{ matrix.wheel_pattern }}
244226
245-
- name: Run tests
246-
shell: bash
227+
- name: Test Python wheel
247228
run: |
248-
pytest -v python/tests/
229+
pytest -v python/tests/ --ignore=python/tests/test_opennmt_tf.py
249230
250231
251232
check-python-style:
252233
runs-on: ubuntu-latest
253234

254235
steps:
255-
- uses: actions/checkout@v3
236+
- uses: actions/checkout@v4
256237

257-
- name: Set up Python 3.8
258-
uses: actions/setup-python@v4
238+
- name: Set up Python 3.10.11
239+
uses: actions/setup-python@v5
259240
with:
260-
python-version: 3.8
241+
python-version: "3.10.11"
261242

262243
- name: Install dependencies
263244
run: |
@@ -282,14 +263,16 @@ jobs:
282263
283264
publish-python-wheels-on-pypi:
284265
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
285-
needs: [build-and-test-cpp-x86_64, build-python-wheels, test-python-wheels, check-python-style]
286-
runs-on: ubuntu-20.04
266+
needs: [build-and-test-cpp-x86_64, build-and-test-cpp-arm64, build-python-wheels, test-python-wheels, check-python-style]
267+
runs-on: ubuntu-22.04
287268

288269
steps:
289270
- name: Download Python wheels
290-
uses: actions/download-artifact@v3
271+
uses: actions/download-artifact@v4
291272
with:
292-
name: python-wheels
273+
pattern: python-wheels-*
274+
merge-multiple: true
275+
path: .
293276

294277
- name: Publish Python wheels to PyPI
295278
uses: pypa/gh-action-pypi-publish@release/v1
@@ -300,12 +283,35 @@ jobs:
300283

301284

302285
build-and-push-docker-images:
303-
runs-on: ubuntu-20.04
286+
runs-on: ubuntu-22.04
304287
steps:
305-
- uses: actions/checkout@v3
288+
- uses: actions/checkout@v4
306289
with:
307290
submodules: recursive
308291

292+
- name: Show disk and docker usage (before cleanup)
293+
run: |
294+
df -h
295+
echo " -= Docker System =-"
296+
docker system df || true
297+
298+
- name: Free disk space (cleanup heavy preinstalled directories + docker prune)
299+
run: |
300+
echo " -= Removing big preinstalled directories (shouldn't remove the needed tools) =-"
301+
sudo rm -rf /opt/hostedtoolcache || true
302+
sudo rm -rf /usr/share/dotnet || true
303+
sudo rm -rf /usr/lib/jvm || true
304+
sudo rm -rf /usr/local/lib/android || true
305+
echo " -= Running docker prune =-"
306+
docker system prune -af --volumes || true
307+
docker builder prune -af || true
308+
309+
- name: Show disk and docker usage (after cleanup)
310+
run: |
311+
df -h
312+
echo " -= Docker System =-"
313+
docker system df || true
314+
309315
- name: Build Docker images
310316
run: |
311317
./docker/build_all.sh
@@ -329,21 +335,23 @@ jobs:
329335
needs: [check-python-style, build-python-wheels]
330336

331337
steps:
332-
- uses: actions/checkout@v3
338+
- uses: actions/checkout@v4
333339

334-
- name: Set up Python 3.8
335-
uses: actions/setup-python@v4
340+
- name: Set up Python 3.10.11
341+
uses: actions/setup-python@v5
336342
with:
337-
python-version: 3.8
343+
python-version: "3.10.11"
338344

339345
- name: Download CTranslate2 wheels
340-
uses: actions/download-artifact@v3
346+
uses: actions/download-artifact@v4
341347
with:
342-
name: python-wheels
348+
pattern: python-wheels-${{ runner.os }}-*
349+
merge-multiple: true
350+
path: .
343351

344352
- name: Install CTranslate2 wheel
345353
run: |
346-
pip install *cp38*manylinux*x86_64.whl
354+
pip install *cp310*manylinux*x86_64.whl
347355
348356
- name: Install dependencies to build docs
349357
working-directory: docs

0 commit comments

Comments
 (0)