Skip to content

Commit 0c096ef

Browse files
Alexsandrussnapetrov
Alexsandruss
authored and
napetrov
committed
CI matrix and sklearn>1.1 fixes (#1086)
* fix for 'normalize' deprecation in linear_model * Add initial jobs matrix * Rework matrix * Dependencies fix * Disable DPCPPROOT; fix clang-format installation * Modify sklearn dep * Modify scikit-learn installation with pip * Disable df examples/tests * fix for sed command * Fix scipy version on Windows * Fix scipy version on Windows 2 * Build docs only on py3.9 * Build docs only on py3.9 n2 * Build docs only on py3.9 n3 * Select one job for docs generation * Increase testing numpy version on Windows * Extend matrix to sklearn 1.2 * PCA n_features_ deprecation * Update linear model __init__ * Correct daal version with fixed decision forest * Move docs building to separate job * Change matrix; update deps; add sklearnex/* tests * Enable onedal iface * Temp.disable dpcpp * Update pybind11 for py311 * Add upgrade for testing deps installation * Add sourcing of MSVS on Windows * Update setup*.py * Correct requirements.txt * Correct conda-recipe/build* - d4p installation * Modify Windows testing * Fix paths for Windows testing * Fix paths for Windows testing 2 * Repair docs job * Repair docs job 2 * Repair docs job 3 * Enable older dpcpp compiler build * Try to fix deselect for daal4py/sklearn tests on Windows * fix daal4py docs build - tbb runtime libs exception * Try to fix deselect for daal4py/sklearn tests on Windows 2 * change dpcpp compiler * Change dep. versions * CI matrix change * Add global patching testing on lnx; remove extra exmples run on mac
1 parent 730f822 commit 0c096ef

File tree

16 files changed

+352
-196
lines changed

16 files changed

+352
-196
lines changed

‎.ci/pipeline/ci.yml‎

Lines changed: 116 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,12 @@
1515
#===============================================================================
1616

1717
variables:
18-
- name: WINDOWS_BASEKIT_URL
19-
value: https://registrationcenter-download.intel.com/akdlm/irc_nas/17768/w_BaseKit_p_2021.2.0.2871_offline.exe
20-
- name: WINDOWS_DPCPP_COMPONENTS
21-
value: intel.oneapi.win.dpcpp-compiler
2218
- name: MACOSX_DEPLOYMENT_TARGET
2319
value: '10.15'
24-
- name: python.version
25-
value: '3.9'
2620
- name: 'PYTHON'
2721
value: python
2822
- name: 'ARGS'
2923
value: '1'
30-
- name: 'OFF_ONEDAL_IFACE'
31-
value: '1'
3224

3325
jobs:
3426
- job: PEP8
@@ -37,62 +29,41 @@ jobs:
3729
steps:
3830
- task: UsePythonVersion@0
3931
inputs:
40-
versionSpec: '$(python.version)'
32+
versionSpec: '3.9'
4133
addToPath: true
4234
- script: |
4335
python -m pip install --upgrade pip setuptools
4436
pip install flake8
4537
flake8 --ignore=E265,E722,E402,F401,F403 --max-line-length=90 --count
4638
displayName: 'PEP 8 check'
47-
- job: Linux
39+
- job: Docs
4840
pool:
4941
vmImage: 'ubuntu-22.04'
5042
steps:
51-
- script: |
52-
bash .ci/scripts/describe_system.sh
53-
displayName: 'System info'
5443
- task: UsePythonVersion@0
55-
displayName: 'Use Python $(python.version)'
5644
inputs:
57-
versionSpec: '$(python.version)'
58-
- script: |
59-
conda update -y -q conda
60-
conda create -q -y -n CB -c intel python=$(python.version) dal-devel impi-devel
61-
displayName: 'Conda create'
45+
versionSpec: '3.9'
46+
addToPath: true
47+
- script: sudo apt-get update && sudo apt-get install -y clang-format
48+
displayName: 'apt-get'
6249
- script: |
63-
. /usr/share/miniconda/etc/profile.d/conda.sh
64-
conda activate CB
50+
pip install daal-devel impi-devel
6551
pip install -r requirements-dev.txt
6652
pip install -r requirements-doc.txt
6753
pip install -r requirements-test.txt
68-
pip install xgboost lightgbm catboost
54+
pip list
6955
displayName: 'Install requirements'
7056
- script: |
71-
. /usr/share/miniconda/etc/profile.d/conda.sh
72-
conda activate CB
73-
export DALROOT=$CONDA_PREFIX
57+
export PREFIX=$(dirname $(dirname $(which python)))
7458
./conda-recipe/build.sh
75-
displayName: 'Build daal4py'
76-
- script: |
77-
. /usr/share/miniconda/etc/profile.d/conda.sh
78-
conda activate CB
79-
python setup_sklearnex.py install --single-version-externally-managed --record=record.txt
80-
displayName: 'Build sklearnex'
81-
- script: |
82-
. /usr/share/miniconda/etc/profile.d/conda.sh
83-
conda activate CB
84-
cd ..
85-
./s/conda-recipe/run_test.sh
86-
displayName: 'Testing'
59+
python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
60+
displayName: 'Build daal4py/sklearnex'
8761
- script: |
88-
. /usr/share/miniconda/etc/profile.d/conda.sh
89-
conda activate CB
62+
export LD_LIBRARY_PATH=$(dirname $(dirname $(which python)))/lib:$LD_LIBRARY_PATH
9063
cd doc/daal4py
9164
make html
9265
displayName: 'Build daal4py documentation'
9366
- script: |
94-
. /usr/share/miniconda/etc/profile.d/conda.sh
95-
conda activate CB
9667
cd doc
9768
make html
9869
displayName: 'Build scikit-learn-intelex documentation'
@@ -106,64 +77,102 @@ jobs:
10677
inputs:
10778
artifactName: 'documentation'
10879
targetPath: '$(Build.ArtifactStagingDirectory)/'
109-
- job: Linux_DPCPP
80+
- job: Linux
81+
strategy:
82+
matrix:
83+
Python3.8_Sklearn0.24:
84+
PYTHON_VERSION: '3.8'
85+
SKLEARN_VERSION: '0.24'
86+
Python3.9_Sklearn1.0:
87+
PYTHON_VERSION: '3.9'
88+
SKLEARN_VERSION: '1.0'
89+
Python3.10_Sklearn1.1:
90+
PYTHON_VERSION: '3.10'
91+
SKLEARN_VERSION: '1.1'
92+
Python3.11_Sklearn1.2:
93+
PYTHON_VERSION: '3.11'
94+
SKLEARN_VERSION: '1.2'
11095
pool:
11196
vmImage: 'ubuntu-22.04'
11297
steps:
113-
- task: UsePythonVersion@0
114-
displayName: 'Use Python $(python.version)'
98+
- script: sudo apt-get update && sudo apt-get install -y clang-format
99+
displayName: 'apt-get'
115100
- script: |
116101
wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
117102
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
118103
rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
119104
echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
120105
sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main"
121106
sudo apt-get update
122-
sudo apt-get install \
123-
intel-oneapi-common-vars \
124-
intel-oneapi-common-licensing \
125-
intel-oneapi-dpcpp-cpp-compiler \
126-
intel-oneapi-dev-utilities \
107+
sudo apt-get install -y \
108+
intel-oneapi-common-vars \
109+
intel-oneapi-common-licensing \
110+
intel-oneapi-dpcpp-cpp-compiler \
111+
intel-oneapi-dev-utilities \
127112
intel-oneapi-libdpstd-devel
128113
sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
129114
sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_
130-
displayName: 'apt-get'
115+
displayName: 'dpcpp installation'
131116
- script: |
132117
bash .ci/scripts/describe_system.sh
133-
displayName: "System info"
118+
displayName: 'System info'
134119
- script: |
135-
conda create -q -y -n CB -c conda-forge conda=4.10 python=$(python.version) conda-build=3.21.8 conda-verify
136-
displayName: Create Anaconda environment
120+
conda update -y -q conda
121+
conda create -q -y -n CB -c conda-forge python=$(PYTHON_VERSION) dal-devel mpich
122+
displayName: 'Conda create'
123+
- script: |
124+
. /usr/share/miniconda/etc/profile.d/conda.sh
125+
conda activate CB
126+
pip install -r requirements-dev.txt
127+
pip list
128+
displayName: 'Install develop requirements'
137129
- script: |
138130
export DPCPPROOT=/opt/intel/oneapi/compiler/latest
139131
. /usr/share/miniconda/etc/profile.d/conda.sh
140132
conda activate CB
141-
conda build --override-channels -c intel -c conda-forge --numpy=1.19 conda-recipe
142-
displayName: conda build
133+
export DALROOT=$CONDA_PREFIX
134+
./conda-recipe/build.sh
135+
displayName: 'Build daal4py'
136+
- script: |
137+
. /usr/share/miniconda/etc/profile.d/conda.sh
138+
conda activate CB
139+
python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
140+
displayName: 'Build sklearnex'
143141
- script: |
144142
. /usr/share/miniconda/etc/profile.d/conda.sh
145143
conda activate CB
146-
pip install daal numpy scikit-learn
147-
pip install -r requirements-test.txt
148-
conda install -y /usr/share/miniconda/envs/CB/conda-bld/linux-64/daal4py*.tar.bz2
149-
python setup_sklearnex.py install --single-version-externally-managed --record=record1.txt
150-
displayName: install sklearnex
144+
sed -i -e "s/scikit-learn==1.2.0/scikit-learn==$(SKLEARN_VERSION).*/" requirements-test.txt
145+
pip install --upgrade -r requirements-test.txt
146+
pip install xgboost lightgbm catboost
147+
pip list
148+
displayName: 'Install testing requirements'
151149
- script: |
152150
. /usr/share/miniconda/etc/profile.d/conda.sh
153151
conda activate CB
154152
cd ..
155-
pytest --pyargs s/sklearnex/tests/
156-
# TODO: remove ignore of test_policy when device names are fixed
157-
pytest --pyargs s/onedal/ --ignore=s/onedal/common/tests/test_policy.py
158-
displayName: test sklearnex
153+
./s/conda-recipe/run_test.sh
154+
displayName: 'Testing'
159155
- script: |
160156
. /usr/share/miniconda/etc/profile.d/conda.sh
161157
conda activate CB
162158
cd ..
163159
python s/.ci/scripts/test_global_patch.py
164160
displayName: global patching testing
165-
166161
- job: MacOS
162+
strategy:
163+
matrix:
164+
Python3.8_Sklearn0.24:
165+
PYTHON_VERSION: '3.8'
166+
SKLEARN_VERSION: '0.24'
167+
Python3.9_Sklearn1.0:
168+
PYTHON_VERSION: '3.9'
169+
SKLEARN_VERSION: '1.0'
170+
Python3.10_Sklearn1.1:
171+
PYTHON_VERSION: '3.10'
172+
SKLEARN_VERSION: '1.1'
173+
Python3.11_Sklearn1.2:
174+
PYTHON_VERSION: '3.11'
175+
SKLEARN_VERSION: '1.2'
167176
pool:
168177
vmImage: 'macos-12'
169178
steps:
@@ -172,74 +181,99 @@ jobs:
172181
sudo chown -R $USER $CONDA
173182
conda config --set always_yes yes --set changeps1 no
174183
conda update -q conda
175-
conda create -n CB python=$(python.version)
176-
conda install -n CB -q cython scipy pytest pandas pyyaml joblib numpydoc jinja2 numpy clang-tools lightgbm scikit-learn
177-
conda install -n CB -q --override-channels -c intel dal dal-include
178-
conda install -n CB -q --override-channels -c conda-forge mpich
184+
conda create -n CB -c conda-forge python=$(PYTHON_VERSION) dal dal-include mpich clang-format
179185
displayName: Create Anaconda environment
180186
- script: |
181187
source activate CB
182188
pip install -q cpufeature
183189
bash .ci/scripts/describe_system.sh
184190
displayName: "System info"
191+
- script: |
192+
source activate CB
193+
pip install -r requirements-dev.txt
194+
pip list
195+
displayName: 'Install develop requirements'
185196
- script: |
186197
source activate CB
187198
export DALROOT=$CONDA_PREFIX
188199
./conda-recipe/build.sh
189-
python setup_sklearnex.py install --single-version-externally-managed --record=record1.txt
200+
python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
190201
displayName: Conda build
202+
- script: |
203+
source activate CB
204+
sed -i.bak -e "s/scikit-learn==1.2.0/scikit-learn==$(SKLEARN_VERSION).*/" requirements-test.txt
205+
pip install --upgrade -r requirements-test.txt
206+
pip install xgboost lightgbm catboost
207+
pip list
208+
displayName: 'Install testing requirements'
191209
- script: |
192210
source activate CB
193211
cd ..
194212
./s/conda-recipe/run_test.sh
195213
displayName: Testing
196-
- script: |
197-
source activate CB
198-
cd examples/daal4py && python run_examples.py
199-
displayName: Examples
200214
- script: |
201215
source activate CB
202216
cd ..
203217
python s/.ci/scripts/test_global_patch.py
204218
displayName: global patching testing
205-
206219
- job: Windows
220+
strategy:
221+
matrix:
222+
Python3.8_Sklearn0.24:
223+
PYTHON_VERSION: '3.8'
224+
SKLEARN_VERSION: '0.24'
225+
Python3.9_Sklearn1.0:
226+
PYTHON_VERSION: '3.9'
227+
SKLEARN_VERSION: '1.0'
228+
Python3.10_Sklearn1.1:
229+
PYTHON_VERSION: '3.10'
230+
SKLEARN_VERSION: '1.1'
231+
Python3.11_Sklearn1.2:
232+
PYTHON_VERSION: '3.11'
233+
SKLEARN_VERSION: '1.2'
207234
pool:
208235
vmImage: 'windows-latest'
209236
steps:
210-
- task: UsePythonVersion@0
211-
displayName: 'Use Python $(python.version)'
212237
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
213238
displayName: Add conda to PATH
214-
- script: conda create -q -y -n CB -c intel -c conda-forge python=$(python.version) dal-devel impi-devel clang-tools
239+
- script: conda create -q -y -n CB -c conda-forge -c intel python=$(PYTHON_VERSION) dal-devel impi-devel clang-format
215240
displayName: 'Create Anaconda environment'
216241
- script: |
217242
call activate CB
218243
pip install --upgrade setuptools
219244
pip install cpufeature
220245
pip install -r requirements-dev.txt
221-
pip install -r requirements-test.txt
246+
pip list
222247
displayName: 'Install requirements'
223248
- script: |
224249
set PATH=C:\msys64\usr\bin;%PATH%
225250
call activate CB
226251
bash .ci/scripts/describe_system.sh
227252
displayName: 'System info'
228253
- script: |
254+
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
229255
call activate CB
230256
set PREFIX=%CONDA_PREFIX%
231257
set PYTHON=python
232258
call conda-recipe\bld.bat
233259
displayName: 'Build daal4py'
234260
- script: |
235261
call activate CB
236-
python setup_sklearnex.py install --single-version-externally-managed --record=record.txt
262+
python setup_sklearnex.py install --single-version-externally-managed --record=record_sklearnex.txt
237263
displayName: 'Build sklearnex'
264+
- script: |
265+
set PATH=C:\msys64\usr\bin;%PATH%
266+
call activate CB
267+
sed -i -e "s/scikit-learn==1.2.0/scikit-learn==$(SKLEARN_VERSION).*/" requirements-test.txt
268+
pip install --upgrade -r requirements-test.txt
269+
pip install xgboost lightgbm catboost
270+
pip list
271+
displayName: 'Install testing requirements'
238272
- script: |
239273
call activate CB
240274
cd ..
241-
pytest --pyargs s/sklearnex/tests/
242-
displayName: sklearnex testing
275+
call s\conda-recipe\run_test.bat s
276+
displayName: daal4py/sklearnex testing
243277
- script: |
244278
call activate CB
245279
cd ..

‎conda-recipe/bld.bat‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ rem See the License for the specific language governing permissions and
1515
rem limitations under the License.
1616
rem ============================================================================
1717

18-
set DAAL4PY_VERSION=%PKG_VERSION%
18+
IF DEFINED PKG_VERSION (set DAAL4PY_VERSION=%PKG_VERSION%)
19+
1920
set MPIROOT=%PREFIX%\Library
2021

2122
IF DEFINED DAALROOT (set DALROOT=%DAALROOT%)
@@ -33,4 +34,4 @@ set PATH=%PATH%;%PREFIX%\Library\bin\libfabric
3334

3435
%PYTHON% setup.py build %BUILD_ARGS%
3536
IF %ERRORLEVEL% neq 0 EXIT /b %ERRORLEVEL%
36-
%PYTHON% setup.py install --single-version-externally-managed --record record.txt
37+
%PYTHON% setup.py install --single-version-externally-managed --record record_daal4py.txt

‎conda-recipe/build.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
#===============================================================================
1717

1818
if [ "$PY3K" == "1" ]; then
19-
ARGS=" --single-version-externally-managed --record=record.txt"
19+
ARGS=" --single-version-externally-managed --record=record_daal4py.txt"
2020
else
2121
ARGS="--old-and-unmanageable"
2222
fi
@@ -40,6 +40,8 @@ if [ "$(uname)" == "Darwin" ]; then
4040
export CXX=g++
4141
fi
4242

43-
export DAAL4PY_VERSION=$PKG_VERSION
43+
if [ ! -z "${PKG_VERSION}" ]; then
44+
export DAAL4PY_VERSION=$PKG_VERSION
45+
fi
4446
export MPIROOT=${PREFIX}
4547
${PYTHON} setup.py install $ARGS

‎conda-recipe/run_test.bat‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ rem See the License for the specific language governing permissions and
1515
rem limitations under the License.
1616
rem ============================================================================
1717

18-
set DAAL4PY_VERSION=%PKG_VERSION%
18+
rem %1 - scikit-learn-intelex repo root
19+
1920
set MPIROOT=%PREFIX%\Library
2021

2122
IF DEFINED DPCPPROOT (
@@ -38,3 +39,8 @@ IF DEFINED TBBROOT (
3839
echo "Sourcing TBBROOT"
3940
call "%TBBROOT%\env\vars.bat"
4041
)
42+
43+
%PYTHON% -m unittest discover -v -s %1\tests -p test*.py
44+
45+
pytest --verbose --pyargs %1\daal4py\sklearn --deselect="daal4py/sklearn/ensemble/tests/test_decision_forest.py::test_classifier_big_estimators_iris[8000]" --deselect="daal4py/sklearn/ensemble/tests/test_decision_forest.py::test_mse_regressor_big_estimators_iris[8000]"
46+
pytest --verbose --pyargs %1\sklearnex

0 commit comments

Comments
 (0)