From b1f3f2c2e25003ba3b5b3a5888cf617929f60238 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 10:49:20 +0200 Subject: [PATCH 01/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 071bec486f..8a468c14a0 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -50,8 +50,10 @@ from onedal.tests.utils._device_selection import get_queues +supported_frameworks = "numpy,pandas,dpnp,dpctl,array_api" + def get_dataframes_and_queues( - dataframe_filter_="numpy,pandas,dpnp,dpctl,array_api", device_filter_="cpu,gpu" + dataframe_filter_=None, device_filter_="cpu,gpu" ): """Get supported dataframes for testing. @@ -59,8 +61,9 @@ def get_dataframes_and_queues( Parameters ---------- - dataframe_filter_ : str, default="numpy,pandas,dpnp,dpctl" + dataframe_filter_ : str, default=None Configure output pytest.params for the certain dataframe formats. + When None, will default to value of `supported_frameworks`. device_filter_ : str, default="cpu,gpu" Configure output pytest.params with certain sycl queue for the dataframe, where it is applicable. @@ -82,6 +85,8 @@ def get_dataframes_and_queues( _convert_to_dataframe : Converted input object to certain dataframe format. """ dataframes_and_queues = [] + if not dataframe_filter_: + dataframe_filter = supported_frameworks if "numpy" in dataframe_filter_: dataframes_and_queues.append(pytest.param("numpy", None, id="numpy")) From dfa286f50b0b337f6fada3b08afc1d55b8cd300f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 11:09:10 +0200 Subject: [PATCH 02/74] Update conftest.py --- sklearnex/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index 4ecad5383b..1a7ff8c2fe 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -23,6 +23,11 @@ from sklearnex import config_context, patch_sklearn, unpatch_sklearn +def pytest_addoption(parser): + + parser.addoption(“--frameworks”, default="numpy,pandas,dpnp,dpctl,array_api") + + def pytest_configure(config): config.addinivalue_line( "markers", "allow_sklearn_fallback: mark test to not check for sklearnex usage" From 55d406c75b732f6093c49d857c4bf8e3f9fb6a02 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 15:29:31 +0200 Subject: [PATCH 03/74] Update conftest.py --- sklearnex/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index 1a7ff8c2fe..507bcf9813 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -16,6 +16,7 @@ import io import logging +import os import pytest @@ -24,8 +25,7 @@ def pytest_addoption(parser): - - parser.addoption(“--frameworks”, default="numpy,pandas,dpnp,dpctl,array_api") + parser.addoption(“--frameworks”, action="store", default="") def pytest_configure(config): @@ -35,6 +35,8 @@ def pytest_configure(config): config.addinivalue_line( "markers", "mpi: mark test to require MPI for distributed testing" ) + if val := config.getoption("--frameworks"): + os.environ["ONEDAL_PYTEST_FRAMEWORKS"] = val @pytest.hookimpl(hookwrapper=True) From b3409fea60de1140d5e64c421e218ecabd1a06de Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 15:34:56 +0200 Subject: [PATCH 04/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 8a468c14a0..786c06f159 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import os import pytest import scipy.sparse as sp @@ -50,7 +51,7 @@ from onedal.tests.utils._device_selection import get_queues -supported_frameworks = "numpy,pandas,dpnp,dpctl,array_api" +test_frameworks = os.environ.get("ONEDAL_PYTEST_FRAMEWORKS", "numpy,pandas,dpnp,dpctl,array_api") def get_dataframes_and_queues( dataframe_filter_=None, device_filter_="cpu,gpu" @@ -63,7 +64,7 @@ def get_dataframes_and_queues( ---------- dataframe_filter_ : str, default=None Configure output pytest.params for the certain dataframe formats. - When None, will default to value of `supported_frameworks`. + When it evaluates False, it will default to value of `test_frameworks`. device_filter_ : str, default="cpu,gpu" Configure output pytest.params with certain sycl queue for the dataframe, where it is applicable. @@ -86,7 +87,7 @@ def get_dataframes_and_queues( """ dataframes_and_queues = [] if not dataframe_filter_: - dataframe_filter = supported_frameworks + dataframe_filter_ = test_frameworks if "numpy" in dataframe_filter_: dataframes_and_queues.append(pytest.param("numpy", None, id="numpy")) From 808b9f00346cb2989b1d5eed8574192a8ec64cf4 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 15:57:04 +0200 Subject: [PATCH 05/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 786c06f159..04d554d73e 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import os +from functools import lru_cache import pytest import scipy.sparse as sp @@ -53,18 +54,20 @@ test_frameworks = os.environ.get("ONEDAL_PYTEST_FRAMEWORKS", "numpy,pandas,dpnp,dpctl,array_api") +@lru_cache def get_dataframes_and_queues( dataframe_filter_=None, device_filter_="cpu,gpu" ): """Get supported dataframes for testing. This is meant to be used for testing purposes only. + It is to be used only for pytest test collection. Parameters ---------- dataframe_filter_ : str, default=None Configure output pytest.params for the certain dataframe formats. - When it evaluates False, it will default to value of `test_frameworks`. + When it evaluates False, it will default to value of ``test_frameworks``. device_filter_ : str, default="cpu,gpu" Configure output pytest.params with certain sycl queue for the dataframe, where it is applicable. @@ -89,6 +92,10 @@ def get_dataframes_and_queues( if not dataframe_filter_: dataframe_filter_ = test_frameworks + # filter dataframe_filter based on available test frameworks + if dataframe_filter_ != test_frameworks: + dataframe_filter_ = ",".join([i for i in dataframe_filter_.split(",") if i in test_frameworks]) + if "numpy" in dataframe_filter_: dataframes_and_queues.append(pytest.param("numpy", None, id="numpy")) if "pandas" in dataframe_filter_: From 8240f9d9a0e5077925e6ddca0b5edc1379bd32fb Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:00:24 +0200 Subject: [PATCH 06/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 04d554d73e..0dbb0385cd 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -51,13 +51,12 @@ from onedal.tests.utils._device_selection import get_queues - -test_frameworks = os.environ.get("ONEDAL_PYTEST_FRAMEWORKS", "numpy,pandas,dpnp,dpctl,array_api") +test_frameworks = os.environ.get( + "ONEDAL_PYTEST_FRAMEWORKS", "numpy,pandas,dpnp,dpctl,array_api" +) @lru_cache -def get_dataframes_and_queues( - dataframe_filter_=None, device_filter_="cpu,gpu" -): +def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): """Get supported dataframes for testing. This is meant to be used for testing purposes only. From d6a34c9e364f19efe9e8559953a78dd2a04daa89 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:02:51 +0200 Subject: [PATCH 07/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 0dbb0385cd..ea594035c6 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -88,12 +88,12 @@ def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): _convert_to_dataframe : Converted input object to certain dataframe format. """ dataframes_and_queues = [] - if not dataframe_filter_: - dataframe_filter_ = test_frameworks # filter dataframe_filter based on available test frameworks - if dataframe_filter_ != test_frameworks: + if dataframe_filter_: dataframe_filter_ = ",".join([i for i in dataframe_filter_.split(",") if i in test_frameworks]) + else: + dataframe_filter_ = test_frameworks if "numpy" in dataframe_filter_: dataframes_and_queues.append(pytest.param("numpy", None, id="numpy")) From 37876fcdba28d91608827a289462ca7c889665a2 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:06:05 +0200 Subject: [PATCH 08/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index ea594035c6..14188430d2 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -91,7 +91,9 @@ def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): # filter dataframe_filter based on available test frameworks if dataframe_filter_: - dataframe_filter_ = ",".join([i for i in dataframe_filter_.split(",") if i in test_frameworks]) + dataframe_filter_ = ",".join( + [i for i in dataframe_filter_.split(",") if i in test_frameworks] + ) else: dataframe_filter_ = test_frameworks From 12906435eb3dfe756d3d8c70ad7af118c9a007d7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:06:19 +0200 Subject: [PATCH 09/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 14188430d2..1a3aeaedf3 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -55,6 +55,7 @@ "ONEDAL_PYTEST_FRAMEWORKS", "numpy,pandas,dpnp,dpctl,array_api" ) + @lru_cache def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): """Get supported dataframes for testing. From 39019d3e181435aad0ef55f471cb13842f59ddf6 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:06:55 +0200 Subject: [PATCH 10/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 1a3aeaedf3..961c4067b5 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -90,7 +90,7 @@ def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): """ dataframes_and_queues = [] - # filter dataframe_filter based on available test frameworks + # filter dataframe_filter_ based on available test frameworks if dataframe_filter_: dataframe_filter_ = ",".join( [i for i in dataframe_filter_.split(",") if i in test_frameworks] From 14b5e89577b273938f65faf59e645f5a027a43de Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:10:41 +0200 Subject: [PATCH 11/74] Update conftest.py --- sklearnex/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index 507bcf9813..8baa44667d 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -25,7 +25,7 @@ def pytest_addoption(parser): - parser.addoption(“--frameworks”, action="store", default="") + parser.addoption("--frameworks", action="store", default="") def pytest_configure(config): From 510791be234411200c0dea372675679373586c65 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:13:48 +0200 Subject: [PATCH 12/74] Update conftest.py --- sklearnex/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index 8baa44667d..fff9367375 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -35,6 +35,7 @@ def pytest_configure(config): config.addinivalue_line( "markers", "mpi: mark test to require MPI for distributed testing" ) + # this assumes onedal.tests.utils._dataframes_support has not been imported yet if val := config.getoption("--frameworks"): os.environ["ONEDAL_PYTEST_FRAMEWORKS"] = val From a3aba9881c9bfaa2f3c422d8f86bb0c330da856a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:25:20 +0200 Subject: [PATCH 13/74] Update conftest.py --- sklearnex/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index fff9367375..9d04f5be35 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -26,7 +26,7 @@ def pytest_addoption(parser): parser.addoption("--frameworks", action="store", default="") - + def pytest_configure(config): config.addinivalue_line( From ed600bd0405e77a3d6204c464bf7556809817441 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 16:57:16 +0200 Subject: [PATCH 14/74] Update _dataframes_support.py --- onedal/tests/utils/_dataframes_support.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/onedal/tests/utils/_dataframes_support.py b/onedal/tests/utils/_dataframes_support.py index 961c4067b5..7a24cbec99 100644 --- a/onedal/tests/utils/_dataframes_support.py +++ b/onedal/tests/utils/_dataframes_support.py @@ -14,7 +14,6 @@ # limitations under the License. # =============================================================================== import os -from functools import lru_cache import pytest import scipy.sparse as sp @@ -56,7 +55,6 @@ ) -@lru_cache def get_dataframes_and_queues(dataframe_filter_=None, device_filter_="cpu,gpu"): """Get supported dataframes for testing. From 90f7e418a02a144b1fd46d0cfbddf3369ae0eed2 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 22:32:38 +0200 Subject: [PATCH 15/74] Update run_test.sh --- conda-recipe/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index 28620adfb5..09cbc95952 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -69,7 +69,7 @@ return_code=$(($return_code + $?)) pytest --verbose --pyargs daal4py $@ $(generate_pytest_args daal4py) return_code=$(($return_code + $?)) -pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex) +pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex) --frameworks=numpy return_code=$(($return_code + $?)) pytest --verbose --pyargs onedal $@ $(generate_pytest_args onedal) From 304cd59c566ffbc7ae2413883e48b66ac4951538 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 23:25:06 +0200 Subject: [PATCH 16/74] Update conftest.py --- sklearnex/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sklearnex/conftest.py b/sklearnex/conftest.py index 9d04f5be35..4ecad5383b 100644 --- a/sklearnex/conftest.py +++ b/sklearnex/conftest.py @@ -16,7 +16,6 @@ import io import logging -import os import pytest @@ -24,10 +23,6 @@ from sklearnex import config_context, patch_sklearn, unpatch_sklearn -def pytest_addoption(parser): - parser.addoption("--frameworks", action="store", default="") - - def pytest_configure(config): config.addinivalue_line( "markers", "allow_sklearn_fallback: mark test to not check for sklearnex usage" @@ -35,9 +30,6 @@ def pytest_configure(config): config.addinivalue_line( "markers", "mpi: mark test to require MPI for distributed testing" ) - # this assumes onedal.tests.utils._dataframes_support has not been imported yet - if val := config.getoption("--frameworks"): - os.environ["ONEDAL_PYTEST_FRAMEWORKS"] = val @pytest.hookimpl(hookwrapper=True) From 4399e5b36b86d6e2544de9bc85b20ac77bbb7fdf Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 24 May 2025 23:26:35 +0200 Subject: [PATCH 17/74] Update run_test.sh --- conda-recipe/run_test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index 09cbc95952..005f538041 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -60,6 +60,8 @@ function generate_pytest_args { printf -- "${ARGS[*]}" } +export ONEDAL_PYTEST_FRAMEWORKS=numpy + ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()" return_code=$(($return_code + $?)) @@ -69,7 +71,7 @@ return_code=$(($return_code + $?)) pytest --verbose --pyargs daal4py $@ $(generate_pytest_args daal4py) return_code=$(($return_code + $?)) -pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex) --frameworks=numpy +pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex) return_code=$(($return_code + $?)) pytest --verbose --pyargs onedal $@ $(generate_pytest_args onedal) From 37ca4fa72a17046c474763c88cbe6405f7a9420a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 25 May 2025 00:18:09 +0200 Subject: [PATCH 18/74] Update run_test.sh --- conda-recipe/run_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index 005f538041..28620adfb5 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -60,8 +60,6 @@ function generate_pytest_args { printf -- "${ARGS[*]}" } -export ONEDAL_PYTEST_FRAMEWORKS=numpy - ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()" return_code=$(($return_code + $?)) From 0d50b627048f322ed112748274c6b4e2dd8214cb Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 00:35:43 +0200 Subject: [PATCH 19/74] Update ci.yml --- .github/workflows/ci.yml | 122 +++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a231191c..75f15956db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,30 @@ env: DPCTL_VERSION: 0.18.1 DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' + UXL_PYTHONVERSION: "3.12" + ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: + + onedal_nightly: + runs-on: ubuntu-24.04 + name: Identify oneDAL nightly + timeout-minutes: 5 + + steps: + - name: Get run ID of "Nightly-build" workflow + id: get-run-id + run: | + WF_NAME="Nightly-build" + JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId' + RUN_ID=`gh run --repo ${{ env.ONEDAL_REPO }} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"` + echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}" + echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + sklearn_lnx: + needs: onedal_nightly strategy: fail-fast: false matrix: @@ -57,23 +78,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.PYTHON_VERSION }} - - name: Get run ID of "Nightly-build" workflow - id: get-run-id - run: | - OTHER_REPO="uxlfoundation/oneDAL" - WF_NAME="Nightly-build" - JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId' - RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"` - echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}" - echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - name: Download oneDAL build artifact uses: actions/download-artifact@v4 with: name: __release_lnx github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL + repository: ${{ env.ONEDAL_REPO }} run-id: ${{ steps.get-run-id.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact @@ -81,7 +91,7 @@ jobs: with: name: oneDAL_env github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL + repository: ${{ env.ONEDAL_REPO }} run-id: ${{ steps.get-run-id.outputs.run-id }} path: .ci/env - name: Set Environment Variables @@ -161,6 +171,7 @@ jobs: bash .ci/scripts/run_sklearn_tests.sh $CPU sklearn_win: + needs: onedal_nightly strategy: fail-fast: false matrix: @@ -182,33 +193,21 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.PYTHON_VERSION }} - - name: Get run ID of "Nightly-build" workflow - id: get-run-id - shell: bash - run: | - OTHER_REPO="uxlfoundation/oneDAL" - WF_NAME="Nightly-build" - JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId' - RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"` - echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}" - echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - name: Download oneDAL build artifact uses: actions/download-artifact@v4 with: name: __release_win github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL - run-id: ${{ steps.get-run-id.outputs.run-id }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.get-run-id.outputs.run-id }} path: ./__release_win - name: Download Intel BaseKit artifact uses: actions/download-artifact@v4 with: name: intel_oneapi_basekit github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL - run-id: ${{ steps.get-run-id.outputs.run-id }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.get-run-id.outputs.run-id }} - name: Decompress Intel BaseKit shell: cmd run: | @@ -313,3 +312,68 @@ jobs: if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu set SKLEARNEX_PREVIEW=YES bash .ci/scripts/run_sklearn_tests.sh %CPU% + + build_uxl: + needs: onedal_nightly + name: LinuxNightly build Python${{ env.UXL_PYTHONVERSION }} + runs-on: uxl-xlarge + timeout-minutes: 30 + + steps: + - name: Checkout Scikit-learn-intelex + uses: actions/checkout@v4 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.UXL_PYTHONVERSION }} + cache: 'pip' + cache-dependency-path: dependencies-dev + - name: Download oneDAL build artifact + uses: actions/download-artifact@v4 + with: + name: __release_lnx + github-token: ${{ github.token }} + repository: uxlfoundation/oneDAL + run-id: ${{ needs.get-run-id.outputs.run-id }} + path: ./__release_lnx + - name: Download oneDAL environment artifact + uses: actions/download-artifact@v4 + with: + name: oneDAL_env + github-token: ${{ github.token }} + repository: uxlfoundation/oneDAL + run-id: ${{ needs.get-run-id.outputs.run-id }} + path: .ci/env + - name: Set Environment Variables + id: set-env + run: | + # Disable SPMD testing + echo "NO_DIST=1" >> "$GITHUB_ENV" + # enable coverage report generation + echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" + echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"; fi + - name: apt-get + run: sudo apt-get update && sudo apt-get install -y clang-format + - name: dpcpp installation + run: | + # This CI system yields oneAPI dependencies from the oneDAL repository + bash .ci/env/apt.sh dpcpp + - name: describe system + run: | + source /opt/intel/oneapi/setvars.sh + bash .ci/scripts/describe_system.sh + - name: Install develop requirements + run: | + pip install -r dependencies-dev + pip list + - name: Build daal4py/sklearnex + run: | + source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} + python setup.py build --single-version-externally-managed --record=record.txt --wheel + - name: Archive sklearnex build + uses: actions/upload-artifact@v4 + with: + name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} + path: | + *.whl + From 586d8b1203986d22e0700781a78bd56da1068de2 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 00:37:41 +0200 Subject: [PATCH 20/74] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75f15956db..131ccbbc2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: name: __release_lnx github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ steps.get-run-id.outputs.run-id }} + run-id: ${{ needs.get-run-id.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact uses: actions/download-artifact@v4 @@ -92,7 +92,7 @@ jobs: name: oneDAL_env github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ steps.get-run-id.outputs.run-id }} + run-id: ${{ needs.get-run-id.outputs.run-id }} path: .ci/env - name: Set Environment Variables id: set-env From cfcb64a9076e36dc61d8a0c9a40184bba95aa073 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 00:45:24 +0200 Subject: [PATCH 21/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 131ccbbc2f..1aa6d3e14f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -315,7 +315,7 @@ jobs: build_uxl: needs: onedal_nightly - name: LinuxNightly build Python${{ env.UXL_PYTHONVERSION }} + name: LinuxNightly GPU build runs-on: uxl-xlarge timeout-minutes: 30 From 955c200670271f8602ea9ffdd2d489d05ac0300f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 00:55:26 +0200 Subject: [PATCH 22/74] Update ci.yml --- .github/workflows/ci.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa6d3e14f..e857eab79d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: name: __release_lnx github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact uses: actions/download-artifact@v4 @@ -92,7 +92,7 @@ jobs: name: oneDAL_env github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: .ci/env - name: Set Environment Variables id: set-env @@ -199,7 +199,7 @@ jobs: name: __release_win github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: ./__release_win - name: Download Intel BaseKit artifact uses: actions/download-artifact@v4 @@ -207,7 +207,7 @@ jobs: name: intel_oneapi_basekit github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} - name: Decompress Intel BaseKit shell: cmd run: | @@ -233,8 +233,8 @@ jobs: with: name: opencl_rt_installer github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL - run-id: ${{ steps.get-run-id.outputs.run-id }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: . - name: Install Intel OpenCL CPU Runtime if: ${{ steps.set-env.outputs.DPCFLAG == '' }} @@ -314,6 +314,7 @@ jobs: bash .ci/scripts/run_sklearn_tests.sh %CPU% build_uxl: + if: github.repository == 'uxlfoundation/scikit-learn-intelex' needs: onedal_nightly name: LinuxNightly GPU build runs-on: uxl-xlarge @@ -333,16 +334,16 @@ jobs: with: name: __release_lnx github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL - run-id: ${{ needs.get-run-id.outputs.run-id }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact uses: actions/download-artifact@v4 with: name: oneDAL_env github-token: ${{ github.token }} - repository: uxlfoundation/oneDAL - run-id: ${{ needs.get-run-id.outputs.run-id }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} path: .ci/env - name: Set Environment Variables id: set-env From f7fd7b5a44d062cb9a01fc1f34150a5e9acc9f67 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:01:30 +0200 Subject: [PATCH 23/74] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e857eab79d..ad880f042b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: name: __release_lnx github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact uses: actions/download-artifact@v4 @@ -92,7 +92,7 @@ jobs: name: oneDAL_env github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: .ci/env - name: Set Environment Variables id: set-env From c6c96b4f4b028f2978da325e61c517b090a39411 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:04:07 +0200 Subject: [PATCH 24/74] Update ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad880f042b..53ab2f4445 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,8 @@ jobs: echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT" env: GH_TOKEN: ${{ github.token }} + outputs: + run-id: ${{ steps.get-run-id.outputs.run-id }} sklearn_lnx: needs: onedal_nightly From bbba3f0cf3641edb5995edc1c156d27d7de81414 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:06:44 +0200 Subject: [PATCH 25/74] Update ci.yml --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53ab2f4445..b55ecb2942 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: name: __release_win github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: ./__release_win - name: Download Intel BaseKit artifact uses: actions/download-artifact@v4 @@ -209,7 +209,7 @@ jobs: name: intel_oneapi_basekit github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} - name: Decompress Intel BaseKit shell: cmd run: | @@ -236,7 +236,7 @@ jobs: name: opencl_rt_installer github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: . - name: Install Intel OpenCL CPU Runtime if: ${{ steps.set-env.outputs.DPCFLAG == '' }} @@ -337,7 +337,7 @@ jobs: name: __release_lnx github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: ./__release_lnx - name: Download oneDAL environment artifact uses: actions/download-artifact@v4 @@ -345,7 +345,7 @@ jobs: name: oneDAL_env github-token: ${{ github.token }} repository: ${{ env.ONEDAL_REPO }} - run-id: ${{ needs.onedal_nightly.get-run-id.outputs.run-id }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} path: .ci/env - name: Set Environment Variables id: set-env From cef844b0d44d52b7e99a8e86b308cfea48c4ffee Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:09:14 +0200 Subject: [PATCH 26/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b55ecb2942..fa13aa8e8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -354,7 +354,7 @@ jobs: echo "NO_DIST=1" >> "$GITHUB_ENV" # enable coverage report generation echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" - echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"; fi + echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format - name: dpcpp installation From 383db206442c4466915774ae703353999bebbd7b Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:14:31 +0200 Subject: [PATCH 27/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa13aa8e8a..fa45920db1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -372,7 +372,7 @@ jobs: - name: Build daal4py/sklearnex run: | source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} - python setup.py build --single-version-externally-managed --record=record.txt --wheel + python setup.py build --record=record.txt --wheel - name: Archive sklearnex build uses: actions/upload-artifact@v4 with: From 4804287d8357dcd15b5dd2ef8820a27b8e1f6998 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:17:59 +0200 Subject: [PATCH 28/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa45920db1..a0a02cb07c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -372,7 +372,7 @@ jobs: - name: Build daal4py/sklearnex run: | source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} - python setup.py build --record=record.txt --wheel + python setup.py build --wheel - name: Archive sklearnex build uses: actions/upload-artifact@v4 with: From 183c9e09dec1ee69b61951c3909d86c920d12c85 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 01:24:59 +0200 Subject: [PATCH 29/74] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0a02cb07c..4b5d594268 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -372,11 +372,11 @@ jobs: - name: Build daal4py/sklearnex run: | source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} - python setup.py build --wheel + python setup.py bdist_wheel - name: Archive sklearnex build uses: actions/upload-artifact@v4 with: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} path: | - *.whl + ./dist/*.whl From 58fe7fafaaf32878e0cb4f0b8cd4025bbefd37cb Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 08:50:34 +0200 Subject: [PATCH 30/74] Update ci.yml --- .github/workflows/ci.yml | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b5d594268..7137101309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ env: DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' UXL_PYTHONVERSION: "3.12" + UXL_SKLEARNVERSION: "1.6" ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: @@ -56,6 +57,8 @@ jobs: GH_TOKEN: ${{ github.token }} outputs: run-id: ${{ steps.get-run-id.outputs.run-id }} + uxl-python: ${{ env.UXL_PYTHONVERSION }} + uxl-sklearn: ${{ env.UXL_SKLEARNVERSION }} sklearn_lnx: needs: onedal_nightly @@ -380,3 +383,98 @@ jobs: path: | ./dist/*.whl + test_uxl: + strategy: + fail-fast: false + matrix: + include: + - OS: uxl-gpu-xlarge + FRAMEWORKS: "numpy,pytorch" + DEVICE: GPU + - OS: uxl-xlarge + FRAMEWORKS: "numpy,pandas" + DEVICE: CPU + needs: build_uxl + name: LinuxNightly Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} test ${{ matrix.FRAMEWORKS }} + runs-on: uxl-xlarge + timeout-minutes: 30 + steps: + - name: Checkout Scikit-learn-intelex + uses: actions/checkout@v4 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.UXL_PYTHONVERSION }} + - name: Download oneDAL build artifact + uses: actions/download-artifact@v4 + with: + name: __release_lnx + github-token: ${{ github.token }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} + path: ./__release_lnx + - name: Download oneDAL environment artifact + uses: actions/download-artifact@v4 + with: + name: oneDAL_env + github-token: ${{ github.token }} + repository: ${{ env.ONEDAL_REPO }} + run-id: ${{ needs.onedal_nightly.outputs.run-id }} + path: .ci/env + - name: Set Environment Variables + id: set-env + run: | + # enable coverage report generation + echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" + echo "ONEDAL_PYTEST_FRAMEWORKS=${{ matrix.FRAMEWORKS }}" >> "$GITHUB_ENV" + - name: apt-get + run: sudo apt-get update + - name: dpcpp installation + run: | + # This CI system yields oneAPI dependencies from the oneDAL repository + bash .ci/env/apt.sh dpcpp + - name: describe system + run: | + source /opt/intel/oneapi/setvars.sh + bash .ci/scripts/describe_system.sh + - name: Install requirements + run: | + pip install -r dependencies-dev + pip list + echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" + bash .ci/scripts/setup_sklearn.sh ${{ env.UXL_SKLEARNVERSION }} + pip install --upgrade -r requirements-test.txt + pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ env.UXL_SKLEARVERSION }}) pyyaml + - name: Download sklearnex wheel + uses: actions/download-artifact@v4 + with: + name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} + path: sklearnex.whl + - name: Install daal4py/sklearnex + run: pip install sklearnex.whl + - name: Sklearnex testing + run: | + source .github/scripts/activate_components.sh + export COVERAGE_FILE=$(pwd)/.coverage.sklearnex + cd .ci + ../conda-recipe/run_test.sh + - name: Sklearn testing + run: | + source .github/scripts/activate_components.sh + export COVERAGE_FILE=$(pwd)/.coverage.sklearn + bash .ci/scripts/run_sklearn_tests.sh ${{ matrix.DEVICE }} + - name: Create coverage report + run: | + source .github/scripts/activate_components.sh + bash .github/scripts/generate_coverage_reports.sh uxl_lnx_${{ matrix.DEVICE }} + - name: Archive coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage_uxl_lnx_${{ matrix.DEVICE }} + path: | + *uxl_lnx_${{ matrix.DEVICE }}.info + - name: Sklearn testing [preview] + run: | + source .github/scripts/activate_components.sh + export SKLEARNEX_PREVIEW='YES' + bash .ci/scripts/run_sklearn_tests.sh ${{ matrix.DEVICE }} From 55d07b2503e0be2b7bda730286779945f195a321 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 09:10:57 +0200 Subject: [PATCH 31/74] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7137101309..9e38d47448 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -394,9 +394,9 @@ jobs: - OS: uxl-xlarge FRAMEWORKS: "numpy,pandas" DEVICE: CPU - needs: build_uxl - name: LinuxNightly Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} test ${{ matrix.FRAMEWORKS }} - runs-on: uxl-xlarge + needs: [onedal_nightly, build_uxl] + name: LinuxNightly ${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} + runs-on: ${{ matrix.OS }} timeout-minutes: 30 steps: - name: Checkout Scikit-learn-intelex From 1326b9a8e0ac3f2303be2261df65cf69459c2095 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 09:33:31 +0200 Subject: [PATCH 32/74] Update ci.yml --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e38d47448..f72afe5740 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -449,9 +449,8 @@ jobs: uses: actions/download-artifact@v4 with: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} - path: sklearnex.whl - name: Install daal4py/sklearnex - run: pip install sklearnex.whl + run: pip install *.whl - name: Sklearnex testing run: | source .github/scripts/activate_components.sh From a786c3d351c9267faaa20d9461aa01c5a02a3145 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 10:09:55 +0200 Subject: [PATCH 33/74] Update ci.yml --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f72afe5740..d0944ab3ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -356,7 +356,6 @@ jobs: # Disable SPMD testing echo "NO_DIST=1" >> "$GITHUB_ENV" # enable coverage report generation - echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format @@ -424,6 +423,7 @@ jobs: - name: Set Environment Variables id: set-env run: | + echo "NO_DIST=1" >> "$GITHUB_ENV" # enable coverage report generation echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" echo "ONEDAL_PYTEST_FRAMEWORKS=${{ matrix.FRAMEWORKS }}" >> "$GITHUB_ENV" @@ -437,14 +437,14 @@ jobs: run: | source /opt/intel/oneapi/setvars.sh bash .ci/scripts/describe_system.sh - - name: Install requirements + - name: Install test requirements run: | pip install -r dependencies-dev - pip list echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" bash .ci/scripts/setup_sklearn.sh ${{ env.UXL_SKLEARNVERSION }} pip install --upgrade -r requirements-test.txt pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ env.UXL_SKLEARVERSION }}) pyyaml + pip list - name: Download sklearnex wheel uses: actions/download-artifact@v4 with: From fc794b649836bd5485ec237558eed6d6b1af0629 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 10:55:29 +0200 Subject: [PATCH 34/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0944ab3ed..88033d62c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: onedal_nightly: runs-on: ubuntu-24.04 name: Identify oneDAL nightly - timeout-minutes: 5 + timeout-minutes: 2 steps: - name: Get run ID of "Nightly-build" workflow From 0e4411ca6f2a366cfd0afe3f22f09bdea093ecb3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 12:03:34 +0200 Subject: [PATCH 35/74] Update ci.yml --- .github/workflows/ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88033d62c0..b15b215830 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,7 +333,9 @@ jobs: with: python-version: ${{ env.UXL_PYTHONVERSION }} cache: 'pip' - cache-dependency-path: dependencies-dev + cache-dependency-path: | + **/dependencies-dev + **/requirements-test.txt - name: Download oneDAL build artifact uses: actions/download-artifact@v4 with: @@ -396,7 +398,7 @@ jobs: needs: [onedal_nightly, build_uxl] name: LinuxNightly ${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} runs-on: ${{ matrix.OS }} - timeout-minutes: 30 + timeout-minutes: 60 steps: - name: Checkout Scikit-learn-intelex uses: actions/checkout@v4 @@ -404,6 +406,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.UXL_PYTHONVERSION }} + cache-dependency-path: | + **/dependencies-dev + **/requirements-test.txt - name: Download oneDAL build artifact uses: actions/download-artifact@v4 with: @@ -449,6 +454,9 @@ jobs: uses: actions/download-artifact@v4 with: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} + - name: Install PyTorch + if: matrix.DEVICE == 'GPU' + run: pip install torch --index-url https://download.pytorch.org/whl/xpu - name: Install daal4py/sklearnex run: pip install *.whl - name: Sklearnex testing From 30cfc150d80eabe9c0c08a798f3d4013a2a28170 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 12:10:20 +0200 Subject: [PATCH 36/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b15b215830..d3be831b1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -455,7 +455,7 @@ jobs: with: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} - name: Install PyTorch - if: matrix.DEVICE == 'GPU' + if: contains(matrix.FRAMEWORKS, 'pytorch') run: pip install torch --index-url https://download.pytorch.org/whl/xpu - name: Install daal4py/sklearnex run: pip install *.whl From f7f32c93a327611833e49ebac06c155dcf25dd36 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 13:15:21 +0200 Subject: [PATCH 37/74] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3be831b1b..afec911f22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,10 +391,10 @@ jobs: include: - OS: uxl-gpu-xlarge FRAMEWORKS: "numpy,pytorch" - DEVICE: GPU + DEVICE: gpu - OS: uxl-xlarge FRAMEWORKS: "numpy,pandas" - DEVICE: CPU + DEVICE: cpu needs: [onedal_nightly, build_uxl] name: LinuxNightly ${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} runs-on: ${{ matrix.OS }} From 39806fb32715628e851185905e9b13616c96e967 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 14:31:38 +0200 Subject: [PATCH 38/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afec911f22..e8fe2c057b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -398,7 +398,7 @@ jobs: needs: [onedal_nightly, build_uxl] name: LinuxNightly ${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} runs-on: ${{ matrix.OS }} - timeout-minutes: 60 + timeout-minutes: 120 steps: - name: Checkout Scikit-learn-intelex uses: actions/checkout@v4 From 46e3901aa5e488079826a02274cf7a855fd85a18 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 18:04:04 +0200 Subject: [PATCH 39/74] Update ci.yml --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8fe2c057b..b5e9197d7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ env: DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' UXL_PYTHONVERSION: "3.12" - UXL_SKLEARNVERSION: "1.6" + UXL_SKLEARNVERSION: "1.5" ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: @@ -359,6 +359,10 @@ jobs: echo "NO_DIST=1" >> "$GITHUB_ENV" # enable coverage report generation echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" + # reduce GPU driver/runner related memory issues + echo "NEOReadDebugKeys=1" >> "$GITHUB_ENV" + echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV" + echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format - name: dpcpp installation From 73df608a77ced471ad3476398d7d4cc9f7d4e8fa Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 20:09:20 +0200 Subject: [PATCH 40/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5e9197d7a..2c0711f3c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ env: DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' UXL_PYTHONVERSION: "3.12" - UXL_SKLEARNVERSION: "1.5" + UXL_SKLEARNVERSION: "1.4" ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: From d9a4a177965e48f587378a39c7ec03d95312b391 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 23:28:07 +0200 Subject: [PATCH 41/74] Update run_sklearn_tests.sh --- .ci/scripts/run_sklearn_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/run_sklearn_tests.sh b/.ci/scripts/run_sklearn_tests.sh index 2c9234627b..782dbb6492 100755 --- a/.ci/scripts/run_sklearn_tests.sh +++ b/.ci/scripts/run_sklearn_tests.sh @@ -29,8 +29,10 @@ export DESELECT_FLAGS="--public ${DESELECT_FLAGS}" if [ -n "${SKLEARNEX_PREVIEW}" ]; then export DESELECT_FLAGS="--preview ${DESELECT_FLAGS}" fi +if [ "$1" == "gpu" ]; then + export DESELECT_FLAGS="--gpu ${DESELECT_FLAGS}" +fi export DESELECTED_TESTS=$(python ../.circleci/deselect_tests.py ../deselected_tests.yaml ${DESELECT_FLAGS}) - # manual setting of OCL_ICD_FILENAMES is required in # specific MSYS environment with conda packages downloaded from intel channel if [[ "$(uname)" =~ "MSYS" ]] && [ -z "${OCL_ICD_FILENAMES}" ] && [ -n "${CONDA_PREFIX}" ]; then From 348d052929fe1172d0a60c9a548ec4979bd1ddb9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 23:52:24 +0200 Subject: [PATCH 42/74] Update ci.yml --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c0711f3c8..adeab302cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: SKLEARN_VERSION: "1.2" - PYTHON_VERSION: "3.11" SKLEARN_VERSION: "1.3" - name: LinuxNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }} + name: LinuxNightly/venv Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }} runs-on: ubuntu-24.04 timeout-minutes: 120 @@ -187,7 +187,7 @@ jobs: SKLEARN_VERSION: "1.2" - PYTHON_VERSION: "3.11" SKLEARN_VERSION: "1.3" - name: WindowsNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }} + name: WindowsNightly/venv Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }} runs-on: windows-2025 timeout-minutes: 120 @@ -321,7 +321,7 @@ jobs: build_uxl: if: github.repository == 'uxlfoundation/scikit-learn-intelex' needs: onedal_nightly - name: LinuxNightly GPU build + name: LinuxNightly build runs-on: uxl-xlarge timeout-minutes: 30 @@ -359,10 +359,6 @@ jobs: echo "NO_DIST=1" >> "$GITHUB_ENV" # enable coverage report generation echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" - # reduce GPU driver/runner related memory issues - echo "NEOReadDebugKeys=1" >> "$GITHUB_ENV" - echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV" - echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format - name: dpcpp installation @@ -436,6 +432,10 @@ jobs: # enable coverage report generation echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" echo "ONEDAL_PYTEST_FRAMEWORKS=${{ matrix.FRAMEWORKS }}" >> "$GITHUB_ENV" + # reduce GPU driver/runner related memory issues + echo "NEOReadDebugKeys=1" >> "$GITHUB_ENV" + echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV" + echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update - name: dpcpp installation From 252bb3ed7bab62b658780e9cbc9e563c480d8fa0 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 27 May 2025 23:55:32 +0200 Subject: [PATCH 43/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adeab302cc..80d10df9da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -321,7 +321,7 @@ jobs: build_uxl: if: github.repository == 'uxlfoundation/scikit-learn-intelex' needs: onedal_nightly - name: LinuxNightly build + name: LinuxNightly build Python${{ needs.onedal_nightly.outputs.uxl-python }} runs-on: uxl-xlarge timeout-minutes: 30 From 44eb9c2c8d3923fa00588c2e11153c2996ffd4fe Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 28 May 2025 02:14:39 +0200 Subject: [PATCH 44/74] Update validation.py --- daal4py/sklearn/utils/validation.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/daal4py/sklearn/utils/validation.py b/daal4py/sklearn/utils/validation.py index 164b046632..cfa9acf877 100644 --- a/daal4py/sklearn/utils/validation.py +++ b/daal4py/sklearn/utils/validation.py @@ -72,21 +72,20 @@ def _assert_all_finite( # Data with small size has too big relative overhead # TODO: tune threshold size - if hasattr(X, "size"): - if X.size < 32768: - if sklearn_check_version("1.1"): - _sklearn_assert_all_finite( - X, - allow_nan=allow_nan, - msg_dtype=msg_dtype, - estimator_name=estimator_name, - input_name=input_name, - ) - else: - _sklearn_assert_all_finite(X, allow_nan=allow_nan, msg_dtype=msg_dtype) - return - is_df = is_DataFrame(X) + if not (is_df or issubclass(np.ndarray)) or X.size < 32768: + if sklearn_check_version("1.1"): + _sklearn_assert_all_finite( + X, + allow_nan=allow_nan, + msg_dtype=msg_dtype, + estimator_name=estimator_name, + input_name=input_name, + ) + else: + _sklearn_assert_all_finite(X, allow_nan=allow_nan, msg_dtype=msg_dtype) + return + num_of_types = get_number_of_types(X) # if X is heterogeneous pandas.DataFrame then From aab30f1f07b4ca572f3a549991afd1d0f0a073b3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 28 May 2025 07:14:21 +0200 Subject: [PATCH 45/74] Update validation.py --- daal4py/sklearn/utils/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daal4py/sklearn/utils/validation.py b/daal4py/sklearn/utils/validation.py index cfa9acf877..e0a790f70b 100644 --- a/daal4py/sklearn/utils/validation.py +++ b/daal4py/sklearn/utils/validation.py @@ -73,7 +73,7 @@ def _assert_all_finite( # Data with small size has too big relative overhead # TODO: tune threshold size is_df = is_DataFrame(X) - if not (is_df or issubclass(np.ndarray)) or X.size < 32768: + if not (is_df or issubclass(X, np.ndarray)) or X.size < 32768: if sklearn_check_version("1.1"): _sklearn_assert_all_finite( X, From dee1b32cea51c279e1bf6fc58156f1049453c57e Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 28 May 2025 08:18:00 +0200 Subject: [PATCH 46/74] Update validation.py --- daal4py/sklearn/utils/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daal4py/sklearn/utils/validation.py b/daal4py/sklearn/utils/validation.py index e0a790f70b..b031b059c6 100644 --- a/daal4py/sklearn/utils/validation.py +++ b/daal4py/sklearn/utils/validation.py @@ -73,7 +73,7 @@ def _assert_all_finite( # Data with small size has too big relative overhead # TODO: tune threshold size is_df = is_DataFrame(X) - if not (is_df or issubclass(X, np.ndarray)) or X.size < 32768: + if not (is_df or isinstance(X, np.ndarray)) or X.size < 32768: if sklearn_check_version("1.1"): _sklearn_assert_all_finite( X, From 5dcdb1ad44eca63c5fed3fbca0e56aba76486c82 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 28 May 2025 08:25:51 +0200 Subject: [PATCH 47/74] Update validation.py --- sklearnex/utils/validation.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sklearnex/utils/validation.py b/sklearnex/utils/validation.py index 1396939ddb..22fabea9a8 100755 --- a/sklearnex/utils/validation.py +++ b/sklearnex/utils/validation.py @@ -74,7 +74,16 @@ def _sklearnex_assert_all_finite( # size check is an initial match to daal4py for performance reasons, can be # optimized later xp, _ = get_namespace(X) - if X.size < 32768 or not _onedal_supported_format(X, xp): + # this try-catch is a PyTorch-specific fix, as Tensor.size is a function. + # The try-catch minimizes changes to most common code path (numpy arrays). + # xp.size is NOT in the array api standard, but is a fix specifically in + # array_api_compat. + try: + too_small = X.size < 32768 + except TypeError: + too_small = xp.size(X) < 32768 + + if too_small or not _onedal_supported_format(X, xp): if sklearn_check_version("1.1"): _sklearn_assert_all_finite(X, allow_nan=allow_nan, input_name=input_name) else: From 5a995dc7f948e0bbe59379b25b23789ee456ff66 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:08:27 +0200 Subject: [PATCH 48/74] Update _device_offload.py --- onedal/_device_offload.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 66a35b2365..44e740e871 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -23,6 +23,7 @@ from sklearn import get_config from ._config import _get_config +from .datatypes import kDLCPU from .utils import _sycl_queue_manager as QM from .utils._array_api import _asarray, _is_numpy_namespace from .utils._dpep_helpers import dpctl_available, dpnp_available @@ -36,7 +37,7 @@ SyclQueue = getattr(_dpc_backend, "SyclQueue", None) logger = logging.getLogger("sklearnex") - +cpu_dlpack_device = (kDLCPU, 0) def supports_queue(func): """Decorator that updates the global queue before function evaluation. @@ -106,7 +107,7 @@ def _transfer_to_host(*data): host_data = [] for item in data: usm_iface = getattr(item, "__sycl_usm_array_interface__", None) - array_api = getattr(item, "__array_namespace__", lambda: None)() + device = getattr(item, "__dlpack_device__", None) if usm_iface is not None: if not dpctl_available: raise RuntimeError( @@ -126,10 +127,26 @@ def _transfer_to_host(*data): order=order, ) has_usm_data = True - elif array_api and not _is_numpy_namespace(array_api): - # `copy`` param for the `asarray`` is not setted. - # The object is copied only if needed. - item = np.asarray(item) + elif device and not isinstance(item, np.ndarray): + # check dlpack data location. + if device != cpu_dlpack_device: + if hasattr(item, "to_device"): + # not officially part of the array api standard but widely supported + item = item.to_device("cpu") + elif hasattr(item, "to"): + # pytorch-specific fix as it is not array api compliant + item = item.to("cpu") + else: + raise TypeError(f"cannot move {type(item)} to cpu") + + # convert to numpy + if hasattr(item, "__array__"): + # `copy`` param for the `asarray`` is not set. + # The object is copied only if needed + item = np.asarray(item) + else: + # requires numpy 1.23 + item = np.from_dlpack(item) has_host_data = True else: has_host_data = True From a1c6003f3e56b897c25d0bcdaa7ea7d4d869ff61 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:11:05 +0200 Subject: [PATCH 49/74] Update _data_conversion.py --- onedal/datatypes/_data_conversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onedal/datatypes/_data_conversion.py b/onedal/datatypes/_data_conversion.py index a779ef8e5a..e7f8448c46 100644 --- a/onedal/datatypes/_data_conversion.py +++ b/onedal/datatypes/_data_conversion.py @@ -18,6 +18,8 @@ from onedal import _default_backend as backend +kDLCPU = backend.kDLCPU +kDLOneAPI = backend.kDLOneAPI def _apply_and_pass(func, *args, **kwargs): if len(args) == 1: From 6798495f28b3c81d586da496b853adc238e0403a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:12:16 +0200 Subject: [PATCH 50/74] Update __init__.py --- onedal/datatypes/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onedal/datatypes/__init__.py b/onedal/datatypes/__init__.py index 5aa190cc20..07a88c15bc 100644 --- a/onedal/datatypes/__init__.py +++ b/onedal/datatypes/__init__.py @@ -14,6 +14,6 @@ # limitations under the License. # ============================================================================== -from ._data_conversion import from_table, to_table +from ._data_conversion import from_table, kDLCPU, kDLOneAPI, to_table -__all__ = ["from_table", "to_table"] +__all__ = ["from_table", "kDLCPU", "kDLOneAPI", "to_table"] From a6893e0e6ccdafd228f155a2f3b9eeda0674ad59 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:14:12 +0200 Subject: [PATCH 51/74] Update run_sklearn_tests.sh --- .ci/scripts/run_sklearn_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/run_sklearn_tests.sh b/.ci/scripts/run_sklearn_tests.sh index 782dbb6492..d1a9650c4f 100755 --- a/.ci/scripts/run_sklearn_tests.sh +++ b/.ci/scripts/run_sklearn_tests.sh @@ -32,6 +32,7 @@ fi if [ "$1" == "gpu" ]; then export DESELECT_FLAGS="--gpu ${DESELECT_FLAGS}" fi + export DESELECTED_TESTS=$(python ../.circleci/deselect_tests.py ../deselected_tests.yaml ${DESELECT_FLAGS}) # manual setting of OCL_ICD_FILENAMES is required in # specific MSYS environment with conda packages downloaded from intel channel From 3acb212bff5909ca3a7309a1ab51a8dadeb3c3a9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:25:26 +0200 Subject: [PATCH 52/74] Update _device_offload.py --- onedal/_device_offload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 44e740e871..812bcdcfb6 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -39,6 +39,7 @@ logger = logging.getLogger("sklearnex") cpu_dlpack_device = (kDLCPU, 0) + def supports_queue(func): """Decorator that updates the global queue before function evaluation. @@ -131,7 +132,8 @@ def _transfer_to_host(*data): # check dlpack data location. if device != cpu_dlpack_device: if hasattr(item, "to_device"): - # not officially part of the array api standard but widely supported + # use of the "cpu" string as device not officially part of + # the array api standard but widely supported item = item.to_device("cpu") elif hasattr(item, "to"): # pytorch-specific fix as it is not array api compliant From fc3622958a11da079b43724c8e63818c78880d32 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:26:12 +0200 Subject: [PATCH 53/74] Update _data_conversion.py --- onedal/datatypes/_data_conversion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onedal/datatypes/_data_conversion.py b/onedal/datatypes/_data_conversion.py index e7f8448c46..cf832b2b3d 100644 --- a/onedal/datatypes/_data_conversion.py +++ b/onedal/datatypes/_data_conversion.py @@ -21,6 +21,7 @@ kDLCPU = backend.kDLCPU kDLOneAPI = backend.kDLOneAPI + def _apply_and_pass(func, *args, **kwargs): if len(args) == 1: return func(args[0], **kwargs) From 8dbf0e1992cdf89aaec2939f071730e70d60f81f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 06:29:40 +0200 Subject: [PATCH 54/74] Update validation.py --- sklearnex/utils/validation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sklearnex/utils/validation.py b/sklearnex/utils/validation.py index 22fabea9a8..0ebff6c815 100755 --- a/sklearnex/utils/validation.py +++ b/sklearnex/utils/validation.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== +import math import numbers import scipy.sparse as sp @@ -76,13 +77,11 @@ def _sklearnex_assert_all_finite( xp, _ = get_namespace(X) # this try-catch is a PyTorch-specific fix, as Tensor.size is a function. # The try-catch minimizes changes to most common code path (numpy arrays). - # xp.size is NOT in the array api standard, but is a fix specifically in - # array_api_compat. try: too_small = X.size < 32768 except TypeError: - too_small = xp.size(X) < 32768 - + too_small = math.prod(X.shape) < 32768 + if too_small or not _onedal_supported_format(X, xp): if sklearn_check_version("1.1"): _sklearn_assert_all_finite(X, allow_nan=allow_nan, input_name=input_name) From 5f2c7ab67ba112b8568bac61ec39c35848b4acd9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 07:09:27 +0200 Subject: [PATCH 55/74] Update table.cpp --- onedal/datatypes/table.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onedal/datatypes/table.cpp b/onedal/datatypes/table.cpp index 12f51dca81..d76171be7b 100644 --- a/onedal/datatypes/table.cpp +++ b/onedal/datatypes/table.cpp @@ -110,7 +110,8 @@ ONEDAL_PY_INIT_MODULE(table) { m.def("dlpack_memory_order", &dlpack::dlpack_memory_order); py::enum_(m, "DLDeviceType") .value("kDLCPU", kDLCPU) - .value("kDLOneAPI", kDLOneAPI); + .value("kDLOneAPI", kDLOneAPI) + .export_values(); } } // namespace oneapi::dal::python From 404cfa6aad53d734928e9a8f7738478f177f5e9b Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 08:04:10 +0200 Subject: [PATCH 56/74] Update _device_offload.py --- onedal/_device_offload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 812bcdcfb6..8566f48e41 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -131,6 +131,7 @@ def _transfer_to_host(*data): elif device and not isinstance(item, np.ndarray): # check dlpack data location. if device != cpu_dlpack_device: + raise TypeError(f"{device}, cpu:{cpu_dlpack_device}") if hasattr(item, "to_device"): # use of the "cpu" string as device not officially part of # the array api standard but widely supported From 9b3788d13c8a09bd142550f38034f42760955516 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 08:47:24 +0200 Subject: [PATCH 57/74] Update _device_offload.py --- onedal/_device_offload.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 8566f48e41..832acdbc48 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -108,7 +108,6 @@ def _transfer_to_host(*data): host_data = [] for item in data: usm_iface = getattr(item, "__sycl_usm_array_interface__", None) - device = getattr(item, "__dlpack_device__", None) if usm_iface is not None: if not dpctl_available: raise RuntimeError( @@ -128,10 +127,10 @@ def _transfer_to_host(*data): order=order, ) has_usm_data = True - elif device and not isinstance(item, np.ndarray): + elif not isinstance(item, np.ndarray) and (device := getattr(item, "__dlpack_device__", None)): # check dlpack data location. - if device != cpu_dlpack_device: - raise TypeError(f"{device}, cpu:{cpu_dlpack_device}") + if device() != cpu_dlpack_device: + print(f"{device}, cpu:{cpu_dlpack_device}") if hasattr(item, "to_device"): # use of the "cpu" string as device not officially part of # the array api standard but widely supported From 5348c05b43b5d9d741242f717f46b30b72439ae4 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 12:32:23 +0200 Subject: [PATCH 58/74] Update deselected_tests.yaml --- deselected_tests.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/deselected_tests.yaml b/deselected_tests.yaml index 31de6c7e33..596a410a0a 100755 --- a/deselected_tests.yaml +++ b/deselected_tests.yaml @@ -40,13 +40,17 @@ deselected_tests: - neighbors/tests/test_neighbors.py::test_neighbor_classifiers_loocv[auto-nn_model0] # Array API support - # sklearnex functional Array API support doesn't guaranty namespace consistency for the estimator's array attributes. + # sklearnex functional Array API support doesn't guarantee namespace consistency for the estimator's array attributes. - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='covariance_eigh')-check_array_api_input_and_values-array_api_strict-None-None] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='covariance_eigh',whiten=True)-check_array_api_input_and_values-array_api_strict-None-None] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='covariance_eigh')-check_array_api_get_precision-array_api_strict-None-None] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='covariance_eigh',whiten=True)-check_array_api_get_precision-array_api_strict-None-None] - linear_model/tests/test_ridge.py::test_ridge_array_api_compliance[Ridge(solver='svd')-check_array_api_attributes-array_api_strict-None-None] - linear_model/tests/test_ridge.py::test_ridge_array_api_compliance[Ridge(solver='svd')-check_array_api_input_and_values-array_api_strict-None-None] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_input_and_values-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_input_and_values-torch-cpu-float32] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_get_precision-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_get_precision-torch-cpu-float32] # `train_test_split` inconsistency for Array API inputs. - model_selection/tests/test_split.py::test_array_api_train_test_split[True-None-array_api_strict-None-None] - model_selection/tests/test_split.py::test_array_api_train_test_split[True-stratify1-array_api_strict-None-None] @@ -56,6 +60,10 @@ deselected_tests: # PCA. InvalidParameterError: The 'M' parameter of randomized_svd must be an instance of 'numpy.ndarray' or a sparse matrix. - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_input_and_values-array_api_strict-None-None] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_get_precision-array_api_strict-None-None] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_input_and_values-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_input_and_values-torch-cpu-float32] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_get_precision-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,power_iteration_normalizer='QR',random_state=0,svd_solver='randomized')-check_array_api_get_precision-torch-cpu-float32] # Ridge regression. Array API functionally supported for all solvers. Not raising error for non-svd solvers. - linear_model/tests/test_ridge.py::test_array_api_error_and_warnings_for_solver_parameter[array_api_strict] From 6a12107c833e24aebe0070d10e8cd47d8615386d Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 13:09:52 +0200 Subject: [PATCH 59/74] Update deselected_tests.yaml --- deselected_tests.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deselected_tests.yaml b/deselected_tests.yaml index 596a410a0a..26b30c3846 100755 --- a/deselected_tests.yaml +++ b/deselected_tests.yaml @@ -51,10 +51,21 @@ deselected_tests: - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_input_and_values-torch-cpu-float32] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_get_precision-torch-cpu-float64] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=2,svd_solver='full')-check_array_api_get_precision-torch-cpu-float32] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_input_and_values-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_input_and_values-torch-cpu-float32] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_get_precision-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_get_precision-torch-cpu-float32] # `train_test_split` inconsistency for Array API inputs. - model_selection/tests/test_split.py::test_array_api_train_test_split[True-None-array_api_strict-None-None] - model_selection/tests/test_split.py::test_array_api_train_test_split[True-stratify1-array_api_strict-None-None] - model_selection/tests/test_split.py::test_array_api_train_test_split[False-None-array_api_strict-None-None] + - model_selection/tests/test_split.py::test_array_api_train_test_split[True-None-torch-cpu-float64] + - model_selection/tests/test_split.py::test_array_api_train_test_split[True-None-torch-cpu-float32] + - model_selection/tests/test_split.py::test_array_api_train_test_split[True-stratify1-torch-cpu-float64] + - model_selection/tests/test_split.py::test_array_api_train_test_split[True-stratify1-torch-cpu-float32] + - model_selection/tests/test_split.py::test_array_api_train_test_split[False-None-torch-cpu-float64] + - model_selection/tests/test_split.py::test_array_api_train_test_split[False-None-torch-cpu-float32] + # PCA. Array API functionally supported for all factorizations. power_iteration_normalizer=["LU", "QR"] - decomposition/tests/test_pca.py::test_array_api_error_and_warnings_on_unsupported_params # PCA. InvalidParameterError: The 'M' parameter of randomized_svd must be an instance of 'numpy.ndarray' or a sparse matrix. From e291673f9302911ef353b1aa862c81629074d565 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 13:11:27 +0200 Subject: [PATCH 60/74] Update _device_offload.py --- onedal/_device_offload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 832acdbc48..1102fe44cb 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -127,10 +127,11 @@ def _transfer_to_host(*data): order=order, ) has_usm_data = True - elif not isinstance(item, np.ndarray) and (device := getattr(item, "__dlpack_device__", None)): + elif not isinstance(item, np.ndarray) and ( + device := getattr(item, "__dlpack_device__", None) + ): # check dlpack data location. if device() != cpu_dlpack_device: - print(f"{device}, cpu:{cpu_dlpack_device}") if hasattr(item, "to_device"): # use of the "cpu" string as device not officially part of # the array api standard but widely supported From 0003d0383a4b6ecff866f4b6f1250d919fa80e46 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 15:15:53 +0200 Subject: [PATCH 61/74] Update deselected_tests.yaml --- deselected_tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deselected_tests.yaml b/deselected_tests.yaml index 26b30c3846..ea72a0fbab 100755 --- a/deselected_tests.yaml +++ b/deselected_tests.yaml @@ -55,6 +55,8 @@ deselected_tests: - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_input_and_values-torch-cpu-float32] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_get_precision-torch-cpu-float64] - decomposition/tests/test_pca.py::test_pca_array_api_compliance[PCA(n_components=0.1,svd_solver='full',whiten=True)-check_array_api_get_precision-torch-cpu-float32] + - decomposition/tests/test_pca.py::test_pca_mle_array_api_compliance[PCA(n_components='mle',svd_solver='full')-check_array_api_get_precision-torch-cpu-float64] + - decomposition/tests/test_pca.py::test_pca_mle_array_api_compliance[PCA(n_components='mle',svd_solver='full')-check_array_api_get_precision-torch-cpu-float32] # `train_test_split` inconsistency for Array API inputs. - model_selection/tests/test_split.py::test_array_api_train_test_split[True-None-array_api_strict-None-None] - model_selection/tests/test_split.py::test_array_api_train_test_split[True-stratify1-array_api_strict-None-None] From 37102429c7f9e812dc24f86411f84f4b352c56e3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 19:41:00 +0200 Subject: [PATCH 62/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d10df9da..a6956a8a54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ env: DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' UXL_PYTHONVERSION: "3.12" - UXL_SKLEARNVERSION: "1.4" + UXL_SKLEARNVERSION: "1.6" ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: From 3946b6522453f2ccea3898f2534a45a42290c7b7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 29 May 2025 23:55:32 +0200 Subject: [PATCH 63/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6956a8a54..80d10df9da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ env: DPNP_VERSION: 0.16.0 DPCTL_PY_VERSIONS: '3.9\|3.11' UXL_PYTHONVERSION: "3.12" - UXL_SKLEARNVERSION: "1.6" + UXL_SKLEARNVERSION: "1.4" ONEDAL_REPO: "uxlfoundation/oneDAL" jobs: From 356804ad0863ead4e3fbf6e7b73ae9cee249863a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 1 Jun 2025 21:06:43 +0200 Subject: [PATCH 64/74] Update deselected_tests.yaml --- deselected_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deselected_tests.yaml b/deselected_tests.yaml index 3264e14c70..a124349018 100755 --- a/deselected_tests.yaml +++ b/deselected_tests.yaml @@ -469,3 +469,6 @@ gpu: # Introduced with RNG forest updates in oneDAL - ensemble/tests/test_voting.py::test_set_estimator_drop + + # Deselection for public GPU runner + - model_selection/tests/test_validation.py::test_learning_curve_some_failing_fits_warning[42] From 883ce450eb5b6ed0e7831634e1d8e65cfefd41fc Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 9 Jun 2025 16:07:09 +0200 Subject: [PATCH 65/74] Update ci.yml --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d10df9da..0f5a440b46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -390,13 +390,13 @@ jobs: matrix: include: - OS: uxl-gpu-xlarge - FRAMEWORKS: "numpy,pytorch" + FRAMEWORKS: "pytorch,numpy" DEVICE: gpu - OS: uxl-xlarge - FRAMEWORKS: "numpy,pandas" + FRAMEWORKS: "pytorch,numpy" DEVICE: cpu needs: [onedal_nightly, build_uxl] - name: LinuxNightly ${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} + name: LinuxNightly [${{ matrix.FRAMEWORKS }}]-${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }} runs-on: ${{ matrix.OS }} timeout-minutes: 120 steps: @@ -460,7 +460,9 @@ jobs: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} - name: Install PyTorch if: contains(matrix.FRAMEWORKS, 'pytorch') - run: pip install torch --index-url https://download.pytorch.org/whl/xpu + run: | + pip install torch --index-url https://download.pytorch.org/whl/xpu + python -c "import torch; _=[print(torch.xpu.get_device_name(i)) for i in range(torch.xpu.device_count())]" - name: Install daal4py/sklearnex run: pip install *.whl - name: Sklearnex testing From 6ac3d949f6c0d5053b5fe6699b2f2d6f38f4076e Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 12 Jun 2025 05:15:03 +0200 Subject: [PATCH 66/74] Update deselected_tests.yaml --- deselected_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deselected_tests.yaml b/deselected_tests.yaml index a124349018..7b4e9ddb62 100755 --- a/deselected_tests.yaml +++ b/deselected_tests.yaml @@ -470,5 +470,5 @@ gpu: # Introduced with RNG forest updates in oneDAL - ensemble/tests/test_voting.py::test_set_estimator_drop - # Deselection for public GPU runner - - model_selection/tests/test_validation.py::test_learning_curve_some_failing_fits_warning[42] + # Deselection for Scikit-Learn 1.4 GPU conformance + - model_selection/tests/test_validation.py::test_learning_curve_some_failing_fits_warning >=1.4 From d212cb9004766997feed820ea7918904ebb8d6d3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 10:43:11 +0200 Subject: [PATCH 67/74] Update validation.py --- sklearnex/utils/validation.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sklearnex/utils/validation.py b/sklearnex/utils/validation.py index 4afe2811ef..9c5f7b2a2f 100755 --- a/sklearnex/utils/validation.py +++ b/sklearnex/utils/validation.py @@ -18,6 +18,7 @@ import numbers import scipy.sparse as sp +from array_api_compat import is_torch_array from sklearn.utils.validation import _assert_all_finite as _sklearn_assert_all_finite from sklearn.utils.validation import _num_samples, check_array, check_non_negative @@ -71,12 +72,8 @@ def _sklearnex_assert_all_finite( # size check is an initial match to daal4py for performance reasons, can be # optimized later xp, _ = get_namespace(X) - # this try-catch is a PyTorch-specific fix, as Tensor.size is a function. - # The try-catch minimizes changes to most common code path (numpy arrays). - try: - too_small = X.size < 32768 - except TypeError: - too_small = math.prod(X.shape) < 32768 + # this is a PyTorch-specific fix, as Tensor.size is a function. + too_small = (math.prod(X.shape) if is_torch_array(X) else X.size) < 32768 if too_small or not _onedal_supported_format(X, xp): if sklearn_check_version("1.1"): From 8becae65daf9905b43acb497b92e854a370d3911 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 10:52:28 +0200 Subject: [PATCH 68/74] Update ci.yml --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f5a440b46..7f736ef4ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -370,8 +370,10 @@ jobs: source /opt/intel/oneapi/setvars.sh bash .ci/scripts/describe_system.sh - name: Install develop requirements + id: install-reqs run: | pip install -r dependencies-dev + echo "numpy-version=$(python -m pip freeze | grep numpy)" >> "$GITHUB_OUTPUT" pip list - name: Build daal4py/sklearnex run: | @@ -383,6 +385,9 @@ jobs: name: sklearnex_build_${{ env.UXL_PYTHONVERSION }} path: | ./dist/*.whl + + outputs: + numpy-version: ${{ steps.install-reqs.outputs.numpy-version }} test_uxl: strategy: @@ -436,6 +441,8 @@ jobs: echo "NEOReadDebugKeys=1" >> "$GITHUB_ENV" echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV" echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV" + # set build numpy version for use in generating code coverage + echo "NUMPY_BUILD=${{ needs.build_uxl.outputs.numpy-version }} >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update - name: dpcpp installation @@ -448,8 +455,6 @@ jobs: bash .ci/scripts/describe_system.sh - name: Install test requirements run: | - pip install -r dependencies-dev - echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" bash .ci/scripts/setup_sklearn.sh ${{ env.UXL_SKLEARNVERSION }} pip install --upgrade -r requirements-test.txt pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ env.UXL_SKLEARVERSION }}) pyyaml From acb645903b395744288a36f1018ac1ac4f03d46b Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 10:59:16 +0200 Subject: [PATCH 69/74] Update __init__.py --- onedal/datatypes/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onedal/datatypes/__init__.py b/onedal/datatypes/__init__.py index 07a88c15bc..5aa190cc20 100644 --- a/onedal/datatypes/__init__.py +++ b/onedal/datatypes/__init__.py @@ -14,6 +14,6 @@ # limitations under the License. # ============================================================================== -from ._data_conversion import from_table, kDLCPU, kDLOneAPI, to_table +from ._data_conversion import from_table, to_table -__all__ = ["from_table", "kDLCPU", "kDLOneAPI", "to_table"] +__all__ = ["from_table", "to_table"] From ea1372b3764f90aa0c692bc334bdb2f9269ef56d Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 10:59:39 +0200 Subject: [PATCH 70/74] Update _data_conversion.py --- onedal/datatypes/_data_conversion.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/onedal/datatypes/_data_conversion.py b/onedal/datatypes/_data_conversion.py index cf832b2b3d..a779ef8e5a 100644 --- a/onedal/datatypes/_data_conversion.py +++ b/onedal/datatypes/_data_conversion.py @@ -18,9 +18,6 @@ from onedal import _default_backend as backend -kDLCPU = backend.kDLCPU -kDLOneAPI = backend.kDLOneAPI - def _apply_and_pass(func, *args, **kwargs): if len(args) == 1: From 97ce7a11cbddae486792286c75462bfd22c2613c Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 11:00:54 +0200 Subject: [PATCH 71/74] Update _device_offload.py --- onedal/_device_offload.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index 1102fe44cb..accc7eb284 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -20,10 +20,11 @@ from functools import wraps import numpy as np +from onedal import _default_backend as backend from sklearn import get_config + from ._config import _get_config -from .datatypes import kDLCPU from .utils import _sycl_queue_manager as QM from .utils._array_api import _asarray, _is_numpy_namespace from .utils._dpep_helpers import dpctl_available, dpnp_available @@ -36,8 +37,9 @@ SyclQueue = getattr(_dpc_backend, "SyclQueue", None) + logger = logging.getLogger("sklearnex") -cpu_dlpack_device = (kDLCPU, 0) +cpu_dlpack_device = (backend.kDLCPU, 0) def supports_queue(func): From e7c667c72023d5f46d5879c027c9d0c15a365699 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 11:19:05 +0200 Subject: [PATCH 72/74] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f736ef4ba..3120e232dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -442,7 +442,7 @@ jobs: echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV" echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV" # set build numpy version for use in generating code coverage - echo "NUMPY_BUILD=${{ needs.build_uxl.outputs.numpy-version }} >> "$GITHUB_ENV" + echo "NUMPY_BUILD=${{ needs.build_uxl.outputs.numpy-version }}" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update - name: dpcpp installation From ae0a0e60cf4aad671578e54d36d128cc462fb8f7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 13 Jun 2025 12:26:17 +0200 Subject: [PATCH 73/74] Update _device_offload.py --- onedal/_device_offload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onedal/_device_offload.py b/onedal/_device_offload.py index accc7eb284..18b165bdf9 100644 --- a/onedal/_device_offload.py +++ b/onedal/_device_offload.py @@ -20,9 +20,9 @@ from functools import wraps import numpy as np -from onedal import _default_backend as backend from sklearn import get_config +from onedal import _default_backend as backend from ._config import _get_config from .utils import _sycl_queue_manager as QM From a293515aed33aa1ef1ab9bd4c04a7c6fe37eaded Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 18 Jun 2025 09:51:01 +0200 Subject: [PATCH 74/74] Update validation.py --- sklearnex/utils/validation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sklearnex/utils/validation.py b/sklearnex/utils/validation.py index 9c5f7b2a2f..95cb0b0e73 100755 --- a/sklearnex/utils/validation.py +++ b/sklearnex/utils/validation.py @@ -18,7 +18,6 @@ import numbers import scipy.sparse as sp -from array_api_compat import is_torch_array from sklearn.utils.validation import _assert_all_finite as _sklearn_assert_all_finite from sklearn.utils.validation import _num_samples, check_array, check_non_negative @@ -72,8 +71,8 @@ def _sklearnex_assert_all_finite( # size check is an initial match to daal4py for performance reasons, can be # optimized later xp, _ = get_namespace(X) - # this is a PyTorch-specific fix, as Tensor.size is a function. - too_small = (math.prod(X.shape) if is_torch_array(X) else X.size) < 32768 + # this is a PyTorch-specific fix, as Tensor.size is a function. It replicates `.size` + too_small = math.prod(X.shape) < 32768 if too_small or not _onedal_supported_format(X, xp): if sklearn_check_version("1.1"):