Skip to content

Commit c36bf45

Browse files
BordaBorda
authored
Merge branch 'master' into feature/19743-tensorboard-histograms
2 parents 48892cb + 6989e15 commit c36bf45

File tree

103 files changed

+3233
-463
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+3233
-463
lines changed

‎.actions/assistant.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,25 @@ def generate_docker_tags(
459459
tags = [f"{docker_project}:{tag}" for tag in tags]
460460
print(",".join(tags))
461461

462+
@staticmethod
463+
def prune_pytest_as_errors(
464+
pyproject_toml: str = "pyproject.toml", errors: tuple = ("FutureWarning", "DeprecationWarning")
465+
) -> None:
466+
"""Prune pytest warnings as errors from the pyproject.toml file."""
467+
import tomlkit
468+
469+
with open(pyproject_toml, encoding="utf-8") as fopen:
470+
content = fopen.read()
471+
pyproject = tomlkit.parse(content)
472+
filterwarnings = pyproject.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("filterwarnings", [])
473+
if not filterwarnings:
474+
return
475+
filterwarnings = [wrn for wrn in filterwarnings if not any(f"error::{err}" in wrn for err in errors)]
476+
pyproject["tool"]["pytest"]["ini_options"]["filterwarnings"] = filterwarnings
477+
478+
with open(pyproject_toml, "w", encoding="utf-8") as fopen:
479+
fopen.write(tomlkit.dumps(pyproject))
480+
462481

463482
if __name__ == "__main__":
464483
import jsonargparse

‎.github/workflows/_build-packages.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
pkg-name: ${{ fromJSON(inputs.pkg-names) }}
2727
steps:
2828
- uses: actions/checkout@v5
29-
- uses: actions/setup-python@v5
29+
- uses: actions/setup-python@v6
3030
with:
3131
python-version: "3.x"
3232

‎.github/workflows/_legacy-checkpoints.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,28 +57,32 @@ jobs:
5757
steps:
5858
- uses: actions/checkout@v5
5959

60-
- uses: actions/setup-python@v5
60+
- name: Install uv and set Python version
61+
uses: astral-sh/setup-uv@v6
6162
with:
62-
# Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt.
6363
python-version: "3.9"
64+
# TODO: Avoid activating environment like this
65+
# see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment
66+
activate-environment: true
67+
enable-cache: true
6468

6569
- name: Install PL from source
6670
env:
6771
PACKAGE_NAME: pytorch
6872
FREEZE_REQUIREMENTS: 1
6973
timeout-minutes: 20
70-
run: pip install . --extra-index-url="${TORCH_URL}"
74+
run: uv pip install . --extra-index-url="${TORCH_URL}"
7175
if: inputs.pl_version == ''
7276

7377
- name: Install PL version
7478
timeout-minutes: 20
75-
run: pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}"
79+
run: uv pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}"
7680
if: inputs.pl_version != ''
7781

7882
- name: Adjust tests -> PL
7983
if: ${{ matrix.pkg-name != 'lightning' }}
8084
run: |
81-
pip install -q -r .actions/requirements.txt
85+
uv pip install -q -r .actions/requirements.txt
8286
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
8387
--source_import="lightning.fabric,lightning.pytorch" \
8488
--target_import="lightning_fabric,pytorch_lightning"
@@ -115,7 +119,7 @@ jobs:
115119
# export to env bool if secrets.AWS_REGION is not empty
116120
run: echo "WITH_SECRETS=$([ -n '${{ secrets.AWS_REGION }}' ] && echo 1 || echo 0)" >> $GITHUB_ENV
117121

118-
- run: pip install -r requirements/ci.txt
122+
- run: uv pip install -r requirements/ci.txt
119123
- name: Upload checkpoints to S3
120124
if: ${{ env.WITH_SECRETS == '1' }}
121125
working-directory: ${{ env.LEGACY_FOLDER }}

‎.github/workflows/ci-pkg-install.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
python-version: ["3.9", "3.11"]
4848
steps:
4949
- uses: actions/checkout@v5
50-
- uses: actions/setup-python@v5
50+
- uses: actions/setup-python@v6
5151
with:
5252
python-version: ${{ matrix.python-version }}
5353
- uses: actions/download-artifact@v5

‎.github/workflows/ci-tests-fabric.yml‎

Lines changed: 66 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -38,113 +38,102 @@ jobs:
3838
strategy:
3939
fail-fast: false
4040
matrix:
41-
include:
41+
os: [macOS-14, ubuntu-22.04, windows-2022]
42+
config:
4243
# only run PyTorch latest
43-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
44-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
45-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
46-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
47-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
48-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
49-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
50-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
51-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
52-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
53-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
54-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
55-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
56-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
57-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
44+
- { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
45+
- { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
46+
- { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
47+
- { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
48+
- { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
49+
5850
# only run PyTorch latest with Python latest, use Fabric scope to limit dependency issues
59-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
60-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
61-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
62-
# "oldest" versions tests, only on minimum Python
63-
- { os: "macOS-14", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
64-
- { os: "ubuntu-22.04", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
65-
- { os: "windows-2022", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
51+
- { pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
52+
6653
# "fabric" installs the standalone package
67-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
68-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
69-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
54+
- { pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
55+
7056
# adding recently cut Torch 2.7 - FUTURE
71-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
72-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
73-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
57+
- { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
58+
59+
# "oldest" versions tests, only on minimum Python
60+
- { pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
7461
timeout-minutes: 25 # because of building grpcio on Mac
7562
env:
76-
PACKAGE_NAME: ${{ matrix.pkg-name }}
63+
PACKAGE_NAME: ${{ matrix.config.pkg-name }}
7764
FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
78-
PYPI_CACHE_DIR: "_pip-wheels"
7965
TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/"
8066
TORCH_URL_TEST: "https://download.pytorch.org/whl/test/cpu/"
8167
# TODO: Remove this - Enable running MPS tests on this platform
8268
DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }}
69+
UV_TORCH_BACKEND: "cpu"
8370
steps:
8471
- uses: actions/checkout@v5
8572

86-
- name: Set up Python ${{ matrix.python-version }}
87-
uses: actions/setup-python@v5
73+
- name: Install uv and set Python version
74+
uses: astral-sh/setup-uv@v6
8875
with:
89-
python-version: ${{ matrix.python-version || '3.9' }}
76+
python-version: ${{ matrix.config.python-version || '3.9' }}
77+
# TODO: Avoid activating environment like this
78+
# see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment
79+
activate-environment: true
80+
enable-cache: true
81+
82+
- name: Basic setup
83+
run: uv pip install -q -r .actions/requirements.txt
9084

91-
- name: basic setup
92-
run: pip install -q -r .actions/requirements.txt
85+
- name: Append Env. vars for Linux
86+
if: ${{ runner.os == 'Linux' }}
87+
run: echo "GLOO_SOCKET_IFNAME=eth0" >> $GITHUB_ENV
88+
89+
- name: Append Env. vars for MacOS
90+
if: ${{ runner.os == 'macOS' }}
91+
run: echo "GLOO_SOCKET_IFNAME=lo0" >> $GITHUB_ENV
92+
93+
- name: Append Env. vars for Windows
94+
if: ${{ runner.os == 'windows' }}
95+
run: |
96+
# Avoid issue on Windows with PyTorch 2.4: "RuntimeError: use_libuv was requested but PyTorch was build without libuv support"
97+
echo "USE_LIBUV=0" >> $GITHUB_ENV
9398
9499
- name: Set min. dependencies
95-
if: ${{ matrix.requires == 'oldest' }}
100+
if: ${{ matrix.config.requires == 'oldest' }}
96101
run: |
97-
cd requirements/fabric
98-
pip install -U "lightning-utilities[cli]"
99-
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt', 'test.txt']"
100-
pip install "cython<3.0" wheel
101-
pip install "pyyaml==5.4" --no-build-isolation
102+
uv pip install -U "lightning-utilities[cli]"
103+
python -m lightning_utilities.cli requirements set-oldest \
104+
--req_files "['requirements/fabric/base.txt', 'requirements/fabric/strategies.txt', 'requirements/fabric/test.txt']"
105+
uv pip install "cython<3.0" wheel
106+
uv pip install "pyyaml==5.4" --no-build-isolation
107+
# This script removes any line containing "error::FutureWarning" from pyproject.toml
108+
uv pip install -r requirements/ci.txt
109+
python .actions/assistant.py prune_pytest_as_errors
102110
103111
- name: Adjust PyTorch versions in requirements files
104-
if: ${{ matrix.requires != 'oldest' }}
112+
if: ${{ matrix.config.requires != 'oldest' }}
105113
run: |
106-
pip install -q -r requirements/ci.txt
114+
uv pip install -q -r requirements/ci.txt
107115
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
108116
for fpath in `ls requirements/**/*.txt`; do \
109-
python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \
117+
python ./adjust-torch-versions.py $fpath ${{ matrix.config.pytorch-version }}; \
110118
done
111119
112-
- name: pip wheels cache
113-
uses: actions/cache/restore@v4
114-
with:
115-
path: ${{ env.PYPI_CACHE_DIR }}
116-
key: pypi_wheels
117-
- run: |
118-
mkdir -p $PYPI_CACHE_DIR
119-
ls -lh $PYPI_CACHE_DIR
120-
121120
- name: Expand Env. variables
122121
run: |
123122
# Switch PyTorch URL between stable and test/future
124-
python -c "print('TORCH_URL=' + str('${{env.TORCH_URL_TEST}}' if '${{ matrix.pytorch-version }}' == '2.7' else '${{env.TORCH_URL_STABLE}}'))" >> $GITHUB_ENV
123+
python -c "print('TORCH_URL=' + str('${{env.TORCH_URL_TEST}}' if '${{ matrix.config.pytorch-version }}' == '2.7' else '${{env.TORCH_URL_STABLE}}'))" >> $GITHUB_ENV
125124
# Switch coverage scope
126-
python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_fabric'))" >> $GITHUB_ENV
125+
python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.config.pkg-name}}' == 'lightning' else 'lightning_fabric'))" >> $GITHUB_ENV
127126
# if you install mono-package set dependency only for this subpackage
128-
python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.pkg-name}}' != 'lightning' else 'fabric-'))" >> $GITHUB_ENV
129-
- name: Append Env. vars for MacOS
130-
if: ${{ runner.os == 'macOS' }}
131-
run: |
132-
# trying to avoid "gloo" issue with SIGABRT
133-
echo "GLOO_SOCKET_IFNAME=lo0" >> $GITHUB_ENV
134-
- name: Append Env. vars for Windows
135-
if: ${{ runner.os == 'windows' }}
136-
run: |
137-
# Avoid issue on Windows with PyTorch 2.4: "RuntimeError: use_libuv was requested but PyTorch was build without libuv support"
138-
echo "USE_LIBUV=0" >> $GITHUB_ENV
127+
python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.config.pkg-name}}' != 'lightning' else 'fabric-'))" >> $GITHUB_ENV
139128
140129
- name: Install package & dependencies
141130
timeout-minutes: 20
142131
run: |
143-
pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
144-
-U --upgrade-strategy=eager --prefer-binary \
145-
--extra-index-url="${TORCH_URL}" \
146-
--find-links="${PYPI_CACHE_DIR}"
147-
pip list
132+
uv pip install ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
133+
--upgrade \
134+
--find-links="${TORCH_URL}"
135+
uv pip list
136+
148137
- name: Dump handy wheels
149138
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
150139
continue-on-error: true
@@ -155,7 +144,7 @@ jobs:
155144
cache-key: "pypi_wheels"
156145

157146
- name: Adjust tests
158-
if: ${{ matrix.pkg-name != 'lightning' }}
147+
if: ${{ matrix.config.pkg-name != 'lightning' }}
159148
run: |
160149
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
161150
--source_import="lightning.fabric" --target_import="lightning_fabric"
@@ -188,10 +177,13 @@ jobs:
188177
with:
189178
token: ${{ secrets.CODECOV_TOKEN }}
190179
file: tests/tests_fabric/coverage.xml
191-
flags: ${{ env.COVERAGE_SCOPE }},cpu,pytest,python${{ matrix.python-version }}
180+
flags: ${{ env.COVERAGE_SCOPE }},cpu,pytest,python${{ matrix.config.python-version }}
192181
name: CPU-coverage
193182
fail_ci_if_error: false
194183

184+
- name: Minimize uv cache
185+
run: uv cache prune --ci
186+
195187
fabric-cpu-guardian:
196188
runs-on: ubuntu-latest
197189
needs: fabric-cpu

0 commit comments

Comments
 (0)