Skip to content

Commit ded8bd2

Browse files
authored
Merge branch 'master' into master
2 parents 2f5cf7b + 6989e15 commit ded8bd2

File tree

172 files changed

+5174
-1046
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+5174
-1046
lines changed

.actions/assistant.py

Lines changed: 19 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fopen:
374-
req = fopen.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fwrite:
376-
fwrite.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,
@@ -500,6 +459,25 @@ def generate_docker_tags(
500459
tags = [f"{docker_project}:{tag}" for tag in tags]
501460
print(",".join(tags))
502461

462+
@staticmethod
463+
def prune_pytest_as_errors(
464+
pyproject_toml: str = "pyproject.toml", errors: tuple = ("FutureWarning", "DeprecationWarning")
465+
) -> None:
466+
"""Prune pytest warnings as errors from the pyproject.toml file."""
467+
import tomlkit
468+
469+
with open(pyproject_toml, encoding="utf-8") as fopen:
470+
content = fopen.read()
471+
pyproject = tomlkit.parse(content)
472+
filterwarnings = pyproject.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("filterwarnings", [])
473+
if not filterwarnings:
474+
return
475+
filterwarnings = [wrn for wrn in filterwarnings if not any(f"error::{err}" in wrn for err in errors)]
476+
pyproject["tool"]["pytest"]["ini_options"]["filterwarnings"] = filterwarnings
477+
478+
with open(pyproject_toml, "w", encoding="utf-8") as fopen:
479+
fopen.write(tomlkit.dumps(pyproject))
480+
503481

504482
if __name__ == "__main__":
505483
import jsonargparse

.azure/gpu-benchmarks.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:

.azure/gpu-tests-fabric.yml

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -99,8 +99,16 @@ jobs:
9999
displayName: "Image info & NVIDIA"
100100
101101
- bash: |
102-
python .actions/assistant.py replace_oldest_ver
102+
set -ex
103103
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
104112
condition: contains(variables['Agent.JobName'], 'oldest')
105113
displayName: "setting oldest dependencies"
106114

.azure/gpu-tests-pytorch.yml

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,16 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
@@ -103,8 +103,16 @@ jobs:
103103
displayName: "Image info & NVIDIA"
104104
105105
- bash: |
106-
python .actions/assistant.py replace_oldest_ver
106+
set -ex
107107
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
108116
condition: contains(variables['Agent.JobName'], 'oldest')
109117
displayName: "setting oldest dependencies"
110118

.github/checkgroup.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ subprojects:
4848
- "!**/*.md"
4949
checks:
5050
- "pytorch-lightning (GPUs) (testing Lightning | latest)"
51+
- "pytorch-lightning (GPUs) (testing PyTorch | oldest)"
5152
- "pytorch-lightning (GPUs) (testing PyTorch | latest)"
5253

5354
- id: "pytorch_lightning: Benchmarks"
@@ -174,6 +175,7 @@ subprojects:
174175
- "!*.md"
175176
- "!**/*.md"
176177
checks:
178+
- "lightning-fabric (GPUs) (testing Fabric | oldest)"
177179
- "lightning-fabric (GPUs) (testing Fabric | latest)"
178180
- "lightning-fabric (GPUs) (testing Lightning | latest)"
179181

.github/workflows/_build-packages.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ jobs:
2525
matrix:
2626
pkg-name: ${{ fromJSON(inputs.pkg-names) }}
2727
steps:
28-
- uses: actions/checkout@v4
29-
- uses: actions/setup-python@v5
28+
- uses: actions/checkout@v5
29+
- uses: actions/setup-python@v6
3030
with:
3131
python-version: "3.x"
3232

.github/workflows/_legacy-checkpoints.yml

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,30 +55,34 @@ jobs:
5555
outputs:
5656
pl-version: ${{ steps.decide-version.outputs.pl-version }}
5757
steps:
58-
- uses: actions/checkout@v4
58+
- uses: actions/checkout@v5
5959

60-
- uses: actions/setup-python@v5
60+
- name: Install uv and set Python version
61+
uses: astral-sh/setup-uv@v6
6162
with:
62-
# Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt.
6363
python-version: "3.9"
64+
# TODO: Avoid activating environment like this
65+
# see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment
66+
activate-environment: true
67+
enable-cache: true
6468

6569
- name: Install PL from source
6670
env:
6771
PACKAGE_NAME: pytorch
6872
FREEZE_REQUIREMENTS: 1
6973
timeout-minutes: 20
70-
run: pip install . --extra-index-url="${TORCH_URL}"
74+
run: uv pip install . --extra-index-url="${TORCH_URL}"
7175
if: inputs.pl_version == ''
7276

7377
- name: Install PL version
7478
timeout-minutes: 20
75-
run: pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}"
79+
run: uv pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}"
7680
if: inputs.pl_version != ''
7781

7882
- name: Adjust tests -> PL
7983
if: ${{ matrix.pkg-name != 'lightning' }}
8084
run: |
81-
pip install -q -r .actions/requirements.txt
85+
uv pip install -q -r .actions/requirements.txt
8286
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
8387
--source_import="lightning.fabric,lightning.pytorch" \
8488
--target_import="lightning_fabric,pytorch_lightning"
@@ -115,7 +119,7 @@ jobs:
115119
# export to env bool if secrets.AWS_REGION is not empty
116120
run: echo "WITH_SECRETS=$([ -n '${{ secrets.AWS_REGION }}' ] && echo 1 || echo 0)" >> $GITHUB_ENV
117121

118-
- run: pip install -r requirements/ci.txt
122+
- run: uv pip install -r requirements/ci.txt
119123
- name: Upload checkpoints to S3
120124
if: ${{ env.WITH_SECRETS == '1' }}
121125
working-directory: ${{ env.LEGACY_FOLDER }}
@@ -135,7 +139,7 @@ jobs:
135139
env:
136140
PL_VERSION: ${{ needs.create-legacy-ckpts.outputs.pl-version }}
137141
steps:
138-
- uses: actions/checkout@v4
142+
- uses: actions/checkout@v5
139143
with:
140144
ref: master
141145

.github/workflows/ci-pkg-install.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ jobs:
4646
pkg-name: ["fabric", "pytorch", "lightning", "notset"]
4747
python-version: ["3.9", "3.11"]
4848
steps:
49-
- uses: actions/checkout@v4
50-
- uses: actions/setup-python@v5
49+
- uses: actions/checkout@v5
50+
- uses: actions/setup-python@v6
5151
with:
5252
python-version: ${{ matrix.python-version }}
5353
- uses: actions/download-artifact@v5

0 commit comments

Comments
 (0)