Skip to content

Commit fd5cbf1

Browse files
SkafteNickiSkafteNicki
authored
Merge branch 'master' into docs/lightning_module_enhancements
2 parents 1b33fc3 + 216f9ec commit fd5cbf1

File tree

168 files changed

+4940
-947
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

168 files changed

+4940
-947
lines changed

‎.actions/assistant.py‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fopen:
374-
req = fopen.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fwrite:
376-
fwrite.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,

‎.azure/gpu-benchmarks.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:
@@ -96,7 +96,7 @@ jobs:
9696
- bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
9797
env:
9898
PL_RUNNING_BENCHMARKS: "1"
99-
PL_RUN_CUDA_TESTS: "1"
99+
RUN_ONLY_CUDA_TESTS: "1"
100100
workingDirectory: tests/
101101
displayName: "Testing: benchmarks"
102102

@@ -105,7 +105,7 @@ jobs:
105105
# without succeeded this could run even if the job has already failed
106106
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
107107
env:
108-
PL_RUN_CUDA_TESTS: "1"
108+
RUN_ONLY_CUDA_TESTS: "1"
109109
PL_RUN_STANDALONE_TESTS: "1"
110110
displayName: "Testing: fabric standalone tasks"
111111
timeoutInMinutes: "10"

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4949
FREEZE_REQUIREMENTS: "1"
5050
PIP_CACHE_DIR: "/var/tmp/pip"
51-
PL_RUN_CUDA_TESTS: "1"
51+
RUN_ONLY_CUDA_TESTS: "1"
5252
container:
5353
image: $(image)
5454
# default shm size is 64m. Increase it to avoid:
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -78,8 +78,6 @@ jobs:
7878
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
7979
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
8080
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
81-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
82-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8381
displayName: "set env. vars"
8482
- bash: |
8583
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -101,8 +99,16 @@ jobs:
10199
displayName: "Image info & NVIDIA"
102100
103101
- bash: |
104-
python .actions/assistant.py replace_oldest_ver
102+
set -ex
105103
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
106112
condition: contains(variables['Agent.JobName'], 'oldest')
107113
displayName: "setting oldest dependencies"
108114

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,23 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
6666
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
6767
FREEZE_REQUIREMENTS: "1"
6868
PIP_CACHE_DIR: "/var/tmp/pip"
69-
PL_RUN_CUDA_TESTS: "1"
69+
RUN_ONLY_CUDA_TESTS: "1"
7070
container:
7171
image: $(image)
7272
# default shm size is 64m. Increase it to avoid:
@@ -82,8 +82,6 @@ jobs:
8282
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
8383
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
8484
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
85-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
86-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8785
displayName: "set env. vars"
8886
- bash: |
8987
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -105,8 +103,16 @@ jobs:
105103
displayName: "Image info & NVIDIA"
106104
107105
- bash: |
108-
python .actions/assistant.py replace_oldest_ver
106+
set -ex
109107
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
110116
condition: contains(variables['Agent.JobName'], 'oldest')
111117
displayName: "setting oldest dependencies"
112118

‎.github/checkgroup.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ subprojects:
4848
- "!**/*.md"
4949
checks:
5050
- "pytorch-lightning (GPUs) (testing Lightning | latest)"
51+
- "pytorch-lightning (GPUs) (testing PyTorch | oldest)"
5152
- "pytorch-lightning (GPUs) (testing PyTorch | latest)"
5253

5354
- id: "pytorch_lightning: Benchmarks"
@@ -174,6 +175,7 @@ subprojects:
174175
- "!*.md"
175176
- "!**/*.md"
176177
checks:
178+
- "lightning-fabric (GPUs) (testing Fabric | oldest)"
177179
- "lightning-fabric (GPUs) (testing Fabric | latest)"
178180
- "lightning-fabric (GPUs) (testing Lightning | latest)"
179181

‎.github/workflows/_build-packages.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
matrix:
2626
pkg-name: ${{ fromJSON(inputs.pkg-names) }}
2727
steps:
28-
- uses: actions/checkout@v4
28+
- uses: actions/checkout@v5
2929
- uses: actions/setup-python@v5
3030
with:
3131
python-version: "3.x"
@@ -51,7 +51,7 @@ jobs:
5151
needs: build-packages
5252
runs-on: ubuntu-22.04
5353
steps:
54-
- uses: actions/download-artifact@v4
54+
- uses: actions/download-artifact@v5
5555
with: # download all build artifacts
5656
pattern: ${{ inputs.artifact-name }}-*
5757
merge-multiple: true

‎.github/workflows/_legacy-checkpoints.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
outputs:
5656
pl-version: ${{ steps.decide-version.outputs.pl-version }}
5757
steps:
58-
- uses: actions/checkout@v4
58+
- uses: actions/checkout@v5
5959

6060
- uses: actions/setup-python@v5
6161
with:
@@ -135,7 +135,7 @@ jobs:
135135
env:
136136
PL_VERSION: ${{ needs.create-legacy-ckpts.outputs.pl-version }}
137137
steps:
138-
- uses: actions/checkout@v4
138+
- uses: actions/checkout@v5
139139
with:
140140
ref: master
141141

‎.github/workflows/call-clear-cache.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,18 @@ on:
2323
jobs:
2424
cron-clear:
2525
if: github.event_name == 'schedule' || github.event_name == 'pull_request'
26-
uses: Lightning-AI/utilities/.github/workflows/[email protected].0
26+
uses: Lightning-AI/utilities/.github/workflows/[email protected].2
2727
with:
28-
scripts-ref: v0.14.3
28+
scripts-ref: v0.15.2
2929
dry-run: ${{ github.event_name == 'pull_request' }}
3030
pattern: "latest|docs"
3131
age-days: 7
3232

3333
direct-clear:
3434
if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
35-
uses: Lightning-AI/utilities/.github/workflows/[email protected].0
35+
uses: Lightning-AI/utilities/.github/workflows/[email protected].2
3636
with:
37-
scripts-ref: v0.14.3
37+
scripts-ref: v0.15.2
3838
dry-run: ${{ github.event_name == 'pull_request' }}
3939
pattern: ${{ inputs.pattern || 'pypi_wheels' }} # setting str in case of PR / debugging
4040
age-days: ${{ fromJSON(inputs.age-days) || 0 }} # setting 0 in case of PR / debugging

‎.github/workflows/ci-pkg-install.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,11 @@ jobs:
4646
pkg-name: ["fabric", "pytorch", "lightning", "notset"]
4747
python-version: ["3.9", "3.11"]
4848
steps:
49-
- uses: actions/checkout@v4
49+
- uses: actions/checkout@v5
5050
- uses: actions/setup-python@v5
5151
with:
5252
python-version: ${{ matrix.python-version }}
53-
- uses: actions/download-artifact@v4
53+
- uses: actions/download-artifact@v5
5454
with:
5555
name: dist-packages-${{ github.sha }}
5656
path: dist

‎.github/workflows/ci-schema.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88

99
jobs:
1010
check:
11-
uses: Lightning-AI/utilities/.github/workflows/[email protected].0
11+
uses: Lightning-AI/utilities/.github/workflows/[email protected].2
1212
with:
1313
# skip azure due to the wrong schema file by MSFT
1414
# https://github.com/Lightning-AI/lightning-flash/pull/1455#issuecomment-1244793607

0 commit comments

Comments
 (0)