Skip to content

Commit 08a3e7c

Browse files
BordaBorda
authored
Merge branch 'master' into torch_28_add_missing_device_id
2 parents 9e46f02 + a08b64e commit 08a3e7c

File tree

64 files changed

+1457
-203
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+1457
-203
lines changed

‎.actions/assistant.py‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fopen:
374-
req = fopen.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fwrite:
376-
fwrite.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,

‎.azure/gpu-benchmarks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.8-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.8-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -99,8 +99,16 @@ jobs:
9999
displayName: "Image info & NVIDIA"
100100
101101
- bash: |
102-
python .actions/assistant.py replace_oldest_ver
102+
set -ex
103103
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
104112
condition: contains(variables['Agent.JobName'], 'oldest')
105113
displayName: "setting oldest dependencies"
106114

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,16 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.8-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.8-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
@@ -103,8 +103,16 @@ jobs:
103103
displayName: "Image info & NVIDIA"
104104
105105
- bash: |
106-
python .actions/assistant.py replace_oldest_ver
106+
set -ex
107107
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
108116
condition: contains(variables['Agent.JobName'], 'oldest')
109117
displayName: "setting oldest dependencies"
110118

‎.github/checkgroup.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ subprojects:
4848
- "!**/*.md"
4949
checks:
5050
- "pytorch-lightning (GPUs) (testing Lightning | latest)"
51+
- "pytorch-lightning (GPUs) (testing PyTorch | oldest)"
5152
- "pytorch-lightning (GPUs) (testing PyTorch | latest)"
5253

5354
- id: "pytorch_lightning: Benchmarks"
@@ -174,6 +175,7 @@ subprojects:
174175
- "!*.md"
175176
- "!**/*.md"
176177
checks:
178+
- "lightning-fabric (GPUs) (testing Fabric | oldest)"
177179
- "lightning-fabric (GPUs) (testing Fabric | latest)"
178180
- "lightning-fabric (GPUs) (testing Lightning | latest)"
179181

‎.github/workflows/_build-packages.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
matrix:
2626
pkg-name: ${{ fromJSON(inputs.pkg-names) }}
2727
steps:
28-
- uses: actions/checkout@v4
28+
- uses: actions/checkout@v5
2929
- uses: actions/setup-python@v5
3030
with:
3131
python-version: "3.x"

‎.github/workflows/_legacy-checkpoints.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
outputs:
5656
pl-version: ${{ steps.decide-version.outputs.pl-version }}
5757
steps:
58-
- uses: actions/checkout@v4
58+
- uses: actions/checkout@v5
5959

6060
- uses: actions/setup-python@v5
6161
with:
@@ -135,7 +135,7 @@ jobs:
135135
env:
136136
PL_VERSION: ${{ needs.create-legacy-ckpts.outputs.pl-version }}
137137
steps:
138-
- uses: actions/checkout@v4
138+
- uses: actions/checkout@v5
139139
with:
140140
ref: master
141141

‎.github/workflows/ci-pkg-install.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
pkg-name: ["fabric", "pytorch", "lightning", "notset"]
4747
python-version: ["3.9", "3.11"]
4848
steps:
49-
- uses: actions/checkout@v4
49+
- uses: actions/checkout@v5
5050
- uses: actions/setup-python@v5
5151
with:
5252
python-version: ${{ matrix.python-version }}

‎.github/workflows/ci-tests-fabric.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ jobs:
8181
# TODO: Remove this - Enable running MPS tests on this platform
8282
DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }}
8383
steps:
84-
- uses: actions/checkout@v4
84+
- uses: actions/checkout@v5
8585

8686
- name: Set up Python ${{ matrix.python-version }}
8787
uses: actions/setup-python@v5
@@ -94,7 +94,9 @@ jobs:
9494
- name: Set min. dependencies
9595
if: ${{ matrix.requires == 'oldest' }}
9696
run: |
97-
python .actions/assistant.py replace_oldest_ver
97+
cd requirements/fabric
98+
pip install -U "lightning-utilities[cli]"
99+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt', 'test.txt']"
98100
pip install "cython<3.0" wheel
99101
pip install "pyyaml==5.4" --no-build-isolation
100102
@@ -140,7 +142,8 @@ jobs:
140142
run: |
141143
pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
142144
-U --upgrade-strategy=eager --prefer-binary \
143-
--extra-index-url="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}"
145+
--extra-index-url="${TORCH_URL}" \
146+
--find-links="${PYPI_CACHE_DIR}"
144147
pip list
145148
- name: Dump handy wheels
146149
if: github.event_name == 'push' && github.ref == 'refs/heads/master'

‎.github/workflows/ci-tests-pytorch.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ jobs:
8686
# TODO: Remove this - Enable running MPS tests on this platform
8787
DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }}
8888
steps:
89-
- uses: actions/checkout@v4
89+
- uses: actions/checkout@v5
9090

9191
- name: Set up Python ${{ matrix.python-version }}
9292
uses: actions/setup-python@v5
@@ -99,7 +99,9 @@ jobs:
9999
- name: Set min. dependencies
100100
if: ${{ matrix.requires == 'oldest' }}
101101
run: |
102-
python .actions/assistant.py replace_oldest_ver
102+
cd requirements/pytorch
103+
pip install -U "lightning-utilities[cli]"
104+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt', 'test.txt']"
103105
pip install "cython<3.0" wheel
104106
pip install "pyyaml==5.4" --no-build-isolation
105107
@@ -139,7 +141,9 @@ jobs:
139141
pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
140142
-U --upgrade-strategy=eager --prefer-binary \
141143
-r requirements/_integrations/accelerators.txt \
142-
--extra-index-url="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}"
144+
--extra-index-url="${TORCH_URL}" \
145+
--find-links="${PYPI_CACHE_DIR}" \
146+
--find-links="https://download.pytorch.org/whl/torch-tensorrt"
143147
pip list
144148
- name: Drop LAI from extensions
145149
if: ${{ matrix.pkg-name != 'lightning' }}

0 commit comments

Comments
 (0)