Skip to content

Commit 9ebbc54

Browse files
KAVYANSHTYAGIKAVYANSHTYAGI
authored
Merge branch 'Lightning-AI:master' into master
2 parents a9478dc + b1cc925 commit 9ebbc54

File tree

240 files changed

+5012
-1130
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

240 files changed

+5012
-1130
lines changed

‎.actions/assistant.py‎

Lines changed: 10 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
154154
155155
"""
156156
path_readme = os.path.join(path_dir, "README.md")
157-
with open(path_readme, encoding="utf-8") as fo:
158-
text = fo.read()
157+
with open(path_readme, encoding="utf-8") as fopen:
158+
text = fopen.read()
159159

160160
# drop images from readme
161161
text = text.replace(
@@ -308,17 +308,17 @@ def copy_replace_imports(
308308
if ext in (".pyc",):
309309
continue
310310
# Try to parse everything else
311-
with open(fp, encoding="utf-8") as fo:
311+
with open(fp, encoding="utf-8") as fopen:
312312
try:
313-
lines = fo.readlines()
313+
lines = fopen.readlines()
314314
except UnicodeDecodeError:
315315
# a binary file, skip
316316
print(f"Skipped replacing imports for {fp}")
317317
continue
318318
lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
319319
os.makedirs(os.path.dirname(fp_new), exist_ok=True)
320-
with open(fp_new, "w", encoding="utf-8") as fo:
321-
fo.writelines(lines)
320+
with open(fp_new, "w", encoding="utf-8") as fopen:
321+
fopen.writelines(lines)
322322

323323

324324
def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fo:
374-
req = fo.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fw:
376-
fw.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,
@@ -471,15 +430,15 @@ def convert_version2nightly(ver_file: str = "src/version.info") -> None:
471430
"""Load the actual version and convert it to the nightly version."""
472431
from datetime import datetime
473432

474-
with open(ver_file) as fo:
475-
version = fo.read().strip()
433+
with open(ver_file) as fopen:
434+
version = fopen.read().strip()
476435
# parse X.Y.Z version and prune any suffix
477436
vers = re.match(r"(\d+)\.(\d+)\.(\d+).*", version)
478437
# create timestamp YYYYMMDD
479438
timestamp = datetime.now().strftime("%Y%m%d")
480439
version = f"{'.'.join(vers.groups())}.dev{timestamp}"
481-
with open(ver_file, "w") as fo:
482-
fo.write(version + os.linesep)
440+
with open(ver_file, "w") as fopen:
441+
fopen.write(version + os.linesep)
483442

484443
@staticmethod
485444
def generate_docker_tags(

‎.actions/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
jsonargparse >=4.16.0, <=4.35.0
1+
jsonargparse
22
requests
33
packaging

‎.azure/gpu-benchmarks.yml‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:
@@ -76,8 +76,13 @@ jobs:
7676
displayName: "Image info & NVIDIA"
7777
7878
- bash: |
79-
pip install -e .[dev] --find-links ${TORCH_URL}
80-
pip install setuptools==75.6.0 jsonargparse==4.35.0
79+
pip install -U -q -r .actions/requirements.txt
80+
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
81+
--source_import="lightning.fabric,lightning.pytorch" \
82+
--target_import="lightning_fabric,pytorch_lightning"
83+
displayName: "Adjust tests"
84+
85+
- bash: pip install -e .[dev] --find-links ${TORCH_URL}
8186
env:
8287
FREEZE_REQUIREMENTS: "1"
8388
displayName: "Install package"
@@ -88,17 +93,10 @@ jobs:
8893
python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
8994
displayName: "Env details"
9095
91-
- bash: |
92-
pip install -q -r .actions/requirements.txt
93-
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
94-
--source_import="lightning.fabric,lightning.pytorch" \
95-
--target_import="lightning_fabric,pytorch_lightning"
96-
displayName: "Adjust tests"
97-
9896
- bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
9997
env:
10098
PL_RUNNING_BENCHMARKS: "1"
101-
PL_RUN_CUDA_TESTS: "1"
99+
RUN_ONLY_CUDA_TESTS: "1"
102100
workingDirectory: tests/
103101
displayName: "Testing: benchmarks"
104102

@@ -107,7 +105,7 @@ jobs:
107105
# without succeeded this could run even if the job has already failed
108106
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
109107
env:
110-
PL_RUN_CUDA_TESTS: "1"
108+
RUN_ONLY_CUDA_TESTS: "1"
111109
PL_RUN_STANDALONE_TESTS: "1"
112110
displayName: "Testing: fabric standalone tasks"
113111
timeoutInMinutes: "10"

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 32 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4949
FREEZE_REQUIREMENTS: "1"
5050
PIP_CACHE_DIR: "/var/tmp/pip"
51-
PL_RUN_CUDA_TESTS: "1"
51+
RUN_ONLY_CUDA_TESTS: "1"
5252
container:
5353
image: $(image)
5454
# default shm size is 64m. Increase it to avoid:
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -78,8 +78,6 @@ jobs:
7878
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
7979
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
8080
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
81-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
82-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8381
displayName: "set env. vars"
8482
- bash: |
8583
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -100,6 +98,20 @@ jobs:
10098
pip list
10199
displayName: "Image info & NVIDIA"
102100
101+
- bash: |
102+
set -ex
103+
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
112+
condition: contains(variables['Agent.JobName'], 'oldest')
113+
displayName: "setting oldest dependencies"
114+
103115
- bash: |
104116
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
105117
pip install -q wget packaging
@@ -109,11 +121,22 @@ jobs:
109121
done
110122
displayName: "Adjust dependencies"
111123
124+
- bash: |
125+
pip install -U -q -r .actions/requirements.txt
126+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
127+
--source_import="lightning.fabric" \
128+
--target_import="lightning_fabric"
129+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
130+
--source_import="lightning.fabric" \
131+
--target_import="lightning_fabric"
132+
# without succeeded this could run even if the job has already failed
133+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
134+
displayName: "Adjust tests & examples"
135+
112136
- bash: |
113137
set -e
114138
extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
115-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
116-
pip install setuptools==75.6.0 jsonargparse==4.35.0
139+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
117140
displayName: "Install package & dependencies"
118141
119142
- bash: |
@@ -130,18 +153,6 @@ jobs:
130153
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
131154
displayName: "Testing: Fabric doctests"
132155

133-
- bash: |
134-
pip install -q -r .actions/requirements.txt
135-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
136-
--source_import="lightning.fabric" \
137-
--target_import="lightning_fabric"
138-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
139-
--source_import="lightning.fabric" \
140-
--target_import="lightning_fabric"
141-
# without succeeded this could run even if the job has already failed
142-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
143-
displayName: "Adjust tests & examples"
144-
145156
- bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_fabric/ -v --durations=50
146157
workingDirectory: tests/
147158
displayName: "Testing: fabric standard"

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,23 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
6666
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
6767
FREEZE_REQUIREMENTS: "1"
6868
PIP_CACHE_DIR: "/var/tmp/pip"
69-
PL_RUN_CUDA_TESTS: "1"
69+
RUN_ONLY_CUDA_TESTS: "1"
7070
container:
7171
image: $(image)
7272
# default shm size is 64m. Increase it to avoid:
@@ -82,8 +82,6 @@ jobs:
8282
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
8383
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
8484
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
85-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
86-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8785
displayName: "set env. vars"
8886
- bash: |
8987
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -104,6 +102,20 @@ jobs:
104102
pip list
105103
displayName: "Image info & NVIDIA"
106104
105+
- bash: |
106+
set -ex
107+
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
116+
condition: contains(variables['Agent.JobName'], 'oldest')
117+
displayName: "setting oldest dependencies"
118+
107119
- bash: |
108120
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
109121
pip install -q wget packaging
@@ -113,11 +125,22 @@ jobs:
113125
done
114126
displayName: "Adjust dependencies"
115127
128+
- bash: |
129+
pip install -U -q -r .actions/requirements.txt
130+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
131+
--source_import="lightning.fabric,lightning.pytorch" \
132+
--target_import="lightning_fabric,pytorch_lightning"
133+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
134+
--source_import="lightning.fabric,lightning.pytorch" \
135+
--target_import="lightning_fabric,pytorch_lightning"
136+
# without succeeded this could run even if the job has already failed
137+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
138+
displayName: "Adjust tests & examples"
139+
116140
- bash: |
117141
set -e
118142
extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
119-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
120-
pip install setuptools==75.6.0 jsonargparse==4.35.0
143+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
121144
displayName: "Install package & dependencies"
122145
123146
- bash: pip uninstall -y lightning
@@ -144,17 +167,6 @@ jobs:
144167
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
145168
displayName: "Testing: PyTorch doctests"
146169

147-
- bash: |
148-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
149-
--source_import="lightning.fabric,lightning.pytorch" \
150-
--target_import="lightning_fabric,pytorch_lightning"
151-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
152-
--source_import="lightning.fabric,lightning.pytorch" \
153-
--target_import="lightning_fabric,pytorch_lightning"
154-
# without succeeded this could run even if the job has already failed
155-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
156-
displayName: "Adjust tests & examples"
157-
158170
- bash: |
159171
bash .actions/pull_legacy_checkpoints.sh
160172
cd tests/legacy

0 commit comments

Comments
 (0)