Skip to content

Commit a0f79ca

Browse files
BordaBorda
authored
Merge branch 'master' into feature/trainer-compile-fn
2 parents 0096755 + 8ff43d4 commit a0f79ca

File tree

252 files changed

+4666
-1126
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

252 files changed

+4666
-1126
lines changed

‎.actions/assistant.py‎

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
154154
155155
"""
156156
path_readme = os.path.join(path_dir, "README.md")
157-
with open(path_readme, encoding="utf-8") as fo:
158-
text = fo.read()
157+
with open(path_readme, encoding="utf-8") as fopen:
158+
text = fopen.read()
159159

160160
# drop images from readme
161161
text = text.replace(
@@ -308,17 +308,17 @@ def copy_replace_imports(
308308
if ext in (".pyc",):
309309
continue
310310
# Try to parse everything else
311-
with open(fp, encoding="utf-8") as fo:
311+
with open(fp, encoding="utf-8") as fopen:
312312
try:
313-
lines = fo.readlines()
313+
lines = fopen.readlines()
314314
except UnicodeDecodeError:
315315
# a binary file, skip
316316
print(f"Skipped replacing imports for {fp}")
317317
continue
318318
lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
319319
os.makedirs(os.path.dirname(fp_new), exist_ok=True)
320-
with open(fp_new, "w", encoding="utf-8") as fo:
321-
fo.writelines(lines)
320+
with open(fp_new, "w", encoding="utf-8") as fopen:
321+
fopen.writelines(lines)
322322

323323

324324
def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
@@ -368,20 +368,6 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
368368
print(final)
369369
path.write_text("\n".join(final) + "\n")
370370

371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fo:
374-
req = fo.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fw:
376-
fw.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385371
@staticmethod
386372
def copy_replace_imports(
387373
source_dir: str,
@@ -471,15 +457,15 @@ def convert_version2nightly(ver_file: str = "src/version.info") -> None:
471457
"""Load the actual version and convert it to the nightly version."""
472458
from datetime import datetime
473459

474-
with open(ver_file) as fo:
475-
version = fo.read().strip()
460+
with open(ver_file) as fopen:
461+
version = fopen.read().strip()
476462
# parse X.Y.Z version and prune any suffix
477463
vers = re.match(r"(\d+)\.(\d+)\.(\d+).*", version)
478464
# create timestamp YYYYMMDD
479465
timestamp = datetime.now().strftime("%Y%m%d")
480466
version = f"{'.'.join(vers.groups())}.dev{timestamp}"
481-
with open(ver_file, "w") as fo:
482-
fo.write(version + os.linesep)
467+
with open(ver_file, "w") as fopen:
468+
fopen.write(version + os.linesep)
483469

484470
@staticmethod
485471
def generate_docker_tags(

‎.actions/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
jsonargparse >=4.16.0, <=4.35.0
1+
jsonargparse
22
requests
33
packaging

‎.azure/gpu-benchmarks.yml‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:
@@ -76,8 +76,13 @@ jobs:
7676
displayName: "Image info & NVIDIA"
7777
7878
- bash: |
79-
pip install -e .[dev] --find-links ${TORCH_URL}
80-
pip install setuptools==75.6.0 jsonargparse==4.35.0
79+
pip install -U -q -r .actions/requirements.txt
80+
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
81+
--source_import="lightning.fabric,lightning.pytorch" \
82+
--target_import="lightning_fabric,pytorch_lightning"
83+
displayName: "Adjust tests"
84+
85+
- bash: pip install -e .[dev] --find-links ${TORCH_URL}
8186
env:
8287
FREEZE_REQUIREMENTS: "1"
8388
displayName: "Install package"
@@ -88,17 +93,10 @@ jobs:
8893
python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
8994
displayName: "Env details"
9095
91-
- bash: |
92-
pip install -q -r .actions/requirements.txt
93-
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
94-
--source_import="lightning.fabric,lightning.pytorch" \
95-
--target_import="lightning_fabric,pytorch_lightning"
96-
displayName: "Adjust tests"
97-
9896
- bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
9997
env:
10098
PL_RUNNING_BENCHMARKS: "1"
101-
PL_RUN_CUDA_TESTS: "1"
99+
RUN_ONLY_CUDA_TESTS: "1"
102100
workingDirectory: tests/
103101
displayName: "Testing: benchmarks"
104102

@@ -107,7 +105,7 @@ jobs:
107105
# without succeeded this could run even if the job has already failed
108106
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
109107
env:
110-
PL_RUN_CUDA_TESTS: "1"
108+
RUN_ONLY_CUDA_TESTS: "1"
111109
PL_RUN_STANDALONE_TESTS: "1"
112110
displayName: "Testing: fabric standalone tasks"
113111
timeoutInMinutes: "10"

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4949
FREEZE_REQUIREMENTS: "1"
5050
PIP_CACHE_DIR: "/var/tmp/pip"
51-
PL_RUN_CUDA_TESTS: "1"
51+
RUN_ONLY_CUDA_TESTS: "1"
5252
container:
5353
image: $(image)
5454
# default shm size is 64m. Increase it to avoid:
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -78,8 +78,6 @@ jobs:
7878
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
7979
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
8080
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
81-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
82-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8381
displayName: "set env. vars"
8482
- bash: |
8583
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -100,6 +98,14 @@ jobs:
10098
pip list
10199
displayName: "Image info & NVIDIA"
102100
101+
- bash: |
102+
cd requirements/fabric
103+
pip install -U "lightning-utilities[cli]"
104+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
105+
pip install "cython<3.0" wheel # for compatibility
106+
condition: contains(variables['Agent.JobName'], 'oldest')
107+
displayName: "setting oldest dependencies"
108+
103109
- bash: |
104110
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
105111
pip install -q wget packaging
@@ -109,11 +115,22 @@ jobs:
109115
done
110116
displayName: "Adjust dependencies"
111117
118+
- bash: |
119+
pip install -U -q -r .actions/requirements.txt
120+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
121+
--source_import="lightning.fabric" \
122+
--target_import="lightning_fabric"
123+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
124+
--source_import="lightning.fabric" \
125+
--target_import="lightning_fabric"
126+
# without succeeded this could run even if the job has already failed
127+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
128+
displayName: "Adjust tests & examples"
129+
112130
- bash: |
113131
set -e
114132
extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
115-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
116-
pip install setuptools==75.6.0 jsonargparse==4.35.0
133+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
117134
displayName: "Install package & dependencies"
118135
119136
- bash: |
@@ -130,18 +147,6 @@ jobs:
130147
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
131148
displayName: "Testing: Fabric doctests"
132149

133-
- bash: |
134-
pip install -q -r .actions/requirements.txt
135-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
136-
--source_import="lightning.fabric" \
137-
--target_import="lightning_fabric"
138-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
139-
--source_import="lightning.fabric" \
140-
--target_import="lightning_fabric"
141-
# without succeeded this could run even if the job has already failed
142-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
143-
displayName: "Adjust tests & examples"
144-
145150
- bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_fabric/ -v --durations=50
146151
workingDirectory: tests/
147152
displayName: "Testing: fabric standard"

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,23 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
6666
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
6767
FREEZE_REQUIREMENTS: "1"
6868
PIP_CACHE_DIR: "/var/tmp/pip"
69-
PL_RUN_CUDA_TESTS: "1"
69+
RUN_ONLY_CUDA_TESTS: "1"
7070
container:
7171
image: $(image)
7272
# default shm size is 64m. Increase it to avoid:
@@ -82,8 +82,6 @@ jobs:
8282
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
8383
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
8484
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
85-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
86-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8785
displayName: "set env. vars"
8886
- bash: |
8987
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -104,6 +102,14 @@ jobs:
104102
pip list
105103
displayName: "Image info & NVIDIA"
106104
105+
- bash: |
106+
cd requirements/pytorch
107+
pip install -U "lightning-utilities[cli]"
108+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
109+
pip install "cython<3.0" wheel # for compatibility
110+
condition: contains(variables['Agent.JobName'], 'oldest')
111+
displayName: "setting oldest dependencies"
112+
107113
- bash: |
108114
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
109115
pip install -q wget packaging
@@ -113,11 +119,22 @@ jobs:
113119
done
114120
displayName: "Adjust dependencies"
115121
122+
- bash: |
123+
pip install -U -q -r .actions/requirements.txt
124+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
125+
--source_import="lightning.fabric,lightning.pytorch" \
126+
--target_import="lightning_fabric,pytorch_lightning"
127+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
128+
--source_import="lightning.fabric,lightning.pytorch" \
129+
--target_import="lightning_fabric,pytorch_lightning"
130+
# without succeeded this could run even if the job has already failed
131+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
132+
displayName: "Adjust tests & examples"
133+
116134
- bash: |
117135
set -e
118136
extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
119-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
120-
pip install setuptools==75.6.0 jsonargparse==4.35.0
137+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
121138
displayName: "Install package & dependencies"
122139
123140
- bash: pip uninstall -y lightning
@@ -144,17 +161,6 @@ jobs:
144161
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
145162
displayName: "Testing: PyTorch doctests"
146163

147-
- bash: |
148-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
149-
--source_import="lightning.fabric,lightning.pytorch" \
150-
--target_import="lightning_fabric,pytorch_lightning"
151-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
152-
--source_import="lightning.fabric,lightning.pytorch" \
153-
--target_import="lightning_fabric,pytorch_lightning"
154-
# without succeeded this could run even if the job has already failed
155-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
156-
displayName: "Adjust tests & examples"
157-
158164
- bash: |
159165
bash .actions/pull_legacy_checkpoints.sh
160166
cd tests/legacy

0 commit comments

Comments
 (0)