Skip to content

Commit b5ef2d3

Browse files
SkafteNickiSkafteNicki
authored
Merge branch 'master' into docs/csv-logger
2 parents 2b2623e + b581924 commit b5ef2d3

File tree

157 files changed

+3577
-790
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

157 files changed

+3577
-790
lines changed

‎.actions/assistant.py‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fopen:
374-
req = fopen.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fwrite:
376-
fwrite.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,

‎.azure/gpu-benchmarks.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
variables:
4747
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4848
container:
49-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
49+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5050
options: "--gpus=all --shm-size=32g"
5151
strategy:
5252
matrix:
@@ -96,7 +96,7 @@ jobs:
9696
- bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
9797
env:
9898
PL_RUNNING_BENCHMARKS: "1"
99-
PL_RUN_CUDA_TESTS: "1"
99+
RUN_ONLY_CUDA_TESTS: "1"
100100
workingDirectory: tests/
101101
displayName: "Testing: benchmarks"
102102

@@ -105,7 +105,7 @@ jobs:
105105
# without succeeded this could run even if the job has already failed
106106
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
107107
env:
108-
PL_RUN_CUDA_TESTS: "1"
108+
RUN_ONLY_CUDA_TESTS: "1"
109109
PL_RUN_STANDALONE_TESTS: "1"
110110
displayName: "Testing: fabric standalone tasks"
111111
timeoutInMinutes: "10"

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
4949
FREEZE_REQUIREMENTS: "1"
5050
PIP_CACHE_DIR: "/var/tmp/pip"
51-
PL_RUN_CUDA_TESTS: "1"
51+
RUN_ONLY_CUDA_TESTS: "1"
5252
container:
5353
image: $(image)
5454
# default shm size is 64m. Increase it to avoid:
@@ -57,16 +57,16 @@ jobs:
5757
strategy:
5858
matrix:
5959
"Fabric | oldest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
60+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
6161
PACKAGE_NAME: "fabric"
6262
"Fabric | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6464
PACKAGE_NAME: "fabric"
6565
#"Fabric | future":
66-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
66+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6767
# PACKAGE_NAME: "fabric"
6868
"Lightning | latest":
69-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
69+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
7070
PACKAGE_NAME: "lightning"
7171
workspace:
7272
clean: all
@@ -78,8 +78,6 @@ jobs:
7878
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
7979
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
8080
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
81-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
82-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8381
displayName: "set env. vars"
8482
- bash: |
8583
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -101,8 +99,16 @@ jobs:
10199
displayName: "Image info & NVIDIA"
102100
103101
- bash: |
104-
python .actions/assistant.py replace_oldest_ver
102+
set -ex
105103
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
106112
condition: contains(variables['Agent.JobName'], 'oldest')
107113
displayName: "setting oldest dependencies"
108114

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,23 @@ jobs:
5050
strategy:
5151
matrix:
5252
"PyTorch | oldest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
53+
image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
5454
PACKAGE_NAME: "pytorch"
5555
"PyTorch | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
5757
PACKAGE_NAME: "pytorch"
5858
#"PyTorch | future":
59-
# image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
59+
# image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
6060
# PACKAGE_NAME: "pytorch"
6161
"Lightning | latest":
62-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.7-cuda12.6.3"
62+
image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
6363
PACKAGE_NAME: "lightning"
6464
pool: lit-rtx-3090
6565
variables:
6666
DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
6767
FREEZE_REQUIREMENTS: "1"
6868
PIP_CACHE_DIR: "/var/tmp/pip"
69-
PL_RUN_CUDA_TESTS: "1"
69+
RUN_ONLY_CUDA_TESTS: "1"
7070
container:
7171
image: $(image)
7272
# default shm size is 64m. Increase it to avoid:
@@ -82,8 +82,6 @@ jobs:
8282
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
8383
scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
8484
echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
85-
python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
86-
echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
8785
displayName: "set env. vars"
8886
- bash: |
8987
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
@@ -105,8 +103,16 @@ jobs:
105103
displayName: "Image info & NVIDIA"
106104
107105
- bash: |
108-
python .actions/assistant.py replace_oldest_ver
106+
set -ex
109107
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
110116
condition: contains(variables['Agent.JobName'], 'oldest')
111117
displayName: "setting oldest dependencies"
112118

‎.github/CONTRIBUTING.md‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices
44
and integrations that the amazing PyTorch team and other research organization rolls out!
55

6-
If you are new to open source, check out [this blog to get started with your first Open Source contribution](https://devblog.pytorchlightning.ai/quick-contribution-guide-86d977171b3a).
6+
If you are new to open source, check out [this blog to get started with your first Open Source contribution](https://medium.com/pytorch-lightning/quick-contribution-guide-86d977171b3a).
77

88
## Main Core Value: One less thing to remember
99

@@ -113,14 +113,28 @@ ______________________________________________________________________
113113

114114
To set up a local development environment, we recommend using `uv`, which can be installed following their [instructions](https://docs.astral.sh/uv/getting-started/installation/).
115115

116-
Once `uv` has been installed, begin by cloning the repository:
116+
Once `uv` has been installed, begin by cloning the forked repository:
117117

118118
```bash
119-
git clone https://github.com/Lightning-AI/lightning.git
120-
cd lightning
119+
git clone https://github.com/{YOUR_GITHUB_USERNAME}/pytorch-lightning.git
120+
cd pytorch-lightning
121121
```
122122

123-
Once in root level of the repository, create a new virtual environment and install the project dependencies.
123+
> If you're using [Lightning Studio](https://lightning.ai) or already have your `uv venv` activated, you can quickly set up the project by running:
124+
125+
```bash
126+
make setup
127+
```
128+
129+
This will:
130+
131+
- Install all required dependencies.
132+
- Perform an editable install of the `pytorch-lightning` project.
133+
- Install and configure `pre-commit`.
134+
135+
#### Manual Setup (Optional)
136+
137+
If you prefer more fine-grained control over the dependencies, you can set up the environment manually:
124138

125139
```bash
126140
uv venv

‎.github/checkgroup.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ subprojects:
4848
- "!**/*.md"
4949
checks:
5050
- "pytorch-lightning (GPUs) (testing Lightning | latest)"
51+
- "pytorch-lightning (GPUs) (testing PyTorch | oldest)"
5152
- "pytorch-lightning (GPUs) (testing PyTorch | latest)"
5253

5354
- id: "pytorch_lightning: Benchmarks"
@@ -135,7 +136,8 @@ subprojects:
135136
- "build-pl (3.11, 2.4, 12.1.1)"
136137
- "build-pl (3.12, 2.5, 12.1.1)"
137138
- "build-pl (3.12, 2.6, 12.4.1)"
138-
- "build-pl (3.12, 2.7, 12.6.3, true)"
139+
- "build-pl (3.12, 2.7, 12.6.3)"
140+
- "build-pl (3.12, 2.8, 12.6.3, true)"
139141

140142
# SECTION: lightning_fabric
141143

@@ -173,6 +175,7 @@ subprojects:
173175
- "!*.md"
174176
- "!**/*.md"
175177
checks:
178+
- "lightning-fabric (GPUs) (testing Fabric | oldest)"
176179
- "lightning-fabric (GPUs) (testing Fabric | latest)"
177180
- "lightning-fabric (GPUs) (testing Lightning | latest)"
178181

‎.github/workflows/_build-packages.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
matrix:
2626
pkg-name: ${{ fromJSON(inputs.pkg-names) }}
2727
steps:
28-
- uses: actions/checkout@v4
28+
- uses: actions/checkout@v5
2929
- uses: actions/setup-python@v5
3030
with:
3131
python-version: "3.x"
@@ -51,7 +51,7 @@ jobs:
5151
needs: build-packages
5252
runs-on: ubuntu-22.04
5353
steps:
54-
- uses: actions/download-artifact@v4
54+
- uses: actions/download-artifact@v5
5555
with: # download all build artifacts
5656
pattern: ${{ inputs.artifact-name }}-*
5757
merge-multiple: true

‎.github/workflows/_legacy-checkpoints.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
outputs:
5656
pl-version: ${{ steps.decide-version.outputs.pl-version }}
5757
steps:
58-
- uses: actions/checkout@v4
58+
- uses: actions/checkout@v5
5959

6060
- uses: actions/setup-python@v5
6161
with:
@@ -135,7 +135,7 @@ jobs:
135135
env:
136136
PL_VERSION: ${{ needs.create-legacy-ckpts.outputs.pl-version }}
137137
steps:
138-
- uses: actions/checkout@v4
138+
- uses: actions/checkout@v5
139139
with:
140140
ref: master
141141

‎.github/workflows/call-clear-cache.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,18 @@ on:
2323
jobs:
2424
cron-clear:
2525
if: github.event_name == 'schedule' || github.event_name == 'pull_request'
26-
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.14.3
26+
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.15.2
2727
with:
28-
scripts-ref: v0.14.3
28+
scripts-ref: v0.15.2
2929
dry-run: ${{ github.event_name == 'pull_request' }}
3030
pattern: "latest|docs"
3131
age-days: 7
3232

3333
direct-clear:
3434
if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
35-
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.14.3
35+
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.15.2
3636
with:
37-
scripts-ref: v0.14.3
37+
scripts-ref: v0.15.2
3838
dry-run: ${{ github.event_name == 'pull_request' }}
3939
pattern: ${{ inputs.pattern || 'pypi_wheels' }} # setting str in case of PR / debugging
4040
age-days: ${{ fromJSON(inputs.age-days) || 0 }} # setting 0 in case of PR / debugging

‎.github/workflows/ci-pkg-install.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,11 @@ jobs:
4646
pkg-name: ["fabric", "pytorch", "lightning", "notset"]
4747
python-version: ["3.9", "3.11"]
4848
steps:
49-
- uses: actions/checkout@v4
49+
- uses: actions/checkout@v5
5050
- uses: actions/setup-python@v5
5151
with:
5252
python-version: ${{ matrix.python-version }}
53-
- uses: actions/download-artifact@v4
53+
- uses: actions/download-artifact@v5
5454
with:
5555
name: dist-packages-${{ github.sha }}
5656
path: dist

0 commit comments

Comments
 (0)