Skip to content

Commit c884d6b

Browse files
LothiraldanLothiraldan
authored
Merge branch 'master' into comet-flush-experiment-after-checkpoint
2 parents 1ef8429 + e29a8cb commit c884d6b

File tree

125 files changed

+766
-315
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

125 files changed

+766
-315
lines changed

‎.actions/assistant.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
154154
155155
"""
156156
path_readme = os.path.join(path_dir, "README.md")
157-
with open(path_readme, encoding="utf-8") as fo:
158-
text = fo.read()
157+
with open(path_readme, encoding="utf-8") as fopen:
158+
text = fopen.read()
159159

160160
# drop images from readme
161161
text = text.replace(
@@ -308,17 +308,17 @@ def copy_replace_imports(
308308
if ext in (".pyc",):
309309
continue
310310
# Try to parse everything else
311-
with open(fp, encoding="utf-8") as fo:
311+
with open(fp, encoding="utf-8") as fopen:
312312
try:
313-
lines = fo.readlines()
313+
lines = fopen.readlines()
314314
except UnicodeDecodeError:
315315
# a binary file, skip
316316
print(f"Skipped replacing imports for {fp}")
317317
continue
318318
lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
319319
os.makedirs(os.path.dirname(fp_new), exist_ok=True)
320-
with open(fp_new, "w", encoding="utf-8") as fo:
321-
fo.writelines(lines)
320+
with open(fp_new, "w", encoding="utf-8") as fopen:
321+
fopen.writelines(lines)
322322

323323

324324
def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
@@ -370,10 +370,10 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
370370

371371
@staticmethod
372372
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fo:
374-
req = fo.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fw:
376-
fw.write(req)
373+
with open(fname, encoding="utf-8") as fopen:
374+
req = fopen.read().replace(">=", "==")
375+
with open(fname, "w", encoding="utf-8") as fwrite:
376+
fwrite.write(req)
377377

378378
@staticmethod
379379
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
@@ -471,15 +471,15 @@ def convert_version2nightly(ver_file: str = "src/version.info") -> None:
471471
"""Load the actual version and convert it to the nightly version."""
472472
from datetime import datetime
473473

474-
with open(ver_file) as fo:
475-
version = fo.read().strip()
474+
with open(ver_file) as fopen:
475+
version = fopen.read().strip()
476476
# parse X.Y.Z version and prune any suffix
477477
vers = re.match(r"(\d+)\.(\d+)\.(\d+).*", version)
478478
# create timestamp YYYYMMDD
479479
timestamp = datetime.now().strftime("%Y%m%d")
480480
version = f"{'.'.join(vers.groups())}.dev{timestamp}"
481-
with open(ver_file, "w") as fo:
482-
fo.write(version + os.linesep)
481+
with open(ver_file, "w") as fopen:
482+
fopen.write(version + os.linesep)
483483

484484
@staticmethod
485485
def generate_docker_tags(

‎.actions/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
jsonargparse >=4.16.0, <=4.35.0
1+
jsonargparse
22
requests
33
packaging

‎.azure/gpu-benchmarks.yml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,13 @@ jobs:
7676
displayName: "Image info & NVIDIA"
7777
7878
- bash: |
79-
pip install -e .[dev] --find-links ${TORCH_URL}
80-
pip install setuptools==75.6.0 jsonargparse==4.35.0
79+
pip install -U -q -r .actions/requirements.txt
80+
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
81+
--source_import="lightning.fabric,lightning.pytorch" \
82+
--target_import="lightning_fabric,pytorch_lightning"
83+
displayName: "Adjust tests"
84+
85+
- bash: pip install -e .[dev] --find-links ${TORCH_URL}
8186
env:
8287
FREEZE_REQUIREMENTS: "1"
8388
displayName: "Install package"
@@ -88,13 +93,6 @@ jobs:
8893
python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
8994
displayName: "Env details"
9095
91-
- bash: |
92-
pip install -q -r .actions/requirements.txt
93-
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
94-
--source_import="lightning.fabric,lightning.pytorch" \
95-
--target_import="lightning_fabric,pytorch_lightning"
96-
displayName: "Adjust tests"
97-
9896
- bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
9997
env:
10098
PL_RUNNING_BENCHMARKS: "1"

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,12 @@ jobs:
100100
pip list
101101
displayName: "Image info & NVIDIA"
102102
103+
- bash: |
104+
python .actions/assistant.py replace_oldest_ver
105+
pip install "cython<3.0" wheel # for compatibility
106+
condition: contains(variables['Agent.JobName'], 'oldest')
107+
displayName: "setting oldest dependencies"
108+
103109
- bash: |
104110
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
105111
pip install -q wget packaging
@@ -109,11 +115,22 @@ jobs:
109115
done
110116
displayName: "Adjust dependencies"
111117
118+
- bash: |
119+
pip install -U -q -r .actions/requirements.txt
120+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
121+
--source_import="lightning.fabric" \
122+
--target_import="lightning_fabric"
123+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
124+
--source_import="lightning.fabric" \
125+
--target_import="lightning_fabric"
126+
# without succeeded this could run even if the job has already failed
127+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
128+
displayName: "Adjust tests & examples"
129+
112130
- bash: |
113131
set -e
114132
extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
115-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
116-
pip install setuptools==75.6.0 jsonargparse==4.35.0
133+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
117134
displayName: "Install package & dependencies"
118135
119136
- bash: |
@@ -130,18 +147,6 @@ jobs:
130147
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
131148
displayName: "Testing: Fabric doctests"
132149

133-
- bash: |
134-
pip install -q -r .actions/requirements.txt
135-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
136-
--source_import="lightning.fabric" \
137-
--target_import="lightning_fabric"
138-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
139-
--source_import="lightning.fabric" \
140-
--target_import="lightning_fabric"
141-
# without succeeded this could run even if the job has already failed
142-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
143-
displayName: "Adjust tests & examples"
144-
145150
- bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_fabric/ -v --durations=50
146151
workingDirectory: tests/
147152
displayName: "Testing: fabric standard"

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,12 @@ jobs:
104104
pip list
105105
displayName: "Image info & NVIDIA"
106106
107+
- bash: |
108+
python .actions/assistant.py replace_oldest_ver
109+
pip install "cython<3.0" wheel # for compatibility
110+
condition: contains(variables['Agent.JobName'], 'oldest')
111+
displayName: "setting oldest dependencies"
112+
107113
- bash: |
108114
PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
109115
pip install -q wget packaging
@@ -113,11 +119,22 @@ jobs:
113119
done
114120
displayName: "Adjust dependencies"
115121
122+
- bash: |
123+
pip install -U -q -r .actions/requirements.txt
124+
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
125+
--source_import="lightning.fabric,lightning.pytorch" \
126+
--target_import="lightning_fabric,pytorch_lightning"
127+
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
128+
--source_import="lightning.fabric,lightning.pytorch" \
129+
--target_import="lightning_fabric,pytorch_lightning"
130+
# without succeeded this could run even if the job has already failed
131+
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
132+
displayName: "Adjust tests & examples"
133+
116134
- bash: |
117135
set -e
118136
extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
119-
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
120-
pip install setuptools==75.6.0 jsonargparse==4.35.0
137+
pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
121138
displayName: "Install package & dependencies"
122139
123140
- bash: pip uninstall -y lightning
@@ -144,17 +161,6 @@ jobs:
144161
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
145162
displayName: "Testing: PyTorch doctests"
146163

147-
- bash: |
148-
python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_pytorch" \
149-
--source_import="lightning.fabric,lightning.pytorch" \
150-
--target_import="lightning_fabric,pytorch_lightning"
151-
python .actions/assistant.py copy_replace_imports --source_dir="./examples/pytorch/basics" \
152-
--source_import="lightning.fabric,lightning.pytorch" \
153-
--target_import="lightning_fabric,pytorch_lightning"
154-
# without succeeded this could run even if the job has already failed
155-
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'pytorch'))
156-
displayName: "Adjust tests & examples"
157-
158164
- bash: |
159165
bash .actions/pull_legacy_checkpoints.sh
160166
cd tests/legacy

‎.github/CONTRIBUTING.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,11 @@ In case you are adding new dependencies, make sure that they are compatible with
130130

131131
### Documentation
132132

133-
To learn about development of docs, check out the docs [README.md](https://github.com/Lightning-AI/lightning/blob/master/docs/README.md).
133+
To learn about development of docs, check out the docs [README.md](https://github.com/Lightning-AI/pytorch-lightning/blob/master/docs/README.md).
134134

135135
### Testing
136136

137-
To learn about tests, check out the tests [README.md](https://github.com/Lightning-AI/lightning/blob/master/tests/README.md).
137+
To learn about tests, check out the tests [README.md](https://github.com/Lightning-AI/pytorch-lightning/blob/master/tests/README.md).
138138

139139
### Pull Request
140140

@@ -165,8 +165,8 @@ We welcome any useful contribution! For your convenience here's a recommended wo
165165

166166
1. If any of the existing tests fail in your PR on our CI, refer to the following READMEs to identify what's failing and try to address it.
167167

168-
- [Test README](https://github.com/Lightning-AI/lightning/blob/master/tests/README.md)
169-
- [CI/CD README](https://github.com/Lightning-AI/lightning/blob/master/.github/workflows/README.md)
168+
- [Test README](https://github.com/Lightning-AI/pytorch-lightning/blob/master/tests/README.md)
169+
- [CI/CD README](https://github.com/Lightning-AI/pytorch-lightning/tree/master/.github/workflows#readme)
170170

171171
1. When you feel ready for integrating your work, mark your PR "Ready for review".
172172

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Fixes #\<issue_number>
1818
<summary><b>Before submitting</b></summary>
1919

2020
- Was this **discussed/agreed** via a GitHub issue? (not for typos and docs)
21-
- [ ] Did you read the [contributor guideline](https://github.com/Lightning-AI/lightning/blob/master/.github/CONTRIBUTING.md), **Pull Request** section?
21+
- [ ] Did you read the [contributor guideline](https://github.com/Lightning-AI/pytorch-lightning/blob/master/.github/CONTRIBUTING.md), **Pull Request** section?
2222
- [ ] Did you make sure your **PR does only one thing**, instead of bundling different changes together?
2323
- Did you make sure to **update the documentation** with your changes? (if necessary)
2424
- Did you write any **new necessary tests**? (not for typos and docs)

‎.github/checkgroup.yml‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ subprojects:
7171
# paths:
7272
# # tpu CI availability is very limited, so we only require tpu tests
7373
# # to pass when their configurations are modified
74-
# - ".github/workflows/tpu-tests.yml"
74+
# - ".github/workflows/tpu-tests.yml.disabled"
7575
# - "tests/tests_pytorch/run_tpu_tests.sh"
7676
# checks:
7777
# - "test-on-tpus (pytorch, pjrt, v4-8)"
@@ -176,14 +176,15 @@ subprojects:
176176
- "lightning-fabric (GPUs) (testing Fabric | latest)"
177177
- "lightning-fabric (GPUs) (testing Lightning | latest)"
178178

179-
- id: "lightning_fabric: TPU workflow"
180-
paths:
181-
# tpu CI availability is very limited, so we only require tpu tests
182-
# to pass when their configurations are modified
183-
- ".github/workflows/tpu-tests.yml"
184-
- "tests/tests_fabric/run_tpu_tests.sh"
185-
checks:
186-
- "test-on-tpus (pytorch, pjrt, v4-8)"
179+
# Temporarily disabled
180+
# - id: "lightning_fabric: TPU workflow"
181+
# paths:
182+
# # tpu CI availability is very limited, so we only require tpu tests
183+
# # to pass when their configurations are modified
184+
# - ".github/workflows/tpu-tests.yml.disabled"
185+
# - "tests/tests_fabric/run_tpu_tests.sh"
186+
# checks:
187+
# - "test-on-tpus (pytorch, pjrt, v4-8)"
187188

188189
# SECTION: common
189190

‎.github/markdown-links-config.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
"ignorePatterns": [
33
{
44
"pattern": "^https://github.com/Lightning-AI/pytorch-lightning/pull/"
5+
},
6+
{
7+
"pattern": "^https://dev.azure.com/Lightning-AI/lightning/_apis/build/status"
8+
},
9+
{
10+
"pattern": "^https://codecov.io/gh/Lightning-AI/pytorch-lightning/graph/badge.svg"
511
}
612
],
713
"httpHeaders": [

‎.github/workflows/_legacy-checkpoints.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,13 @@ jobs:
111111
retention-days: ${{ env.KEEP_DAYS }}
112112
include-hidden-files: true
113113

114+
- name: access secrets
115+
# export to env bool if secrets.AWS_REGION is not empty
116+
run: echo "WITH_SECRETS=$([ -n '${{ secrets.AWS_REGION }}' ] && echo 1 || echo 0)" >> $GITHUB_ENV
117+
114118
- run: pip install -r requirements/ci.txt
115119
- name: Upload checkpoints to S3
116-
if: ${{ secrets[AWS_REGION] != '' }}
120+
if: ${{ env.WITH_SECRETS == '1' }}
117121
working-directory: ${{ env.LEGACY_FOLDER }}
118122
env:
119123
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
@@ -145,13 +149,11 @@ jobs:
145149
title: Adding test for legacy checkpoint created with ${{ env.PL_VERSION }}
146150
committer: GitHub <[email protected]>
147151
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
148-
commit-message: "adding `${{ env.PL_VERSION }}` checkpoint"
149-
body: "**This is automated addition of created checkpoints with the latest `lightning` release!**"
152+
body: "**This is automated addition of created checkpoint with the latest `${{ env.PL_VERSION }}` `lightning` release!**"
150153
delete-branch: true
151154
token: ${{ secrets.PAT_GHOST }}
152155
labels: |
153156
checkpointing
154157
tests
155158
pl
156159
assignees: borda
157-
reviewers: borda

0 commit comments

Comments
 (0)