Skip to content

Commit a58039e

Browse files
BordaBorda
authored
Merge branch 'master' into bugfix/gan-ddp-training
2 parents ab9b2dd + 216f9ec commit a58039e

File tree

80 files changed

+1969
-335
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+1969
-335
lines changed

‎.actions/assistant.py‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
341341

342342

343343
class AssistantCLI:
344-
@staticmethod
345-
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
346-
"""Remove some packages from given requirement files."""
347-
if isinstance(req_files, str):
348-
req_files = [req_files]
349-
for req in req_files:
350-
AssistantCLI._prune_packages(req, packages)
351-
352-
@staticmethod
353-
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
354-
"""Remove some packages from given requirement files."""
355-
path = Path(req_file)
356-
assert path.exists()
357-
text = path.read_text()
358-
lines = text.splitlines()
359-
final = []
360-
for line in lines:
361-
ln_ = line.strip()
362-
if not ln_ or ln_.startswith("#"):
363-
final.append(line)
364-
continue
365-
req = list(_parse_requirements([ln_]))[0]
366-
if req.name not in packages:
367-
final.append(line)
368-
print(final)
369-
path.write_text("\n".join(final) + "\n")
370-
371-
@staticmethod
372-
def _replace_min(fname: str) -> None:
373-
with open(fname, encoding="utf-8") as fopen:
374-
req = fopen.read().replace(">=", "==")
375-
with open(fname, "w", encoding="utf-8") as fwrite:
376-
fwrite.write(req)
377-
378-
@staticmethod
379-
def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
380-
"""Replace the min package version by fixed one."""
381-
for fname in requirement_fnames:
382-
print(fname)
383-
AssistantCLI._replace_min(fname)
384-
385344
@staticmethod
386345
def copy_replace_imports(
387346
source_dir: str,

‎.azure/gpu-tests-fabric.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,16 @@ jobs:
9999
displayName: "Image info & NVIDIA"
100100
101101
- bash: |
102-
python .actions/assistant.py replace_oldest_ver
102+
set -ex
103103
pip install "cython<3.0" wheel # for compatibility
104+
pip install -U "lightning-utilities[cli]"
105+
cd requirements/fabric
106+
# replace range by pin minimal requirements
107+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
108+
# drop deepspeed since it is not supported by our minimal Torch requirements
109+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
110+
# uninstall deepspeed since some older docker images have it pre-installed
111+
pip uninstall -y deepspeed
104112
condition: contains(variables['Agent.JobName'], 'oldest')
105113
displayName: "setting oldest dependencies"
106114

‎.azure/gpu-tests-pytorch.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,8 +103,16 @@ jobs:
103103
displayName: "Image info & NVIDIA"
104104
105105
- bash: |
106-
python .actions/assistant.py replace_oldest_ver
106+
set -ex
107107
pip install "cython<3.0" wheel # for compatibility
108+
pip install -U "lightning-utilities[cli]"
109+
cd requirements/pytorch
110+
# replace range by pin minimal requirements
111+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
112+
# drop deepspeed since it is not supported by our minimal Torch requirements
113+
python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
114+
# uninstall deepspeed since some older docker images have it pre-installed
115+
pip uninstall -y deepspeed
108116
condition: contains(variables['Agent.JobName'], 'oldest')
109117
displayName: "setting oldest dependencies"
110118

‎.github/workflows/ci-tests-fabric.yml‎

Lines changed: 32 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -38,42 +38,29 @@ jobs:
3838
strategy:
3939
fail-fast: false
4040
matrix:
41-
include:
41+
os: [macOS-14, ubuntu-22.04, windows-2022]
42+
config:
4243
# only run PyTorch latest
43-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
44-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
45-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
46-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
47-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
48-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
49-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
50-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
51-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
52-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
53-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
54-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
55-
- { os: "macOS-14", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
56-
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
57-
- { os: "windows-2022", pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
44+
- { pkg-name: "lightning", python-version: "3.10", pytorch-version: "2.1" }
45+
- { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.2.2" }
46+
- { pkg-name: "lightning", python-version: "3.11", pytorch-version: "2.3" }
47+
- { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.4.1" }
48+
- { pkg-name: "lightning", python-version: "3.12.7", pytorch-version: "2.5.1" }
49+
5850
# only run PyTorch latest with Python latest, use Fabric scope to limit dependency issues
59-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
60-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
61-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
62-
# "oldest" versions tests, only on minimum Python
63-
- { os: "macOS-14", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
64-
- { os: "ubuntu-22.04", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
65-
- { os: "windows-2022", pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
51+
- { pkg-name: "fabric", python-version: "3.12.7", pytorch-version: "2.6" }
52+
6653
# "fabric" installs the standalone package
67-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
68-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
69-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
54+
- { pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.7" }
55+
7056
# adding recently cut Torch 2.7 - FUTURE
71-
- { os: "macOS-14", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
72-
- { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
73-
- { os: "windows-2022", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
57+
- { pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.8" }
58+
59+
# "oldest" versions tests, only on minimum Python
60+
- { pkg-name: "fabric", pytorch-version: "2.1", requires: "oldest" }
7461
timeout-minutes: 25 # because of building grpcio on Mac
7562
env:
76-
PACKAGE_NAME: ${{ matrix.pkg-name }}
63+
PACKAGE_NAME: ${{ matrix.config.pkg-name }}
7764
FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
7865
PYPI_CACHE_DIR: "_pip-wheels"
7966
TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/"
@@ -83,28 +70,30 @@ jobs:
8370
steps:
8471
- uses: actions/checkout@v5
8572

86-
- name: Set up Python ${{ matrix.python-version }}
73+
- name: Set up Python ${{ matrix.config.python-version }}
8774
uses: actions/setup-python@v5
8875
with:
89-
python-version: ${{ matrix.python-version || '3.9' }}
76+
python-version: ${{ matrix.config.python-version || '3.9' }}
9077

9178
- name: basic setup
9279
run: pip install -q -r .actions/requirements.txt
9380

9481
- name: Set min. dependencies
95-
if: ${{ matrix.requires == 'oldest' }}
82+
if: ${{ matrix.config.requires == 'oldest' }}
9683
run: |
97-
python .actions/assistant.py replace_oldest_ver
84+
cd requirements/fabric
85+
pip install -U "lightning-utilities[cli]"
86+
python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt', 'test.txt']"
9887
pip install "cython<3.0" wheel
9988
pip install "pyyaml==5.4" --no-build-isolation
10089
10190
- name: Adjust PyTorch versions in requirements files
102-
if: ${{ matrix.requires != 'oldest' }}
91+
if: ${{ matrix.config.requires != 'oldest' }}
10392
run: |
10493
pip install -q -r requirements/ci.txt
10594
python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
10695
for fpath in `ls requirements/**/*.txt`; do \
107-
python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \
96+
python ./adjust-torch-versions.py $fpath ${{ matrix.config.pytorch-version }}; \
10897
done
10998
11099
- name: pip wheels cache
@@ -119,11 +108,11 @@ jobs:
119108
- name: Expand Env. variables
120109
run: |
121110
# Switch PyTorch URL between stable and test/future
122-
python -c "print('TORCH_URL=' + str('${{env.TORCH_URL_TEST}}' if '${{ matrix.pytorch-version }}' == '2.7' else '${{env.TORCH_URL_STABLE}}'))" >> $GITHUB_ENV
111+
python -c "print('TORCH_URL=' + str('${{env.TORCH_URL_TEST}}' if '${{ matrix.config.pytorch-version }}' == '2.7' else '${{env.TORCH_URL_STABLE}}'))" >> $GITHUB_ENV
123112
# Switch coverage scope
124-
python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_fabric'))" >> $GITHUB_ENV
113+
python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.config.pkg-name}}' == 'lightning' else 'lightning_fabric'))" >> $GITHUB_ENV
125114
# if you install mono-package set dependency only for this subpackage
126-
python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.pkg-name}}' != 'lightning' else 'fabric-'))" >> $GITHUB_ENV
115+
python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.config.pkg-name}}' != 'lightning' else 'fabric-'))" >> $GITHUB_ENV
127116
- name: Append Env. vars for MacOS
128117
if: ${{ runner.os == 'macOS' }}
129118
run: |
@@ -140,7 +129,8 @@ jobs:
140129
run: |
141130
pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
142131
-U --upgrade-strategy=eager --prefer-binary \
143-
--extra-index-url="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}"
132+
--extra-index-url="${TORCH_URL}" \
133+
--find-links="${PYPI_CACHE_DIR}"
144134
pip list
145135
- name: Dump handy wheels
146136
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
@@ -152,7 +142,7 @@ jobs:
152142
cache-key: "pypi_wheels"
153143

154144
- name: Adjust tests
155-
if: ${{ matrix.pkg-name != 'lightning' }}
145+
if: ${{ matrix.config.pkg-name != 'lightning' }}
156146
run: |
157147
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
158148
--source_import="lightning.fabric" --target_import="lightning_fabric"
@@ -185,7 +175,7 @@ jobs:
185175
with:
186176
token: ${{ secrets.CODECOV_TOKEN }}
187177
file: tests/tests_fabric/coverage.xml
188-
flags: ${{ env.COVERAGE_SCOPE }},cpu,pytest,python${{ matrix.python-version }}
178+
flags: ${{ env.COVERAGE_SCOPE }},cpu,pytest,python${{ matrix.config.python-version }}
189179
name: CPU-coverage
190180
fail_ci_if_error: false
191181

0 commit comments

Comments
 (0)