Merge branch 'master' into throughput-callback-variable-batch-size

Author: SkafteNicki

.actions/assistant.py

@@ -18,10 +18,11 @@
 import shutil
 import tempfile
 import urllib.request
+from collections.abc import Iterable, Iterator, Sequence
 from itertools import chain
 from os.path import dirname, isfile
 from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
+from typing import Any, Optional
 
 from packaging.requirements import Requirement
 from packaging.version import Version
@@ -127,7 +128,7 @@ def _parse_requirements(lines: Iterable[str]) -> Iterator[_RequirementWithCommen
         pip_argument = None
 
 
-def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> List[str]:
+def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> list[str]:
     """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
@@ -153,8 +154,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
 
     """
     path_readme = os.path.join(path_dir, "README.md")
-    with open(path_readme, encoding="utf-8") as fo:
-        text = fo.read()
+    with open(path_readme, encoding="utf-8") as fopen:
+        text = fopen.read()
 
     # drop images from readme
     text = text.replace(
@@ -222,7 +223,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
         fp.writelines([ln + os.linesep for ln in requires] + [os.linesep])
 
 
-def _retrieve_files(directory: str, *ext: str) -> List[str]:
+def _retrieve_files(directory: str, *ext: str) -> list[str]:
     all_files = []
     for root, _, files in os.walk(directory):
         for fname in files:
@@ -232,7 +233,7 @@ def _retrieve_files(directory: str, *ext: str) -> List[str]:
     return all_files
 
 
-def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning_by: str = "") -> List[str]:
+def _replace_imports(lines: list[str], mapping: list[tuple[str, str]], lightning_by: str = "") -> list[str]:
     """Replace imports of standalone package to lightning.
 
     >>> lns = [
@@ -307,20 +308,20 @@ def copy_replace_imports(
         if ext in (".pyc",):
             continue
         # Try to parse everything else
-        with open(fp, encoding="utf-8") as fo:
+        with open(fp, encoding="utf-8") as fopen:
             try:
-                lines = fo.readlines()
+                lines = fopen.readlines()
             except UnicodeDecodeError:
                 # a binary file, skip
                 print(f"Skipped replacing imports for {fp}")
                 continue
         lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
         os.makedirs(os.path.dirname(fp_new), exist_ok=True)
-        with open(fp_new, "w", encoding="utf-8") as fo:
-            fo.writelines(lines)
+        with open(fp_new, "w", encoding="utf-8") as fopen:
+            fopen.writelines(lines)
 
 
-def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None:
+def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
     """Create a mirror package with adjusted imports."""
     # replace imports and copy the code
     mapping = package_mapping.copy()
@@ -340,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> N
 
 
 class AssistantCLI:
-    @staticmethod
-    def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
-        """Remove some packages from given requirement files."""
-        if isinstance(req_files, str):
-            req_files = [req_files]
-        for req in req_files:
-            AssistantCLI._prune_packages(req, packages)
-
-    @staticmethod
-    def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
-        """Remove some packages from given requirement files."""
-        path = Path(req_file)
-        assert path.exists()
-        text = path.read_text()
-        lines = text.splitlines()
-        final = []
-        for line in lines:
-            ln_ = line.strip()
-            if not ln_ or ln_.startswith("#"):
-                final.append(line)
-                continue
-            req = list(_parse_requirements([ln_]))[0]
-            if req.name not in packages:
-                final.append(line)
-        print(final)
-        path.write_text("\n".join(final) + "\n")
-
-    @staticmethod
-    def _replace_min(fname: str) -> None:
-        with open(fname, encoding="utf-8") as fo:
-            req = fo.read().replace(">=", "==")
-        with open(fname, "w", encoding="utf-8") as fw:
-            fw.write(req)
-
-    @staticmethod
-    def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
-        """Replace the min package version by fixed one."""
-        for fname in requirement_fnames:
-            print(fname)
-            AssistantCLI._replace_min(fname)
-
     @staticmethod
     def copy_replace_imports(
         source_dir: str,
@@ -470,15 +430,53 @@ def convert_version2nightly(ver_file: str = "src/version.info") -> None:
         """Load the actual version and convert it to the nightly version."""
         from datetime import datetime
 
-        with open(ver_file) as fo:
-            version = fo.read().strip()
+        with open(ver_file) as fopen:
+            version = fopen.read().strip()
         # parse X.Y.Z version and prune any suffix
         vers = re.match(r"(\d+)\.(\d+)\.(\d+).*", version)
         # create timestamp  YYYYMMDD
         timestamp = datetime.now().strftime("%Y%m%d")
         version = f"{'.'.join(vers.groups())}.dev{timestamp}"
-        with open(ver_file, "w") as fo:
-            fo.write(version + os.linesep)
+        with open(ver_file, "w") as fopen:
+            fopen.write(version + os.linesep)
+
+    @staticmethod
+    def generate_docker_tags(
+        release_version: str,
+        python_version: str,
+        torch_version: str,
+        cuda_version: str,
+        docker_project: str = "pytorchlightning/pytorch_lightning",
+        add_latest: bool = False,
+    ) -> None:
+        """Generate docker tags for the given versions."""
+        tags = [f"latest-py{python_version}-torch{torch_version}-cuda{cuda_version}"]
+        if release_version:
+            tags += [f"{release_version}-py{python_version}-torch{torch_version}-cuda{cuda_version}"]
+        if add_latest:
+            tags += ["latest"]
+
+        tags = [f"{docker_project}:{tag}" for tag in tags]
+        print(",".join(tags))
+
+    @staticmethod
+    def prune_pytest_as_errors(
+        pyproject_toml: str = "pyproject.toml", errors: tuple = ("FutureWarning", "DeprecationWarning")
+    ) -> None:
+        """Prune pytest warnings as errors from the pyproject.toml file."""
+        import tomlkit
+
+        with open(pyproject_toml, encoding="utf-8") as fopen:
+            content = fopen.read()
+        pyproject = tomlkit.parse(content)
+        filterwarnings = pyproject.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("filterwarnings", [])
+        if not filterwarnings:
+            return
+        filterwarnings = [wrn for wrn in filterwarnings if not any(f"error::{err}" in wrn for err in errors)]
+        pyproject["tool"]["pytest"]["ini_options"]["filterwarnings"] = filterwarnings
+
+        with open(pyproject_toml, "w", encoding="utf-8") as fopen:
+            fopen.write(tomlkit.dumps(pyproject))
 
 
 if __name__ == "__main__":

.actions/requirements.txt

@@ -1,3 +1,3 @@
-jsonargparse >=4.16.0, <4.28.0
+jsonargparse
 requests
 packaging

.azure/gpu-benchmarks.yml

@@ -46,7 +46,7 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.4-cuda12.1.0"
+      image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
       options: "--gpus=all --shm-size=32g"
     strategy:
       matrix:
@@ -75,6 +75,13 @@ jobs:
           pip list
         displayName: "Image info & NVIDIA"
 
+      - bash: |
+          pip install -U -q -r .actions/requirements.txt
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning.fabric,lightning.pytorch" \
+            --target_import="lightning_fabric,pytorch_lightning"
+        displayName: "Adjust tests"
+
       - bash: pip install -e .[dev] --find-links ${TORCH_URL}
         env:
           FREEZE_REQUIREMENTS: "1"
@@ -86,17 +93,10 @@ jobs:
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
         displayName: "Env details"
 
-      - bash: |
-          pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
-            --source_import="lightning.fabric,lightning.pytorch" \
-            --target_import="lightning_fabric,pytorch_lightning"
-        displayName: "Adjust tests"
-
       - bash: python -m pytest parity_$(PACKAGE_NAME) -v --durations=0
         env:
           PL_RUNNING_BENCHMARKS: "1"
-          PL_RUN_CUDA_TESTS: "1"
+          RUN_ONLY_CUDA_TESTS: "1"
         workingDirectory: tests/
         displayName: "Testing: benchmarks"
 
@@ -105,6 +105,7 @@ jobs:
         # without succeeded this could run even if the job has already failed
         condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
         env:
-          PL_RUN_CUDA_TESTS: "1"
+          RUN_ONLY_CUDA_TESTS: "1"
+          PL_RUN_STANDALONE_TESTS: "1"
         displayName: "Testing: fabric standalone tasks"
         timeoutInMinutes: "10"

.azure/gpu-tests-fabric.yml

@@ -48,19 +48,25 @@ jobs:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
       FREEZE_REQUIREMENTS: "1"
       PIP_CACHE_DIR: "/var/tmp/pip"
-      PL_RUN_CUDA_TESTS: "1"
+      RUN_ONLY_CUDA_TESTS: "1"
     container:
       image: $(image)
       # default shm size is 64m. Increase it to avoid:
       # 'Error while creating shared memory: unhandled system error, NCCL version 2.7.8'
       options: "--gpus=all --shm-size=2gb  -v /var/tmp:/var/tmp"
     strategy:
       matrix:
+        "Fabric | oldest":
+          image: "pytorchlightning/pytorch_lightning:base-cuda12.1.1-py3.10-torch2.1"
+          PACKAGE_NAME: "fabric"
         "Fabric | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.11-torch2.3-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
           PACKAGE_NAME: "fabric"
+        #"Fabric | future":
+        #  image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.7"
+        #  PACKAGE_NAME: "fabric"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.4-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda12.6.3-py3.12-torch2.8"
           PACKAGE_NAME: "lightning"
     workspace:
       clean: all
@@ -72,14 +78,11 @@ jobs:
           echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
           scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
           echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
-          python_ver=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
-          echo "##vso[task.setvariable variable=PYTHON_VERSION_MM]$python_ver"
         displayName: "set env. vars"
       - bash: |
           echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
-          echo "##vso[task.setvariable variable=TORCHVISION_URL]https://download.pytorch.org/whl/test/cu124/torchvision-0.19.0%2Bcu124-cp${PYTHON_VERSION_MM}-cp${PYTHON_VERSION_MM}-linux_x86_64.whl"
         condition: endsWith(variables['Agent.JobName'], 'future')
-        displayName: "set env. vars 4 future"
+        displayName: "extend env. vars 4 future"
 
       - bash: |
           echo $(DEVICES)
@@ -95,6 +98,20 @@ jobs:
           pip list
         displayName: "Image info & NVIDIA"
 
+      - bash: |
+          set -ex
+          pip install "cython<3.0" wheel  # for compatibility
+          pip install -U "lightning-utilities[cli]"
+          cd requirements/fabric
+          # replace range by pin minimal requirements
+          python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
+          # drop deepspeed since it is not supported by our minimal Torch requirements
+          python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
+          # uninstall deepspeed since some older docker images have it pre-installed
+          pip uninstall -y deepspeed
+        condition: contains(variables['Agent.JobName'], 'oldest')
+        displayName: "setting oldest dependencies"
+
       - bash: |
           PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
           pip install -q wget packaging
@@ -105,14 +122,28 @@ jobs:
         displayName: "Adjust dependencies"
 
       - bash: |
+          pip install -U -q -r .actions/requirements.txt
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
+            --source_import="lightning.fabric" \
+            --target_import="lightning_fabric"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
+            --source_import="lightning.fabric" \
+            --target_import="lightning_fabric"
+        # without succeeded this could run even if the job has already failed
+        condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
+        displayName: "Adjust tests & examples"
+
+      - bash: |
+          set -e
           extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}" --find-links="${TORCHVISION_URL}"
+          pip install -e ".[${extra}dev]" -U --upgrade-strategy=eager --extra-index-url="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: |
           set -e
           python requirements/collect_env_details.py
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
+          python requirements/pytorch/check-avail-extras.py
           python -c "import bitsandbytes"
         displayName: "Env details"
 
@@ -122,27 +153,17 @@ jobs:
         condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
         displayName: "Testing: Fabric doctests"
 
-      - bash: |
-          pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py copy_replace_imports --source_dir="./tests/tests_fabric" \
-            --source_import="lightning.fabric" \
-            --target_import="lightning_fabric"
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples/fabric" \
-            --source_import="lightning.fabric" \
-            --target_import="lightning_fabric"
-        # without succeeded this could run even if the job has already failed
-        condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
-        displayName: "Adjust tests & examples"
-
-      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest . -v --durations=50
-        workingDirectory: tests/tests_fabric/
+      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest tests_fabric/ -v --durations=50
+        workingDirectory: tests/
         displayName: "Testing: fabric standard"
         timeoutInMinutes: "10"
 
-      - bash: bash ../run_standalone_tests.sh "."
-        workingDirectory: tests/tests_fabric/
+      - bash: |
+          wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/run_standalone_tests.sh
+          bash ./run_standalone_tests.sh "tests_fabric"
+        workingDirectory: tests/
         env:
-          PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
+          PL_RUN_STANDALONE_TESTS: "1"
         displayName: "Testing: fabric standalone"
         timeoutInMinutes: "10"
 
@@ -157,7 +178,7 @@ jobs:
           ./codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) \
             --flags=gpu,pytest,${COVERAGE_SOURCE} --name="GPU-coverage" --env=linux,azure
           ls -l
-        workingDirectory: tests/tests_fabric/
+        workingDirectory: tests/
         displayName: "Statistics"
 
       - script: |