Merge branch 'master' into milesial/master

Author: Borda

.actions/assistant.py

@@ -234,7 +234,7 @@ def _download_frontend(pkg_path: str, version: str = "v0.0.0"):
         response = urllib.request.urlopen(frontend_release_url)
 
         file = tarfile.open(fileobj=response, mode="r|gz")
-        file.extractall(path=download_dir)
+        file.extractall(path=download_dir)  # noqa: S202
 
         shutil.move(download_dir, frontend_dir)
         print("The Lightning UI has successfully been downloaded!")
@@ -442,9 +442,20 @@ def pull_docs_files(
         target_dir: str = "docs/source-pytorch/XXX",
         checkout: str = "refs/tags/1.0.0",
         source_dir: str = "docs/source",
+        single_page: Optional[str] = None,
         as_orphan: bool = False,
     ) -> None:
-        """Pull docs pages from external source and append to local docs."""
+        """Pull docs pages from external source and append to local docs.
+
+        Args:
+            gh_user_repo: standard GitHub user/repo string
+            target_dir: relative location inside the docs folder
+            checkout: specific tag or branch to checkout
+            source_dir: relative location inside the remote / external repo
+            single_page: copy only single page from the remote repo and name it as the repo name
+            as_orphan: append orphan statement to the page
+
+        """
         import zipfile
 
         zip_url = f"https://github.com/{gh_user_repo}/archive/{checkout}.zip"
@@ -457,13 +468,21 @@ def pull_docs_files(
                 raise RuntimeError(f"Requesting file '{zip_url}' does not exist or it is just unavailable.")
 
             with zipfile.ZipFile(zip_file, "r") as zip_ref:
-                zip_ref.extractall(tmp)
+                zip_ref.extractall(tmp)  # noqa: S202
 
             zip_dirs = [d for d in glob.glob(os.path.join(tmp, "*")) if os.path.isdir(d)]
             # check that the extracted archive has only repo folder
             assert len(zip_dirs) == 1
             repo_dir = zip_dirs[0]
 
+            if single_page:  # special case for copying single page
+                single_page = os.path.join(repo_dir, source_dir, single_page)
+                assert os.path.isfile(single_page), f"File '{single_page}' does not exist."
+                name = re.sub(r"lightning[-_]?", "", gh_user_repo.split("/")[-1])
+                new_rst = os.path.join(_PROJECT_ROOT, target_dir, f"{name}.rst")
+                AssistantCLI._copy_rst(single_page, new_rst, as_orphan=as_orphan)
+                return
+            # continue with copying all pages
             ls_pages = glob.glob(os.path.join(repo_dir, source_dir, "*.rst"))
             ls_pages += glob.glob(os.path.join(repo_dir, source_dir, "**", "*.rst"))
             for rst in ls_pages:

.actions/requirements.txt

@@ -1,2 +1,2 @@
-jsonargparse>=4.16.0
+jsonargparse >=4.16.0, <4.28.0
 requests

.azure/gpu-benchmarks.yml

@@ -46,7 +46,8 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.8.0"
+      # TODO: Upgrade to Python 3.11
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
       options: "--gpus=all --shm-size=32g"
     strategy:
       matrix:

.azure/gpu-tests-fabric.yml

@@ -23,8 +23,7 @@ pr:
       - ".azure/gpu-tests-fabric.yml"
       - "examples/fabric/**"
       - "examples/run_fabric_examples.sh"
-      - "tests/tests_fabric/run_standalone_*.sh"
-      - "tests/tests_pytorch/run_standalone_tests.sh" # used by fabric through a symlink
+      - "tests/run_standalone_*.sh"
       - "requirements/fabric/**"
       - "src/lightning/__init__.py"
       - "src/lightning/__setup__.py"
@@ -49,18 +48,20 @@ jobs:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
       FREEZE_REQUIREMENTS: "1"
       PIP_CACHE_DIR: "/var/tmp/pip"
+      PL_RUN_CUDA_TESTS: "1"
     container:
       image: $(image)
       # default shm size is 64m. Increase it to avoid:
       # 'Error while creating shared memory: unhandled system error, NCCL version 2.7.8'
       options: "--gpus=all --shm-size=2gb  -v /var/tmp:/var/tmp"
     strategy:
       matrix:
+        # TODO: Upgrade to Python 3.11
         "Fabric | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "fabric"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     workspace:
       clean: all
@@ -73,6 +74,10 @@ jobs:
           scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
           echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
         displayName: "set env. vars"
+      - bash: |
+          echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}/torch_test.html"
+        condition: endsWith(variables['Agent.JobName'], 'future')
+        displayName: "set env. vars 4 future"
 
       - bash: |
           echo $(DEVICES)
@@ -99,13 +104,14 @@ jobs:
 
       - bash: |
           extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links ${TORCH_URL}
+          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: |
           set -e
           python requirements/collect_env_details.py
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
+          python -c "import bitsandbytes"
         displayName: "Env details"
 
       - bash: python -m pytest lightning_fabric
@@ -126,19 +132,16 @@ jobs:
         condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
         displayName: "Adjust tests & examples"
 
-      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest -v --durations=50
-        workingDirectory: tests/tests_fabric
-        env:
-          PL_RUN_CUDA_TESTS: "1"
+      - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest . -v --durations=50
+        workingDirectory: tests/tests_fabric/
         displayName: "Testing: fabric standard"
         timeoutInMinutes: "10"
 
-      - bash: bash run_standalone_tests.sh
-        workingDirectory: tests/tests_fabric
+      - bash: bash ../run_standalone_tests.sh "."
+        workingDirectory: tests/tests_fabric/
         env:
-          PL_RUN_CUDA_TESTS: "1"
           PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
-        displayName: "Testing: fabric standalone tests"
+        displayName: "Testing: fabric standalone"
         timeoutInMinutes: "10"
 
       - bash: |
@@ -152,12 +155,12 @@ jobs:
           ./codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) \
             --flags=gpu,pytest,${COVERAGE_SOURCE} --name="GPU-coverage" --env=linux,azure
           ls -l
-        workingDirectory: tests/tests_fabric
+        workingDirectory: tests/tests_fabric/
         displayName: "Statistics"
 
       - script: |
           set -e
           bash run_fabric_examples.sh --accelerator=cuda --devices=1
           bash run_fabric_examples.sh --accelerator=cuda --devices=2 --strategy ddp
-        workingDirectory: examples
+        workingDirectory: examples/
         displayName: "Testing: fabric examples"

.azure/gpu-tests-pytorch.yml

@@ -48,17 +48,19 @@ jobs:
     cancelTimeoutInMinutes: "2"
     strategy:
       matrix:
+        # TODO: Upgrade to Python 3.11
         "PyTorch | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "pytorch"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     pool: lit-rtx-3090
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
       FREEZE_REQUIREMENTS: "1"
       PIP_CACHE_DIR: "/var/tmp/pip"
+      PL_RUN_CUDA_TESTS: "1"
     container:
       image: $(image)
       # default shm size is 64m. Increase it to avoid:
@@ -75,6 +77,10 @@ jobs:
           scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
           echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
         displayName: "set env. vars"
+      - bash: |
+          echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}/torch_test.html"
+        condition: endsWith(variables['Agent.JobName'], 'future')
+        displayName: "set env. vars 4 future"
 
       - bash: |
           echo $(DEVICES)
@@ -102,13 +108,13 @@ jobs:
       - bash: |
           pip install -q -r .actions/requirements.txt
           python .actions/assistant.py requirements_prune_pkgs \
-            --packages="[lightning-colossalai,lightning-bagua]" \
+            --packages="[lightning-colossalai]" \
             --req_files="[requirements/_integrations/strategies.txt]"
         displayName: "Prune packages" # these have installation issues
 
       - bash: |
           extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links ${TORCH_URL}
+          pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: pip uninstall -y lightning
@@ -126,6 +132,7 @@ jobs:
           python requirements/collect_env_details.py
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
           python requirements/pytorch/check-avail-extras.py
+          python -c "import bitsandbytes"
         displayName: "Env details"
 
       - bash: python -m pytest pytorch_lightning
@@ -154,16 +161,13 @@ jobs:
 
       - bash: python -m coverage run --source ${COVERAGE_SOURCE} -m pytest -v --durations=50
         workingDirectory: tests/tests_pytorch
-        env:
-          PL_RUN_CUDA_TESTS: "1"
         displayName: "Testing: PyTorch standard"
         timeoutInMinutes: "35"
 
-      - bash: bash run_standalone_tests.sh
+      - bash: bash ../run_standalone_tests.sh "."
         workingDirectory: tests/tests_pytorch
         env:
           PL_USE_MOCKED_MNIST: "1"
-          PL_RUN_CUDA_TESTS: "1"
           PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
         displayName: "Testing: PyTorch standalone tests"
         timeoutInMinutes: "35"
@@ -172,7 +176,6 @@ jobs:
         workingDirectory: tests/tests_pytorch
         env:
           PL_USE_MOCKED_MNIST: "1"
-          PL_RUN_CUDA_TESTS: "1"
         displayName: "Testing: PyTorch standalone tasks"
         timeoutInMinutes: "10"
 

.github/ISSUE_TEMPLATE/1_bug_report.yaml

@@ -36,6 +36,7 @@ body:
         - "v1.9"
         - "v2.0"
         - "v2.1"
+        - "v2.2"
         - "master"
     validations:
       required: true

.github/actions/pip-wheels/action.yml

@@ -19,7 +19,7 @@ runs:
   using: "composite"
   steps:
     - name: install dev. env
-      run: pip install setuptools wheel
+      run: pip install -r requirements/ci.txt
       shell: bash
 
     - name: Freeze local emv.

.github/actions/pkg-check/action.yml

@@ -14,7 +14,7 @@ runs:
   using: "composite"
   steps:
     - name: install dev. env
-      run: pip install "twine==4.0.1" setuptools wheel flake8
+      run: pip install -r requirements/ci.txt
       shell: bash
 
     - name: Set PACKAGE_NAME envvar

.github/checkgroup.yml

@@ -19,21 +19,18 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pl-cpu (macOS-11, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (macOS-11, lightning, 3.9, 1.12)"
+      - "pl-cpu (macOS-11, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (macOS-11, lightning, 3.10, 1.13)"
-      - "pl-cpu (macOS-11, lightning, 3.10, 2.0)"
       - "pl-cpu (macOS-11, lightning, 3.10, 2.1)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.9, 1.12)"
+      - "pl-cpu (macOS-11, lightning, 3.10, 2.2)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.13)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.0)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "pl-cpu (windows-2022, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (windows-2022, lightning, 3.9, 1.12)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.2)"
+      - "pl-cpu (windows-2022, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (windows-2022, lightning, 3.10, 1.13)"
-      - "pl-cpu (windows-2022, lightning, 3.10, 2.0)"
       - "pl-cpu (windows-2022, lightning, 3.10, 2.1)"
+      - "pl-cpu (windows-2022, lightning, 3.10, 2.2)"
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.13)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.13)"
       - "pl-cpu (windows-2022, pytorch, 3.8, 1.13)"
@@ -95,7 +92,6 @@ subprojects:
       - ".github/workflows/tpu-tests.yml"
       - "tests/tests_pytorch/run_tpu_tests.sh"
     checks:
-      - "test-on-tpus (pytorch, xrt, v4-8)"
       - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   - id: "fabric: Docs"
@@ -144,13 +140,11 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "build-cuda (3.9, 1.12, 11.7.1)"
       - "build-cuda (3.9, 1.13, 11.8.0)"
       - "build-cuda (3.9, 1.13, 12.0.1)"
       - "build-cuda (3.10, 2.0, 11.8.0)"
       - "build-cuda (3.10, 2.1, 12.1.0)"
       #- "build-NGC"
-      - "build-pl (3.9, 1.12, 11.7.1)"
       - "build-pl (3.9, 1.13, 11.8.0)"
       - "build-pl (3.9, 1.13, 12.0.1)"
       - "build-pl (3.10, 2.0, 11.8.0)"
@@ -194,21 +188,18 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "fabric-cpu (macOS-11, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (macOS-11, lightning, 3.9, 1.12)"
+      - "fabric-cpu (macOS-11, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (macOS-11, lightning, 3.10, 1.13)"
-      - "fabric-cpu (macOS-11, lightning, 3.10, 2.0)"
       - "fabric-cpu (macOS-11, lightning, 3.11, 2.1)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.9, 1.12)"
+      - "fabric-cpu (macOS-11, lightning, 3.11, 2.2)"
+      - "fabric-cpu (ubuntu-20.04, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.10, 1.13)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.10, 2.0)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.1)"
-      - "fabric-cpu (windows-2022, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (windows-2022, lightning, 3.9, 1.12)"
+      - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.2)"
+      - "fabric-cpu (windows-2022, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (windows-2022, lightning, 3.10, 1.13)"
-      - "fabric-cpu (windows-2022, lightning, 3.10, 2.0)"
       - "fabric-cpu (windows-2022, lightning, 3.11, 2.1)"
+      - "fabric-cpu (windows-2022, lightning, 3.11, 2.2)"
       - "fabric-cpu (macOS-11, fabric, 3.8, 1.13)"
       - "fabric-cpu (ubuntu-20.04, fabric, 3.8, 1.13)"
       - "fabric-cpu (windows-2022, fabric, 3.8, 1.13)"
@@ -225,8 +216,7 @@ subprojects:
       - ".azure/gpu-tests-fabric.yml"
       - "examples/fabric/**"
       - "examples/run_fabric_examples.sh"
-      - "tests/tests_fabric/run_standalone_*.sh"
-      - "tests/tests_pytorch/run_standalone_tests.sh" # used by Fabric through a symlink
+      - "tests/run_standalone_*.sh"
       - "requirements/fabric/**"
       - "src/lightning/__init__.py"
       - "src/lightning/__setup__.py"
@@ -249,7 +239,6 @@ subprojects:
       - ".github/workflows/tpu-tests.yml"
       - "tests/tests_fabric/run_tpu_tests.sh"
     checks:
-      - "test-on-tpus (fabric, xrt, v4-8)"
       - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   # SECTION: lightning_app