Merge branch 'master' into bug/18727_reset_trainer_should_stop_on_fit

Author: Borda

.actions/assistant.py

@@ -234,7 +234,7 @@ def _download_frontend(pkg_path: str, version: str = "v0.0.0"):
         response = urllib.request.urlopen(frontend_release_url)
 
         file = tarfile.open(fileobj=response, mode="r|gz")
-        file.extractall(path=download_dir)
+        file.extractall(path=download_dir)  # noqa: S202
 
         shutil.move(download_dir, frontend_dir)
         print("The Lightning UI has successfully been downloaded!")
@@ -442,9 +442,20 @@ def pull_docs_files(
         target_dir: str = "docs/source-pytorch/XXX",
         checkout: str = "refs/tags/1.0.0",
         source_dir: str = "docs/source",
+        single_page: Optional[str] = None,
         as_orphan: bool = False,
     ) -> None:
-        """Pull docs pages from external source and append to local docs."""
+        """Pull docs pages from external source and append to local docs.
+
+        Args:
+            gh_user_repo: standard GitHub user/repo string
+            target_dir: relative location inside the docs folder
+            checkout: specific tag or branch to checkout
+            source_dir: relative location inside the remote / external repo
+            single_page: copy only single page from the remote repo and name it as the repo name
+            as_orphan: append orphan statement to the page
+
+        """
         import zipfile
 
         zip_url = f"https://github.com/{gh_user_repo}/archive/{checkout}.zip"
@@ -457,13 +468,21 @@ def pull_docs_files(
                 raise RuntimeError(f"Requesting file '{zip_url}' does not exist or it is just unavailable.")
 
             with zipfile.ZipFile(zip_file, "r") as zip_ref:
-                zip_ref.extractall(tmp)
+                zip_ref.extractall(tmp)  # noqa: S202
 
             zip_dirs = [d for d in glob.glob(os.path.join(tmp, "*")) if os.path.isdir(d)]
             # check that the extracted archive has only repo folder
             assert len(zip_dirs) == 1
             repo_dir = zip_dirs[0]
 
+            if single_page:  # special case for copying single page
+                single_page = os.path.join(repo_dir, source_dir, single_page)
+                assert os.path.isfile(single_page), f"File '{single_page}' does not exist."
+                name = re.sub(r"lightning[-_]?", "", gh_user_repo.split("/")[-1])
+                new_rst = os.path.join(_PROJECT_ROOT, target_dir, f"{name}.rst")
+                AssistantCLI._copy_rst(single_page, new_rst, as_orphan=as_orphan)
+                return
+            # continue with copying all pages
             ls_pages = glob.glob(os.path.join(repo_dir, source_dir, "*.rst"))
             ls_pages += glob.glob(os.path.join(repo_dir, source_dir, "**", "*.rst"))
             for rst in ls_pages:

.azure/gpu-benchmarks.yml

@@ -46,7 +46,8 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.0-cuda11.8.0"
+      # TODO: Upgrade to Python 3.11
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
       options: "--gpus=all --shm-size=32g"
     strategy:
       matrix:

.azure/gpu-tests-fabric.yml

@@ -56,11 +56,12 @@ jobs:
       options: "--gpus=all --shm-size=2gb  -v /var/tmp:/var/tmp"
     strategy:
       matrix:
+        # TODO: Upgrade to Python 3.11
         "Fabric | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "fabric"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     workspace:
       clean: all
@@ -73,6 +74,10 @@ jobs:
           scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(fabric="lightning_fabric").get(n, n))')
           echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
         displayName: "set env. vars"
+      - bash: |
+          echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}/torch_test.html"
+        condition: endsWith(variables['Agent.JobName'], 'future')
+        displayName: "set env. vars 4 future"
 
       - bash: |
           echo $(DEVICES)
@@ -99,13 +104,14 @@ jobs:
 
       - bash: |
           extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links ${TORCH_URL}
+          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: |
           set -e
           python requirements/collect_env_details.py
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
+          python -c "import bitsandbytes"
         displayName: "Env details"
 
       - bash: python -m pytest lightning_fabric

.azure/gpu-tests-pytorch.yml

@@ -48,11 +48,12 @@ jobs:
     cancelTimeoutInMinutes: "2"
     strategy:
       matrix:
+        # TODO: Upgrade to Python 3.11
         "PyTorch | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "pytorch"
         "Lightning | latest":
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.0"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
           PACKAGE_NAME: "lightning"
     pool: lit-rtx-3090
     variables:
@@ -76,6 +77,10 @@ jobs:
           scope=$(python -c 'n = "$(PACKAGE_NAME)" ; print(dict(pytorch="pytorch_lightning").get(n, n))')
           echo "##vso[task.setvariable variable=COVERAGE_SOURCE]$scope"
         displayName: "set env. vars"
+      - bash: |
+          echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}/torch_test.html"
+        condition: endsWith(variables['Agent.JobName'], 'future')
+        displayName: "set env. vars 4 future"
 
       - bash: |
           echo $(DEVICES)
@@ -103,13 +108,13 @@ jobs:
       - bash: |
           pip install -q -r .actions/requirements.txt
           python .actions/assistant.py requirements_prune_pkgs \
-            --packages="[lightning-colossalai,lightning-bagua]" \
+            --packages="[lightning-colossalai]" \
             --req_files="[requirements/_integrations/strategies.txt]"
         displayName: "Prune packages" # these have installation issues
 
       - bash: |
           extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links ${TORCH_URL}
+          pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: pip uninstall -y lightning
@@ -127,6 +132,7 @@ jobs:
           python requirements/collect_env_details.py
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
           python requirements/pytorch/check-avail-extras.py
+          python -c "import bitsandbytes"
         displayName: "Env details"
 
       - bash: python -m pytest pytorch_lightning

.github/ISSUE_TEMPLATE/1_bug_report.yaml

@@ -36,6 +36,7 @@ body:
         - "v1.9"
         - "v2.0"
         - "v2.1"
+        - "v2.2"
         - "master"
     validations:
       required: true

.github/checkgroup.yml

@@ -19,21 +19,18 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pl-cpu (macOS-11, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (macOS-11, lightning, 3.9, 1.12)"
+      - "pl-cpu (macOS-11, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (macOS-11, lightning, 3.10, 1.13)"
-      - "pl-cpu (macOS-11, lightning, 3.10, 2.0)"
       - "pl-cpu (macOS-11, lightning, 3.10, 2.1)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.9, 1.12)"
+      - "pl-cpu (macOS-11, lightning, 3.10, 2.2)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.13)"
-      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.0)"
       - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "pl-cpu (windows-2022, lightning, 3.8, 1.12, oldest)"
-      - "pl-cpu (windows-2022, lightning, 3.9, 1.12)"
+      - "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.2)"
+      - "pl-cpu (windows-2022, lightning, 3.8, 1.13, oldest)"
       - "pl-cpu (windows-2022, lightning, 3.10, 1.13)"
-      - "pl-cpu (windows-2022, lightning, 3.10, 2.0)"
       - "pl-cpu (windows-2022, lightning, 3.10, 2.1)"
+      - "pl-cpu (windows-2022, lightning, 3.10, 2.2)"
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.13)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.13)"
       - "pl-cpu (windows-2022, pytorch, 3.8, 1.13)"
@@ -95,7 +92,6 @@ subprojects:
       - ".github/workflows/tpu-tests.yml"
       - "tests/tests_pytorch/run_tpu_tests.sh"
     checks:
-      - "test-on-tpus (pytorch, xrt, v4-8)"
       - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   - id: "fabric: Docs"
@@ -144,13 +140,11 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "build-cuda (3.9, 1.12, 11.7.1)"
       - "build-cuda (3.9, 1.13, 11.8.0)"
       - "build-cuda (3.9, 1.13, 12.0.1)"
       - "build-cuda (3.10, 2.0, 11.8.0)"
       - "build-cuda (3.10, 2.1, 12.1.0)"
       #- "build-NGC"
-      - "build-pl (3.9, 1.12, 11.7.1)"
       - "build-pl (3.9, 1.13, 11.8.0)"
       - "build-pl (3.9, 1.13, 12.0.1)"
       - "build-pl (3.10, 2.0, 11.8.0)"
@@ -194,21 +188,18 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "fabric-cpu (macOS-11, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (macOS-11, lightning, 3.9, 1.12)"
+      - "fabric-cpu (macOS-11, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (macOS-11, lightning, 3.10, 1.13)"
-      - "fabric-cpu (macOS-11, lightning, 3.10, 2.0)"
       - "fabric-cpu (macOS-11, lightning, 3.11, 2.1)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.9, 1.12)"
+      - "fabric-cpu (macOS-11, lightning, 3.11, 2.2)"
+      - "fabric-cpu (ubuntu-20.04, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.10, 1.13)"
-      - "fabric-cpu (ubuntu-20.04, lightning, 3.10, 2.0)"
       - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.1)"
-      - "fabric-cpu (windows-2022, lightning, 3.8, 1.12, oldest)"
-      - "fabric-cpu (windows-2022, lightning, 3.9, 1.12)"
+      - "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.2)"
+      - "fabric-cpu (windows-2022, lightning, 3.8, 1.13, oldest)"
       - "fabric-cpu (windows-2022, lightning, 3.10, 1.13)"
-      - "fabric-cpu (windows-2022, lightning, 3.10, 2.0)"
       - "fabric-cpu (windows-2022, lightning, 3.11, 2.1)"
+      - "fabric-cpu (windows-2022, lightning, 3.11, 2.2)"
       - "fabric-cpu (macOS-11, fabric, 3.8, 1.13)"
       - "fabric-cpu (ubuntu-20.04, fabric, 3.8, 1.13)"
       - "fabric-cpu (windows-2022, fabric, 3.8, 1.13)"
@@ -248,7 +239,6 @@ subprojects:
       - ".github/workflows/tpu-tests.yml"
       - "tests/tests_fabric/run_tpu_tests.sh"
     checks:
-      - "test-on-tpus (fabric, xrt, v4-8)"
       - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   # SECTION: lightning_app

.github/workflows/README.md

@@ -27,7 +27,6 @@ Brief description of all our automation tools used for boosting development perf
 
   - GPU: 2 x NVIDIA RTX 3090
   - TPU: [Google TPU v4-8](https://cloud.google.com/tpu/docs)
-  - IPU: [Colossus MK1 IPU](https://www.graphcore.ai/products/ipu)
 
 - To check which versions of Python or PyTorch are used for testing in our CI, see the corresponding workflow files or checkgroup config file at [`.github/checkgroup.yml`](../checkgroup.yml).
 

.github/workflows/_build-packages.yml

@@ -47,7 +47,7 @@ jobs:
         with:
           name: ${{ inputs.artifact-name }}
           path: pypi
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
 

.github/workflows/_legacy-checkpoints.yml

@@ -42,7 +42,7 @@ on:
         type: boolean
 
 env:
-  legacy_dir: "tests/legacy"
+  LEGACY_FOLDER: "tests/legacy"
   TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
 
 defaults:
@@ -57,7 +57,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           # Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt.
           python-version: 3.8
@@ -91,11 +91,11 @@ jobs:
         run: python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT
 
       - name: Generate checkpoints
-        working-directory: ${{ env.legacy_dir }}
+        working-directory: ${{ env.LEGACY_FOLDER }}
         run: bash generate_checkpoints.sh ${{ inputs.pl_version }}
       - name: Rename local to actual version
         if: inputs.upload_local
-        working-directory: ${{ env.legacy_dir }}
+        working-directory: ${{ env.LEGACY_FOLDER }}
         run: mv checkpoints/local checkpoints/${{ steps.decide-version.outputs.pl-version }}
 
       - name: "Determine: Keep artifact & DryRun"
@@ -107,12 +107,12 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: checkpoints-${{ github.sha }}
-          path: ${{ env.legacy_dir }}/checkpoints/
+          path: ${{ env.LEGACY_FOLDER }}/checkpoints/
           retention-days: ${{ env.KEEP_DAYS }}
 
       - run: pip install -r requirements/ci.txt
       - name: Upload checkpoints to S3
-        working-directory: ${{ env.legacy_dir }}
+        working-directory: ${{ env.LEGACY_FOLDER }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUB_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUB_SECRET_KEY }}
@@ -134,11 +134,11 @@ jobs:
           ref: master
 
       - name: Append a new version to legacy checkpoint list
-        working-directory: ${{ env.legacy_dir }}
+        working-directory: ${{ env.LEGACY_FOLDER }}
         run: echo ${PL_VERSION} >> back-compatible-versions.txt
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           title: Adding test for legacy checkpoint created with ${{ env.PL_VERSION }}
           committer: GitHub <[email protected]>

.github/workflows/call-clear-cache.yml

@@ -12,12 +12,12 @@ on:
 jobs:
   cron-clear:
     if: github.event_name == 'schedule'
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].0
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].1
     with:
       pattern: "latest|docs"
 
   direct-clear:
     if: github.event_name == 'workflow_dispatch'
-    uses: Lightning-AI/utilities/.github/workflows/[email protected].0
+    uses: Lightning-AI/utilities/.github/workflows/[email protected].1
     with:
       pattern: ${{ inputs.pattern }}