Merge branch 'master' into patch-2

Author: amorehead

.github/checkgroup.yml

@@ -19,7 +19,7 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "pl-cpu (macOS-13, lightning, 3.9, 2.1, oldest)"
+      - "pl-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
       - "pl-cpu (macOS-14, lightning, 3.10, 2.1)"
       - "pl-cpu (macOS-14, lightning, 3.11, 2.2.2)"
       - "pl-cpu (macOS-14, lightning, 3.11, 2.3)"
@@ -40,9 +40,9 @@ subprojects:
       - "pl-cpu (macOS-14, pytorch, 3.9, 2.1)"
       - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 2.1)"
       - "pl-cpu (windows-2022, pytorch, 3.9, 2.1)"
-      - "pl-cpu (macOS-13, pytorch, 3.10, 2.1)"
-      - "pl-cpu (ubuntu-22.04, pytorch, 3.10, 2.1)"
-      - "pl-cpu (windows-2022, pytorch, 3.10, 2.1)"
+      - "pl-cpu (macOS-14, pytorch, 3.12, 2.5.1)"
+      - "pl-cpu (ubuntu-22.04, pytorch, 3.12, 2.5.1)"
+      - "pl-cpu (windows-2022, pytorch, 3.12, 2.5.1)"
 
   - id: "pytorch_lightning: Azure GPU"
     paths:
@@ -89,14 +89,15 @@ subprojects:
     checks:
       - "lightning.Benchmarks"
 
-  - id: "pytorch-lightning: TPU workflow"
-    paths:
-      # tpu CI availability is very limited, so we only require tpu tests
-      # to pass when their configurations are modified
-      - ".github/workflows/tpu-tests.yml"
-      - "tests/tests_pytorch/run_tpu_tests.sh"
-    checks:
-      - "test-on-tpus (pytorch, pjrt, v4-8)"
+  # Temporarily disabled
+  #  - id: "pytorch-lightning: TPU workflow"
+  #    paths:
+  #      # tpu CI availability is very limited, so we only require tpu tests
+  #      # to pass when their configurations are modified
+  #      - ".github/workflows/tpu-tests.yml"
+  #      - "tests/tests_pytorch/run_tpu_tests.sh"
+  #    checks:
+  #      - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   - id: "fabric: Docs"
     paths:
@@ -171,7 +172,7 @@ subprojects:
       - "!*.md"
       - "!**/*.md"
     checks:
-      - "fabric-cpu (macOS-13, lightning, 3.9, 2.1, oldest)"
+      - "fabric-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
       - "fabric-cpu (macOS-14, lightning, 3.10, 2.1)"
       - "fabric-cpu (macOS-14, lightning, 3.11, 2.2.2)"
       - "fabric-cpu (macOS-14, lightning, 3.11, 2.3)"
@@ -192,9 +193,9 @@ subprojects:
       - "fabric-cpu (macOS-14, fabric, 3.9, 2.1)"
       - "fabric-cpu (ubuntu-20.04, fabric, 3.9, 2.1)"
       - "fabric-cpu (windows-2022, fabric, 3.9, 2.1)"
-      - "fabric-cpu (macOS-13, fabric, 3.10, 2.1)"
-      - "fabric-cpu (ubuntu-22.04, fabric, 3.10, 2.1)"
-      - "fabric-cpu (windows-2022, fabric, 3.10, 2.1)"
+      - "fabric-cpu (macOS-14, fabric, 3.12, 2.5.1)"
+      - "fabric-cpu (ubuntu-22.04, fabric, 3.12, 2.5.1)"
+      - "fabric-cpu (windows-2022, fabric, 3.12, 2.5.1)"
 
   - id: "lightning_fabric: Azure GPU"
     paths:
@@ -266,14 +267,14 @@ subprojects:
       - "install-pkg (ubuntu-22.04, lightning, 3.11)"
       - "install-pkg (ubuntu-22.04, notset, 3.9)"
       - "install-pkg (ubuntu-22.04, notset, 3.11)"
-      - "install-pkg (macOS-13, fabric, 3.9)"
-      - "install-pkg (macOS-13, fabric, 3.11)"
-      - "install-pkg (macOS-13, pytorch, 3.9)"
-      - "install-pkg (macOS-13, pytorch, 3.11)"
-      - "install-pkg (macOS-13, lightning, 3.9)"
-      - "install-pkg (macOS-13, lightning, 3.11)"
-      - "install-pkg (macOS-13, notset, 3.9)"
-      - "install-pkg (macOS-13, notset, 3.11)"
+      - "install-pkg (macOS-14, fabric, 3.9)"
+      - "install-pkg (macOS-14, fabric, 3.11)"
+      - "install-pkg (macOS-14, pytorch, 3.9)"
+      - "install-pkg (macOS-14, pytorch, 3.11)"
+      - "install-pkg (macOS-14, lightning, 3.9)"
+      - "install-pkg (macOS-14, lightning, 3.11)"
+      - "install-pkg (macOS-14, notset, 3.9)"
+      - "install-pkg (macOS-14, notset, 3.11)"
       - "install-pkg (windows-2022, fabric, 3.9)"
       - "install-pkg (windows-2022, fabric, 3.11)"
       - "install-pkg (windows-2022, pytorch, 3.9)"

.github/workflows/ci-pkg-install.yml

@@ -42,7 +42,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "macOS-13", "windows-2022"]
+        os: ["ubuntu-22.04", "macOS-14", "windows-2022"]
         pkg-name: ["fabric", "pytorch", "lightning", "notset"]
         python-version: ["3.9", "3.11"]
     steps:

.github/workflows/ci-tests-fabric.yml

@@ -56,11 +56,11 @@ jobs:
           - { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12", pytorch-version: "2.5.1" }
           - { os: "windows-2022", pkg-name: "lightning", python-version: "3.12", pytorch-version: "2.5.1" }
           # only run PyTorch latest with Python latest, use Fabric scope to limit dependency issues
-          - { os: "macOS-13", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.1" }
-          - { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.1" }
-          - { os: "windows-2022", pkg-name: "fabric", python-version: "3.10", pytorch-version: "2.1" }
+          - { os: "macOS-14", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.5.1" }
+          - { os: "ubuntu-22.04", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.5.1" }
+          - { os: "windows-2022", pkg-name: "fabric", python-version: "3.12", pytorch-version: "2.5.1" }
           # "oldest" versions tests, only on minimum Python
-          - { os: "macOS-13", pkg-name: "lightning", python-version: "3.9", pytorch-version: "2.1", requires: "oldest" }
+          - { os: "macOS-14", pkg-name: "lightning", python-version: "3.9", pytorch-version: "2.1", requires: "oldest" }
           - {
               os: "ubuntu-20.04",
               pkg-name: "lightning",
@@ -101,7 +101,10 @@ jobs:
 
       - name: Set min. dependencies
         if: ${{ matrix.requires == 'oldest' }}
-        run: python .actions/assistant.py replace_oldest_ver
+        run: |
+          python .actions/assistant.py replace_oldest_ver
+          pip install "cython<3.0" wheel
+          pip install "pyyaml==5.4" --no-build-isolation
 
       - name: Adjust PyTorch versions in requirements files
         if: ${{ matrix.requires != 'oldest' }}

.github/workflows/ci-tests-pytorch.yml

@@ -60,11 +60,11 @@ jobs:
           - { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.12", pytorch-version: "2.5.1" }
           - { os: "windows-2022", pkg-name: "lightning", python-version: "3.12", pytorch-version: "2.5.1" }
           # only run PyTorch latest with Python latest, use PyTorch scope to limit dependency issues
-          - { os: "macOS-13", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "2.1" }
-          - { os: "ubuntu-22.04", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "2.1" }
-          - { os: "windows-2022", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "2.1" }
+          - { os: "macOS-14", pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.5.1" }
+          - { os: "ubuntu-22.04", pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.5.1" }
+          - { os: "windows-2022", pkg-name: "pytorch", python-version: "3.12", pytorch-version: "2.5.1" }
           # "oldest" versions tests, only on minimum Python
-          - { os: "macOS-13", pkg-name: "lightning", python-version: "3.9", pytorch-version: "2.1", requires: "oldest" }
+          - { os: "macOS-14", pkg-name: "lightning", python-version: "3.9", pytorch-version: "2.1", requires: "oldest" }
           - {
               os: "ubuntu-20.04",
               pkg-name: "lightning",
@@ -106,7 +106,10 @@ jobs:
 
       - name: Set min. dependencies
         if: ${{ matrix.requires == 'oldest' }}
-        run: python .actions/assistant.py replace_oldest_ver
+        run: |
+          python .actions/assistant.py replace_oldest_ver
+          pip install "cython<3.0" wheel
+          pip install "pyyaml==5.4" --no-build-isolation
 
       - name: Adjust PyTorch versions in requirements files
         if: ${{ matrix.requires != 'oldest' }}

.github/workflows/tpu-tests.yml renamed to .github/workflows/tpu-tests.yml.disabled

@@ -41,7 +41,8 @@ def training_step(self, batch, batch_idx: int):
 
     def configure_optimizers(self):
         optim = torch.optim.Adam(self.parameters(), lr=1e-4)
-        return optim, {
+        return {
+            "optimizer": optim,
             "scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(optim, mode="max", verbose=True),
             "monitor": "val_accuracy",
             "interval": "epoch",

examples/fabric/build_your_own_trainer/run.py

@@ -39,13 +39,13 @@ def teardown(self) -> None:
 
     @staticmethod
     @override
-    def parse_devices(devices: Union[int, str, List[int]]) -> int:
+    def parse_devices(devices: Union[int, str]) -> int:
         """Accelerator device parsing logic."""
         return _parse_cpu_cores(devices)
 
     @staticmethod
     @override
-    def get_parallel_devices(devices: Union[int, str, List[int]]) -> List[torch.device]:
+    def get_parallel_devices(devices: Union[int, str]) -> List[torch.device]:
         """Gets parallel devices for the Accelerator."""
         devices = _parse_cpu_cores(devices)
         return [torch.device("cpu")] * devices
@@ -72,12 +72,12 @@ def register_accelerators(cls, accelerator_registry: _AcceleratorRegistry) -> No
         )
 
 
-def _parse_cpu_cores(cpu_cores: Union[int, str, List[int]]) -> int:
+def _parse_cpu_cores(cpu_cores: Union[int, str]) -> int:
     """Parses the cpu_cores given in the format as accepted by the ``devices`` argument in the
     :class:`~lightning.pytorch.trainer.trainer.Trainer`.
 
     Args:
-        cpu_cores: An int > 0.
+        cpu_cores: An int > 0 or a string that can be converted to an int > 0.
 
     Returns:
         An int representing the number of processes

src/lightning/fabric/accelerators/cpu.py

@@ -347,6 +347,14 @@ def measure_flops(
         torch.int8: 389.9e12,
         "int4": 779.8e12,
     },
+    "rtx 4080 super": {
+        torch.float32: 52.2e12,
+        "tfloat32": 52.2e12,
+        torch.bfloat16: 52.2e12,
+        torch.float16: 52.2e12,
+        torch.int8: 417.6e12,
+        "int4": 835.2e12,
+    },
     "l4": {
         torch.float32: 30.3e12,
         "tfloat32": 60e12,

src/lightning/fabric/utilities/throughput.py

@@ -48,13 +48,13 @@ def teardown(self) -> None:
 
     @staticmethod
     @override
-    def parse_devices(devices: Union[int, str, List[int]]) -> int:
+    def parse_devices(devices: Union[int, str]) -> int:
         """Accelerator device parsing logic."""
         return _parse_cpu_cores(devices)
 
     @staticmethod
     @override
-    def get_parallel_devices(devices: Union[int, str, List[int]]) -> List[torch.device]:
+    def get_parallel_devices(devices: Union[int, str]) -> List[torch.device]:
         """Gets parallel devices for the Accelerator."""
         devices = _parse_cpu_cores(devices)
         return [torch.device("cpu")] * devices

src/lightning/pytorch/accelerators/cpu.py

@@ -88,7 +88,7 @@ def forward(self, x: Tensor) -> Tensor:
             # TODO: Could make this a `nn.Parameter` with `requires_grad=False`
             self.pe = self._init_pos_encoding(device=x.device)
 
-        x = x + self.pe[: x.size(0), :]
+        x = x + self.pe[:, x.size(1)]
         return self.dropout(x)
 
     def _init_pos_encoding(self, device: torch.device) -> Tensor:
@@ -97,7 +97,7 @@ def _init_pos_encoding(self, device: torch.device) -> Tensor:
         div_term = torch.exp(torch.arange(0, self.dim, 2, device=device).float() * (-math.log(10000.0) / self.dim))
         pe[:, 0::2] = torch.sin(position * div_term)
         pe[:, 1::2] = torch.cos(position * div_term)
-        pe = pe.unsqueeze(0).transpose(0, 1)
+        pe = pe.unsqueeze(0)
         return pe