Merge branch 'master' into weights-only-compatibility

matsumotosan · web-flow · commit eb70cf7db7e9 · 2025-10-29T08:27:06.000-04:00
diff --git a/.github/actions/pip-wheels/action.yml b/.github/actions/pip-wheels/action.yml
diff --git a/.github/workflows/_build-packages.yml b/.github/workflows/_build-packages.yml
@@ -41,7 +41,7 @@ jobs:
           mkdir -p pypi/${{ matrix.pkg-name }}
           cp dist/* pypi/${{ matrix.pkg-name }}/
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: ${{ inputs.artifact-name }}-${{ matrix.pkg-name }}
           path: pypi
@@ -51,7 +51,7 @@ jobs:
     needs: build-packages
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with: # download all build artifacts
           pattern: ${{ inputs.artifact-name }}-*
           merge-multiple: true
@@ -62,7 +62,7 @@ jobs:
 
       - name: Keep artifact
         run: python -c "print('DAYS=' + str(5 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_ENV
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: ${{ inputs.artifact-name }}
           path: pypi
diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml
@@ -108,7 +108,7 @@ jobs:
           python -c "print('AWS_RUN=' + str('' if '${{inputs.push_to_s3}}' == 'true' else '--dryrun'))" >> $GITHUB_ENV
 
       - name: Upload checkpoints to GitHub Actions artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: checkpoints-${{ github.sha }}
           path: ${{ env.LEGACY_FOLDER }}/checkpoints/
diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml
@@ -50,7 +50,7 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
diff --git a/.github/workflows/ci-tests-fabric.yml b/.github/workflows/ci-tests-fabric.yml
@@ -134,15 +134,6 @@ jobs:
             --find-links="${TORCH_URL}"
           uv pip list
 
-      - name: Dump handy wheels
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-        continue-on-error: true
-        uses: ./.github/actions/pip-wheels
-        with:
-          wheel-dir: ${{ env.PYPI_CACHE_DIR }}
-          torch-url: ${{ env.TORCH_URL }}
-          cache-key: "pypi_wheels"
-
       - name: Adjust tests
         if: ${{ matrix.config.pkg-name != 'lightning' }}
         run: |
diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml
@@ -147,15 +147,6 @@ jobs:
         if: ${{ matrix.config.pkg-name == 'lightning' }}
         run: uv pip uninstall pytorch-lightning
 
-      - name: Dump handy wheels
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-        continue-on-error: true
-        uses: ./.github/actions/pip-wheels
-        with:
-          wheel-dir: ${{ env.PYPI_CACHE_DIR }}
-          torch-url: ${{ env.TORCH_URL }}
-          cache-key: "pypi_wheels"
-
       - name: Cache datasets
         uses: actions/cache@v4
         with:
diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml
@@ -47,7 +47,6 @@ defaults:
 env:
   FREEZE_REQUIREMENTS: "1"
   TORCH_URL: "https://download.pytorch.org/whl/cpu/"
-  PYPI_CACHE_DIR: "_pip-wheels"
   PYPI_LOCAL_DIR: "pypi_pkgs/"
 
 jobs:
@@ -95,12 +94,6 @@ jobs:
           aws s3 sync --no-sign-request s3://sphinx-packages/ ${PYPI_LOCAL_DIR}
           uv pip install lai-sphinx-theme -U -f ${PYPI_LOCAL_DIR}
 
-      - name: pip wheels cache
-        uses: actions/cache/restore@v4
-        with:
-          path: ${{ env.PYPI_CACHE_DIR }}
-          key: pypi_wheels
-
       - name: Install pandoc & texlive
         if: ${{ matrix.pkg-name == 'pytorch' }}
         timeout-minutes: 5
@@ -111,10 +104,8 @@ jobs:
       - name: Install package & dependencies
         timeout-minutes: 20
         run: |
-          mkdir -p ${PYPI_CACHE_DIR}  # in case cache was not hit
-          ls -lh ${PYPI_CACHE_DIR}
           uv pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
-            -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} --extra-index-url="${TORCH_URL}"
+            -f ${PYPI_LOCAL_DIR} --extra-index-url="${TORCH_URL}"
           uv pip list
 
       - name: Install req. for Notebooks/tutorials
@@ -142,22 +133,13 @@ jobs:
 
       - name: Upload built docs
         if: ${{ matrix.target == 'html' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: docs-${{ matrix.pkg-name }}-${{ github.sha }}
           path: docs/build/html/
           retention-days: ${{ env.ARTIFACT_DAYS }}
           include-hidden-files: true
 
-      #- name: Dump handy wheels
-      #  if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-      #  continue-on-error: true
-      #  uses: ./.github/actions/pip-wheels
-      #  with:
-      #    wheel-dir: ${{ env.PYPI_CACHE_DIR }}
-      #    torch-url: ${{ env.TORCH_URL }}
-      #    cache-key: "pypi_wheels"
-
   deploy-docs:
     needs: docs-make
     if: github.repository_owner == 'Lightning-AI' && github.event_name != 'pull_request'
@@ -171,7 +153,7 @@ jobs:
       # use input if dispatch or git tag
       VERSION: ${{ inputs.version || github.ref_name }}
     steps:
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: docs-${{ matrix.pkg-name }}-${{ github.sha }}
           path: docs/build/html/
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
@@ -40,7 +40,7 @@ jobs:
           nb-dirs: ${{ env.NB_DIRS }}
           allow-local-changes: "true"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: nightly-packages-${{ github.sha }}
           path: dist
@@ -54,7 +54,7 @@ jobs:
       PKG_NAME: "lightning"
     steps:
       - uses: actions/checkout@v5 # needed to use local composite action
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: nightly-packages-${{ github.sha }}
           path: dist
diff --git a/.github/workflows/release-pkg.yml b/.github/workflows/release-pkg.yml
@@ -38,7 +38,7 @@ jobs:
     if: github.event_name == 'release'
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
@@ -140,7 +140,7 @@ jobs:
         name: ["FABRIC", "PYTORCH", "LIGHTNING"]
     steps:
       - uses: actions/checkout@v5 # needed for local action below
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
@@ -165,7 +165,7 @@ jobs:
         name: ["FABRIC", "PYTORCH", "LIGHTNING"]
     steps:
       - uses: actions/checkout@v5 # needed for local action below
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
diff --git a/docs/source-pytorch/community/ecosystem-ci.rst b/docs/source-pytorch/community/ecosystem-ci.rst
diff --git a/docs/source-pytorch/community/index.rst b/docs/source-pytorch/community/index.rst
@@ -7,7 +7,6 @@
    ../generated/CONTRIBUTING.md
    ../generated/BECOMING_A_CORE_CONTRIBUTOR.md
    governance
-   ecosystem-ci
    ../versioning
    ../past_versions
    ../generated/CHANGELOG.md
@@ -70,13 +69,6 @@ Community
    :button_link: ../generated/CHANGELOG.html
    :height: 100
 
-.. displayitem::
-   :header: Ecosystem CI
-   :description: Automate issue discovery for your projects against Lightning nightly and releases
-   :col_css: col-md-12
-   :button_link: ecosystem-ci.html
-   :height: 100
-
 .. raw:: html
 
         </div>
diff --git a/requirements/fabric/examples.txt b/requirements/fabric/examples.txt
@@ -1,5 +1,5 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-torchvision >=0.16.0, <0.24.0
+torchvision >=0.16.0, <0.25.0
 torchmetrics >=0.10.0, <1.9.0
diff --git a/requirements/pytorch/examples.txt b/requirements/pytorch/examples.txt
@@ -2,6 +2,6 @@
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
 requests <2.33.0
-torchvision >=0.16.0, <0.24.0
+torchvision >=0.16.0, <0.25.0
 ipython[all] >=8.0.0, <10.0.0
 torchmetrics >=0.10.0, <1.9.0
diff --git a/src/lightning/pytorch/CHANGELOG.md b/src/lightning/pytorch/CHANGELOG.md
@@ -64,6 +64,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed how `ThroughputMonitor` calculated training time ([#21291](https://github.com/Lightning-AI/pytorch-lightning/pull/21291))
 
 
+- Fixed synchronization of gradients in manual optimization with `DDPStrategy(static_graph=True)` ([#21251](https://github.com/Lightning-AI/pytorch-lightning/pull/21251))
+
+
+
 ---
 
 ## [2.5.5] - 2025-09-05
diff --git a/src/lightning/pytorch/strategies/ddp.py b/src/lightning/pytorch/strategies/ddp.py
@@ -103,6 +103,7 @@ def __init__(
         self._process_group_backend: Optional[str] = process_group_backend
         self._timeout: Optional[timedelta] = timeout
         self._start_method = start_method
+        self._pl_static_graph_delay_done = False
 
     @property
     def is_distributed(self) -> bool:  # pragma: no-cover
@@ -319,6 +320,27 @@ def pre_backward(self, closure_loss: Tensor) -> None:
         if not self.lightning_module.automatic_optimization:
             prepare_for_backward(self.model, closure_loss)
 
+    @override
+    def post_backward(self, closure_loss: Tensor) -> None:
+        # Only for first static-graph iteration with manual optimization
+        model = self.model
+        lm = self.lightning_module
+        if not isinstance(model, DistributedDataParallel):
+            return
+        if lm is None or lm.automatic_optimization:
+            return
+        if not getattr(model, "static_graph", False):
+            return
+        if self._pl_static_graph_delay_done:
+            return
+
+        # Call DDP's own first-iter static-graph flush.
+        # This is what actually launches the bucket all-reduces.
+        reducer = model.reducer
+        reducer._delay_all_reduce()
+
+        self._pl_static_graph_delay_done = True
+
     @override
     def model_to_device(self) -> None:
         log.debug(f"{self.__class__.__name__}: moving model to device [{self.root_device}]...")
diff --git a/tests/tests_pytorch/strategies/test_ddp_integration.py b/tests/tests_pytorch/strategies/test_ddp_integration.py
