Merge branch 'master' into debug/ddp-fork

Author: Borda

.actions/assistant.py

@@ -341,47 +341,6 @@ def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> N
 
 
 class AssistantCLI:
-    @staticmethod
-    def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
-        """Remove some packages from given requirement files."""
-        if isinstance(req_files, str):
-            req_files = [req_files]
-        for req in req_files:
-            AssistantCLI._prune_packages(req, packages)
-
-    @staticmethod
-    def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
-        """Remove some packages from given requirement files."""
-        path = Path(req_file)
-        assert path.exists()
-        text = path.read_text()
-        lines = text.splitlines()
-        final = []
-        for line in lines:
-            ln_ = line.strip()
-            if not ln_ or ln_.startswith("#"):
-                final.append(line)
-                continue
-            req = list(_parse_requirements([ln_]))[0]
-            if req.name not in packages:
-                final.append(line)
-        print(final)
-        path.write_text("\n".join(final) + "\n")
-
-    @staticmethod
-    def _replace_min(fname: str) -> None:
-        with open(fname, encoding="utf-8") as fopen:
-            req = fopen.read().replace(">=", "==")
-        with open(fname, "w", encoding="utf-8") as fwrite:
-            fwrite.write(req)
-
-    @staticmethod
-    def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
-        """Replace the min package version by fixed one."""
-        for fname in requirement_fnames:
-            print(fname)
-            AssistantCLI._replace_min(fname)
-
     @staticmethod
     def copy_replace_imports(
         source_dir: str,

.azure/gpu-tests-fabric.yml

@@ -99,8 +99,16 @@ jobs:
         displayName: "Image info & NVIDIA"
 
       - bash: |
-          python .actions/assistant.py replace_oldest_ver
+          set -ex
           pip install "cython<3.0" wheel  # for compatibility
+          pip install -U "lightning-utilities[cli]"
+          cd requirements/fabric
+          # replace range by pin minimal requirements
+          python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
+          # drop deepspeed since it is not supported by our minimal Torch requirements
+          python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
+          # uninstall deepspeed since some older docker images have it pre-installed
+          pip uninstall -y deepspeed
         condition: contains(variables['Agent.JobName'], 'oldest')
         displayName: "setting oldest dependencies"
 

.azure/gpu-tests-pytorch.yml

@@ -103,8 +103,16 @@ jobs:
         displayName: "Image info & NVIDIA"
 
       - bash: |
-          python .actions/assistant.py replace_oldest_ver
+          set -ex
           pip install "cython<3.0" wheel  # for compatibility
+          pip install -U "lightning-utilities[cli]"
+          cd requirements/pytorch
+          # replace range by pin minimal requirements
+          python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
+          # drop deepspeed since it is not supported by our minimal Torch requirements
+          python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files strategies.txt
+          # uninstall deepspeed since some older docker images have it pre-installed
+          pip uninstall -y deepspeed
         condition: contains(variables['Agent.JobName'], 'oldest')
         displayName: "setting oldest dependencies"
 

.github/workflows/ci-tests-fabric.yml

@@ -94,7 +94,9 @@ jobs:
       - name: Set min. dependencies
         if: ${{ matrix.requires == 'oldest' }}
         run: |
-          python .actions/assistant.py replace_oldest_ver
+          cd requirements/fabric
+          pip install -U "lightning-utilities[cli]"
+          python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt', 'test.txt']"
           pip install "cython<3.0" wheel
           pip install "pyyaml==5.4" --no-build-isolation
 
@@ -140,7 +142,8 @@ jobs:
         run: |
           pip install -e ".[${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
             -U --upgrade-strategy=eager --prefer-binary \
-            --extra-index-url="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}"
+            --extra-index-url="${TORCH_URL}" \
+            --find-links="${PYPI_CACHE_DIR}"
           pip list
       - name: Dump handy wheels
         if: github.event_name == 'push' && github.ref == 'refs/heads/master'

.github/workflows/ci-tests-pytorch.yml

@@ -99,7 +99,9 @@ jobs:
       - name: Set min. dependencies
         if: ${{ matrix.requires == 'oldest' }}
         run: |
-          python .actions/assistant.py replace_oldest_ver
+          cd requirements/pytorch
+          pip install -U "lightning-utilities[cli]"
+          python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt', 'test.txt']"
           pip install "cython<3.0" wheel
           pip install "pyyaml==5.4" --no-build-isolation
 
@@ -139,7 +141,9 @@ jobs:
           pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \
             -U --upgrade-strategy=eager --prefer-binary \
             -r requirements/_integrations/accelerators.txt \
-            --extra-index-url="${TORCH_URL}" --find-links="${PYPI_CACHE_DIR}" --find-links="https://download.pytorch.org/whl/torch-tensorrt"
+            --extra-index-url="${TORCH_URL}" \
+            --find-links="${PYPI_CACHE_DIR}" \
+            --find-links="https://download.pytorch.org/whl/torch-tensorrt"
           pip list
       - name: Drop LAI from extensions
         if: ${{ matrix.pkg-name != 'lightning' }}

.lightning/workflows/fabric.yml

@@ -43,7 +43,10 @@ run: |
 
   if [ "${TORCH_VER}" == "2.1" ]; then
     echo "Set oldest versions"
-    python .actions/assistant.py replace_oldest_ver
+    cd requirements/fabric
+    pip install -U "lightning-utilities[cli]"
+    python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'strategies.txt']"
+    cd ../..
     pip install "cython<3.0" wheel  # for compatibility
   fi
 

.lightning/workflows/pytorch.yml

@@ -43,7 +43,10 @@ run: |
 
   if [ "${TORCH_VER}" == "2.1" ]; then
     recho "Set oldest versions"
-    python .actions/assistant.py replace_oldest_ver
+    cd requirements/pytorch
+    pip install -U "lightning-utilities[cli]"
+    python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt']"
+    cd ../..
     pip install "cython<3.0" wheel  # for compatibility
   fi
 

dockers/release/Dockerfile

@@ -21,6 +21,7 @@ FROM pytorchlightning/pytorch_lightning:base-cuda${CUDA_VERSION}-py${PYTHON_VERS
 LABEL maintainer="Lightning-AI <https://github.com/Lightning-AI>"
 
 ARG LIGHTNING_VERSION=""
+ARG PYTORCH_VERSION
 
 COPY ./ /home/pytorch-lightning/
 
@@ -39,7 +40,14 @@ RUN \
     fi && \
     # otherwise there is collision with folder name and pkg name on Pypi
     cd pytorch-lightning && \
-    pip install setuptools==75.6.0 && \
+    # pip install setuptools==75.6.0 && \
+    pip install -U "lightning-utilities[cli]" && \
+    # drop deepspeed since it is not supported by our minimal Torch requirements \
+    echo "PYTORCH_VERSION is: '$PYTORCH_VERSION'" && \
+    if [[ "$PYTORCH_VERSION" =~ ^(2\.1|2\.2|2\.3|2\.4)$ ]]; then \
+        python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/fabric/strategies.txt ; \
+        python -m lightning_utilities.cli requirements prune-pkgs --packages deepspeed --req_files requirements/pytorch/strategies.txt ; \
+    fi && \
     PACKAGE_NAME=lightning pip install '.[extra,loggers,strategies]' --no-cache-dir && \
     PACKAGE_NAME=pytorch pip install '.[extra,loggers,strategies]' --no-cache-dir && \
     cd .. && \

docs/source-fabric/advanced/compile.rst

@@ -417,7 +417,7 @@ Additional Resources
 
 Here are a few resources for further reading after you complete this tutorial:
 
-- `PyTorch 2.0 Paper <https://pytorch.org/blog/pytorch-2-paper-tutorial/>`_
+- `PyTorch 2.0 Paper <https://pytorch.org/get-started/pytorch-2-x/>`_
 - `GenAI with PyTorch 2.0 blog post series <https://pytorch.org/blog/accelerating-generative-ai-4/>`_
 - `Training Production AI Models with PyTorch 2.0 <https://pytorch.org/blog/training-production-ai-models/>`_
 - `Empowering Models with Performance: The Art of Generalized Model Transformation Approach <https://pytorch.org/blog/empowering-models-performance/>`_

docs/source-pytorch/advanced/compile.rst

@@ -396,7 +396,7 @@ Additional Resources
 
 Here are a few resources for further reading after you complete this tutorial:
 
-- `PyTorch 2.0 Paper <https://pytorch.org/blog/pytorch-2-paper-tutorial/>`_
+- `PyTorch 2.0 Paper <https://pytorch.org/get-started/pytorch-2-x/>`_
 - `GenAI with PyTorch 2.0 blog post series <https://pytorch.org/blog/accelerating-generative-ai-4/>`_
 - `Training Production AI Models with PyTorch 2.0 <https://pytorch.org/blog/training-production-ai-models/>`_
 - `Empowering Models with Performance: The Art of Generalized Model Transformation Approach <https://pytorch.org/blog/empowering-models-performance/>`_