ci: update install lite & cut pkg dependency (#14517)

* ci: update install lite

* try without lite in req file

* ci: install

* app

* init

* Revert "app"

This reverts commit f3f09e7.

* ci: cpu

* ci: gpu

* pkg

* env

* bench

* trigger

* notes

* prune

* set version

* fix version

* git reset

* hpu, ipu

* adjust

* --hard

* git checkout

* Apply suggestions from code review

Co-authored-by: Carlos Mocholí <[email protected]>
Co-authored-by: Akihiro Nitta <[email protected]>

* rc2

* L

* docs

* hpu

Co-authored-by: awaelchli <[email protected]>
Co-authored-by: Carlos Mocholí <[email protected]>
Co-authored-by: Akihiro Nitta <[email protected]>
Co-authored-by: Luca Antiga <[email protected]>

Author: 5 people

.actions/setup_tools.py

@@ -19,17 +19,16 @@
 import tarfile
 import tempfile
 import urllib.request
-from datetime import datetime
 from distutils.version import LooseVersion
 from importlib.util import module_from_spec, spec_from_file_location
 from itertools import chain
 from types import ModuleType
-from typing import Dict, List, Sequence
+from typing import Dict, List
 
 from pkg_resources import parse_requirements
 
 _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
-_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"}
+_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"}
 
 # TODO: remove this once lightning-ui package is ready as a dependency
 _LIGHTNING_FRONTEND_RELEASE_URL = "https://storage.googleapis.com/grid-packages/lightning-ui/v0.0.0/build.tar.gz"
@@ -83,7 +82,7 @@ def _augment_requirement(ln: str, comment_char: str = "#", unfreeze: str = "all"
         is_strict = False
     req = ln.strip()
     # skip directly installed dependencies
-    if not req or req.startswith("http") or "@http" in req:
+    if not req or req.startswith("http") or "@" in req:
         return ""
     # extract the major version from all listed versions
     if unfreeze == "major":
@@ -237,20 +236,6 @@ def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None:
         copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping)
 
 
-def set_version_today(fpath: str) -> None:
-    """Replace the template date with today."""
-    with open(fpath) as fp:
-        lines = fp.readlines()
-
-    def _replace_today(ln):
-        today = datetime.now()
-        return ln.replace("YYYY.-M.-D", f"{today.year}.{today.month}.{today.day}")
-
-    lines = list(map(_replace_today, lines))
-    with open(fpath, "w") as fp:
-        fp.writelines(lines)
-
-
 def _download_frontend(pkg_path: str):
     """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
     directory."""
@@ -273,27 +258,6 @@ def _download_frontend(pkg_path: str):
         print("The Lightning UI downloading has failed!")
 
 
-def _relax_require_versions(
-    source_dir: str = "src", req_dir: str = "requirements", strict_pkgs: Sequence[str] = ("lightning_app",)
-) -> None:
-    """Parse the base requirements and append  as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`.
-
-    >>> _relax_require_versions("../src", "../requirements")
-    """
-    strict_pkgs = strict_pkgs or tuple()
-    reqs = load_requirements(req_dir, file_name="base.txt")
-    for i, req in enumerate(parse_requirements(reqs)):
-        ver = parse_version_from_file(os.path.join(source_dir, req.name))
-        if not ver:
-            continue
-        if req.name not in strict_pkgs:
-            ver = ".".join(ver.split(".")[:2] + ["*"])
-        reqs[i] = f"{req}, =={ver}"
-
-    with open(os.path.join(req_dir, "base.txt"), "w") as fp:
-        fp.writelines([ln + os.linesep for ln in reqs])
-
-
 def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None:
     """Load all base requirements from all particular packages and prune duplicates."""
     requires = [
@@ -308,3 +272,19 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
     requires = list(chain(*requires))
     with open(os.path.join(req_dir, "base.txt"), "w") as fp:
         fp.writelines([ln + os.linesep for ln in requires])
+
+
+def set_actual_version_from_src(req_path: str, src_root: str, pkg_name: str) -> None:
+    """Setting actual version from source code for a given package."""
+    with open(req_path, encoding="utf-8") as fo:
+        lines = fo.readlines()
+    ver = parse_version_from_file(os.path.join(src_root, pkg_name.replace("-", "_")))
+    for i, ln in enumerate(lines):
+        reqs = list(parse_requirements([ln]))
+        if not reqs:
+            continue
+        if reqs[0].name == pkg_name:
+            lines[i] = f"{pkg_name}=={ver}{os.linesep}"
+
+    with open(req_path, "w", encoding="utf-8") as fw:
+        fw.writelines(lines)

.azure/gpu-benchmark.yml

@@ -13,7 +13,14 @@ trigger:
       - "release/*"
       - "refs/tags/*"
 
-pr: none
+pr:
+  branches:
+    include:
+      - "master"
+      - "release/*"
+  paths:
+    include:
+      - ".azure/gpu-benchmark.yml"
 
 schedules:
   - cron: "0 0 * * *" # At the end of every day
@@ -26,17 +33,36 @@ jobs:
   - job: benchmarks
     timeoutInMinutes: "90"
     cancelTimeoutInMinutes: "2"
-    pool: azure-jirka-spot
+    pool: lit-rtx-3090
+    variables:
+      DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
       image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
-      options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g"
+      options: "--gpus=all --shm-size=32g"
     workspace:
       clean: all
 
     steps:
 
     - bash: |
-        pip install -e . -r requirements/pytorch/strategies.txt
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+        cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
+        echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver"
+        echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
+      displayName: 'set env. vars'
+
+    - bash: |
+        pip install -e . --find-links ${TORCH_URL}
+        git checkout -- setup.py MANIFEST.in
+      env:
+        PACKAGE_NAME: lite
+        FREEZE_REQUIREMENTS: 1
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
+      displayName: 'Install Lite pkg'
+
+    - bash: |
+        pip install -e .[strategies] --find-links ${TORCH_URL}
         pip list
       env:
         PACKAGE_NAME: pytorch
@@ -45,7 +71,7 @@ jobs:
 
     - bash: python -m pytest benchmarks -v --durations=0
       env:
-        PL_RUNNING_BENCHMARKS: 1
+        PL_RUNNING_BENCHMARKS: "1"
         PL_RUN_CUDA_TESTS: "1"
       workingDirectory: tests/tests_pytorch
       displayName: 'Testing: PyTorch benchmarks'

.azure/gpu-tests-pytorch.yml

@@ -61,7 +61,18 @@ jobs:
       clean: all
 
     steps:
+
     - bash: |
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+        cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
+        echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver"
+        echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
+      displayName: 'set env. vars'
+
+    - bash: |
+        echo $CUDA_VISIBLE_DEVICES
+        echo $CUDA_VERSION_MM
+        echo $TORCH_URL
         lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
@@ -72,11 +83,6 @@ jobs:
       displayName: 'Image info & NVIDIA'
 
     - bash: |
-        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
-      displayName: 'set visible devices'
-
-    - bash: |
-        set -e
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'bagua' not in line] ; open(fname, 'w').writelines(lines)"
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)"
@@ -85,12 +91,26 @@ jobs:
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${PYTORCH_VERSION}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt ${PYTORCH_VERSION}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION}
+      displayName: 'Adjust dependencies'
 
-        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        pip install -e .[strategies] --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
-        pip install --requirement requirements/pytorch/devel.txt --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
-        pip install --requirement requirements/pytorch/examples.txt --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
+    - bash: |
+        pip install -e . --find-links ${TORCH_URL}
+        git checkout -- setup.py MANIFEST.in
+      env:
+        PACKAGE_NAME: "lite"
+        FREEZE_REQUIREMENTS: "1"
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
+      displayName: 'Install Lite pkg'
+
+    - bash: pip install -e .[strategies] -r requirements/pytorch/devel.txt -r requirements/pytorch/examples.txt --find-links ${TORCH_URL}
+      env:
+        PACKAGE_NAME: "pytorch"
+        FREEZE_REQUIREMENTS: "1"
+      displayName: 'Install package'
 
+    - bash: |
+        set -e
         CUDA_VERSION_BAGUA=$(python -c "print([ver for ver in [116,113,111,102] if $CUDA_VERSION_MM >= ver][0])")
         pip install "bagua-cuda$CUDA_VERSION_BAGUA"
 
@@ -100,14 +120,10 @@ jobs:
         pip install "colossalai==0.1.10+torch${PYTORCH_VERSION_COLOSSALAI}cu${CUDA_VERSION_COLOSSALAI}" --find-links https://release.colossalai.org
 
         pip list
-      env:
-        PACKAGE_NAME: pytorch
-        FREEZE_REQUIREMENTS: 1
       displayName: 'Install dependencies'
 
     - bash: |
         set -e
-        echo $CUDA_VISIBLE_DEVICES
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
         python requirements/pytorch/check-avail-strategies.py

.azure/hpu-tests.yml

@@ -58,6 +58,16 @@ jobs:
         sudo pip install pip -U
       displayName: 'Instance HW info'
 
+    - bash: |
+        pip install -e .
+        git checkout -- setup.py MANIFEST.in
+      env:
+        PACKAGE_NAME: "lite"
+        FREEZE_REQUIREMENTS: "1"
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
+      displayName: 'Install Lite pkg'
+
     - bash: |
         set -e
         pip --version
@@ -67,9 +77,9 @@ jobs:
         pip install ".[extra,test]"
         pip list
       env:
-        PACKAGE_NAME: pytorch
-        FREEZE_REQUIREMENTS: 1
-      displayName: 'Install dependencies'
+        PACKAGE_NAME: "pytorch"
+        FREEZE_REQUIREMENTS: "1"
+      displayName: 'Install package & dependencies'
 
     - bash: |
         hl-smi -L

.azure/ipu-tests.yml

@@ -62,6 +62,16 @@ jobs:
         done
       displayName: "Reset IPU devices"
 
+    - bash: |
+        pip install -e .
+        git checkout -- setup.py MANIFEST.in
+      env:
+        PACKAGE_NAME: "lite"
+        FREEZE_REQUIREMENTS: "1"
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
+      displayName: 'Install Lite pkg'
+
     - bash: |
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt

.github/workflows/ci-pkg-install.yml

@@ -23,7 +23,46 @@ defaults:
 
 jobs:
 
+  init-pypi:
+    runs-on: ubuntu-20.04
+    steps:
+    - run: mkdir pypi && touch pypi/.placeholder
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ci-packages-${{ github.sha }}
+        path: pypi
+
+  build-pypi:
+    needs: init-pypi
+    # This serves to create packages for potential internal dependencies
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      max-parallel: 1
+      matrix:
+        pkg: ["lite"]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - uses: actions/download-artifact@v3
+        with:
+          name: ci-packages-${{ github.sha }}
+          path: pypi
+      - name: Build package
+        env:
+          PACKAGE_NAME: ${{ matrix.pkg }}
+        run: |
+          python setup.py sdist
+          cp dist/* pypi/
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ci-packages-${{ github.sha }}
+          path: pypi
+
   install-pkg:
+    needs: build-pypi
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -49,7 +88,14 @@ jobs:
           pkg-name: ${{ matrix.pkg }}
           nb-dirs: ${{ env.NB_DIRS }}
 
+      - uses: actions/download-artifact@v3
+        with:
+          name: ci-packages-${{ github.sha }}
+          path: pypi
+      - run: ls -l pypi/
       - uses: ./.github/actions/pkg-install
+        with:
+          pip-flags: "--pre --find-links ../pypi/"
 
       - name: Run CLI
         # todo: add testing for `lightning_app`

.github/workflows/ci-pytorch-tests-slow.yml

@@ -55,9 +55,18 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-py${{ matrix.python-version }}-
 
-    - name: Install dependencies
+    - name: Install lite for PL
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package to prevent accidental interaction with cache
       env:
-        PACKAGE_NAME: pytorch
+        PACKAGE_NAME: "lite"
+      run: |
+        pip install -e . --upgrade --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+        git checkout -- setup.py MANIFEST.in
+
+    - name: Install package & dependencies
+      env:
+        PACKAGE_NAME: "pytorch"
         FREEZE_REQUIREMENTS: 1
       run: |
         # adjust versions according installed Torch version

.github/workflows/ci-pytorch-tests.yml

@@ -103,6 +103,7 @@ jobs:
     - name: Adjust PyTorch versions in requirements files
       if: ${{ matrix.requires != 'oldest' }}
       run: |
+        python ./requirements/pytorch/adjust-versions.py requirements/lite/base.txt ${{ matrix.pytorch-version }}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${{ matrix.pytorch-version }}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${{ matrix.pytorch-version }}
         cat requirements/pytorch/base.txt
@@ -122,6 +123,16 @@ jobs:
     - name: Switch PyTorch URL
       run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
 
+    - name: Install lite for PL
+      # Lite shall have pin version, so to keep development aligned we need to install lite from source not from pypi
+      # also installing from source not from on-the-fly created package tp prevent accidental interaction with cache
+      if: ${{ matrix.pkg-name != 'lightning' }}
+      env:
+        PACKAGE_NAME: "lite"
+      run: |
+        pip install -e . --upgrade --find-links ${TORCH_URL}
+        git checkout -- setup.py MANIFEST.in
+
     - name: Install package & dependencies
       env:
         PACKAGE_NAME: ${{ matrix.pkg-name }}