Merge branch 'master' into patch-2

Author: Borda

.actions/assistant.py

@@ -14,19 +14,18 @@
 import glob
 import logging
 import os
-import pathlib
 import re
 import shutil
-import tarfile
 import tempfile
 import urllib.request
-from distutils.version import LooseVersion
+from collections.abc import Iterable, Iterator, Sequence
 from itertools import chain
 from os.path import dirname, isfile
 from pathlib import Path
-from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union
+from typing import Any, Optional
 
-from pkg_resources import Requirement, parse_requirements, yield_lines
+from packaging.requirements import Requirement
+from packaging.version import Version
 
 REQUIREMENT_FILES = {
     "pytorch": (
@@ -35,20 +34,11 @@
         "requirements/pytorch/strategies.txt",
         "requirements/pytorch/examples.txt",
     ),
-    "app": (
-        "requirements/app/app.txt",
-        "requirements/app/cloud.txt",
-        "requirements/app/ui.txt",
-    ),
     "fabric": (
         "requirements/fabric/base.txt",
         "requirements/fabric/strategies.txt",
     ),
-    "data": (
-        "requirements/data/data.txt",
-        "requirements/data/cloud.txt",
-        "requirements/data/examples.txt",
-    ),
+    "data": ("requirements/data/data.txt",),
 }
 REQUIREMENT_FILES_ALL = list(chain(*REQUIREMENT_FILES.values()))
 
@@ -91,14 +81,15 @@ def adjust(self, unfreeze: str) -> str:
         out = str(self)
         if self.strict:
             return f"{out}  {self.strict_string}"
+        specs = [(spec.operator, spec.version) for spec in self.specifier]
         if unfreeze == "major":
-            for operator, version in self.specs:
+            for operator, version in specs:
                 if operator in ("<", "<="):
-                    major = LooseVersion(version).version[0]
+                    major = Version(version).major
                     # replace upper bound with major version increased by one
                     return out.replace(f"{operator}{version}", f"<{major + 1}.0")
         elif unfreeze == "all":
-            for operator, version in self.specs:
+            for operator, version in specs:
                 if operator in ("<", "<="):
                     # drop upper bound
                     return out.replace(f"{operator}{version},", "")
@@ -107,33 +98,25 @@ def adjust(self, unfreeze: str) -> str:
         return out
 
 
-def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_RequirementWithComment]:
+def _parse_requirements(lines: Iterable[str]) -> Iterator[_RequirementWithComment]:
     """Adapted from `pkg_resources.parse_requirements` to include comments.
 
     >>> txt = ['# ignored', '', 'this # is an', '--piparg', 'example', 'foo # strict', 'thing', '-r different/file.txt']
     >>> [r.adjust('none') for r in _parse_requirements(txt)]
     ['this', 'example', 'foo  # strict', 'thing']
-    >>> txt = '\\n'.join(txt)
-    >>> [r.adjust('none') for r in _parse_requirements(txt)]
-    ['this', 'example', 'foo  # strict', 'thing']
 
     """
-    lines = yield_lines(strs)
     pip_argument = None
     for line in lines:
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
         # Drop comments -- a hash without a space may be in a URL.
         if " #" in line:
             comment_pos = line.find(" #")
             line, comment = line[:comment_pos], line[comment_pos:]
         else:
             comment = ""
-        # If there is a line continuation, drop it, and append the next line.
-        if line.endswith("\\"):
-            line = line[:-2].strip()
-            try:
-                line += next(lines)
-            except StopIteration:
-                return
         # If there's a pip argument, save it
         if line.startswith("--"):
             pip_argument = line
@@ -145,7 +128,7 @@ def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_Requiremen
         pip_argument = None
 
 
-def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> List[str]:
+def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str = "all") -> list[str]:
     """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
@@ -159,7 +142,7 @@ def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str
         logging.warning(f"Folder {path_dir} does not have any base requirements.")
         return []
     assert path.exists(), (path_dir, file_name, path)
-    text = path.read_text()
+    text = path.read_text().splitlines()
     return [req.adjust(unfreeze) for req in _parse_requirements(text)]
 
 
@@ -171,8 +154,8 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
 
     """
     path_readme = os.path.join(path_dir, "README.md")
-    with open(path_readme, encoding="utf-8") as fo:
-        text = fo.read()
+    with open(path_readme, encoding="utf-8") as fopen:
+        text = fopen.read()
 
     # drop images from readme
     text = text.replace(
@@ -220,30 +203,6 @@ def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
         shutil.copy2(ver_template, fpath)
 
 
-def _download_frontend(pkg_path: str, version: str = "v0.0.0"):
-    """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
-    directory."""
-
-    try:
-        frontend_dir = pathlib.Path(pkg_path, "ui")
-        download_dir = tempfile.mkdtemp()
-
-        shutil.rmtree(frontend_dir, ignore_errors=True)
-        # TODO: remove this once lightning-ui package is ready as a dependency
-        frontend_release_url = f"https://lightning-packages.s3.amazonaws.com/ui/{version}.tar.gz"
-        response = urllib.request.urlopen(frontend_release_url)
-
-        file = tarfile.open(fileobj=response, mode="r|gz")
-        file.extractall(path=download_dir)  # noqa: S202
-
-        shutil.move(download_dir, frontend_dir)
-        print("The Lightning UI has successfully been downloaded!")
-
-    # If installing from source without internet connection, we don't want to break the installation
-    except Exception:
-        print("The Lightning UI downloading has failed!")
-
-
 def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None:
     """Load all base requirements from all particular packages and prune duplicates.
 
@@ -264,7 +223,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
         fp.writelines([ln + os.linesep for ln in requires] + [os.linesep])
 
 
-def _retrieve_files(directory: str, *ext: str) -> List[str]:
+def _retrieve_files(directory: str, *ext: str) -> list[str]:
     all_files = []
     for root, _, files in os.walk(directory):
         for fname in files:
@@ -274,7 +233,7 @@ def _retrieve_files(directory: str, *ext: str) -> List[str]:
     return all_files
 
 
-def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning_by: str = "") -> List[str]:
+def _replace_imports(lines: list[str], mapping: list[tuple[str, str]], lightning_by: str = "") -> list[str]:
     """Replace imports of standalone package to lightning.
 
     >>> lns = [
@@ -349,31 +308,33 @@ def copy_replace_imports(
         if ext in (".pyc",):
             continue
         # Try to parse everything else
-        with open(fp, encoding="utf-8") as fo:
+        with open(fp, encoding="utf-8") as fopen:
             try:
-                lines = fo.readlines()
+                lines = fopen.readlines()
             except UnicodeDecodeError:
                 # a binary file, skip
                 print(f"Skipped replacing imports for {fp}")
                 continue
         lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
         os.makedirs(os.path.dirname(fp_new), exist_ok=True)
-        with open(fp_new, "w", encoding="utf-8") as fo:
-            fo.writelines(lines)
+        with open(fp_new, "w", encoding="utf-8") as fopen:
+            fopen.writelines(lines)
 
 
-def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None:
+def create_mirror_package(source_dir: str, package_mapping: dict[str, str]) -> None:
+    """Create a mirror package with adjusted imports."""
     # replace imports and copy the code
     mapping = package_mapping.copy()
     mapping.pop("lightning", None)  # pop this key to avoid replacing `lightning` to `lightning.lightning`
 
     mapping = {f"lightning.{sp}": sl for sp, sl in mapping.items()}
     for pkg_from, pkg_to in mapping.items():
+        source_imports, target_imports = zip(*mapping.items())
         copy_replace_imports(
             source_dir=os.path.join(source_dir, pkg_from.replace(".", os.sep)),
             # pytorch_lightning uses lightning_fabric, so we need to replace all imports for all directories
-            source_imports=mapping.keys(),
-            target_imports=mapping.values(),
+            source_imports=source_imports,
+            target_imports=target_imports,
             target_dir=os.path.join(source_dir, pkg_to.replace(".", os.sep)),
             lightning_by=pkg_from,
         )
@@ -401,26 +362,12 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
             if not ln_ or ln_.startswith("#"):
                 final.append(line)
                 continue
-            req = list(parse_requirements(ln_))[0]
+            req = list(_parse_requirements([ln_]))[0]
             if req.name not in packages:
                 final.append(line)
         print(final)
         path.write_text("\n".join(final) + "\n")
 
-    @staticmethod
-    def _replace_min(fname: str) -> None:
-        with open(fname, encoding="utf-8") as fo:
-            req = fo.read().replace(">=", "==")
-        with open(fname, "w", encoding="utf-8") as fw:
-            fw.write(req)
-
-    @staticmethod
-    def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
-        """Replace the min package version by fixed one."""
-        for fname in requirement_fnames:
-            print(fname)
-            AssistantCLI._replace_min(fname)
-
     @staticmethod
     def copy_replace_imports(
         source_dir: str,
@@ -468,7 +415,7 @@ def pull_docs_files(
                 raise RuntimeError(f"Requesting file '{zip_url}' does not exist or it is just unavailable.")
 
             with zipfile.ZipFile(zip_file, "r") as zip_ref:
-                zip_ref.extractall(tmp)  # noqa: S202
+                zip_ref.extractall(tmp)
 
             zip_dirs = [d for d in glob.glob(os.path.join(tmp, "*")) if os.path.isdir(d)]
             # check that the extracted archive has only repo folder
@@ -505,6 +452,40 @@ def _copy_rst(rst_in, rst_out, as_orphan: bool = False):
         with open(rst_out, "w", encoding="utf-8") as fopen:
             fopen.write(page)
 
+    @staticmethod
+    def convert_version2nightly(ver_file: str = "src/version.info") -> None:
+        """Load the actual version and convert it to the nightly version."""
+        from datetime import datetime
+
+        with open(ver_file) as fopen:
+            version = fopen.read().strip()
+        # parse X.Y.Z version and prune any suffix
+        vers = re.match(r"(\d+)\.(\d+)\.(\d+).*", version)
+        # create timestamp  YYYYMMDD
+        timestamp = datetime.now().strftime("%Y%m%d")
+        version = f"{'.'.join(vers.groups())}.dev{timestamp}"
+        with open(ver_file, "w") as fopen:
+            fopen.write(version + os.linesep)
+
+    @staticmethod
+    def generate_docker_tags(
+        release_version: str,
+        python_version: str,
+        torch_version: str,
+        cuda_version: str,
+        docker_project: str = "pytorchlightning/pytorch_lightning",
+        add_latest: bool = False,
+    ) -> None:
+        """Generate docker tags for the given versions."""
+        tags = [f"latest-py{python_version}-torch{torch_version}-cuda{cuda_version}"]
+        if release_version:
+            tags += [f"{release_version}-py{python_version}-torch{torch_version}-cuda{cuda_version}"]
+        if add_latest:
+            tags += ["latest"]
+
+        tags = [f"{docker_project}:{tag}" for tag in tags]
+        print(",".join(tags))
+
 
 if __name__ == "__main__":
     import jsonargparse

.actions/requirements.txt

@@ -1,2 +1,3 @@
-jsonargparse >=4.16.0, <4.28.0
+jsonargparse
 requests
+packaging