Merge branch 'master' into master

Author: Borda

.actions/assistant.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import glob
+import logging
 import os
 import pathlib
 import re
@@ -25,7 +26,7 @@
 from pathlib import Path
 from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union
 
-from pkg_resources import parse_requirements, Requirement, yield_lines
+from pkg_resources import Requirement, parse_requirements, yield_lines
 
 REQUIREMENT_FILES = {
     "pytorch": (
@@ -35,14 +36,19 @@
         "requirements/pytorch/examples.txt",
     ),
     "app": (
-        "requirements/app/base.txt",
-        "requirements/app/ui.txt",
+        "requirements/app/app.txt",
         "requirements/app/cloud.txt",
+        "requirements/app/ui.txt",
     ),
     "fabric": (
         "requirements/fabric/base.txt",
         "requirements/fabric/strategies.txt",
     ),
+    "data": (
+        "requirements/data/data.txt",
+        "requirements/data/cloud.txt",
+        "requirements/data/examples.txt",
+    ),
 }
 REQUIREMENT_FILES_ALL = list(chain(*REQUIREMENT_FILES.values()))
 
@@ -80,6 +86,7 @@ def adjust(self, unfreeze: str) -> str:
         'arrow>=1.2.0'
         >>> _RequirementWithComment("arrow").adjust("major")
         'arrow'
+
         """
         out = str(self)
         if self.strict:
@@ -109,6 +116,7 @@ def _parse_requirements(strs: Union[str, Iterable[str]]) -> Iterator[_Requiremen
     >>> txt = '\\n'.join(txt)
     >>> [r.adjust('none') for r in _parse_requirements(txt)]
     ['this', 'example', 'foo  # strict', 'thing']
+
     """
     lines = yield_lines(strs)
     pip_argument = None
@@ -142,10 +150,14 @@ def load_requirements(path_dir: str, file_name: str = "base.txt", unfreeze: str
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
     >>> load_requirements(path_req, "docs.txt", unfreeze="major")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-    ['sphinx<6.0,>=4.0', ...]
+    ['sphinx<...]
+
     """
     assert unfreeze in {"none", "major", "all"}
     path = Path(path_dir) / file_name
+    if not path.exists():
+        logging.warning(f"Folder {path_dir} does not have any base requirements.")
+        return []
     assert path.exists(), (path_dir, file_name, path)
     text = path.read_text()
     return [req.adjust(unfreeze) for req in _parse_requirements(text)]
@@ -156,6 +168,7 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
 
     >>> load_readme_description(_PROJECT_ROOT, "", "")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     '...PyTorch Lightning is just organized PyTorch...'
+
     """
     path_readme = os.path.join(path_dir, "README.md")
     with open(path_readme, encoding="utf-8") as fo:
@@ -187,13 +200,12 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
     skip_begin = r"<!-- following section will be skipped from PyPI description -->"
     skip_end = r"<!-- end skipping PyPI description -->"
     # todo: wrap content as commented description
-    text = re.sub(rf"{skip_begin}.+?{skip_end}", "<!--  -->", text, flags=re.IGNORECASE + re.DOTALL)
+    return re.sub(rf"{skip_begin}.+?{skip_end}", "<!--  -->", text, flags=re.IGNORECASE + re.DOTALL)
 
     # # https://github.com/Borda/pytorch-lightning/releases/download/1.1.0a6/codecov_badge.png
     # github_release_url = os.path.join(homepage, "releases", "download", version)
     # # download badge and replace url with local file
     # text = _parse_for_badge(text, github_release_url)
-    return text
 
 
 def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
@@ -208,7 +220,7 @@ def distribute_version(src_folder: str, ver_file: str = "version.info") -> None:
         shutil.copy2(ver_template, fpath)
 
 
-def _download_frontend(pkg_path: str):
+def _download_frontend(pkg_path: str, version: str = "v0.0.0"):
     """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
     directory."""
 
@@ -218,13 +230,13 @@ def _download_frontend(pkg_path: str):
 
         shutil.rmtree(frontend_dir, ignore_errors=True)
         # TODO: remove this once lightning-ui package is ready as a dependency
-        frontend_release_url = "https://storage.googleapis.com/grid-packages/lightning-ui/v0.0.0/build.tar.gz"
+        frontend_release_url = f"https://lightning-packages.s3.amazonaws.com/ui/{version}.tar.gz"
         response = urllib.request.urlopen(frontend_release_url)
 
         file = tarfile.open(fileobj=response, mode="r|gz")
         file.extractall(path=download_dir)
 
-        shutil.move(os.path.join(download_dir, "build"), frontend_dir)
+        shutil.move(download_dir, frontend_dir)
         print("The Lightning UI has successfully been downloaded!")
 
     # If installing from source without internet connection, we don't want to break the installation
@@ -236,11 +248,12 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme
     """Load all base requirements from all particular packages and prune duplicates.
 
     >>> _load_aggregate_requirements(os.path.join(_PROJECT_ROOT, "requirements"))
+
     """
     requires = [
         load_requirements(d, unfreeze="none" if freeze_requirements else "major")
         for d in glob.glob(os.path.join(req_dir, "*"))
-        # skip empty folder as git artefacts, and resolving Will's special issue
+        # skip empty folder (git artifacts), and resolving Will's special issue
         if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and not os.path.basename(d).startswith("_")
     ]
     if not requires:
@@ -292,6 +305,7 @@ def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning
      'http://pytorch_lightning.ai', \
      'from lightning_fabric import __version__', \
      '@lightning.ai']
+
     """
     out = lines[:]
     for source_import, target_import in mapping:
@@ -404,6 +418,7 @@ def _replace_min(fname: str) -> None:
     def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
         """Replace the min package version by fixed one."""
         for fname in requirement_fnames:
+            print(fname)
             AssistantCLI._replace_min(fname)
 
     @staticmethod
@@ -421,6 +436,56 @@ def copy_replace_imports(
             source_dir, source_imports, target_imports, target_dir=target_dir, lightning_by=lightning_by
         )
 
+    @staticmethod
+    def pull_docs_files(
+        gh_user_repo: str,
+        target_dir: str = "docs/source-pytorch/XXX",
+        checkout: str = "refs/tags/1.0.0",
+        source_dir: str = "docs/source",
+        as_orphan: bool = False,
+    ) -> None:
+        """Pull docs pages from external source and append to local docs."""
+        import zipfile
+
+        zip_url = f"https://github.com/{gh_user_repo}/archive/{checkout}.zip"
+
+        with tempfile.TemporaryDirectory() as tmp:
+            zip_file = os.path.join(tmp, "repo.zip")
+            try:
+                urllib.request.urlretrieve(zip_url, zip_file)
+            except urllib.error.HTTPError:
+                raise RuntimeError(f"Requesting file '{zip_url}' does not exist or it is just unavailable.")
+
+            with zipfile.ZipFile(zip_file, "r") as zip_ref:
+                zip_ref.extractall(tmp)
+
+            zip_dirs = [d for d in glob.glob(os.path.join(tmp, "*")) if os.path.isdir(d)]
+            # check that the extracted archive has only repo folder
+            assert len(zip_dirs) == 1
+            repo_dir = zip_dirs[0]
+
+            ls_pages = glob.glob(os.path.join(repo_dir, source_dir, "*.rst"))
+            ls_pages += glob.glob(os.path.join(repo_dir, source_dir, "**", "*.rst"))
+            for rst in ls_pages:
+                rel_rst = rst.replace(os.path.join(repo_dir, source_dir) + os.path.sep, "")
+                rel_dir = os.path.dirname(rel_rst)
+                os.makedirs(os.path.join(_PROJECT_ROOT, target_dir, rel_dir), exist_ok=True)
+                new_rst = os.path.join(_PROJECT_ROOT, target_dir, rel_rst)
+                if os.path.isfile(new_rst):
+                    logging.warning(f"Page {new_rst} already exists in the local tree so it will be skipped.")
+                    continue
+                AssistantCLI._copy_rst(rst, new_rst, as_orphan=as_orphan)
+
+    @staticmethod
+    def _copy_rst(rst_in, rst_out, as_orphan: bool = False):
+        """Copy RST page with optional inserting orphan statement."""
+        with open(rst_in, encoding="utf-8") as fopen:
+            page = fopen.read()
+        if as_orphan and ":orphan:" not in page:
+            page = ":orphan:\n\n" + page
+        with open(rst_out, "w", encoding="utf-8") as fopen:
+            fopen.write(page)
+
 
 if __name__ == "__main__":
     import jsonargparse

.azure/README.md

@@ -44,3 +44,27 @@ done
 ```bash
 ps aux | grep start.sh
 ```
+
+# Machine maintenance
+
+Since most of our jobs/checks are running in a Docker container, the OS/machine can become polluted and fail to run with errors such as:
+
+```
+No space left on device : '/azp/agent-litGPU-21_0,1/_diag/pages/8bb191f4-a8c2-419a-8788-66e3f0522bea_1.log'
+```
+
+In such cases, you need to log in to the machine and run `docker system prune`.
+
+## Automated ways
+
+Let's explore adding a cron job for periodically removing all Docker caches:
+
+1. Open your user's cron tab for editing: `crontab -e`
+1. Schedule/add the command with the `--force` flag to force pruning without interactive confirmation:
+   ```bash
+   # every day at 2:00 AM clean docker caches
+   0 2 * * * docker system prune --force
+   ```
+1. Verify the entry: `crontab -l`
+
+Note: You may need to add yourself to the Docker group by running `sudo usermod -aG docker <your_username>` to have permission to execute this command without needing `sudo` and entering the password.