environment config

Author: hephaex

python-pytorchlightning/adjust_version.py

@@ -0,0 +1,50 @@
+import os
+import re
+import sys
+from typing import Any, Dict
+
+VERSIONS_LUT: Dict[str, Dict[str, Any]] = {
+    "1.4.0": dict(torchvision="0.5.0", torchtext="0.5"),
+    "1.5.0": dict(torchvision="0.6.0", torchtext="0.6"),
+    "1.5.1": dict(torchvision="0.6.1", torchtext="0.6"),
+    "1.6.0": dict(torchvision="0.7.0", torchtext="0.7"),
+    "1.7.0": dict(torchvision="0.8.1", torchtext="0.8"),
+    "1.7.1": dict(torchvision="0.8.2", torchtext="0.8.1"),
+    "1.8.0": dict(torchvision="0.9.0", torchtext="0.9"),
+    "1.8.1": dict(torchvision="0.9.0", torchtext="0.9"),
+}
+
+
+def find_latest(ver: str, versions_all: list) -> str:
+    # drop all except semantic version
+    ver = re.search(r'([\.\d]+)', ver).groups()[0]
+    # find candidates, by starting version pattern
+    options = [v for v in versions_all if v.startswith(ver)]
+    assert options, f"missing {ver} among {versions_all}"
+    # take the last one...
+    return sorted(options)[-1]
+
+
+def main(path_req: str, torch_version: str = None) -> None:
+    with open(path_req, "r") as fp:
+        req = fp.read()
+
+    if not torch_version:
+        import torch
+        torch_version = torch.__version__
+    assert torch_version, f"invalid/missing Torch: {torch_version}"
+
+    torch_version = find_latest(torch_version, list(VERSIONS_LUT.keys()))
+    dep_versions = VERSIONS_LUT[torch_version]
+    dep_versions["torch"] = torch_version
+    for lib in dep_versions:
+        version = dep_versions[lib]
+        replace = f"{lib}=={version}\n"
+        req = re.sub(rf"{lib}[>=]*[\d\.]*{os.linesep}", replace, req)
+
+    with open(path_req, "w") as fp:
+        fp.write(req)
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])

python-pytorchlightning/environment.yml

@@ -0,0 +1,39 @@
+name:
+    lightning
+
+channels:
+    - conda-forge
+    - pytorch
+    - pytorch-test
+    - pytorch-nightly
+
+dependencies:
+    - python>=3.6
+    - pip>20.1
+    - numpy>=1.16.4
+    - pytorch>=1.4
+    - future>=0.17.1
+    - PyYAML>=5.1
+    - tqdm>=4.41.0
+    - fsspec[http]>=0.8.1
+    #- tensorboard>=2.2.0  # not needed, already included in pytorch
+
+    # Optional
+    #- nvidia-apex  # missing for py3.8
+    - scikit-learn>=0.20.0
+    - matplotlib>=3.1.1
+    - omegaconf>=2.0.0
+    - torchtext>=0.5
+
+    # Examples
+    - torchvision>=0.5
+
+    - pip:
+        - test-tube>=0.7.5
+        - mlflow>=1.0.0
+        - comet_ml>=3.1.12
+        - wandb>=0.8.21
+        - neptune-client>=0.4.109
+        - horovod>=0.21.2
+        - onnxruntime>=1.3.0
+        - gym>=0.17.0

python-pytorchlightning/extra.txt

@@ -0,0 +1,12 @@
+# extended list of package dependencies to reach full functionality
+
+matplotlib>3.1
+horovod>=0.21.2  # no need to install with [pytorch] as pytorch is already installed
+omegaconf>=2.0.1
+torchtext>=0.5
+# onnx>=1.7.0
+onnxruntime>=1.3.0
+hydra-core>=1.0
+# todo: when switch to standard package stream, drop `fairscale` from hard mocked docs libs
+https://github.com/PyTorchLightning/fairscale/archive/pl_1.2.0.zip
+jsonargparse[signatures]>=3.9.0

python-pytorchlightning/runner-scripts/bootstrap.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+BAI_MULTINODE_CONFIG_TF=$(/opt/backend.ai/bin/python /opt/container/setup_multinode.py)
+if [ -z "$BAI_MULTINODE_CONFIG_TF" ];
+then
+	echo "";
+else
+	echo ${BAI_MULTINODE_CONFIG_TF}
+	export TF_CONFIG="${BAI_MULTINODE_CONFIG_TF}"
+fi

python-pytorchlightning/runner-scripts/setup_multinode.py

@@ -0,0 +1,19 @@
+import os
+import json
+
+if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
+    env = {}
+    env['cluster'] = {}
+    env['cluster']['worker'] = []
+    for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
+        env['cluster']['worker'].append(container + ":2220")
+    env['task'] = {}
+    if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
+        env['task']['type'] = "worker" # Was chief. but recent TF choose first worker as chief.
+        env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0
+    else:
+        env['task']['type'] = "worker"
+        env['task']["index"] = os.environ['BACKENDAI_CLUSTER_IDX'] 
+    print(json.dumps(env))
+else:
+    print("")

python-pytorchlightning/service-defs/digits.json

@@ -0,0 +1,5 @@
+{
+    "command": [
+        "{runtime_path}", "-m", "digits"
+    ]
+}

python-pytorchlightning/service-defs/ipython.json

@@ -0,0 +1,5 @@
+{
+    "command": [
+        "{runtime_path}", "-m", "IPython"
+    ]
+}

python-pytorchlightning/service-defs/jupyter.json

@@ -0,0 +1,20 @@
+{
+  "prestart": [
+    {
+      "action": "write_tempfile",
+      "args": {
+        "body": [
+          "c.NotebookApp.allow_root = True\n",
+          "c.NotebookApp.ip = \"0.0.0.0\"\n",
+          "c.NotebookApp.port = {ports[0]}\n",
+          "c.NotebookApp.token = \"\"\n",
+          "c.FileContentsManager.delete_to_trash = False\n"
+        ]
+      },
+      "ref": "jupyter_cfg"
+    }
+  ],
+  "command": [
+    "{runtime_path}", "-m", "notebook", "--no-browser", "--config", "{jupyter_cfg}"
+  ]
+}

python-pytorchlightning/service-defs/jupyterlab.json

@@ -0,0 +1,20 @@
+{
+  "prestart": [
+    {
+      "action": "write_tempfile",
+      "args": {
+        "body": [
+          "c.NotebookApp.allow_root = True\n",
+          "c.NotebookApp.ip = \"0.0.0.0\"\n",
+          "c.NotebookApp.port = {ports[0]}\n",
+          "c.NotebookApp.token = \"\"\n",
+          "c.FileContentsManager.delete_to_trash = False\n"
+        ]
+      },
+      "ref": "jupyter_cfg"
+    }
+  ],
+  "command": [
+    "{runtime_path}", "-m", "jupyterlab", "--no-browser", "--config", "{jupyter_cfg}"
+  ]
+}

python-pytorchlightning/service-defs/mlflow-ui.json

@@ -0,0 +1,9 @@
+{
+  "prestart": [],
+  "command": [
+    "/usr/local/bin/mlflow",
+    "ui",
+    "--host",
+    "0.0.0.0"
+  ]
+}