support multi node

Author: hephaex

vendor/ngc-tensorflow/runner-scripts/bootstrap.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+BAI_MULTINODE_CONFIG_TF=$(/opt/backend.ai/bin/python /opt/container/setup_multinode.py)
+if [ -z "$BAI_MULTINODE_CONFIG_TF" ];
+then
+	echo "";
+else
+	echo ${BAI_MULTINODE_CONFIG_TF}
+	export TF_CONFIG="${BAI_MULTINODE_CONFIG_TF}"
+fi

vendor/ngc-tensorflow/runner-scripts/setup_multinode.py

@@ -0,0 +1,19 @@
+import os
+import json
+
+if 'BACKENDAI_CLUSTER_HOST' in os.environ: # Start mutli-instance setup.
+    env = {}
+    env['cluster'] = {}
+    env['cluster']['worker'] = []
+    for container in os.environ['BACKENDAI_CLUSTER_HOSTS'].split(","):
+        env['cluster']['worker'].append(container + ":2220")
+    env['task'] = {}
+    if os.environ['BACKENDAI_CLUSTER_ROLE'] == 'main':
+        env['task']['type'] = "worker" # Was chief. but recent TF choose first worker as chief.
+        env['task']["index"] = str(int(os.environ['BACKENDAI_CLUSTER_IDX']) - 1) # Index starts from 0
+    else:
+        env['task']['type'] = "worker"
+        env['task']["index"] = os.environ['BACKENDAI_CLUSTER_IDX'] 
+    print(json.dumps(env))
+else:
+    print("")