Merge pull request #22 from sql-machine-learning/lhw_develop

Add all-in-one start.sh

Author: lhw362950217

Vagrantfile

@@ -5,14 +5,25 @@ Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu/bionic64"
   config.vm.provision "shell", path: "provision.bash"
 
+  # Enlarge disk size from default '10G' to '20G'
+  # This need the vagrant-disksize plugin which is installed in install.bash
+  config.disksize.size = '20GB'
+
   # Don't forward 22.  Even if we do so, the exposed port only binds
   # to 127.0.0.1, but not 0.0.0.0.  Other ports binds to all IPs.
   config.vm.network "forwarded_port", guest: 3306, host: 3306,
                     auto_correct: true
   config.vm.network "forwarded_port", guest: 50051, host: 50051,
                     auto_correct: true
+  # Jupyter Notebook
   config.vm.network "forwarded_port", guest: 8888, host: 8888,
                     auto_correct: true
+  # minikube dashboard
+  config.vm.network "forwarded_port", guest: 9000, host: 9000,
+                    auto_correct: true
+  # Argo dashboard
+  config.vm.network "forwarded_port", guest: 9001, host: 9001,
+                    auto_correct: true
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 16384

install.bash

@@ -38,6 +38,11 @@ if [[ -n "$(vagrant global-status --prune | grep -o 'playground')" ]]; then
   exit 0
 fi
 
+echo "Installing vagrant disk size plugin..."
+if [[ -z "$(vagrant plugin list | grep -o 'vagrant-disksize')" ]]; then
+    vagrant plugin install vagrant-disksize
+fi
+
 if [[ "$WE_ARE_IN_CHINA" ]]; then
   if [[ -z "$(vagrant box list | grep -o ubuntu/bionic64)" ]]; then
     echo "Download ubuntu box beforehand..."

provision.bash

@@ -5,20 +5,22 @@ set -e  # Exit script if any error
 # The shared folder is specified in Vagrantfile.
 VAGRANT_SHARED_FOLDER=/home/vagrant/desktop
 
-. $VAGRANT_SHARED_FOLDER/sqlflow/docker/dev/find_fastest_resources.sh
+source $VAGRANT_SHARED_FOLDER/sqlflow/docker/dev/find_fastest_resources.sh
 
 echo "Setting apt-get mirror..."
-$(find_fastest_apt_source >/etc/apt/sources.list)
+find_fastest_apt_source >/etc/apt/sources.list
+apt-get update
 
 echo "Installing Docker ..."
 # c.f. https://dockr.ly/3cExcay
-if which docker > /dev/null; then
+if ! which docker > /dev/null; then
     echo "Docker had been installed. Skip."
 else
     best_install_url=$(find_fastest_docker_url)
+    docker_ce_mirror=$(find_fastest_docker_ce_mirror)
     echo "Using ${best_install_url}..."
-    curl -sSL ${best_install_url} | sh -
-    best_docker_mirror=$(find_fastest_docker_mirror)
+    curl -sSL "${best_install_url}" | DOWNLOAD_URL=$docker_ce_mirror bash -
+    best_docker_mirror=$(find_fastest_docker_registry)
     if [[ -n "${best_docker_mirror}" ]]; then
         mkdir -p /etc/docker
         cat <<-EOF >/etc/docker/daemon.json

sqlflow

@@ -12,18 +12,125 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
-
 echo "Docker pull SQLFlow images ..."
 # c.f. https://github.com/sql-machine-learning/sqlflow/blob/develop/.travis.yml
-docker pull sqlflow/sqlflow:latest
+docker pull sqlflow/sqlflow:jupyter
+docker pull sqlflow/sqlflow:mysql
+docker pull sqlflow/sqlflow:server
+docker pull sqlflow/sqlflow:step
 echo "Done."
 
 # NOTE: According to https://stackoverflow.com/a/16619261/724872,
 # source is very necessary here.
 source $(dirname $0)/sqlflow/scripts/travis/export_k8s_vars.sh
+source $(dirname $0)/sqlflow/docker/dev/find_fastest_resources.sh
+
+# (FIXME:lhw) If grep match nothing and return 1, do not exit
+# Find a way that we do not need to use 'set -e'
+set +e
+
+# Use a faster kube image and docker registry
+echo "Start minikube cluster ..."
+minikube_status=$(minikube status | grep "apiserver: Running")
+if [[ "$minikube_status" == "apiserver: Running" ]]; then
+  echo "Already in running."
+else
+    ali_kube="http://kubernetes.oss-cn-hangzhou.aliyuncs.com"
+    google_kube="http://k8s.gcr.io"
+    fast_kube_site=$(find_fastest_url $ali_kube $google_kube)
+    if [[ "$fast_kube_site" == "$ali_kube" ]]; then
+        sudo minikube start --image-mirror-country cn \
+          --registry-mirror=https://registry.docker-cn.com --driver=none \
+          --kubernetes-version=v"$K8S_VERSION"
+    else
+        sudo minikube start \
+          --vm-driver=none \
+          --kubernetes-version=v"$K8S_VERSION"
+    fi
+fi
+
+# Test if a Kubernetes resource is alive
+# "$1" shoulde be namespace id e.g. argo
+# "$2" should be resource id e.g. pod/argo-server
+function is_resource_alive() {
+    local type=$(echo "$2" | cut -d / -f1)
+    local name=$(echo "$2" | cut -d / -f2)
+    if kubectl get -n "$1" "$2" | grep -q -o "$name" >/dev/null; then
+        # make sure relative pod is alive
+        if kubectl get pod -n "$1" | grep "$name" | grep "Running" >/dev/null; then
+            echo "yes"
+        else
+            echo "no"
+        fi
+    else
+        echo "no"
+    fi
+}
+
+echo "Start argo ..."
+argo_server_alive=$(is_resource_alive "argo" "service/argo-server")
+if [[ "$argo_server_alive" == "yes" ]]; then
+    echo "Already in running."
+else
+    $(dirname $0)/sqlflow/scripts/travis/start_argo.sh
+fi
+
+echo "Strat Kubernetes Dashboard..."
+dashboard_alive=$(is_resource_alive "kubernetes-dashboard" "service/kubernetes-dashboard")
+if [[ "$dashboard_alive" == "yes" ]]; then
+    echo "Already in running."
+else
+    nohup minikube dashboard &
+fi
+
+echo "Strat SQLFlow ..."
+sqlflow_alive=$(is_resource_alive "default" "pod/sqlflow-server")
+if [[ "$sqlflow_alive" == "yes" ]]; then
+    echo "Already in running."
+else
+    kubectl apply -f sqlflow/doc/run/k8s/install-sqlflow.yaml
+fi
+
+# Kill port exposing if it already exist
+function stop_expose() {
+    ps -elf | grep "kubectl port-forward" | grep "$1" | grep "$2" | awk '{print $4}' | xargs kill  >/dev/null
+}
+
+# Kubernetes port-forwarding
+# "$1" should be namespace
+# "$2" should be resource, e.g. service/argo-server
+# "$3" should be port mapping, e.g. 8000:80
+function expose() {
+    echo "Stop exposing $3..."
+    stop_expose "$2" "$3"
+    echo "Detecting service and exposing port ..."
+    while [[ true ]]; do
+        local alive=$(is_resource_alive "$1" "$2")
+        if [[ "$alive" == "yes" ]]; then
+            break
+        fi
+        sleep 1
+    done
+    nohup kubectl port-forward -n $1 --address='0.0.0.0' $2 $3 >>port-forward-log 2>&1 &
+}
+
+# (NOTE) after re-deploy sqlflow we have to re-expose the service ports.
+expose kubernetes-dashboard service/kubernetes-dashboard 9000:80
+expose argo service/argo-server 9001:2746
+expose default pod/sqlflow-server 8888:8888
+expose default pod/sqlflow-server 3306:3306
+
+jupyter_addr=$(kubectl logs pod/sqlflow-server notebook | grep -o -E "http://127.0.0.1[^?]+\?token=.*" | head -1)
+
+cat <<EOF
+Congratulations, SQLFlow playground is up!
+
+Access Jupyter Notebook at: $jupyter_addr
+Access Kubernetes Dashboard at: http://localhost:9000
+Access Argo Dashboard at: http://localhost:9001
+Access SQLFlow with cli: refer to https://github.com/sql-machine-learning/sqlflow/blob/develop/doc/run/cli.md
 
-$(dirname $0)/sqlflow/scripts/travis/start_minikube.sh
-sudo chown -R vagrant: $HOME/.minikube/
+Stop minikube with: minikube stop
+Stop vagrant with: vagrant halt
+EOF
 
-$(dirname $0)/sqlflow/scripts/travis/start_argo.sh