Merge pull request #254 from istalker2/etcd-app-ha

etcd app with HA

Author: pigmej

examples/etcd/README.md

@@ -5,7 +5,7 @@ etcd cluster application
 
     1. Start AppController Pod. Suppose its name is k8s-appcontroller
     2. Upload etcd graph definition resources to k8s by running create.sh.
-       This script uploads two AppController flows: `etcd-bootstrap` and `etcd-scale`.
+       This script uploads three AppController flows: `etcd-bootstrap`, `etcd-scale` and `etcd-recover`.
     3. This application uses custom etcd docker image which is the original etcd image plus kubectl binary.
        Use build-image.sh script to build the image `etcd-kubectl` and then publish it to the docker repository
        used by your k8s environment. 
@@ -19,6 +19,8 @@ etcd cluster application
 `--arg clusterName=my-cluster` - cluster name. This allows to deploy several independent cluster in one k8s namespace. 
 If omitted, `etcd` name is used by default.
 
+`--arg livenessProbeInterval=30` - how often monitoring job examines cluster health (in seconds, default is 30).
+
 3. Scale the cluster:
 
 `kubectl exec k8s-appcontroller kubeac run etcd-scale -n +1 --arg clusterName=my-cluster`
@@ -28,3 +30,14 @@ added node is going to be deleted. At the moment it is only possible to scale cl
 However, any number of nodes can be removed. Note, that this can also remove nodes created upon initial deployment. 
 
 `--arg clusterName=my-cluster` - name of the cluster to scale (`etcd` if not specified).
+
+4. Recover broken cluster nodes
+
+`kubectl exec k8s-appcontroller kubeac run etcd-recover --arg clusterName=my-cluster --arg nodeSuffix=abcde`
+
+`--arg clusterName=my-cluster` - name of the cluster to scale (`etcd` if not specified).
+
+`--arg nodeSuffix=abcde` - name suffix of the broken node (XXX in http://etcd-XXX:2379)
+
+However, there is no need to invoke this flow manually as the bootstrap flow creates a job that checks and
+recovers broken nodes on the regular basis

examples/etcd/create.sh

@@ -11,7 +11,5 @@ do
   cat $file | $KUBECTL_NAME exec -i $AC_POD_NAME kubeac wrap | $KUBECTL_NAME create -f-
 done
 
-for file in dependencies/*.yaml
-do 
-  cat $file | $KUBECTL_NAME create -f-
-done
+$KUBECTL_NAME create -f dependencies/
+$KUBECTL_NAME create -f resources/

examples/etcd/dependencies/etcd-bootstrap.yaml

@@ -30,6 +30,17 @@ kind: Dependency
 metadata:
   generateName: dependency-
   labels:
-    flow: cleanup
+    flow: etcd-bootstrap
+parent: pod/initial-etcd-pod-$AC_NAME
+child: job/monitor-$clusterName-job
+---
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Dependency
+metadata:
+  generateName: dependency-
+  labels:
+    flow: etcd-bootstrap-cleanup
 parent: flow/etcd-bootstrap
-child: job/delete-node-$AC_NAME-job
+child: job/delete-node-$NAME-job
+args:
+  NAME: $AC_NAME

examples/etcd/dependencies/etcd-recover.yaml

@@ -0,0 +1,21 @@
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Dependency
+metadata:
+  generateName: dependency-
+  labels:
+    flow: etcd-recover
+parent: flow/etcd-recover
+child: job/delete-node-$NAME-job
+args:
+  NAME: $nodeSuffix
+---
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Dependency
+metadata:
+  generateName: dependency-
+  labels:
+    flow: etcd-recover
+parent: job/delete-node-$NAME-job
+child: flow/etcd-scale
+args:
+  clusterName: $clusterName

examples/etcd/dependencies/etcd-scale.yaml

@@ -30,7 +30,9 @@ kind: Dependency
 metadata:
   generateName: dependency-
   labels:
-    flow: cleanup
+    flow: etcd-scale-cleanup
 parent: flow/etcd-scale
-child: job/delete-node-$AC_NAME-job
+child: job/delete-node-$NAME-job
+args:
+  NAME: $AC_NAME
 

examples/etcd/resdefs/bootstrap-flow.yaml

@@ -7,11 +7,14 @@ exported: true
 construction:
   flow: etcd-bootstrap
 destruction:
-  flow: cleanup
+  flow: etcd-bootstrap-cleanup
 
 replicaSpace: etcd-$clusterName-node
 
 parameters:
   clusterName:
     description: etcd cluster name
     default: etcd
+  livenessProbeInterval:
+    description: how often monitoring job examines cluster health (in seconds)
+    default: "30"

examples/etcd/resdefs/delete-node-job.yaml

@@ -0,0 +1,17 @@
+apiVersion: appcontroller.k8s/v1alpha1
+kind: Flow
+metadata:
+  name: etcd-recover
+
+exported: true
+construction:
+  flow: etcd-recover
+
+replicaSpace: etcd-recover-$clusterName
+
+parameters:
+  clusterName:
+    description: etcd cluster name
+    default: etcd
+  nodeSuffix:
+    description: node name suffix (X in http://etcd-X:2379)

examples/etcd/resdefs/recover-flow.yaml

@@ -7,7 +7,7 @@ exported: true
 construction:
   flow: etcd-scale
 destruction:
-  flow: cleanup
+  flow: etcd-scale-cleanup
 
 replicaSpace: etcd-$clusterName-node
 

examples/etcd/resdefs/scale-flow.yaml

@@ -7,6 +7,7 @@ metadata:
     serviceRole: cluster
   name: $clusterName-client
 spec:
+  type: NodePort
   ports:
   - name: etcd-client-port
     port: 2379