[New Scheduler] Etcd installation & Implements EtcdClient (#5031)

* Implements EtcdClient

* Add license header

* Add configuration for tests

* Exclude etcd config from controller & invoker

* Add config for etcd test

* Separate etcd test from whisk unit test

* Apply scala compilation

Author: KeonHee

ansible/environments/docker-machine/hosts.j2.ini

@@ -39,6 +39,11 @@ invoker1                    ansible_host={{ docker_machine_ip }}
 [elasticsearch:children]
 db
 
+[etcd]
+etcd0            ansible_host={{ docker_machine_ip }}
+{% if mode is defined and 'HA' in mode %}
+etcd1            ansible_host={{ docker_machine_ip }}
+
 ; define variables
 [all:vars]
 ansible_connection=ssh

ansible/environments/local/hosts.j2.ini

@@ -39,3 +39,6 @@ db
 
 [apigateway]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
+
+[etcd]
+etcd0            ansible_host=172.17.0.1 ansible_connection=local

ansible/environments/vagrant/hosts

@@ -27,3 +27,6 @@ invoker0            ansible_host=172.17.0.1 ansible_connection=local
 
 [apigateway]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
+
+[etcd]
+etcd0            ansible_host=172.17.0.1 ansible_connection=local

ansible/etcd.yml

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This playbook deploys Openwhisk Invokers.
+
+- hosts: etcd
+  roles:
+  - etcd

ansible/group_vars/all

@@ -51,6 +51,7 @@ whisk:
   feature_flags:
     require_api_key_annotation: "{{ require_api_key_annotation | default(true) | lower }}"
     require_response_payload: "{{ require_response_payload | default(true) | lower }}"
+  cluster_name: "{{ whisk_cluster_name | default('whisk') }}"
 
 ##
 # configuration parameters related to support runtimes (see org.apache.openwhisk.core.entity.ExecManifest for schema of the manifest).
@@ -424,3 +425,28 @@ user_events: "{{ user_events_enabled | default(false) | lower }}"
 
 durationChecker:
     timeWindow: "{{ duration_checker_time_window | default('1 d') }}"
+
+etcd:
+  version: "{{ etcd_version | default('v3.4.0') }}"
+  client:
+    port: 2379
+  server:
+    port: 2480
+  cluster:
+    token: "{{ etcd_cluster_token | default('openwhisk-etcd-token') }}"
+  dir:
+    data: "{{ etcd_data_dir | default('') }}"
+  lease:
+    timeout: "{{ etcd_lease_timeout | default(1) }}"
+  loglevel: "{{ etcd_log_level | default('info') }}"
+  quota_backend_bytes: "{{ etcd_quota_backend_bytes | default(0) }}"
+  snapshot_count: "{{ etcd_snapshot_count | default(100000) }}"
+  auto_compaction_retention: "{{ etcd_auto_compaction_retention | default(1) }}"
+  auto_compaction_mode: "{{ etcd_auto_compaction_mode | default('periodic') }}"
+  pool_threads: "{{ etcd_pool_threads | default(10) }}"
+
+etcd_connect_string: "{% set ret = [] %}\
+                      {% for host in groups['etcd'] %}\
+                        {{ ret.append( hostvars[host].ansible_host + ':' + ((etcd.client.port+loop.index-1)|string) ) }}\
+                      {% endfor %}\
+                      {{ ret | join(',') }}"

ansible/roles/etcd/tasks/clean.yml

@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# Remove etcd containers.
+
+- name: remove etcd
+  docker_container:
+    name: etcd{{ groups['etcd'].index(inventory_hostname) }}
+    keep_volumes: True
+    state: absent
+  ignore_errors: True

ansible/roles/etcd/tasks/deploy.yml

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will install etcd in group 'etcd' in the environment inventory
+
+- name: "Set the name of the etcd node"
+  set_fact:
+    name: "etcd{{ groups['etcd'].index(inventory_hostname) }}"
+
+- name: "set the volume_dir"
+  set_fact:
+    volume_dir: "{{ etcd.dir.data }}/etcd{{ groups['etcd'].index(inventory_hostname) }}:/etcd-data"
+  when: etcd_data_dir is defined
+
+
+- name: "Set the cluster of the etcd cluster"
+  set_fact:
+    cluster: "{% set etcdhosts = [] %}
+              {% for host in groups['etcd'] %}
+                  {{ etcdhosts.append('etcd' + ((loop.index-1)|string) + '=' + 'http://' + hostvars[host].ansible_host + ':' + ((2480+loop.index-1)|string) ) }}
+              {% endfor %}
+              {{ etcdhosts | join(',') }}"
+
+- name: (re)start etcd
+  docker_container:
+    name: etcd{{ groups['etcd'].index(inventory_hostname) }}
+    image: quay.io/coreos/etcd:{{ etcd.version }}
+    state: started
+    recreate: true
+    restart_policy: "{{ docker.restart.policy }}"
+    volumes: "{{volume_dir | default([])}}"
+    command: "/usr/local/bin/etcd \
+               --data-dir=/etcd-data --name '{{ name }}' \
+               --initial-advertise-peer-urls http://{{ ansible_host }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
+               --advertise-client-urls http://{{ ansible_host }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
+               --listen-peer-urls http://0.0.0.0:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
+               --listen-client-urls http://0.0.0.0:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
+               --initial-cluster {{ cluster }} \
+               --initial-cluster-state new --initial-cluster-token {{ etcd.cluster.token }} \
+               --quota-backend-bytes {{ etcd.quota_backend_bytes }} \
+               --snapshot-count {{ etcd.snapshot_count }} \
+               --auto-compaction-retention {{ etcd.auto_compaction_retention }} \
+               --auto-compaction-mode {{ etcd.auto_compaction_mode }} \
+               --log-level {{ etcd.loglevel }}"
+    ports:
+      - "{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}"
+      - "{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}"
+    pull: "{{ etcd.pull_etcd | default(true) }}"
+
+- name: wait until etcd in this host is up and running
+  uri:
+    url: "http://{{ ansible_host }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}/health"
+  register: result
+  until: result.status == 200
+  retries: 12
+  delay: 5

ansible/roles/etcd/tasks/main.yml

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will install etcd in group 'etcd' in the environment inventory
+# In deploy mode it will deploy etcd containers.
+# In clean mode it will remove etcd containers.
+
+- import_tasks: deploy.yml
+  when: mode == "deploy"
+
+- import_tasks: clean.yml
+  when: mode == "clean"

common/scala/build.gradle

@@ -73,6 +73,9 @@ dependencies {
     //for mesos
     compile "com.adobe.api.platform.runtime:mesos-actor:0.0.17"
 
+    // for etcd
+    compile("com.ibm.etcd:etcd-java:0.0.13")
+
     //tracing support
     compile "io.opentracing:opentracing-api:0.31.0"
     compile "io.opentracing:opentracing-util:0.31.0"

common/scala/src/main/resources/application.conf

@@ -383,6 +383,11 @@ whisk {
         local-image-prefix = "whisk"
     }
 
+    # cluster name related etcd configuration
+    cluster {
+        name = "whisk"
+    }
+
     user-events {
         enabled = false
     }