Implement an ElasticSearchActivationStore (#4724)

* Implement an ElasticSearchActivationStore

* Remove short headers and mismatched comments

* typo fix

* Use TestContainers

* Neat changes

* Update build.gradle

* Fix TestContainers

* fix mistake

* Remove cache and add configuration example

* Fix pureconfig

* Apply review comments

* Add result and annotations to mock WhiskActivations

* Add instructions about how to use it

* Update ansible/README.md

* Remove vagrant changes

* Add elasticsearch to `redo` and remove whitespace

* Fix tests

Author: jiangpengcheng

ansible/README.md

@@ -191,6 +191,46 @@ ansible-playbook -i environments/<environment> routemgmt.yml
 - To use the API Gateway, you'll need to run `apigateway.yml` and `routemgmt.yml`.
 - Use `ansible-playbook -i environments/<environment> openwhisk.yml` to avoid wiping the data store. This is useful to start OpenWhisk after restarting your Operating System.
 
+### Using ElasticSearch to Store Activations
+
+You can use ElasticSearch (ES) to store activations separately while other entities remain stored in CouchDB. There is an Ansible playbook to setup a simple ES cluster for testing and development purposes.
+
+-   Provide your custom ES related ansible arguments:
+
+```
+elastic_protocol="http"
+elastic_index_pattern="openwhisk-%s" // this will be combined with namespace's name, so different namespace can use different index
+elastic_base_volume="esdata" // name of docker volume to store ES data
+elastic_cluster_name="openwhisk"
+elastic_java_opts="-Xms1g -Xmx1g"
+elastic_loglevel="INFO"
+elastic_username="admin"
+elastic_password="admin"
+elasticsearch_connect_string="x.x.x.x:9200,y.y.y.y:9200" // if you want to use an external ES cluster, add it
+```
+
+-  Then execute:
+
+```
+cd <openwhisk_home>
+./gradlew distDocker
+cd ansible
+# couchdb is still needed to store subjects and actions
+ansible-playbook -i environments/<environment> couchdb.yml
+ansible-playbook -i environments/<environment> initdb.yml
+ansible-playbook -i environments/<environment> wipe.yml
+# this will deploy a simple ES cluster, you can skip this to use external ES cluster
+ansible-playbook -i environments/<environment> elasticsearch.yml
+ansible-playbook -i environments/<environment> openwhisk.yml -e db_activation_backend=ElasticSearch
+
+# installs a catalog of public packages and actions
+ansible-playbook -i environments/<environment> postdeploy.yml
+
+# to use the API gateway
+ansible-playbook -i environments/<environment> apigateway.yml
+ansible-playbook -i environments/<environment> routemgmt.yml
+```
+
 ### Configuring the installation of `wsk` CLI
 There are two installation modes to install `wsk` CLI: remote and local.
 

ansible/elasticsearch.yml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This playbook deploys a ElasticSearch cluster
+
+- hosts: elasticsearch
+  vars:
+    #
+    # host_group - usually "{{ groups['...'] }}" where '...' is what was used
+    #   for 'hosts' above.  The hostname of each host will be looked up in this
+    #   group to assign a zero-based index.  That index will be used in concert
+    #   with 'name_prefix' below to assign a host/container name.
+    host_group: "{{ groups['elasticsearch'] }}"
+    #
+    # name_prefix - a unique prefix for this set of elasticsearches.  The prefix
+    # will be used in combination with an index (determined using
+    # 'host_group' above) to name host/elasticsearcher.
+    name_prefix: "elasticsearch"
+  roles:
+  - elasticsearch

ansible/environments/docker-machine/hosts.j2.ini

@@ -36,6 +36,9 @@ invoker1                    ansible_host={{ docker_machine_ip }}
 [apigateway]
 {{ docker_machine_ip }}     ansible_host={{ docker_machine_ip }}
 
+[elasticsearch:children]
+db
+
 ; define variables
 [all:vars]
 ansible_connection=ssh

ansible/environments/local/hosts.j2.ini

@@ -31,6 +31,9 @@ invoker1            ansible_host=172.17.0.1 ansible_connection=local
 [db]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
 
+[elasticsearch:children]
+db
+
 [redis]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
 

ansible/group_vars/all

@@ -272,6 +272,27 @@ db:
     invoker:
       user: "{{ db_invoker_user | default(lookup('ini', 'db_username section=invoker file={{ playbook_dir }}/db_local.ini')) }}"
       pass: "{{ db_invoker_pass | default(lookup('ini', 'db_password section=invoker file={{ playbook_dir }}/db_local.ini')) }}"
+  activation_store:
+    backend: "{{ db_activation_backend | default('CouchDB') }}"
+  elasticsearch:
+    protocol: "{{ elastic_protocol | default('http') }}"
+    port: 9200
+    index_pattern: "{{ elastic_index_pattern | default('openwhisk-%s') }}"
+    base_transport_port: 9300
+    confdir: "{{ config_root_dir }}/elasticsearch"
+    dir:
+      become: "{{ elastic_dir_become | default(false) }}"
+    base_volume: "{{ elastic_base_volume | default('esdata') }}"
+    cluster_name: "{{ elastic_cluster_name | default('openwhisk') }}"
+    java_opts: "{{ elastic_java_opts | default('-Xms1g -Xmx1g') }}"
+    loglevel: "{{ elastic_loglevel | default('INFO') }}"
+    # the user id of elasticsearch process, default is 1000, if you have enabled user namespace
+    # for docker daemon, this need to be changed correspondingly
+    uid: "{{ elastic_uid | default(1000) }}"
+    auth:
+      admin:
+        username: "{{ elastic_username | default('admin') }}"
+        password: "{{ elastic_password | default('admin') }}"
 
 apigateway:
   port:
@@ -291,6 +312,15 @@ linux:
 couchdb:
   version: 2.3
 
+elasticsearch:
+  version: 6.7.2
+
+elasticsearch_connect_string: "{% set ret = [] %}\
+                               {% for host in groups['elasticsearch'] %}\
+                               {{ ret.append( hostvars[host].ansible_host + ':' + ((db.elasticsearch.port+loop.index-1)|string) ) }}\
+                               {% endfor %}\
+                               {{ ret | join(',') }}"
+
 docker:
   # The user to install docker for. Defaults to the ansible user if not set. This will be the user who is able to run
   # docker commands on a machine setup with prereq_build.yml

ansible/roles/controller/tasks/deploy.yml

@@ -275,6 +275,22 @@
   set_fact:
     env: "{{ env | combine(controller.extraEnv) }}"
 
+- name: setup elasticsearch activation store env
+  set_fact:
+    elastic_env:
+      "CONFIG_whisk_activationStore_elasticsearch_protocol": "{{ db.elasticsearch.protocol}}"
+      "CONFIG_whisk_activationStore_elasticsearch_hosts": "{{ elasticsearch_connect_string }}"
+      "CONFIG_whisk_activationStore_elasticsearch_indexPattern": "{{ db.elasticsearch.index_pattern }}"
+      "CONFIG_whisk_activationStore_elasticsearch_username": "{{ db.elasticsearch.auth.admin.username }}"
+      "CONFIG_whisk_activationStore_elasticsearch_password": "{{ db.elasticsearch.auth.admin.password }}"
+      "CONFIG_whisk_spi_ActivationStoreProvider": "org.apache.openwhisk.core.database.elasticsearch.ElasticSearchActivationStoreProvider"
+  when: db.activation_store.backend == "ElasticSearch"
+
+- name: merge elasticsearch activation store env
+  set_fact:
+    env: "{{ env | combine(elastic_env) }}"
+  when: db.activation_store.backend == "ElasticSearch"
+
 - name: populate volumes for controller
   set_fact:
     controller_volumes:

ansible/roles/elasticsearch/tasks/clean.yml

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# Remove ElasticSearch server
+
+- name: set elasticsearch container name and volume
+  set_fact:
+    elasticsearch_name: "{{ name_prefix ~ host_group.index(inventory_hostname) }}"
+    volume_name: "{{ db.elasticsearch.base_volume ~ host_group.index(inventory_hostname) }}"
+
+- name: remove ElasticSearch
+  vars:
+    elasticsearch_image: "{{ elasticsearch.docker_image | default('docker.elastic.co/elasticsearch/elasticsearch:' ~ elasticsearch.version ) }}"
+  docker_container:
+    name: "{{ elasticsearch_name }}"
+    image: "{{ elasticsearch_image }}"
+    keep_volumes: False
+    state: absent
+  ignore_errors: True
+
+- name: remove ElasticSearch conf dir
+  file:
+    path: "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}"
+    state: absent
+  become: "{{ db.elasticsearch.dir.become }}"

ansible/roles/elasticsearch/tasks/deploy.yml

@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will run a ElasticSearch server on the db group
+
+- name: set the vm.max_map_count to 262144
+  sysctl:
+    name: vm.max_map_count
+    value: '262144'
+  become: true
+
+- name: set elasticsearch container name, volume and port
+  set_fact:
+    elasticsearch_name: "{{ name_prefix ~ host_group.index(inventory_hostname) }}"
+    volume_name: "{{ db.elasticsearch.base_volume ~ host_group.index(inventory_hostname) }}"
+    http_port: "{{ (db.elasticsearch.port|int) + host_group.index(inventory_hostname) }}"
+    transport_port: "{{ (db.elasticsearch.base_transport_port|int) + host_group.index(inventory_hostname) }}"
+
+- name: ensure elasticserach config directory is created with permissions
+  file:
+    path: "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}"
+    state: directory
+    mode: 0755
+  become: "{{ db.elasticsearch.dir.become }}"
+
+# create volume direcotry if it's a directory path(not a named volume)
+- name: ensure elasticserach volume directory is created with permissions
+  file:
+    path: "{{ volume_name }}"
+    state: directory
+    mode: 0700
+    owner: "{{ db.elasticsearch.uid }}"
+  become: true
+  when: volume_name is search("/")
+
+- name: copy elasticsearch config file
+  template:
+    src: "elasticsearch.yml.j2"
+    dest: "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}/elasticsearch.yml"
+    mode: 0644
+  become: "{{ db.elasticsearch.dir.become }}"
+
+- name: copy elasticsearch log config file
+  template:
+    src: "log4j2.properties.j2"
+    dest: "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}/log4j2.properties"
+    mode: 0644
+  become: "{{ db.elasticsearch.dir.become }}"
+
+- name: "(re)start ElasticSearch from '{{ elasticsearch_image }} ' "
+  vars:
+    elasticsearch_image: "{{ elasticsearch.docker_image | default('docker.elastic.co/elasticsearch/elasticsearch:' ~ elasticsearch.version ) }}"
+  docker_container:
+    name: "{{ elasticsearch_name }}"
+    image: "{{ elasticsearch_image }}"
+    state: started
+    recreate: true
+    restart_policy: "{{ docker.restart.policy }}"
+    ports:
+      - "{{ http_port }}:9200"
+      - "{{ transport_port }}:{{ transport_port }}"
+    volumes:
+      - "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
+      - "{{ db.elasticsearch.confdir }}/{{ elasticsearch_name }}/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties"
+      - "{{ volume_name }}:/usr/share/elasticsearch/data"
+    pull: "{{ docker.pull_elasticsearch | default(true) }}"
+    ulimits:
+      - "nofile:262144:262144"
+      - "memlock:-1:-1"
+    env:
+      TZ: "{{ docker.timezone }}"
+      ES_JAVA_OPTS: "{{ db.elasticsearch.java_opts }}"
+
+- name: wait until ElasticSearch in this host is up and running
+  uri:
+    url: "{{ db.elasticsearch.protocol }}://{{ ansible_host }}:{{ http_port }}"
+  register: result
+  until: result.status == 200
+  retries: 12
+  delay: 5

ansible/roles/elasticsearch/tasks/main.yml

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will deploy a database server. Use the role if you want to use ElasticSearch locally.
+# In deploy mode it will start the ElasticSearch container.
+# In clean mode it will remove the ElasticSearch container.
+
+- import_tasks: deploy.yml
+  when: mode == "deploy"
+
+- import_tasks: clean.yml
+  when: mode == "clean"

ansible/roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -0,0 +1,23 @@
+cluster.name: "{{ db.elasticsearch.cluster_name }}"
+node.name: "{{ elasticsearch_name }}"
+network.host: 0.0.0.0
+network.publish_host: {{ ansible_default_ipv4.address }}
+
+http.port: 9200
+transport.tcp.port: {{ transport_port }}
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# set to 1 to allow single node clusters
+# Details: https://github.com/elastic/elasticsearch/pull/17282
+discovery.zen.ping.unicast.hosts:
+{% for es in groups['elasticsearch'] %}
+   - {{ hostvars[es].ansible_host }}:{{ db.elasticsearch.base_transport_port + host_group.index(es)|int }}
+{% endfor %}
+discovery.zen.minimum_master_nodes: {{ (host_group|length / 2 + 1) | int}}
+
+gateway.recover_after_nodes: {{ (host_group|length / 2 + 1) | int }}
+gateway.expected_nodes: {{ host_group|length }}
+gateway.recover_after_time: 5m
+
+xpack.security.enabled: false
+bootstrap.memory_lock: true