Skip to content

Commit 08a5d7d

Browse files
authored
Merge pull request #139 from cloud-gov/separate-release-for-platform
Separate release for platform
2 parents 15d1672 + f060eec commit 08a5d7d

File tree

32 files changed

+1059
-0
lines changed

32 files changed

+1059
-0
lines changed

config/blobs.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ elasticsearch/elasticsearch-7.9.3-no-jdk-linux-x86_64.tar.gz:
7878
size: 162808745
7979
object_id: e3856ece-c8a3-4c2b-6097-53a75ee5eeca
8080
sha: sha256:2ab0e23277e2fd9365b53af2653e6107ab46db2117390e84ba53385a96f3559f
81+
elasticsearch/elasticsearch-7.17.5-no-jdk-linux-x86_64.tar.gz:
82+
size: 167410729
83+
object_id: de00c51b-5ecf-4c0d-53fd-04c6a6aaf5bc
84+
sha: sha256:d301e4e270bbb3391e3aa721adc6c4cf86c3dcc5521a914120408d32ac30a92e
8185
haproxy/haproxy-1.7.5.tar.gz:
8286
size: 1743979
8387
object_id: 4ee72933-de11-4d3c-4657-b6b284388def
@@ -90,6 +94,10 @@ kibana/kibana-7.9.3-linux-x86_64.tar.gz:
9094
size: 295755102
9195
object_id: 74b514c5-0669-4b75-71f0-9b1f81240e97
9296
sha: sha256:20106e084f717270ce38290d1d3908db7de09b16233688ce0ec41e1d0dffddd6
97+
kibana/kibana-7.17.5-linux-x86_64.tar.gz:
98+
size: 263788794
99+
object_id: b0aef29a-f939-4632-65a3-0de6b26d4d9c
100+
sha: sha256:3a86dbb377acd407a04c59b31b6967fa2b8969166b38d6a6cc340e5ca445d939
93101
logstash/logstash-7.17.5.tar.gz:
94102
size: 363609474
95103
object_id: 6d8f78c8-3d0a-4517-7026-b4353ebdc05f
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
If you're needing to tweak and improve elasticsearch memory usage, you might
2+
want to enable GC logging. We've found the following set to be verbose and
3+
helpful...
4+
5+
elasticsearch.exec.options:
6+
- "-XX:+PrintCommandLineFlags"
7+
- "-XX:+PrintFlagsFinal"
8+
- "-XX:+PrintPromotionFailure"
9+
- "-XX:+PrintReferenceGC"
10+
- "-XX:+PrintGCDateStamps"
11+
- "-XX:+PrintGCDetails"
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
{
2+
"properties" : {
3+
"level" : {
4+
"type" : "string",
5+
"index" : "not_analyzed"
6+
},
7+
"logger" : {
8+
"type" : "string",
9+
"index" : "not_analyzed"
10+
},
11+
"message" : {
12+
"type" : "string"
13+
},
14+
"node" : {
15+
"type" : "string",
16+
"index" : "not_analyzed"
17+
},
18+
"slowlog_extra_source" : {
19+
"type" : "string",
20+
"index" : "not_analyzed"
21+
},
22+
"slowlog_index" : {
23+
"type" : "string",
24+
"index" : "not_analyzed"
25+
},
26+
"slowlog_search_type" : {
27+
"type" : "string",
28+
"index" : "not_analyzed"
29+
},
30+
"slowlog_shard" : {
31+
"type" : "long"
32+
},
33+
"slowlog_source" : {
34+
"type" : "string",
35+
"index" : "not_analyzed"
36+
},
37+
"slowlog_took_millis" : {
38+
"type" : "long"
39+
},
40+
"slowlog_total_shards" : {
41+
"type" : "long"
42+
},
43+
"slowlog_types" : {
44+
"type" : "string",
45+
"index" : "not_analyzed"
46+
}
47+
}
48+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
grok {
2+
match => [ "@message", "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{WORD:level}\s*\]\[(?<logger>[^\]\s]+)\s*\]\s+\[(?<node>[^\]\s]+)\s*\] %{GREEDYDATA:message}" ]
3+
tag_on_failure => [ "fail/logsearch/elasticsearch/stdout/v1" ]
4+
}
5+
6+
# date {} wants 2015-03-13 19:17:23.557 not 2015-03-13 19:17:23,557
7+
mutate {
8+
gsub => [ "timestamp", ",", "." ]
9+
}
10+
11+
date {
12+
match => [ "timestamp", "ISO8601" ]
13+
timezone => "UTC"
14+
remove_field => [ "timestamp" ]
15+
}
16+
17+
if [logger] == "index.search.slowlog.query" {
18+
# see https://github.com/elastic/elasticsearch/blob/efbda318d061867c098be5112561f8099227b9d4/src/main/java/org/elasticsearch/index/search/slowlog/ShardSlowLogSearchService.java#L185
19+
grok {
20+
match => [ "message", "^\[(?<slowlog_index>[^\]]+)\]\[%{WORD:slowlog_shard:int}\] took\[([^\]]+)\], took_millis\[%{POSINT:slowlog_took_millis:int}\], types\[(?<slowlog_types>[^\]\s]*)\], stats\[(?<slowlog_stats>[^\]\s]*)\], search_type\[%{WORD:slowlog_search_type}\], total_shards\[%{POSINT:slowlog_total_shards:int}\], source\[(?<slowlog_source>.*)\], extra_source\[(?<slowlog_extra_source>.*)\],?$" ]
21+
tag_on_failure => [ "fail/logsearch/elasticsearch/stdout/v1/slowlog" ]
22+
}
23+
24+
mutate {
25+
split => [ "slowlog_types", "," ]
26+
split => [ "slowlog_stats", "," ]
27+
}
28+
}
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# encoding: utf-8
2+
require 'spec_helper'
3+
require 'logstash/filters/date'
4+
require 'logstash/filters/grok'
5+
require 'logstash/filters/mutate'
6+
7+
describe LogStash::Filters::Grok do
8+
9+
describe 'logsearch/elasticsearch/stdout/v1' do
10+
config 'filter {' + File.read("#{File.dirname(__FILE__)}/logstash-filters.conf") + '}'
11+
12+
sample('@message' => '[2015-03-13 19:17:23,557][INFO ][cluster.routing.allocation.decider] [log_parser/0] updating [cluster.routing.allocation.enable] from [PRIMARIES] to [ALL]') do
13+
insist { subject['tags'] }.nil?
14+
#insist { subject['@timestamp'] } === Time.iso8601('2015-03-13T19:17:23.557Z')
15+
16+
insist { subject['level'] } === 'INFO'
17+
insist { subject['logger'] } === 'cluster.routing.allocation.decider'
18+
insist { subject['node'] } === 'log_parser/0'
19+
insist { subject['message'] } === 'updating [cluster.routing.allocation.enable] from [PRIMARIES] to [ALL]'
20+
21+
insist { subject.to_hash.keys.sort } === [
22+
'@message',
23+
'@timestamp',
24+
'@version',
25+
'level',
26+
'logger',
27+
'message',
28+
'node',
29+
]
30+
end
31+
32+
describe 'slowlog' do
33+
sample('@message' => '[2015-04-07 13:18:55,920][DEBUG][index.search.slowlog.query] [elasticsearch_eu-west-1b/2] [logstash-2015.04.07][1] took[5.4s], took_millis[5417], types[type1,type2], stats[stat1,stat2], search_type[QUERY_THEN_FETCH], total_shards[4], source[{"query":{"match_all":{}},"size":0}], extra_source[{"timeout":15000}],') do
34+
insist { subject['tags'] }.nil?
35+
#insist { subject['@timestamp'] } === Time.iso8601('2015-04-07T13:18:55.920Z')
36+
37+
insist { subject['level'] } === 'DEBUG'
38+
insist { subject['logger'] } === 'index.search.slowlog.query'
39+
insist { subject['node'] } === 'elasticsearch_eu-west-1b/2'
40+
insist { subject['message'] } === '[logstash-2015.04.07][1] took[5.4s], took_millis[5417], types[type1,type2], stats[stat1,stat2], search_type[QUERY_THEN_FETCH], total_shards[4], source[{"query":{"match_all":{}},"size":0}], extra_source[{"timeout":15000}],'
41+
42+
insist { subject['slowlog_index'] } === 'logstash-2015.04.07'
43+
insist { subject['slowlog_shard'] } === 1
44+
insist { subject['slowlog_took_millis'] } === 5417
45+
insist { subject['slowlog_types'] } === [ 'type1', 'type2' ]
46+
insist { subject['slowlog_stats'] } === [ 'stat1', 'stat2' ]
47+
insist { subject['slowlog_search_type'] } === 'QUERY_THEN_FETCH'
48+
insist { subject['slowlog_total_shards'] } === 4
49+
insist { subject['slowlog_source'] } === '{"query":{"match_all":{}},"size":0}'
50+
insist { subject['slowlog_extra_source'] } === '{"timeout":15000}'
51+
52+
insist { subject.to_hash.keys.sort } === [
53+
'@message',
54+
'@timestamp',
55+
'@version',
56+
'level',
57+
'logger',
58+
'message',
59+
'node',
60+
'slowlog_extra_source',
61+
'slowlog_index',
62+
'slowlog_search_type',
63+
'slowlog_shard',
64+
'slowlog_source',
65+
'slowlog_stats',
66+
'slowlog_took_millis',
67+
'slowlog_total_shards',
68+
'slowlog_types',
69+
]
70+
end
71+
72+
sample('@message' => '[2015-04-07 13:18:55,920][DEBUG][index.search.slowlog.query] [elasticsearch_eu-west-1b/2] [logstash-2015.04.07][1] took[5.4s], took_millis[5417], types[type1], stats[], search_type[QUERY_THEN_FETCH], total_shards[4], source[{"query":{"match_all":{}},"size":0}], extra_source[],') do
73+
insist { subject['tags'] }.nil?
74+
#insist { subject['@timestamp'] } === Time.iso8601('2015-04-07T13:18:55.920Z')
75+
76+
insist { subject['level'] } === 'DEBUG'
77+
insist { subject['logger'] } === 'index.search.slowlog.query'
78+
insist { subject['node'] } === 'elasticsearch_eu-west-1b/2'
79+
insist { subject['message'] } === '[logstash-2015.04.07][1] took[5.4s], took_millis[5417], types[type1], stats[], search_type[QUERY_THEN_FETCH], total_shards[4], source[{"query":{"match_all":{}},"size":0}], extra_source[],'
80+
81+
insist { subject['slowlog_index'] } === 'logstash-2015.04.07'
82+
insist { subject['slowlog_shard'] } === 1
83+
insist { subject['slowlog_took_millis'] } === 5417
84+
insist { subject['slowlog_types'] } === [ 'type1' ]
85+
insist { subject['slowlog_stats'] }.nil?
86+
insist { subject['slowlog_search_type'] } === 'QUERY_THEN_FETCH'
87+
insist { subject['slowlog_total_shards'] } === 4
88+
insist { subject['slowlog_source'] } === '{"query":{"match_all":{}},"size":0}'
89+
insist { subject['slowlog_extra_source'] }.nil?
90+
91+
insist { subject.to_hash.keys.sort } === [
92+
'@message',
93+
'@timestamp',
94+
'@version',
95+
'level',
96+
'logger',
97+
'message',
98+
'node',
99+
'slowlog_index',
100+
'slowlog_search_type',
101+
'slowlog_shard',
102+
'slowlog_source',
103+
'slowlog_took_millis',
104+
'slowlog_total_shards',
105+
'slowlog_types',
106+
]
107+
end
108+
end
109+
end
110+
end

jobs/elasticsearch-platform/monit

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
check process elasticsearch
2+
with pidfile /var/vcap/sys/run/bpm/elasticsearch/elasticsearch.pid
3+
start program "/var/vcap/jobs/bpm/bin/bpm start elasticsearch"
4+
stop program "/var/vcap/jobs/bpm/bin/bpm stop elasticsearch"
5+
group vcap
6+
7+
check device es-ephemeral_disk with path /var/vcap/data
8+
if SPACE usage > 80% then alert
9+
10+
check device es-persistent_disk with path /var/vcap/store
11+
if SPACE usage > 80% then alert

jobs/elasticsearch-platform/spec

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
---
2+
name: elasticsearch-platform
3+
4+
description: This job runs Elasticsearch node (master or data)
5+
6+
packages:
7+
- elasticsearch-platform
8+
- openjdk-11
9+
10+
templates:
11+
bin/drain.erb: bin/drain
12+
bin/post-deploy.erb: bin/post-deploy
13+
bin/post-start.erb: bin/post-start
14+
bin/pre-start: bin/pre-start
15+
bin/elasticsearch: bin/elasticsearch.sh
16+
config/bpm.yml.erb: config/bpm.yml
17+
config/config.yml.erb: config/elasticsearch.yml
18+
config/ca.erb: config/ssl/elasticsearch.ca
19+
config/node-crt.erb: config/ssl/elasticsearch-node.crt
20+
config/node-pem.erb: config/ssl/elasticsearch-node.pem
21+
config/admin-crt.erb: config/ssl/elasticsearch-admin.crt
22+
config/admin-pem.erb: config/ssl/elasticsearch-admin.pem
23+
config/jvm.options.erb: config/jvm.options.d/custom
24+
config/log4j2.properties.erb: config/log4j2.properties
25+
26+
provides:
27+
- name: elasticsearch-platform
28+
type: elasticsearch
29+
properties:
30+
- elasticsearch.port
31+
- elasticsearch.cluster_name
32+
33+
consumes:
34+
- name: elasticsearch-platform
35+
type: elasticsearch
36+
optional: true
37+
38+
properties:
39+
elasticsearch.master_hosts:
40+
description: The list of elasticsearch master node IPs
41+
elasticsearch.port:
42+
description: Port address of elasticsearch host to proxy requests for
43+
default: 9200
44+
elasticsearch.cluster_name:
45+
description: The name of the elastic search cluster
46+
default: "logsearch"
47+
elasticsearch.log_level:
48+
description: The default logging level (e.g. WARN, DEBUG, INFO)
49+
default: INFO
50+
elasticsearch.jvm_options:
51+
description: Additional Java Virtual Machine options
52+
default: []
53+
elasticsearch.heap_size:
54+
description: sets jvm heap sized
55+
elasticsearch.heap_percentage:
56+
description: The percentage value used in the calculation to set the heap size.
57+
default: 46
58+
elasticsearch.path_repo:
59+
description: |
60+
Shared file system to store snapshots.
61+
In order to register the shared file system repository it is
62+
necessary to mount the same shared filesystem to the same location
63+
on all master and data nodes.
64+
default: ''
65+
elasticsearch.node.allow_master:
66+
description: Allow node to become master? (true / false)
67+
default: false
68+
elasticsearch.node.allow_data:
69+
description: Allow node to store data? (true / false)
70+
default: false
71+
elasticsearch.node.allow_ingest:
72+
description: Allow node to become ingest node? (true / false)
73+
default: false
74+
elasticsearch.node.ssl.ca:
75+
description: CA certificate for SSL plugin
76+
elasticsearch.node.ssl.certificate:
77+
description: Node certificate
78+
elasticsearch.node.ssl.private_key:
79+
description: Private key for node certificate
80+
elasticsearch.node.ssl.dn:
81+
description: Nodes DN pattern
82+
elasticsearch.admin.certificate:
83+
description: Admin certificate to operate SSL plugin
84+
elasticsearch.admin.private_key:
85+
description: Private key for admin certificate
86+
elasticsearch.admin.dn:
87+
description: Admin DN to operate SSL plugin
88+
elasticsearch.health.timeout:
89+
description: Post-start timeout for node to join cluster (seconds)
90+
default: 300
91+
elasticsearch.health.interval:
92+
description: Post-start interval for node to join cluster (seconds)
93+
default: 15
94+
elasticsearch.health.connect_timeout:
95+
description: Post-start timeout for node to become available (seconds)
96+
default: 60
97+
elasticsearch.health.connect_interval:
98+
description: Post-start interval for node to become available (seconds)
99+
default: 5
100+
elasticsearch.node.tags:
101+
description: A hash of additional tags for the node
102+
elasticsearch.exec.environment:
103+
description: A hash of additional environment variables for the process
104+
elasticsearch.exec.options:
105+
description: An array of additional options to pass when starting elasticsearch
106+
default: []
107+
elasticsearch.limits.fd:
108+
description: Maximum file descriptors
109+
default: 65536
110+
elasticsearch.recovery.delay_allocation:
111+
description: Delay allocation interval
112+
default: "1m"
113+
elasticsearch.recovery.delay_allocation_restart:
114+
description: Delay allocation interval during restart
115+
default: "5m"
116+
elasticsearch.discovery.minimum_master_nodes:
117+
description: The minimum number of master eligible nodes a node should "see" in order to operate within the cluster. Recommended to set it to a higher value than 1 when running more than 2 nodes in the cluster.
118+
default: "auto"
119+
elasticsearch.config_options:
120+
description: "Additional options to append to elasticsearch's config.yml (YAML format)."
121+
default: ~
122+
elasticsearch.logging_options:
123+
description: "Additional options to append to elasticsearch's logging.yml (YAML format)."
124+
default: ~
125+
elasticsearch.plugins:
126+
description: "Plugins to run elasticsearch with (array[] = { plugin-name: install-source }; e.g. [ { kopf: 'lmenezes/elasticsearch-kopf' } ])"
127+
default: []
128+
elasticsearch.http_host:
129+
description: "The host address to bind the elasticsearch HTTP service to and to publish for HTTP clients to connect to"
130+
default: 0.0.0.0
131+
elasticsearch.health.disable_post_start:
132+
description: Allow node to run post-start script? (true / false)
133+
default: false
134+
135+
elasticsearch.cloud.aws.access_key:
136+
description: Access key ID for AWS account
137+
elasticsearch.cloud.aws.secret_key:
138+
description: Secret access key for AWS account
139+
elasticsearch.cloud.aws.region:
140+
description: Specify the AWS region to use
141+
elasticsearch.cloud.aws.protocol:
142+
description: Protocol to use for all API calls to AWS endpoints
143+
default: "https"
144+
elasticsearch.cloud.aws.read_timeout:
145+
description: The amount of time to wait for data to be transferred over an established, open connection before the connection is timed out
146+
default: 50s
147+
elasticsearch.cloud.aws.bucket:
148+
description: Bucket name on S3 where to keep snapshots
149+
default: ''
150+
elasticsearch.snapshots.repository:
151+
description: Repository name for automatic snapshots
152+
default: ''

0 commit comments

Comments
 (0)