Skip to content

Commit 1349f7d

Browse files
shroffkshroffk
committed
Adding support to auto generate the elastic indices needed by CF
1 parent da1f686 commit 1349f7d

File tree

7 files changed

+130
-67
lines changed

7 files changed

+130
-67
lines changed

‎src/main/java/org/phoebus/channelfinder/ElasticConfig.java‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,19 @@
1414
* #L%
1515
*/
1616

17+
import java.io.IOException;
18+
import java.io.InputStream;
19+
import java.util.concurrent.atomic.AtomicBoolean;
20+
import java.util.logging.Level;
1721
import java.util.logging.Logger;
1822

1923
import javax.servlet.ServletContextEvent;
2024
import javax.servlet.ServletContextListener;
2125

26+
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
27+
import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
28+
import co.elastic.clients.elasticsearch.indices.ExistsRequest;
29+
import co.elastic.clients.transport.endpoints.BooleanResponse;
2230
import org.apache.http.HttpHost;
2331
import org.elasticsearch.client.RestClient;
2432
import org.springframework.beans.factory.annotation.Value;
@@ -46,27 +54,23 @@ public class ElasticConfig implements ServletContextListener {
4654

4755
private ElasticsearchClient searchClient;
4856
private ElasticsearchClient indexClient;
57+
private static final AtomicBoolean esInitialized = new AtomicBoolean();
4958

5059
@Value("${elasticsearch.cluster.name:elasticsearch}")
5160
private String clusterName;
5261
@Value("${elasticsearch.network.host:localhost}")
5362
private String host;
5463
@Value("${elasticsearch.http.port:9200}")
5564
private int port;
65+
@Value("${elasticsearch.create.indices:true}")
66+
private String createIndices;
5667

5768
@Value("${elasticsearch.tag.index:cf_tags}")
5869
private String ES_TAG_INDEX;
59-
@Value("${elasticsearch.tag.type:cf_tag}")
60-
private String ES_TAG_TYPE;
6170
@Value("${elasticsearch.property.index:cf_properties}")
6271
private String ES_PROPERTY_INDEX;
63-
@Value("${elasticsearch.property.type:cf_property}")
64-
private String ES_PROPERTY_TYPE;
6572
@Value("${elasticsearch.channel.index:channelfinder}")
6673
private String ES_CHANNEL_INDEX;
67-
@Value("${elasticsearch.channel.type:cf_channel}")
68-
private String ES_CHANNEL_TYPE;
69-
7074
@Value("${elasticsearch.query.size}")
7175
private String ES_QUERY_SIZE;
7276

@@ -81,6 +85,10 @@ public ElasticsearchClient getSearchClient() {
8185

8286
searchClient = new ElasticsearchClient(transport);
8387
}
88+
esInitialized.set(!Boolean.parseBoolean(createIndices));
89+
if (esInitialized.compareAndSet(false, true)) {
90+
elasticIndexValidation(searchClient);
91+
}
8492
return searchClient;
8593
}
8694

@@ -94,6 +102,10 @@ public ElasticsearchClient getIndexClient() {
94102
ElasticsearchTransport transport = new RestClientTransport(httpClient, new JacksonJsonpMapper());
95103
indexClient = new ElasticsearchClient(transport);
96104
}
105+
esInitialized.set(!Boolean.parseBoolean(createIndices));
106+
if (esInitialized.compareAndSet(false, true)) {
107+
elasticIndexValidation(indexClient);
108+
}
97109
return indexClient;
98110
}
99111

@@ -111,4 +123,52 @@ public void contextDestroyed(ServletContextEvent sce) {
111123
indexClient.shutdown();
112124
}
113125

126+
127+
/**
128+
* Create the olog indices and templates if they don't exist
129+
* @param client
130+
*/
131+
void elasticIndexValidation(ElasticsearchClient client) {
132+
// ChannelFinder Index
133+
try (InputStream is = ElasticConfig.class.getResourceAsStream("/channel_mapping.json")) {
134+
BooleanResponse exits = client.indices().exists(ExistsRequest.of(e -> e.index(ES_CHANNEL_INDEX)));
135+
if(!exits.value()) {
136+
137+
CreateIndexResponse result = client.indices().create(
138+
CreateIndexRequest.of(
139+
c -> c.index(ES_CHANNEL_INDEX).withJson(is)));
140+
log.info("Created index: " + ES_CHANNEL_INDEX + " : acknowledged " + result.acknowledged());
141+
}
142+
} catch (IOException e) {
143+
log.log(Level.WARNING, "Failed to create index " + ES_CHANNEL_INDEX, e);
144+
}
145+
146+
// ChannelFinder tag Index
147+
try (InputStream is = ElasticConfig.class.getResourceAsStream("/tag_mapping.json")) {
148+
BooleanResponse exits = client.indices().exists(ExistsRequest.of(e -> e.index(ES_TAG_INDEX)));
149+
if(!exits.value()) {
150+
151+
CreateIndexResponse result = client.indices().create(
152+
CreateIndexRequest.of(
153+
c -> c.index(ES_TAG_INDEX).withJson(is)));
154+
log.info("Created index: " + ES_TAG_INDEX + " : acknowledged " + result.acknowledged());
155+
}
156+
} catch (IOException e) {
157+
log.log(Level.WARNING, "Failed to create index " + ES_TAG_INDEX, e);
158+
}
159+
160+
// ChannelFinder property Index
161+
try (InputStream is = ElasticConfig.class.getResourceAsStream("/properties_mapping.json")) {
162+
BooleanResponse exits = client.indices().exists(ExistsRequest.of(e -> e.index(ES_PROPERTY_INDEX)));
163+
if(!exits.value()) {
164+
165+
CreateIndexResponse result = client.indices().create(
166+
CreateIndexRequest.of(
167+
c -> c.index(ES_PROPERTY_INDEX).withJson(is)));
168+
log.info("Created index: " + ES_PROPERTY_INDEX + " : acknowledged " + result.acknowledged());
169+
}
170+
} catch (IOException e) {
171+
log.log(Level.WARNING, "Failed to create index " + ES_PROPERTY_INDEX, e);
172+
}
173+
}
114174
}

‎src/main/resources/application.properties‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ ldap.groups.search.base = ou=Group
2626
ldap.groups.search.pattern = (memberUid= {1})
2727

2828
############## LDAP - Embedded ##############
29-
embedded_ldap.enabled = true
29+
embedded_ldap.enabled = false
3030
embedded_ldap.urls = ldap://localhost:8389/dc=cf,dc=local
3131
embedded_ldap.base.dn = dc=cf,dc=local
3232
embedded_ldap.user.dn.pattern = uid={0},ou=People
@@ -39,7 +39,7 @@ spring.ldap.embedded.validation.enabled=false
3939

4040

4141
############## Demo Auth ##############
42-
demo_auth.enabled = false
42+
demo_auth.enabled = true
4343

4444
############## Role --> group Mapping ##############
4545
# Customize group names here
@@ -53,23 +53,20 @@ tag-groups=cf-tags,USER
5353
# Elasticsearch host
5454
#elasticsearch.network.host: 169.254.42.56
5555
# Set a custom port for the node to node communication (9300 by default):
56-
elasticsearch.transport.tcp.port: 9300
56+
#elasticsearch.transport.tcp.port: 9300
5757
# Set a custom port to listen for HTTP traffic:
5858
elasticsearch.http.port: 9200
5959

6060
# Elasticsearch index names and types used by channelfinder, ensure that any changes here should be replicated in the mapping_definitions.sh
6161
elasticsearch.tag.index = cf_tags
62-
elasticsearch.tag.type = cf_tag
63-
6462
elasticsearch.property.index = cf_properties
65-
elasticsearch.property.type = cf_property
66-
6763
elasticsearch.channel.index = channelfinder
68-
elasticsearch.channel.type = cf_channel
6964

7065
# maximum query result size
7166
elasticsearch.query.size = 10000
7267

68+
# Create the Channel Finder indices if they do not exist
69+
elasticsearch.create.indices: true
7370

7471
############################## Service Info ###############################
7572
channelfinder.version = 4.0.0

‎src/main/resources/channel_mapping.json‎

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,40 @@
11
{
2-
"properties": {
3-
"name": {
4-
"type": "keyword"
5-
},
6-
"owner": {
7-
"type": "keyword"
8-
},
9-
"script": {
10-
"type": "keyword"
11-
},
12-
"properties": {
13-
"type": "nested",
14-
"properties": {
15-
"name": {
16-
"type": "keyword"
17-
},
18-
"owner": {
19-
"type": "keyword"
20-
},
21-
"value": {
22-
"type": "keyword"
23-
}
24-
}
25-
},
26-
"tags": {
27-
"type": "nested",
28-
"properties": {
29-
"name": {
30-
"type": "keyword"
31-
},
32-
"owner": {
33-
"type": "keyword"
34-
}
35-
}
36-
}
37-
}
2+
"mappings": {
3+
"properties": {
4+
"name": {
5+
"type": "keyword"
6+
},
7+
"owner": {
8+
"type": "keyword"
9+
},
10+
"script": {
11+
"type": "keyword"
12+
},
13+
"properties": {
14+
"type": "nested",
15+
"properties": {
16+
"name": {
17+
"type": "keyword"
18+
},
19+
"owner": {
20+
"type": "keyword"
21+
},
22+
"value": {
23+
"type": "keyword"
24+
}
25+
}
26+
},
27+
"tags": {
28+
"type": "nested",
29+
"properties": {
30+
"name": {
31+
"type": "keyword"
32+
},
33+
"owner": {
34+
"type": "keyword"
35+
}
36+
}
37+
}
38+
}
39+
}
3840
}

‎src/main/resources/es8_mapping_definitions.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/sh
22

33
es_host=localhost
4-
es_port=9208
4+
es_port=9200
55

66
###
77
# #%L

‎src/main/resources/properties_mapping.json‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
2-
"properties": {
3-
"name": {
2+
"mappings": {
3+
"properties": {
4+
"name": {
45
"type": "keyword"
5-
},
6-
"owner": {
6+
},
7+
"owner": {
78
"type": "keyword"
9+
}
810
}
9-
}
10-
}
11+
}
12+
}

‎src/main/resources/tag_mapping.json‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
2-
"properties": {
3-
"name": {
2+
"mappings": {
3+
"properties": {
4+
"name": {
45
"type": "keyword"
5-
},
6-
"owner": {
6+
},
7+
"owner": {
78
"type": "keyword"
9+
}
810
}
9-
}
10-
}
11+
}
12+
}

‎src/test/resources/application_test.properties‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ ldap.groups.search.base = ou=Group
2424
ldap.groups.search.pattern = (memberUid= {1})
2525

2626
############## LDAP - Embedded ##############
27-
embedded_ldap.enabled = true
27+
embedded_ldap.enabled = false
2828
embedded_ldap.urls = ldap://localhost:8389/dc=olog,dc=local
2929
embedded_ldap.base.dn = dc=olog,dc=local
3030
embedded_ldap.user.dn.pattern = uid={0},ou=People
@@ -37,7 +37,7 @@ spring.ldap.embedded.validation.enabled=false
3737

3838

3939
############## Demo Auth ##############
40-
demo_auth.enabled = false
40+
demo_auth.enabled = true
4141

4242
############## Group-->Role Mapping ##############
4343
# Customize group names here
@@ -64,7 +64,7 @@ tag-groups=cf-tags
6464

6565
# Set both 'bind_host' and 'publish_host':
6666
#
67-
elasticsearch.network.host: 169.254.42.56
67+
elasticsearch.network.host: localhost
6868

6969
# Set a custom port for the node to node communication (9300 by default):
7070
#

0 commit comments

Comments
 (0)