Skip to content

Commit 6cfde8a

Browse files
shroffkshroffk
committed
#2333 fixing the IndexNameHelper
1 parent 8701646 commit 6cfde8a

File tree

7 files changed

+81
-64
lines changed

7 files changed

+81
-64
lines changed

‎core/util/src/main/java/org/phoebus/util/indexname/IndexNameHelper.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class IndexNameHelper
2222
private String baseIndexName;
2323
private String currentDateSpan;
2424
private String dateSpanUnit;
25-
private Integer dateSpanValue;
25+
private Integer dateSpanValue = 1;
2626

2727
private Instant spanStart;
2828
private Instant spanEnd;
@@ -38,17 +38,22 @@ public class IndexNameHelper
3838
/**
3939
*
4040
* @param baseIndexName : Index base name that the date will be appended to.
41+
* @param useDatedIndexNames : A flag to indicate if the helper should generate date based index names
4142
* @param dateSpanUnit : The unit of the date span. Years (Y/y), Months (M/m), Weeks (W/w), and Days (D/d) are supported.
42-
* @param dateSpanValue : The integer number of date range units that each index will span.
4343
* @throws Exception : If any parameters are invalid or null.
4444
*/
45-
public IndexNameHelper(final String baseIndexName, final String dateSpanUnit, final Integer dateSpanValue) throws Exception
45+
public IndexNameHelper(final String baseIndexName, final boolean useDatedIndexNames, final String dateSpanUnit) throws Exception
4646
{
4747
if (null != baseIndexName)
4848
this.baseIndexName = baseIndexName;
4949
else
5050
throw new Exception("Base Index Name is null.");
5151

52+
if (!useDatedIndexNames)
53+
this.dateSpanValue = -1;
54+
else
55+
this.dateSpanValue = 1;
56+
5257
if (null != dateSpanUnit)
5358
{
5459
if (! acceptedDateUnits.contains(dateSpanUnit.toUpperCase()))
@@ -58,10 +63,6 @@ public IndexNameHelper(final String baseIndexName, final String dateSpanUnit, fi
5863
else
5964
throw new Exception("Date Span Unit is null.");
6065

61-
if (null != dateSpanValue)
62-
this.dateSpanValue = dateSpanValue;
63-
else
64-
throw new Exception("Date Span Value is null.");
6566
}
6667

6768
/**

‎core/util/src/test/java/org/phoebus/util/indexname/IndexNameHelperTest.java‎

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public void baseIndexNameNull()
3939
{
4040
try
4141
{
42-
new IndexNameHelper(null, null, null);
42+
new IndexNameHelper(null, true, null);
4343
}
4444
catch (Exception ex)
4545
{
@@ -52,7 +52,7 @@ public void dateSpanUnitNull()
5252
{
5353
try
5454
{
55-
new IndexNameHelper("index", null, 10);
55+
new IndexNameHelper("index", true,null);
5656
}
5757
catch (Exception ex)
5858
{
@@ -65,7 +65,7 @@ public void dateSpanUnitInvalid()
6565
{
6666
try
6767
{
68-
new IndexNameHelper("index", "Q", 5);
68+
new IndexNameHelper("index", true, "Q");
6969
}
7070
catch (Exception ex)
7171
{
@@ -74,31 +74,9 @@ public void dateSpanUnitInvalid()
7474
}
7575

7676
@Test
77-
public void dateSpanValueNull()
77+
public void useDatedIndexNamesFalse() throws Exception
7878
{
79-
try
80-
{
81-
new IndexNameHelper("index", "y", null);
82-
}
83-
catch (Exception ex)
84-
{
85-
assertEquals("Date Span Value is null.", ex.getMessage());
86-
}
87-
}
88-
89-
@Test
90-
public void dateSpanValueZero() throws Exception
91-
{
92-
IndexNameHelper inh = new IndexNameHelper("test_index", "y", 0);
93-
String indexName = inh.getIndexName(Instant.now());
94-
95-
assertEquals("test_index", indexName);
96-
}
97-
98-
@Test
99-
public void dateSpanValueLessThanZero() throws Exception
100-
{
101-
IndexNameHelper inh = new IndexNameHelper("test_index", "y", -5);
79+
IndexNameHelper inh = new IndexNameHelper("test_index", false, "y");
10280
String indexName = inh.getIndexName(Instant.now());
10381

10482
assertEquals("test_index", indexName);
@@ -114,7 +92,7 @@ public void dateInCurrentYear() throws Exception
11492

11593
expectedSpanStart.atZone(ZoneId.systemDefault()).toInstant();
11694

117-
IndexNameHelper inh = new IndexNameHelper("test_index", "y", 1);
95+
IndexNameHelper inh = new IndexNameHelper("test_index", true, "y");
11896

11997
assertNull(inh.getCurrentDateSpanStart());
12098
assertNull(inh.getCurrentDateSpanEnd());
@@ -136,7 +114,7 @@ public void dateInNextYear() throws Exception
136114
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 13, 0, 0, 0);
137115
LocalDateTime newSpanTime = LocalDateTime.of(2019, 1, 1, 0, 0, 1);
138116

139-
IndexNameHelper inh = new IndexNameHelper("test_index", "y", 1);
117+
IndexNameHelper inh = new IndexNameHelper("test_index", true,"y");
140118

141119
assertNull(inh.getCurrentDateSpanStart());
142120
assertNull(inh.getCurrentDateSpanEnd());
@@ -163,7 +141,7 @@ public void dateInCurrentMonth() throws Exception
163141
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 15, 0, 0, 0);
164142
LocalDateTime newSpanTime = LocalDateTime.of(2018, 9, 17, 0, 0, 0);
165143

166-
IndexNameHelper inh = new IndexNameHelper("test_index", "m", 1);
144+
IndexNameHelper inh = new IndexNameHelper("test_index", true, "m");
167145

168146
assertNull(inh.getCurrentDateSpanStart());
169147
assertNull(inh.getCurrentDateSpanEnd());
@@ -185,7 +163,7 @@ public void dateInNextMonth() throws Exception
185163
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 13, 0, 0, 0);
186164
LocalDateTime newSpanTime = LocalDateTime.of(2018, 10, 1, 0, 0, 1);
187165

188-
IndexNameHelper inh = new IndexNameHelper("test_index", "m", 1);
166+
IndexNameHelper inh = new IndexNameHelper("test_index", true,"m");
189167

190168
assertNull(inh.getCurrentDateSpanStart());
191169
assertNull(inh.getCurrentDateSpanEnd());
@@ -212,7 +190,7 @@ public void dateInCurrentWeek() throws Exception
212190
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 13, 0, 0, 0);
213191
LocalDateTime newSpanTime = LocalDateTime.of(2018, 9, 14, 0, 0, 0);
214192

215-
IndexNameHelper inh = new IndexNameHelper("test_index", "w", 1);
193+
IndexNameHelper inh = new IndexNameHelper("test_index", true,"w");
216194

217195
assertNull(inh.getCurrentDateSpanStart());
218196
assertNull(inh.getCurrentDateSpanEnd());
@@ -234,7 +212,7 @@ public void dateInNextWeek() throws Exception
234212
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 13, 0, 0, 0);
235213
LocalDateTime newSpanTime = LocalDateTime.of(2018, 9, 16, 0, 0, 1);
236214

237-
IndexNameHelper inh = new IndexNameHelper("test_index", "w", 1);
215+
IndexNameHelper inh = new IndexNameHelper("test_index", true, "w");
238216

239217
assertNull(inh.getCurrentDateSpanStart());
240218
assertNull(inh.getCurrentDateSpanEnd());
@@ -261,7 +239,7 @@ public void dateInCurrentDay() throws Exception
261239
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 9, 0, 1, 0);
262240
LocalDateTime newSpanTime = LocalDateTime.of(2018, 9, 9, 0, 2, 0);
263241

264-
IndexNameHelper inh = new IndexNameHelper("test_index", "d", 1);
242+
IndexNameHelper inh = new IndexNameHelper("test_index", true, "d");
265243

266244
assertNull(inh.getCurrentDateSpanStart());
267245
assertNull(inh.getCurrentDateSpanEnd());
@@ -283,7 +261,7 @@ public void dateInNextDay() throws Exception
283261
LocalDateTime oldSpanTime = LocalDateTime.of(2018, 9, 13, 0, 0, 0);
284262
LocalDateTime newSpanTime = LocalDateTime.of(2018, 9, 14, 0, 0, 1);
285263

286-
IndexNameHelper inh = new IndexNameHelper("test_index", "d", 1);
264+
IndexNameHelper inh = new IndexNameHelper("test_index", true, "d");
287265

288266
assertNull(inh.getCurrentDateSpanStart());
289267
assertNull(inh.getCurrentDateSpanEnd());

‎services/alarm-logger/src/main/java/org/phoebus/alarm/logging/AlarmCmdLogger.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.phoebus.applications.alarm.client.KafkaHelper;
2525
import org.phoebus.applications.alarm.messages.AlarmCommandMessage;
2626
import org.phoebus.applications.alarm.messages.MessageParser;
27+
import org.phoebus.framework.preferences.PreferencesReader;
2728
import org.phoebus.util.indexname.IndexNameHelper;
2829

2930
/**
@@ -35,6 +36,8 @@
3536
*/
3637
public class AlarmCmdLogger implements Runnable {
3738

39+
private static final PreferencesReader prefs = new PreferencesReader(AlarmLoggingService.class, "/application.properties");
40+
3841
private static final String INDEX_FORMAT = "_alarms_cmd";
3942
private final String topic;
4043
private final Serde<AlarmCommandMessage> alarmCommandMessageSerde;
@@ -86,10 +89,10 @@ public long extract(ConsumerRecord<Object, Object> record, long previousTimestam
8689
}));
8790

8891
final String indexDateSpanUnits = props.getProperty("date_span_units");
89-
final Integer indexDateSpanValue = Integer.parseInt(props.getProperty("date_span_value"));
92+
final boolean useDatedIndexNames = Boolean.getBoolean(props.getProperty("use_dated_index_names"));
9093

9194
try {
92-
indexNameHelper = new IndexNameHelper(topic + INDEX_FORMAT , indexDateSpanUnits, indexDateSpanValue);
95+
indexNameHelper = new IndexNameHelper(topic + INDEX_FORMAT, useDatedIndexNames, indexDateSpanUnits);
9396
} catch (Exception ex) {
9497
logger.log(Level.SEVERE, "Time based index creation failed.", ex);
9598
}

‎services/alarm-logger/src/main/java/org/phoebus/alarm/logging/AlarmMessageLogger.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,11 @@ public void run() {
7878

7979

8080
final String indexDateSpanUnits = props.getProperty("date_span_units");
81-
final Integer indexDateSpanValue = Integer.parseInt(props.getProperty("date_span_value"));
81+
final boolean useDatedIndexNames = Boolean.parseBoolean(props.getProperty("use_dated_index_names"));
8282

8383
try {
84-
stateIndexNameHelper = new IndexNameHelper(topic + STATE_INDEX_FORMAT, indexDateSpanUnits, indexDateSpanValue);
85-
configIndexNameHelper = new IndexNameHelper(topic + CONFIG_INDEX_FORMAT , indexDateSpanUnits, indexDateSpanValue);
84+
stateIndexNameHelper = new IndexNameHelper(topic + STATE_INDEX_FORMAT, useDatedIndexNames, indexDateSpanUnits);
85+
configIndexNameHelper = new IndexNameHelper(topic + CONFIG_INDEX_FORMAT , useDatedIndexNames, indexDateSpanUnits);
8686
} catch (Exception ex) {
8787
logger.log(Level.SEVERE, "Time based index creation failed.", ex);
8888
}

‎services/alarm-logger/src/main/java/org/phoebus/alarm/logging/ElasticClientHelper.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import static org.phoebus.alarm.logging.AlarmLoggingService.logger;
4545

4646
/**
47+
* A Utility service to allow for batched indexing of alarm state, config, and command messages to an elastic backend
48+
*
4749
* @author Kunal Shroff {@literal <[email protected]>}
4850
*
4951
*/
@@ -240,6 +242,10 @@ public void run() {
240242
private String ALARM_CONFIG_TEMPLATE = props.getProperty("elasticsearch.alarm.config.template","alarms_config_template");
241243
private String ALARM_CONFIG_TEMPLATE_PATTERN = props.getProperty("elasticsearch.alarm.config.template.pattern","*_alarms_config*");
242244

245+
/**
246+
* Check if the required templated for the phoebus alarm logs exists, if not create them.
247+
* @throws IOException
248+
*/
243249
public void initializeIndices() throws IOException {
244250
// Create the alarm state messages index template
245251
boolean exists = client.indices().existsIndexTemplate(ExistsIndexTemplateRequest.of(i -> i.name(ALARM_STATE_TEMPLATE))).value();

‎services/alarm-logger/src/main/java/org/phoebus/alarm/logging/rest/AlarmLogSearchUtil.java‎

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
1717
import org.phoebus.alarm.logging.AlarmLoggingService;
1818
import org.phoebus.framework.preferences.PreferencesReader;
19+
import org.phoebus.util.indexname.IndexNameHelper;
1920
import org.phoebus.util.time.TimeParser;
2021

2122
import java.io.IOException;
@@ -62,6 +63,12 @@ public class AlarmLogSearchUtil {
6263
private static final String STARTTIME = "start";
6364
private static final String ENDTIME = "end";
6465

66+
private static final String CONFIG_INDEX_FORMAT = "_alarms_config";
67+
private static final String STATE_INDEX_FORMAT = "_alarms_state";
68+
69+
private IndexNameHelper stateIndexNameHelper;
70+
private IndexNameHelper configIndexNameHelper;
71+
6572
/**
6673
* Find all the log (state and config) messages which match the search criteria
6774
*
@@ -74,33 +81,37 @@ public static List<AlarmLogMessage> search(ElasticsearchClient client,
7481
logger.info("searching for alarm log entires : " +
7582
searchParameters.entrySet().stream().map(e -> e.getKey() + ": " + e.getValue()).collect(Collectors.joining()));
7683

77-
String from = formatter.format(Instant.now().minus(7, ChronoUnit.DAYS));
78-
String to = formatter.format(Instant.now());
84+
Instant fromInstant = Instant.now().minus(7, ChronoUnit.DAYS);
85+
Instant toInstant = Instant.now();
7986

8087
// The maximum search result size
8188
int maxSize = prefs.getInt("es_max_size");
89+
final String indexDateSpanUnits = prefs.get("date_span_units");
90+
final boolean useDatedIndexNames = prefs.getBoolean("use_dated_index_names");
91+
8292
boolean configSet = false;
8393
boolean temporalSearch = false;
8494

8595
BoolQuery.Builder boolQuery = new BoolQuery.Builder();
96+
List<String> indexList = new ArrayList<>();
8697

8798
for (Map.Entry<String, String> parameter : searchParameters.entrySet()) {
8899
switch (parameter.getKey().strip().toLowerCase()) {
89100
case STARTTIME:
90101
Object startTime = TimeParser.parseInstantOrTemporalAmount(parameter.getValue().strip());
91102
if (startTime instanceof Instant) {
92-
from = formatter.format((Instant) startTime);
103+
fromInstant = (Instant) startTime;
93104
} else if (startTime instanceof TemporalAmount) {
94-
from = formatter.format(Instant.now().minus((TemporalAmount) startTime));
105+
fromInstant = Instant.now().minus((TemporalAmount) startTime);
95106
}
96107
temporalSearch = true;
97108
break;
98109
case ENDTIME:
99110
Object endTime = TimeParser.parseInstantOrTemporalAmount(parameter.getValue().strip());
100111
if (endTime instanceof Instant) {
101-
to = formatter.format((Instant) endTime);
112+
toInstant = (Instant) endTime;
102113
} else if (endTime instanceof TemporalAmount) {
103-
to = formatter.format(Instant.now().minus((TemporalAmount) endTime));
114+
toInstant = Instant.now().minus((TemporalAmount) endTime);
104115
}
105116
temporalSearch = true;
106117
break;
@@ -187,19 +198,38 @@ public static List<AlarmLogMessage> search(ElasticsearchClient client,
187198
// Add the temporal queries
188199
if (temporalSearch) {
189200
// TODO check that the start is before the end
201+
if(fromInstant.isBefore(toInstant)) {
202+
} else {
203+
//
204+
logger.log(Level.SEVERE,
205+
"Failed to search for alarm logs: invalid time range from: " + formatter.format(fromInstant) + " to: " + formatter.format(toInstant));
206+
}
190207
//Effectively final
191-
String finalFrom = from;
192-
String finalTo = to;
208+
Instant finalFromInstant = fromInstant;
209+
Instant finalToInstant = toInstant;
193210
boolQuery.must(
194211
Query.of(q -> q
195212
.range(RangeQuery.of(r -> r
196213
.field("message_time")
197-
.from(finalFrom)
198-
.to(finalTo)
214+
.from(formatter.format(finalFromInstant))
215+
.to(formatter.format(finalToInstant))
199216
)
200217
)
201218
)
202219
);
220+
221+
try {
222+
IndexNameHelper indexNameHelper = new IndexNameHelper("*", useDatedIndexNames, indexDateSpanUnits);
223+
String fromIndex = indexNameHelper.getIndexName(fromInstant);
224+
String toIndex = indexNameHelper.getIndexName(toInstant);
225+
if(fromIndex.equalsIgnoreCase(toIndex)) {
226+
indexList.add(fromIndex);
227+
}
228+
} catch (Exception e) {
229+
e.printStackTrace();
230+
}
231+
232+
203233
}
204234

205235
int finalSize = maxSize; //Effectively final

‎services/alarm-logger/src/main/resources/application.properties‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# The server port for the rest service
2-
server.port=9000
2+
server.port=8080
33

44
# Disable the spring banner
55
spring.main.banner-mode=off
@@ -12,7 +12,7 @@ alarm_topics=Accelerator
1212

1313
# location of elastic node/s
1414
es_host=localhost
15-
es_port=9228
15+
es_port=9200
1616
# max default size for es queries
1717
es_max_size=1000
1818
# set to 'true' if sniffing to be enabled to discover other cluster nodes
@@ -22,17 +22,16 @@ es_sniff=false
2222
es_create_templates=true
2323

2424
# Kafka server location
25-
bootstrap.servers=localhost:19092
25+
bootstrap.servers=localhost:9092
2626

2727
# Kafka client properties file
2828
kafka_properties=
2929

30+
# A flag indicating if the service should use index names based on date/time generated periodically based on date_span_units
31+
use_dated_index_names=true
32+
3033
# The units of the indices date span: Days (D), Weeks(W), Months(M), Years(Y).
3134
date_span_units=M
3235

33-
# The value of the indices date span.
34-
# An integer value to be combined with the units. For example, if months were selected and the value set to 2, each index would span 2 months.
35-
date_span_value=1
36-
3736
# Size of the thread pool for message and command loggers. Two threads per topic/configuration are required
3837
thread_pool_size=4

0 commit comments

Comments
 (0)