Skip to content

Commit 46d1071

Browse files
stas-panasiukstas-panasiuk
authored
[CYB-183] Marked output fields where needed (#60)
1 parent 2614b42 commit 46d1071

File tree

9 files changed

+65
-39
lines changed

9 files changed

+65
-39
lines changed

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,12 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
16-
import static java.lang.String.format;
1715
import com.cloudera.parserchains.core.FieldName;
1816
import com.cloudera.parserchains.core.FieldValue;
1917
import com.cloudera.parserchains.core.Message;
2018
import com.cloudera.parserchains.core.Parser;
2119
import com.cloudera.parserchains.core.catalog.Configurable;
2220
import com.cloudera.parserchains.core.catalog.MessageParser;
23-
import java.io.ByteArrayInputStream;
24-
import java.io.IOException;
25-
import java.util.Optional;
2621
import lombok.extern.slf4j.Slf4j;
2722
import org.apache.avro.AvroRuntimeException;
2823
import org.apache.avro.Schema;
@@ -35,6 +30,13 @@
3530
import org.apache.flink.core.fs.FileSystem;
3631
import org.apache.flink.core.fs.Path;
3732

33+
import java.io.ByteArrayInputStream;
34+
import java.io.IOException;
35+
import java.util.Optional;
36+
37+
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
38+
import static java.lang.String.format;
39+
3840
@MessageParser(
3941
name = "Simple Avro parser",
4042
description = "Parses Avro data by creating a field for each Avro element.")
@@ -55,7 +57,8 @@ public AvroParser() {
5557
key = "input",
5658
label = "Input Field",
5759
description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
58-
defaultValue = DEFAULT_INPUT_FIELD)
60+
defaultValue = DEFAULT_INPUT_FIELD,
61+
isOutputName = true)
5962
public AvroParser inputField(String fieldName) {
6063
if (StringUtils.isNotBlank(fieldName)) {
6164
this.inputField = FieldName.of(fieldName);

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
16-
import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
17-
import static java.lang.String.format;
1815
import com.cloudera.parserchains.core.Constants;
1916
import com.cloudera.parserchains.core.FieldName;
2017
import com.cloudera.parserchains.core.FieldValue;
@@ -29,13 +26,18 @@
2926
import com.fasterxml.jackson.databind.ObjectReader;
3027
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
3128
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
29+
import org.apache.commons.lang3.StringUtils;
30+
3231
import java.util.ArrayList;
3332
import java.util.Collections;
3433
import java.util.List;
3534
import java.util.Objects;
3635
import java.util.Optional;
3736
import java.util.function.Supplier;
38-
import org.apache.commons.lang3.StringUtils;
37+
38+
import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
39+
import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
40+
import static java.lang.String.format;
3941

4042
/**
4143
* Parses delimited text like CSV.
@@ -95,6 +97,7 @@ public CsvTextParser withInputField(FieldName inputField) {
9597
@Configurable(key = "inputField",
9698
label = "Input Field",
9799
description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
100+
isOutputName = true,
98101
defaultValue = Constants.DEFAULT_INPUT_FIELD)
99102
public CsvTextParser withInputField(String fieldName) {
100103
if (StringUtils.isNotEmpty(fieldName)) {

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static java.lang.String.format;
1615
import com.cloudera.parserchains.core.Constants;
1716
import com.cloudera.parserchains.core.FieldName;
1817
import com.cloudera.parserchains.core.FieldValue;
@@ -22,10 +21,13 @@
2221
import com.cloudera.parserchains.core.StringFieldValue;
2322
import com.cloudera.parserchains.core.catalog.Configurable;
2423
import com.cloudera.parserchains.core.catalog.MessageParser;
25-
import java.util.Optional;
2624
import lombok.extern.slf4j.Slf4j;
2725
import org.apache.commons.lang3.StringUtils;
2826

27+
import java.util.Optional;
28+
29+
import static java.lang.String.format;
30+
2931
@MessageParser(
3032
name="Delimited Key Values",
3133
description="Parses delimited key-value pairs."
@@ -57,7 +59,8 @@ public DelimitedKeyValueParser() {
5759
key="input",
5860
label="Input Field",
5961
description="The input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
60-
defaultValue=Constants.DEFAULT_INPUT_FIELD)
62+
defaultValue=Constants.DEFAULT_INPUT_FIELD,
63+
isOutputName = true)
6164
public DelimitedKeyValueParser inputField(String inputField) {
6265
if(StringUtils.isNotBlank(inputField)) {
6366
this.inputField = FieldName.of(inputField);

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static java.lang.String.format;
1615
import com.cloudera.parserchains.core.Constants;
1716
import com.cloudera.parserchains.core.FieldName;
1817
import com.cloudera.parserchains.core.Message;
@@ -23,11 +22,14 @@
2322
import com.cloudera.parserchains.core.catalog.WidgetType;
2423
import io.krakens.grok.api.Grok;
2524
import io.krakens.grok.api.GrokCompiler;
25+
import org.apache.commons.lang3.StringUtils;
26+
2627
import java.time.ZoneOffset;
2728
import java.util.ArrayList;
2829
import java.util.Collections;
2930
import java.util.List;
30-
import org.apache.commons.lang3.StringUtils;
31+
32+
import static java.lang.String.format;
3133

3234
@MessageParser(
3335
name = "Grok",
@@ -74,7 +76,8 @@ private void doParse(String textToParse, Message.Builder output) {
7476

7577
@Configurable(key = "grokPattern",
7678
description = "Define a Grok pattern that can be referenced from an expression.",
77-
orderPriority = 1)
79+
orderPriority = 1,
80+
isOutputName = true)
7881
public GrokParser pattern(
7982
@Parameter(key = "name", label = "Pattern Name") String patternName,
8083
@Parameter(key = "regex", label = "Pattern Regex", widgetType = WidgetType.TEXTAREA) String patternRegex) {

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
16-
import static java.lang.String.format;
1715
import com.cloudera.parserchains.core.FieldName;
1816
import com.cloudera.parserchains.core.Message;
1917
import com.cloudera.parserchains.core.Parser;
@@ -23,6 +21,8 @@
2321
import com.fasterxml.jackson.core.JsonProcessingException;
2422
import com.fasterxml.jackson.databind.ObjectMapper;
2523
import com.fasterxml.jackson.databind.ObjectReader;
24+
import org.apache.commons.lang3.StringUtils;
25+
2626
import java.io.IOException;
2727
import java.util.ArrayList;
2828
import java.util.HashMap;
@@ -31,7 +31,9 @@
3131
import java.util.Objects;
3232
import java.util.Optional;
3333
import java.util.stream.Collectors;
34-
import org.apache.commons.lang3.StringUtils;
34+
35+
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
36+
import static java.lang.String.format;
3537

3638
@MessageParser(
3739
name="Simple JSON",
@@ -52,7 +54,8 @@ public JSONParser() {
5254
key="input",
5355
label="Input Field",
5456
description= "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
55-
defaultValue=DEFAULT_INPUT_FIELD)
57+
defaultValue=DEFAULT_INPUT_FIELD,
58+
isOutputName = true)
5659
public JSONParser inputField(String fieldName) {
5760
if(StringUtils.isNotBlank(fieldName)) {
5861
this.inputField = FieldName.of(fieldName);

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
1615
import com.cloudera.cyber.parser.MessageToParse;
1716
import com.cloudera.cyber.stellar.MetronCompatibilityParser;
1817
import com.cloudera.parserchains.core.FieldName;
@@ -21,11 +20,6 @@
2120
import com.cloudera.parserchains.core.Parser;
2221
import com.cloudera.parserchains.core.catalog.Configurable;
2322
import com.cloudera.parserchains.core.catalog.MessageParser;
24-
import java.io.IOException;
25-
import java.util.List;
26-
import java.util.Map;
27-
import java.util.Optional;
28-
import java.util.Set;
2923
import lombok.extern.slf4j.Slf4j;
3024
import org.apache.commons.collections.CollectionUtils;
3125
import org.apache.commons.lang3.StringUtils;
@@ -35,6 +29,14 @@
3529
import org.apache.metron.parsers.interfaces.MessageParserResult;
3630
import org.json.simple.JSONObject;
3731

32+
import java.io.IOException;
33+
import java.util.List;
34+
import java.util.Map;
35+
import java.util.Optional;
36+
import java.util.Set;
37+
38+
import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
39+
3840
@MessageParser(
3941
name = "Metron Stellar parser",
4042
description = "Metron compatibility parser.")
@@ -53,7 +55,8 @@ public StellarParser() {
5355
key = "input",
5456
label = "Input Field",
5557
description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
56-
defaultValue = DEFAULT_INPUT_FIELD)
58+
defaultValue = DEFAULT_INPUT_FIELD,
59+
isOutputName = true)
5760
public StellarParser inputField(String fieldName) {
5861
if (StringUtils.isNotBlank(fieldName)) {
5962
this.inputField = FieldName.of(fieldName);

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static java.lang.String.format;
1615
import com.cloudera.parserchains.core.Constants;
1716
import com.cloudera.parserchains.core.FieldName;
1817
import com.cloudera.parserchains.core.Message;
@@ -22,9 +21,12 @@
2221
import com.cloudera.parserchains.core.catalog.MessageParser;
2322
import com.github.palindromicity.syslog.SyslogParserBuilder;
2423
import com.github.palindromicity.syslog.SyslogSpecification;
25-
import java.util.Objects;
2624
import org.apache.commons.lang3.StringUtils;
2725

26+
import java.util.Objects;
27+
28+
import static java.lang.String.format;
29+
2830
@MessageParser(
2931
name="Syslog",
3032
description="Parses Syslog according to RFC 3164 and 5424.")
@@ -62,7 +64,8 @@ public SyslogSpecification getSpecification() {
6264
@Configurable(key="inputField",
6365
label="Input Field",
6466
description="The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
65-
defaultValue = Constants.DEFAULT_INPUT_FIELD)
67+
defaultValue = Constants.DEFAULT_INPUT_FIELD,
68+
isOutputName = true)
6669
public SyslogParser withInputField(String inputField) {
6770
if(StringUtils.isNotBlank(inputField)) {
6871
this.inputField = FieldName.of(inputField);

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static java.util.stream.Collectors.toList;
1615
import com.cloudera.parserchains.core.FieldName;
1716
import com.cloudera.parserchains.core.FieldValue;
1817
import com.cloudera.parserchains.core.Message;
@@ -21,6 +20,8 @@
2120
import com.cloudera.parserchains.core.catalog.Configurable;
2221
import com.cloudera.parserchains.core.catalog.MessageParser;
2322
import com.cloudera.parserchains.core.catalog.Parameter;
23+
import org.apache.commons.lang3.StringUtils;
24+
2425
import java.io.Serializable;
2526
import java.time.DateTimeException;
2627
import java.time.Instant;
@@ -32,7 +33,8 @@
3233
import java.util.Iterator;
3334
import java.util.List;
3435
import java.util.TimeZone;
35-
import org.apache.commons.lang3.StringUtils;
36+
37+
import static java.util.stream.Collectors.toList;
3638

3739
/**
3840
* A parser to extract formatted timestamps and express them as epoch time, preserving the
@@ -82,7 +84,7 @@ private Long parseDate(String inputValue, DateTimeFormatter format, String tz) t
8284
description = "The field that will contain the timestamp.",
8385
multipleValues = true)
8486
public TimestampFormatParser withOutputField(
85-
@Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true) String fieldName,
87+
@Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true, isOutputName = true) String fieldName,
8688
@Parameter(key = "format", label = "Time format", description = "A compatible time format", required = true) String format,
8789
@Parameter(key = "tz", label = "Timezone", description = "Optionally set the expected timezone", required = true) String tz
8890
) {

‎flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212

1313
package com.cloudera.parserchains.parsers;
1414

15-
import static java.lang.String.format;
16-
import static org.apache.commons.lang3.StringUtils.isNotBlank;
1715
import com.cloudera.parserchains.core.Constants;
1816
import com.cloudera.parserchains.core.FieldName;
1917
import com.cloudera.parserchains.core.FieldValue;
@@ -23,16 +21,20 @@
2321
import com.cloudera.parserchains.core.catalog.Configurable;
2422
import com.cloudera.parserchains.core.catalog.MessageParser;
2523
import com.github.wnameless.json.flattener.JsonFlattener;
26-
import java.util.Map;
27-
import java.util.Objects;
28-
import java.util.Optional;
2924
import lombok.extern.slf4j.Slf4j;
3025
import org.apache.commons.lang3.StringUtils;
3126
import org.json.JSONException;
3227
import org.json.JSONObject;
3328
import org.json.XML;
3429
import org.json.XMLParserConfiguration;
3530

31+
import java.util.Map;
32+
import java.util.Objects;
33+
import java.util.Optional;
34+
35+
import static java.lang.String.format;
36+
import static org.apache.commons.lang3.StringUtils.isNotBlank;
37+
3638
@MessageParser(
3739
name = "XML Flattener",
3840
description = "Flattens XML data."
@@ -53,7 +55,8 @@ public XMLFlattener() {
5355
@Configurable(key = "inputField",
5456
label = "Input Field",
5557
description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
56-
defaultValue = Constants.DEFAULT_INPUT_FIELD)
58+
defaultValue = Constants.DEFAULT_INPUT_FIELD,
59+
isOutputName = true)
5760
public XMLFlattener inputField(String fieldName) {
5861
if (StringUtils.isNotEmpty(fieldName)) {
5962
this.inputField = FieldName.of(fieldName);

0 commit comments

Comments
 (0)