Skip to content

Commit 2c00a1d

Browse files
carolyndubycarolynduby
committed
merge in CSA 1.12.0 with 7.1.9 CDH
2 parents 46f8087 + 629a2c1 commit 2c00a1d

File tree

37 files changed

+459
-180
lines changed

37 files changed

+459
-180
lines changed

‎flink-cyber/caracal-generator/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@
3333
<artifactId>flink-connector-kafka</artifactId>
3434
</dependency>
3535

36+
<dependency>
37+
<groupId>org.apache.flink</groupId>
38+
<artifactId>flink-clients</artifactId>
39+
<scope>provided</scope>
40+
<version>${flink.version}</version>
41+
</dependency>
42+
3643
<dependency>
3744
<groupId>com.hortonworks.smm</groupId>
3845
<artifactId>monitoring-interceptors</artifactId>

‎flink-cyber/caracal-parser/pom.xml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,12 @@
126126

127127
<dependency>
128128
<groupId>org.apache.flink</groupId>
129-
<artifactId>flink-cloudera-registry</artifactId>
130-
<version>${flink.version}</version>
129+
<artifactId>flink-connector-cloudera-registry</artifactId>
130+
</dependency>
131+
132+
<dependency>
133+
<groupId>com.hortonworks.registries</groupId>
134+
<artifactId>schema-registry-serdes</artifactId>
131135
</dependency>
132136

133137
<dependency>
@@ -164,6 +168,11 @@
164168
<artifactId>flink-orc</artifactId>
165169
</dependency>
166170

171+
<dependency>
172+
<groupId>org.apache.orc</groupId>
173+
<artifactId>orc-core</artifactId>
174+
</dependency>
175+
167176
<dependency>
168177
<groupId>org.apache.parquet</groupId>
169178
<artifactId>parquet-avro</artifactId>

‎flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public long getTs() {
5454
return message.getTs();
5555
}
5656

57-
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName())
57+
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null)
5858
.fields()
5959
.name("message").type(Message.SCHEMA$).noDefault()
6060
.name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault()

‎flink-cyber/flink-alert-scoring/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@
9595
<groupId>org.apache.flink</groupId>
9696
<artifactId>flink-statebackend-rocksdb</artifactId>
9797
</dependency>
98+
<dependency>
99+
<groupId>com.google.guava</groupId>
100+
<artifactId>guava</artifactId>
101+
<scope>test</scope>
102+
</dependency>
98103

99104
</dependencies>
100105
<build>

‎flink-cyber/flink-commands/json-commands/pom.xml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
<dependency>
3737
<groupId>junit</groupId>
3838
<artifactId>junit</artifactId>
39-
<scope>test</scope>
4039
</dependency>
4140

4241
<dependency>
@@ -70,15 +69,14 @@
7069
<version>${log4j.version}</version>
7170
<scope>provided</scope>
7271
</dependency>
73-
<dependency>
74-
<groupId>com.google.guava</groupId>
75-
<artifactId>guava</artifactId>
76-
<version>${guava.version}</version>
77-
</dependency>
7872
<dependency>
7973
<groupId>com.fasterxml.jackson.core</groupId>
8074
<artifactId>jackson-databind</artifactId>
81-
<version>${jackson.version}</version>
75+
</dependency>
76+
<dependency>
77+
<groupId>com.google.guava</groupId>
78+
<artifactId>guava</artifactId>
79+
<scope>test</scope>
8280
</dependency>
8381
</dependencies>
8482

‎flink-cyber/flink-commands/scoring-commands/pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@
6060
<version>${jackson.datatype.version}</version>
6161
<scope>compile</scope>
6262
</dependency>
63+
<dependency>
64+
<groupId>com.hortonworks.registries</groupId>
65+
<artifactId>schema-registry-serdes</artifactId>
66+
</dependency>
67+
<dependency>
68+
<groupId>com.google.guava</groupId>
69+
<artifactId>guava</artifactId>
70+
</dependency>
6371
<!-- Logging and Metrics -->
6472
<dependency>
6573
<groupId>org.slf4j</groupId>
@@ -90,6 +98,10 @@
9098
<artifactId>avro</artifactId>
9199
<scope>compile</scope>
92100
</dependency>
101+
<dependency>
102+
<groupId>org.apache.flink</groupId>
103+
<artifactId>flink-java</artifactId>
104+
</dependency>
93105
</dependencies>
94106
<build>
95107
<plugins>
@@ -128,6 +140,12 @@
128140
</excludes>
129141
</filter>
130142
</filters>
143+
<relocations>
144+
<relocation>
145+
<pattern>com.google.</pattern>
146+
<shadedPattern>com.cloudera.cyber.com.google.</shadedPattern>
147+
</relocation>
148+
</relocations>
131149
<transformers>
132150
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
133151
<mainClass>com.cloudera.cyber.scoring.ScoringJobKafka</mainClass>

‎flink-cyber/flink-common/pom.xml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,22 @@
4343
<groupId>org.apache.flink</groupId>
4444
<artifactId>flink-connector-kafka</artifactId>
4545
</dependency>
46+
<dependency>
47+
<groupId>org.apache.kafka</groupId>
48+
<artifactId>kafka-clients</artifactId>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.apache.flink</groupId>
52+
<artifactId>flink-connector-cloudera-registry</artifactId>
53+
</dependency>
54+
<dependency>
55+
<groupId>com.hortonworks.registries</groupId>
56+
<artifactId>schema-registry-serdes</artifactId>
57+
</dependency>
4658
<dependency>
4759
<groupId>org.apache.flink</groupId>
48-
<artifactId>flink-cloudera-registry</artifactId>
60+
<artifactId>flink-connector-base</artifactId>
61+
<version>${flink.version}</version>
4962
</dependency>
5063
<dependency>
5164
<groupId>org.apache.avro</groupId>

‎flink-cyber/flink-cyber-api/pom.xml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,19 @@
3232
</plugins>
3333
</build>
3434

35+
<dependencyManagement>
36+
<dependencies>
37+
<dependency>
38+
<!-- Import dependency management from Spring Boot -->
39+
<groupId>org.springframework.boot</groupId>
40+
<artifactId>spring-boot-dependencies</artifactId>
41+
<version>${spring-boot.version}</version>
42+
<type>pom</type>
43+
<scope>import</scope>
44+
</dependency>
45+
</dependencies>
46+
</dependencyManagement>
47+
3548
<dependencies>
3649
<dependency>
3750
<groupId>org.projectlombok</groupId>
@@ -55,10 +68,15 @@
5568
<artifactId>flink-table-common</artifactId>
5669
</dependency>
5770

71+
72+
<dependency>
73+
<groupId>org.springframework</groupId>
74+
<artifactId>spring-core</artifactId>
75+
</dependency>
76+
5877
<dependency>
5978
<groupId>org.apache.avro</groupId>
6079
<artifactId>avro</artifactId>
61-
<version>${avro.version}</version>
6280
</dependency>
6381

6482
<dependency>

‎flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import lombok.Data;
1919
import lombok.NoArgsConstructor;
2020
import lombok.NonNull;
21+
import org.apache.avro.LogicalTypes;
2122
import org.apache.avro.Schema;
2223
import org.apache.avro.SchemaBuilder;
2324
import org.apache.avro.specific.SpecificRecord;
@@ -56,7 +57,7 @@ public class Message extends SpecificRecordBase implements SpecificRecord, Ident
5657
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName())
5758
.fields()
5859
.requiredString("id")
59-
.requiredLong("ts")
60+
.name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault()
6061
.name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault()
6162
.requiredString("message")
6263
.name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$)))

‎flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,16 @@
55
public class AvroSchemas {
66

77
public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName) {
8-
return SchemaBuilder.record(recordName).namespace(namespace)
9-
.prop("ssb.rowtimeAttribute", "ts")
10-
.prop("ssb.watermarkExpression", "`ts` - INTERVAL '30' SECOND");
8+
return createRecordBuilder(namespace, recordName, "ts");
9+
}
10+
11+
public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName, String tsFieldName) {
12+
SchemaBuilder.RecordBuilder<org.apache.avro.Schema> recordBuilder = SchemaBuilder.record(recordName).namespace(namespace);
13+
if (tsFieldName != null) {
14+
recordBuilder
15+
.prop("ssb.rowtimeAttribute", tsFieldName)
16+
.prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName));
17+
}
18+
return recordBuilder;
1119
}
1220
}

0 commit comments

Comments
 (0)