Skip to content

Commit db0cfbf

Browse files
carolyndubycarolynduby
authored
[CYB-216] upgrade cybersec toolkit for csa 1.14.0.0 and cdh 7.3.1 (#90)
1 parent 8fbc43d commit db0cfbf

File tree

6 files changed

+65
-55
lines changed

6 files changed

+65
-55
lines changed

‎.github/workflows/publish_release.yml‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@
99
name: Publish Release
1010

1111
on:
12-
schedule:
13-
# At 0:00am each day on the default branch
14-
- cron: '0 0 * * *'
1512
workflow_dispatch:
1613
inputs:
1714
tag:
@@ -29,25 +26,7 @@ on:
2926
type: boolean
3027

3128
jobs:
32-
# checks if there's a commit in last 24hr
33-
check_date:
34-
runs-on: ubuntu-latest
35-
steps:
36-
- uses: actions/checkout@v2
37-
- name: print latest_commit
38-
run: echo ${{ github.sha }}
39-
40-
- id: should_run
41-
continue-on-error: true
42-
name: check latest commit is less than a day
43-
run: |
44-
test -z $(git rev-list --after="24 hours" ${{ github.sha }}) \
45-
&& (gh run cancel ${{ github.run_id }} \
46-
&& gh run watch ${{ github.run_id }})
47-
env:
48-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4929
publish_release:
50-
needs: check_date
5130
runs-on: ubuntu-latest
5231
container: maven:3-eclipse-temurin-8
5332
steps:

‎flink-cyber/caracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ public void processBroadcastElement(SplitConfig splitConfig, Context context, Co
7676
log.info(String.format("Adding splitter %s on thread %d ", splitConfig, Thread.currentThread().getId()));
7777
context.getBroadcastState(SplitJob.Descriptors.broadcastState).put(splitConfig.getTopic(), splitConfig);
7878
SplittingFlatMapFunction f = new SplittingFlatMapFunction(splitConfig, signKey);
79-
f.open(null);
79+
f.open((Configuration)null);
8080
splitters.put(splitConfig.getTopic(), f);
8181
}
8282
}

‎flink-cyber/cyber-jobs/src/main/resources/scripts/env.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ function get_kerberos_config() {
149149
if [ -f "${kerberos_properties}" ]; then
150150
read_properties_into_variables "${kerberos_properties}"
151151
read_properties_into_variables "${internal_ssl_properties}"
152-
security_options+=("-yD" "security.kerberos.login.keytab=${kerberos_keytab}")
152+
kerberos_keytab_abs=$(readlink -f "${kerberos_keytab}")
153+
security_options+=("-yD" "security.kerberos.login.keytab=${kerberos_keytab_abs}")
153154
security_options+=("-yD" "security.kerberos.login.principal=${kerberos_principal}")
154155
security_options+=("-yD" "security.ssl.internal.enabled=true")
155156
security_options+=("-yD" "security.ssl.internal.keystore=${flink_internal_keystore}")

‎flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.io.IOException;
2222
import java.io.StringWriter;
2323
import java.io.Writer;
24+
import java.nio.charset.StandardCharsets;
2425
import java.time.LocalDateTime;
2526
import java.time.ZoneOffset;
2627
import java.util.List;
@@ -36,16 +37,13 @@ public ThreatGenerator() {
3637
cfg.setClassLoaderForTemplateLoading(Thread.currentThread().getContextClassLoader(), "");
3738
cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50, 50));
3839
cfg.setTemplateUpdateDelayMilliseconds(3600 * 24 * 1000);
39-
try {
40-
lines = IOUtils.readLines(getClass().getResourceAsStream("/threats/threatq.json"));
41-
StringTemplateLoader stringLoader = new StringTemplateLoader();
42-
IntStream.range(0, lines.size()).forEach(i -> {
43-
stringLoader.putTemplate(String.valueOf(i), lines.get(i));
44-
});
45-
cfg.setTemplateLoader(stringLoader);
46-
} catch (IOException ioException) {
47-
ioException.printStackTrace();
48-
}
40+
41+
lines = IOUtils.readLines(getClass().getResourceAsStream("/threats/threatq.json"), StandardCharsets.UTF_8);
42+
StringTemplateLoader stringLoader = new StringTemplateLoader();
43+
IntStream.range(0, lines.size()).forEach(i -> {
44+
stringLoader.putTemplate(String.valueOf(i), lines.get(i));
45+
});
46+
cfg.setTemplateLoader(stringLoader);
4947
}
5048

5149
private static final RandomGenerators utils = new RandomGenerators();

‎flink-cyber/flink-indexing/flink-indexing-hive/pom.xml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,18 @@
257257
<groupId>org.slf4j</groupId>
258258
<artifactId>slf4j-log4j12</artifactId>
259259
</exclusion>
260+
<exclusion>
261+
<groupId>org.apache.atlas</groupId>
262+
<artifactId>atlas-intg</artifactId>
263+
</exclusion>
264+
<exclusion>
265+
<groupId>org.cloudera.logredactor</groupId>
266+
<artifactId>logredactor</artifactId>
267+
</exclusion>
268+
<exclusion>
269+
<groupId>com.cloudera</groupId>
270+
<artifactId>jwtprovider-api</artifactId>
271+
</exclusion>
260272
</exclusions>
261273
</dependency>
262274

@@ -274,6 +286,18 @@
274286
<groupId>commons-cli</groupId>
275287
<artifactId>commons-cli</artifactId>
276288
</exclusion>
289+
<exclusion>
290+
<groupId>org.apache.hive</groupId>
291+
<artifactId>hive-serde</artifactId>
292+
</exclusion>
293+
<exclusion>
294+
<groupId>org.apache.hive</groupId>
295+
<artifactId>hive-metastore</artifactId>
296+
</exclusion>
297+
<exclusion>
298+
<groupId>org.apache.hive</groupId>
299+
<artifactId>hive-exec</artifactId>
300+
</exclusion>
277301
</exclusions>
278302
</dependency>
279303
<dependency>
@@ -290,6 +314,10 @@
290314
<groupId>commons-cli</groupId>
291315
<artifactId>commons-cli</artifactId>
292316
</exclusion>
317+
<exclusion>
318+
<groupId>com.cloudera</groupId>
319+
<artifactId>jwtprovider-api</artifactId>
320+
</exclusion>
293321
</exclusions>
294322
</dependency>
295323

@@ -303,6 +331,22 @@
303331
<groupId>org.glassfish</groupId>
304332
<artifactId>javax.el</artifactId>
305333
</exclusion>
334+
<exclusion>
335+
<groupId>org.apache.hive</groupId>
336+
<artifactId>hive-cli</artifactId>
337+
</exclusion>
338+
<exclusion>
339+
<groupId>org.apache.hive</groupId>
340+
<artifactId>hive-common</artifactId>
341+
</exclusion>
342+
<exclusion>
343+
<groupId>org.apache.hive</groupId>
344+
<artifactId>hive-metastore</artifactId>
345+
</exclusion>
346+
<exclusion>
347+
<groupId>org.apache.hive</groupId>
348+
<artifactId>hive-exec</artifactId>
349+
</exclusion>
306350
</exclusions>
307351
</dependency>
308352

‎flink-cyber/pom.xml‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -55,23 +55,23 @@
5555
<properties>
5656
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
5757
<java.version>1.8</java.version>
58-
<csa.version>csa1.12.0.0</csa.version>
59-
<flink.version>1.18.0-csa1.12.0.0</flink.version>
58+
<csa.version>csa1.14.1.0</csa.version>
59+
<flink.version>1.19.2-${csa.version}</flink.version>
6060
<cloudera.registry>1.0-${csa.version}</cloudera.registry>
6161
<flink.hbase.version>2.4</flink.hbase.version>
62-
<cdh.version>7.1.9.0-387</cdh.version>
62+
<cdh.version>7.3.1.300-81</cdh.version>
6363
<lombok.version>1.18.22</lombok.version>
6464
<lombok.plugin.version>1.18.16.0</lombok.plugin.version>
6565
<guava.version>21.0</guava.version>
66-
<avro.version>1.11.3</avro.version>
66+
<avro.version>1.11.4</avro.version>
6767
<log4j.kafka.version>3.4.1.${cdh.version}</log4j.kafka.version>
6868
<scala.binary.version>2.12</scala.binary.version>
6969
<kafka.version>3.4.1.${cdh.version}</kafka.version>
7070
<nifi.version>1.11.4</nifi.version>
7171
<jackson.version>2.11.2</jackson.version>
7272
<jackson.datatype.version>2.10.1</jackson.datatype.version>
7373
<smm.intercepter.version>2.3.0.${cdh.version}</smm.intercepter.version>
74-
<flink.format.parquet.version>1.11.1</flink.format.parquet.version>
74+
<flink.format.parquet.version>1.12.2</flink.format.parquet.version>
7575
<slf4j.version>1.7.36</slf4j.version>
7676
<log4j.version>2.17.2</log4j.version>
7777
<spring-boot.version>2.7.6</spring-boot.version>
@@ -107,7 +107,7 @@
107107
<graalvm.version>21.3.10</graalvm.version>
108108
<junit.version>4.12</junit.version>
109109
<junit5.version>5.7.0</junit5.version>
110-
<orc.version>1.5.1.${cdh.version}</orc.version>
110+
<orc.version>1.8.3.${cdh.version}</orc.version>
111111
<swagger-annotations.version>1.6.0</swagger-annotations.version>
112112

113113
<!--Commons-->
@@ -119,7 +119,7 @@
119119

120120
<!--Maven-->
121121
<maven.build.timestamp.format>yyMMddHHmm</maven.build.timestamp.format>
122-
<maven-shade-plugin.version>3.3.0</maven-shade-plugin.version>
122+
<maven-shade-plugin.version>3.4.1</maven-shade-plugin.version>
123123
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
124124
<maven-source-plugin.version>3.3.1</maven-source-plugin.version>
125125
<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
@@ -181,7 +181,7 @@
181181
<dependency>
182182
<groupId>org.apache.flink</groupId>
183183
<artifactId>flink-connector-kafka</artifactId>
184-
<version>3.1-${csa.version}</version>
184+
<version>3.2-${csa.version}</version>
185185
<exclusions>
186186
<exclusion>
187187
<groupId>org.apache.kafka</groupId>
@@ -290,18 +290,6 @@
290290
<artifactId>flink-connector-cloudera-registry</artifactId>
291291
<version>1.0-${csa.version}</version>
292292
<exclusions>
293-
<exclusion>
294-
<groupId>org.apache.avro</groupId>
295-
<artifactId>avro</artifactId>
296-
</exclusion>
297-
<exclusion>
298-
<groupId>org.apache.avro</groupId>
299-
<artifactId>avro</artifactId>
300-
</exclusion>
301-
<exclusion>
302-
<groupId>com.hortonworks.registries</groupId>
303-
<artifactId>schema-registry-serdes</artifactId>
304-
</exclusion>
305293
<exclusion>
306294
<groupId>org.everit.json</groupId>
307295
<artifactId>org.everit.json.schema</artifactId>
@@ -316,7 +304,7 @@
316304
<dependency>
317305
<groupId>com.hortonworks.smm</groupId>
318306
<artifactId>monitoring-interceptors</artifactId>
319-
<version>[${smm.intercepter.version},)</version>
307+
<version>${smm.intercepter.version}</version>
320308
<exclusions>
321309
<exclusion>
322310
<groupId>org.slf4j</groupId>
@@ -738,7 +726,7 @@
738726
<dependency>
739727
<groupId>org.apache.flink</groupId>
740728
<artifactId>flink-connector-kafka</artifactId>
741-
<version>3.1-${csa.version}</version>
729+
<version>3.2-${csa.version}</version>
742730
</dependency>
743731
</dependencies>
744732
</profile>

0 commit comments

Comments
 (0)