Skip to content

Commit 3078158

Browse files
stas-panasiukstas-panasiuk
committed
Merge branch 'develop' into codestyle
# Conflicts: # flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml # flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java # flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java # flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java # flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/service/JobService.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/StartupComponent.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/config/kafka/KafkaConfig.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ChainController.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ClusterController.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserController.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/controller/ParserSampleController.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/ClusterService.java # flink-cyber/metron-parser-chain/parser-chains-config-service/src/main/java/com/cloudera/parserchains/queryservice/service/KafkaService.java
2 parents 5d5fee4 + 491eaae commit 3078158

File tree

124 files changed

+4016
-2822
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

124 files changed

+4016
-2822
lines changed

‎flink-cyber/cyber-csd/src/main/aux/templates/parser-ui.properties.j2‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
server.servlet.context-path=/
2+
server.tomcat.accesslog.enabled=true
3+
server.port={{ globals['server.port'] }}
4+
config.path={{ globals['config.path'] }}
5+
pipelines.path={{ globals['server.pipelineDir'] }}
6+
springdoc.api-doc.path=/api-docs
7+
springdoc.swagger-ui.path=/swagger-ui
8+
spring.profiles.active=kafka
9+
logging.config={{ globals['logging_config'] }}
10+
11+
{% for obj in globals['clusters_kafka_map'] | fromjson %}
12+
13+
spring.kafka.external-clusters.{{ obj['name'] }}.bootstrap-servers={{obj['bootstrap']}}
14+
spring.kafka.external-clusters.{{ obj['name'] }}.reply-topic={{ globals['spring.kafka.replyTopic'] }}
15+
spring.kafka.external-clusters.{{ obj['name'] }}.request-topic={{ globals['spring.kafka.requestTopic'] }}
16+
spring.kafka.external-clusters.{{ obj['name'] }}.consumer.group-id={{ globals['server.group.id'] }}
17+
spring.kafka.external-clusters.{{ obj['name'] }}.consumer.enable-auto-commit={{ globals['spring.kafka.consumer.enable-auto-commit'] }}
18+
spring.kafka.external-clusters.{{ obj['name'] }}.consumer.auto-offset-reset={{ globals['spring.kafka.consumer.auto-offset-reset'] }}
19+
spring.kafka.external-clusters.{{ obj['name'] }}.consumer.key-deserializer={{ globals['spring.kafka.consumer.key-deserializer'] }}
20+
spring.kafka.external-clusters.{{ obj['name'] }}.consumer.value-deserializer={{ globals['spring.kafka.consumer.value-deserializer'] }}
21+
spring.kafka.external-clusters.{{ obj['name'] }}.producer.key-serializer={{ globals['spring.kafka.producer.key-deserialize'] }}
22+
spring.kafka.external-clusters.{{ obj['name'] }}.producer.value-serializer={{ globals['spring.kafka.producer.value-serializer'] }}
23+
{% endfor %}

‎flink-cyber/cyber-csd/src/main/descriptor/service.sdl‎

Lines changed: 523 additions & 48 deletions
Large diffs are not rendered by default.

‎flink-cyber/cyber-csd/src/main/scripts/control.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ case $CMD in
2525
get_generic_java_opts
2626
exec ${CYBERSEC_BIN}/cs-start-parser-ui start
2727
;;
28+
(start-worker)
29+
echo "Worker echo $CYBERSEC_BIN/cs-start-worker"
30+
get_generic_java_opts
31+
exec ${CYBERSEC_BIN}/cs-start-worker start
32+
;;
2833
(*)
2934
echo "Don't understand [$CMD]"
3035
;;

‎flink-cyber/cyber-jobs/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,12 @@
9191
<version>${project.parent.version}</version>
9292
<scope>provided</scope>
9393
</dependency>
94+
<dependency>
95+
<groupId>com.cloudera.cyber</groupId>
96+
<artifactId>cyber-worker-service</artifactId>
97+
<version>${project.parent.version}</version>
98+
<scope>provided</scope>
99+
</dependency>
94100
</dependencies>
95101

96102
<build>

‎flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml‎

Lines changed: 169 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -17,171 +17,173 @@
1717
-->
1818

1919
<assembly>
20-
<!-- Do not depend on the output of the bin.xml as assemblies are executed during the same maven
21-
lifecycle phase it is safer to create independently. This also enables us to filter unnecessary files.
22-
-->
23-
<id>cloudera</id>
24-
<formats>
25-
<format>tar.gz</format>
26-
</formats>
27-
28-
<includeBaseDirectory>true</includeBaseDirectory>
29-
<baseDirectory>${cloudera.tar.name}</baseDirectory>
30-
31-
<dependencySets>
32-
<dependencySet>
33-
<outputDirectory>lib</outputDirectory>
34-
<unpack>false</unpack>
35-
<useProjectArtifact>false</useProjectArtifact>
36-
<useProjectAttachments>false</useProjectAttachments>
37-
<useTransitiveDependencies>true</useTransitiveDependencies>
38-
<useTransitiveFiltering>true</useTransitiveFiltering>
39-
<includes>
40-
<include>org.slf4j:slf4j-api</include>
41-
<include>org.apache.logging.log4j:log4j-api</include>
42-
<include>org.apache.logging.log4j:log4j-core</include>
43-
<include>org.apache.logging.log4j:log4j-slf4j-impl</include>
44-
</includes>
45-
</dependencySet>
46-
</dependencySets>
47-
48-
<files>
49-
<file>
50-
<source>../caracal-generator/target/caracal-generator-${project.version}.jar</source>
51-
<outputDirectory>jobs/</outputDirectory>
52-
<destName>caracal-generator-${cybersec.full.version}.jar</destName>
53-
<fileMode>0644</fileMode>
54-
</file>
55-
56-
<file>
57-
<source>
58-
../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar
59-
</source>
60-
<outputDirectory>jobs/</outputDirectory>
61-
<destName>flink-enrichment-combined-${cybersec.full.version}.jar</destName>
62-
<fileMode>0644</fileMode>
63-
</file>
64-
65-
<file>
66-
<source>../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar
67-
</source>
68-
<outputDirectory>jobs/</outputDirectory>
69-
<destName>flink-enrichment-loading-${cybersec.full.version}.jar</destName>
70-
<fileMode>0644</fileMode>
71-
</file>
72-
73-
<file>
74-
<source>../flink-profiler-java/target/flink-profiler-java-${project.version}.jar</source>
75-
<outputDirectory>jobs/</outputDirectory>
76-
<destName>flink-profiler-java-${cybersec.full.version}.jar</destName>
77-
<fileMode>0644</fileMode>
78-
</file>
79-
<file>
80-
<source>../parser-chains-flink/target/parser-chains-flink-${project.version}.jar</source>
81-
<outputDirectory>jobs/</outputDirectory>
82-
<destName>parser-chains-flink-${cybersec.full.version}.jar</destName>
83-
<fileMode>0644</fileMode>
84-
</file>
85-
86-
<file>
87-
<source>../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar</source>
88-
<outputDirectory>jobs/</outputDirectory>
89-
<destName>flink-indexing-hive-${cybersec.full.version}.jar</destName>
90-
<fileMode>0644</fileMode>
91-
</file>
92-
93-
<file>
94-
<source>../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar</source>
95-
<outputDirectory>tools/</outputDirectory>
96-
<destName>scoring-commands-${cybersec.full.version}.jar</destName>
97-
<fileMode>0644</fileMode>
98-
</file>
99-
100-
<file>
101-
<source>../flink-commands/scoring-commands/README.md</source>
102-
<outputDirectory>tools/</outputDirectory>
103-
<destName>scoring-commands-README.md</destName>
104-
<fileMode>0644</fileMode>
105-
</file>
106-
107-
<file>
108-
<source>../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar</source>
109-
<outputDirectory>tools/</outputDirectory>
110-
<destName>kafka-commands-${cybersec.full.version}.jar</destName>
111-
<fileMode>0644</fileMode>
112-
</file>
113-
114-
<file>
115-
<source>../flink-commands/json-commands/target/json-commands-${project.version}.jar</source>
116-
<outputDirectory>tools/</outputDirectory>
117-
<destName>json-commands-${cybersec.full.version}.jar</destName>
118-
<fileMode>0644</fileMode>
119-
</file>
120-
121-
<file>
122-
<source>
123-
../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar
124-
</source>
125-
<outputDirectory>tools/</outputDirectory>
126-
<destName>parser-chains-config-service-${project.version}.jar</destName>
127-
<fileMode>0644</fileMode>
128-
</file>
129-
130-
</files>
131-
132-
<fileSets>
133-
<fileSet>
134-
<directory>src/main/resources/conf</directory>
135-
<outputDirectory>conf</outputDirectory>
136-
<fileMode>0644</fileMode>
137-
<excludes>
138-
<exclude>*.json</exclude>
139-
<exclude>log4j.properties</exclude>
140-
</excludes>
141-
</fileSet>
142-
<fileSet>
143-
<directory>src/main/resources/conf/templates</directory>
144-
<outputDirectory>conf/templates</outputDirectory>
145-
<fileMode>0644</fileMode>
146-
</fileSet>
147-
<fileSet>
148-
<directory>src/main/resources/conf/templates/generate</directory>
149-
<outputDirectory>conf/templates/generate</outputDirectory>
150-
<fileMode>0644</fileMode>
151-
</fileSet>
152-
<fileSet>
153-
<directory>src/main/resources/conf/templates/index</directory>
154-
<outputDirectory>conf/templates/index</outputDirectory>
155-
<fileMode>0644</fileMode>
156-
</fileSet>
157-
<fileSet>
158-
<directory>src/main/resources/conf/templates/parse</directory>
159-
<outputDirectory>conf/templates/parse</outputDirectory>
160-
<fileMode>0644</fileMode>
161-
</fileSet>
162-
<fileSet>
163-
<directory>src/main/resources/conf/templates/profile</directory>
164-
<outputDirectory>conf/templates/profile</outputDirectory>
165-
<fileMode>0644</fileMode>
166-
</fileSet>
167-
<fileSet>
168-
<directory>src/main/resources/conf/templates/triage</directory>
169-
<outputDirectory>conf/templates/triage</outputDirectory>
170-
<fileMode>0644</fileMode>
171-
</fileSet>
172-
<fileSet>
173-
<directory>target/classes/scripts</directory>
174-
<outputDirectory>bin</outputDirectory>
175-
<fileMode>0755</fileMode>
176-
</fileSet>
177-
<fileSet>
178-
<directory>src/main/resources/conf</directory>
179-
<outputDirectory>tools</outputDirectory>
180-
<fileMode>0644</fileMode>
181-
<includes>
182-
<include>sample-rule.json</include>
183-
<include>log4j.properties</include>
184-
</includes>
185-
</fileSet>
186-
</fileSets>
20+
<!-- Do not depend on the output of the bin.xml as assemblies are executed during the same maven
21+
lifecycle phase it is safer to create independently. This also enables us to filter unnecessary files.
22+
-->
23+
<id>cloudera</id>
24+
<formats>
25+
<format>tar.gz</format>
26+
</formats>
27+
28+
<includeBaseDirectory>true</includeBaseDirectory>
29+
<baseDirectory>${cloudera.tar.name}</baseDirectory>
30+
31+
<dependencySets>
32+
<dependencySet>
33+
<outputDirectory>lib</outputDirectory>
34+
<unpack>false</unpack>
35+
<useProjectArtifact>false</useProjectArtifact>
36+
<useProjectAttachments>false</useProjectAttachments>
37+
<useTransitiveDependencies>true</useTransitiveDependencies>
38+
<useTransitiveFiltering>true</useTransitiveFiltering>
39+
<includes>
40+
<include>org.slf4j:slf4j-api</include>
41+
<include>org.apache.logging.log4j:log4j-api</include>
42+
<include>org.apache.logging.log4j:log4j-core</include>
43+
<include>org.apache.logging.log4j:log4j-slf4j-impl</include>
44+
</includes>
45+
</dependencySet>
46+
</dependencySets>
47+
48+
<files>
49+
<file>
50+
<source>../caracal-generator/target/caracal-generator-${project.version}.jar</source>
51+
<outputDirectory>jobs/</outputDirectory>
52+
<destName>caracal-generator-${cybersec.full.version}.jar</destName>
53+
<fileMode>0644</fileMode>
54+
</file>
55+
56+
<file>
57+
<source>../flink-enrichment/flink-enrichment-combined/target/flink-enrichment-combined-${project.version}.jar</source>
58+
<outputDirectory>jobs/</outputDirectory>
59+
<destName>flink-enrichment-combined-${cybersec.full.version}.jar</destName>
60+
<fileMode>0644</fileMode>
61+
</file>
62+
63+
<file>
64+
<source>../flink-enrichment/flink-enrichment-load/target/flink-enrichment-loading-${project.version}.jar</source>
65+
<outputDirectory>jobs/</outputDirectory>
66+
<destName>flink-enrichment-loading-${cybersec.full.version}.jar</destName>
67+
<fileMode>0644</fileMode>
68+
</file>
69+
70+
<file>
71+
<source>../flink-profiler-java/target/flink-profiler-java-${project.version}.jar</source>
72+
<outputDirectory>jobs/</outputDirectory>
73+
<destName>flink-profiler-java-${cybersec.full.version}.jar</destName>
74+
<fileMode>0644</fileMode>
75+
</file>
76+
<file>
77+
<source>../parser-chains-flink/target/parser-chains-flink-${project.version}.jar</source>
78+
<outputDirectory>jobs/</outputDirectory>
79+
<destName>parser-chains-flink-${cybersec.full.version}.jar</destName>
80+
<fileMode>0644</fileMode>
81+
</file>
82+
83+
<file>
84+
<source>../flink-indexing/flink-indexing-hive/target/flink-indexing-hive-${project.version}.jar</source>
85+
<outputDirectory>jobs/</outputDirectory>
86+
<destName>flink-indexing-hive-${cybersec.full.version}.jar</destName>
87+
<fileMode>0644</fileMode>
88+
</file>
89+
90+
<file>
91+
<source>../cyber-services/cyber-worker-service/target/cyber-worker-service-${project.version}.jar</source>
92+
<outputDirectory>tools/</outputDirectory>
93+
<destName>cyber-worker-service-${cybersec.full.version}.jar</destName>
94+
<fileMode>0644</fileMode>
95+
</file>
96+
97+
<file>
98+
<source>../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar</source>
99+
<outputDirectory>tools/</outputDirectory>
100+
<destName>scoring-commands-${cybersec.full.version}.jar</destName>
101+
<fileMode>0644</fileMode>
102+
</file>
103+
104+
<file>
105+
<source>../flink-commands/scoring-commands/README.md</source>
106+
<outputDirectory>tools/</outputDirectory>
107+
<destName>scoring-commands-README.md</destName>
108+
<fileMode>0644</fileMode>
109+
</file>
110+
111+
<file>
112+
<source>../flink-commands/kafka-commands/target/kafka-commands-${project.version}.jar</source>
113+
<outputDirectory>tools/</outputDirectory>
114+
<destName>kafka-commands-${cybersec.full.version}.jar</destName>
115+
<fileMode>0644</fileMode>
116+
</file>
117+
118+
<file>
119+
<source>../flink-commands/json-commands/target/json-commands-${project.version}.jar</source>
120+
<outputDirectory>tools/</outputDirectory>
121+
<destName>json-commands-${cybersec.full.version}.jar</destName>
122+
<fileMode>0644</fileMode>
123+
</file>
124+
125+
<file>
126+
<source>../metron-parser-chain/parser-chains-config-service/target/parser-chains-config-service-${project.version}.jar</source>
127+
<outputDirectory>tools/</outputDirectory>
128+
<destName>parser-chains-config-service-${project.version}.jar</destName>
129+
<fileMode>0644</fileMode>
130+
</file>
131+
132+
</files>
133+
134+
<fileSets>
135+
<fileSet>
136+
<directory>src/main/resources/conf</directory>
137+
<outputDirectory>conf</outputDirectory>
138+
<fileMode>0644</fileMode>
139+
<excludes>
140+
<exclude>*.json</exclude>
141+
<exclude>log4j.properties</exclude>
142+
</excludes>
143+
</fileSet>
144+
<fileSet>
145+
<directory>src/main/resources/conf/templates</directory>
146+
<outputDirectory>conf/templates</outputDirectory>
147+
<fileMode>0644</fileMode>
148+
</fileSet>
149+
<fileSet>
150+
<directory>src/main/resources/conf/templates/generate</directory>
151+
<outputDirectory>conf/templates/generate</outputDirectory>
152+
<fileMode>0644</fileMode>
153+
</fileSet>
154+
<fileSet>
155+
<directory>src/main/resources/conf/templates/index</directory>
156+
<outputDirectory>conf/templates/index</outputDirectory>
157+
<fileMode>0644</fileMode>
158+
</fileSet>
159+
<fileSet>
160+
<directory>src/main/resources/conf/templates/parse</directory>
161+
<outputDirectory>conf/templates/parse</outputDirectory>
162+
<fileMode>0644</fileMode>
163+
</fileSet>
164+
<fileSet>
165+
<directory>src/main/resources/conf/templates/profile</directory>
166+
<outputDirectory>conf/templates/profile</outputDirectory>
167+
<fileMode>0644</fileMode>
168+
</fileSet>
169+
<fileSet>
170+
<directory>src/main/resources/conf/templates/triage</directory>
171+
<outputDirectory>conf/templates/triage</outputDirectory>
172+
<fileMode>0644</fileMode>
173+
</fileSet>
174+
<fileSet>
175+
<directory>target/classes/scripts</directory>
176+
<outputDirectory>bin</outputDirectory>
177+
<fileMode>0755</fileMode>
178+
</fileSet>
179+
<fileSet>
180+
<directory>src/main/resources/conf</directory>
181+
<outputDirectory>tools</outputDirectory>
182+
<fileMode>0644</fileMode>
183+
<includes>
184+
<include>sample-rule.json</include>
185+
<include>log4j.properties</include>
186+
</includes>
187+
</fileSet>
188+
</fileSets>
187189
</assembly>

0 commit comments

Comments
 (0)