Skip to content

Commit 1fb3673

Browse files
author
Sumedh Wale
committed
[SNAPPYDATA] updated dependencies to address major CVEs
- also exclude log4j2 from yarn shuffle shadow jar - fixed occasional failures in kafka suite
1 parent d2ab68a commit 1fb3673

File tree

7 files changed

+36
-28
lines changed

7 files changed

+36
-28
lines changed

‎build.gradle

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,30 +60,29 @@ allprojects {
6060
scalaBinaryVersion = '2.11'
6161
scalaVersion = scalaBinaryVersion + '.8'
6262
hadoopVersion = '3.2.0'
63-
protobufVersion = '3.6.1'
64-
jerseyVersion = '2.22.2'
63+
protobufVersion = '3.16.1'
64+
jerseyVersion = '2.35'
6565
sunJerseyVersion = '1.19.4'
6666
jettyVersion = '9.4.43.v20210629'
6767
yarnTestJettyVersion = '9.3.24.v20180605' // only used by yarn tests
6868
jettyOldVersion = '6.1.26'
6969
log4j2Version = '2.17.2'
7070
slf4jVersion = '1.7.32'
71-
junitVersion = '4.12'
71+
junitVersion = '4.13.2'
7272
mockitoVersion = '1.10.19'
7373
javaxServletVersion = '3.1.0'
7474
guavaVersion = '14.0.1'
7575
hiveVersion = '1.21.2.7.0.3.2-3'
7676
chillVersion = '0.8.5'
7777
kryoVersion = '4.0.2'
78-
nettyVersion = '3.10.6.Final'
79-
nettyAllVersion = '4.1.73.Final'
78+
nettyVersion = '4.1.77.Final'
8079
derbyVersion = '10.14.2.0'
81-
httpClientVersion = '4.5.6'
80+
httpClientVersion = '4.5.13'
8281
httpCoreVersion = '4.4.10'
8382
levelDbJniVersion = '1.8'
8483
jackson1Version = '1.9.14-atlassian-6'
85-
jacksonVersion = '2.13.1'
86-
jacksonDatabindVersion = '2.13.1'
84+
jacksonVersion = '2.13.3'
85+
jacksonDatabindVersion = '2.13.3'
8786
snappyJavaVersion = '1.1.8.4'
8887
lz4Version = '1.8.0'
8988
lzfVersion = '1.1'
@@ -92,11 +91,13 @@ allprojects {
9291
metricsVersion = '3.2.6'
9392
janinoVersion = '3.0.8'
9493
thriftVersion = '0.9.3'
94+
fb303Version = '0.9.3'
9595
antlrVersion = '4.5.3'
9696
jpamVersion = '1.1'
9797
seleniumVersion = '2.52.0'
9898
curatorVersion = '2.7.1'
99-
commonsCodecVersion = '1.11'
99+
commonsCodecVersion = '1.15'
100+
commonsCompressVersion = '1.21'
100101
commonsCryptoVersion = '1.0.0'
101102
commonsLang3Version = '3.8.1'
102103
commonsMath3Version = '3.6.1'
@@ -120,7 +121,7 @@ allprojects {
120121
dockerClientVersion = '8.14.5'
121122
mysqlVersion = '8.0.13'
122123
postgresqlVersion = '42.2.5'
123-
ojdbc6Version = '11.2.0.4'
124+
ojdbc8Version = '12.2.0.1'
124125
zookeeperVersion = '3.6.2'
125126
activationVersion = '1.1.1'
126127
roaringBitmapVersion = '0.6.66'
@@ -181,7 +182,9 @@ allprojects {
181182
exclude(group: 'org.apache.ant', module: 'ant')
182183
exclude(group: 'com.esotericsoftware.kryo', module: 'kryo')
183184
exclude(group: 'commons-codec', module: 'commons-codec')
185+
exclude(group: 'org.apache.commons', module: 'commons-compress')
184186
exclude(group: 'commons-httpclient', module: 'commons-httpclient')
187+
exclude(group: 'org.apache.avro', module: 'avro')
185188
exclude(group: 'org.apache.avro', module: 'avro-mapred')
186189
exclude(group: 'org.apache.calcite', module: 'calcite-core')
187190
exclude(group: 'org.apache.curator', module: 'apache-curator')
@@ -232,7 +235,7 @@ allprojects {
232235
exclude(group: 'org.apache.velocity', module: 'velocity')
233236
exclude(group: 'org.apache.avro', module: 'avro-ipc')
234237
},
235-
netty_all: dependencies.create("io.netty:netty-all:${nettyAllVersion}") {
238+
netty_all: dependencies.create("io.netty:netty-all:${nettyVersion}") {
236239
exclude(group: 'io.netty', module: 'netty-codec-dns')
237240
exclude(group: 'io.netty', module: 'netty-codec-haproxy')
238241
exclude(group: 'io.netty', module: 'netty-codec-http')

‎common/network-yarn/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ shadowJar {
4141
exclude(dependency('org.scala-lang:.*'))
4242
exclude(dependency('org.scala-lang.modules:.*'))
4343
exclude(dependency('org.slf4j:.*'))
44-
exclude(dependency('log4j:.*'))
44+
exclude(dependency('org.apache.logging.log4j:.*'))
4545
exclude(dependency('org.scalatest:.*'))
4646
}
4747
//configurations = [ project.configurations.runtimeJar ]

‎core/build.gradle

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ dependencies {
2424
compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion)
2525
compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion)
2626

27+
compile group: 'org.apache.commons', name: 'commons-compress', version: commonsCompressVersion
2728
compile libraries.avro_ipc
2829
compile libraries.avro_mapred
2930
compile group: 'com.google.guava', name: 'guava', version: guavaVersion
@@ -35,14 +36,11 @@ dependencies {
3536
exclude(group: 'com.esotericsoftware', module: 'kryo-shaded')
3637
}
3738
compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version
38-
// explicitly include netty from akka-remote to not let zookeeper override it
39-
compile(group: 'io.netty', name: 'netty', version: nettyVersion) {
40-
exclude(group: 'log4j', module: 'log4j')
41-
}
4239
// explicitly exclude old netty from zookeeper
4340
compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion) {
4441
exclude(group: 'org.jboss.netty', module: 'netty')
4542
exclude(group: 'jline', module: 'jline')
43+
exclude(group: 'io.netty', module: 'netty')
4644
exclude(group: 'io.netty', module: 'netty-handler')
4745
exclude(group: 'io.netty', module: 'netty-transport-native-epoll')
4846
exclude(group: 'com.github.spotbugs', module: 'spotbugs-annotations')
@@ -93,10 +91,15 @@ dependencies {
9391
compile group: 'commons-net', name: 'commons-net', version: commonsNetVersion
9492
compile group: 'org.json4s', name: 'json4s-jackson_' + scalaBinaryVersion, version: json4sVersion
9593
compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: jerseyVersion
96-
compile group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion
97-
compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion
94+
compile(group: 'org.glassfish.jersey.core', name: 'jersey-common', version: jerseyVersion) {
95+
exclude(group: 'com.sun.activation', module: 'jakarta.activation')
96+
}
97+
compile(group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion) {
98+
exclude(group: 'jakarta.xml.bind', module: 'jakarta.xml.bind-api')
99+
}
98100
compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion
99101
compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion
102+
compile group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: jerseyVersion
100103
compile(group: 'com.clearspring.analytics', name: 'stream', version: streamVersion) {
101104
exclude(group: 'it.unimi.dsi', module: 'fastutil')
102105
}
@@ -137,6 +140,7 @@ dependencies {
137140
exclude(group: 'com.google.guava', module: 'guava')
138141
exclude(group: 'io.netty', module: 'netty')
139142
}
143+
testCompile group: 'org.glassfish.jersey.test-framework.providers', name: 'jersey-test-framework-provider-simple', version: jerseyVersion
140144
testCompile(group: 'org.seleniumhq.selenium', name: 'selenium-htmlunit-driver', version: seleniumVersion) {
141145
exclude(group: 'com.google.guava', module: 'guava')
142146
}

‎external/docker-integration-tests/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ dependencies {
3737
testCompile group: 'org.apache.httpcomponents', name: 'httpcore', version: httpCoreVersion
3838
testCompile group: 'mysql', name: 'mysql-connector-java', version: mysqlVersion
3939
testCompile group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion
40-
testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc6', version: ojdbc6Version
40+
testCompile group: 'com.oracle.database.jdbc', name: 'ojdbc8', version: ojdbc8Version
4141
testCompile group: 'com.sun.jersey', name: 'jersey-server', version: sunJerseyVersion
4242
testCompile group: 'com.sun.jersey', name: 'jersey-core', version: sunJerseyVersion
4343
testCompile group: 'com.sun.jersey', name: 'jersey-servlet', version: sunJerseyVersion

‎external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceSuite.scala

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ class KafkaSourceSuite extends KafkaSourceTest {
432432
.format("kafka")
433433
.option("kafka.bootstrap.servers", testUtils.brokerAddress)
434434
.option("kafka.metadata.max.age.ms", "1")
435-
.option("kafka.default.api.timeout.ms", "3000")
435+
.option("kafka.default.api.timeout.ms", "10000")
436436
.option("subscribePattern", s"$topicPrefix-.*")
437437
.option("failOnDataLoss", "false")
438438

@@ -577,16 +577,16 @@ class KafkaSourceSuite extends KafkaSourceTest {
577577
.readStream
578578
.format("kafka")
579579
.option("kafka.bootstrap.servers", testUtils.brokerAddress)
580-
.option("kafka.default.api.timeout.ms", "3000")
580+
.option("kafka.default.api.timeout.ms", "10000")
581581
.option("kafka.metadata.max.age.ms", "1")
582582
.option("subscribe", topic)
583583
// If a topic is deleted and we try to poll data starting from offset 0,
584584
// the Kafka consumer will just block until timeout and return an empty result.
585585
// So set the timeout to 1 second to make this test fast.
586-
.option("kafkaConsumer.pollTimeoutMs", "1000")
586+
.option("kafkaConsumer.pollTimeoutMs", "3000")
587587
.option("startingOffsets", "earliest")
588588
.option("failOnDataLoss", "false")
589-
.option("kafka.default.api.timeout.ms", "3000")
589+
.option("kafka.default.api.timeout.ms", "10000")
590590
val kafka = reader.load()
591591
.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
592592
.as[(String, String)]
@@ -888,7 +888,7 @@ class KafkaSourceStressSuite extends KafkaSourceTest {
888888
.option("kafka.metadata.max.age.ms", "1")
889889
.option("subscribePattern", "stress.*")
890890
.option("failOnDataLoss", "false")
891-
.option("kafka.default.api.timeout.ms", "3000")
891+
.option("kafka.default.api.timeout.ms", "10000")
892892
.load()
893893
.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
894894
.as[(String, String)]
@@ -985,11 +985,11 @@ class KafkaSourceStressForDontFailOnDataLossSuite extends StreamTest with Shared
985985
.format("kafka")
986986
.option("kafka.bootstrap.servers", testUtils.brokerAddress)
987987
.option("kafka.metadata.max.age.ms", "1")
988-
.option("kafka.default.api.timeout.ms", "3000")
988+
.option("kafka.default.api.timeout.ms", "10000")
989989
.option("subscribePattern", "failOnDataLoss.*")
990990
.option("startingOffsets", "earliest")
991991
.option("failOnDataLoss", "false")
992-
.option("fetchOffset.retryIntervalMs", "3000")
992+
.option("fetchOffset.retryIntervalMs", "10000")
993993
val kafka = reader.load()
994994
.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
995995
.as[(String, String)]

‎sql/hive/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ dependencies {
5757
compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) {
5858
exclude(group: 'org.slf4j', module: 'slf4j-api')
5959
}
60-
compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) {
60+
compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) {
6161
exclude(group: 'org.slf4j', module: 'slf4j-api')
6262
}
6363

‎yarn/build.gradle

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ dependencies {
3535
exclude(group: 'org.ow2.asm', module: 'asm')
3636
exclude(group: 'org.jboss.netty', module: 'netty')
3737
exclude(group: 'javax.servlet', module: 'servlet-api')
38+
exclude(group: 'org.apache.commons', module: 'commons-compress')
3839
exclude(group: 'commons-logging', module: 'commons-logging')
3940
exclude(group: 'com.sun.jersey')
4041
exclude(group: 'com.sun.jersey.jersey-test-framework')
@@ -78,7 +79,7 @@ dependencies {
7879
compile(group: 'org.apache.thrift', name: 'libthrift', version: thriftVersion) {
7980
exclude(group: 'org.slf4j', module: 'slf4j-api')
8081
}
81-
compile(group: 'org.apache.thrift', name: 'libfb303', version: thriftVersion) {
82+
compile(group: 'org.apache.thrift', name: 'libfb303', version: fb303Version) {
8283
exclude(group: 'org.slf4j', module: 'slf4j-api')
8384
}
8485

0 commit comments

Comments
 (0)