Skip to content

Commit ab1650d

Browse files
Fokko DriesprongHyukjinKwon
Fokko Driesprong
authored and
HyukjinKwon
committed
[SPARK-24601] Update Jackson to 2.9.6
Hi all, Jackson is incompatible with upstream versions, therefore bump the Jackson version to a more recent one. I bumped into some issues with Azure CosmosDB that is using a more recent version of Jackson. This can be fixed by adding exclusions and then it works without any issues. So no breaking changes in the API's. I would also consider bumping the version of Jackson in Spark. I would suggest to keep up to date with the dependencies, since in the future this issue will pop up more frequently. ## What changes were proposed in this pull request? Bump Jackson to 2.9.6 ## How was this patch tested? Compiled and tested it locally to see if anything broke. Please review http://spark.apache.org/contributing.html before opening a pull request. Closes apache#21596 from Fokko/fd-bump-jackson. Authored-by: Fokko Driesprong <[email protected]> Signed-off-by: hyukjinkwon <[email protected]>
1 parent 4597007 commit ab1650d

File tree

11 files changed

+59
-47
lines changed

11 files changed

+59
-47
lines changed

‎core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import org.apache.spark.util.Utils
3636
* (2) the Spark version of the client / server
3737
* (3) an optional message
3838
*/
39-
@JsonInclude(Include.NON_NULL)
39+
@JsonInclude(Include.NON_ABSENT)
4040
@JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY)
4141
@JsonPropertyOrder(alphabetic = true)
4242
private[rest] abstract class SubmitRestProtocolMessage {

‎core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging
4141
* There is no particular relationship between an operation scope and a stage or a job.
4242
* A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take).
4343
*/
44-
@JsonInclude(Include.NON_NULL)
44+
@JsonInclude(Include.NON_ABSENT)
4545
@JsonPropertyOrder(Array("id", "name", "parent"))
4646
private[spark] class RDDOperationScope(
4747
val name: String,

‎core/src/main/scala/org/apache/spark/status/KVUtils.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging {
4242
private[spark] class KVStoreScalaSerializer extends KVStoreSerializer {
4343

4444
mapper.registerModule(DefaultScalaModule)
45-
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
45+
mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
4646

4747
}
4848

‎core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{
4949
}
5050
mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
5151
mapper.enable(SerializationFeature.INDENT_OUTPUT)
52-
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
52+
mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
5353
mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)
5454

5555
override def isWriteable(

‎core/src/main/scala/org/apache/spark/status/api/v1/api.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@ private[spark] class ExecutorMetricsJsonSerializer
139139
jsonGenerator.writeObject(metricsMap)
140140
}
141141
}
142+
143+
override def isEmpty(provider: SerializerProvider, value: Option[ExecutorMetrics]): Boolean =
144+
value.isEmpty
142145
}
143146

144147
class JobData private[spark](

‎dev/deps/spark-deps-hadoop-2.6

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -87,16 +87,16 @@ htrace-core-3.0.4.jar
8787
httpclient-4.5.6.jar
8888
httpcore-4.4.10.jar
8989
ivy-2.4.0.jar
90-
jackson-annotations-2.6.7.jar
91-
jackson-core-2.6.7.jar
90+
jackson-annotations-2.9.6.jar
91+
jackson-core-2.9.6.jar
9292
jackson-core-asl-1.9.13.jar
93-
jackson-databind-2.6.7.1.jar
94-
jackson-dataformat-yaml-2.6.7.jar
93+
jackson-databind-2.9.6.jar
94+
jackson-dataformat-yaml-2.9.6.jar
9595
jackson-jaxrs-1.9.13.jar
9696
jackson-mapper-asl-1.9.13.jar
97-
jackson-module-jaxb-annotations-2.6.7.jar
98-
jackson-module-paranamer-2.7.9.jar
99-
jackson-module-scala_2.11-2.6.7.1.jar
97+
jackson-module-jaxb-annotations-2.9.6.jar
98+
jackson-module-paranamer-2.9.6.jar
99+
jackson-module-scala_2.11-2.9.6.jar
100100
jackson-xc-1.9.13.jar
101101
janino-3.0.10.jar
102102
javassist-3.18.1-GA.jar
@@ -177,7 +177,7 @@ scala-xml_2.11-1.0.5.jar
177177
shapeless_2.11-2.3.2.jar
178178
slf4j-api-1.7.16.jar
179179
slf4j-log4j12-1.7.16.jar
180-
snakeyaml-1.15.jar
180+
snakeyaml-1.18.jar
181181
snappy-0.2.jar
182182
snappy-java-1.1.7.1.jar
183183
spire-macros_2.11-0.13.0.jar

‎dev/deps/spark-deps-hadoop-2.7

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -87,16 +87,16 @@ htrace-core-3.1.0-incubating.jar
8787
httpclient-4.5.6.jar
8888
httpcore-4.4.10.jar
8989
ivy-2.4.0.jar
90-
jackson-annotations-2.6.7.jar
91-
jackson-core-2.6.7.jar
90+
jackson-annotations-2.9.6.jar
91+
jackson-core-2.9.6.jar
9292
jackson-core-asl-1.9.13.jar
93-
jackson-databind-2.6.7.1.jar
94-
jackson-dataformat-yaml-2.6.7.jar
93+
jackson-databind-2.9.6.jar
94+
jackson-dataformat-yaml-2.9.6.jar
9595
jackson-jaxrs-1.9.13.jar
9696
jackson-mapper-asl-1.9.13.jar
97-
jackson-module-jaxb-annotations-2.6.7.jar
98-
jackson-module-paranamer-2.7.9.jar
99-
jackson-module-scala_2.11-2.6.7.1.jar
97+
jackson-module-jaxb-annotations-2.9.6.jar
98+
jackson-module-paranamer-2.9.6.jar
99+
jackson-module-scala_2.11-2.9.6.jar
100100
jackson-xc-1.9.13.jar
101101
janino-3.0.10.jar
102102
javassist-3.18.1-GA.jar
@@ -178,7 +178,7 @@ scala-xml_2.11-1.0.5.jar
178178
shapeless_2.11-2.3.2.jar
179179
slf4j-api-1.7.16.jar
180180
slf4j-log4j12-1.7.16.jar
181-
snakeyaml-1.15.jar
181+
snakeyaml-1.18.jar
182182
snappy-0.2.jar
183183
snappy-java-1.1.7.1.jar
184184
spire-macros_2.11-0.13.0.jar

‎dev/deps/spark-deps-hadoop-3.1

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,17 +86,17 @@ htrace-core4-4.1.0-incubating.jar
8686
httpclient-4.5.6.jar
8787
httpcore-4.4.10.jar
8888
ivy-2.4.0.jar
89-
jackson-annotations-2.6.7.jar
90-
jackson-core-2.6.7.jar
89+
jackson-annotations-2.9.6.jar
90+
jackson-core-2.9.6.jar
9191
jackson-core-asl-1.9.13.jar
92-
jackson-databind-2.6.7.1.jar
93-
jackson-dataformat-yaml-2.6.7.jar
92+
jackson-databind-2.9.6.jar
93+
jackson-dataformat-yaml-2.9.6.jar
9494
jackson-jaxrs-base-2.7.8.jar
9595
jackson-jaxrs-json-provider-2.7.8.jar
9696
jackson-mapper-asl-1.9.13.jar
97-
jackson-module-jaxb-annotations-2.6.7.jar
98-
jackson-module-paranamer-2.7.9.jar
99-
jackson-module-scala_2.11-2.6.7.1.jar
97+
jackson-module-jaxb-annotations-2.9.6.jar
98+
jackson-module-paranamer-2.9.6.jar
99+
jackson-module-scala_2.11-2.9.6.jar
100100
janino-3.0.10.jar
101101
javassist-3.18.1-GA.jar
102102
javax.annotation-api-1.2.jar
@@ -197,7 +197,7 @@ scala-xml_2.11-1.0.5.jar
197197
shapeless_2.11-2.3.2.jar
198198
slf4j-api-1.7.16.jar
199199
slf4j-log4j12-1.7.16.jar
200-
snakeyaml-1.15.jar
200+
snakeyaml-1.18.jar
201201
snappy-0.2.jar
202202
snappy-java-1.1.7.1.jar
203203
spire-macros_2.11-0.13.0.jar

‎pom.xml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,8 +158,7 @@
158158
<scala.version>2.11.12</scala.version>
159159
<scala.binary.version>2.11</scala.binary.version>
160160
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
161-
<fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
162-
<fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version>
161+
<fasterxml.jackson.version>2.9.6</fasterxml.jackson.version>
163162
<snappy.version>1.1.7.1</snappy.version>
164163
<netlib.java.version>1.1.2</netlib.java.version>
165164
<calcite.version>1.2.0-incubating</calcite.version>
@@ -629,7 +628,7 @@
629628
<dependency>
630629
<groupId>com.fasterxml.jackson.core</groupId>
631630
<artifactId>jackson-databind</artifactId>
632-
<version>${fasterxml.jackson.databind.version}</version>
631+
<version>${fasterxml.jackson.version}</version>
633632
</dependency>
634633
<dependency>
635634
<groupId>com.fasterxml.jackson.core</groupId>
@@ -641,7 +640,7 @@
641640
<dependency>
642641
<groupId>com.fasterxml.jackson.module</groupId>
643642
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
644-
<version>${fasterxml.jackson.databind.version}</version>
643+
<version>${fasterxml.jackson.version}</version>
645644
<exclusions>
646645
<exclusion>
647646
<groupId>com.google.guava</groupId>

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,13 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
244244
"1234")
245245
}
246246

247+
test("some big value") {
248+
val value = "x" * 3000
249+
checkEvaluation(
250+
GetJsonObject(NonFoldableLiteral((s"""{"big": "$value"}""")),
251+
NonFoldableLiteral("$.big")), value)
252+
}
253+
247254
val jsonTupleQuery = Literal("f1") ::
248255
Literal("f2") ::
249256
Literal("f3") ::

0 commit comments

Comments
 (0)