Skip to content

Commit 25ac793

Browse files
garyrussellartembilan
garyrussell
authored and
artembilan
committed
Private Header Type for DeserializationExceptions
Use a package-private header for deserialization exceptions. **cherry-pick to 2.9.x** # Conflicts: # spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java # spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerUtils.java # spring-kafka/src/test/java/org/springframework/kafka/listener/DeadLetterPublishingRecovererTests.java # spring-kafka/src/test/java/org/springframework/kafka/listener/ErrorHandlingDeserializerTests.java
1 parent d370d2d commit 25ac793

File tree

12 files changed

+268
-46
lines changed

12 files changed

+268
-46
lines changed

‎spring-kafka-docs/src/main/asciidoc/kafka.adoc‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4655,10 +4655,15 @@ void listen(List<Thing> in, @Header(KafkaHeaders.BATCH_CONVERTED_HEADERS) List<M
46554655
Thing thing = in.get(i);
46564656
if (thing == null
46574657
&& headers.get(i).get(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER) != null) {
4658-
DeserializationException deserEx = ListenerUtils.byteArrayToDeserializationException(this.logger,
4659-
(byte[]) headers.get(i).get(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER));
4660-
if (deserEx != null) {
4661-
logger.error(deserEx, "Record at index " + i + " could not be deserialized");
4658+
try {
4659+
DeserializationException deserEx = SerializationUtils.byteArrayToDeserializationException(this.logger,
4660+
headers.get(i).get(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER));
4661+
if (deserEx != null) {
4662+
logger.error(deserEx, "Record at index " + i + " could not be deserialized");
4663+
}
4664+
}
4665+
catch (Exception ex) {
4666+
logger.error(ex, "Record at index " + i + " could not be deserialized");
46624667
}
46634668
throw new BatchListenerFailedException("Deserialization", deserEx, i);
46644669
}
@@ -4668,9 +4673,9 @@ void listen(List<Thing> in, @Header(KafkaHeaders.BATCH_CONVERTED_HEADERS) List<M
46684673
----
46694674
====
46704675

4671-
`ListenerUtils.byteArrayToDeserializationException()` can be used to convert the header to a `DeserializationException`.
4676+
`SerializationUtils.byteArrayToDeserializationException()` can be used to convert the header to a `DeserializationException`.
46724677

4673-
When consuming `List<ConsumerRecord<?, ?>`, `ListenerUtils.getExceptionFromHeader()` is used instead:
4678+
When consuming `List<ConsumerRecord<?, ?>`, `SerializationUtils.getExceptionFromHeader()` is used instead:
46744679

46754680
====
46764681
[source, java]
@@ -4680,7 +4685,7 @@ void listen(List<ConsumerRecord<String, Thing>> in) {
46804685
for (int i = 0; i < in.size(); i++) {
46814686
ConsumerRecord<String, Thing> rec = in.get(i);
46824687
if (rec.value() == null) {
4683-
DeserializationException deserEx = ListenerUtils.getExceptionFromHeader(rec,
4688+
DeserializationException deserEx = SerializationUtils.getExceptionFromHeader(rec,
46844689
SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER, this.logger);
46854690
if (deserEx != null) {
46864691
logger.error(deserEx, "Record at offset " + rec.offset() + " could not be deserialized");

‎spring-kafka/src/main/java/org/springframework/kafka/listener/DeadLetterPublishingRecoverer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -486,9 +486,9 @@ public void accept(ConsumerRecord<?, ?> record, @Nullable Consumer<?, ?> consume
486486
if (consumer != null && this.verifyPartition) {
487487
tp = checkPartition(tp, consumer);
488488
}
489-
DeserializationException vDeserEx = ListenerUtils.getExceptionFromHeader(record,
489+
DeserializationException vDeserEx = SerializationUtils.getExceptionFromHeader(record,
490490
SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER, this.logger);
491-
DeserializationException kDeserEx = ListenerUtils.getExceptionFromHeader(record,
491+
DeserializationException kDeserEx = SerializationUtils.getExceptionFromHeader(record,
492492
SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, this.logger);
493493
Headers headers = new RecordHeaders(record.headers().toArray());
494494
addAndEnhanceHeaders(record, exception, vDeserEx, kDeserEx, headers);

‎spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2929,8 +2929,9 @@ private void fixStackTrace(Exception ex, Exception toHandle) {
29292929
}
29302930
}
29312931

2932-
public void checkDeser(final ConsumerRecord<K, V> record, String headerName) {
2933-
DeserializationException exception = ListenerUtils.getExceptionFromHeader(record, headerName, this.logger);
2932+
public void checkDeser(final ConsumerRecord<K, V> cRecord, String headerName) {
2933+
DeserializationException exception = SerializationUtils.getExceptionFromHeader(cRecord, headerName,
2934+
this.logger);
29342935
if (exception != null) {
29352936
/*
29362937
* Wrapping in a LEFE is not strictly correct, but required for backwards compatibility.

‎spring-kafka/src/main/java/org/springframework/kafka/listener/ListenerUtils.java‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,11 @@
2626
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
2727
import org.apache.kafka.common.Metric;
2828
import org.apache.kafka.common.MetricName;
29-
import org.apache.kafka.common.header.Header;
30-
import org.apache.kafka.common.header.Headers;
31-
import org.apache.kafka.common.header.internals.RecordHeaders;
3229

3330
import org.springframework.core.log.LogAccessor;
3431
import org.springframework.kafka.support.KafkaUtils;
3532
import org.springframework.kafka.support.serializer.DeserializationException;
33+
import org.springframework.kafka.support.serializer.SerializationUtils;
3634
import org.springframework.lang.Nullable;
3735
import org.springframework.util.Assert;
3836
import org.springframework.util.backoff.BackOff;
@@ -96,23 +94,15 @@ else if (listener instanceof GenericMessageListener) {
9694
* @param logger the logger for logging errors.
9795
* @return the exception or null.
9896
* @since 2.3
97+
* @deprecated in favor of
98+
* {@link SerializationUtils#getExceptionFromHeader(ConsumerRecord, String, LogAccessor)}.
9999
*/
100+
@Deprecated
100101
@Nullable
101102
public static DeserializationException getExceptionFromHeader(final ConsumerRecord<?, ?> record,
102103
String headerName, LogAccessor logger) {
103104

104-
Header header = record.headers().lastHeader(headerName);
105-
if (header != null) {
106-
byte[] value = header.value();
107-
DeserializationException exception = byteArrayToDeserializationException(logger, value);
108-
if (exception != null) {
109-
Headers headers = new RecordHeaders(record.headers().toArray());
110-
headers.remove(headerName);
111-
exception.setHeaders(headers);
112-
}
113-
return exception;
114-
}
115-
return null;
105+
return SerializationUtils.getExceptionFromHeader(record, headerName, logger);
116106
}
117107

118108
/**
@@ -122,7 +112,11 @@ public static DeserializationException getExceptionFromHeader(final ConsumerReco
122112
* @param value the bytes.
123113
* @return the exception or null if deserialization fails.
124114
* @since 2.8.1
115+
* @deprecated in favor of
116+
* {@link SerializationUtils#getExceptionFromHeader(ConsumerRecord, String, LogAccessor)} or
117+
* {@link SerializationUtils#byteArrayToDeserializationException(LogAccessor, org.apache.kafka.common.header.Header)}.
125118
*/
119+
@Deprecated
126120
@Nullable
127121
public static DeserializationException byteArrayToDeserializationException(LogAccessor logger, byte[] value) {
128122
try {

‎spring-kafka/src/main/java/org/springframework/kafka/requestreply/ReplyingKafkaTemplate.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2018-2022 the original author or authors.
2+
* Copyright 2018-2023 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -48,7 +48,6 @@
4848
import org.springframework.kafka.listener.ConsumerSeekAware;
4949
import org.springframework.kafka.listener.ContainerProperties;
5050
import org.springframework.kafka.listener.GenericMessageListenerContainer;
51-
import org.springframework.kafka.listener.ListenerUtils;
5251
import org.springframework.kafka.support.KafkaHeaders;
5352
import org.springframework.kafka.support.KafkaUtils;
5453
import org.springframework.kafka.support.TopicPartitionOffset;
@@ -531,7 +530,7 @@ protected Exception checkForErrors(ConsumerRecord<K, R> record) {
531530
* Return a {@link DeserializationException} if either the key or value failed
532531
* deserialization; null otherwise. If you need to determine whether it was the key or
533532
* value, call
534-
* {@link ListenerUtils#getExceptionFromHeader(ConsumerRecord, String, LogAccessor)}
533+
* {@link SerializationUtils#getExceptionFromHeader(ConsumerRecord, String, LogAccessor)}
535534
* with {@link SerializationUtils#KEY_DESERIALIZER_EXCEPTION_HEADER} and
536535
* {@link SerializationUtils#VALUE_DESERIALIZER_EXCEPTION_HEADER} instead.
537536
* @param record the record.
@@ -541,14 +540,14 @@ protected Exception checkForErrors(ConsumerRecord<K, R> record) {
541540
*/
542541
@Nullable
543542
public static DeserializationException checkDeserialization(ConsumerRecord<?, ?> record, LogAccessor logger) {
544-
DeserializationException exception = ListenerUtils.getExceptionFromHeader(record,
543+
DeserializationException exception = SerializationUtils.getExceptionFromHeader(record,
545544
SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER, logger);
546545
if (exception != null) {
547546
logger.error(exception, () -> "Reply value deserialization failed for " + record.topic() + "-"
548547
+ record.partition() + "@" + record.offset());
549548
return exception;
550549
}
551-
exception = ListenerUtils.getExceptionFromHeader(record,
550+
exception = SerializationUtils.getExceptionFromHeader(record,
552551
SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, logger);
553552
if (exception != null) {
554553
logger.error(exception, () -> "Reply key deserialization failed for " + record.topic() + "-"

‎spring-kafka/src/main/java/org/springframework/kafka/support/serializer/DeserializationExceptionHeader.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.kafka.support.serializer;
18+
19+
import org.apache.kafka.common.header.internals.RecordHeader;
20+
21+
/**
22+
* A package-protected header used to contain serialized
23+
* {@link DeserializationException}s. Only headers of this type will be examined for
24+
* deserialization.
25+
*
26+
* @author Gary Russell
27+
* @since 2.9.11
28+
*/
29+
class DeserializationExceptionHeader extends RecordHeader {
30+
31+
/**
32+
* Construct an instance with the provided properties.
33+
* @param key the key.
34+
* @param value the value;
35+
*/
36+
DeserializationExceptionHeader(String key, byte[] value) {
37+
super(key, value);
38+
}
39+
40+
}

‎spring-kafka/src/main/java/org/springframework/kafka/support/serializer/SerializationUtils.java‎

Lines changed: 83 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2022 the original author or authors.
2+
* Copyright 2020-2023 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -16,16 +16,24 @@
1616

1717
package org.springframework.kafka.support.serializer;
1818

19+
import java.io.ByteArrayInputStream;
1920
import java.io.ByteArrayOutputStream;
2021
import java.io.IOException;
22+
import java.io.ObjectInputStream;
2123
import java.io.ObjectOutputStream;
24+
import java.io.ObjectStreamClass;
2225
import java.lang.reflect.InvocationTargetException;
2326
import java.lang.reflect.Method;
2427
import java.util.function.BiFunction;
2528

29+
import org.apache.kafka.clients.consumer.ConsumerRecord;
30+
import org.apache.kafka.common.header.Header;
2631
import org.apache.kafka.common.header.Headers;
27-
import org.apache.kafka.common.header.internals.RecordHeader;
32+
import org.apache.kafka.common.header.internals.RecordHeaders;
2833

34+
import org.springframework.core.log.LogAccessor;
35+
import org.springframework.kafka.support.KafkaUtils;
36+
import org.springframework.lang.Nullable;
2937
import org.springframework.util.Assert;
3038
import org.springframework.util.ClassUtils;
3139

@@ -166,10 +174,82 @@ data, isForKeyArg, new RuntimeException("Could not serialize type "
166174
}
167175
}
168176
headers.add(
169-
new RecordHeader(isForKeyArg
177+
new DeserializationExceptionHeader(isForKeyArg
170178
? KEY_DESERIALIZER_EXCEPTION_HEADER
171179
: VALUE_DESERIALIZER_EXCEPTION_HEADER,
172180
stream.toByteArray()));
173181
}
174182

183+
/**
184+
* Extract a {@link DeserializationException} from the supplied header name, if
185+
* present.
186+
* @param record the consumer record.
187+
* @param headerName the header name.
188+
* @param logger the logger for logging errors.
189+
* @return the exception or null.
190+
* @since 2.9.11
191+
*/
192+
@Nullable
193+
public static DeserializationException getExceptionFromHeader(final ConsumerRecord<?, ?> record,
194+
String headerName, LogAccessor logger) {
195+
196+
Header header = record.headers().lastHeader(headerName);
197+
if (!(header instanceof DeserializationExceptionHeader)) {
198+
logger.warn(
199+
() -> String.format("Foreign deserialization exception header in (%s) ignored; possible attack?",
200+
KafkaUtils.format(record)));
201+
return null;
202+
}
203+
if (header != null) {
204+
byte[] value = header.value();
205+
DeserializationException exception = byteArrayToDeserializationException(logger, header);
206+
if (exception != null) {
207+
Headers headers = new RecordHeaders(record.headers().toArray());
208+
headers.remove(headerName);
209+
exception.setHeaders(headers);
210+
}
211+
return exception;
212+
}
213+
return null;
214+
}
215+
216+
/**
217+
* Convert a byte array containing a serialized {@link DeserializationException} to the
218+
* {@link DeserializationException}.
219+
* @param logger a log accessor to log errors.
220+
* @param header the header.
221+
* @return the exception or null if deserialization fails.
222+
* @since 2.9.11
223+
*/
224+
@Nullable
225+
public static DeserializationException byteArrayToDeserializationException(LogAccessor logger, Header header) {
226+
227+
if (!(header instanceof DeserializationExceptionHeader)) {
228+
throw new IllegalStateException("Foreign deserialization exception header ignored; possible attack?");
229+
}
230+
try {
231+
ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(header.value())) {
232+
233+
boolean first = true;
234+
235+
@Override
236+
protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException {
237+
if (this.first) {
238+
this.first = false;
239+
Assert.state(desc.getName().equals(DeserializationException.class.getName()),
240+
"Header does not contain a DeserializationException");
241+
}
242+
return super.resolveClass(desc);
243+
}
244+
245+
246+
};
247+
return (DeserializationException) ois.readObject();
248+
}
249+
catch (IOException | ClassNotFoundException | ClassCastException e) {
250+
logger.error(e, "Failed to deserialize a deserialization exception");
251+
return null;
252+
}
253+
}
254+
175255
}

‎spring-kafka/src/test/java/org/springframework/kafka/listener/DeadLetterPublishingRecovererTests.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import org.springframework.kafka.support.KafkaHeaders;
7272
import org.springframework.kafka.support.SendResult;
7373
import org.springframework.kafka.support.serializer.DeserializationException;
74+
import org.springframework.kafka.support.serializer.SerializationTestUtils;
7475
import org.springframework.kafka.support.serializer.SerializationUtils;
7576
import org.springframework.kafka.test.utils.KafkaTestUtils;
7677
import org.springframework.util.concurrent.ListenableFuture;
@@ -172,8 +173,10 @@ void valueHeaderStripped() {
172173
KafkaOperations<?, ?> template = mock(KafkaOperations.class);
173174
DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(template);
174175
Headers headers = new RecordHeaders();
175-
headers.add(new RecordHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER, header(false)));
176-
headers.add(new RecordHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, header(true)));
176+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER,
177+
header(false)));
178+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER,
179+
header(true)));
177180
Headers custom = new RecordHeaders();
178181
custom.add(new RecordHeader("foo", "bar".getBytes()));
179182
recoverer.setHeadersFunction((rec, ex) -> custom);
@@ -202,7 +205,8 @@ void keyHeaderStripped() {
202205
KafkaOperations<?, ?> template = mock(KafkaOperations.class);
203206
DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(template);
204207
Headers headers = new RecordHeaders();
205-
headers.add(new RecordHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, header(true)));
208+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER,
209+
header(true)));
206210
SettableListenableFuture future = new SettableListenableFuture();
207211
future.set(new Object());
208212
willReturn(future).given(template).send(any(ProducerRecord.class));
@@ -222,8 +226,8 @@ void keyDeserOnly() {
222226
DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(template);
223227
Headers headers = new RecordHeaders();
224228
DeserializationException deserEx = createDeserEx(true);
225-
headers.add(
226-
new RecordHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, header(true, deserEx)));
229+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER,
230+
header(true, deserEx)));
227231
SettableListenableFuture future = new SettableListenableFuture();
228232
future.set(new Object());
229233
willReturn(future).given(template).send(any(ProducerRecord.class));
@@ -245,8 +249,10 @@ void headersNotStripped() {
245249
DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(template);
246250
recoverer.setRetainExceptionHeader(true);
247251
Headers headers = new RecordHeaders();
248-
headers.add(new RecordHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER, header(false)));
249-
headers.add(new RecordHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER, header(true)));
252+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER,
253+
header(false)));
254+
headers.add(SerializationTestUtils.deserializationHeader(SerializationUtils.KEY_DESERIALIZER_EXCEPTION_HEADER,
255+
header(true)));
250256
SettableListenableFuture future = new SettableListenableFuture();
251257
future.set(new Object());
252258
willReturn(future).given(template).send(any(ProducerRecord.class));

‎spring-kafka/src/test/java/org/springframework/kafka/listener/ErrorHandlingDeserializerTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2018-2022 the original author or authors.
2+
* Copyright 2018-2023 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -128,8 +128,8 @@ public String deserialize(String topic, Headers headers, byte[] data) {
128128
ErrorHandlingDeserializer<String> ehd = new ErrorHandlingDeserializer<>(new MyDes());
129129
Headers headers = new RecordHeaders();
130130
ehd.deserialize("foo", headers, new byte[1]);
131-
DeserializationException dex = ListenerUtils.byteArrayToDeserializationException(null,
132-
headers.lastHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER).value());
131+
DeserializationException dex = SerializationUtils.byteArrayToDeserializationException(null,
132+
headers.lastHeader(SerializationUtils.VALUE_DESERIALIZER_EXCEPTION_HEADER));
133133
assertThat(dex.getMessage())
134134
.contains("Could not serialize")
135135
.contains("original exception message");

0 commit comments

Comments
 (0)