Honour ObjectMapper feature in Jackson2Tokenizer

After this commit, Jackson2Tokenizer honours ObjectMapper's
DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS feature when creating
TokenBuffers.

Closes gh-24479

Author: poutsma

spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.core.async.ByteArrayFeeder;
 import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.deser.DefaultDeserializationContext;
 import com.fasterxml.jackson.databind.util.TokenBuffer;
@@ -36,6 +37,7 @@
 import org.springframework.core.io.buffer.DataBuffer;
 import org.springframework.core.io.buffer.DataBufferLimitException;
 import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.lang.Nullable;
 
 /**
  * {@link Function} to transform a JSON stream of arbitrary size, byte array
@@ -55,34 +57,39 @@ final class Jackson2Tokenizer {
 
 	private final boolean tokenizeArrayElements;
 
-	private TokenBuffer tokenBuffer;
+	private final boolean forceUseOfBigDecimal;
+
+	private final int maxInMemorySize;
 
 	private int objectDepth;
 
 	private int arrayDepth;
 
-	private final int maxInMemorySize;
-
 	private int byteCount;
 
+	@Nullable // yet initialized by calling createToken() in the constructor
+	private TokenBuffer tokenBuffer;
+
 
 	// TODO: change to ByteBufferFeeder when supported by Jackson
 	// See https://github.com/FasterXML/jackson-core/issues/478
 	private final ByteArrayFeeder inputFeeder;
 
 
 	private Jackson2Tokenizer(JsonParser parser, DeserializationContext deserializationContext,
-			boolean tokenizeArrayElements, int maxInMemorySize) {
+			boolean tokenizeArrayElements, boolean forceUseOfBigDecimal, int maxInMemorySize) {
 
 		this.parser = parser;
 		this.deserializationContext = deserializationContext;
 		this.tokenizeArrayElements = tokenizeArrayElements;
-		this.tokenBuffer = new TokenBuffer(parser, deserializationContext);
+		this.forceUseOfBigDecimal = forceUseOfBigDecimal;
 		this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder();
 		this.maxInMemorySize = maxInMemorySize;
+		createToken();
 	}
 
 
+
 	private Flux<TokenBuffer> tokenize(DataBuffer dataBuffer) {
 		int bufferSize = dataBuffer.readableByteCount();
 		byte[] bytes = new byte[dataBuffer.readableByteCount()];
@@ -132,6 +139,9 @@ else if (token == null ) { // !previousNull
 				previousNull = true;
 				continue;
 			}
+			else {
+				previousNull = false;
+			}
 			updateDepth(token);
 			if (!this.tokenizeArrayElements) {
 				processTokenNormal(token, result);
@@ -165,7 +175,7 @@ private void processTokenNormal(JsonToken token, List<TokenBuffer> result) throw
 
 		if ((token.isStructEnd() || token.isScalarValue()) && this.objectDepth == 0 && this.arrayDepth == 0) {
 			result.add(this.tokenBuffer);
-			this.tokenBuffer = new TokenBuffer(this.parser, this.deserializationContext);
+			createToken();
 		}
 
 	}
@@ -178,10 +188,15 @@ private void processTokenArray(JsonToken token, List<TokenBuffer> result) throws
 		if (this.objectDepth == 0 && (this.arrayDepth == 0 || this.arrayDepth == 1) &&
 				(token == JsonToken.END_OBJECT || token.isScalarValue())) {
 			result.add(this.tokenBuffer);
-			this.tokenBuffer = new TokenBuffer(this.parser, this.deserializationContext);
+			createToken();
 		}
 	}
 
+	private void createToken() {
+		this.tokenBuffer = new TokenBuffer(this.parser, this.deserializationContext);
+		this.tokenBuffer.forceUseOfBigDecimal(this.forceUseOfBigDecimal);
+	}
+
 	private boolean isTopLevelArrayToken(JsonToken token) {
 		return this.objectDepth == 0 && ((token == JsonToken.START_ARRAY && this.arrayDepth == 1) ||
 				(token == JsonToken.END_ARRAY && this.arrayDepth == 0));
@@ -229,7 +244,9 @@ public static Flux<TokenBuffer> tokenize(Flux<DataBuffer> dataBuffers, JsonFacto
 				context = ((DefaultDeserializationContext) context).createInstance(
 						objectMapper.getDeserializationConfig(), parser, objectMapper.getInjectableValues());
 			}
-			Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrays, maxInMemorySize);
+			boolean forceUseOfBigDecimal = objectMapper.isEnabled(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
+			Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrays, forceUseOfBigDecimal,
+					maxInMemorySize);
 			return dataBuffers.flatMap(tokenizer::tokenize, Flux::error, tokenizer::endOfInput);
 		}
 		catch (IOException ex) {

spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2019 the original author or authors.
+ * Copyright 2002-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,10 +19,14 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 
 import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.util.TokenBuffer;
 import org.json.JSONException;
@@ -37,8 +41,10 @@
 import org.springframework.core.io.buffer.DataBuffer;
 import org.springframework.core.io.buffer.DataBufferLimitException;
 
-import static java.util.Arrays.*;
-import static java.util.Collections.*;
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 /**
  * @author Arjen Poutsma
@@ -259,6 +265,36 @@ public void jsonEOFExceptionIsWrappedAsDecodingError() {
 				.verify();
 	}
 
+	@Test
+	public void useBigDecimalForFloats() {
+		for (boolean useBigDecimalForFloats : Arrays.asList(false, true)) {
+			this.objectMapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, useBigDecimalForFloats);
+
+			Flux<DataBuffer> source = Flux.just(stringBuffer("1E+2"));
+			Flux<TokenBuffer> tokens =
+					Jackson2Tokenizer.tokenize(source, this.jsonFactory, this.objectMapper, false, -1);
+
+			StepVerifier.create(tokens)
+					.assertNext(tokenBuffer -> {
+						try {
+							JsonParser parser = tokenBuffer.asParser();
+							JsonToken token = parser.nextToken();
+							assertEquals(JsonToken.VALUE_NUMBER_FLOAT, token);
+							JsonParser.NumberType numberType = parser.getNumberType();
+							if (useBigDecimalForFloats) {
+								assertEquals(JsonParser.NumberType.BIG_DECIMAL, numberType);
+							}
+							else {
+								assertEquals(JsonParser.NumberType.DOUBLE, numberType);
+							}
+						}
+						catch (IOException ex) {
+							fail(ex.getMessage());
+						}
+					})
+					.verifyComplete();
+		}
+	}
 
 	private Flux<String> decode(List<String> source, boolean tokenize, int maxInMemorySize) {