Support array of scalar values in Jackson2Tokenizer

Issue: SPR-16407

Author: poutsma

spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2017 the original author or authors.
+ * Copyright 2002-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -173,8 +173,9 @@ private void processTokenArray(JsonToken token, List<TokenBuffer> result) throws
 			this.tokenBuffer.copyCurrentEvent(this.parser);
 		}
 
-		if ((token == JsonToken.END_OBJECT &&  this.objectDepth == 0 && (this.arrayDepth == 1 || this.arrayDepth == 0)) ||
-				(token.isScalarValue()) && this.objectDepth == 0 && this.arrayDepth == 0) {
+		if (this.objectDepth == 0 &&
+				(this.arrayDepth == 0 || this.arrayDepth == 1) &&
+				(token == JsonToken.END_OBJECT || token.isScalarValue())) {
 			result.add(this.tokenBuffer);
 			this.tokenBuffer = new TokenBuffer(this.parser);
 		}

spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2017 the original author or authors.
+ * Copyright 2002-2018 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -168,14 +168,18 @@ public void tokenizeArrayElements() {
 
 		testTokenize(asList("12.", "34")
 			,singletonList("12.34"), true);
+
+		// SPR-16407
+		testTokenize(asList("[1", ",2,", "3]"),
+				asList("1", "2", "3"), true);
 	}
 
 	private void testTokenize(List<String> source, List<String> expected, boolean tokenizeArrayElements) {
 		Flux<DataBuffer> sourceFlux = Flux.fromIterable(source)
 				.map(this::stringBuffer);
 
 		Flux<TokenBuffer> tokenBufferFlux =
-				Jackson2Tokenizer.tokenize(sourceFlux, jsonFactory, tokenizeArrayElements);
+				Jackson2Tokenizer.tokenize(sourceFlux, this.jsonFactory, tokenizeArrayElements);
 
 		Flux<String> result = tokenBufferFlux
 				.map(tokenBuffer -> {