Skip to content

Commit a2ed191

Browse files
atallahadeatallahade
authored
[core] Fix JSON text deserialization in ARRAYs and ROWs (#5049)
1 parent 183ca8f commit a2ed191

File tree

3 files changed

+367
-3
lines changed

3 files changed

+367
-3
lines changed

‎paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
172172
List<Object> resultList = new ArrayList<>();
173173
for (JsonNode elementNode : arrayNode) {
174174
if (!elementNode.isNull()) {
175-
String elementJson = elementNode.toString();
175+
String elementJson;
176+
if (elementNode.isTextual()) {
177+
elementJson = elementNode.asText();
178+
} else {
179+
elementJson = elementNode.toString();
180+
}
176181
Object elementObject =
177182
castFromStringInternal(elementJson, elementType, isCdcValue);
178183
resultList.add(elementObject);
@@ -260,7 +265,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
260265
DataField field = rowType.getFields().get(pos);
261266
JsonNode fieldNode = rowNode.get(field.name());
262267
if (fieldNode != null && !fieldNode.isNull()) {
263-
String fieldJson = fieldNode.toString();
268+
String fieldJson;
269+
if (fieldNode.isTextual()) {
270+
fieldJson = fieldNode.asText();
271+
} else {
272+
fieldJson = fieldNode.toString();
273+
}
264274
Object fieldObject =
265275
castFromStringInternal(fieldJson, field.type(), isCdcValue);
266276
genericRow.setField(pos, fieldObject);

‎paimon-common/src/test/java/org/apache/paimon/utils/TypeUtilsTest.java‎

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.utils;
20+
21+
import org.apache.paimon.data.BinaryString;
22+
import org.apache.paimon.data.Decimal;
23+
import org.apache.paimon.data.GenericArray;
24+
import org.apache.paimon.data.GenericMap;
25+
import org.apache.paimon.data.GenericRow;
26+
import org.apache.paimon.data.Timestamp;
27+
import org.apache.paimon.types.DataField;
28+
import org.apache.paimon.types.DataTypes;
29+
30+
import org.junit.jupiter.api.AfterAll;
31+
import org.junit.jupiter.api.BeforeAll;
32+
import org.junit.jupiter.api.Test;
33+
34+
import java.math.BigDecimal;
35+
import java.nio.charset.StandardCharsets;
36+
import java.time.LocalDateTime;
37+
import java.util.Arrays;
38+
import java.util.Base64;
39+
import java.util.Collections;
40+
import java.util.HashMap;
41+
import java.util.TimeZone;
42+
43+
import static org.assertj.core.api.Assertions.assertThat;
44+
45+
/** Test for {@link TypeUtils}. */
46+
public class TypeUtilsTest {
47+
private static TimeZone originalTimeZone;
48+
49+
@BeforeAll
50+
public static void setUp() {
51+
originalTimeZone = TimeZone.getDefault();
52+
TimeZone.setDefault(TimeZone.getTimeZone("Asia/Tokyo"));
53+
}
54+
55+
@AfterAll
56+
public static void tearDown() {
57+
TimeZone.setDefault(originalTimeZone);
58+
}
59+
60+
@Test
61+
public void testCastFromString() {
62+
String value =
63+
"[{\"key1\":null,\"key2\":\"value\"},{\"key1\":{\"nested_key1\":0},\"key2\":null}]";
64+
Object result =
65+
TypeUtils.castFromString(
66+
value,
67+
DataTypes.ARRAY(
68+
DataTypes.ROW(
69+
new DataField(
70+
0,
71+
"key1",
72+
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
73+
new DataField(1, "key2", DataTypes.STRING()))));
74+
GenericArray expected =
75+
new GenericArray(
76+
Arrays.asList(
77+
GenericRow.of(null, BinaryString.fromString("value")),
78+
GenericRow.of(
79+
new GenericMap(
80+
Collections.singletonMap(
81+
BinaryString.fromString(
82+
"nested_key1"),
83+
0)),
84+
null))
85+
.toArray());
86+
assertThat(result).isEqualTo(expected);
87+
}
88+
89+
@Test
90+
public void testStringCastFromString() {
91+
String value = "value";
92+
Object result = TypeUtils.castFromString(value, DataTypes.STRING());
93+
BinaryString expected = BinaryString.fromString("value");
94+
assertThat(result).isEqualTo(expected);
95+
}
96+
97+
@Test
98+
public void testArrayIntCastFromString() {
99+
String value = "[0, 1, 2]";
100+
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.INT()));
101+
GenericArray expected = new GenericArray(new Integer[] {0, 1, 2});
102+
assertThat(result).isEqualTo(expected);
103+
}
104+
105+
@Test
106+
public void testArrayStringCastFromString() {
107+
String value = "[\"0\", \"1\", \"2\"]";
108+
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.STRING()));
109+
GenericArray expected =
110+
new GenericArray(
111+
Arrays.asList(
112+
BinaryString.fromString("0"),
113+
BinaryString.fromString("1"),
114+
BinaryString.fromString("2"))
115+
.toArray());
116+
assertThat(result).isEqualTo(expected);
117+
}
118+
119+
@Test
120+
public void testLongCastFromString() {
121+
String value = "12";
122+
Object result = TypeUtils.castFromString(value, DataTypes.BIGINT());
123+
long expected = 12;
124+
assertThat(result).isEqualTo(expected);
125+
}
126+
127+
@Test
128+
public void testBinaryCastFromString() {
129+
String value = "abc";
130+
Object result = TypeUtils.castFromString(value, DataTypes.BINARY(3));
131+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
132+
assertThat(result).isEqualTo(expected);
133+
}
134+
135+
@Test
136+
public void testBinaryCastFromCdcValueString() {
137+
String value = Base64.getEncoder().encodeToString("abc".getBytes(StandardCharsets.UTF_8));
138+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BINARY(3));
139+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
140+
assertThat(result).isEqualTo(expected);
141+
}
142+
143+
@Test
144+
public void testBooleanTrueCastFromString() {
145+
String value = "true";
146+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
147+
boolean expected = true;
148+
assertThat(result).isEqualTo(expected);
149+
}
150+
151+
@Test
152+
public void testBooleanFalseCastFromString() {
153+
String value = "false";
154+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
155+
boolean expected = false;
156+
assertThat(result).isEqualTo(expected);
157+
}
158+
159+
@Test
160+
public void testCharCastFromString() {
161+
String value = "abc";
162+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.CHAR(3));
163+
BinaryString expected = BinaryString.fromString("abc");
164+
assertThat(result).isEqualTo(expected);
165+
}
166+
167+
@Test
168+
public void testDateCastFromString() {
169+
String value = "2017-12-12 09:30:00.0";
170+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
171+
int expected = 17512;
172+
assertThat(result).isEqualTo(expected);
173+
}
174+
175+
@Test
176+
public void testDateNumericCastFromString() {
177+
String value = "17512";
178+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
179+
int expected = 17512;
180+
assertThat(result).isEqualTo(expected);
181+
}
182+
183+
@Test
184+
public void testDecimalCastFromString() {
185+
String value = "123";
186+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DECIMAL(5, 0));
187+
Decimal expected = Decimal.fromBigDecimal(new BigDecimal("123"), 5, 0);
188+
assertThat(result).isEqualTo(expected);
189+
}
190+
191+
@Test
192+
public void testDoubleCastFromString() {
193+
String value = "123.456";
194+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DOUBLE());
195+
Double expected = 123.456;
196+
assertThat(result).isEqualTo(expected);
197+
}
198+
199+
@Test
200+
public void testFloatCastFromString() {
201+
String value = "123.456";
202+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
203+
Float expected = 123.456f;
204+
assertThat(result).isEqualTo(expected);
205+
}
206+
207+
@Test
208+
public void testLargeFloatCastFromString() {
209+
String value = "123.45678";
210+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
211+
Float expected = 123.45678f;
212+
assertThat(result).isEqualTo(expected);
213+
}
214+
215+
@Test
216+
public void testIntCastFromString() {
217+
String value = "12";
218+
Object result = TypeUtils.castFromString(value, DataTypes.INT());
219+
int expected = 12;
220+
assertThat(result).isEqualTo(expected);
221+
}
222+
223+
@Test
224+
public void testLocalZonedTimestampCastFromString() {
225+
String value = "2017-12-12 09:30:00";
226+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE());
227+
Timestamp expected =
228+
Timestamp.fromEpochMillis(
229+
LocalDateTime.parse("2017-12-12T09:30:00")
230+
.atZone(TimeZone.getTimeZone("Asia/Tokyo").toZoneId())
231+
.toEpochSecond()
232+
* 1000);
233+
assertThat(result).isEqualTo(expected);
234+
}
235+
236+
@Test
237+
public void testMapStringStringCastFromString() {
238+
String value = "{\"a\":\"b\", \"c\":\"d\"}";
239+
Object result =
240+
TypeUtils.castFromString(
241+
value, DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()));
242+
GenericMap expected =
243+
new GenericMap(
244+
new HashMap<BinaryString, BinaryString>() {
245+
{
246+
put(BinaryString.fromString("a"), BinaryString.fromString("b"));
247+
put(BinaryString.fromString("c"), BinaryString.fromString("d"));
248+
}
249+
});
250+
assertThat(result).isEqualTo(expected);
251+
}
252+
253+
@Test
254+
public void testMapStringIntCastFromString() {
255+
String value = "{\"a\":0, \"c\":1}";
256+
Object result =
257+
TypeUtils.castFromString(value, DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()));
258+
GenericMap expected =
259+
new GenericMap(
260+
new HashMap<BinaryString, Integer>() {
261+
{
262+
put(BinaryString.fromString("a"), 0);
263+
put(BinaryString.fromString("c"), 1);
264+
}
265+
});
266+
assertThat(result).isEqualTo(expected);
267+
}
268+
269+
@Test
270+
public void testRowCastFromString() {
271+
String value = "{\"key1\":{\"nested_key1\":0},\"key2\":\"value\"}";
272+
Object result =
273+
TypeUtils.castFromString(
274+
value,
275+
DataTypes.ROW(
276+
new DataField(
277+
0,
278+
"key1",
279+
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
280+
new DataField(1, "key2", DataTypes.STRING())));
281+
GenericRow expected =
282+
GenericRow.of(
283+
new GenericMap(
284+
Collections.singletonMap(
285+
BinaryString.fromString("nested_key1"), 0)),
286+
BinaryString.fromString("value"));
287+
assertThat(result).isEqualTo(expected);
288+
}
289+
290+
@Test
291+
public void testSmallIntCastFromString() {
292+
String value = "12";
293+
Object result = TypeUtils.castFromString(value, DataTypes.SMALLINT());
294+
short expected = 12;
295+
assertThat(result).isEqualTo(expected);
296+
}
297+
298+
@Test
299+
public void testTimestampCastFromString() {
300+
String value = "2017-12-12 09:30:00";
301+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
302+
Timestamp expected =
303+
Timestamp.fromLocalDateTime(LocalDateTime.parse("2017-12-12T09:30:00"));
304+
assertThat(result).isEqualTo(expected);
305+
}
306+
307+
@Test
308+
public void testTimestampNumericCastFromString() {
309+
String value = "123456789000000";
310+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
311+
Timestamp expected = Timestamp.fromMicros(123456789000000L);
312+
assertThat(result).isEqualTo(expected);
313+
}
314+
315+
@Test
316+
public void testTimeCastFromString() {
317+
String value = "13:09:42.123456+01:00";
318+
Object result = TypeUtils.castFromString(value, DataTypes.TIME(3));
319+
int expected = 14 * 60 * 60 * 1000 + 9 * 60 * 1000 + 42 * 1000 + 123;
320+
assertThat(result).isEqualTo(expected);
321+
}
322+
323+
@Test
324+
public void testTimeNumericCastFromString() {
325+
String value = "123456789";
326+
Object result = TypeUtils.castFromString(value, DataTypes.TIME());
327+
int expected = 123456789;
328+
assertThat(result).isEqualTo(expected);
329+
}
330+
331+
@Test
332+
public void testTinyIntCastFromString() {
333+
String value = "6";
334+
Object result = TypeUtils.castFromString(value, DataTypes.TINYINT());
335+
byte expected = 6;
336+
assertThat(result).isEqualTo(expected);
337+
}
338+
339+
@Test
340+
public void testVarBinaryCastFromString() {
341+
String value = "abc";
342+
Object result = TypeUtils.castFromString(value, DataTypes.VARBINARY(3));
343+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
344+
assertThat(result).isEqualTo(expected);
345+
}
346+
347+
@Test
348+
public void testVarCharCastFromString() {
349+
String value = "abc";
350+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.VARCHAR(3));
351+
BinaryString expected = BinaryString.fromString("abc");
352+
assertThat(result).isEqualTo(expected);
353+
}
354+
}

‎paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ private void testAllTypesImpl() throws Exception {
423423
+ "Paimon , Apache Paimon, Apache Paimon PostgreSQL Test Data, "
424424
+ "[98, 121, 116, 101, 115], "
425425
+ "{\"a\": \"b\"}, "
426-
+ "[\"item1\", \"item2\"]"
426+
+ "[item1, item2]"
427427
+ "]",
428428
"+I["
429429
+ "2, 2.2, "

0 commit comments

Comments
 (0)