Skip to content

Commit 323f66d

Browse files
atallahadeatallahade
committed
[core] Fix JSON text deserialization in ARRAYs and ROWs
1 parent 908a4e6 commit 323f66d

File tree

2 files changed

+347
-2
lines changed

2 files changed

+347
-2
lines changed

‎paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
172172
List<Object> resultList = new ArrayList<>();
173173
for (JsonNode elementNode : arrayNode) {
174174
if (!elementNode.isNull()) {
175-
String elementJson = elementNode.toString();
175+
String elementJson;
176+
if (elementNode.isTextual()) {
177+
elementJson = elementNode.asText();
178+
} else {
179+
elementJson = elementNode.toString();
180+
}
176181
Object elementObject =
177182
castFromStringInternal(elementJson, elementType, isCdcValue);
178183
resultList.add(elementObject);
@@ -260,7 +265,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
260265
DataField field = rowType.getFields().get(pos);
261266
JsonNode fieldNode = rowNode.get(field.name());
262267
if (fieldNode != null && !fieldNode.isNull()) {
263-
String fieldJson = fieldNode.toString();
268+
String fieldJson;
269+
if (fieldNode.isTextual()) {
270+
fieldJson = fieldNode.asText();
271+
} else {
272+
fieldJson = fieldNode.toString();
273+
}
264274
Object fieldObject =
265275
castFromStringInternal(fieldJson, field.type(), isCdcValue);
266276
genericRow.setField(pos, fieldObject);

‎paimon-common/src/test/java/org/apache/paimon/utils/TypeUtilsTest.java

Lines changed: 335 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.utils;
20+
21+
import org.apache.paimon.data.BinaryString;
22+
import org.apache.paimon.data.Decimal;
23+
import org.apache.paimon.data.GenericArray;
24+
import org.apache.paimon.data.GenericMap;
25+
import org.apache.paimon.data.GenericRow;
26+
import org.apache.paimon.data.Timestamp;
27+
import org.apache.paimon.types.DataField;
28+
import org.apache.paimon.types.DataTypes;
29+
30+
import org.junit.jupiter.api.Test;
31+
32+
import java.math.BigDecimal;
33+
import java.nio.charset.StandardCharsets;
34+
import java.time.LocalDateTime;
35+
import java.util.Arrays;
36+
import java.util.Base64;
37+
import java.util.Collections;
38+
import java.util.HashMap;
39+
40+
import static org.assertj.core.api.Assertions.assertThat;
41+
42+
/** Test for {@link TypeUtils}. */
43+
public class TypeUtilsTest {
44+
45+
@Test
46+
public void testCastFromString() {
47+
String value =
48+
"[{\"key1\":null,\"key2\":\"value\"},{\"key1\":{\"nested_key1\":0},\"key2\":null}]";
49+
Object result =
50+
TypeUtils.castFromString(
51+
value,
52+
DataTypes.ARRAY(
53+
DataTypes.ROW(
54+
new DataField(
55+
0,
56+
"key1",
57+
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
58+
new DataField(1, "key2", DataTypes.STRING()))));
59+
GenericArray expected =
60+
new GenericArray(
61+
Arrays.asList(
62+
GenericRow.of(null, BinaryString.fromString("value")),
63+
GenericRow.of(
64+
new GenericMap(
65+
Collections.singletonMap(
66+
BinaryString.fromString(
67+
"nested_key1"),
68+
0)),
69+
null))
70+
.toArray());
71+
assertThat(result).isEqualTo(expected);
72+
}
73+
74+
@Test
75+
public void testStringCastFromString() {
76+
String value = "value";
77+
Object result = TypeUtils.castFromString(value, DataTypes.STRING());
78+
BinaryString expected = BinaryString.fromString("value");
79+
assertThat(result).isEqualTo(expected);
80+
}
81+
82+
@Test
83+
public void testArrayIntCastFromString() {
84+
String value = "[0, 1, 2]";
85+
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.INT()));
86+
GenericArray expected = new GenericArray(new Integer[] {0, 1, 2});
87+
assertThat(result).isEqualTo(expected);
88+
}
89+
90+
@Test
91+
public void testArrayStringCastFromString() {
92+
String value = "[\"0\", \"1\", \"2\"]";
93+
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.STRING()));
94+
GenericArray expected =
95+
new GenericArray(
96+
Arrays.asList(
97+
BinaryString.fromString("0"),
98+
BinaryString.fromString("1"),
99+
BinaryString.fromString("2"))
100+
.toArray());
101+
assertThat(result).isEqualTo(expected);
102+
}
103+
104+
@Test
105+
public void testLongCastFromString() {
106+
String value = "12";
107+
Object result = TypeUtils.castFromString(value, DataTypes.BIGINT());
108+
long expected = 12;
109+
assertThat(result).isEqualTo(expected);
110+
}
111+
112+
@Test
113+
public void testBinaryCastFromString() {
114+
String value = "abc";
115+
Object result = TypeUtils.castFromString(value, DataTypes.BINARY(3));
116+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
117+
assertThat(result).isEqualTo(expected);
118+
}
119+
120+
@Test
121+
public void testBinaryCastFromCdcValueString() {
122+
String value = Base64.getEncoder().encodeToString("abc".getBytes(StandardCharsets.UTF_8));
123+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BINARY(3));
124+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
125+
assertThat(result).isEqualTo(expected);
126+
}
127+
128+
@Test
129+
public void testBooleanTrueCastFromString() {
130+
String value = "true";
131+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
132+
boolean expected = true;
133+
assertThat(result).isEqualTo(expected);
134+
}
135+
136+
@Test
137+
public void testBooleanFalseCastFromString() {
138+
String value = "false";
139+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
140+
boolean expected = false;
141+
assertThat(result).isEqualTo(expected);
142+
}
143+
144+
@Test
145+
public void testCharCastFromString() {
146+
String value = "abc";
147+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.CHAR(3));
148+
BinaryString expected = BinaryString.fromString("abc");
149+
assertThat(result).isEqualTo(expected);
150+
}
151+
152+
@Test
153+
public void testDateCastFromString() {
154+
String value = "2017-12-12 09:30:00.0";
155+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
156+
int expected = 17512;
157+
assertThat(result).isEqualTo(expected);
158+
}
159+
160+
@Test
161+
public void testDateNumericCastFromString() {
162+
String value = "17512";
163+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
164+
int expected = 17512;
165+
assertThat(result).isEqualTo(expected);
166+
}
167+
168+
@Test
169+
public void testDecimalCastFromString() {
170+
String value = "123";
171+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DECIMAL(5, 0));
172+
Decimal expected = Decimal.fromBigDecimal(new BigDecimal("123"), 5, 0);
173+
assertThat(result).isEqualTo(expected);
174+
}
175+
176+
@Test
177+
public void testDoubleCastFromString() {
178+
String value = "123.456";
179+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DOUBLE());
180+
Double expected = 123.456;
181+
assertThat(result).isEqualTo(expected);
182+
}
183+
184+
@Test
185+
public void testFloatCastFromString() {
186+
String value = "123.456";
187+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
188+
Float expected = 123.456f;
189+
assertThat(result).isEqualTo(expected);
190+
}
191+
192+
@Test
193+
public void testLargeFloatCastFromString() {
194+
String value = "123.45678";
195+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
196+
Float expected = 123.45678f;
197+
assertThat(result).isEqualTo(expected);
198+
}
199+
200+
@Test
201+
public void testIntCastFromString() {
202+
String value = "12";
203+
Object result = TypeUtils.castFromString(value, DataTypes.INT());
204+
int expected = 12;
205+
assertThat(result).isEqualTo(expected);
206+
}
207+
208+
@Test
209+
public void testLocalZonedTimestampCastFromString() {
210+
String value = "2017-12-12 09:30:00";
211+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE());
212+
Timestamp expected =
213+
Timestamp.fromLocalDateTime(LocalDateTime.parse("2017-12-12T09:30:00"));
214+
assertThat(result).isEqualTo(expected);
215+
}
216+
217+
@Test
218+
public void testMapStringStringCastFromString() {
219+
String value = "{\"a\":\"b\", \"c\":\"d\"}";
220+
Object result =
221+
TypeUtils.castFromString(
222+
value, DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()));
223+
GenericMap expected =
224+
new GenericMap(
225+
new HashMap<BinaryString, BinaryString>() {
226+
{
227+
put(BinaryString.fromString("a"), BinaryString.fromString("b"));
228+
put(BinaryString.fromString("c"), BinaryString.fromString("d"));
229+
}
230+
});
231+
assertThat(result).isEqualTo(expected);
232+
}
233+
234+
@Test
235+
public void testMapStringIntCastFromString() {
236+
String value = "{\"a\":0, \"c\":1}";
237+
Object result =
238+
TypeUtils.castFromString(value, DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()));
239+
GenericMap expected =
240+
new GenericMap(
241+
new HashMap<BinaryString, Integer>() {
242+
{
243+
put(BinaryString.fromString("a"), 0);
244+
put(BinaryString.fromString("c"), 1);
245+
}
246+
});
247+
assertThat(result).isEqualTo(expected);
248+
}
249+
250+
@Test
251+
public void testRowCastFromString() {
252+
String value = "{\"key1\":{\"nested_key1\":0},\"key2\":\"value\"}";
253+
Object result =
254+
TypeUtils.castFromString(
255+
value,
256+
DataTypes.ROW(
257+
new DataField(
258+
0,
259+
"key1",
260+
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
261+
new DataField(1, "key2", DataTypes.STRING())));
262+
GenericRow expected =
263+
GenericRow.of(
264+
new GenericMap(
265+
Collections.singletonMap(
266+
BinaryString.fromString("nested_key1"), 0)),
267+
BinaryString.fromString("value"));
268+
assertThat(result).isEqualTo(expected);
269+
}
270+
271+
@Test
272+
public void testSmallIntCastFromString() {
273+
String value = "12";
274+
Object result = TypeUtils.castFromString(value, DataTypes.SMALLINT());
275+
short expected = 12;
276+
assertThat(result).isEqualTo(expected);
277+
}
278+
279+
@Test
280+
public void testTimestampCastFromString() {
281+
String value = "2017-12-12 09:30:00";
282+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
283+
Timestamp expected =
284+
Timestamp.fromLocalDateTime(LocalDateTime.parse("2017-12-12T09:30:00"));
285+
assertThat(result).isEqualTo(expected);
286+
}
287+
288+
@Test
289+
public void testTimestampNumericCastFromString() {
290+
String value = "123456789000000";
291+
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
292+
Timestamp expected = Timestamp.fromMicros(123456789000000L);
293+
assertThat(result).isEqualTo(expected);
294+
}
295+
296+
@Test
297+
public void testTimeCastFromString() {
298+
String value = "13:09:42.123456+01:00";
299+
Object result = TypeUtils.castFromString(value, DataTypes.TIME(3));
300+
int expected = 14 * 60 * 60 * 1000 + 9 * 60 * 1000 + 42 * 1000 + 123;
301+
assertThat(result).isEqualTo(expected);
302+
}
303+
304+
@Test
305+
public void testTimeNumericCastFromString() {
306+
String value = "123456789";
307+
Object result = TypeUtils.castFromString(value, DataTypes.TIME());
308+
int expected = 123456789;
309+
assertThat(result).isEqualTo(expected);
310+
}
311+
312+
@Test
313+
public void testTinyIntCastFromString() {
314+
String value = "6";
315+
Object result = TypeUtils.castFromString(value, DataTypes.TINYINT());
316+
byte expected = 6;
317+
assertThat(result).isEqualTo(expected);
318+
}
319+
320+
@Test
321+
public void testVarBinaryCastFromString() {
322+
String value = "abc";
323+
Object result = TypeUtils.castFromString(value, DataTypes.VARBINARY(3));
324+
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
325+
assertThat(result).isEqualTo(expected);
326+
}
327+
328+
@Test
329+
public void testVarCharCastFromString() {
330+
String value = "abc";
331+
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.VARCHAR(3));
332+
BinaryString expected = BinaryString.fromString("abc");
333+
assertThat(result).isEqualTo(expected);
334+
}
335+
}

0 commit comments

Comments
 (0)