Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[core] Fix JSON text deserialization in ARRAYs and ROWs #5049

Merged
merged 3 commits into from
Feb 13, 2025
Merged

[core] Fix JSON text deserialization in ARRAYs and ROWs #5049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
323f66d
[core] Fix JSON text deserialization in ARRAYs and ROWs
atallahade Feb 10, 2025
668c353
[core] Fix TypeUtils test with local datetime
atallahade Feb 10, 2025
43379d8
[cdc] Fix test in PostgresSyncTableActionITCase for strings inside ar…
atallahade Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
List<Object> resultList = new ArrayList<>();
for (JsonNode elementNode : arrayNode) {
if (!elementNode.isNull()) {
String elementJson = elementNode.toString();
String elementJson;
if (elementNode.isTextual()) {
elementJson = elementNode.asText();
} else {
elementJson = elementNode.toString();
}
Object elementObject =
castFromStringInternal(elementJson, elementType, isCdcValue);
resultList.add(elementObject);
Expand Down Expand Up @@ -260,7 +265,12 @@ public static Object castFromStringInternal(String s, DataType type, boolean isC
DataField field = rowType.getFields().get(pos);
JsonNode fieldNode = rowNode.get(field.name());
if (fieldNode != null && !fieldNode.isNull()) {
String fieldJson = fieldNode.toString();
String fieldJson;
if (fieldNode.isTextual()) {
fieldJson = fieldNode.asText();
} else {
fieldJson = fieldNode.toString();
}
Object fieldObject =
castFromStringInternal(fieldJson, field.type(), isCdcValue);
genericRow.setField(pos, fieldObject);
Expand Down
354 changes: 354 additions & 0 deletions paimon-common/src/test/java/org/apache/paimon/utils/TypeUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,354 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.paimon.utils;

import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.TimeZone;

import static org.assertj.core.api.Assertions.assertThat;

/** Test for {@link TypeUtils}. */
public class TypeUtilsTest {
private static TimeZone originalTimeZone;

@BeforeAll
public static void setUp() {
originalTimeZone = TimeZone.getDefault();
TimeZone.setDefault(TimeZone.getTimeZone("Asia/Tokyo"));
}

@AfterAll
public static void tearDown() {
TimeZone.setDefault(originalTimeZone);
}

@Test
public void testCastFromString() {
String value =
"[{\"key1\":null,\"key2\":\"value\"},{\"key1\":{\"nested_key1\":0},\"key2\":null}]";
Object result =
TypeUtils.castFromString(
value,
DataTypes.ARRAY(
DataTypes.ROW(
new DataField(
0,
"key1",
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
new DataField(1, "key2", DataTypes.STRING()))));
GenericArray expected =
new GenericArray(
Arrays.asList(
GenericRow.of(null, BinaryString.fromString("value")),
GenericRow.of(
new GenericMap(
Collections.singletonMap(
BinaryString.fromString(
"nested_key1"),
0)),
null))
.toArray());
assertThat(result).isEqualTo(expected);
}

@Test
public void testStringCastFromString() {
String value = "value";
Object result = TypeUtils.castFromString(value, DataTypes.STRING());
BinaryString expected = BinaryString.fromString("value");
assertThat(result).isEqualTo(expected);
}

@Test
public void testArrayIntCastFromString() {
String value = "[0, 1, 2]";
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.INT()));
GenericArray expected = new GenericArray(new Integer[] {0, 1, 2});
assertThat(result).isEqualTo(expected);
}

@Test
public void testArrayStringCastFromString() {
String value = "[\"0\", \"1\", \"2\"]";
Object result = TypeUtils.castFromString(value, DataTypes.ARRAY(DataTypes.STRING()));
GenericArray expected =
new GenericArray(
Arrays.asList(
BinaryString.fromString("0"),
BinaryString.fromString("1"),
BinaryString.fromString("2"))
.toArray());
assertThat(result).isEqualTo(expected);
}

@Test
public void testLongCastFromString() {
String value = "12";
Object result = TypeUtils.castFromString(value, DataTypes.BIGINT());
long expected = 12;
assertThat(result).isEqualTo(expected);
}

@Test
public void testBinaryCastFromString() {
String value = "abc";
Object result = TypeUtils.castFromString(value, DataTypes.BINARY(3));
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
assertThat(result).isEqualTo(expected);
}

@Test
public void testBinaryCastFromCdcValueString() {
String value = Base64.getEncoder().encodeToString("abc".getBytes(StandardCharsets.UTF_8));
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BINARY(3));
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
assertThat(result).isEqualTo(expected);
}

@Test
public void testBooleanTrueCastFromString() {
String value = "true";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
boolean expected = true;
assertThat(result).isEqualTo(expected);
}

@Test
public void testBooleanFalseCastFromString() {
String value = "false";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.BOOLEAN());
boolean expected = false;
assertThat(result).isEqualTo(expected);
}

@Test
public void testCharCastFromString() {
String value = "abc";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.CHAR(3));
BinaryString expected = BinaryString.fromString("abc");
assertThat(result).isEqualTo(expected);
}

@Test
public void testDateCastFromString() {
String value = "2017-12-12 09:30:00.0";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
int expected = 17512;
assertThat(result).isEqualTo(expected);
}

@Test
public void testDateNumericCastFromString() {
String value = "17512";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DATE());
int expected = 17512;
assertThat(result).isEqualTo(expected);
}

@Test
public void testDecimalCastFromString() {
String value = "123";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DECIMAL(5, 0));
Decimal expected = Decimal.fromBigDecimal(new BigDecimal("123"), 5, 0);
assertThat(result).isEqualTo(expected);
}

@Test
public void testDoubleCastFromString() {
String value = "123.456";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.DOUBLE());
Double expected = 123.456;
assertThat(result).isEqualTo(expected);
}

@Test
public void testFloatCastFromString() {
String value = "123.456";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
Float expected = 123.456f;
assertThat(result).isEqualTo(expected);
}

@Test
public void testLargeFloatCastFromString() {
String value = "123.45678";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.FLOAT());
Float expected = 123.45678f;
assertThat(result).isEqualTo(expected);
}

@Test
public void testIntCastFromString() {
String value = "12";
Object result = TypeUtils.castFromString(value, DataTypes.INT());
int expected = 12;
assertThat(result).isEqualTo(expected);
}

@Test
public void testLocalZonedTimestampCastFromString() {
String value = "2017-12-12 09:30:00";
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE());
Timestamp expected =
Timestamp.fromEpochMillis(
LocalDateTime.parse("2017-12-12T09:30:00")
.atZone(TimeZone.getTimeZone("Asia/Tokyo").toZoneId())
.toEpochSecond()
* 1000);
assertThat(result).isEqualTo(expected);
}

@Test
public void testMapStringStringCastFromString() {
String value = "{\"a\":\"b\", \"c\":\"d\"}";
Object result =
TypeUtils.castFromString(
value, DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()));
GenericMap expected =
new GenericMap(
new HashMap<BinaryString, BinaryString>() {
{
put(BinaryString.fromString("a"), BinaryString.fromString("b"));
put(BinaryString.fromString("c"), BinaryString.fromString("d"));
}
});
assertThat(result).isEqualTo(expected);
}

@Test
public void testMapStringIntCastFromString() {
String value = "{\"a\":0, \"c\":1}";
Object result =
TypeUtils.castFromString(value, DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()));
GenericMap expected =
new GenericMap(
new HashMap<BinaryString, Integer>() {
{
put(BinaryString.fromString("a"), 0);
put(BinaryString.fromString("c"), 1);
}
});
assertThat(result).isEqualTo(expected);
}

@Test
public void testRowCastFromString() {
String value = "{\"key1\":{\"nested_key1\":0},\"key2\":\"value\"}";
Object result =
TypeUtils.castFromString(
value,
DataTypes.ROW(
new DataField(
0,
"key1",
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())),
new DataField(1, "key2", DataTypes.STRING())));
GenericRow expected =
GenericRow.of(
new GenericMap(
Collections.singletonMap(
BinaryString.fromString("nested_key1"), 0)),
BinaryString.fromString("value"));
assertThat(result).isEqualTo(expected);
}

@Test
public void testSmallIntCastFromString() {
String value = "12";
Object result = TypeUtils.castFromString(value, DataTypes.SMALLINT());
short expected = 12;
assertThat(result).isEqualTo(expected);
}

@Test
public void testTimestampCastFromString() {
String value = "2017-12-12 09:30:00";
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
Timestamp expected =
Timestamp.fromLocalDateTime(LocalDateTime.parse("2017-12-12T09:30:00"));
assertThat(result).isEqualTo(expected);
}

@Test
public void testTimestampNumericCastFromString() {
String value = "123456789000000";
Object result = TypeUtils.castFromString(value, DataTypes.TIMESTAMP());
Timestamp expected = Timestamp.fromMicros(123456789000000L);
assertThat(result).isEqualTo(expected);
}

@Test
public void testTimeCastFromString() {
String value = "13:09:42.123456+01:00";
Object result = TypeUtils.castFromString(value, DataTypes.TIME(3));
int expected = 14 * 60 * 60 * 1000 + 9 * 60 * 1000 + 42 * 1000 + 123;
assertThat(result).isEqualTo(expected);
}

@Test
public void testTimeNumericCastFromString() {
String value = "123456789";
Object result = TypeUtils.castFromString(value, DataTypes.TIME());
int expected = 123456789;
assertThat(result).isEqualTo(expected);
}

@Test
public void testTinyIntCastFromString() {
String value = "6";
Object result = TypeUtils.castFromString(value, DataTypes.TINYINT());
byte expected = 6;
assertThat(result).isEqualTo(expected);
}

@Test
public void testVarBinaryCastFromString() {
String value = "abc";
Object result = TypeUtils.castFromString(value, DataTypes.VARBINARY(3));
byte[] expected = "abc".getBytes(StandardCharsets.UTF_8);
assertThat(result).isEqualTo(expected);
}

@Test
public void testVarCharCastFromString() {
String value = "abc";
Object result = TypeUtils.castFromCdcValueString(value, DataTypes.VARCHAR(3));
BinaryString expected = BinaryString.fromString("abc");
assertThat(result).isEqualTo(expected);
}
}
2 changes: 1 addition & 1 deletion .../test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ private void testAllTypesImpl() throws Exception {
+ "Paimon , Apache Paimon, Apache Paimon PostgreSQL Test Data, "
+ "[98, 121, 116, 101, 115], "
+ "{\"a\": \"b\"}, "
+ "[\"item1\", \"item2\"]"
+ "[item1, item2]"
+ "]",
"+I["
+ "2, 2.2, "
Expand Down
Loading