Skip to content

fix: list inner type for json and valueref, refactor type to ref for struct/list #7113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 21, 2025
Merged

fix: list inner type for json and valueref, refactor type to ref for struct/list #7113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c31a36c
refactor: use arc for struct type
sunng87 Oct 18, 2025
fb4a95c
fix: inner type of list value and ref
sunng87 Oct 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions src/api/src/helper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

use std::collections::HashSet;
use std::sync::Arc;

use common_decimal::Decimal128;
use common_decimal::decimal128::{DECIMAL128_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION};
Expand Down Expand Up @@ -185,7 +186,7 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
datatype: d.datatype(),
datatype_ext: d.datatype_extension.clone().map(|d| *d),
};
ConcreteDataType::list_datatype(item_type.into())
ConcreteDataType::list_datatype(Arc::new(item_type.into()))
} else {
// invalid state: type extension not found
ConcreteDataType::null_datatype()
Expand All @@ -208,7 +209,7 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
StructField::new(f.name.clone(), field_type.into(), true)
})
.collect::<Vec<_>>();
ConcreteDataType::struct_datatype(StructType::from(fields))
ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields)))
} else {
// invalid state: type extension not found
ConcreteDataType::null_datatype()
Expand Down Expand Up @@ -445,7 +446,7 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
ColumnDataType::Struct => {
if let Some(struct_type) = datatype.as_struct() {
let mut fields = Vec::with_capacity(struct_type.fields().len());
for field in struct_type.fields() {
for field in struct_type.fields().iter() {
let field_type =
ColumnDataTypeWrapper::try_from(field.data_type().clone())?;
let proto_field = crate::v1::StructField {
Expand Down Expand Up @@ -755,7 +756,7 @@ pub fn pb_value_to_value_ref<'a>(

let list_value = ListValueRef::RefList {
val: items,
item_datatype: item_type.clone(),
item_datatype: Arc::new(item_type.clone()),
};
ValueRef::List(list_value)
}
Expand Down Expand Up @@ -794,7 +795,7 @@ pub fn pb_value_to_value_ref<'a>(

let struct_value_ref = StructValueRef::RefList {
val: items,
fields: StructType::new(struct_fields),
fields: StructType::new(Arc::new(struct_fields)),
};
ValueRef::Struct(struct_value_ref)
}
Expand Down Expand Up @@ -1351,10 +1352,10 @@ mod tests {
ColumnDataTypeWrapper::vector_datatype(3).into()
);
assert_eq!(
ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()),
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::string_datatype())),
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::string_datatype()).into()
);
let struct_type = StructType::new(vec![
let struct_type = StructType::new(Arc::new(vec![
StructField::new("id".to_string(), ConcreteDataType::int64_datatype(), true),
StructField::new(
"name".to_string(),
Expand All @@ -1367,7 +1368,7 @@ mod tests {
ConcreteDataType::string_datatype(),
true,
),
]);
]));
assert_eq!(
ConcreteDataType::struct_datatype(struct_type.clone()),
ColumnDataTypeWrapper::struct_datatype(vec![
Expand Down Expand Up @@ -1534,7 +1535,7 @@ mod tests {

assert_eq!(
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::int16_datatype()),
ConcreteDataType::list_datatype(ConcreteDataType::int16_datatype())
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int16_datatype()))
.try_into()
.expect("Failed to create column datatype from List(ListType { item_type: Int16(Int16Type) })")
);
Expand All @@ -1547,16 +1548,16 @@ mod tests {
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::string_datatype())
)
]),
ConcreteDataType::struct_datatype(StructType::new(vec![
ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![
StructField::new("a".to_string(), ConcreteDataType::int64_datatype(), true),
StructField::new(
"a.a".to_string(),
ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()), true
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::string_datatype())), true
)
])).try_into().expect("Failed to create column datatype from Struct(StructType { fields: [StructField { name: \"a\", data_type: Int64(Int64Type) }, StructField { name: \"a.a\", data_type: List(ListType { item_type: String(StringType) }) }] })")
]))).try_into().expect("Failed to create column datatype from Struct(StructType { fields: [StructField { name: \"a\", data_type: Int64(Int64Type) }, StructField { name: \"a.a\", data_type: List(ListType { item_type: String(StringType) }) }] })")
);

let struct_type = StructType::new(vec![
let struct_type = StructType::new(Arc::new(vec![
StructField::new("id".to_string(), ConcreteDataType::int64_datatype(), true),
StructField::new(
"name".to_string(),
Expand All @@ -1569,7 +1570,7 @@ mod tests {
ConcreteDataType::string_datatype(),
true,
),
]);
]));
assert_eq!(
ColumnDataTypeWrapper::new(
ColumnDataType::Json,
Expand Down Expand Up @@ -1736,7 +1737,7 @@ mod tests {
fn test_list_to_pb_value() {
let value = Value::List(ListValue::new(
vec![Value::Boolean(true)],
ConcreteDataType::boolean_datatype(),
Arc::new(ConcreteDataType::boolean_datatype()),
));

let pb_value = to_proto_value(value);
Expand All @@ -1756,14 +1757,14 @@ mod tests {
let value = Value::Struct(
StructValue::try_new(
items,
StructType::new(vec![
StructType::new(Arc::new(vec![
StructField::new(
"a.a".to_string(),
ConcreteDataType::boolean_datatype(),
true,
),
StructField::new("a.b".to_string(), ConcreteDataType::string_datatype(), true),
]),
])),
)
.unwrap(),
);
Expand Down
30 changes: 17 additions & 13 deletions src/datatypes/src/data_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,9 +456,9 @@ impl TryFrom<&ArrowDataType> for ConcreteDataType {
}
ArrowDataType::Utf8 | ArrowDataType::Utf8View => Self::string_datatype(),
ArrowDataType::LargeUtf8 => Self::large_string_datatype(),
ArrowDataType::List(field) => Self::List(ListType::new(
ArrowDataType::List(field) => Self::List(ListType::new(Arc::new(
ConcreteDataType::from_arrow_type(field.data_type()),
)),
))),
ArrowDataType::Dictionary(key_type, value_type) => {
let key_type = ConcreteDataType::from_arrow_type(key_type);
let value_type = ConcreteDataType::from_arrow_type(value_type);
Expand Down Expand Up @@ -641,7 +641,7 @@ impl ConcreteDataType {
}
}

pub fn list_datatype(item_type: ConcreteDataType) -> ConcreteDataType {
pub fn list_datatype(item_type: Arc<ConcreteDataType>) -> ConcreteDataType {
ConcreteDataType::List(ListType::new(item_type))
}

Expand Down Expand Up @@ -794,7 +794,7 @@ mod tests {
ArrowDataType::Int32,
true,
)))),
ConcreteDataType::List(ListType::new(ConcreteDataType::int32_datatype()))
ConcreteDataType::List(ListType::new(Arc::new(ConcreteDataType::int32_datatype())))
);
assert!(matches!(
ConcreteDataType::from_arrow_type(&ArrowDataType::Date32),
Expand Down Expand Up @@ -985,9 +985,10 @@ mod tests {

#[test]
fn test_as_list() {
let list_type = ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype());
let list_type =
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()));
assert_eq!(
ListType::new(ConcreteDataType::int32_datatype()),
ListType::new(Arc::new(ConcreteDataType::int32_datatype())),
*list_type.as_list().unwrap()
);
assert!(ConcreteDataType::int32_datatype().as_list().is_none());
Expand Down Expand Up @@ -1032,21 +1033,24 @@ mod tests {
);
// Nested types
assert_eq!(
ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype()).to_string(),
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()))
.to_string(),
"List<Int32>"
);
assert_eq!(
ConcreteDataType::list_datatype(ConcreteDataType::Dictionary(DictionaryType::new(
ConcreteDataType::int32_datatype(),
ConcreteDataType::string_datatype()
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::Dictionary(
DictionaryType::new(
ConcreteDataType::int32_datatype(),
ConcreteDataType::string_datatype()
)
)))
.to_string(),
"List<Dictionary<Int32, String>>"
);
assert_eq!(
ConcreteDataType::list_datatype(ConcreteDataType::list_datatype(
ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype())
))
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::list_datatype(Arc::new(
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()))
))))
.to_string(),
"List<List<List<Int32>>>"
);
Expand Down
Loading