Skip to content

Commit 8ea835c

Browse files
jayzhan211jayzhan211
committed
clippy
Signed-off-by: jayzhan211 <jayzhan211@gmail.com>
1 parent 94015e2 commit 8ea835c

3 files changed

Lines changed: 174 additions & 55 deletions

File tree

‎datafusion/core/tests/sqllogictests/test_files/array.slt‎

Lines changed: 63 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -947,25 +947,29 @@ select array_has_all([true, true, false, true, false], [true, false, false]), ar
947947
true true
948948

949949
# array_has_all scalar function #6
950-
query BB
951-
select array_has_all(make_array(true, true, true), make_array(false, false)), array_has_all([false, false, false], [true, true]);
950+
query BBB
951+
select array_has_all(make_array(true, true, true), make_array(false, false)),
952+
array_has_all([false, false, false], [true, true]),
953+
array_has_all(make_array(true, true, true), make_array(true, false));
952954
----
953-
false false
955+
false false false
954956

955-
query ??
956-
select column1, column2 from arrays;
957-
----
958-
[[, 2], [3, ]] [1.1, 2.2, 3.3]
959-
[[3, 4], [5, 6]] [, 5.5, 6.6]
960-
[[5, 6], [7, 8]] [7.7, 8.8, 9.9]
961-
[[7, ], [9, 10]] [10.1, , 12.2]
962-
NULL [13.3, 14.4, 15.5]
963-
[[11, 12], [13, 14]] NULL
964-
[[15, 16], [, 18]] [16.6, 17.7, 18.8]
957+
# For reference
958+
# query ??
959+
# select column1, column2 from arrays;
960+
# ----
961+
# [[, 2], [3, ]] [1.1, 2.2, 3.3]
962+
# [[3, 4], [5, 6]] [, 5.5, 6.6]
963+
# [[5, 6], [7, 8]] [7.7, 8.8, 9.9]
964+
# [[7, ], [9, 10]] [10.1, , 12.2]
965+
# NULL [13.3, 14.4, 15.5]
966+
# [[11, 12], [13, 14]] NULL
967+
# [[15, 16], [, 18]] [16.6, 17.7, 18.8]
965968

966969
# array_has_all column-wise #1
967970
query BB
968-
select array_has_all(column1, make_array(7)), array_has_all(column1, make_array(3, 4, 5, 6)) from arrays;
971+
select array_has_all(column1, make_array(7)),
972+
array_has_all(column1, make_array(3, 4, 5, 6)) from arrays;
969973
----
970974
false false
971975
false true
@@ -976,8 +980,51 @@ false false
976980
false false
977981

978982
# array_has column-wise #1
979-
# query B
980-
# select array_has(column1, 7) from arrays;
983+
query B
984+
select array_has(column1, 7) from arrays;
985+
----
986+
false
987+
false
988+
true
989+
true
990+
false
991+
false
992+
false
993+
994+
# array_has_any column-wise #1
995+
query B
996+
select array_has_any(column1, make_array(3, 5, 7, 11)) from arrays;
997+
----
998+
true
999+
true
1000+
true
1001+
true
1002+
false
1003+
true
1004+
false
1005+
1006+
# array_has_any column-wise #2
1007+
query BBBBB
1008+
select array_has_any(make_array(true, true, true), make_array(false)),
1009+
array_has_any(make_array(true, true, true), make_array(true)),
1010+
array_has_any(make_array(true, false), make_array(true)),
1011+
array_has_any(make_array(true, false), make_array(false)),
1012+
array_has_any(make_array(false, false), make_array(true, false));
1013+
----
1014+
false true true true true
1015+
1016+
# array_has_any column-wise #3
1017+
query BBB
1018+
select array_has_any(column2, make_array(1.0, 1.2, 1.4)),
1019+
array_has_any(column2, make_array(1.1, 1.2, 1.4)),
1020+
array_has_any(column2, make_array(1.1, 16.6, 1.4)) from arrays;
1021+
----
1022+
false true true
1023+
false false false
1024+
false false false
1025+
false false false
1026+
false false false
1027+
false false true
9811028

9821029
### Array operators tests
9831030
## array concatenate operator

‎datafusion/physical-expr/src/array_expressions.rs‎

Lines changed: 106 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,14 +1352,25 @@ macro_rules! contains {
13521352
($FIRST_ARRAY:expr, $SECOND_ARRAY:expr, $ARRAY_TYPE:ident) => {{
13531353
let first_array = downcast_arg!($FIRST_ARRAY, $ARRAY_TYPE);
13541354
let second_array = downcast_arg!($SECOND_ARRAY, $ARRAY_TYPE);
1355-
let mut res = true;
13561355
for x in second_array.values().iter().dedup() {
13571356
if !first_array.values().contains(x) {
1358-
res = false;
1359-
break;
1357+
return Ok(false);
13601358
}
13611359
}
1362-
res
1360+
Ok(true)
1361+
}};
1362+
}
1363+
1364+
macro_rules! overlap {
1365+
($FIRST_ARRAY:expr, $SECOND_ARRAY:expr, $ARRAY_TYPE:ident) => {{
1366+
let first_array = downcast_arg!($FIRST_ARRAY, $ARRAY_TYPE);
1367+
let second_array = downcast_arg!($SECOND_ARRAY, $ARRAY_TYPE);
1368+
for x in second_array.values().iter().dedup() {
1369+
if first_array.values().contains(x) {
1370+
return Ok(true);
1371+
}
1372+
}
1373+
Ok(false)
13631374
}};
13641375
}
13651376

@@ -1374,7 +1385,7 @@ fn flatten_list_array<OffsetSize: OffsetSizeTrait>(
13741385
let (_, offsets, values, _) = list_array.clone().into_parts();
13751386
let arr_offsets = offsets.to_vec();
13761387
let inner_arr = flatten_list_array::<OffsetSize>(values)?;
1377-
let (field, offsets, values, nulls) = inner_arr.clone().into_parts();
1388+
let (field, offsets, values, nulls) = inner_arr.into_parts();
13781389

13791390
let inner_arr_offsets = offsets.to_vec();
13801391
let flatten_offsets: Vec<OffsetSize> = arr_offsets
@@ -1394,41 +1405,59 @@ fn flatten_list_array<OffsetSize: OffsetSizeTrait>(
13941405
}
13951406
_ => Ok(list_array.clone()),
13961407
},
1397-
_ => Err(DataFusionError::Internal(format!("array should be list"))),
1408+
_ => Err(DataFusionError::Internal(
1409+
"array should be list".to_string(),
1410+
)),
13981411
}
13991412
}
14001413

1401-
/// Array_has_any SQL function
1402-
pub fn array_has_any(args: &[ArrayRef]) -> Result<ArrayRef> {
1403-
assert_eq!(args.len(), 2);
1404-
let array = flatten_list_array::<i32>(args[0].clone())?;
1405-
Ok(Arc::new(array) as ArrayRef)
1406-
}
1407-
1408-
/// Array_has SQL function
1409-
pub fn array_has(args: &[ArrayRef]) -> Result<ArrayRef> {
1410-
assert_eq!(args.len(), 2);
1411-
let array = flatten_list_array::<i32>(args[0].clone())?;
1412-
Ok(Arc::new(array) as ArrayRef)
1414+
fn overlap_internal(arr: ArrayRef, sub_arr: ArrayRef) -> Result<bool> {
1415+
match (arr.data_type(), sub_arr.data_type()) {
1416+
(DataType::Utf8, DataType::Utf8) => overlap!(arr, sub_arr, StringArray),
1417+
(DataType::LargeUtf8, DataType::LargeUtf8) => overlap!(arr, sub_arr, LargeStringArray),
1418+
(DataType::Boolean, DataType::Boolean) => {
1419+
let first_array = downcast_arg!(arr, BooleanArray);
1420+
let second_array = downcast_arg!(sub_arr, BooleanArray);
1421+
if second_array.true_count() > 0 && first_array.true_count() > 0 {
1422+
return Ok(true);
1423+
}
1424+
if second_array.false_count() > 0 &&first_array.false_count() > 0 {
1425+
return Ok(true);
1426+
}
1427+
Ok(false)
1428+
}
1429+
(DataType::Float32, DataType::Float32) => overlap!(arr, sub_arr, Float32Array),
1430+
(DataType::Float64, DataType::Float64) => overlap!(arr, sub_arr, Float64Array),
1431+
(DataType::Int8, DataType::Int8) => overlap!(arr, sub_arr, Int8Array),
1432+
(DataType::Int16, DataType::Int16) => overlap!(arr, sub_arr, Int16Array),
1433+
(DataType::Int32, DataType::Int32) => overlap!(arr, sub_arr, Int32Array),
1434+
(DataType::Int64, DataType::Int64) => overlap!(arr, sub_arr, Int64Array),
1435+
(DataType::UInt8, DataType::UInt8) => overlap!(arr, sub_arr, UInt8Array),
1436+
(DataType::UInt16, DataType::UInt16) => overlap!(arr, sub_arr, UInt16Array),
1437+
(DataType::UInt32, DataType::UInt32) => overlap!(arr, sub_arr, UInt32Array),
1438+
(DataType::UInt64, DataType::UInt64) => overlap!(arr, sub_arr, UInt64Array),
1439+
(first_array_data_type, second_array_data_type) => {
1440+
Err(DataFusionError::NotImplemented(format!(
1441+
"Array_has_all is not implemented for types '{first_array_data_type:?}' and '{second_array_data_type:?}'."
1442+
)))
1443+
}
1444+
}
14131445
}
14141446

1415-
/// Array_has_all SQL function
1416-
pub fn array_has_all(args: &[ArrayRef]) -> Result<ArrayRef> {
1417-
assert_eq!(args.len(), 2);
1418-
let array = flatten_list_array::<i32>(args[0].clone())?;
1419-
// TODO: Dont need to flatten rhs array
1420-
let sub_array = flatten_list_array::<i32>(args[1].clone())?;
1421-
let mut boolean_array = Vec::with_capacity(array.len());
1422-
1423-
for (arr, sub_arr) in array.iter().zip(sub_array.iter()) {
1424-
if let (Some(arr), Some(sub_arr)) = (arr, sub_arr) {
1425-
let res = match (arr.data_type(), sub_arr.data_type()) {
1447+
fn contains_internal(arr: ArrayRef, sub_arr: ArrayRef) -> Result<bool> {
1448+
match (arr.data_type(), sub_arr.data_type()) {
14261449
(DataType::Utf8, DataType::Utf8) => contains!(arr, sub_arr, StringArray),
14271450
(DataType::LargeUtf8, DataType::LargeUtf8) => contains!(arr, sub_arr, LargeStringArray),
14281451
(DataType::Boolean, DataType::Boolean) => {
14291452
let first_array = downcast_arg!(arr, BooleanArray);
14301453
let second_array = downcast_arg!(sub_arr, BooleanArray);
1431-
compute::bool_or(first_array) == compute::bool_or(second_array)
1454+
if second_array.true_count() > 0 && first_array.true_count() == 0 {
1455+
return Ok(false);
1456+
}
1457+
if second_array.false_count() > 0 && first_array.false_count() == 0 {
1458+
return Ok(false);
1459+
}
1460+
Ok(true)
14321461
}
14331462
(DataType::Float32, DataType::Float32) => contains!(arr, sub_arr, Float32Array),
14341463
(DataType::Float64, DataType::Float64) => contains!(arr, sub_arr, Float64Array),
@@ -1441,11 +1470,56 @@ pub fn array_has_all(args: &[ArrayRef]) -> Result<ArrayRef> {
14411470
(DataType::UInt32, DataType::UInt32) => contains!(arr, sub_arr, UInt32Array),
14421471
(DataType::UInt64, DataType::UInt64) => contains!(arr, sub_arr, UInt64Array),
14431472
(first_array_data_type, second_array_data_type) => {
1444-
return Err(DataFusionError::NotImplemented(format!(
1473+
Err(DataFusionError::NotImplemented(format!(
14451474
"Array_has_all is not implemented for types '{first_array_data_type:?}' and '{second_array_data_type:?}'."
14461475
)))
14471476
}
1448-
};
1477+
}
1478+
}
1479+
1480+
/// Array_has_any SQL function
1481+
pub fn array_has_any(args: &[ArrayRef]) -> Result<ArrayRef> {
1482+
assert_eq!(args.len(), 2);
1483+
let array = flatten_list_array::<i32>(args[0].clone())?;
1484+
// TODO: Dont need to flatten rhs array
1485+
let sub_array = flatten_list_array::<i32>(args[1].clone())?;
1486+
let mut boolean_array = Vec::with_capacity(array.len());
1487+
1488+
for (arr, sub_arr) in array.iter().zip(sub_array.iter()) {
1489+
if let (Some(arr), Some(sub_arr)) = (arr, sub_arr) {
1490+
let res = overlap_internal(arr.clone(), sub_arr.clone())?;
1491+
boolean_array.push(res);
1492+
}
1493+
}
1494+
Ok(Arc::new(BooleanArray::from(boolean_array)))
1495+
}
1496+
1497+
/// Array_has SQL function
1498+
pub fn array_has(args: &[ArrayRef]) -> Result<ArrayRef> {
1499+
assert_eq!(args.len(), 2);
1500+
let array = flatten_list_array::<i32>(args[0].clone())?;
1501+
let sub_array = args[1].clone();
1502+
let mut boolean_array = Vec::with_capacity(array.len());
1503+
1504+
for arr in array.iter().flatten() {
1505+
let res = contains_internal(arr.clone(), sub_array.clone())?;
1506+
boolean_array.push(res);
1507+
}
1508+
1509+
Ok(Arc::new(BooleanArray::from(boolean_array)))
1510+
}
1511+
1512+
/// Array_has_all SQL function
1513+
pub fn array_has_all(args: &[ArrayRef]) -> Result<ArrayRef> {
1514+
assert_eq!(args.len(), 2);
1515+
let array = flatten_list_array::<i32>(args[0].clone())?;
1516+
// TODO: Dont need to flatten rhs array
1517+
let sub_array = flatten_list_array::<i32>(args[1].clone())?;
1518+
let mut boolean_array = Vec::with_capacity(array.len());
1519+
1520+
for (arr, sub_arr) in array.iter().zip(sub_array.iter()) {
1521+
if let (Some(arr), Some(sub_arr)) = (arr, sub_arr) {
1522+
let res = contains_internal(arr.clone(), sub_arr.clone())?;
14491523
boolean_array.push(res);
14501524
}
14511525
}

‎datafusion/proto/src/logical_plan/from_proto.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,11 @@ use datafusion_common::{
3737
use datafusion_expr::expr::{Alias, Placeholder};
3838
use datafusion_expr::{
3939
abs, acos, acosh, array, array_append, array_concat, array_dims, array_fill,
40-
array_has,
41-
array_has_any,
42-
array_has_all, array_length, array_ndims, array_position, array_positions,
43-
array_prepend, array_remove, array_replace, array_to_string, ascii, asin, asinh,
44-
atan, atan2, atanh, bit_length, btrim, cardinality, cbrt, ceil, character_length,
45-
chr, coalesce, concat_expr, concat_ws_expr, cos, cosh, cot, current_date,
46-
current_time, date_bin, date_part, date_trunc, degrees, digest, exp,
40+
array_has, array_has_all, array_has_any, array_length, array_ndims, array_position,
41+
array_positions, array_prepend, array_remove, array_replace, array_to_string, ascii,
42+
asin, asinh, atan, atan2, atanh, bit_length, btrim, cardinality, cbrt, ceil,
43+
character_length, chr, coalesce, concat_expr, concat_ws_expr, cos, cosh, cot,
44+
current_date, current_time, date_bin, date_part, date_trunc, degrees, digest, exp,
4745
expr::{self, InList, Sort, WindowFunction},
4846
factorial, floor, from_unixtime, gcd, lcm, left, ln, log, log10, log2,
4947
logical_plan::{PlanType, StringifiedPlan},

0 commit comments

Comments
 (0)