Skip to content

Commit b970752

Browse files
jpschorrjpschorr
authored
Update ion tests and add null & nan comparisions (#543)
* Update ion tests and add null & nan comparisions * Fix deep-equality as per specification of `eqg`
1 parent f684ea9 commit b970752

File tree

6 files changed

+230
-44
lines changed

6 files changed

+230
-44
lines changed

‎extension/partiql-extension-ion/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ unicase = "2.7"
3434
rust_decimal = { version = "1.36.0", default-features = false, features = ["std"] }
3535
rust_decimal_macros = "1.36"
3636
ion-rs_old = { version = "0.18", package = "ion-rs" }
37-
ion-rs = { version = "1.0.0-rc.11", features = ["experimental"] }
37+
ion-rs = { version = "1.0.0-rc.11", features = ["experimental", "experimental-ion-hash", "sha2"] }
3838

3939
time = { version = "0.3", features = ["macros"] }
4040
once_cell = "1"

‎extension/partiql-extension-ion/src/boxed_ion.rs

Lines changed: 188 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
use crate::util::{PartiqlValueTarget, ToPartiqlValue};
22
use ion_rs::{
3-
AnyEncoding, Element, ElementReader, IonResult, IonType, OwnedSequenceIterator, Reader,
4-
Sequence,
3+
AnyEncoding, Element, ElementReader, IonData, IonResult, IonType, OwnedSequenceIterator,
4+
Reader, Sequence, Struct, Symbol,
55
};
6+
use itertools::Itertools;
67
use partiql_value::boxed_variant::{
78
BoxedVariant, BoxedVariantResult, BoxedVariantType, BoxedVariantTypeTag,
89
BoxedVariantValueIntoIterator, DynBoxedVariant,
@@ -12,14 +13,14 @@ use partiql_value::datum::{
1213
DatumSeqRef, DatumTupleOwned, DatumTupleRef, DatumValueOwned, DatumValueRef, OwnedSequenceView,
1314
OwnedTupleView, RefSequenceView, RefTupleView, SequenceDatum, TupleDatum,
1415
};
15-
use partiql_value::{Bag, BindingsName, List, Tuple, Value, Variant};
16+
use partiql_value::{Bag, BindingsName, List, NullableEq, Tuple, Value, Variant};
1617
use peekmore::{PeekMore, PeekMoreIterator};
1718
#[cfg(feature = "serde")]
1819
use serde::{Deserialize, Deserializer, Serialize, Serializer};
1920
use std::any::Any;
2021
use std::borrow::Cow;
2122
use std::cell::RefCell;
22-
use std::cmp::Ordering;
23+
use std::collections::HashMap;
2324
use std::fmt::{Debug, Display, Formatter};
2425
use std::hash::{Hash, Hasher};
2526
use std::ops::DerefMut;
@@ -40,16 +41,33 @@ impl BoxedVariantType for BoxedIonType {
4041
}
4142

4243
fn value_eq(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> bool {
43-
let (l, r) = get_values(l, r);
44-
45-
l.eq(r)
44+
wrap_eq::<true, false>(l, r) == Value::Boolean(true)
45+
}
46+
47+
fn value_eq_param(
48+
&self,
49+
l: &DynBoxedVariant,
50+
r: &DynBoxedVariant,
51+
nulls_eq: bool,
52+
nans_eq: bool,
53+
) -> bool {
54+
let res = match (nulls_eq, nans_eq) {
55+
(true, true) => wrap_eq::<true, true>(l, r),
56+
(true, false) => wrap_eq::<true, false>(l, r),
57+
(false, true) => wrap_eq::<false, true>(l, r),
58+
(false, false) => wrap_eq::<false, false>(l, r),
59+
};
60+
res == Value::Boolean(true)
4661
}
62+
}
4763

48-
fn value_cmp(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> Ordering {
49-
let (l, r) = get_values(l, r);
50-
51-
l.cmp(r)
52-
}
64+
fn wrap_eq<const NULLS_EQUAL: bool, const NAN_EQUAL: bool>(
65+
l: &DynBoxedVariant,
66+
r: &DynBoxedVariant,
67+
) -> Value {
68+
let (l, r) = get_values(l, r);
69+
let wrap = IonEqualityValue::<'_, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
70+
NullableEq::eq(&wrap(l), &wrap(r))
5371
}
5472

5573
#[inline]
@@ -139,8 +157,8 @@ impl<'de> Deserialize<'de> for BoxedIon {
139157
}
140158

141159
impl Hash for BoxedIon {
142-
fn hash<H: Hasher>(&self, _: &mut H) {
143-
todo!("BoxedIon.hash")
160+
fn hash<H: Hasher>(&self, state: &mut H) {
161+
self.doc.hash(state);
144162
}
145163
}
146164

@@ -206,23 +224,21 @@ impl BoxedVariant for BoxedIon {
206224
}
207225
}
208226

209-
impl PartialEq<Self> for BoxedIon {
210-
fn eq(&self, other: &Self) -> bool {
211-
self.doc.eq(&other.doc)
212-
}
213-
}
214-
215-
impl Eq for BoxedIon {}
227+
/// A wrapper on [`T`] that specifies if missing and null values should be equal.
228+
#[derive(Eq, PartialEq)]
229+
pub struct IonEqualityValue<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool, T>(pub &'a T);
216230

217-
impl PartialOrd for BoxedIon {
218-
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
219-
Some(self.cmp(other))
231+
impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
232+
for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, BoxedIon>
233+
{
234+
fn eq(&self, rhs: &Self) -> Value {
235+
let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
236+
NullableEq::eq(&wrap(&self.0.doc), &wrap(&rhs.0.doc))
220237
}
221-
}
222-
impl Ord for BoxedIon {
223-
fn cmp(&self, other: &Self) -> Ordering {
224-
// TODO lowering just to compare is costly... Either find a better way, or lift this out of the extension
225-
self.lower().unwrap().cmp(&other.lower().unwrap())
238+
#[inline(always)]
239+
fn eqg(&self, rhs: &Self) -> Value {
240+
let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
241+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
226242
}
227243
}
228244

@@ -531,17 +547,153 @@ enum BoxedIonValue {
531547
Sequence(Sequence),
532548
}
533549

534-
impl PartialEq<Self> for BoxedIonValue {
535-
fn eq(&self, other: &Self) -> bool {
536-
match (self, other) {
537-
(BoxedIonValue::Value(l), BoxedIonValue::Value(r)) => l == r,
538-
(BoxedIonValue::Sequence(l), BoxedIonValue::Sequence(r)) => l == r,
539-
_ => false,
550+
impl Hash for BoxedIonValue {
551+
fn hash<H: Hasher>(&self, state: &mut H) {
552+
match self {
553+
BoxedIonValue::Stream() => {
554+
todo!("stream not hashable? ")
555+
}
556+
BoxedIonValue::Value(val) => {
557+
let sha = ion_rs::ion_hash::sha256(val).expect("ion hash");
558+
state.write(&sha);
559+
}
560+
BoxedIonValue::Sequence(_) => todo!("ion seq hash"),
540561
}
541562
}
542563
}
543564

544-
impl Eq for BoxedIonValue {}
565+
impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
566+
for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, BoxedIonValue>
567+
{
568+
#[inline(always)]
569+
fn eq(&self, other: &Self) -> Value {
570+
let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Element>;
571+
let wrap_seq = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Sequence>;
572+
match (self.0, other.0) {
573+
(BoxedIonValue::Value(l), BoxedIonValue::Value(r)) => {
574+
NullableEq::eq(&wrap(l), &wrap(r))
575+
}
576+
(BoxedIonValue::Sequence(l), BoxedIonValue::Sequence(r)) => {
577+
NullableEq::eq(&wrap_seq(l), &wrap_seq(r))
578+
}
579+
_ => Value::Boolean(false),
580+
}
581+
}
582+
583+
#[inline(always)]
584+
fn eqg(&self, rhs: &Self) -> Value {
585+
let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
586+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
587+
}
588+
}
589+
590+
impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
591+
for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Element>
592+
{
593+
fn eq(&self, other: &Self) -> Value {
594+
let wrap_seq = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Sequence>;
595+
let wrap_struct = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, Struct>;
596+
let (l, r) = (self.0, other.0);
597+
let (lty, rty) = (l.ion_type(), r.ion_type());
598+
599+
let result = if l.is_null() && r.is_null() {
600+
NULLS_EQUAL && l.annotations().eq(r.annotations())
601+
} else {
602+
match (lty, rty) {
603+
(IonType::Float, IonType::Float) => {
604+
let (lf, rf) = (l.as_float().unwrap(), r.as_float().unwrap());
605+
if lf.is_nan() && rf.is_nan() {
606+
NAN_EQUAL && l.annotations().eq(r.annotations())
607+
} else {
608+
lf == rf
609+
}
610+
}
611+
612+
(IonType::List, IonType::List) => {
613+
let (ls, rs) = (l.as_list().unwrap(), r.as_list().unwrap());
614+
l.annotations().eq(r.annotations())
615+
&& NullableEq::eq(&wrap_seq(ls), &wrap_seq(rs)) == Value::Boolean(true)
616+
}
617+
(IonType::SExp, IonType::SExp) => {
618+
let (ls, rs) = (l.as_sexp().unwrap(), r.as_sexp().unwrap());
619+
l.annotations().eq(r.annotations())
620+
&& NullableEq::eq(&wrap_seq(ls), &wrap_seq(rs)) == Value::Boolean(true)
621+
}
622+
623+
(IonType::Struct, IonType::Struct) => {
624+
let (ls, rs) = (l.as_struct().unwrap(), r.as_struct().unwrap());
625+
l.annotations().eq(r.annotations())
626+
&& NullableEq::eq(&wrap_struct(ls), &wrap_struct(rs))
627+
== Value::Boolean(true)
628+
}
629+
630+
_ => l == r,
631+
}
632+
};
633+
634+
Value::Boolean(result)
635+
}
636+
637+
#[inline(always)]
638+
fn eqg(&self, rhs: &Self) -> Value {
639+
let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
640+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
641+
}
642+
}
643+
644+
impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
645+
for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Sequence>
646+
{
647+
fn eq(&self, other: &Self) -> Value {
648+
let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
649+
let (l, r) = (self.0, other.0);
650+
let l = l.iter().map(wrap);
651+
let r = r.iter().map(wrap);
652+
let res = l.zip(r).all(|(l, r)| l.eqg(&r) == Value::Boolean(true));
653+
Value::Boolean(res)
654+
}
655+
656+
#[inline(always)]
657+
fn eqg(&self, rhs: &Self) -> Value {
658+
let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
659+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
660+
}
661+
}
662+
663+
impl<'a, const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
664+
for IonEqualityValue<'a, NULLS_EQUAL, NAN_EQUAL, Struct>
665+
{
666+
fn eq(&self, other: &Self) -> Value {
667+
if self.0.len() != other.0.len() {
668+
return Value::Boolean(false);
669+
}
670+
671+
let (l, r) = (self.0, other.0);
672+
let l = l.iter();
673+
let r = r.iter();
674+
675+
let sort_fn = |(ls, le): &(&Symbol, &Element), (rs, re): &(&Symbol, &Element)| {
676+
ls.cmp(rs).then(IonData::from(le).cmp(&IonData::from(re)))
677+
};
678+
for ((ls, lv), (rs, rv)) in l.sorted_by(sort_fn).zip(r.sorted_by(sort_fn)) {
679+
if ls != rs {
680+
return Value::Boolean(false);
681+
}
682+
683+
let wrap = IonEqualityValue::<'a, { NULLS_EQUAL }, { NAN_EQUAL }, _>;
684+
if NullableEq::eqg(&wrap(lv), &wrap(rv)) != Value::Boolean(true) {
685+
return Value::Boolean(false);
686+
}
687+
}
688+
Value::Boolean(true)
689+
}
690+
691+
#[inline(always)]
692+
fn eqg(&self, rhs: &Self) -> Value {
693+
let wrap = IonEqualityValue::<'_, true, { NAN_EQUAL }, _>;
694+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
695+
}
696+
}
545697

546698
impl From<Element> for BoxedIonValue {
547699
fn from(value: Element) -> Self {

‎partiql-conformance-tests/partiql-tests

‎partiql-value/src/boxed_variant.rs

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ use dyn_clone::DynClone;
22
use dyn_hash::DynHash;
33
use partiql_common::pretty::PrettyDoc;
44
use std::any::Any;
5+
use std::borrow::Cow;
56
use std::cmp::Ordering;
67
use std::error::Error;
78

@@ -28,7 +29,14 @@ pub trait BoxedVariantType: Debug + DynClone {
2829
fn name(&self) -> &'static str;
2930

3031
fn value_eq(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> bool;
31-
fn value_cmp(&self, l: &DynBoxedVariant, r: &DynBoxedVariant) -> Ordering;
32+
33+
fn value_eq_param(
34+
&self,
35+
l: &DynBoxedVariant,
36+
r: &DynBoxedVariant,
37+
nulls_eq: bool,
38+
nans_eq: bool,
39+
) -> bool;
3240
}
3341

3442
dyn_clone::clone_trait_object!(BoxedVariantType);
@@ -90,9 +98,12 @@ impl PartialOrd for DynBoxedVariant {
9098
}
9199
impl Ord for DynBoxedVariant {
92100
fn cmp(&self, other: &Self) -> Ordering {
93-
self.type_tag()
94-
.cmp(&other.type_tag())
95-
.then_with(|| self.type_tag().value_cmp(self, other))
101+
let missing = |_| Cow::Owned(Value::Missing);
102+
self.type_tag().cmp(&other.type_tag()).then_with(|| {
103+
self.lower()
104+
.unwrap_or_else(missing)
105+
.cmp(&other.lower().unwrap_or_else(missing))
106+
})
96107
}
97108
}
98109

‎partiql-value/src/comparison.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
use crate::Value;
21
use crate::{util, Bag, List, Tuple};
2+
use crate::{Value, Variant};
33

44
pub trait Comparable {
55
fn is_comparable_to(&self, rhs: &Self) -> bool;
@@ -72,6 +72,7 @@ impl<const GROUP_NULLS: bool, const NAN_EQUAL: bool> NullableEq
7272
let wrap_list = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, List>;
7373
let wrap_bag = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Bag>;
7474
let wrap_tuple = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Tuple>;
75+
let wrap_var = EqualityValue::<'_, { GROUP_NULLS }, { NAN_EQUAL }, Variant>;
7576
if GROUP_NULLS {
7677
if let (Value::Missing | Value::Null, Value::Missing | Value::Null) = (self.0, rhs.0) {
7778
return Value::Boolean(true);
@@ -110,6 +111,7 @@ impl<const GROUP_NULLS: bool, const NAN_EQUAL: bool> NullableEq
110111
(Value::List(l), Value::List(r)) => NullableEq::eq(&wrap_list(l), &wrap_list(r)),
111112
(Value::Bag(l), Value::Bag(r)) => NullableEq::eq(&wrap_bag(l), &wrap_bag(r)),
112113
(Value::Tuple(l), Value::Tuple(r)) => NullableEq::eq(&wrap_tuple(l), &wrap_tuple(r)),
114+
(Value::Variant(l), Value::Variant(r)) => NullableEq::eq(&wrap_var(l), &wrap_var(r)),
113115
(_, _) => Value::from(self.0 == rhs.0),
114116
}
115117
}

‎partiql-value/src/variant.rs

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::datum::{
77
DatumValue,
88
};
99

10-
use crate::{Comparable, NullSortedValue, Value};
10+
use crate::{Comparable, EqualityValue, NullSortedValue, NullableEq, Value};
1111
use delegate::delegate;
1212
use partiql_common::pretty::{pretty_surrounded_doc, PrettyDoc, ToPretty};
1313
use pretty::{DocAllocator, DocBuilder};
@@ -202,6 +202,27 @@ impl PartialEq<Self> for Variant {
202202

203203
impl Eq for Variant {}
204204

205+
impl<const NULLS_EQUAL: bool, const NAN_EQUAL: bool> NullableEq
206+
for EqualityValue<'_, NULLS_EQUAL, NAN_EQUAL, Variant>
207+
{
208+
#[inline(always)]
209+
fn eq(&self, other: &Self) -> Value {
210+
let l = &self.0.variant;
211+
let r = &other.0.variant;
212+
let lty = l.type_tag();
213+
let rty = r.type_tag();
214+
215+
let res = lty == rty && lty.value_eq_param(l, r, NULLS_EQUAL, NAN_EQUAL);
216+
Value::Boolean(res)
217+
}
218+
219+
#[inline(always)]
220+
fn eqg(&self, rhs: &Self) -> Value {
221+
let wrap = EqualityValue::<'_, true, { NAN_EQUAL }, _>;
222+
NullableEq::eq(&wrap(self.0), &wrap(rhs.0))
223+
}
224+
}
225+
205226
#[cfg(feature = "serde")]
206227
impl Serialize for Variant {
207228
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>

0 commit comments

Comments
 (0)