Skip to content

Commit debc5bf

Browse files
XiangpengHaoalamb
andauthored
[Parquet] Add projection utility functions (#6931)
* projection utilities * improve docs --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 1f639f6 commit debc5bf

File tree

1 file changed

+101
-0
lines changed

1 file changed

+101
-0
lines changed

parquet/src/arrow/mod.rs

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,45 @@ impl ProjectionMask {
281281
pub fn leaf_included(&self, leaf_idx: usize) -> bool {
282282
self.mask.as_ref().map(|m| m[leaf_idx]).unwrap_or(true)
283283
}
284+
285+
/// Union two projection masks
286+
///
287+
/// Example:
288+
/// ```text
289+
/// mask1 = [true, false, true]
290+
/// mask2 = [false, true, true]
291+
/// union(mask1, mask2) = [true, true, true]
292+
/// ```
293+
pub fn union(&mut self, other: &Self) {
294+
match (self.mask.as_ref(), other.mask.as_ref()) {
295+
(None, _) | (_, None) => self.mask = None,
296+
(Some(a), Some(b)) => {
297+
debug_assert_eq!(a.len(), b.len());
298+
let mask = a.iter().zip(b.iter()).map(|(&a, &b)| a || b).collect();
299+
self.mask = Some(mask);
300+
}
301+
}
302+
}
303+
304+
/// Intersect two projection masks
305+
///
306+
/// Example:
307+
/// ```text
308+
/// mask1 = [true, false, true]
309+
/// mask2 = [false, true, true]
310+
/// intersect(mask1, mask2) = [false, false, true]
311+
/// ```
312+
pub fn intersect(&mut self, other: &Self) {
313+
match (self.mask.as_ref(), other.mask.as_ref()) {
314+
(None, _) => self.mask = other.mask.clone(),
315+
(_, None) => {}
316+
(Some(a), Some(b)) => {
317+
debug_assert_eq!(a.len(), b.len());
318+
let mask = a.iter().zip(b.iter()).map(|(&a, &b)| a && b).collect();
319+
self.mask = Some(mask);
320+
}
321+
}
322+
}
284323
}
285324

286325
/// Lookups up the parquet column by name
@@ -551,4 +590,66 @@ mod test {
551590
let mask = ProjectionMask::columns(&schema, ["a", "e"]);
552591
assert_eq!(mask.mask.unwrap(), [true, false, true, false, true]);
553592
}
593+
594+
#[test]
595+
fn test_projection_mask_union() {
596+
let mut mask1 = ProjectionMask {
597+
mask: Some(vec![true, false, true]),
598+
};
599+
let mask2 = ProjectionMask {
600+
mask: Some(vec![false, true, true]),
601+
};
602+
mask1.union(&mask2);
603+
assert_eq!(mask1.mask, Some(vec![true, true, true]));
604+
605+
let mut mask1 = ProjectionMask { mask: None };
606+
let mask2 = ProjectionMask {
607+
mask: Some(vec![false, true, true]),
608+
};
609+
mask1.union(&mask2);
610+
assert_eq!(mask1.mask, None);
611+
612+
let mut mask1 = ProjectionMask {
613+
mask: Some(vec![true, false, true]),
614+
};
615+
let mask2 = ProjectionMask { mask: None };
616+
mask1.union(&mask2);
617+
assert_eq!(mask1.mask, None);
618+
619+
let mut mask1 = ProjectionMask { mask: None };
620+
let mask2 = ProjectionMask { mask: None };
621+
mask1.union(&mask2);
622+
assert_eq!(mask1.mask, None);
623+
}
624+
625+
#[test]
626+
fn test_projection_mask_intersect() {
627+
let mut mask1 = ProjectionMask {
628+
mask: Some(vec![true, false, true]),
629+
};
630+
let mask2 = ProjectionMask {
631+
mask: Some(vec![false, true, true]),
632+
};
633+
mask1.intersect(&mask2);
634+
assert_eq!(mask1.mask, Some(vec![false, false, true]));
635+
636+
let mut mask1 = ProjectionMask { mask: None };
637+
let mask2 = ProjectionMask {
638+
mask: Some(vec![false, true, true]),
639+
};
640+
mask1.intersect(&mask2);
641+
assert_eq!(mask1.mask, Some(vec![false, true, true]));
642+
643+
let mut mask1 = ProjectionMask {
644+
mask: Some(vec![true, false, true]),
645+
};
646+
let mask2 = ProjectionMask { mask: None };
647+
mask1.intersect(&mask2);
648+
assert_eq!(mask1.mask, Some(vec![true, false, true]));
649+
650+
let mut mask1 = ProjectionMask { mask: None };
651+
let mask2 = ProjectionMask { mask: None };
652+
mask1.intersect(&mask2);
653+
assert_eq!(mask1.mask, None);
654+
}
554655
}

0 commit comments

Comments
 (0)