Skip to content

Add iai versions of the multi-like evaluation benchmarks #296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+454 −314
Draft

Add iai versions of the multi-like evaluation benchmarks #296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion deny.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,5 +152,9 @@ unknown-git = "deny"
# if not specified. If it is specified but empty, no registries are allowed.
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# List of URLs for allowed Git repositories
allow-git = []
allow-git = [
# Allow running iai benchmarks on MacOS
# Cf. "https://github.com/bheisler/iai/issues/25#issuecomment-1378856308"
"https://github.com/bheisler/iai.git",
]

6 changes: 6 additions & 0 deletions partiql/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,14 @@ partiql-eval = { path = "../partiql-eval" }

itertools = "0.10"
criterion = "0.4"
iai = { git = "https://github.com/bheisler/iai" }
rand = "0.8"
once_cell = "1.17"

[[bench]]
name = "bench_eval_multi_like"
harness = false

[[bench]]
name = "bench_eval_multi_like_iai"
harness = false
317 changes: 4 additions & 313 deletions partiql/benches/bench_eval_multi_like.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
use std::time::Duration;

use criterion::{black_box, criterion_group, criterion_main, Criterion};
use itertools::Itertools;
use rand::{Rng, SeedableRng};

use partiql_eval::env::basic::MapBindings;
use partiql_eval::eval::EvalPlan;
use partiql_eval::plan::EvaluatorPlanner;
use partiql_logical::{BindingsOp, LogicalPlan};

use crate::multi_like_data::{employee_data, QUERY_1, QUERY_15, QUERY_30};
use partiql_parser::{Parser, ParserResult};
use partiql_value::{partiql_tuple, Bag, Tuple, Value};
use partiql_value::Value;

// Benchmarks:
// - parsing,
Expand All @@ -21,315 +20,7 @@ use partiql_value::{partiql_tuple, Bag, Tuple, Value};
// of queries that filter against 1, 15, or 30 `OR`ed `LIKE` expressions
// over 10201 rows of tuples containing an id and a string

fn employee_data() -> Vec<Value> {
let name1 = vec![
"Bob",
"Madden",
"Brycen",
"Bryanna",
"Zayne",
"Jocelynn",
"Breanna",
"Margaret",
"Jasmine",
"Kenyon",
"Aryanna",
"Zackery",
"Jorden",
"Malia",
"Raven",
"Neveah",
"Finley",
"Austin",
"Jaxson",
"Tobias",
"Dominique",
"Devan",
"Colby",
"Tanner",
"Mckenna",
"Kristina",
"Cristal",
"River",
"Taliyah",
"Abagail",
"Spencer",
"Gage",
"Ronnie",
"Amari",
"Jabari",
"Alanna",
"Anderson",
"Saniya",
"Baylee",
"Elisa",
"Savannah",
"Jakobe",
"Sandra",
"Simone",
"Frank",
"Braedon",
"Clark",
"Francisco",
"Roman",
"Matias",
"Messi",
"Elisha",
"Alexander",
"Kadence",
"Karsyn",
"Adonis",
"Ishaan",
"Trevon",
"Ryan",
"Jaelynn",
"Marilyn",
"Emma",
"Avah",
"Jordan",
"Riley",
"Amelie",
"Denisse",
"Darion",
"Lydia",
"Marley",
"Brogan",
"Trace",
"Maeve",
"Elijah",
"Kareem",
"Erick",
"Hope",
"Elisabeth",
"Antwan",
"Francesca",
"Layla",
"Jase",
"Angel",
"Addyson",
"Mckinley",
"Julianna",
"Winston",
"Royce",
"Paola",
"Issac",
"Zachary",
"Niko",
"Shania",
"Colin",
"Jesse",
"Pedro",
"Cheyenne",
"Ashley",
"Karli",
"Bianca",
"Mario",
];
let name2 = vec![
"Smith",
"Oconnell",
"Whitehead",
"Carrillo",
"Parrish",
"Monroe",
"Summers",
"Hurst",
"Durham",
"Hardin",
"Hunt",
"Mitchell",
"Pennington",
"Woodward",
"Franklin",
"Martinez",
"Shepard",
"Khan",
"Mcfarland",
"Frey",
"Mckenzie",
"Blair",
"Mercer",
"Callahan",
"Cameron",
"Gilmore",
"Bowers",
"Donovan",
"Meyers",
"Horne",
"Rice",
"Castillo",
"Cain",
"Dickson",
"Valenzuela",
"Silva",
"Prince",
"Vance",
"Berry",
"Coffey",
"Young",
"Walker",
"Burch",
"Ross",
"Mejia",
"Zuniga",
"Haney",
"Jordan",
"Love",
"Larsen",
"Bowman",
"Werner",
"Greer",
"Krause",
"Bishop",
"Day",
"Luna",
"Patrick",
"Adkins",
"Benson",
"Mcconnell",
"Sanchez",
"Villa",
"Wu",
"Duke",
"Fisher",
"Hess",
"Lawrence",
"Perry",
"Hardy",
"Wyatt",
"Mcknight",
"Thomas",
"Trevino",
"Flowers",
"Cisneros",
"Coleman",
"Sanders",
"Good",
"Newton",
"Carpenter",
"Garza",
"Barber",
"Swanson",
"Owen",
"Anderson",
"Bright",
"Beck",
"Lawson",
"Jones",
"Davila",
"Porter",
"Dougherty",
"Stevenson",
"Malone",
"Garrison",
"Bates",
"Wheeler",
"Petty",
"Rojas",
"Townsend",
];

// cartesian product of name1 x name2 (e.g., "Bob Smith", ... "Mario Townsend")
let combined = name1
.iter()
.cartesian_product(name2.iter())
.map(|(n1, n2)| format!("{n1} {n2}"));

// seed the rng with a known value to assure same data across runs
let mut rng = rand::rngs::StdRng::from_seed([42; 32]);
use rand::distributions::Distribution;
let chars = rand::distributions::Alphanumeric;
let random_size = rand::distributions::uniform::Uniform::from(5..=100);

// add random string prefix and suffix to each combined name
let employee_data: Vec<Value> = combined
.enumerate()
.map(|(id, person)| {
let prefix_size = random_size.sample(&mut rng);
let suffix_size = random_size.sample(&mut rng);
let prefix: String = (0..prefix_size)
.map(|_| rng.sample(chars) as char)
.collect();
let suffix: String = (0..suffix_size)
.map(|_| rng.sample(chars) as char)
.collect();
let full_name = format!("{prefix} {person} {suffix}");
partiql_tuple![("id", id), ("name", full_name)].into()
})
.collect_vec();

employee_data
}

fn data() -> MapBindings<Value> {
let data = partiql_tuple![(
"hr",
partiql_tuple![("employees", Bag::from(employee_data()))]
)];

data.into()
}

const QUERY_1: &str = "
SELECT *
FROM hr.employees as emp
WHERE lower(emp.name) LIKE '%bob smith%'
";

const QUERY_15: &str = "
SELECT *
FROM hr.employees as emp
WHERE lower(emp.name) LIKE '%bob smith%'
OR lower(emp.name) LIKE '%gage swanson%'
OR lower(emp.name) LIKE '%riley perry%'
OR lower(emp.name) LIKE '%sandra woodward%'
OR lower(emp.name) LIKE '%abagail oconnell%'
OR lower(emp.name) LIKE '%amari duke%'
OR lower(emp.name) LIKE '%elisha wyatt%'
OR lower(emp.name) LIKE '%aryanna hess%'
OR lower(emp.name) LIKE '%bryanna jones%'
OR lower(emp.name) LIKE '%trace gilmore%'
OR lower(emp.name) LIKE '%antwan stevenson%'
OR lower(emp.name) LIKE '%julianna callahan%'
OR lower(emp.name) LIKE '%jaelynn trevino%'
OR lower(emp.name) LIKE '%kadence bates%'
OR lower(emp.name) LIKE '%jakobe townsend%'
";

const QUERY_30: &str = "
SELECT *
FROM hr.employees as emp
WHERE lower(emp.name) LIKE '%bob smith%'
OR lower(emp.name) LIKE '%gage swanson%'
OR lower(emp.name) LIKE '%riley perry%'
OR lower(emp.name) LIKE '%sandra woodward%'
OR lower(emp.name) LIKE '%abagail oconnell%'
OR lower(emp.name) LIKE '%amari duke%'
OR lower(emp.name) LIKE '%elisha wyatt%'
OR lower(emp.name) LIKE '%aryanna hess%'
OR lower(emp.name) LIKE '%bryanna jones%'
OR lower(emp.name) LIKE '%trace gilmore%'
OR lower(emp.name) LIKE '%antwan stevenson%'
OR lower(emp.name) LIKE '%julianna callahan%'
OR lower(emp.name) LIKE '%jaelynn trevino%'
OR lower(emp.name) LIKE '%kadence bates%'
OR lower(emp.name) LIKE '%jakobe townsend%'
OR lower(emp.name) LIKE '%austin pennington%'
OR lower(emp.name) LIKE '%colby woodward%'
OR lower(emp.name) LIKE '%brycen blair%'
OR lower(emp.name) LIKE '%cristal mercer%'
OR lower(emp.name) LIKE '%river gilmore%'
OR lower(emp.name) LIKE '%saniya bowers%'
OR lower(emp.name) LIKE '%braedon ross%'
OR lower(emp.name) LIKE '%clark mejia%'
OR lower(emp.name) LIKE '%ryan day%'
OR lower(emp.name) LIKE '%marilyn luna%'
OR lower(emp.name) LIKE '%avah sanchez%'
OR lower(emp.name) LIKE '%amelie wu%'
OR lower(emp.name) LIKE '%paola duke%'
OR lower(emp.name) LIKE '%jesse trevino%'
OR lower(emp.name) LIKE '%bianca cisneros%'
";
mod multi_like_data;

#[inline]
fn parse(text: &str) -> ParserResult {
Expand Down Expand Up @@ -413,7 +104,7 @@ fn bench_eval(c: &mut Criterion) {
let compiled_15 = compile(&parse(QUERY_15).unwrap());
let compiled_30 = compile(&parse(QUERY_30).unwrap());

let bindings = data();
let bindings = employee_data();

c.bench_function("eval-1", |b| {
b.iter(|| {
Expand Down
Loading