Skip to content

Commit

Permalink
feat: add batch size options (apache#248)
Browse files Browse the repository at this point in the history
  • Loading branch information
jiacai2050 authored Sep 14, 2022
1 parent e39650e commit 5c14f00
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 2 deletions.
2 changes: 2 additions & 0 deletions analytic_engine/src/instance/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,8 @@ pub struct Instance {
pub(crate) space_write_buffer_size: usize,
/// replay wal batch size
pub(crate) replay_batch_size: usize,
/// batch size for scan sst
pub(crate) scan_batch_size: usize,
}

impl Instance {
Expand Down
1 change: 1 addition & 0 deletions analytic_engine/src/instance/open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ impl Instance {
db_write_buffer_size: ctx.config.db_write_buffer_size,
space_write_buffer_size: ctx.config.space_write_buffer_size,
replay_batch_size: ctx.config.replay_batch_size,
scan_batch_size: ctx.config.scan_batch_size,
});

Ok(instance)
Expand Down
2 changes: 1 addition & 1 deletion analytic_engine/src/instance/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ impl Instance {
// Collect metrics.
table_data.metrics.on_read_request_begin();

let iter_options = IterOptions::default();
let iter_options = IterOptions::new(self.scan_batch_size);
let table_options = table_data.table_options();

if need_merge_sort_streams(&table_data.table_options(), &request) {
Expand Down
4 changes: 4 additions & 0 deletions analytic_engine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ pub struct Config {
pub db_write_buffer_size: usize,
// End of global write buffer options.

// Batch size for scan sst
pub scan_batch_size: usize,

// Obkv wal config.
pub obkv_wal: ObkvWalConfig,
}
Expand All @@ -95,6 +98,7 @@ impl Default for Config {
/// Zero means disabling this param, give a positive value to enable
/// it.
db_write_buffer_size: 0,
scan_batch_size: 500,
obkv_wal: ObkvWalConfig::default(),
}
}
Expand Down
8 changes: 7 additions & 1 deletion analytic_engine/src/row_iter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,15 @@ pub struct IterOptions {
pub batch_size: usize,
}

impl IterOptions {
pub fn new(batch_size: usize) -> Self {
Self { batch_size }
}
}

impl Default for IterOptions {
fn default() -> Self {
Self { batch_size: 500 }
Self::new(500)
}
}

Expand Down

0 comments on commit 5c14f00

Please sign in to comment.