From 9c161d145f3e3702eb8219c687e55ae486db914a Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 2 Dec 2021 10:51:42 +0800 Subject: [PATCH 1/7] Introduce helper next_sqe() and move_forward() for SQ Introduce helper next_sqe() and move_forward() for SQ, it will be reused in following patches. Signed-off-by: Liu Jiang --- src/squeue.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/squeue.rs b/src/squeue.rs index 2afaa241..26b7ebd0 100644 --- a/src/squeue.rs +++ b/src/squeue.rs @@ -213,11 +213,8 @@ impl SubmissionQueue<'_> { #[inline] pub unsafe fn push(&mut self, Entry(entry): &Entry) -> Result<(), PushError> { if !self.is_full() { - *self - .queue - .sqes - .add((self.tail & self.queue.ring_mask) as usize) = *entry; - self.tail = self.tail.wrapping_add(1); + *self.next_sqe() = *entry; + self.move_forward(); Ok(()) } else { Err(PushError) @@ -240,15 +237,28 @@ impl SubmissionQueue<'_> { } for Entry(entry) in entries { - *self - .queue - .sqes - .add((self.tail & self.queue.ring_mask) as usize) = *entry; - self.tail = self.tail.wrapping_add(1); + *self.next_sqe() = *entry; + self.move_forward(); } Ok(()) } + + // Unsafe because it may return entry being used by kernel and make kernel use the + // uninitialized entry. + #[inline] + unsafe fn next_sqe(&mut self) -> &mut sys::io_uring_sqe { + &mut *self + .queue + .sqes + .add((self.tail & self.queue.ring_mask) as usize) + } + + // Unsafe because it may cause kernel to access uninitialized entry. + #[inline] + unsafe fn move_forward(&mut self) { + self.tail = self.tail.wrapping_add(1); + } } impl Drop for SubmissionQueue<'_> { From fef41a8047011bbbe95a2c1fe8a0ae0b80cbe2c4 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 2 Dec 2021 10:55:52 +0800 Subject: [PATCH 2/7] Introduce trait PrepareSQE for opcode Introduce trait PrepareSQE to support prepare SQE in place by calling opcode.prepare(&mut sqe). Signed-off-by: Liu Jiang --- src/opcode.rs | 606 ++++++++++++++++++-------------------------------- 1 file changed, 214 insertions(+), 392 deletions(-) diff --git a/src/opcode.rs b/src/opcode.rs index 824fd3ac..737fec18 100644 --- a/src/opcode.rs +++ b/src/opcode.rs @@ -9,6 +9,12 @@ use crate::squeue::Entry; use crate::sys; use crate::types::{self, sealed}; +/// Trait to prepare an SQE from an opcode object. +pub trait PrepareSQE { + /// Prepare an SQE from an opcode object. + fn prepare(&self, sqe: &mut sys::io_uring_sqe); +} + macro_rules! assign_fd { ( $sqe:ident . fd = $opfd:expr ) => { match $opfd { @@ -52,8 +58,7 @@ macro_rules! opcode { pub const CODE = $opcode:expr; - $( #[$build_meta:meta] )* - pub fn build($self:ident) -> Entry $build_block:block + fn set_used_fields(&$self:ident, $sqe1:ident : &mut sys::io_uring_sqe) $set_used_fields:block ) => { $( #[$outer] )* pub struct $name { @@ -85,9 +90,23 @@ macro_rules! opcode { } )* - $( #[$build_meta] )* #[inline] - pub fn build($self) -> Entry $build_block + pub fn build(self) -> Entry { + let mut sqe = sqe_zeroed(); + self.set_used_fields(&mut sqe); + Entry(sqe) + } + + #[inline] + fn set_used_fields(&$self, $sqe1: &mut sys::io_uring_sqe) $set_used_fields + } + + impl PrepareSQE for $name { + #[inline] + fn prepare(&self, sqe: &mut sys::io_uring_sqe) { + *sqe = unsafe { std::mem::zeroed() }; + self.set_used_fields(sqe); + } } } } @@ -107,13 +126,9 @@ opcode!( pub const CODE = sys::IORING_OP_NOP; - pub fn build(self) -> Entry { - let Nop {} = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - Entry(sqe) } ); @@ -134,22 +149,14 @@ opcode!( pub const CODE = sys::IORING_OP_READV; - pub fn build(self) -> Entry { - let Readv { - fd, - iovec, len, offset, - ioprio, rw_flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = iovec as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.iovec as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; } ); @@ -170,22 +177,14 @@ opcode!( pub const CODE = sys::IORING_OP_WRITEV; - pub fn build(self) -> Entry { - let Writev { - fd, - iovec, len, offset, - ioprio, rw_flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = iovec as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.iovec as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; } ); @@ -210,14 +209,10 @@ opcode!( pub const CODE = sys::IORING_OP_FSYNC; - pub fn build(self) -> Entry { - let Fsync { fd, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_3.fsync_flags = flags.bits(); - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_3.fsync_flags = self.flags.bits(); } ); @@ -244,24 +239,15 @@ opcode!( pub const CODE = sys::IORING_OP_READ_FIXED; - pub fn build(self) -> Entry { - let ReadFixed { - fd, - buf, len, offset, - buf_index, - ioprio, rw_flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - sqe.__bindgen_anon_4.buf_index = buf_index; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + sqe.__bindgen_anon_4.buf_index = self.buf_index; } ); @@ -288,24 +274,15 @@ opcode!( pub const CODE = sys::IORING_OP_WRITE_FIXED; - pub fn build(self) -> Entry { - let WriteFixed { - fd, - buf, len, offset, - buf_index, - ioprio, rw_flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - sqe.__bindgen_anon_4.buf_index = buf_index; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + sqe.__bindgen_anon_4.buf_index = self.buf_index; } ); @@ -325,25 +302,20 @@ opcode!( pub const CODE = sys::IORING_OP_POLL_ADD; - pub fn build(self) -> Entry { - let PollAdd { fd, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); + assign_fd!(sqe.fd = self.fd); #[cfg(target_endian = "little")] { - sqe.__bindgen_anon_3.poll32_events = flags; + sqe.__bindgen_anon_3.poll32_events = self.flags; } #[cfg(target_endian = "big")] { - let x = flags << 16; - let y = flags >> 16; + let x = self.flags << 16; + let y = self.flags >> 16; let flags = x | y; sqe.__bindgen_anon_3.poll32_events = flags; } - - Entry(sqe) } ); @@ -360,14 +332,10 @@ opcode!( pub const CODE = sys::IORING_OP_POLL_REMOVE; - pub fn build(self) -> Entry { - let PollRemove { user_data } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = user_data as _; - Entry(sqe) + sqe.__bindgen_anon_2.addr = self.user_data as _; } ); @@ -386,20 +354,12 @@ opcode!( pub const CODE = sys::IORING_OP_SYNC_FILE_RANGE; - pub fn build(self) -> Entry { - let SyncFileRange { - fd, - len, offset, - flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.len = len as _; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.sync_range_flags = flags; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.len = self.len as _; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.sync_range_flags = self.flags; } ); @@ -419,17 +379,13 @@ opcode!( pub const CODE = sys::IORING_OP_SENDMSG; - pub fn build(self) -> Entry { - let SendMsg { fd, msg, ioprio, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = msg as _; + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.msg as _; sqe.len = 1; - sqe.__bindgen_anon_3.msg_flags = flags; - Entry(sqe) + sqe.__bindgen_anon_3.msg_flags = self.flags; } ); @@ -448,17 +404,13 @@ opcode!( pub const CODE = sys::IORING_OP_RECVMSG; - pub fn build(self) -> Entry { - let RecvMsg { fd, msg, ioprio, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = msg as _; + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.msg as _; sqe.len = 1; - sqe.__bindgen_anon_3.msg_flags = flags; - Entry(sqe) + sqe.__bindgen_anon_3.msg_flags = self.flags; } ); @@ -484,17 +436,13 @@ opcode!( pub const CODE = sys::IORING_OP_TIMEOUT; - pub fn build(self) -> Entry { - let Timeout { timespec, count, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = timespec as _; + sqe.__bindgen_anon_2.addr = self.timespec as _; sqe.len = 1; - sqe.__bindgen_anon_1.off = count as _; - sqe.__bindgen_anon_3.timeout_flags = flags.bits(); - Entry(sqe) + sqe.__bindgen_anon_1.off = self.count as _; + sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); } ); @@ -510,15 +458,11 @@ opcode!( pub const CODE = sys::IORING_OP_TIMEOUT_REMOVE; - pub fn build(self) -> Entry { - let TimeoutRemove { user_data, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = user_data as _; - sqe.__bindgen_anon_3.timeout_flags = flags.bits(); - Entry(sqe) + sqe.__bindgen_anon_2.addr = self.user_data as _; + sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); } ); @@ -534,16 +478,12 @@ opcode!( pub const CODE = sys::IORING_OP_ACCEPT; - pub fn build(self) -> Entry { - let Accept { fd, addr, addrlen, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_2.addr = addr as _; - sqe.__bindgen_anon_1.addr2 = addrlen as _; - sqe.__bindgen_anon_3.accept_flags = flags as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_2.addr = self.addr as _; + sqe.__bindgen_anon_1.addr2 = self.addrlen as _; + sqe.__bindgen_anon_3.accept_flags = self.flags as _; } ); @@ -558,14 +498,10 @@ opcode!( pub const CODE = sys::IORING_OP_ASYNC_CANCEL; - pub fn build(self) -> Entry { - let AsyncCancel { user_data } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = user_data as _; - Entry(sqe) + sqe.__bindgen_anon_2.addr = self.user_data as _; } ); @@ -581,16 +517,12 @@ opcode!( pub const CODE = sys::IORING_OP_LINK_TIMEOUT; - pub fn build(self) -> Entry { - let LinkTimeout { timespec, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = timespec as _; + sqe.__bindgen_anon_2.addr = self.timespec as _; sqe.len = 1; - sqe.__bindgen_anon_3.timeout_flags = flags.bits(); - Entry(sqe) + sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); } ); @@ -605,15 +537,11 @@ opcode!( pub const CODE = sys::IORING_OP_CONNECT; - pub fn build(self) -> Entry { - let Connect { fd, addr, addrlen } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_2.addr = addr as _; - sqe.__bindgen_anon_1.off = addrlen as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_2.addr = self.addr as _; + sqe.__bindgen_anon_1.off = self.addrlen as _; } ); @@ -631,16 +559,12 @@ opcode!( pub const CODE = sys::IORING_OP_FALLOCATE; - pub fn build(self) -> Entry { - let Fallocate { fd, len, offset, mode } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_2.addr = len as _; - sqe.len = mode as _; - sqe.__bindgen_anon_1.off = offset as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_2.addr = self.len as _; + sqe.len = self.mode as _; + sqe.__bindgen_anon_1.off = self.offset as _; } ); @@ -656,16 +580,12 @@ opcode!( pub const CODE = sys::IORING_OP_OPENAT; - pub fn build(self) -> Entry { - let OpenAt { dirfd, pathname, flags, mode } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = dirfd; - sqe.__bindgen_anon_2.addr = pathname as _; - sqe.len = mode; - sqe.__bindgen_anon_3.open_flags = flags as _; - Entry(sqe) + sqe.fd = self.dirfd; + sqe.__bindgen_anon_2.addr = self.pathname as _; + sqe.len = self.mode; + sqe.__bindgen_anon_3.open_flags = self.flags as _; } ); @@ -678,13 +598,9 @@ opcode!( pub const CODE = sys::IORING_OP_CLOSE; - pub fn build(self) -> Entry { - let Close { fd } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = fd; - Entry(sqe) + sqe.fd = self.fd; } ); @@ -701,16 +617,12 @@ opcode!( pub const CODE = sys::IORING_OP_FILES_UPDATE; - pub fn build(self) -> Entry { - let FilesUpdate { fds, len, offset } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = fds as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - Entry(sqe) + sqe.__bindgen_anon_2.addr = self.fds as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; } ); @@ -727,20 +639,13 @@ opcode!( pub const CODE = sys::IORING_OP_STATX; - pub fn build(self) -> Entry { - let Statx { - dirfd, pathname, statxbuf, - flags, mask - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = dirfd; - sqe.__bindgen_anon_2.addr = pathname as _; - sqe.len = mask; - sqe.__bindgen_anon_1.off = statxbuf as _; - sqe.__bindgen_anon_3.statx_flags = flags as _; - Entry(sqe) + sqe.fd = self.dirfd; + sqe.__bindgen_anon_2.addr = self.pathname as _; + sqe.len = self.mask; + sqe.__bindgen_anon_1.off = self.statxbuf as _; + sqe.__bindgen_anon_3.statx_flags = self.flags as _; } ); @@ -773,24 +678,15 @@ opcode!( pub const CODE = sys::IORING_OP_READ; - pub fn build(self) -> Entry { - let Read { - fd, - buf, len, offset, - ioprio, rw_flags, - buf_group - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - sqe.__bindgen_anon_4.buf_group = buf_group; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + sqe.__bindgen_anon_4.buf_group = self.buf_group; } ); @@ -822,22 +718,14 @@ opcode!( pub const CODE = sys::IORING_OP_WRITE; - pub fn build(self) -> Entry { - let Write { - fd, - buf, len, offset, - ioprio, rw_flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.ioprio = ioprio; - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.rw_flags = rw_flags; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.ioprio = self.ioprio; + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.rw_flags = self.rw_flags; } ); @@ -853,16 +741,12 @@ opcode!( pub const CODE = sys::IORING_OP_FADVISE; - pub fn build(self) -> Entry { - let Fadvise { fd, len, advice, offset } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.len = len as _; - sqe.__bindgen_anon_1.off = offset as _; - sqe.__bindgen_anon_3.fadvise_advice = advice as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.len = self.len as _; + sqe.__bindgen_anon_1.off = self.offset as _; + sqe.__bindgen_anon_3.fadvise_advice = self.advice as _; } ); @@ -877,16 +761,12 @@ opcode!( pub const CODE = sys::IORING_OP_MADVISE; - pub fn build(self) -> Entry { - let Madvise { addr, len, advice } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; sqe.fd = -1; - sqe.__bindgen_anon_2.addr = addr as _; - sqe.len = len as _; - sqe.__bindgen_anon_3.fadvise_advice = advice as _; - Entry(sqe) + sqe.__bindgen_anon_2.addr = self.addr as _; + sqe.len = self.len as _; + sqe.__bindgen_anon_3.fadvise_advice = self.advice as _; } ); @@ -902,16 +782,12 @@ opcode!( pub const CODE = sys::IORING_OP_SEND; - pub fn build(self) -> Entry { - let Send { fd, buf, len, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_3.msg_flags = flags as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_3.msg_flags = self.flags as _; } ); @@ -928,17 +804,13 @@ opcode!( pub const CODE = sys::IORING_OP_RECV; - pub fn build(self) -> Entry { - let Recv { fd, buf, len, flags, buf_group } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.__bindgen_anon_2.addr = buf as _; - sqe.len = len; - sqe.__bindgen_anon_3.msg_flags = flags as _; - sqe.__bindgen_anon_4.buf_group = buf_group; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.__bindgen_anon_2.addr = self.buf as _; + sqe.len = self.len; + sqe.__bindgen_anon_3.msg_flags = self.flags as _; + sqe.__bindgen_anon_4.buf_group = self.buf_group; } ); @@ -953,16 +825,12 @@ opcode!( pub const CODE = sys::IORING_OP_OPENAT2; - pub fn build(self) -> Entry { - let OpenAt2 { dirfd, pathname, how } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = dirfd; - sqe.__bindgen_anon_2.addr = pathname as _; + sqe.fd = self.dirfd; + sqe.__bindgen_anon_2.addr = self.pathname as _; sqe.len = mem::size_of::() as _; - sqe.__bindgen_anon_1.off = how as _; - Entry(sqe) + sqe.__bindgen_anon_1.off = self.how as _; } ); @@ -978,16 +846,12 @@ opcode!( pub const CODE = sys::IORING_OP_EPOLL_CTL; - pub fn build(self) -> Entry { - let EpollCtl { epfd, fd, op, ev } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = epfd); - sqe.__bindgen_anon_2.addr = ev as _; - sqe.len = op as _; - sqe.__bindgen_anon_1.off = fd as _; - Entry(sqe) + assign_fd!(sqe.fd = self.epfd); + sqe.__bindgen_anon_2.addr = self.ev as _; + sqe.len = self.op as _; + sqe.__bindgen_anon_1.off = self.fd as _; } ); @@ -1011,16 +875,15 @@ opcode!( pub const CODE = sys::IORING_OP_SPLICE; - pub fn build(self) -> Entry { - let Splice { fd_in, off_in, fd_out, off_out, len, mut flags } = self; + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { + let mut flags = self.flags; - let mut sqe = sqe_zeroed(); sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd_out); - sqe.len = len; - sqe.__bindgen_anon_1.off = off_out as _; + assign_fd!(sqe.fd = self.fd_out); + sqe.len = self.len; + sqe.__bindgen_anon_1.off = self.off_out as _; - sqe.__bindgen_anon_5.splice_fd_in = match fd_in { + sqe.__bindgen_anon_5.splice_fd_in = match self.fd_in { sealed::Target::Fd(fd) => fd, sealed::Target::Fixed(i) => { flags |= sys::SPLICE_F_FD_IN_FIXED; @@ -1028,9 +891,8 @@ opcode!( } }; - sqe.__bindgen_anon_2.splice_off_in = off_in as _; + sqe.__bindgen_anon_2.splice_off_in = self.off_in as _; sqe.__bindgen_anon_3.splice_flags = flags; - Entry(sqe) } ); @@ -1052,17 +914,13 @@ opcode!( pub const CODE = sys::IORING_OP_PROVIDE_BUFFERS; - pub fn build(self) -> Entry { - let ProvideBuffers { addr, len, nbufs, bgid, bid } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = nbufs as _; - sqe.__bindgen_anon_2.addr = addr as _; - sqe.len = len as _; - sqe.__bindgen_anon_1.off = bid as _; - sqe.__bindgen_anon_4.buf_group = bgid; - Entry(sqe) + sqe.fd = self.nbufs as _; + sqe.__bindgen_anon_2.addr = self.addr as _; + sqe.len = self.len as _; + sqe.__bindgen_anon_1.off = self.bid as _; + sqe.__bindgen_anon_4.buf_group = self.bgid; } ); @@ -1080,14 +938,10 @@ opcode!( pub const CODE = sys::IORING_OP_REMOVE_BUFFERS; - pub fn build(self) -> Entry { - let RemoveBuffers { nbufs, bgid } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = nbufs as _; - sqe.__bindgen_anon_4.buf_group = bgid; - Entry(sqe) + sqe.fd = self.nbufs as _; + sqe.__bindgen_anon_4.buf_group = self.bgid; } ); @@ -1108,16 +962,14 @@ opcode!( pub const CODE = sys::IORING_OP_TEE; - pub fn build(self) -> Entry { - let Tee { fd_in, fd_out, len, mut flags } = self; + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { + let mut flags = self.flags; - let mut sqe = sqe_zeroed(); sqe.opcode = Self::CODE; + assign_fd!(sqe.fd = self.fd_out); + sqe.len = self.len; - assign_fd!(sqe.fd = fd_out); - sqe.len = len; - - sqe.__bindgen_anon_5.splice_fd_in = match fd_in { + sqe.__bindgen_anon_5.splice_fd_in = match self.fd_in { sealed::Target::Fd(fd) => fd, sealed::Target::Fixed(i) => { flags |= sys::SPLICE_F_FD_IN_FIXED; @@ -1126,8 +978,6 @@ opcode!( }; sqe.__bindgen_anon_3.splice_flags = flags; - - Entry(sqe) } ); @@ -1143,14 +993,10 @@ opcode!( pub const CODE = sys::IORING_OP_SHUTDOWN; - pub fn build(self) -> Entry { - let Shutdown { fd, how } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = fd); - sqe.len = how as _; - Entry(sqe) + assign_fd!(sqe.fd = self.fd); + sqe.len = self.how as _; } ); @@ -1167,21 +1013,13 @@ opcode!( pub const CODE = sys::IORING_OP_RENAMEAT; - pub fn build(self) -> Entry { - let RenameAt { - olddirfd, oldpath, - newdirfd, newpath, - flags - } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = olddirfd; - sqe.__bindgen_anon_2.addr = oldpath as _; - sqe.len = newdirfd as _; - sqe.__bindgen_anon_1.off = newpath as _; - sqe.__bindgen_anon_3.rename_flags = flags; - Entry(sqe) + sqe.fd = self.olddirfd; + sqe.__bindgen_anon_2.addr = self.oldpath as _; + sqe.len = self.newdirfd as _; + sqe.__bindgen_anon_1.off = self.newpath as _; + sqe.__bindgen_anon_3.rename_flags = self.flags; } ); @@ -1196,15 +1034,11 @@ opcode!( pub const CODE = sys::IORING_OP_UNLINKAT; - pub fn build(self) -> Entry { - let UnlinkAt { dirfd, pathname, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = dirfd; - sqe.__bindgen_anon_2.addr = pathname as _; - sqe.__bindgen_anon_3.unlink_flags = flags as _; - Entry(sqe) + sqe.fd = self.dirfd; + sqe.__bindgen_anon_2.addr = self.pathname as _; + sqe.__bindgen_anon_3.unlink_flags = self.flags as _; } ); @@ -1224,15 +1058,11 @@ opcode!( pub const CODE = sys::IORING_OP_MKDIRAT; - pub fn build(self) -> Entry { - let MkDirAt { dirfd, pathname, mode } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = dirfd; - sqe.__bindgen_anon_2.addr = pathname as _; - sqe.len = mode; - Entry(sqe) + sqe.fd = self.dirfd; + sqe.__bindgen_anon_2.addr = self.pathname as _; + sqe.len = self.mode; } ); @@ -1250,15 +1080,11 @@ opcode!( pub const CODE = sys::IORING_OP_SYMLINKAT; - pub fn build(self) -> Entry { - let SymlinkAt { newdirfd, target, linkpath } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = newdirfd; - sqe.__bindgen_anon_2.addr = target as _; - sqe.__bindgen_anon_1.addr2 = linkpath as _; - Entry(sqe) + sqe.fd = self.newdirfd; + sqe.__bindgen_anon_2.addr = self.target as _; + sqe.__bindgen_anon_1.addr2 = self.linkpath as _; } ); @@ -1278,16 +1104,12 @@ opcode!( pub const CODE = sys::IORING_OP_LINKAT; - pub fn build(self) -> Entry { - let LinkAt { olddirfd, oldpath, newdirfd, newpath, flags } = self; - - let mut sqe = sqe_zeroed(); + fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { sqe.opcode = Self::CODE; - sqe.fd = olddirfd as _; - sqe.__bindgen_anon_2.addr = oldpath as _; - sqe.len = newdirfd as _; - sqe.__bindgen_anon_1.addr2 = newpath as _; - sqe.__bindgen_anon_3.hardlink_flags = flags as _; - Entry(sqe) + sqe.fd = self.olddirfd as _; + sqe.__bindgen_anon_2.addr = self.oldpath as _; + sqe.len = self.newdirfd as _; + sqe.__bindgen_anon_1.addr2 = self.newpath as _; + sqe.__bindgen_anon_3.hardlink_flags = self.flags as _; } ); From e6c7b679f6c99721bf2f19de45eca417ed3f6021 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 2 Dec 2021 13:49:10 +0800 Subject: [PATCH 3/7] Add SubmissionQueue::push_command()/push_commands() Add SubmissionQueue::push_command()/push_commands() to generate SQE in place. Signed-off-by: Liu Jiang --- src/squeue.rs | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/src/squeue.rs b/src/squeue.rs index 26b7ebd0..4f6d2429 100644 --- a/src/squeue.rs +++ b/src/squeue.rs @@ -4,6 +4,8 @@ use std::error::Error; use std::fmt::{self, Debug, Display, Formatter}; use std::sync::atomic; +#[cfg(feature = "unstable")] +use crate::opcode::PrepareSQE; use crate::sys; use crate::util::{unsync_load, Mmap}; @@ -34,6 +36,15 @@ pub struct SubmissionQueue<'a> { #[derive(Clone)] pub struct Entry(pub(crate) sys::io_uring_sqe); +/// Common options for Submission Queue Entry. +#[derive(Clone, Copy, Default, Debug)] +pub struct SqeCommonOptions { + pub user_data: u64, + pub personality: u16, + pub flags_set: u8, + pub flags_clear: u8, +} + bitflags! { /// Submission flags pub struct Flags: u8 { @@ -244,6 +255,59 @@ impl SubmissionQueue<'_> { Ok(()) } + /// Attempts to push an opcode into the submission queue. + /// If the queue is full, an error is returned. + /// + /// # Safety + /// + /// Developers must ensure that parameters of the opcode (such as buffer) are valid and will + /// be valid for the entire duration of the operation, otherwise it may cause memory problems. + #[cfg(feature = "unstable")] + #[inline] + pub unsafe fn push_command<'a, T: PrepareSQE>( + &'a mut self, + opcode: &T, + options: Option<&SqeCommonOptions>, + ) -> Result<(), PushError> { + if !self.is_full() { + let sqe = self.next_sqe(); + options.map(|v| v.set(sqe)); + opcode.prepare(sqe); + self.move_forward(); + Ok(()) + } else { + Err(PushError) + } + } + + /// Attempts to push several opcodes into the queue. + /// If the queue does not have space for all of the entries, an error is returned. + /// + /// # Safety + /// + /// Developers must ensure that parameters of all the entries (such as buffer) are valid and + /// will be valid for the entire duration of the operation, otherwise it may cause memory + /// problems. + #[cfg(feature = "unstable")] + #[inline] + pub unsafe fn push_commands<'a, T: PrepareSQE>( + &'a mut self, + ops: &[(T, Option<&SqeCommonOptions>)], + ) -> Result<(), PushError> { + if self.capacity() - self.len() < ops.len() { + return Err(PushError); + } + + for (opcode, options) in ops { + let sqe = self.next_sqe(); + options.map(|v| v.set(sqe)); + opcode.prepare(sqe); + self.move_forward(); + } + + Ok(()) + } + // Unsafe because it may return entry being used by kernel and make kernel use the // uninitialized entry. #[inline] @@ -295,6 +359,60 @@ impl Entry { } } +impl SqeCommonOptions { + /// Create a new instance of `OptionValues`. + pub fn new(user_data: u64, personality: u16, flags_set: Flags, flags_clear: Flags) -> Self { + SqeCommonOptions { + user_data, + personality, + flags_set: flags_set.bits(), + flags_clear: flags_clear.bits(), + } + } + + /// Set the user data. + /// + /// This is an application-supplied value that will be passed straight through into the + /// [completion queue entry](crate::cqueue::Entry::user_data). + #[inline] + pub fn user_data(mut self, user_data: u64) -> Self { + self.user_data = user_data; + self + } + + /// Set the personality of this event. + /// + /// You can obtain a personality using + /// [`Submitter::register_personality`](crate::Submitter::register_personality). + #[inline] + pub fn personality(mut self, personality: u16) -> Self { + self.personality = personality; + self + } + + /// Mark the flags to set on the submission event's [flags](Flags). + #[inline] + pub fn set_flags(mut self, flags: Flags) -> Self { + self.flags_set |= flags.bits(); + self + } + + /// Mark the flags to cleared on the submission event's [flags](Flags). + #[inline] + pub fn clear_flags(mut self, flags: Flags) -> Self { + self.flags_clear |= flags.bits(); + self + } + + /// Set common options for a submission queue entry. + pub fn set(&self, sqe: &mut sys::io_uring_sqe) { + sqe.personality = self.personality; + sqe.user_data = self.user_data; + sqe.flags |= self.flags_set; + sqe.flags &= !self.flags_clear; + } +} + /// An error pushing to the submission queue due to it being full. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] From 0e6299cc3faeec8358594113ee6331a1bb0e38ee Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Wed, 8 Dec 2021 00:23:31 +0800 Subject: [PATCH 4/7] Add unit test cases and benchmark for push_command() Add unit test cases and benchmark for push_command(). Signed-off-by: Liu Jiang --- io-uring-bench/src/nop.rs | 29 ++++++++++++++++++++++++++++- io-uring-test/src/main.rs | 2 ++ io-uring-test/src/tests/queue.rs | 28 ++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/io-uring-bench/src/nop.rs b/io-uring-bench/src/nop.rs index 8eebaf31..19ad2ae7 100644 --- a/io-uring-bench/src/nop.rs +++ b/io-uring-bench/src/nop.rs @@ -41,5 +41,32 @@ fn bench_normal(c: &mut Criterion) { }); } -criterion_group!(squeue, bench_normal); +fn bench_prepare(c: &mut Criterion) { + let mut io_uring = IoUring::new(16).unwrap(); + + c.bench_function("prepare", |b| { + b.iter(|| { + let mut queue = TaskQueue(128); + + while queue.want() { + { + let mut sq = io_uring.submission(); + while queue.want() { + unsafe { + match sq.push_command(&black_box(opcode::Nop::new()), None) { + Ok(_) => queue.pop(), + Err(_) => break, + } + } + } + } + + io_uring.submit_and_wait(16).unwrap(); + + io_uring.completion().map(black_box).for_each(drop); + } + }); + }); +} +criterion_group!(squeue, bench_normal, bench_prepare); criterion_main!(squeue); diff --git a/io-uring-test/src/main.rs b/io-uring-test/src/main.rs index adecda05..4b52d193 100644 --- a/io-uring-test/src/main.rs +++ b/io-uring-test/src/main.rs @@ -29,6 +29,8 @@ fn main() -> anyhow::Result<()> { tests::queue::test_nop(&mut ring, &test)?; tests::queue::test_queue_split(&mut ring, &test)?; tests::queue::test_debug_print(&mut ring, &test)?; + #[cfg(feature = "unstable")] + tests::queue::test_nop_prepare(&mut ring, &test)?; #[cfg(feature = "unstable")] tests::queue::test_batch(&mut ring, &test)?; diff --git a/io-uring-test/src/tests/queue.rs b/io-uring-test/src/tests/queue.rs index c9d2a083..d067525b 100644 --- a/io-uring-test/src/tests/queue.rs +++ b/io-uring-test/src/tests/queue.rs @@ -1,4 +1,5 @@ use crate::Test; +use io_uring::squeue::SqeCommonOptions; use io_uring::{opcode, IoUring}; pub fn test_nop(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { @@ -26,6 +27,33 @@ pub fn test_nop(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { Ok(()) } +#[cfg(feature = "unstable")] +pub fn test_nop_prepare(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { + require! { + test; + } + + println!("test nop_prepare"); + + let nop = opcode::Nop::new(); + let opt = SqeCommonOptions::default().user_data(0x42); + + unsafe { + let mut queue = ring.submission(); + queue.push_command(&nop, Some(&opt)).expect("queue is full"); + } + + ring.submit_and_wait(1)?; + + let cqes = ring.completion().collect::>(); + + assert_eq!(cqes.len(), 1); + assert_eq!(cqes[0].user_data(), 0x42); + assert_eq!(cqes[0].result(), 0); + + Ok(()) +} + #[cfg(feature = "unstable")] pub fn test_batch(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { use std::mem::MaybeUninit; From 54abde8a506ce93d0f887994a364bbc512c1b6d6 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 3 Jan 2022 16:26:15 +0800 Subject: [PATCH 5/7] Enhance opcode to setup SQE in place Currently the opcode module provides a data structure for each io-uring operation code. And it works in following way to submit an SQE: - generate an opcode object, such Nop, Readv etc. - convert the opcode object into an SQE object - copy the generated SQE object onto the next available SQE in the submission queue One previous patch in this series implements PrepareSQE to optimize the process to submit a request as: - generate an opcode object, such Nop, Readv etc. - convert the opcode object to an SQE in the submission queue in place The process to submit a request could be further optimized as: - get the next available SQE from the submission queue - prepare the SQE for io uring operation - commit the SQE This patch enhance the opcode module to prepare SQE in place, without involving the intermedia opcode structures. The way to achieve the goal is to introduce a structure for each opcode to support in-place preparatio. #[repr(transparent)] pub struct ReadvSqe { sqe: sys::io_uring_sqe, } impl ReadvSqe { #[inline] pub fn prepare(&mut self, fd: sealed::Target, iovec: *const libc::iovec, len: u32) { } #[inline] pub fn ioprio(mut self, ioprio: u16) -> Self { self.sqe.ioprio = ioprio.as_sqe_value() as _; self } } impl<'a> From<&'a mut sys::io_uring_sqe> for &'a mut ReadvSqe { #[inline] fn from(sqe: &'a mut sys::io_uring_sqe) -> &'a mut ReadvSqe { unsafe { mem::transmute(sqe) } } } Signed-off-by: Liu Jiang --- src/opcode.rs | 1103 +++++++++++++++++++++++++++++-------------------- 1 file changed, 653 insertions(+), 450 deletions(-) diff --git a/src/opcode.rs b/src/opcode.rs index 737fec18..c13c3162 100644 --- a/src/opcode.rs +++ b/src/opcode.rs @@ -15,14 +15,103 @@ pub trait PrepareSQE { fn prepare(&self, sqe: &mut sys::io_uring_sqe); } -macro_rules! assign_fd { - ( $sqe:ident . fd = $opfd:expr ) => { - match $opfd { - sealed::Target::Fd(fd) => $sqe.fd = fd, - sealed::Target::Fixed(i) => { - $sqe.fd = i as _; - $sqe.flags |= crate::squeue::Flags::FIXED_FILE.bits(); - } +trait AsSqeValue { + type T; + fn as_sqe_value(&self) -> Self::T; +} + +impl AsSqeValue for sealed::Target { + type T = i32; + + fn as_sqe_value(&self) -> Self::T { + match self { + sealed::Target::Fd(fd) => *fd as _, + sealed::Target::Fixed(i) => *i as _, + } + } +} + +impl AsSqeValue for types::FsyncFlags { + type T = u32; + + fn as_sqe_value(&self) -> Self::T { + self.bits() + } +} + +impl AsSqeValue for types::TimeoutFlags { + type T = u32; + + fn as_sqe_value(&self) -> Self::T { + self.bits() + } +} + +impl AsSqeValue for u64 { + type T = u64; + + fn as_sqe_value(&self) -> Self::T { + *self as Self::T + } +} + +impl AsSqeValue for i64 { + type T = u64; + + fn as_sqe_value(&self) -> Self::T { + *self as Self::T + } +} + +impl AsSqeValue for u32 { + type T = u32; + + fn as_sqe_value(&self) -> Self::T { + *self + } +} + +impl AsSqeValue for i32 { + type T = i32; + + fn as_sqe_value(&self) -> Self::T { + *self + } +} + +impl AsSqeValue for u16 { + type T = u16; + + fn as_sqe_value(&self) -> Self::T { + *self + } +} + +impl AsSqeValue for *const U { + type T = u64; + + fn as_sqe_value(&self) -> Self::T { + *self as _ + } +} + +impl AsSqeValue for *mut U { + type T = u64; + + fn as_sqe_value(&self) -> Self::T { + *self as _ + } +} + +macro_rules! set_fd_flags { + ( $sqe:ident, $opfd:expr ) => { + if matches!($opfd, sealed::Target::Fixed(_)) { + $sqe.flags |= crate::squeue::Flags::FIXED_FILE.bits(); + } + }; + ( $self:ident . $sqe:ident, $opfd:expr ) => { + if matches!($opfd, sealed::Target::Fixed(_)) { + $self.$sqe.flags |= crate::squeue::Flags::FIXED_FILE.bits(); } }; } @@ -42,7 +131,7 @@ macro_rules! opcode { pub struct $name:ident { $( #[$new_meta:meta] )* - $( $field:ident : { $( $tnt:tt )+ } ),* + $( $field:ident : { $( $tnt:tt )+ } => sqe $(. $sqe_field:ident)+ ),* $(,)? @@ -50,15 +139,21 @@ macro_rules! opcode { $( $( #[$opt_meta:meta] )* - $opt_field:ident : $opt_tname:ty = $default:expr + $opt_field:ident : $opt_tname:ty = $default:expr => sqe $(. $opt_sqe_field:ident)+ ),* $(,)? } + $( #[$outer_sqe:meta] )* + pub struct $name_sqe:ident { } + pub const CODE = $opcode:expr; - fn set_used_fields(&$self:ident, $sqe1:ident : &mut sys::io_uring_sqe) $set_used_fields:block + fn set_special_fields(&mut $self:ident, $sqe1:ident : &mut sys::io_uring_sqe) { + $( op: $set_op_special_fields:block )? + $( op_sqe: $set_op_sqe_special_fields:block )? + } ) => { $( #[$outer] )* pub struct $name { @@ -67,6 +162,11 @@ macro_rules! opcode { } impl $name { + /// The opcode of the operation. This can be passed to + /// [`Probe::is_supported`](crate::Probe::is_supported) to check if this operation is + /// supported with the current kernel. + pub const CODE: u8 = $opcode as _; + $( #[$new_meta] )* #[inline] pub fn new($( $field : $( $tnt )* ),*) -> Self { @@ -76,11 +176,6 @@ macro_rules! opcode { } } - /// The opcode of the operation. This can be passed to - /// [`Probe::is_supported`](crate::Probe::is_supported) to check if this operation is - /// supported with the current kernel. - pub const CODE: u8 = $opcode as _; - $( $( #[$opt_meta] )* #[inline] @@ -93,19 +188,57 @@ macro_rules! opcode { #[inline] pub fn build(self) -> Entry { let mut sqe = sqe_zeroed(); - self.set_used_fields(&mut sqe); + self.prepare(&mut sqe); Entry(sqe) } + } + impl PrepareSQE for $name { #[inline] - fn set_used_fields(&$self, $sqe1: &mut sys::io_uring_sqe) $set_used_fields + fn prepare(& $self, $sqe1: &mut sys::io_uring_sqe) { + $sqe1 .opcode = $name::CODE; + $( $sqe1 $(. $sqe_field)* = $self. $field .as_sqe_value() as _; )* + $( $sqe1 $(. $opt_sqe_field)* = $self. $opt_field .as_sqe_value() as _; )* + $( $set_op_special_fields )? + } } - impl PrepareSQE for $name { + $( #[$outer_sqe] )* + #[repr(transparent)] + pub struct $name_sqe { + sqe: sys::io_uring_sqe, + } + + impl $name_sqe { + /// The opcode of the operation. This can be passed to + /// [`Probe::is_supported`](crate::Probe::is_supported) to check if this operation is + /// supported with the current kernel. + pub const CODE: u8 = $opcode as _; + #[inline] - fn prepare(&self, sqe: &mut sys::io_uring_sqe) { - *sqe = unsafe { std::mem::zeroed() }; - self.set_used_fields(sqe); + pub fn prepare(&mut $self, $( $field : opcode!(@type $( $tnt )*), )* ) { + $self.sqe.opcode = Self::CODE; + $( $self.sqe $(. $sqe_field)* = $field .as_sqe_value() as _; )* + $( $self.sqe $(. $opt_sqe_field)* = $default .as_sqe_value() as _; )* + $( $set_op_sqe_special_fields )? + } + + $( + $( #[$opt_meta] )* + #[inline] + pub fn $opt_field(mut self, $opt_field: $opt_tname) -> Self { + self.sqe$(. $opt_sqe_field )* = $opt_field .as_sqe_value() as _; + self + } + )* + } + + impl<'a> From<&'a mut sys::io_uring_sqe> for &'a mut $name_sqe { + #[inline] + fn from(sqe: &'a mut sys::io_uring_sqe) -> &'a mut $name_sqe { + unsafe { + mem::transmute(sqe) + } } } } @@ -113,7 +246,7 @@ macro_rules! opcode { /// inline zeroed to improve codegen #[inline(always)] -fn sqe_zeroed() -> sys::io_uring_sqe { +pub(crate) fn sqe_zeroed() -> sys::io_uring_sqe { unsafe { std::mem::zeroed() } } @@ -124,11 +257,16 @@ opcode!( #[derive(Debug)] pub struct Nop { ;; } + /// Do not perform any I/O. + /// + /// This is useful for testing the performance of the io_uring implementation itself. + pub struct NopSqe { } + pub const CODE = sys::IORING_OP_NOP; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); @@ -136,27 +274,25 @@ opcode!( /// Vectored read, equivalent to `preadv2(2)`. #[derive(Debug)] pub struct Readv { - fd: { impl sealed::UseFixed }, - iovec: { *const libc::iovec }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + iovec: { *const libc::iovec } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; - ioprio: u16 = 0, - offset: libc::off_t = 0, + ioprio: u16 = 0u16 => sqe.ioprio, + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, /// specified for read operations, contains a bitwise OR of per-I/O flags, /// as described in the `preadv2(2)` man page. - rw_flags: types::RwFlags = 0 + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, } + /// Vectored read, equivalent to `preadv2(2)`. + pub struct ReadvSqe { } + pub const CODE = sys::IORING_OP_READV; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.iovec as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -164,27 +300,25 @@ opcode!( /// Vectored write, equivalent to `pwritev2(2)`. #[derive(Debug)] pub struct Writev { - fd: { impl sealed::UseFixed }, - iovec: { *const libc::iovec }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + iovec: { *const libc::iovec } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; - ioprio: u16 = 0, - offset: libc::off_t = 0, + ioprio: u16 = 0u16 => sqe.ioprio, + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, /// specified for write operations, contains a bitwise OR of per-I/O flags, /// as described in the `preadv2(2)` man page. - rw_flags: types::RwFlags = 0 + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, } + /// Vectored write, equivalent to `pwritev2(2)`. + pub struct WritevSqe { } + pub const CODE = sys::IORING_OP_WRITEV; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.iovec as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -199,20 +333,29 @@ opcode!( /// the fsync. #[derive(Debug)] pub struct Fsync { - fd: { impl sealed::UseFixed }, + fd: { impl sealed::UseFixed } => sqe.fd, ;; /// The `flags` bit mask may contain either 0, for a normal file integrity sync, /// or [types::FsyncFlags::DATASYNC] to provide data sync only semantics. /// See the descriptions of `O_SYNC` and `O_DSYNC` in the `open(2)` manual page for more information. - flags: types::FsyncFlags = types::FsyncFlags::empty() + flags: types::FsyncFlags = types::FsyncFlags::empty() => sqe.__bindgen_anon_3.fsync_flags, } + /// File sync, equivalent to `fsync(2)`. + /// + /// Note that, while I/O is initiated in the order in which it appears in the submission queue, + /// completions are unordered. For example, an application which places a write I/O followed by + /// an fsync in the submission queue cannot expect the fsync to apply to the write. The two + /// operations execute in parallel, so the fsync may complete before the write is issued to the + /// storage. The same is also true for previously issued writes that have not completed prior to + /// the fsync. + pub struct FsyncSqe { } + pub const CODE = sys::IORING_OP_FSYNC; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_3.fsync_flags = self.flags.bits(); + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -225,29 +368,29 @@ opcode!( pub struct ReadFixed { /// The `buf_index` is an index into an array of fixed buffers, /// and is only valid if fixed buffers were registered. - fd: { impl sealed::UseFixed }, - buf: { *mut u8 }, - len: { u32 }, - buf_index: { u16 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *mut u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, + buf_index: { u16 } => sqe.__bindgen_anon_4.buf_index, ;; - offset: libc::off_t = 0, - ioprio: u16 = 0, + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, + ioprio: u16 = 0u16 => sqe.ioprio, /// specified for read operations, contains a bitwise OR of per-I/O flags, /// as described in the `preadv2(2)` man page. - rw_flags: types::RwFlags = 0 + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, } + /// Read from pre-mapped buffers that have been previously registered with + /// [`Submitter::register_buffers`](crate::Submitter::register_buffers). + /// + /// The return values match those documented in the `preadv2(2)` man pages. + pub struct ReadFixedSqe { } + pub const CODE = sys::IORING_OP_READ_FIXED; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; - sqe.__bindgen_anon_4.buf_index = self.buf_index; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -260,29 +403,29 @@ opcode!( pub struct WriteFixed { /// The `buf_index` is an index into an array of fixed buffers, /// and is only valid if fixed buffers were registered. - fd: { impl sealed::UseFixed }, - buf: { *const u8 }, - len: { u32 }, - buf_index: { u16 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *const u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, + buf_index: { u16 } => sqe.__bindgen_anon_4.buf_index, ;; - ioprio: u16 = 0, - offset: libc::off_t = 0, + ioprio: u16 = 0u16 => sqe.ioprio, + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, /// specified for write operations, contains a bitwise OR of per-I/O flags, /// as described in the `preadv2(2)` man page. - rw_flags: types::RwFlags = 0 + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, } + /// Write to pre-mapped buffers that have been previously registered with + /// [`Submitter::register_buffers`](crate::Submitter::register_buffers). + /// + /// The return values match those documented in the `pwritev2(2)` man pages. + pub struct WriteFixedSqe { } + pub const CODE = sys::IORING_OP_WRITE_FIXED; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; - sqe.__bindgen_anon_4.buf_index = self.buf_index; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -295,26 +438,37 @@ opcode!( pub struct PollAdd { /// The bits that may be set in `flags` are defined in ``, /// and documented in `poll(2)`. - fd: { impl sealed::UseFixed }, - flags: { u32 } + fd: { impl sealed::UseFixed } => sqe.fd, + flags: { u32 } => sqe.__bindgen_anon_3.poll32_events ;; } - pub const CODE = sys::IORING_OP_POLL_ADD; + /// Poll the specified fd. + /// + /// Unlike poll or epoll without `EPOLLONESHOT`, this interface always works in one shot mode. + /// That is, once the poll operation is completed, it will have to be resubmitted. + pub struct PollAddSqe { } - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); + pub const CODE = sys::IORING_OP_POLL_ADD; - #[cfg(target_endian = "little")] { - sqe.__bindgen_anon_3.poll32_events = self.flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + set_fd_flags!(sqe, self.fd); + #[cfg(target_endian = "big")] { + let x = self.flags << 16; + let y = self.flags >> 16; + let flags = x | y; + sqe.__bindgen_anon_3.poll32_events = flags; + } } - - #[cfg(target_endian = "big")] { - let x = self.flags << 16; - let y = self.flags >> 16; - let flags = x | y; - sqe.__bindgen_anon_3.poll32_events = flags; + op_sqe: { + set_fd_flags!(self.sqe, fd); + #[cfg(target_endian = "big")] { + let x = flags << 16; + let y = flags >> 16; + let flags = x | y; + self.sqe.__bindgen_anon_3.poll32_events = flags; + } } } ); @@ -326,16 +480,21 @@ opcode!( /// If not found, `result` will return `-libc::ENOENT`. #[derive(Debug)] pub struct PollRemove { - user_data: { u64 } + user_data: { u64 } => sqe.__bindgen_anon_2.addr, ;; } + /// Remove an existing [poll](PollAdd) request. + /// + /// If found, the `result` method of the `cqueue::Entry` will return 0. + /// If not found, `result` will return `-libc::ENOENT`. + pub struct PollRemoveSqe { } + pub const CODE = sys::IORING_OP_POLL_REMOVE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.user_data as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); @@ -343,23 +502,23 @@ opcode!( /// Sync a file segment with disk, equivalent to `sync_file_range(2)`. #[derive(Debug)] pub struct SyncFileRange { - fd: { impl sealed::UseFixed }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + len: { u32 } => sqe.len, ;; /// the offset method holds the offset in bytes - offset: libc::off64_t = 0, + offset: libc::off64_t = 0i64 => sqe.__bindgen_anon_1.off, /// the flags method holds the flags for the command - flags: u32 = 0 + flags: u32 = 0u32 => sqe.__bindgen_anon_3.sync_range_flags, } + /// Sync a file segment with disk, equivalent to `sync_file_range(2)`. + pub struct SyncFileRangeSqe { } + pub const CODE = sys::IORING_OP_SYNC_FILE_RANGE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.len = self.len as _; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.sync_range_flags = self.flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -370,22 +529,30 @@ opcode!( /// structure, and flags holds the flags associated with the system call. #[derive(Debug)] pub struct SendMsg { - fd: { impl sealed::UseFixed }, - msg: { *const libc::msghdr }, + fd: { impl sealed::UseFixed } => sqe.fd, + msg: { *const libc::msghdr } => sqe.__bindgen_anon_2.addr, ;; - ioprio: u16 = 0, - flags: u32 = 0 + ioprio: u16 = 0u16 => sqe.ioprio, + flags: u32 = 0u32 => sqe.__bindgen_anon_3.msg_flags, } + /// Send a message on a socket, equivalent to `send(2)`. + /// + /// fd must be set to the socket file descriptor, addr must contains a pointer to the msghdr + /// structure, and flags holds the flags associated with the system call. + pub struct SendMsgSqe { } + pub const CODE = sys::IORING_OP_SENDMSG; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.msg as _; - sqe.len = 1; - sqe.__bindgen_anon_3.msg_flags = self.flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + set_fd_flags!(sqe, self.fd); + sqe.len = 1; + } + op_sqe: { + set_fd_flags!(self.sqe, fd); + self.sqe.len = 1; + } } ); @@ -395,22 +562,29 @@ opcode!( /// See also the description of [`SendMsg`]. #[derive(Debug)] pub struct RecvMsg { - fd: { impl sealed::UseFixed }, - msg: { *mut libc::msghdr }, + fd: { impl sealed::UseFixed } => sqe.fd, + msg: { *mut libc::msghdr } => sqe.__bindgen_anon_2.addr, ;; - ioprio: u16 = 0, - flags: u32 = 0 + ioprio: u16 = 0u16 => sqe.ioprio, + flags: u32 = 0u32 => sqe.__bindgen_anon_3.msg_flags, } + /// Receive a message on a socket, equivalent to `recvmsg(2)`. + /// + /// See also the description of [`SendMsg`]. + pub struct RecvMsgSqe { } + pub const CODE = sys::IORING_OP_RECVMSG; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.msg as _; - sqe.len = 1; - sqe.__bindgen_anon_3.msg_flags = self.flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + set_fd_flags!(sqe, self.fd); + sqe.len = 1; + } + op_sqe: { + set_fd_flags!(self.sqe, fd); + self.sqe.len = 1; + } } ); @@ -425,24 +599,37 @@ opcode!( /// If the timeout was cancelled before it expired, the request will complete with `-ECANCELED`. #[derive(Debug)] pub struct Timeout { - timespec: { *const types::Timespec }, + timespec: { *const types::Timespec } => sqe.__bindgen_anon_2.addr, ;; /// `count` may contain a completion event count. - count: u32 = 0, + count: u32 = 0u32 => sqe.__bindgen_anon_1.off, /// `flags` may contain [types::TimeoutFlags::ABS] for an absolute timeout value, or 0 for a relative timeout. - flags: types::TimeoutFlags = types::TimeoutFlags::empty() + flags: types::TimeoutFlags = types::TimeoutFlags::empty() => sqe.__bindgen_anon_3.timeout_flags, } + /// Register a timeout operation. + /// + /// A timeout will trigger a wakeup event on the completion ring for anyone waiting for events. + /// A timeout condition is met when either the specified timeout expires, or the specified number of events have completed. + /// Either condition will trigger the event. + /// The request will complete with `-ETIME` if the timeout got completed through expiration of the timer, + /// or 0 if the timeout got completed through requests completing on their own. + /// If the timeout was cancelled before it expired, the request will complete with `-ECANCELED`. + pub struct TimeoutSqe { } + pub const CODE = sys::IORING_OP_TIMEOUT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.timespec as _; - sqe.len = 1; - sqe.__bindgen_anon_1.off = self.count as _; - sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + sqe.fd = -1; + sqe.len = 1; + } + op_sqe: { + let sqe = &mut self.sqe; + sqe.fd = -1; + sqe.len = 1; + } } ); @@ -451,57 +638,60 @@ opcode!( opcode!( /// Attempt to remove an existing [timeout operation](Timeout). pub struct TimeoutRemove { - user_data: { u64 }, + user_data: { u64 } => sqe.__bindgen_anon_2.addr, ;; - flags: types::TimeoutFlags = types::TimeoutFlags::empty() + flags: types::TimeoutFlags = types::TimeoutFlags::empty() => sqe.__bindgen_anon_3.timeout_flags, } + /// Attempt to remove an existing [timeout operation](Timeout). + pub struct TimeoutRemoveSqe { } + pub const CODE = sys::IORING_OP_TIMEOUT_REMOVE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.user_data as _; - sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); opcode!( /// Accept a new connection on a socket, equivalent to `accept4(2)`. pub struct Accept { - fd: { impl sealed::UseFixed }, - addr: { *mut libc::sockaddr }, - addrlen: { *mut libc::socklen_t }, + fd: { impl sealed::UseFixed } => sqe.fd, + addr: { *mut libc::sockaddr } => sqe.__bindgen_anon_2.addr, + addrlen: { *mut libc::socklen_t } => sqe.__bindgen_anon_1.addr2, ;; - flags: i32 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.accept_flags, } + /// Accept a new connection on a socket, equivalent to `accept4(2)`. + pub struct AcceptSqe { } + pub const CODE = sys::IORING_OP_ACCEPT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_2.addr = self.addr as _; - sqe.__bindgen_anon_1.addr2 = self.addrlen as _; - sqe.__bindgen_anon_3.accept_flags = self.flags as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Attempt to cancel an already issued request. pub struct AsyncCancel { - user_data: { u64 } + user_data: { u64 } => sqe.__bindgen_anon_2.addr, ;; // TODO flags } + /// Attempt to cancel an already issued request. + pub struct AsyncCancelSqe { } + pub const CODE = sys::IORING_OP_ASYNC_CANCEL; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.user_data as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); @@ -510,38 +700,47 @@ opcode!( /// [`Flags::IO_LINK`](crate::squeue::Flags::IO_LINK) which is described below. /// Unlike [`Timeout`], [`LinkTimeout`] acts on the linked request, not the completion queue. pub struct LinkTimeout { - timespec: { *const types::Timespec }, + timespec: { *const types::Timespec } => sqe.__bindgen_anon_2.addr, ;; - flags: types::TimeoutFlags = types::TimeoutFlags::empty() + flags: types::TimeoutFlags = types::TimeoutFlags::empty() => sqe.__bindgen_anon_3.timeout_flags, } + /// This request must be linked with another request through + /// [`Flags::IO_LINK`](crate::squeue::Flags::IO_LINK) which is described below. + /// Unlike [`Timeout`], [`LinkTimeout`] acts on the linked request, not the completion queue. + pub struct LinkTimeoutSqe { } + pub const CODE = sys::IORING_OP_LINK_TIMEOUT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.timespec as _; - sqe.len = 1; - sqe.__bindgen_anon_3.timeout_flags = self.flags.bits(); + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + sqe.fd = -1; + sqe.len = 1; + } + op_sqe: { + self.sqe.fd = -1; + self.sqe.fd = 1; + } } ); opcode!( /// Connect a socket, equivalent to `connect(2)`. pub struct Connect { - fd: { impl sealed::UseFixed }, - addr: { *const libc::sockaddr }, - addrlen: { libc::socklen_t } + fd: { impl sealed::UseFixed } => sqe.fd, + addr: { *const libc::sockaddr } => sqe.__bindgen_anon_2.addr, + addrlen: { libc::socklen_t } => sqe.__bindgen_anon_1.off, ;; } + /// Connect a socket, equivalent to `connect(2)`. + pub struct ConnectSqe { } + pub const CODE = sys::IORING_OP_CONNECT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_2.addr = self.addr as _; - sqe.__bindgen_anon_1.off = self.addrlen as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -550,58 +749,55 @@ opcode!( opcode!( /// Preallocate or deallocate space to a file, equivalent to `fallocate(2)`. pub struct Fallocate { - fd: { impl sealed::UseFixed }, - len: { libc::off_t }, + fd: { impl sealed::UseFixed } => sqe.fd, + len: { libc::off_t } => sqe.__bindgen_anon_2.addr, ;; - offset: libc::off_t = 0, - mode: i32 = 0 + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, + mode: i32 = 0i32 => sqe.len, } + /// Preallocate or deallocate space to a file, equivalent to `fallocate(2)`. + pub struct FallocateSqe { } + pub const CODE = sys::IORING_OP_FALLOCATE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_2.addr = self.len as _; - sqe.len = self.mode as _; - sqe.__bindgen_anon_1.off = self.offset as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Open a file, equivalent to `openat(2)`. pub struct OpenAt { - dirfd: { impl sealed::UseFd }, - pathname: { *const libc::c_char }, + dirfd: { impl sealed::UseFd } => sqe.fd, + pathname: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, ;; - flags: i32 = 0, - mode: libc::mode_t = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.open_flags, + mode: libc::mode_t = 0u32 => sqe.len, } + /// Open a file, equivalent to `openat(2)`. + pub struct OpenAtSqe { } + pub const CODE = sys::IORING_OP_OPENAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.dirfd; - sqe.__bindgen_anon_2.addr = self.pathname as _; - sqe.len = self.mode; - sqe.__bindgen_anon_3.open_flags = self.flags as _; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); opcode!( /// Close a file descriptor, equivalent to `close(2)`. pub struct Close { - fd: { impl sealed::UseFd } + fd: { impl sealed::UseFd } => sqe.fd, ;; } + /// Close a file descriptor, equivalent to `close(2)`. + pub struct CloseSqe { } + pub const CODE = sys::IORING_OP_CLOSE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.fd; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); opcode!( @@ -609,44 +805,42 @@ opcode!( /// [`Submitter::register_files_update`](crate::Submitter::register_files_update) which then /// works in an async fashion, like the rest of the io_uring commands. pub struct FilesUpdate { - fds: { *const RawFd }, - len: { u32 }, + fds: { *const RawFd } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; - offset: i32 = 0 + offset: i32 = 0i32 => sqe.__bindgen_anon_1.off, } + /// This command is an alternative to using + /// [`Submitter::register_files_update`](crate::Submitter::register_files_update) which then + /// works in an async fashion, like the rest of the io_uring commands. + pub struct FilesUpdateSqe { } + pub const CODE = sys::IORING_OP_FILES_UPDATE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.fds as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); opcode!( /// Get file status, equivalent to `statx(2)`. pub struct Statx { - dirfd: { impl sealed::UseFd }, - pathname: { *const libc::c_char }, - statxbuf: { *mut types::statx }, + dirfd: { impl sealed::UseFd } => sqe.fd, + pathname: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, + statxbuf: { *mut types::statx } => sqe.__bindgen_anon_1.off, ;; - flags: i32 = 0, - mask: u32 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.statx_flags, + mask: u32 = 0u32 => sqe.len, } + /// Get file status, equivalent to `statx(2)`. + pub struct StatxSqe { } + pub const CODE = sys::IORING_OP_STATX; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.dirfd; - sqe.__bindgen_anon_2.addr = self.pathname as _; - sqe.len = self.mask; - sqe.__bindgen_anon_1.off = self.statxbuf as _; - sqe.__bindgen_anon_3.statx_flags = self.flags as _; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); opcode!( @@ -661,32 +855,38 @@ opcode!( /// /// Available since 5.6. pub struct Read { - fd: { impl sealed::UseFixed }, - buf: { *mut u8 }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *mut u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; /// `offset` contains the read or write offset. /// /// If `fd` does not refer to a seekable file, `offset` must be set to zero. /// If `offsett` is set to `-1`, the offset will use (and advance) the file position, /// like the `read(2)` and `write(2)` system calls. - offset: libc::off_t = 0, - ioprio: u16 = 0, - rw_flags: types::RwFlags = 0, - buf_group: u16 = 0 + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, + ioprio: u16 = 0u16 => sqe.ioprio, + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, + buf_group: u16 = 0u16 => sqe.__bindgen_anon_4.buf_group, } + /// Issue the equivalent of a `pread(2)` or `pwrite(2)` system call + /// + /// * `fd` is the file descriptor to be operated on, + /// * `addr` contains the buffer in question, + /// * `len` contains the length of the IO operation, + /// + /// These are non-vectored versions of the `IORING_OP_READV` and `IORING_OP_WRITEV` opcodes. + /// See also `read(2)` and `write(2)` for the general description of the related system call. + /// + /// Available since 5.6. + pub struct ReadSqe { } + pub const CODE = sys::IORING_OP_READ; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; - sqe.__bindgen_anon_4.buf_group = self.buf_group; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); @@ -702,156 +902,162 @@ opcode!( /// /// Available since 5.6. pub struct Write { - fd: { impl sealed::UseFixed }, - buf: { *const u8 }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *const u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; /// `offset` contains the read or write offset. /// /// If `fd` does not refer to a seekable file, `offset` must be set to zero. /// If `offsett` is set to `-1`, the offset will use (and advance) the file position, /// like the `read(2)` and `write(2)` system calls. - offset: libc::off_t = 0, - ioprio: u16 = 0, - rw_flags: types::RwFlags = 0 + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, + ioprio: u16 = 0u16 => sqe.ioprio, + rw_flags: types::RwFlags = 0i32 => sqe.__bindgen_anon_3.rw_flags, } + /// Issue the equivalent of a `pread(2)` or `pwrite(2)` system call + /// + /// * `fd` is the file descriptor to be operated on, + /// * `addr` contains the buffer in question, + /// * `len` contains the length of the IO operation, + /// + /// These are non-vectored versions of the `IORING_OP_READV` and `IORING_OP_WRITEV` opcodes. + /// See also `read(2)` and `write(2)` for the general description of the related system call. + /// + /// Available since 5.6. + pub struct WriteSqe { } + pub const CODE = sys::IORING_OP_WRITE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.ioprio = self.ioprio; - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.rw_flags = self.rw_flags; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Predeclare an access pattern for file data, equivalent to `posix_fadvise(2)`. pub struct Fadvise { - fd: { impl sealed::UseFixed }, - len: { libc::off_t }, - advice: { i32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + len: { libc::off_t } => sqe.len, + advice: { i32 } => sqe.__bindgen_anon_3.fadvise_advice, ;; - offset: libc::off_t = 0, + offset: libc::off_t = 0i64 => sqe.__bindgen_anon_1.off, } + /// Predeclare an access pattern for file data, equivalent to `posix_fadvise(2)`. + pub struct FadviseSqe { } + pub const CODE = sys::IORING_OP_FADVISE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.len = self.len as _; - sqe.__bindgen_anon_1.off = self.offset as _; - sqe.__bindgen_anon_3.fadvise_advice = self.advice as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Give advice about use of memory, equivalent to `madvise(2)`. pub struct Madvise { - addr: { *const libc::c_void }, - len: { libc::off_t }, - advice: { i32 }, + addr: { *const libc::c_void } => sqe.__bindgen_anon_2.addr, + len: { libc::off_t } => sqe.len, + advice: { i32 } => sqe.__bindgen_anon_3.fadvise_advice, ;; } + /// Give advice about use of memory, equivalent to `madvise(2)`. + pub struct MadviseSqe { } + pub const CODE = sys::IORING_OP_MADVISE; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = -1; - sqe.__bindgen_anon_2.addr = self.addr as _; - sqe.len = self.len as _; - sqe.__bindgen_anon_3.fadvise_advice = self.advice as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.fd = -1; } + op_sqe: { self.sqe.fd = -1; } } ); opcode!( /// Send a message on a socket, equivalent to `send(2)`. pub struct Send { - fd: { impl sealed::UseFixed }, - buf: { *const u8 }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *const u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; - flags: i32 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.msg_flags, } + /// Send a message on a socket, equivalent to `send(2)`. + pub struct SendSqe { } + pub const CODE = sys::IORING_OP_SEND; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_3.msg_flags = self.flags as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Receive a message from a socket, equivalent to `recv(2)`. pub struct Recv { - fd: { impl sealed::UseFixed }, - buf: { *mut u8 }, - len: { u32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + buf: { *mut u8 } => sqe.__bindgen_anon_2.addr, + len: { u32 } => sqe.len, ;; - flags: i32 = 0, - buf_group: u16 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.msg_flags, + buf_group: u16 = 0u16 => sqe.__bindgen_anon_4.buf_group, } + /// Receive a message from a socket, equivalent to `recv(2)`. + pub struct RecvSqe { } + pub const CODE = sys::IORING_OP_RECV; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.__bindgen_anon_2.addr = self.buf as _; - sqe.len = self.len; - sqe.__bindgen_anon_3.msg_flags = self.flags as _; - sqe.__bindgen_anon_4.buf_group = self.buf_group; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); opcode!( /// Open a file, equivalent to `openat2(2)`. pub struct OpenAt2 { - dirfd: { impl sealed::UseFd }, - pathname: { *const libc::c_char }, - how: { *const types::OpenHow } + dirfd: { impl sealed::UseFd } => sqe.fd, + pathname: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, + how: { *const types::OpenHow } => sqe.__bindgen_anon_1.off, ;; } + /// Open a file, equivalent to `openat2(2)`. + pub struct OpenAt2Sqe { } + pub const CODE = sys::IORING_OP_OPENAT2; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.dirfd; - sqe.__bindgen_anon_2.addr = self.pathname as _; - sqe.len = mem::size_of::() as _; - sqe.__bindgen_anon_1.off = self.how as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { sqe.len = mem::size_of::() as _; } + op_sqe: { self.sqe.len = mem::size_of::() as _;} } ); opcode!( /// Modify an epoll file descriptor, equivalent to `epoll_ctl(2)`. pub struct EpollCtl { - epfd: { impl sealed::UseFixed }, - fd: { impl sealed::UseFd }, - op: { i32 }, - ev: { *const types::epoll_event }, + epfd: { impl sealed::UseFixed } => sqe.fd, + fd: { impl sealed::UseFd } => sqe.__bindgen_anon_1.off, + op: { i32 } => sqe.len, + ev: { *const types::epoll_event } => sqe.__bindgen_anon_2.addr, ;; } + /// Modify an epoll file descriptor, equivalent to `epoll_ctl(2)`. + pub struct EpollCtlSqe { } + pub const CODE = sys::IORING_OP_EPOLL_CTL; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.epfd); - sqe.__bindgen_anon_2.addr = self.ev as _; - sqe.len = self.op as _; - sqe.__bindgen_anon_1.off = self.fd as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.epfd); } + op_sqe: { set_fd_flags!(self.sqe, epfd); } } ); @@ -863,36 +1069,39 @@ opcode!( /// if `fd_in` refers to a pipe, `off_in` must be `-1`; /// The description of `off_in` also applied to `off_out`. pub struct Splice { - fd_in: { impl sealed::UseFixed }, - off_in: { i64 }, - fd_out: { impl sealed::UseFixed }, - off_out: { i64 }, - len: { u32 }, + fd_in: { impl sealed::UseFixed } => sqe.__bindgen_anon_5.splice_fd_in, + off_in: { i64 } => sqe.__bindgen_anon_2.splice_off_in, + fd_out: { impl sealed::UseFixed } => sqe.fd, + off_out: { i64 } => sqe.__bindgen_anon_1.off, + len: { u32 } => sqe.len, ;; /// see man `splice(2)` for description of flags. - flags: u32 = 0 + flags: u32 = 0u32 => sqe.__bindgen_anon_3.splice_flags, } - pub const CODE = sys::IORING_OP_SPLICE; - - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - let mut flags = self.flags; + /// Splice data to/from a pipe, equivalent to `splice(2)`. + /// + /// if `fd_in` refers to a pipe, `off_in` must be `-1`; + /// The description of `off_in` also applied to `off_out`. + pub struct SpliceSqe { } - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd_out); - sqe.len = self.len; - sqe.__bindgen_anon_1.off = self.off_out as _; + pub const CODE = sys::IORING_OP_SPLICE; - sqe.__bindgen_anon_5.splice_fd_in = match self.fd_in { - sealed::Target::Fd(fd) => fd, - sealed::Target::Fixed(i) => { - flags |= sys::SPLICE_F_FD_IN_FIXED; - i as _ + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + set_fd_flags!(sqe, self.fd_out); + if matches!(self.fd_in, sealed::Target::Fixed(_)) { + sqe.__bindgen_anon_3.splice_flags = self.flags | sys::SPLICE_F_FD_IN_FIXED; } - }; - - sqe.__bindgen_anon_2.splice_off_in = self.off_in as _; - sqe.__bindgen_anon_3.splice_flags = flags; + } + op_sqe: { + set_fd_flags!(self.sqe, fd_out); + if matches!(fd_in, sealed::Target::Fixed(_)) { + unsafe { + self.sqe.__bindgen_anon_3.splice_flags |= sys::SPLICE_F_FD_IN_FIXED; + } + } + } } ); @@ -904,24 +1113,24 @@ opcode!( /// /// Requires the `unstable` feature. pub struct ProvideBuffers { - addr: { *mut u8 }, - len: { i32 }, - nbufs: { u16 }, - bgid: { u16 }, - bid: { u16 } + addr: { *mut u8 } => sqe.__bindgen_anon_2.addr, + len: { i32 } => sqe.len, + nbufs: { u16 } => sqe.fd, + bgid: { u16 } => sqe.__bindgen_anon_4.buf_group, + bid: { u16 } => sqe.__bindgen_anon_1.off, ;; } + /// Register `nbufs` buffers that each have the length `len` with ids starting from `big` in the + /// group `bgid` that can be used for any request. See + /// [`BUFFER_SELECT`](crate::squeue::Flags::BUFFER_SELECT) for more info. + /// + /// Requires the `unstable` feature. + pub struct ProvideBuffersSqe { } + pub const CODE = sys::IORING_OP_PROVIDE_BUFFERS; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.nbufs as _; - sqe.__bindgen_anon_2.addr = self.addr as _; - sqe.len = self.len as _; - sqe.__bindgen_anon_1.off = self.bid as _; - sqe.__bindgen_anon_4.buf_group = self.bgid; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); #[cfg(feature = "unstable")] @@ -931,18 +1140,20 @@ opcode!( /// /// Requires the `unstable` feature. pub struct RemoveBuffers { - nbufs: { u16 }, - bgid: { u16 } + nbufs: { u16 } => sqe.fd, + bgid: { u16 } => sqe.__bindgen_anon_4.buf_group, ;; } + /// Remove some number of buffers from a buffer group. See + /// [`BUFFER_SELECT`](crate::squeue::Flags::BUFFER_SELECT) for more info. + /// + /// Requires the `unstable` feature. + pub struct RemoveBuffersSqe { } + pub const CODE = sys::IORING_OP_REMOVE_BUFFERS; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.nbufs as _; - sqe.__bindgen_anon_4.buf_group = self.bgid; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); // === 5.8 === @@ -953,31 +1164,35 @@ opcode!( /// /// Requires the `unstable` feature. pub struct Tee { - fd_in: { impl sealed::UseFixed }, - fd_out: { impl sealed::UseFixed }, - len: { u32 } + fd_in: { impl sealed::UseFixed } => sqe.__bindgen_anon_5.splice_fd_in, + fd_out: { impl sealed::UseFixed } => sqe.fd, + len: { u32 } => sqe.len, ;; - flags: u32 = 0 + flags: u32 = 0 => sqe.__bindgen_anon_3.splice_flags, } - pub const CODE = sys::IORING_OP_TEE; - - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - let mut flags = self.flags; + /// Duplicate pipe content, equivalent to `tee(2)`. + /// + /// Requires the `unstable` feature. + pub struct TeeSqe { } - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd_out); - sqe.len = self.len; + pub const CODE = sys::IORING_OP_TEE; - sqe.__bindgen_anon_5.splice_fd_in = match self.fd_in { - sealed::Target::Fd(fd) => fd, - sealed::Target::Fixed(i) => { - flags |= sys::SPLICE_F_FD_IN_FIXED; - i as _ + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { + set_fd_flags!(sqe, self.fd_out); + if matches!(self.fd_in, sealed::Target::Fixed(_)) { + sqe.__bindgen_anon_3.splice_flags = self.flags | sys::SPLICE_F_FD_IN_FIXED; } - }; - - sqe.__bindgen_anon_3.splice_flags = flags; + } + op_sqe: { + set_fd_flags!(self.sqe, fd_out); + if matches!(fd_in, sealed::Target::Fixed(_)) { + unsafe { + self.sqe.__bindgen_anon_3.splice_flags |= sys::SPLICE_F_FD_IN_FIXED; + } + } + } } ); @@ -986,60 +1201,53 @@ opcode!( #[cfg(feature = "unstable")] opcode!( pub struct Shutdown { - fd: { impl sealed::UseFixed }, - how: { i32 }, + fd: { impl sealed::UseFixed } => sqe.fd, + how: { i32 } => sqe.len, ;; } + pub struct ShutdownSqe { } + pub const CODE = sys::IORING_OP_SHUTDOWN; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - assign_fd!(sqe.fd = self.fd); - sqe.len = self.how as _; + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { + op: { set_fd_flags!(sqe, self.fd); } + op_sqe: { set_fd_flags!(self.sqe, fd); } } ); #[cfg(feature = "unstable")] opcode!( pub struct RenameAt { - olddirfd: { impl sealed::UseFd }, - oldpath: { *const libc::c_char }, - newdirfd: { impl sealed::UseFd }, - newpath: { *const libc::c_char }, + olddirfd: { impl sealed::UseFd } => sqe.fd, + oldpath: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, + newdirfd: { impl sealed::UseFd } => sqe.len, + newpath: { *const libc::c_char } => sqe.__bindgen_anon_1.off, ;; - flags: u32 = 0 + flags: u32 = 0u32 => sqe.__bindgen_anon_3.rename_flags, } + pub struct RenameAtSqe { } + pub const CODE = sys::IORING_OP_RENAMEAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.olddirfd; - sqe.__bindgen_anon_2.addr = self.oldpath as _; - sqe.len = self.newdirfd as _; - sqe.__bindgen_anon_1.off = self.newpath as _; - sqe.__bindgen_anon_3.rename_flags = self.flags; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); #[cfg(feature = "unstable")] opcode!( pub struct UnlinkAt { - dirfd: { impl sealed::UseFd }, - pathname: { *const libc::c_char }, + dirfd: { impl sealed::UseFd } => sqe.fd, + pathname: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, ;; - flags: i32 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.unlink_flags, } + pub struct UnlinkAtSqe { } + pub const CODE = sys::IORING_OP_UNLINKAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.dirfd; - sqe.__bindgen_anon_2.addr = self.pathname as _; - sqe.__bindgen_anon_3.unlink_flags = self.flags as _; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); // === 5.15 === @@ -1050,20 +1258,17 @@ opcode!( /// /// Requires the `unstable` feature. pub struct MkDirAt { - dirfd: { impl sealed::UseFd }, - pathname: { *const libc::c_char }, + dirfd: { impl sealed::UseFd } => sqe.fd, + pathname: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, ;; - mode: libc::mode_t = 0 + mode: libc::mode_t = 0u32 => sqe.len, } + pub struct MkDirAtSqe { } + pub const CODE = sys::IORING_OP_MKDIRAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.dirfd; - sqe.__bindgen_anon_2.addr = self.pathname as _; - sqe.len = self.mode; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); #[cfg(feature = "unstable")] @@ -1072,20 +1277,20 @@ opcode!( /// /// Requires the `unstable` feature. pub struct SymlinkAt { - newdirfd: { impl sealed::UseFd }, - target: { *const libc::c_char }, - linkpath: { *const libc::c_char }, + newdirfd: { impl sealed::UseFd } => sqe.fd, + target: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, + linkpath: { *const libc::c_char } => sqe.__bindgen_anon_1.addr2, ;; } + /// Create a symlink, equivalent to `symlinkat2(2)`. + /// + /// Requires the `unstable` feature. + pub struct SymlinkAtSqe { } + pub const CODE = sys::IORING_OP_SYMLINKAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.newdirfd; - sqe.__bindgen_anon_2.addr = self.target as _; - sqe.__bindgen_anon_1.addr2 = self.linkpath as _; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); #[cfg(feature = "unstable")] @@ -1094,22 +1299,20 @@ opcode!( /// /// Requires the `unstable` feature. pub struct LinkAt { - olddirfd: { impl sealed::UseFd }, - oldpath: { *const libc::c_char }, - newdirfd: { impl sealed::UseFd }, - newpath: { *const libc::c_char }, + olddirfd: { impl sealed::UseFd } => sqe.fd, + oldpath: { *const libc::c_char } => sqe.__bindgen_anon_2.addr, + newdirfd: { impl sealed::UseFd } => sqe.len, + newpath: { *const libc::c_char } => sqe.__bindgen_anon_1.addr2 , ;; - flags: i32 = 0 + flags: i32 = 0i32 => sqe.__bindgen_anon_3.hardlink_flags, } + /// Create a hard link, equivalent to `linkat2(2)`. + /// + /// Requires the `unstable` feature. + pub struct LinkAtSqe { } + pub const CODE = sys::IORING_OP_LINKAT; - fn set_used_fields(&self, sqe: &mut sys::io_uring_sqe) { - sqe.opcode = Self::CODE; - sqe.fd = self.olddirfd as _; - sqe.__bindgen_anon_2.addr = self.oldpath as _; - sqe.len = self.newdirfd as _; - sqe.__bindgen_anon_1.addr2 = self.newpath as _; - sqe.__bindgen_anon_3.hardlink_flags = self.flags as _; - } + fn set_special_fields(&mut self, sqe: &mut sys::io_uring_sqe) { } ); From be14c7124c000be7b12dbb57829a1b7fbd60d30f Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 3 Jan 2022 16:42:52 +0800 Subject: [PATCH 6/7] Enhance SubmissionQueue to support prepare SQE in place Export SubmissionQueue::get_available_sqe() to prepare available SQE for in-place preparation. And export SubmissionQueue::move_forward() to commit prepared SQEs. Sample code to use the new interface: pub fn prepare_sqe(mut sq: SubmissionQueue<'_>) { unsafe { match sq.get_available_sqe(0) { Ok(sqe) => { let nop_sqe: &mut crate::opcode::NopSqe = sqe.into(); nop_sqe.prepare(); sq.move_forward(1); } Err(_) => return, } } } Signed-off-by: Liu Jiang --- src/opcode.rs | 5 +++ src/squeue.rs | 117 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/src/opcode.rs b/src/opcode.rs index c13c3162..b296a456 100644 --- a/src/opcode.rs +++ b/src/opcode.rs @@ -223,6 +223,11 @@ macro_rules! opcode { $( $set_op_sqe_special_fields )? } + #[inline] + pub fn get_mut_sqe(&mut self) -> &mut sys::io_uring_sqe { + &mut self.sqe + } + $( $( #[$opt_meta] )* #[inline] diff --git a/src/squeue.rs b/src/squeue.rs index 4f6d2429..1eb408c2 100644 --- a/src/squeue.rs +++ b/src/squeue.rs @@ -4,6 +4,7 @@ use std::error::Error; use std::fmt::{self, Debug, Display, Formatter}; use std::sync::atomic; +use crate::opcode::sqe_zeroed; #[cfg(feature = "unstable")] use crate::opcode::PrepareSQE; use crate::sys; @@ -223,13 +224,10 @@ impl SubmissionQueue<'_> { /// be valid for the entire duration of the operation, otherwise it may cause memory problems. #[inline] pub unsafe fn push(&mut self, Entry(entry): &Entry) -> Result<(), PushError> { - if !self.is_full() { - *self.next_sqe() = *entry; - self.move_forward(); - Ok(()) - } else { - Err(PushError) - } + let sqe = self.get_available_sqe(0)?; + *sqe = *entry; + self.move_forward(1); + Ok(()) } /// Attempts to push several [entries](Entry) into the queue. @@ -248,13 +246,54 @@ impl SubmissionQueue<'_> { } for Entry(entry) in entries { - *self.next_sqe() = *entry; - self.move_forward(); + let sqe = self.get_available_sqe(0)?; + *sqe = *entry; + self.move_forward(1); } Ok(()) } +} + +#[cfg(not(feature = "unstable"))] +impl SubmissionQueue<'_> { + /// Try to get a mutable reference to the SQE at `next_available + offset`. + /// + /// # Safety + /// + /// The returned mutable reference has been zeroed-out. Developers must ensure that legal + /// values are set onto the SQE and call `move_forward()` to commit the prepared SQE. + #[inline] + unsafe fn get_available_sqe( + &mut self, + offset: u32, + ) -> Result<&mut sys::io_uring_sqe, PushError> { + if self.capacity() - self.len() <= offset as usize { + Err(PushError) + } else { + let sqe = &mut *self + .queue + .sqes + .add((self.tail.wrapping_add(offset) & self.queue.ring_mask) as usize); + *sqe = sqe_zeroed(); + Ok(sqe) + } + } + + /// Move the submission queue forward by `count` steps. + /// + /// # Safety + /// + /// Developers must ensure that `count` is valid and the next `count` SQEs have been correctly + /// initialized. + #[inline] + unsafe fn move_forward(&mut self, count: u32) { + self.tail = self.tail.wrapping_add(count); + } +} +#[cfg(feature = "unstable")] +impl SubmissionQueue<'_> { /// Attempts to push an opcode into the submission queue. /// If the queue is full, an error is returned. /// @@ -262,22 +301,17 @@ impl SubmissionQueue<'_> { /// /// Developers must ensure that parameters of the opcode (such as buffer) are valid and will /// be valid for the entire duration of the operation, otherwise it may cause memory problems. - #[cfg(feature = "unstable")] #[inline] pub unsafe fn push_command<'a, T: PrepareSQE>( &'a mut self, opcode: &T, options: Option<&SqeCommonOptions>, ) -> Result<(), PushError> { - if !self.is_full() { - let sqe = self.next_sqe(); - options.map(|v| v.set(sqe)); - opcode.prepare(sqe); - self.move_forward(); - Ok(()) - } else { - Err(PushError) - } + let sqe = self.get_available_sqe(0)?; + opcode.prepare(sqe); + options.map(|v| v.set(sqe)); + self.move_forward(1); + Ok(()) } /// Attempts to push several opcodes into the queue. @@ -288,7 +322,6 @@ impl SubmissionQueue<'_> { /// Developers must ensure that parameters of all the entries (such as buffer) are valid and /// will be valid for the entire duration of the operation, otherwise it may cause memory /// problems. - #[cfg(feature = "unstable")] #[inline] pub unsafe fn push_commands<'a, T: PrepareSQE>( &'a mut self, @@ -299,29 +332,47 @@ impl SubmissionQueue<'_> { } for (opcode, options) in ops { - let sqe = self.next_sqe(); - options.map(|v| v.set(sqe)); + let sqe = self.get_available_sqe(0)?; opcode.prepare(sqe); - self.move_forward(); + options.map(|v| v.set(sqe)); + self.move_forward(1); } Ok(()) } - // Unsafe because it may return entry being used by kernel and make kernel use the - // uninitialized entry. + /// Try to get a mutable reference to the SQE at `next_available + offset`. + /// + /// # Safety + /// + /// The returned mutable reference has been zeroed-out. Developers must ensure that legal + /// values are set onto the SQE and call `move_forward()` to commit the prepared SQE. #[inline] - unsafe fn next_sqe(&mut self) -> &mut sys::io_uring_sqe { - &mut *self - .queue - .sqes - .add((self.tail & self.queue.ring_mask) as usize) + pub unsafe fn get_available_sqe( + &mut self, + offset: u32, + ) -> Result<&mut sys::io_uring_sqe, PushError> { + if self.capacity() - self.len() <= offset as usize { + Err(PushError) + } else { + let sqe = &mut *self + .queue + .sqes + .add((self.tail.wrapping_add(offset) & self.queue.ring_mask) as usize); + *sqe = sqe_zeroed(); + Ok(sqe) + } } - // Unsafe because it may cause kernel to access uninitialized entry. + /// Move the submission queue forward by `count` steps. + /// + /// # Safety + /// + /// Developers must ensure that `count` is valid and the next `count` SQEs have been correctly + /// initialized. #[inline] - unsafe fn move_forward(&mut self) { - self.tail = self.tail.wrapping_add(1); + pub unsafe fn move_forward(&mut self, count: u32) { + self.tail = self.tail.wrapping_add(count); } } From 3c6f30d01ac0caecd65edf87bf49944285ebdcc7 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sat, 1 Jan 2022 11:12:19 +0800 Subject: [PATCH 7/7] Add benchmark for preparing SQE in place Add benchmark for preparing SQE in place. Signed-off-by: Liu Jiang --- io-uring-bench/src/nop.rs | 36 +++++++++++++++++++++++++++++++- io-uring-test/src/main.rs | 2 ++ io-uring-test/src/tests/queue.rs | 30 ++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/io-uring-bench/src/nop.rs b/io-uring-bench/src/nop.rs index 19ad2ae7..198f1622 100644 --- a/io-uring-bench/src/nop.rs +++ b/io-uring-bench/src/nop.rs @@ -68,5 +68,39 @@ fn bench_prepare(c: &mut Criterion) { }); }); } -criterion_group!(squeue, bench_normal, bench_prepare); + +fn bench_prepare_sqe(c: &mut Criterion) { + let mut io_uring = IoUring::new(16).unwrap(); + + c.bench_function("prepare_sqe", |b| { + b.iter(|| { + let mut queue = TaskQueue(128); + + while queue.want() { + { + let mut sq = io_uring.submission(); + while queue.want() { + unsafe { + match sq.get_available_sqe(0) { + Ok(sqe) => { + let nop_sqe: &mut opcode::NopSqe = black_box(sqe.into()); + nop_sqe.prepare(); + sq.move_forward(1); + queue.pop(); + } + Err(_) => break, + }; + } + } + } + + io_uring.submit_and_wait(16).unwrap(); + + io_uring.completion().map(black_box).for_each(drop); + } + }); + }); +} + +criterion_group!(squeue, bench_normal, bench_prepare, bench_prepare_sqe); criterion_main!(squeue); diff --git a/io-uring-test/src/main.rs b/io-uring-test/src/main.rs index 4b52d193..87bea908 100644 --- a/io-uring-test/src/main.rs +++ b/io-uring-test/src/main.rs @@ -31,6 +31,8 @@ fn main() -> anyhow::Result<()> { tests::queue::test_debug_print(&mut ring, &test)?; #[cfg(feature = "unstable")] tests::queue::test_nop_prepare(&mut ring, &test)?; + #[cfg(feature = "unstable")] + tests::queue::test_nop_prepare_sqe(&mut ring, &test)?; #[cfg(feature = "unstable")] tests::queue::test_batch(&mut ring, &test)?; diff --git a/io-uring-test/src/tests/queue.rs b/io-uring-test/src/tests/queue.rs index d067525b..e49e1507 100644 --- a/io-uring-test/src/tests/queue.rs +++ b/io-uring-test/src/tests/queue.rs @@ -54,6 +54,36 @@ pub fn test_nop_prepare(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { Ok(()) } +#[cfg(feature = "unstable")] +pub fn test_nop_prepare_sqe(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { + require! { + test; + } + + println!("test nop_prepare_sqe"); + + let opt = SqeCommonOptions::default().user_data(0x42); + + unsafe { + let mut queue = ring.submission(); + let sqe = queue.get_available_sqe(0).unwrap(); + let nop_sqe: &mut opcode::NopSqe = sqe.into(); + nop_sqe.prepare(); + opt.set(nop_sqe.get_mut_sqe()); + queue.move_forward(1); + } + + ring.submit_and_wait(1)?; + + let cqes = ring.completion().collect::>(); + + assert_eq!(cqes.len(), 1); + assert_eq!(cqes[0].user_data(), 0x42); + assert_eq!(cqes[0].result(), 0); + + Ok(()) +} + #[cfg(feature = "unstable")] pub fn test_batch(ring: &mut IoUring, test: &Test) -> anyhow::Result<()> { use std::mem::MaybeUninit;