From 8d2aaacebe0d1cf4b66a1bec11e7077ec04e31ec Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 11:06:45 +0100 Subject: [PATCH 01/36] add `field_projection` v2 rfc --- text/3735-field-projections.md | 1344 ++++++++++++++++++++++++++++++++ 1 file changed, 1344 insertions(+) create mode 100644 text/3735-field-projections.md diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md new file mode 100644 index 00000000000..9f308711749 --- /dev/null +++ b/text/3735-field-projections.md @@ -0,0 +1,1344 @@ +- Feature Name: `field_projection` +- Start Date: 2024-10-24 +- RFC PR: [rust-lang/rfcs#3735](https://github.com/rust-lang/rfcs/pull/3735) +- Rust Issue: [rust-lang/rust#0000](https://github.com/rust-lang/rust/issues/0000) + +# Summary +[summary]: #summary + +Field projections are a very general concept. In simple terms, it is a new operator that turns a +generic container type `C` containing a struct `T` into a container `C` where `F` is a +field of the struct `T`. For example given the struct: + +```rust +struct Foo { + bar: i32, +} +``` + +One can project from `&mut MaybeUninit` to `&mut MaybeUninit` by using the new field +projection operator: + +```rust +impl Foo { + fn initialize(this: &mut MaybeUninit) { + let bar: &mut MaybeUninit = this->bar; + bar.write(42); + } +} +``` + +Special cases of field projections are [pin projections], or projecting raw pointers to fields +`*mut Foo` to `*mut i32` with improved syntax over `&raw mut (*ptr).bar`. + +# Motivation +[motivation]: #motivation + +Field projections are a unifying solution to several problems: +- [pin projections], +- ergonomic pointer-to-field access operations for pointer-types (`*const T`, `&mut MaybeUninit`, + `NonNull`, `&UnsafeCell`, etc.), +- projecting custom references and container types. + +[Pin projections] have been a constant pain point and this feature solves them elegantly while at +the same time solving a much broader problem space. For example, field projections enable the +ergonomic use of `NonNull` over `*mut T` for accessing fields. + +In the following sections, we will cover the basic usage first. And then we will go over the most +complex version that is required for [pin projections] as well as allowing custom projections such +as the abstraction for RCU from the Rust for Linux project (also given below). + +[pin projections]: https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning +[Pin projections]: https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning + +## Ergonomic Pointer-to-Field Operations + +We will use the struct from the summary as a simple example: + +```rust +struct Foo { + bar: i32, +} +``` + +References and raw pointers already possess pointer-to-field operations. Given a variable `foo: &T` +one can write `&foo.bar` to obtain a `&i32` pointing to the field `bar` of `Foo`. The same can be +done for `foo: *const T`: `&raw (*foo).bar` (although this operation is `unsafe`) and their mutable +versions. + +However, the other pointer-like types such as `NonNull`, `&mut MaybeUninit` and +`&UnsafeCell` don't natively support this operation. Of course one can write: + +```rust +unsafe fn project(foo: NonNull) -> NonNull { + let foo = foo.as_ptr(); + unsafe { NonNull::new_unchecked(&raw (*foo).bar) } +} +``` + +But this is very annoying to use in practice, since the code depends on the name of the field and +can thus not be written using a single generic function. For this reason, many people use raw +pointers even though `NonNull` would be more fitting. The same can be said about `&mut +MaybeUninit`. + +There are a lot of types that can benefit from this operation: +- `NonNull` +- `*const T`, `*mut T` +- `&T`, `&mut T` +- `&Cell`, `&UnsafeCell` +- `&mut MaybeUninit`, `*mut MaybeUninit` +- `cell::Ref<'_, T>`, `cell::RefMut<'_, T>` +- `Cow<'_, T>` + +## Pin Projections + +The examples from the previous section are very simple, since they all follow the pattern `C -> +C` where `C` is the respective generic container type and `F` is a field of `T`. + +In order to handle `Pin<&mut T>`, the return type of the field projection operator needs to depend +on the field itself. This is needed in order to be able to project structurally pinned fields from +`Pin<&mut T>` to `Pin<&mut F1>` while simultaneously projecting not structurally pinned fields from +`Pin<&mut T>` to `&mut F2`. + +Fields marked with `#[pin]` are structurally pinned field. For example, consider the following +future: + +```rust +struct FairRaceFuture { + #[pin] + fut1: F1, + #[pin] + fut2: F2, + fair: bool, +} +``` + +One can utilize the following projections when given `fut: Pin<&mut FairRaceFuture>`: +- `fut->fut1: Pin<&mut F1>` +- `fut->fut2: Pin<&mut F2>` +- `fut->fair: &mut bool` + +Using these, one can concisely implement `Future` for `FairRaceFuture`: + +```rust +impl> Future for FairRaceFuture { + type Output = F1::Output; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll { + let fair: &mut bool = self->fair; + *fair = !*fair; + if *fair { + // self->fut1: Pin<&mut F1> + match self->fut1.poll(ctx) { + Poll::Pending => self->fut2.poll(ctx), + Poll::Ready(res) => Poll::Ready(res), + } + } else { + // self->fut2: Pin<&mut F2> + match self->fut2.poll(ctx) { + Poll::Pending => self->fut1.poll(ctx), + Poll::Ready(res) => Poll::Ready(res), + } + } + } +} +``` + +Without field projection, one would either have to use `unsafe` or reach for a third party library +like [`pin-project`] or [`pin-project-lite`] and then use the provided `project` function. + +[`pin-project`]: https://crates.io/crates/pin-project +[`pin-project-lite`]: https://crates.io/crates/pin-project-lite + +## Custom Projections + +This proposal also aims to allow custom field projections. For example a custom pointer type for +"always valid pointers" i.e. mutable references that are allowed to alias and that have no +guarantees with respect to race conditions. Those would be rather annoying to use without field +projection, since one would always have to convert them into raw pointers to go to a field. + +In this section, three examples are presented of custom field projections in the Rust for Linux +project. The first is volatile memory, a pointer that ensures only volatile access to the pointee. +The second is untrusted data, requiring validation before the data can be used for logic. And the +last example is a sketch for a safe abstraction (an API that provides only safe functions to use the +underlying feature) of RCU. It probably requires field projection in order to be able to provide +such a safe abstraction. Also note that this example requires to use field projection as pin +projections, so it is beneficial to read that section first. + +### Rust for Linux Example: Volatile Memory + +In the kernel, sometimes there is the need to solely access memory via volatile operations. Since +combining normal and volatile memory accesses will lead to undefined behavior, a safe abstraction is +required. + +```rust +pub struct VolatileMem { + inner: *mut T, +} + +impl VolatileMem { + pub fn write(&mut self, value: T) + // Required, since we can't drop the value + where T: Copy + { + unsafe { std::ptr::write_volatile(self.inner, value) } + } + + pub fn read(&self) -> T + // Required, since we can't drop the value + where T: Copy + { + unsafe { std::ptr::read_volatile(self.inner) } + } +} +``` + +This design is problematic when `T` is a big struct and one is either only interested in reading a +single field or in modifying a single field. + +```rust +#[derive(Clone, Copy)] +struct Data { + x: i64, + y: u64, + /* imagine lots more fields */ +} + +let data: VolatileMem = /* ... */; +data.write(Data { /* ... */ }); + +// later in the program + +// we only want to change `x`, but have to first read and then write the entire struct. +let d = data.read(); +data.write(Data { x: 42, ..d }); +``` + +This is a big problem, also for correctness, since in some applications of volatile memory, the +value of `data` might change after the read, but before the write. Additionally it is very +inefficient, when the struct is very big. + +Any projection operation would have to be `unsafe`, because the pointer stored in `VolatileMem` is a +raw pointer and there is no way to ensure that the resulting, user-supplied pointer points to a +field of the original value. + +But with custom field projections, one could simply do this instead: + +```rust +data->x.write(42); +``` + +### Rust for Linux Example: Untrusted Data + +In the Linux kernel, data coming from hardware or userspace is untrusted. This means that the data +must be validated before it is used for *logic* inside the kernel. Copying it into userspace is fine +without validation, but indexing some structure requires to first validate the index. + +For the exact details, see the [untrusted data patch +series](https://lore.kernel.org/rust-for-linux/20240925205244.873020-1-benno.lossin@proton.me/). It +introduces the `Untrusted` type used to mark data as untrusted. Kernel developers are supposed to +validate such data before it is used to drive logic within the kernel. Thus this type prevents +reading the data without validating it first. + +One use case of untrusted data will be ioctls. They were discussed in version 1 in [this +reply](https://lore.kernel.org/rust-for-linux/ZvU6JQEN_92nPH4k@phenom.ffwll.local/) (slightly +adapted the code): +> Example in pseudo-rust: +> +> ```rust +> struct IoctlParams { +> input: u32, +> ouptut: u32, +> } +> ``` +> +> The thing is that ioctl that use the struct approach like drm does, use the same struct if there's +> both input and output paramterers, and furthermore we are not allowed to overwrite the entire +> struct because that breaks ioctl restarting. So the flow is roughly +> +> ```rust +> let userptr: UserSlice; +> let params: Untrusted; +> +> userptr.read(params)); +> +> // validate params, do something interesting with it params.input +> +> // this is _not_ allowed to overwrite params.input but must leave it +> // unchanged +> +> params.write(|x| { x.output = 42; }); +> +> userptr.write(params); +> ``` +> +> Your current write doesn't allow this case, and I think that's not good enough. The one I propsed +> in private does: +> +> ```rust +> Untrusted::write(&mut self, impl Fn(&mut T)) +> ``` + +Importantly, we would like to only overwrite the `output` field of the `IoctlParams` struct. This is +the exact pattern that field projections can help with, instead of exposing a mutable reference to +the untrusted data via the `write` function, we can have: + +```rust +impl Untrusted { + fn write(&mut self, value: T); +} +``` + +In addition to allowing projections of `&mut Untrusted` to `&mut Untrusted`, thus +allowing to overwrite parts of a struct with field projections. + +### Rust for Linux Example: RCU + +RCU stands for read, copy, update. It is a creative locking mechanism that is very efficient for +data that is seldomly updated, but read very often. Below you can find a small summary of how I +understand it to work. No guarantees that I am 100% correct, if you want to make sure that you have +a correct understanding of how RCU works, please read the sources provided in the next section. + +It requires quite a lot of explaining until I can express why field projection comes up in this +instance. However, in this case (similar to `Pin`) it is (to my knowledge) impossible to write a +safe API without field projections, so they would be invaluable for this use case. + +#### RCU Explained + +For a much more extensive explanation, please see . +Since the first paragraph of the first section is invaluable in understanding RCU, it is quoted here +for the reader's convenience: + +> The basic idea behind RCU is to split updates into “removal” and “reclamation” phases. The removal +> phase removes references to data items within a data structure (possibly by replacing them with +> references to new versions of these data items), and can run concurrently with readers. The reason +> that it is safe to run the removal phase concurrently with readers is the semantics of modern CPUs +> guarantee that readers will see either the old or the new version of the data structure rather +> than a partially updated reference. The reclamation phase does the work of reclaiming (e.g., +> freeing) the data items removed from the data structure during the removal phase. Because +> reclaiming data items can disrupt any readers concurrently referencing those data items, the +> reclamation phase must not start until readers no longer hold references to those data items. + +In C, RCU is used like this: +- the data protected by RCU sits behind a pointer, +- readers must use the [`rcu_read_lock()`](https://docs.kernel.org/RCU/whatisRCU.html#rcu-read-lock) + and [`rcu_read_unlock()`](https://docs.kernel.org/RCU/whatisRCU.html#rcu-read-unlock) functions + when accessing any data protected by RCU, within this critical section, blocking is forbidden. +- read accesses of the pointer must only be done after calling + [`rcu_dereference()`](https://docs.kernel.org/RCU/whatisRCU.html#rcu-dereference). +- write accesses of the pointer must be done via [`rcu_assign_pointer(, + )`](https://docs.kernel.org/RCU/whatisRCU.html#rcu-assign-pointer). +- before a writer frees the old value (i.e. it enters into the reclamation phase), they must call + [`synchronize_rcu()`](https://docs.kernel.org/RCU/whatisRCU.html#synchronize-rcu). +- multiple writers **still require** some other kind of locking mechanism. + +`synchronize_rcu()` waits for all existing read-side critical sections to complete. It does not have +to wait for new read-side critical sections that are begun after it has been called. + +The big advantage of RCU is that in certain kernel configurations, (un)locking the RCU read lock is +achieved with absolutely no instructions. + +#### A Safe Abstraction for RCU + +In Rust, we will of course use a guard for the RCU read lock, so we have: + +```rust +mod rcu { + pub struct Guard(/* ... */); + + impl Drop for Guard { /* ... */ } + + pub fn read_lock() -> Guard; +} +``` + +The pointers that are protected by RCU must be specially tagged, so we introduce the `Rcu` type. It +exposes the Rust equivalents of `rcu_dereference` and `rcu_assign_pointer` [^1]: + +[^1]: Note that the requirement of not blocking in a critical RCU section is not expressed in code. + Instead we use an external tool called [`klint`] for that purpose. + +[`klint`]: https://rust-for-linux.com/klint + +```rust +mod rcu { + pub struct Rcu

{ + inner: UnsafeCell

, + // we require this to opt-out of uniqueness of `&mut`. + // if `UnsafePinned` were available, we would use that instead. + _phantom: PhantomPinned, + } + + impl Rcu

{ + pub fn read<'a>(&'a self, _guard: &'a RcuGuard) -> &'a P::Target; + pub fn set(self: Pin<&mut Self>, new: P) -> Old

; + } + + pub struct Old

(/* ... */); + + impl

Drop for Old

{ + fn drop() { + unsafe { bindings::synchronize_rcu() }; + } + } +} +``` + +The `Old` type is responsible for calling `synchronize_rcu` before dropping the old value. + +Note that `set` takes a pinned mutable reference to `Rcu`. This is important, since it might not be +obvious why there is pinning involved here. Firstly, we need to take a mutable reference, since +writers still need to be synchronized. Secondly, since there are still concurrent shared references, +we must not allow users to use `mem::swap`, since that would change the value without the required +compiler and CPU barriers in place. + +Now to the crux of the issue and why field projection comes up here: A common use-case of RCU is to +protect data inside of a struct that is itself protected by a lock. Since the data is protected by +RCU, we don't need to hold the lock to read the data. However, locks do not allow access to the +inner value without locking it (that's kind of their whole point...). So we need a way to get to the +`Rcu

` without locking the lock. Using field projection, we would allow projections for fields of +type `Rcu` from `&Lock` to `&Rcu

`. + +This way, readers can use field projection and the `Rcu::read` function and writers can continue to +lock the lock and then use `Rcu::set`. + +#### RCU API Usage Examples + +```rust +struct BufferConfig { + flush_sensitivity: u8, +} + +struct Buffer { + // We also require `Rcu` to be pinned, because `&mut Rcu` must not exist (otherwise one could + // call mem::swap). + #[pin] + cfg: Rcu>, + buf: Vec, +} + +struct MyDriver { + // The `Mutex` in the kernel needs to be pinned. + #[pin] + buf: Mutex, +} +``` + +Here the struct that is protected by the lock is `Buffer` and the data that is protected by RCU +inside of this struct is `BufferConfig`. To read the config, we now don't have to lock the lock, +instead we can read it using field projection: + +```rust +impl MyDriver { + fn buffer_config<'a>(&'a self, rcu_guard: &'a RcuGuard) -> &'a BufferConfig { + let buf: &Mutex = &self.buf; + // Here we use the special projections set up for `Mutex` with fields of type `Rcu`. + let cfg: &Rcu> = buf->cfg; + cfg.read(rcu_guard) + } +} +``` + +To set the buffer config, one has to hold the lock: + +```rust +impl MyDriver { + fn set_buffer_config(&self, flush_sensitivity: u8) { + // Our `Mutex` pins the value. + let mut guard: Pin> = self.buf.lock(); + let buf: Pin<&mut Buffer> = guard.as_mut(); + // We can use pin-projections since we marked `cfg` as `#[pin]` + let cfg: Pin<&mut Rcu>> = buf->cfg; + cfg.set(Box::new(BufferConfig { flush_sensitivity })); + // ^^ this returns an `Old>` and runs `synchronize_rcu` on drop. + } +} +``` + +And of course one can still use other fields normally, but now requires field projection, since +`Pin<&mut T>` is involved: + +```rust +impl MyDriver { + fn read_to_buffer(&self, data: &[u8]) -> Result { + let mut buf: Pin> = self.buf.lock(); + // This method allocates, so it must be fallible. + // `buf.as_mut()->buf` again uses the field projection for `Pin` to yield a `&mut Vec`. + buf.as_mut()->buf.extend_from_slice(data) + } +} +``` + +# Guide-level explanation +[guide-level-explanation]: #guide-level-explanation + +## Rust Book Chapter: Field Projections + +When programming in Rust, one often has the need to access only a single field of a struct. In the +usual cases of `&T` or `&mut T`, this is simple. Just use dot syntax and you can create a reference +to a field of the struct `&t.field`. + +However, when one has a different type that "contains" or "points" at a `T`, one has to reach for +*field projections* via the *field projection operator* `->`. In this chapter, we will learn what +field projections are and how to use them for the most common types from the standard library. For +example for pointer-like types and [pin projections]. + +### Simple Uses of Field Projections + +Let's say we have a big struct that doesn't fit onto the stack: + +```rust +struct Data { + flags: u32, + buf: [u8; 1024 * 1024], +} +``` + +We would like to initialize the bytes in `buf` to `0xff` and `flags` should be `0x0f`. We start with +a new function returning memory on the heap: + +```rust +impl Data { + fn new() -> Box { + let mut data = Box::new_uninit(); + { + let data: &mut MaybeUninit = &mut *data; +``` + +Now we can use field projection to turn `&mut MaybeUninit` into `&mut MaybeUninit` that +points to the `flags` field: + +```rust + let flags: &mut MaybeUninit = data->flags; + flags.write(0x0f); +``` + +And to initialize `buf`, we can do the same: + +```rust + let buf: &mut MaybeUninit<[u8]> = data->buf; + let buf: &mut [MaybeUninit] = MaybeUninit::slice_as_bytes_mut(buf); + MaybeUninit::fill(buf, 0xff); + } +``` + +Now we only need to unsafely assert that we initialized everything. + +```rust + unsafe { data.assume_init() } + } +} +``` + +A more general explanation of field projection is that it is an operation that turns a generic +container type `C` containing a struct `T` into a container `C` where `F` is a field of the +struct `T`. + +#### Raw Pointers + +Similarly to `&mut MaybeUninit`, raw pointers also support projections. Given a raw pointer +`ptr: *mut Data`, one can use field projection to obtain a pointer to a field: +`ptr->flags: *mut u32`. Essentially `ptr->field` is a shorthand for `&raw mut (*ptr).field` (for +`*const` the same is true except for the `mut`). However, there is a small difference between the +two: the latter has to be `unsafe`, since `*ptr` requires that `ptr` be dereferencable. But field +projection is a safe operation and thus it uses [`wrapping_add`] under the hood. This is less +efficient, as it prevents certain optimizations. If that is a problem, either use `&raw [mut] +(*ptr).field` or create a custom pointer type that represents an always dereferencable pointer and +implement field projections using `unsafe`. + +[`wrapping_add`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.wrapping_add + +Another pointer type that supports field projection is `NonNull`. For example, if we had to add a +function that sets the `flags` field given only a `NonNull`, we could do so: + +```rust +impl Data { + unsafe fn set_flags_raw(this: NonNull, flags: u32) { + let ptr: NonNull = this->flags; + unsafe { ptr.write(flags) }; + } +} +``` + +#### `RefCell`'s References + +Even the "exotic" references of `RefCell` i.e. [`cell::Ref<'_, T>`] and [`cell::RefMut<'_, T>`] +are supporting field projection. + +[`cell::Ref<'_, T>`]: https://doc.rust-lang.org/std/cell/struct.Ref.html +[`cell::RefMut<'_, T>`]: https://doc.rust-lang.org/std/cell/struct.RefMut.html + +In this example, we create a buffer that tracks the various operations done to it for debug +purposes. + +```rust +struct Buffer { + stats: RefCell, + buf: VecDeque, +} + +struct Stats { + ops: Vec, + elements_pushed: usize, + elements_popped: usize, +} +``` + +There are three operations, one for pushing a number of elements, one other for popping them and the +last one for peeking at the elements in the buffer. + +```rust +enum Operation { + Push(usize), + Pop(usize), + Peek, +} +``` + +When pushing and popping, we have a mutable reference to the buffer and could just use `Stats` +without the `RefCell`. But in the peek case, we only have a shared reference and still require to +record the statistic. + +Pushing and popping are very simple: + +```rust +impl Buffer { + fn push(&mut self, items: &[T]) + where + T: Clone, + { + let mut stats = self.stats.borrow_mut(); + stats.ops.push(Operation::Push(items.len())); + stats.elements_pushed += items.len(); + self.buf.extend(items.iter().cloned()); + } + + fn pop(&mut self, count: usize) -> Option> { + let count = count.min(self.buf.len()); + let mut stats = self.stats.borrow_mut(); + stats.ops.push(Operation::Pop(count)); + stats.elements_popped += count; + if count == 0 { + return None; + } + let mut res = Box::new_uninit_slice(count); + for i in 0..count { + let Some(val) = self.buf.pop_front() else { + // we took the minimum above. + unreachable!() + }; + res[i].write(val); + } + Some(unsafe { res.assume_init() }) + } +} +``` + +Peeking also is rather easy: + +```rust +impl Buffer { + fn peek(&self) -> Option<&T> { + self.stats.borrow_mut().ops.push(Operation::Peek); + self.buf.front() + } +} +``` + +Now we come to the part where we need field projections. We would like to be able to access the +operation statistics from other code. But because it is wrapped in `RefCell`, we cannot give a +reference out: + +```rust +impl Buffer { + fn stats(&self) -> &Vec { +// error[E0515]: cannot return value referencing temporary value + &self.stats.borrow().ops +// ^-------------------^^^^ +// || +// |temporary value created here +// returns a value referencing data owned by the current function + } +} +``` + +That is because the value returned by `borrow` is placed on the stack and must be kept alive for +bookkeeping purposes of `RefCell` until the borrow ends. But using field projection, we can return a +`cell::Ref`: + +```rust +impl Buffer { + fn stats(&self) -> cell::Ref<'_, Vec> { + self.stats.borrow()->ops + } +} +``` + +We could even hide the fact that the stats are implemented using `RefCell` using an opaque type: + +```rust +impl Buffer { + fn stats(&self) -> impl Deref> + '_ { + self.stats.borrow()->ops + } +} +``` + +### Complicated Field Projections + +Field projection is even more powerful than what we have seen until now. The returned type of the +projection operator can even depend on the field itself! + +This enables them to be used for making [pin projections] ergonomic. We will discuss how to use +this way of pin projection in the next section. + +#### Pin Projections + +For this section, you should understand what [pin projections] are. If not, then you can just skip +this section. + +Structurally pinned fields are marked with `#[pin]` using the derive macro `PinProject`. For example +consider a future that alternatingly polls two futures: + +```rust +#[derive(PinProject)] +struct FairRaceFuture { + #[pin] + fut1: F1, + #[pin] + fut2: F2, + fair: bool, +} +``` + +Now, it's possible to project a `fut: Pin<&mut FairRaceFuture>`: +- `fut->fut1: Pin<&mut F1>` +- `fut->fut2: Pin<&mut F2>` +- `fut->fair: &mut bool` + +Using these, one can concisely implement `Future` for `FairRaceFuture` without any `unsafe` code: + +```rust +impl> Future for FairRaceFuture { + type Output = F1::Output; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll { + let fair: &mut bool = self->fair; + *fair = !*fair; + if *fair { + // self->fut1: Pin<&mut F1> + match self->fut1.poll(ctx) { + Poll::Pending => self->fut2.poll(ctx), + Poll::Ready(res) => Poll::Ready(res), + } + } else { + // self->fut2: Pin<&mut F2> + match self->fut2.poll(ctx) { + Poll::Pending => self->fut1.poll(ctx), + Poll::Ready(res) => Poll::Ready(res), + } + } + } +} +``` + +## Impact of this Feature + +Overall this feature improves readability of code, because it replaces more complex to parse syntax +with simpler syntax: +- `&raw mut (*ptr).foo` is turned into `ptr->foo` +- using `NonNull` as a replacement for `*mut T` becomes a lot better when accessing fields, + +There is of course a cost associated with introducing a new operator along with the concept of field +projections. + +# Reference-level explanation +[reference-level-explanation]: #reference-level-explanation + +## Implementation Details + +In order to facilitate field projections, several interlinked concepts have to be introduced. These +concepts are: + +- [field types], + - `Field` trait, + - `field_of!` macro, + - [`#[projecting]`](#projecting-attribute) attribute +- projection operator `->`, + - `Project` trait, + - `Projectable` trait, + +To ease understanding, here is a short explanation of the interactions between these concepts. The +following subsections explain them in more detail, so refer to them in cases of ambiguity. The +projection operator `->` is governed by the `Project` trait that has `Projectable` as a super trait. +`Projectable` helps to select the struct whose fields are used for projection. Field types store +information about a field (such as the base struct and the field type) via the `Field` trait and the +`field_of!` macro makes it possible to name the [field types]. Finally, the +[`#[projecting]`](#projecting-attribute) attribute allows `repr(transparent)` structs to be ignored +when looking for fields for projection. + +### Field Types +[field type]: #field-types +[field types]: #field-types + +The compiler generates a compiler-internal type for every field of every struct. These types can +only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields +accessible to the current scope can be projected. These types are called *field types*. + +Field types implement the `Field` trait: + +```rust +pub trait Field { + type Base: ?Sized; + type Type: ?Sized; + + const OFFSET: usize; +} +``` + +In the implementation of this trait, `Base` is set to the struct that the field is part of and +`Type` is set to the type of the field. `OFFSET` is set to the offset in bytes of the field in +the struct (i.e. `OFFSET = offset_of!(Base, ident)`). + +In addition to all fields of all structs, field types are generated for transparent, +[`#[projecting]`](#projecting-attribute) container types as follows: given a transparent, generic type annotated with +[`#[projecting]`](#projecting-attribute) and a struct contained in it: + +```rust +#[projecting] +#[repr(transparent)] +pub struct Container { + inner: T, +} + +struct Foo { + bar: i32, +} +``` + +The type `Container` inherits all fields of `Foo` with `Base` and `Type` adjusted accordingly +(i.e. wrapped by `Container`): + +```rust +fn project(r: &F::Base) -> &F::Type; + +let x: Container; +let _: &Container = project::, bar)>(&x); +``` + +The implementation of the `Field` trait sets the associated types and constant like this: +- `Base = Container` +- `Type = Container` +- `OFFSET = offset_of!(Foo, bar)` + +This can even be done for multiple levels: `Container>` also has a [field type] `bar` +of type `Container>`. Mixing different container types is also possible. + +Annotating a struct with [`#[projecting]`](#projecting-attribute) disables projection via that structs own fields. +Continuing the example from above: + +```rust +struct Bar {} + +// ERROR: `Container` does not have a field `inner`. `Container` is annotated with +// `#[projecting]` and thus the field types it exposes are changed to the wrapped type. `Bar` does +// not have a field `inner`. +type X = field_of!(Container, inner); + +struct Baz { + inner: Foo, +} + +// this refers to the field `inner` of `Baz`. +type Y = field_of!(Container, inner); +// it has the following implementation of `Field`: +impl Field for Y { + type Base = Container; + type Type = Container; + + const OFFSET: usize = offset_of!(Baz, inner); +} +``` + +#### `Field` Trait + +The field trait is added to `core::marker` and cannot be implemented manually. + +```rust +pub trait Field { + type Base: ?Sized; + type Type: ?Sized; + + const OFFSET: usize; +} +``` + +The compiler automatically implements it for all [field types]. Users of the trait are allowed to rely +on the associated types and constants in `unsafe` code. So for example this piece of code is sound: + +```rust +fn get_field(base: &F::Base) -> &F::Type +where + // required to be able to `cast` + F::Type: Sized, + F::Base: Sized, +{ + let ptr: *const Base = base; + let ptr: *const u8 = base.cast::(); + // SAFETY: `ptr` is derived from a reference and the `Field` trait is guaranteed to contain + // correct values. So `F::OFFSET` is still within the `F::Base` type. + let ptr: *const u8 = unsafe { ptr.add(F::OFFSET) }; + let ptr: *const F::Type = ptr.cast::(); + // SAFETY: The `Field` trait guarantees that at `F::OFFSET` we find a field of type `F::Type`. + unsafe { &*ptr } +} +``` + +#### `field_of!` Macro + +Also added to `core::marker` is the following built-in macro: + +```rust +pub macro field_of($Container:ty, $($fields:expr)+ $(,)?) { + /* built-in macro */ +} +``` + +It has the same syntax as the `offset_of!` macro and returns the respective [field type]. It emits an +error in the following cases: + +```rust +pub mod foo { + pub struct Foo { + bar: u32, + pub baz: i32, + } + + // Error: unknown field `barr` of type `Foo` + type FooBar = field_of!(Foo, barr); +} + +pub mod bar { + // Error: field `bar` of type `Foo` is private + type FooBar = field_of!(Foo, bar); +} +``` + +#### `#[projecting]` Attribute + +The `#[projecting]` attribute can be put on a struct or union declaration. It requires that the +type is `#[repr(transparent)]` and that there is a unique field that has non-zero size in general. +It results in fields of the single non-zero sized type being considered fields of the outer type. + +So for example: + +```rust +#[projecting] +#[repr(transparent)] +pub struct Container { + inner: T, +} + +struct Foo { + bar: i32, +} +``` + +Now `Container` has a [field type] associated with `bar` implementing `Field` with: +- `Base = Container` +- `Type = Container` +- `OFFSET = offset_of!(Foo, bar)` + +The same is true for the non-generic case: + +```rust +#[projecting] +#[repr(transparent)] +pub struct Container { + inner: Foo, +} + +struct Foo { + bar: i32, +} +``` + +Here are some error examples: + +```rust +// ERROR: missing `#[repr(transparent)]` +#[projecting] +pub struct Container { + inner: T, +} + +// ERROR: no field to project onto found, the struct has no fields +#[projecting] +#[repr(transparent)] +pub struct Container {} + +// ERROR: no field to project onto found, all fields are always zero-sized +#[projecting] +#[repr(transparent)] +pub struct Container { + phantom1: PhantomData T>, + phantom2: PhantomData, +} +``` + +### Field Projection Operator + +The field projection operator `->` has the following syntax: + +> **Syntax**\ +> _ProjectionExpression_ :\ +>    [_Expression_] `->` _ProjectionMember_ +> +> _ProjectionMember_ :\ +>       [IDENTIFIER] +>    | [TUPLE_INDEX] + +[IDENTIFIER]: https://doc.rust-lang.org/reference/identifiers.html +[_Expression_]: https://doc.rust-lang.org/reference/expressions.html +[TUPLE_INDEX]: https://doc.rust-lang.org/reference/tokens.html#tuple-index + +#### Desugaring + +```rust +struct T { + field: F, +} + +let t: C = /* ... */; +let _ = t->field; + +// becomes + +let _ = Project:: as Projectable>::Inner, field)>::project(t); +``` + +The `C` in the `C as Projectable` comes from a type inference variable over the expression +`t`. + + +#### `Projectable` & `Project` Traits + +Added to `core::ops`: + +```rust +pub trait Projectable: Sized { + type Inner: ?Sized; +} + +pub trait Project: Projectable +where + F: Field, +{ + type Output; + + fn project(self) -> Self::Output; +} +``` + +## Stdlib Field Projections + +All examples from the guide-level explanation work when the standard library is extended with the +implementations detailed below. + +The following pointer types get an implementation for `Projectable` with `Inner = T`. They support +projections for any field and perform the obvious offset operation. + +- `&mut T` +- `&T` +- `*mut T` +- `*const T` +- `NonNull` +- `Cow<'_, T>` + +The following types get annotated with [`#[projecting]`](#projecting-attribute): + +- `MaybeUninit` +- `Cell` +- `UnsafeCell` +- `SyncUnsafeCell` + +### Pin Projections + +In order to provide [pin projections], a new derive macro `PinProject` and a trait `PinField` is +required: + +```rust +pub unsafe trait PinField: Field { + type Projected<'a>; + + fn from_pinned_ref<'a>(r: &'a mut Self::Type) -> Self::Projected<'a>; +} +``` + +The trait is `unsafe`, because it must only be implemented by the `PinProject` derive marco. + +```rust +#[derive(PinProject)] +struct FairRaceFuture { + #[pin] + fut1: F1, + #[pin] + fut2: F2, + fair: bool, +} +``` + +It expands the above to: + +```rust +struct FairRaceFuture { + fut1: F1, + fut2: F2, + fair: bool, +} + +unsafe impl PinField for field_of!(FairRaceFuture, fut1) { + type Projected<'a> = Pin<&'a mut F1>; + + fn from_pinned_ref<'a>(r: &'a mut F1) -> Pin<&'a mut F1> { + unsafe { Pin::new_unchecked(r) } + } +} + +unsafe impl PinField for field_of!(FairRaceFuture, fut2) { + type Projected<'a> = Pin<&'a mut F2>; + + fn from_pinned_ref<'a>(r: &'a mut F2) -> Pin<&'a mut F2> { + unsafe { Pin::new_unchecked(r) } + } +} + +unsafe impl PinField for field_of!(FairRaceFuture, fut2) { + type Projected<'a> = &'a mut bool; + + fn from_pinned_ref<'a>(r: &'a mut bool) -> &'a mut bool { + r + } +} +``` + +Now the only component that is left is an implementation of `Projectable` and `Project` for +`Pin<&mut T>`: + +```rust +impl<'a, T: ?Sized> Projectable for Pin<&'a mut T> { + type Inner = T; +} + +impl<'a, T, F> Project for Pin<&'a mut T> +where + F: Field + PinField, +{ + type Output = ::Projected<'a>; + + fn project(self) -> Self::Output { + let r: &mut T = unsafe { Pin::into_inner_unchecked(self) }; + let r: &mut F::Type = <&mut T as Project::>::project(r); + ::from_pinned_ref(r) + } +} +``` + +## Interactions + +There aren't a lot of interactions with other features. + +The projection operator binds very tightly: + +```rust +*ctr->field = *(ctr->field); + +&mut ctr->field = &mut (ctr->field); + +ctr->field.foo() = (ctr->field).foo(); + +ctr.foo()->field = (ctr.foo())->field; + +ctr->field->bar = (ctr->field)->bar; +``` + +# Drawbacks +[drawbacks]: #drawbacks + +- [Pin projections] still require library level support via a proc macro and a trait solely for + [field types]. + +# Rationale and alternatives +[rationale-and-alternatives]: #rationale-and-alternatives + +This proposal is a lot more general than just improving [pin projections]. It not only covers +pointer-like types, but also permits all sorts of operations generic over fields. + +Not adding this feature will result in the proliferation of `*mut T` over more suitable pointer +types that better express the invariants of the pointer. The ergonomic cost of +`unsafe { MyPtr::new_unchecked(&raw mut (*my_ptr.as_ptr()).field) }` is just to great to be useful +in practice. + +While [pin projections] can be addressed via a library or a separate feature, not having them in the +language takes a toll on projects trying to minimize dependencies. The Rust for Linux project is +already using pinning extensively, since all locking primitives require it; a library solution will +never be as ergonomic as a language-level construct. Thus that project would benefit greatly from +this feature. + +Additionally, safe RCU abstractions are likely impossible without field projections, since they +require being generic over the fields of structs. + +Field projections are on first contact rather difficult to understand, especially the instantiation +as [pin projections]. However, they are a very natural operation, extending the already existent +features of raw pointers and references. Therefore they are fairly easy to adjust to; and in turn, +they provide a big increase in readability of the code, expressing the concept of field projection +concisely. The compiler changes are rather manageable, reusing several already existing systems, +thus increasing the maintenance burden only slightly if at all. + +# Prior art +[prior-art]: #prior-art + +Most importantly, see the [old field projection RFC](http://github.com/rust-lang/rfcs/pull/3318). +There also was a [pre-old-RFC +discussion](https://internals.rust-lang.org/t/pre-rfc-field-projection/17383/57) and the +list of crates in the next section is also from the old RFC. + +## Crates + +There are several crates implementing projections for different types. + +- [pin projections] + - [`pin-project`] provides pin projections via a proc macro on the type specifying the + structurally pinned fields. At the projection-site the user calls a projection function + `.project()` and then receives a type with each field replaced with the respective projected + field. + - [cell-project] provides cell projection via a macro at the projection-site: the user writes + `cell_project!($ty, $val.$field)` where `$ty` is the type of `$val`. Internally, it uses unsafe + to facilitate the projection. + - [pin-projections] provides pin projections, it differs from [`pin-project`] by providing + explicit projection functions for each field. It also can generate other types of getters for + fields. [`pin-project`] seems like a more mature solution. +- `&[mut] MaybeUninit` projections + - [project-uninit] provides uninit projections via macros at the projection-site uses `unsafe` + internally. +- multiple of the above + - [`field-project`] provides projection for `Pin<&[mut] T>` and `&[mut] MaybeUninit` via a macro + at the projection-site: the user writes `proj!($var.$field)` to project to `$field`. + - [`field-projection`] is an experimental crate that implements general field projections via a + proc-macro that hashes the name of the field to create unique types for each field that can then + implement traits to make different output types for projections. + +[`field-project`]: https://crates.io/crates/field-project +[`cell-project`]: https://crates.io/crates/cell-project +[`pin-projections`]: https://crates.io/crates/pin-projections +[`project-uninit`]: https://crates.io/crates/project-uninit +[`field-projection`]: https://crates.io/crates/field-projection + +## Blog Posts and Discussions + +- [Safe Cell field projection in + Rust](https://www.abubalay.com/blog/2020/01/05/cell-field-projection) +- [Field projdection for `Rc` and + `Arc`](https://internals.rust-lang.org/t/field-projection-for-rc-and-arc/15827) +- [Generic Field Projection](https://internals.rust-lang.org/t/generic-field-projection/16204) + +Blog posts about pin (projections): +- [Pinned places](https://without.boats/blog/pinned-places/) +- [Overwrite trait](https://smallcultfollowing.com/babysteps/series/overwrite-trait/) + +## Rust and Other Languages + +Rust already has a precedent for compiler-generated types. All functions and closures have a unique, +unnameable type. + +In C++ there are field projections supported on `std::shared_ptr`, it consists of two pointers, one +pointing to the reference count and the other to the data. Making it possible to project down to a +field and still take a reference count on the entire struct, keeping also the field alive. + +# Unresolved questions +[unresolved-questions]: #unresolved-questions + +## Syntax Bikeshedding + +What is the right syntax for the various operations given in this RFC? + +Ideally we would have a strong opinion when this feature is implemented. But the decision should +only be finalized when stabilizing the feature. + +### Field Projection Operator + +Current favorite: `$base:expr->$field:ident`. + +Alternatives: +- use `~` instead of `->` + +### Naming Field Types + +Current favorite: `field_of!($Container:ty, $($fields:expr)+ $(,)?)` macro with `offset_of!` syntax. + +Alternatives: +- Introduce a more native syntax on the level of types `$Container:ty->$field:ident` akin to + projecting an expression. + +### Declaring a Transparent Container Type + +Current favorite: [`#[projecting]`](#projecting-attribute) attribute. + +Alternatives: +- extend the `repr` attribute with + - `projecting` + - `flatten` +- use `#[flatten]` instead. + +## Other + +- should the [`#[projecting]`](#projecting-attribute) attribute have an associated field attribute + to mark the field that is projected onto? + +# Future possibilities +[future-possibilities]: #future-possibilities + +## Enums + +Enums are difficult to support with the same framework as structs. The problem is that many +containers don't provide sufficient guarantees to read the discriminant (for example raw pointers +and `&mut MaybeUninit`). However, for types that do provide sufficient guarantees, one could cook +up a similar feature. Let's call them *enum projections*. They could work like this: projecting is +done via a new kind of match operator: + +```rust +enum MyEnum { + A(i32, String), + B(#[pin] F), +} +type F = impl Future; +let x: Pin<&mut MyEnum>; +match_proj x { + MyEnum::A(n, s) => { + let _: &mut i32 = n; + let _: &mut String = s; + } + MyEnum::B(fut) => { + let _: Pin<&mut F> = fut; + } +} +``` + +Here `match_proj` would need to be a new keyword. I dislike the name and syntax, but haven't come up +with something better. + +## Arrays + +Arrays can be thought of structs/tuples where each index is a field. Supporting them would simply +follow tuples. They might need additional syntax or just use the tuple syntax. + +## Unions + +Since field access for unions is `unsafe`, projection would also have to be `unsafe`. Since unions +are rarely used directly, this probably isn't important. + +## More Stdlib Additions + +Types that might be good candidates for [`#[projecting]`](#projecting-attribute): + +- `ManuallyDrop` From a3a774982e6b9c94804c82b97335d3dc371c474b Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 14:17:08 +0100 Subject: [PATCH 02/36] fix typos --- text/3735-field-projections.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 9f308711749..5d2858fcfb7 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -260,7 +260,7 @@ adapted the code): > let userptr: UserSlice; > let params: Untrusted; > -> userptr.read(params)); +> userptr.read(params); > > // validate params, do something interesting with it params.input > @@ -344,11 +344,11 @@ In Rust, we will of course use a guard for the RCU read lock, so we have: ```rust mod rcu { - pub struct Guard(/* ... */); + pub struct RcuGuard(/* ... */); - impl Drop for Guard { /* ... */ } + impl Drop for RcuGuard { /* ... */ } - pub fn read_lock() -> Guard; + pub fn read_lock() -> RcuGuard; } ``` From bf0d342ad3b7009d04ce95777f0bc96265e1a215 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 14:19:25 +0100 Subject: [PATCH 03/36] add attibution for enum projections --- text/3735-field-projections.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 5d2858fcfb7..bf33264bdd7 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1324,6 +1324,10 @@ match_proj x { } ``` +I got this idea from reading the [Pinned places](https://without.boats/blog/pinned-places/) blog +post from boats. There, enum projections for pinned references (i.e. just pin projections) are +discussed. + Here `match_proj` would need to be a new keyword. I dislike the name and syntax, but haven't come up with something better. From ae64e46818a1886d254e607b85db5fbbb6db0b69 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 18:35:27 +0100 Subject: [PATCH 04/36] fix `PinField` trait - make from_pinned_ref `unsafe` - add safety documentation --- text/3735-field-projections.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index bf33264bdd7..1ced3b47e5f 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1068,14 +1068,22 @@ In order to provide [pin projections], a new derive macro `PinProject` and a tra required: ```rust +/// # Safety +/// +/// - `Self::Projected` is set to either `&'a mut Self::Type` or `Pin<&'a mut Self::Type>`, +/// - `from_pinned_ref` must either be the identity function, or return the argument wrapped in +/// `Pin` (either with `Pin::new_unchecked` or `Pin::new`) pub unsafe trait PinField: Field { type Projected<'a>; - fn from_pinned_ref<'a>(r: &'a mut Self::Type) -> Self::Projected<'a>; + /// # Safety + /// + /// `r` must point at a field of the struct `Self::Base`. That struct value must be pinned. + unsafe fn from_pinned_ref<'a>(r: &'a mut Self::Type) -> Self::Projected<'a>; } ``` -The trait is `unsafe`, because it must only be implemented by the `PinProject` derive marco. +An example use is: ```rust #[derive(PinProject)] From 7cdee51eb53c46f9a96863c08bdc330889d74581 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 18:58:56 +0100 Subject: [PATCH 05/36] add prior art: design meeting field projection --- text/3735-field-projections.md | 1 + 1 file changed, 1 insertion(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 1ced3b47e5f..fe4e5a9a547 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1244,6 +1244,7 @@ There are several crates implementing projections for different types. ## Blog Posts and Discussions +- [Design Meeting Field Projection](https://hackmd.io/@y86-dev/SkkfRkzWh) - [Safe Cell field projection in Rust](https://www.abubalay.com/blog/2020/01/05/cell-field-projection) - [Field projdection for `Rc` and From fb938d5c2e9a2ee866315556b2a5a68ec3e5c6d7 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 20:19:10 +0100 Subject: [PATCH 06/36] add impl of `Projectable` and `Project` for `&T` and `Cow<'_, T>` --- text/3735-field-projections.md | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index fe4e5a9a547..cb552b3ac86 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1055,6 +1055,70 @@ projections for any field and perform the obvious offset operation. - `NonNull` - `Cow<'_, T>` +For example, `&T` would be implemented like this: + +```rust +impl<'a, T: ?Sized> Projectable for &'a T { + type Inner = T; +} + +impl<'a, T: ?Sized, F> Project for &'a T +where + F: Field, + // Needed to be able to `.cast` below + F::Type: Sized, +{ + type Output = &'a F::Type; + + fn project(self) -> Self::Output { + let ptr: *const T = self; + let ptr = ptr.cast::(); + let ptr = unsafe { ptr.add(F::OFFSET) }; + let ptr = ptr.cast::(); + unsafe { &*ptr } + } +} +``` + +And `Cow` would be implemented like this: + +```rust +impl<'a, T: ?Sized> Projectable for Cow<'a, T> { + type Inner = T; +} + +impl<'a, T: ?Sized, F> Project for Cow<'a, T> +where + F: Field, + F::Type: Sized, +{ + type Output = Cow<'a, F::Type>; + + fn project(self) -> Self::Output { + match self { + Cow::Borrowed(this) => Cow::Borrowed(<&T as Project::>::project(this)), + Cow::Owned(this) => Cow::Owned(project_value::(this)), + } + } +} +``` + +Where `project_value` is defined as: + +```rust +fn project_value(value: T) -> F::Type +where + F: Field, + F::Type: Sized, +{ + let r = &value; + let r = <&T as Project>::project(r); + let res = std::ptr::read(r); + std::mem::forget(value); + res +} +``` + The following types get annotated with [`#[projecting]`](#projecting-attribute): - `MaybeUninit` From 27d514617f94ba766334e7e65f59878c1ae39f2e Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 20:36:55 +0100 Subject: [PATCH 07/36] fix `Project` impl for `Cow<'_, T>` --- text/3735-field-projections.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index cb552b3ac86..76188e32ceb 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1083,11 +1083,11 @@ where And `Cow` would be implemented like this: ```rust -impl<'a, T: ?Sized> Projectable for Cow<'a, T> { +impl<'a, T: ?Sized + ToOwned> Projectable for Cow<'a, T> { type Inner = T; } -impl<'a, T: ?Sized, F> Project for Cow<'a, T> +impl<'a, T: ?Sized + ToOwned, F> Project for Cow<'a, T> where F: Field, F::Type: Sized, From 9b5b3f295661de165f38cf9a2ed5664b345c54ee Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 20:56:02 +0100 Subject: [PATCH 08/36] add alternative operator syntax `~` --- text/3735-field-projections.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 76188e32ceb..f69900be4ac 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1267,6 +1267,11 @@ they provide a big increase in readability of the code, expressing the concept o concisely. The compiler changes are rather manageable, reusing several already existing systems, thus increasing the maintenance burden only slightly if at all. +We could consider other operators rather than `->`. `->` has associations in C/C++ with performing a +dereference, while field projection doesn't necessarily perform a dereference. However, in C++ the +operator is also overloadable, so it isn't always a dereference. As an alternative to `->`, we could +consider `~` instead. + # Prior art [prior-art]: #prior-art From 8142a7822d0043b5ce78fe121fcb6ca73cbdf1ed Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 21:25:38 +0100 Subject: [PATCH 09/36] add `ArcRef` future possibility --- text/3735-field-projections.md | 109 +++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index f69900be4ac..e1a9fa68e22 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1424,3 +1424,112 @@ are rarely used directly, this probably isn't important. Types that might be good candidates for [`#[projecting]`](#projecting-attribute): - `ManuallyDrop` + +### `ArcRef` for Stdlib + +Using field projections, we can implement an `Arc` reference type, a pointer that owns a refcount on +an `Arc`, but points not at the entire struct in the `Arc`, but rather a field of that struct. + +```rust +pub struct ArcRef { + ptr: NonNull, + count: NonNull, +} + +impl Drop for ArcRef { + fn drop(&mut self) { + todo!() + // decrement the refcount + } +} + +impl Deref for ArcRef { + type Target = T; + + fn deref(&self) -> &T { + unsafe { self.ptr.as_ref() } + } +} +``` + +We can then make it have field projections: + +```rust +impl Projectable for ArcRef { + type Inner = T; +} + +impl Project for ArcRef +where + F: Field +{ + type Output = ArcRef; + + fn project(self) -> Self::Output { + // We give the refcount to the output, so we have to forget `self`. + let this = ManuallyDrop::new(self); + ArcRef { + ptr: NonNull::project(this.ptr), + count: this.count + } + } +} +``` + +And to get an `ArcRef` from an `Arc`, we can also use field projections: + +```rust +impl Projectable for Arc { + type Inner = T; +} + +impl Project for Arc +where + F: Field +{ + type Output = ArcRef; + + fn project(self) -> Self::Output { + ArcRef::project(self.into_arc_ref()) + } +} +``` + +Where `into_arc_ref` is implemented like this: + +```rust +impl Arc { + fn into_arc_ref(self) -> ArcRef { + let this = ManuallyDrop::new(self); + let ptr = Arc::as_ptr(&*this); + ArcRef { + ptr, + count: /* get ptr to strong refcount */ + } + } +} +``` + +Maybe, the count ptr should also point to the weak count. + +Now one can use it like this: + +```rust +struct DataContainer { + one: Data, + two: Data, +} + +struct Data { + flags: u32, + buf: [u8; 1024 * 1024], +} + +let x = Arc::::new_zeroed(); +let x: Arc = unsafe { x.assume_init() }; + +let one: ArcRef = x.clone()->one; +let two: ArcRef = x->two; + +let flags: ArcRef = one->flags; +``` From 51cf2e28cdc04b58c04779150627bc3193ba202b Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 23:12:31 +0100 Subject: [PATCH 10/36] make `Field` trait `unsafe` --- text/3735-field-projections.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index e1a9fa68e22..e1e3f6cac4d 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -788,7 +788,11 @@ accessible to the current scope can be projected. These types are called *field Field types implement the `Field` trait: ```rust -pub trait Field { +/// # Safety +/// +/// In any well-aligned instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there +/// exists a well-aligned field of type `Self::Type`. +pub unsafe trait Field { type Base: ?Sized; type Type: ?Sized; From d528aa44182b93c1b9d4663c338a306a5ba16a0d Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 23:13:34 +0100 Subject: [PATCH 11/36] don't generate field types for misaligned fields --- text/3735-field-projections.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index e1e3f6cac4d..30fcce4e21a 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -785,6 +785,8 @@ The compiler generates a compiler-internal type for every field of every struct. only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields accessible to the current scope can be projected. These types are called *field types*. +Field types are not generated for fields of `#[repr(packed)]` structs that are misaligned. + Field types implement the `Field` trait: ```rust From 1c9278f49c034990407ea9306e1d1d9fc34f6c22 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 4 Dec 2024 23:15:50 +0100 Subject: [PATCH 12/36] fix typo --- text/3735-field-projections.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 30fcce4e21a..bee0255a709 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1254,7 +1254,7 @@ pointer-like types, but also permits all sorts of operations generic over fields Not adding this feature will result in the proliferation of `*mut T` over more suitable pointer types that better express the invariants of the pointer. The ergonomic cost of -`unsafe { MyPtr::new_unchecked(&raw mut (*my_ptr.as_ptr()).field) }` is just to great to be useful +`unsafe { MyPtr::new_unchecked(&raw mut (*my_ptr.as_ptr()).field) }` is just too great to be useful in practice. While [pin projections] can be addressed via a library or a separate feature, not having them in the From 32f18807350b1fea22d3e153c41fa53dda2bf593 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 11:21:16 +0100 Subject: [PATCH 13/36] move `Cow<'_, T>` into future possibilities and fix its implementation --- text/3735-field-projections.md | 76 ++++++++++++++++------------------ 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index bee0255a709..19b494fe92a 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -88,7 +88,6 @@ There are a lot of types that can benefit from this operation: - `&Cell`, `&UnsafeCell` - `&mut MaybeUninit`, `*mut MaybeUninit` - `cell::Ref<'_, T>`, `cell::RefMut<'_, T>` -- `Cow<'_, T>` ## Pin Projections @@ -1059,7 +1058,6 @@ projections for any field and perform the obvious offset operation. - `*mut T` - `*const T` - `NonNull` -- `Cow<'_, T>` For example, `&T` would be implemented like this: @@ -1086,45 +1084,6 @@ where } ``` -And `Cow` would be implemented like this: - -```rust -impl<'a, T: ?Sized + ToOwned> Projectable for Cow<'a, T> { - type Inner = T; -} - -impl<'a, T: ?Sized + ToOwned, F> Project for Cow<'a, T> -where - F: Field, - F::Type: Sized, -{ - type Output = Cow<'a, F::Type>; - - fn project(self) -> Self::Output { - match self { - Cow::Borrowed(this) => Cow::Borrowed(<&T as Project::>::project(this)), - Cow::Owned(this) => Cow::Owned(project_value::(this)), - } - } -} -``` - -Where `project_value` is defined as: - -```rust -fn project_value(value: T) -> F::Type -where - F: Field, - F::Type: Sized, -{ - let r = &value; - let r = <&T as Project>::project(r); - let res = std::ptr::read(r); - std::mem::forget(value); - res -} -``` - The following types get annotated with [`#[projecting]`](#projecting-attribute): - `MaybeUninit` @@ -1539,3 +1498,38 @@ let two: ArcRef = x->two; let flags: ArcRef = one->flags; ``` + +### `Cow<'_, T>` + +For `Cow<'_, T>`, we need a new property for field types: + +```rust +pub unsafe trait MoveableField: Field { + fn move_out(base: Self::Base) -> Self::Type; +} +``` + +The `move_out` function is implemented by just moving out the field in question. Using this, we can +now implement field projections for `Cow<'_, T>`: + +```rust +impl<'a, T: ?Sized + ToOwned> Projectable for Cow<'a, T> { + type Inner = T; +} + +impl<'a, T: ?Sized + ToOwned, F> Project for Cow<'a, T> +where + F: Field, + F: MoveableField, + F::Type: Sized, +{ + type Output = Cow<'a, F::Type>; + + fn project(self) -> Self::Output { + match self { + Cow::Borrowed(this) => Cow::Borrowed(<&T as Project::>::project(this)), + Cow::Owned(this) => Cow::Owned(F::move_out(this)), + } + } +} +``` From d73e2293c41352b1584826d49418772b314a0d5e Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 11:24:09 +0100 Subject: [PATCH 14/36] add unresolved questions for `Field` trait and enum projections Co-authored-by: Josh Triplett --- text/3735-field-projections.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 19b494fe92a..710932ea077 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1333,6 +1333,14 @@ Alternatives: - `flatten` - use `#[flatten]` instead. +## `Field` trait stability + +Should we allow user implementations of the `Field` trait and have user-visible internals for it, or should we make it more opaque and sealed to reserve the possibility of supporting enums or similar? + +## Generalized enum projection + +Is there a generalization of enum projection that allows for runtime-conditional projection, for structures that sometimes-but-not-always have a given field? This could guarantee the type and identity of the field, but require a projection to be validated against a runtime instance of the value before confirming that the field exists and providing the projected field type. This would allow enums, as well as runtime equivalents such as C-style unions with discriminants or similar mechanisms for identifying variants at runtime. + ## Other - should the [`#[projecting]`](#projecting-attribute) attribute have an associated field attribute From 95ebc5788ceb92fe5e6bb823bbec2d6ce4999e45 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 11:24:55 +0100 Subject: [PATCH 15/36] text reflow --- text/3735-field-projections.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 710932ea077..58bb9289aee 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1335,11 +1335,17 @@ Alternatives: ## `Field` trait stability -Should we allow user implementations of the `Field` trait and have user-visible internals for it, or should we make it more opaque and sealed to reserve the possibility of supporting enums or similar? +Should we allow user implementations of the `Field` trait and have user-visible internals for it, or +should we make it more opaque and sealed to reserve the possibility of supporting enums or similar? ## Generalized enum projection -Is there a generalization of enum projection that allows for runtime-conditional projection, for structures that sometimes-but-not-always have a given field? This could guarantee the type and identity of the field, but require a projection to be validated against a runtime instance of the value before confirming that the field exists and providing the projected field type. This would allow enums, as well as runtime equivalents such as C-style unions with discriminants or similar mechanisms for identifying variants at runtime. +Is there a generalization of enum projection that allows for runtime-conditional projection, for +structures that sometimes-but-not-always have a given field? This could guarantee the type and +identity of the field, but require a projection to be validated against a runtime instance of the +value before confirming that the field exists and providing the projected field type. This would +allow enums, as well as runtime equivalents such as C-style unions with discriminants or similar +mechanisms for identifying variants at runtime. ## Other From d24e7c70d6e3da7170cc793fe9219faa19700668 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 11:45:24 +0100 Subject: [PATCH 16/36] add deref pattern similarity and propose not having a `match_proj` operator --- text/3735-field-projections.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 58bb9289aee..f7a24f450d7 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1388,6 +1388,12 @@ discussed. Here `match_proj` would need to be a new keyword. I dislike the name and syntax, but haven't come up with something better. +A similar issue comes up in the design of [deref patterns]. Since the types `Pin` and `MyEnum` are +distinct, they can be used to differentiate the kind of `match` the user wants to make. Thus making +it possible to only have the `match` operator and not a separate `match_proj` operator. + +[deref patterns]: https://hackmd.io/4qDDMcvyQ-GDB089IPcHGg + ## Arrays Arrays can be thought of structs/tuples where each index is a field. Supporting them would simply From c250db0256f3f320ef19887e978da10fc963b93b Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:22:51 +0100 Subject: [PATCH 17/36] add `UnalignedField` trait --- text/3735-field-projections.md | 64 +++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index f7a24f450d7..6a6e0916ffa 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -760,7 +760,7 @@ In order to facilitate field projections, several interlinked concepts have to b concepts are: - [field types], - - `Field` trait, + - `Field` traits, - `field_of!` macro, - [`#[projecting]`](#projecting-attribute) attribute - projection operator `->`, @@ -771,8 +771,8 @@ To ease understanding, here is a short explanation of the interactions between t following subsections explain them in more detail, so refer to them in cases of ambiguity. The projection operator `->` is governed by the `Project` trait that has `Projectable` as a super trait. `Projectable` helps to select the struct whose fields are used for projection. Field types store -information about a field (such as the base struct and the field type) via the `Field` trait and the -`field_of!` macro makes it possible to name the [field types]. Finally, the +information about a field (such as the base struct and the field type) via the `UnalignedField` +trait and the `field_of!` macro makes it possible to name the [field types]. Finally, the [`#[projecting]`](#projecting-attribute) attribute allows `repr(transparent)` structs to be ignored when looking for fields for projection. @@ -784,16 +784,14 @@ The compiler generates a compiler-internal type for every field of every struct. only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields accessible to the current scope can be projected. These types are called *field types*. -Field types are not generated for fields of `#[repr(packed)]` structs that are misaligned. - -Field types implement the `Field` trait: +Field types implement the `UnalignedField` trait: ```rust /// # Safety /// -/// In any well-aligned instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there -/// exists a well-aligned field of type `Self::Type`. -pub unsafe trait Field { +/// In any instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there exists a +/// (possibly misaligned) field of type `Self::Type`. +pub unsafe trait UnalignedField { type Base: ?Sized; type Type: ?Sized; @@ -805,6 +803,17 @@ In the implementation of this trait, `Base` is set to the struct that the field `Type` is set to the type of the field. `OFFSET` is set to the offset in bytes of the field in the struct (i.e. `OFFSET = offset_of!(Base, ident)`). +For aligned fields (such as all fields of non-`#[repr(packed)]` structs), their field types also implement the +`Field` trait: + +```rust +/// # Safety +/// +/// In any well-aligned instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there +/// exists a well-aligned field of type `Self::Type`. +pub unsafe trait Field: UnalignedField {} +``` + In addition to all fields of all structs, field types are generated for transparent, [`#[projecting]`](#projecting-attribute) container types as follows: given a transparent, generic type annotated with [`#[projecting]`](#projecting-attribute) and a struct contained in it: @@ -831,7 +840,7 @@ let x: Container; let _: &Container = project::, bar)>(&x); ``` -The implementation of the `Field` trait sets the associated types and constant like this: +The implementation of the `UnalignedField` trait sets the associated types and constant like this: - `Base = Container` - `Type = Container` - `OFFSET = offset_of!(Foo, bar)` @@ -856,8 +865,8 @@ struct Baz { // this refers to the field `inner` of `Baz`. type Y = field_of!(Container, inner); -// it has the following implementation of `Field`: -impl Field for Y { +// it has the following implementation of `UnalignedField`: +impl UnalignedField for Y { type Base = Container; type Type = Container; @@ -865,17 +874,27 @@ impl Field for Y { } ``` -#### `Field` Trait +#### `Field` Traits -The field trait is added to `core::marker` and cannot be implemented manually. +The fields trait are added to `core::marker` and cannot be implemented manually. ```rust -pub trait Field { +/// # Safety +/// +/// In any instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there exists a +/// (possibly misaligned) field of type `Self::Type`. +pub unsafe trait UnalignedField { type Base: ?Sized; type Type: ?Sized; const OFFSET: usize; } + +/// # Safety +/// +/// In any well-aligned instance of the type `Self::Base`, at byte offset `Self::OFFSET`, there +/// exists a well-aligned field of type `Self::Type`. +pub unsafe trait Field: UnalignedField {} ``` The compiler automatically implements it for all [field types]. Users of the trait are allowed to rely @@ -890,8 +909,8 @@ where { let ptr: *const Base = base; let ptr: *const u8 = base.cast::(); - // SAFETY: `ptr` is derived from a reference and the `Field` trait is guaranteed to contain - // correct values. So `F::OFFSET` is still within the `F::Base` type. + // SAFETY: `ptr` is derived from a reference and the `UnalignedField` trait is guaranteed to + // contain correct values. So `F::OFFSET` is still within the `F::Base` type. let ptr: *const u8 = unsafe { ptr.add(F::OFFSET) }; let ptr: *const F::Type = ptr.cast::(); // SAFETY: The `Field` trait guarantees that at `F::OFFSET` we find a field of type `F::Type`. @@ -1053,12 +1072,15 @@ implementations detailed below. The following pointer types get an implementation for `Projectable` with `Inner = T`. They support projections for any field and perform the obvious offset operation. -- `&mut T` -- `&T` - `*mut T` - `*const T` - `NonNull` +The same is true for the following types, except that they only allow projecting aligned fields: + +- `&mut T` +- `&T` + For example, `&T` would be implemented like this: ```rust @@ -1068,7 +1090,7 @@ impl<'a, T: ?Sized> Projectable for &'a T { impl<'a, T: ?Sized, F> Project for &'a T where - F: Field, + F: UnalignedField + Field, // Needed to be able to `.cast` below F::Type: Sized, { @@ -1169,7 +1191,7 @@ impl<'a, T: ?Sized> Projectable for Pin<&'a mut T> { impl<'a, T, F> Project for Pin<&'a mut T> where - F: Field + PinField, + F: UnalignedField + PinField, { type Output = ::Projected<'a>; From 6a7149fad188a0e37d5b0c3ad14de4ab6821635b Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:26:05 +0100 Subject: [PATCH 18/36] add `Mapped{Mutex,RwLock{Read,Write}}Guard` to projections and `cell::Ref[Mut]` to reference level-explanation --- text/3735-field-projections.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 6a6e0916ffa..4a72698d8b5 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -88,6 +88,7 @@ There are a lot of types that can benefit from this operation: - `&Cell`, `&UnsafeCell` - `&mut MaybeUninit`, `*mut MaybeUninit` - `cell::Ref<'_, T>`, `cell::RefMut<'_, T>` +- `MappedMutexGuard`, `MappedRwLockReadGuard` and `MappedRwLockWriteGuard` ## Pin Projections @@ -1078,8 +1079,10 @@ projections for any field and perform the obvious offset operation. The same is true for the following types, except that they only allow projecting aligned fields: -- `&mut T` -- `&T` +- `&T`, `&mut T` +- `cell::Ref`, `cell::RefMut` +- `MappedMutexGuard`, `MappedRwLockReadGuard` and `MappedRwLockWriteGuard` + For example, `&T` would be implemented like this: From 84e53e4b2cfaa8bb8dff2a29611c7687eb853988 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:27:43 +0100 Subject: [PATCH 19/36] fix typos --- text/3735-field-projections.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 4a72698d8b5..691443e8fb9 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -908,8 +908,8 @@ where F::Type: Sized, F::Base: Sized, { - let ptr: *const Base = base; - let ptr: *const u8 = base.cast::(); + let ptr: *const F::Base = base; + let ptr: *const u8 = ptr.cast::(); // SAFETY: `ptr` is derived from a reference and the `UnalignedField` trait is guaranteed to // contain correct values. So `F::OFFSET` is still within the `F::Base` type. let ptr: *const u8 = unsafe { ptr.add(F::OFFSET) }; @@ -1095,7 +1095,7 @@ impl<'a, T: ?Sized, F> Project for &'a T where F: UnalignedField + Field, // Needed to be able to `.cast` below - F::Type: Sized, + F::Type: Sized + 'a, { type Output = &'a F::Type; From 491d8231a2309c03fab58ebbe769f72ef1aa7c2c Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:36:40 +0100 Subject: [PATCH 20/36] always require a generic for `#[projecting]` and allow zero-sized container types --- text/3735-field-projections.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 691443e8fb9..d3882e02af5 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -951,9 +951,10 @@ pub mod bar { #### `#[projecting]` Attribute -The `#[projecting]` attribute can be put on a struct or union declaration. It requires that the -type is `#[repr(transparent)]` and that there is a unique field that has non-zero size in general. -It results in fields of the single non-zero sized type being considered fields of the outer type. +The `#[projecting]` attribute can be put on a struct or union declaration. It requires that the type +has at least one generic type parameter, here we call this parameter `T`. Additionally, the type +must be `#[repr(transparent)]` and either always zero-sized or there must be a unique non-zero-sized +field of type `T` (or a type that is `#[projecting]` over `T`). So for example: @@ -974,17 +975,22 @@ Now `Container` has a [field type] associated with `bar` implementing `Fiel - `Type = Container` - `OFFSET = offset_of!(Foo, bar)` -The same is true for the non-generic case: +Some more examples: ```rust +// This type is always zero-sized, but "contains" a field. #[projecting] #[repr(transparent)] -pub struct Container { - inner: Foo, +pub struct Container2 { + _phantom: PhantomData, } -struct Foo { - bar: i32, +#[projecting] +#[repr(transparent)] +pub struct Container3 { + // nesting multiple containers is fine as long as all of them are `#[projecting]` over the same + // generic. + ctr: Container>, } ``` @@ -1002,12 +1008,11 @@ pub struct Container { #[repr(transparent)] pub struct Container {} -// ERROR: no field to project onto found, all fields are always zero-sized +// ERROR: no generic type parameter found #[projecting] #[repr(transparent)] -pub struct Container { - phantom1: PhantomData T>, - phantom2: PhantomData, +pub struct Container { + foo: Foo, } ``` From 4de45c8430420ace22b8d12d328c2df8dd6d67e3 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:37:57 +0100 Subject: [PATCH 21/36] show that `#[projecting]` still allows other non-zero-sized fields --- text/3735-field-projections.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index d3882e02af5..89b9c5a3d4f 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -991,6 +991,8 @@ pub struct Container3 { // nesting multiple containers is fine as long as all of them are `#[projecting]` over the same // generic. ctr: Container>, + // other zero-sized fields still are allowed: + _variance: PhantomData, } ``` From 358de373a59670c6d372cb1fcdce8d74dbc1075d Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:41:38 +0100 Subject: [PATCH 22/36] text reflow --- text/3735-field-projections.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 89b9c5a3d4f..9a4dde49923 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -804,8 +804,8 @@ In the implementation of this trait, `Base` is set to the struct that the field `Type` is set to the type of the field. `OFFSET` is set to the offset in bytes of the field in the struct (i.e. `OFFSET = offset_of!(Base, ident)`). -For aligned fields (such as all fields of non-`#[repr(packed)]` structs), their field types also implement the -`Field` trait: +For aligned fields (such as all fields of non-`#[repr(packed)]` structs), their field types also +implement the `Field` trait: ```rust /// # Safety From dae2b041323d331e490682864c94383bff734698 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Thu, 5 Dec 2024 23:44:41 +0100 Subject: [PATCH 23/36] add sized caveat for field types --- text/3735-field-projections.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 9a4dde49923..251fc350e71 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -781,10 +781,14 @@ when looking for fields for projection. [field type]: #field-types [field types]: #field-types -The compiler generates a compiler-internal type for every field of every struct. These types can -only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields +The compiler generates a compiler-internal type for every sized[^2] field of every struct. These types +can only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields accessible to the current scope can be projected. These types are called *field types*. +[^2]: This restriction can be lifted in the future to include unsized types with statically known + alignment, but that would have to be done in unison with adding support for those fields in + `offset_of!`. + Field types implement the `UnalignedField` trait: ```rust From fc6279d097c42f76ff87cab9b01c2d1ebf6fc99e Mon Sep 17 00:00:00 2001 From: y86-dev Date: Fri, 6 Dec 2024 10:29:33 +0100 Subject: [PATCH 24/36] add guide-level explanation of implementing custom projections --- text/3735-field-projections.md | 156 +++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 251fc350e71..73427e03e8f 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -742,6 +742,162 @@ impl> Future for FairRaceFuture` traits, or +- annotating it with `#[projecting]`. + +They are used for pointers and container types respectively. + +#### Pointer-Like + +Pointer-like types can be projected using `Projectable` and `Project`. For example, if we create +a custom reference type that simultaneously points at two instances of the same type. + +```rust +pub struct DoubleRef<'a, T> { + first: &'a T, + second: &'a T, +} + +impl<'a, T> DoubleRef<'a, T> { + pub fn new(first: &'a T, second: &'a T) -> Self { + Self { first, second } + } + + pub fn read(&self) -> (T, T) + where + T: Copy + { + (*self.first, *self.second) + } +} +``` + +We can now allow its users to use field projection by implementing the above mentioned traits: + +```rust +impl<'a, T> Projectable for DoubleRef<'a, T> { + type Inner = T; +} +``` + +The `Projectable` trait is what tells the compiler which types' fields to consider for projecting. +When you write `a->b`, then `a` has to implement `Projectable` in order for the compiler to know +which type to look up the field `b` for. + +The actual projection operation is governed by `Project`: + +```rust +impl<'a, T, F> Project for DoubleRef<'a, T> +where + // Since we want to project both of our references, we want the field to be aligned. + F: UnalignedField + Field, +{ + type Output = DoubleRef<'a, F::Type>; + + fn project(self) -> Self::Output { + DoubleRef { + first: self.first.project(), + second: self.second.project(), + } + } +} +``` + +The `Project` trait governs projections for fields represented by the field type `F`. These field +types are generated for all[^3] fields of all structs. A field type always implements the +`UnalignedField` trait: + +[^3]: Almost all, fields that don't have a size, are not included. + +```rust +pub unsafe trait UnalignedField { + type Base: ?Sized; + type Type: ?Sized; + const OFFSET: usize; +} +``` + +`Base` is set to the parent struct containing the field, `Type` is set to the type of the field +itself and `OFFSET` is the offset in bytes as returned by `offset_of!`. + + +With the above projections in place, users can write the following code: + +```rust +struct Foo { + bar: i32, + baz: u32, +} + +let x: &Foo = &Foo { bar: 42, baz: 43 }; +let y: &Foo = &Foo { bar: 24, baz: 25 }; +let d: DoubleRef<'_, Foo> = DoubleRef::new(x, y); +let bars: DoubleRef<'_, i32> = d->bar; +assert_eq!((42, 24), bars.read()); +``` + +#### Containers + +The other kind of projections are that of *containers*, for example `UnsafeCell` or +`MaybeUninit`. They are governed by the `#[projecting]` attribute. + +If we want to combine the two containers from the previous sections, we can do it in the following +way: + +```rust +#[projecting] +#[repr(transparent)] +pub struct Opaque { + inner: UnsafeCell>, +} + +impl Opaque { + pub fn new(value: T) -> Self { + Self { inner: UnsafeCell::new(MaybeUninit::new(value)) } + } + + pub fn uninit() -> Self { + Self { inner: UnsafeCell::new(MaybeUninit::uninit()) } + } + + pub fn write(&mut self, value: T) { + self.inner.get_mut().write(value); + } +} +``` + +Now `&Opaque` can represent a reference that points at data from other languages, such as C. + +The `#[projecting]` attribute changes the way the field types are generated for this type. Instead +of having a field type representing the `inner` field, this type will "project" through to the type +`T`, inheriting all fields that `T` has. So if we consider the type `Foo` from above: + +```rust +struct Foo { + bar: i32, + baz: u32, +} +``` + +Then there are two field types for `Opaque`: +- one representing `bar` with `Base = Opaque` and `Type = Opaque` +- the other representing `baz` with `Base = Opaque` and `Type = Opaque` + +So the both the base type and the type of the fields are wrapped with the container. Now users can +write: + +```rust +fn init(foo: &mut Opaque) { + let bar: &mut Opaque = foo->bar; + bar.write(42); + foo->baz.write(24); +} +``` + + ## Impact of this Feature Overall this feature improves readability of code, because it replaces more complex to parse syntax From 6282d95d6968913b6fea4d6aa642cbfcdf1d595d Mon Sep 17 00:00:00 2001 From: y86-dev Date: Fri, 6 Dec 2024 10:32:26 +0100 Subject: [PATCH 25/36] remove using `repr` as an alternative to `#[projecting]` --- text/3735-field-projections.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 73427e03e8f..a7e6758e215 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1520,9 +1520,6 @@ Alternatives: Current favorite: [`#[projecting]`](#projecting-attribute) attribute. Alternatives: -- extend the `repr` attribute with - - `projecting` - - `flatten` - use `#[flatten]` instead. ## `Field` trait stability From 612c91056bc7f8e34b009297930083bf291c82ca Mon Sep 17 00:00:00 2001 From: y86-dev Date: Fri, 6 Dec 2024 11:10:20 +0100 Subject: [PATCH 26/36] add prior art: Field Projection Use Cases document --- text/3735-field-projections.md | 1 + 1 file changed, 1 insertion(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index a7e6758e215..2b1f61631a3 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1476,6 +1476,7 @@ There are several crates implementing projections for different types. - [Field projdection for `Rc` and `Arc`](https://internals.rust-lang.org/t/field-projection-for-rc-and-arc/15827) - [Generic Field Projection](https://internals.rust-lang.org/t/generic-field-projection/16204) +- [Field Projection Use Cases](https://hackmd.io/@y86-dev/SkSB48hCR) Blog posts about pin (projections): - [Pinned places](https://without.boats/blog/pinned-places/) From 04b5193daa708adc4772b2de8e8e017c22d31cf0 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Fri, 6 Dec 2024 15:02:03 +0100 Subject: [PATCH 27/36] make code more concise --- text/3735-field-projections.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 2b1f61631a3..22346354de4 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -792,8 +792,7 @@ The actual projection operation is governed by `Project`: ```rust impl<'a, T, F> Project for DoubleRef<'a, T> where - // Since we want to project both of our references, we want the field to be aligned. - F: UnalignedField + Field, + F: Field, { type Output = DoubleRef<'a, F::Type>; @@ -1260,7 +1259,7 @@ impl<'a, T: ?Sized> Projectable for &'a T { impl<'a, T: ?Sized, F> Project for &'a T where - F: UnalignedField + Field, + F: Field, // Needed to be able to `.cast` below F::Type: Sized + 'a, { From 7df52395a4fbb4ae2d40ad9e230963c7354c6cb8 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sat, 7 Dec 2024 20:33:01 +0100 Subject: [PATCH 28/36] expand text on tuple support --- text/3735-field-projections.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 22346354de4..65ce74f84dd 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -936,9 +936,9 @@ when looking for fields for projection. [field type]: #field-types [field types]: #field-types -The compiler generates a compiler-internal type for every sized[^2] field of every struct. These types -can only be named via the `field_of!` macro that has the same syntax as `offset_of!`. Only fields -accessible to the current scope can be projected. These types are called *field types*. +The compiler generates a compiler-internal type for every sized[^2] field of every struct and tuple. +These types can only be named via the `field_of!` macro that has the same syntax as `offset_of!`. +Only fields accessible to the current scope can be projected. These types are called *field types*. [^2]: This restriction can be lifted in the future to include unsized types with statically known alignment, but that would have to be done in unison with adding support for those fields in @@ -974,9 +974,9 @@ implement the `Field` trait: pub unsafe trait Field: UnalignedField {} ``` -In addition to all fields of all structs, field types are generated for transparent, -[`#[projecting]`](#projecting-attribute) container types as follows: given a transparent, generic type annotated with -[`#[projecting]`](#projecting-attribute) and a struct contained in it: +In addition to all fields of all structs and tuples, field types are generated for transparent, +[`#[projecting]`](#projecting-attribute) container types as follows: given a transparent, generic +type annotated with [`#[projecting]`](#projecting-attribute) and a struct contained in it: ```rust #[projecting] @@ -1088,8 +1088,9 @@ pub macro field_of($Container:ty, $($fields:expr)+ $(,)?) { } ``` -It has the same syntax as the `offset_of!` macro and returns the respective [field type]. It emits an -error in the following cases: +It has the same syntax as the `offset_of!` macro also supporting tuples. `field_of!` returns the +[field type] of the field `$fields` of the `$Container` struct or tuple type. It emits an error in +the following cases: ```rust pub mod foo { From 7d505032b41a3129e2e11728412bbde65bf006ee Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sun, 8 Dec 2024 23:00:52 +0100 Subject: [PATCH 29/36] add future possibility `Option` stdlib --- text/3735-field-projections.md | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 65ce74f84dd..8babb84b6de 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1743,3 +1743,47 @@ where } } ``` + +### `Option` + +`Option` is a bit of an interesting case, as it cannot be annotated with `#[projecting]`, since +it is not a transparent wrapper type. If we again consider an example struct: + +```rust +struct Foo { + bar: i32, + baz: u32, +} +``` + +Then `Option` does not have a field of type `Option`, since `Option` adds an additional +bit of information that needs to be represented in the raw bits of the type. + +However, we can implement field projections for `Option` when `T` has field projections +available. In the `None` case, we just project to `None` and in the `Some` case, we can use `T`'s +projection: + +```rust +impl Projectable for Option { + type Inner = ::Inner; +} + +impl Project for Option +where + T: Project, + F: Field, + F::Type: Sized, +{ + type Output = Option; + + fn project(self) -> Self::Output { + match self { + Some(v) => Some(v.project()) + None => None + } + } +} +``` + +Now we are able to project for example `Option<&mut MaybeUninit` to +`Option<&mut MaybeUninit>`. From 329175907f635f8cd164ee1ab6c4fdc4e8434581 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sun, 15 Dec 2024 11:36:43 +0100 Subject: [PATCH 30/36] fix typos --- text/3735-field-projections.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 8babb84b6de..849fb8efdd1 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -72,7 +72,7 @@ However, the other pointer-like types such as `NonNull`, `&mut MaybeUninit ```rust unsafe fn project(foo: NonNull) -> NonNull { let foo = foo.as_ptr(); - unsafe { NonNull::new_unchecked(&raw (*foo).bar) } + unsafe { NonNull::new_unchecked(&raw mut (*foo).bar) } } ``` @@ -253,7 +253,7 @@ adapted the code): > ``` > > The thing is that ioctl that use the struct approach like drm does, use the same struct if there's -> both input and output paramterers, and furthermore we are not allowed to overwrite the entire +> both input and output parameters, and furthermore we are not allowed to overwrite the entire > struct because that breaks ioctl restarting. So the flow is roughly > > ```rust From b7f6cb9987504c1236db0ca572161ac1a8ff5b5c Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sat, 1 Mar 2025 15:42:34 +0100 Subject: [PATCH 31/36] move desugaring below trait definitions --- text/3735-field-projections.md | 37 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 849fb8efdd1..877f2e5a28c 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1194,25 +1194,6 @@ The field projection operator `->` has the following syntax: [_Expression_]: https://doc.rust-lang.org/reference/expressions.html [TUPLE_INDEX]: https://doc.rust-lang.org/reference/tokens.html#tuple-index -#### Desugaring - -```rust -struct T { - field: F, -} - -let t: C = /* ... */; -let _ = t->field; - -// becomes - -let _ = Project:: as Projectable>::Inner, field)>::project(t); -``` - -The `C` in the `C as Projectable` comes from a type inference variable over the expression -`t`. - - #### `Projectable` & `Project` Traits Added to `core::ops`: @@ -1232,6 +1213,24 @@ where } ``` +#### Desugaring + +```rust +struct T { + field: F, +} + +let t: C = /* ... */; +let _ = t->field; + +// becomes + +let _ = Project:: as Projectable>::Inner, field)>::project(t); +``` + +The `C` in the `C as Projectable` comes from a type inference variable over the expression +`t`. + ## Stdlib Field Projections All examples from the guide-level explanation work when the standard library is extended with the From 5a5bb21aba4651c5e50f259da6abd0cfcb0cc785 Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sat, 1 Mar 2025 15:49:22 +0100 Subject: [PATCH 32/36] change projection traits and add simultaneous projections --- text/3735-field-projections.md | 177 +++++++++++++++++++++++++++++---- 1 file changed, 156 insertions(+), 21 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 877f2e5a28c..69b9ec7edbd 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -1196,11 +1196,15 @@ The field projection operator `->` has the following syntax: #### `Projectable` & `Project` Traits -Added to `core::ops`: +The projection operator is governed by three traits added to `core::ops`: ```rust pub trait Projectable: Sized { type Inner: ?Sized; + // name-bikeshed needed + type Inter; + + fn start_projection(self) -> Self::Inter; } pub trait Project: Projectable @@ -1209,12 +1213,33 @@ where { type Output; - fn project(self) -> Self::Output; + fn project_last(inter: Self::Inter) -> Self::Output; +} + +// name-bikeshed needed +pub unsafe trait SimultaneousProject: Project +where + F: Field, +{ + fn project(inter: Self::Inter) -> Self::Output; } ``` +`Project` is responsible for the actual projection operation while `Projectable` identifies if a +type has any kind of projection and for which fields there could be projections. So if the compiler +sees `x->y`, the type of `x` has to implement `Projectable` in order for the compiler to verify that +the associated type `Inner` of that impl has a field named `y`. + +`SimultaneousProject` can be implemented in addition to `Project` in order to allow projecting the +same expression for different fields at the same time. This is also the reason for the existence of +`Projectable::Inter`, since multiple values of `Self` "pointing" to the same value might not be +allowed to coexist (this is the case for `&mut T`). + #### Desugaring +When only a single projection operation using that variable is done, the desugaring is simpler. For +example: + ```rust struct T { field: F, @@ -1225,12 +1250,45 @@ let _ = t->field; // becomes -let _ = Project:: as Projectable>::Inner, field)>::project(t); +let _ = Project:: as Projectable>::Inner, field)>::project_last( + Projectable::start_projection(t), +); ``` The `C` in the `C as Projectable` comes from a type inference variable over the expression `t`. +When the same projection base is used multiple times, the desugaring is as follows: + +```rust +struct T { + field: F, + x: X, + y: Y, +} + +let t: C = /* ... */; +let _ = t->field; +let _ = t->x; +let _ = t->y; + +// becomes + +let __inter_t = Projectable::start_projection(t); +let _ = SimultaneousProject:: as Projectable>::Inner, field)>::project( + unsafe { ptr::read(&__inter_t) }, +); +let _ = SimultaneousProject:: as Projectable>::Inner, x)>::project( + unsafe { ptr::read(&__inter_t) }, +); +let _ = Project:: as Projectable>::Inner, y)>::project_last( + Projectable::start_projection(t), +); +``` + +Essentially, the compiler starts projecting the value and then re-uses the same `Inter` value for +the various projections, consuming it on the last one. + ## Stdlib Field Projections All examples from the guide-level explanation work when the standard library is extended with the @@ -1255,6 +1313,11 @@ For example, `&T` would be implemented like this: ```rust impl<'a, T: ?Sized> Projectable for &'a T { type Inner = T; + type Inter = *const T; + + fn start_projection(self) -> Self::Inter { + self + } } impl<'a, T: ?Sized, F> Project for &'a T @@ -1265,8 +1328,18 @@ where { type Output = &'a F::Type; - fn project(self) -> Self::Output { - let ptr: *const T = self; + fn project_last(ptr: *const T) -> Self::Output { + Self::project(ptr) + } +} + +unsafe impl<'a, T: ?Sized, F> SimultaneousProject for &'a T +where + F: Field, + // Needed to be able to `.cast` below + F::Type: Sized + 'a, +{ + fn project(ptr: *const T) -> Self::Output { let ptr = ptr.cast::(); let ptr = unsafe { ptr.add(F::OFFSET) }; let ptr = ptr.cast::(); @@ -1356,6 +1429,11 @@ Now the only component that is left is an implementation of `Projectable` and `P ```rust impl<'a, T: ?Sized> Projectable for Pin<&'a mut T> { type Inner = T; + type Inter = *mut T; + + fn start_projection(self) -> Self::Inter { + unsafe { Pin::into_inner_unchecked(self) } + } } impl<'a, T, F> Project for Pin<&'a mut T> @@ -1364,9 +1442,18 @@ where { type Output = ::Projected<'a>; - fn project(self) -> Self::Output { - let r: &mut T = unsafe { Pin::into_inner_unchecked(self) }; - let r: &mut F::Type = <&mut T as Project::>::project(r); + fn project_last(inter: Self::Inter) -> Self::Output { + let r: &mut F::Type = <&mut T as Project>::project(inter); + ::from_pinned_ref(r) + } +} + +unsafe impl<'a, T, F> SimultaneousProject for Pin<&'a mut T> +where + F: UnalignedField + PinField, +{ + fn project(inter: Self::Inter) -> Self::Output { + let r: &mut F::Type = <&mut T as Project>::project(inter); ::from_pinned_ref(r) } } @@ -1631,6 +1718,11 @@ We can then make it have field projections: ```rust impl Projectable for ArcRef { type Inner = T; + type Inter = Self; + + fn start_projection(self) -> Self { + self + } } impl Project for ArcRef @@ -1639,15 +1731,26 @@ where { type Output = ArcRef; - fn project(self) -> Self::Output { + fn project_last(inter: Self) -> Self::Output { // We give the refcount to the output, so we have to forget `self`. - let this = ManuallyDrop::new(self); + let this = ManuallyDrop::new(inter); ArcRef { ptr: NonNull::project(this.ptr), count: this.count } } } + +impl SimultaneousProject for ArcRef +where + F: Field +{ + fn project(inter: Self) -> Self::Output { + let this = inter.clone(); + mem::forget(inter); + >::project_last(this) + } +} ``` And to get an `ArcRef` from an `Arc`, we can also use field projections: @@ -1655,6 +1758,11 @@ And to get an `ArcRef` from an `Arc`, we can also use field projections: ```rust impl Projectable for Arc { type Inner = T; + type Inter = ArcRef; + + fn start_projection(self) -> ArcRef { + self.into_arc_ref() + } } impl Project for Arc @@ -1663,8 +1771,17 @@ where { type Output = ArcRef; - fn project(self) -> Self::Output { - ArcRef::project(self.into_arc_ref()) + fn project_last(inter: Self::Inter) -> Self::Output { + as Project>::project_last(inter) + } +} + +impl SimultaneousProject for Arc +where + F: Field +{ + fn project(inter: Self::Inter) -> Self::Output { + as SimultaneousProject>::project(inter) } } ``` @@ -1673,7 +1790,7 @@ Where `into_arc_ref` is implemented like this: ```rust impl Arc { - fn into_arc_ref(self) -> ArcRef { + pub fn into_arc_ref(self) -> ArcRef { let this = ManuallyDrop::new(self); let ptr = Arc::as_ptr(&*this); ArcRef { @@ -1724,6 +1841,11 @@ now implement field projections for `Cow<'_, T>`: ```rust impl<'a, T: ?Sized + ToOwned> Projectable for Cow<'a, T> { type Inner = T; + type Inter = Self; + + fn start_projection(self) -> Self { + self + } } impl<'a, T: ?Sized + ToOwned, F> Project for Cow<'a, T> @@ -1734,9 +1856,9 @@ where { type Output = Cow<'a, F::Type>; - fn project(self) -> Self::Output { - match self { - Cow::Borrowed(this) => Cow::Borrowed(<&T as Project::>::project(this)), + fn project_last(inter: Self) -> Self::Output { + match inter { + Cow::Borrowed(this) => Cow::Borrowed(<&T as Project>::project(this)), Cow::Owned(this) => Cow::Owned(F::move_out(this)), } } @@ -1765,6 +1887,11 @@ projection: ```rust impl Projectable for Option { type Inner = ::Inner; + type Inter = Option<::Inter>; + + fn start_projection(self) -> Self::Inter { + self.map(T::start_projection) + } } impl Project for Option @@ -1775,11 +1902,19 @@ where { type Output = Option; - fn project(self) -> Self::Output { - match self { - Some(v) => Some(v.project()) - None => None - } + fn project_last(inter: Self::Inter) -> Self::Output { + inter.map(>::project_last) + } +} + +impl SimultaneousProject for Option +where + T: SimultaneousProject, + F: Field, + F::Type: Sized, +{ + fn project(inter: Self::Inter) -> Self::Output { + inter.map(>::project) } } ``` From e923b540609da541299a612a9ea20fe29bdbab9c Mon Sep 17 00:00:00 2001 From: y86-dev Date: Sat, 1 Mar 2025 18:16:00 +0100 Subject: [PATCH 33/36] add limitation section to `#[projecting]` attribute to explain the need for `#[repr(transparent)]` --- text/3735-field-projections.md | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 69b9ec7edbd..c4a687b74f8 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -896,6 +896,44 @@ fn init(foo: &mut Opaque) { } ``` +##### Limitations + +A type can only be annotated with `#[projecting]` if it is also `#[repr(transparent)]`, because of +the following problem: + +Assume that we could annotate the following container with `#[projecting]`: + +```rust +#[projecting] +struct Container { + count: u64, + value: T, +} +``` + +Now the memory layout of a `Container` could look like this (one character represents one byte): + +```text +|count---|bar-|baz-| +``` + +If we now want to project `&Container` to `&Container`, we would have to project to `baz`. +But the problem is that the memory layout of `Container` looks like this: + +```text +|count---|u32-| +``` + +Now projecting becomes impossible for two reasons: +- this layout is not contained as a direct sublayout of the above (ie with `count` mapped to `count` + and `u32` mapped to `baz`), +- projecting for references is done simply via offsetting. + +So is not possible to project to the correct layout with both of these constraints. + +This is also the reason for why `Arc` cannot be projected to `Arc` (the reference count is +stored in front of the `T`). However, it is possible to create an [`ArcRef`](arcref) that +tracks the reference count separately from the field. ## Impact of this Feature @@ -1687,6 +1725,7 @@ Types that might be good candidates for [`#[projecting]`](#projecting-attribute) - `ManuallyDrop` ### `ArcRef` for Stdlib +[arcref]: #arcref-t--for-stdlib Using field projections, we can implement an `Arc` reference type, a pointer that owns a refcount on an `Arc`, but points not at the entire struct in the `Arc`, but rather a field of that struct. From 13b8a477874f27739448df253a2bbe66fec674af Mon Sep 17 00:00:00 2001 From: y86-dev Date: Mon, 10 Mar 2025 22:20:52 +0100 Subject: [PATCH 34/36] simultaneous projections rework --- text/3735-field-projections.md | 222 +++++++++++++++++++++------------ 1 file changed, 141 insertions(+), 81 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index c4a687b74f8..91ec2efcfee 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -838,6 +838,65 @@ let bars: DoubleRef<'_, i32> = d->bar; assert_eq!((42, 24), bars.read()); ``` +#### Simultaneous Projections + +One important detail of the `Projectable` and `Project` traits is that they only enable support for +a single projection. So the value `x` will be consumed when doing `x->y`. + +Simultaneous projections are governed by the `SimultaneousProjectable` and `SimultaneousProject` +traits. When projecting a value whose type implements these traits, it can be projected once for +each field. So if our `DoubleRef` implemented them, we could also check the value of the two `baz` +fields: + +```rust +struct Foo { + bar: i32, + baz: u32, +} + +let x: &Foo = &Foo { bar: 42, baz: 43 }; +let y: &Foo = &Foo { bar: 24, baz: 25 }; +let d: DoubleRef<'_, Foo> = DoubleRef::new(x, y); +let bars: DoubleRef<'_, i32> = d->bar; +let bazes: DoubleRef<'_, u32> = d->baz; +assert_eq!((42, 24), bars.read()); +assert_eq!((43, 25), bazes.read()); +``` + +Implementing these traits for `DoubleRef` looks like this: + +```rust +impl<'a, T> SimultaneousProjectable for DoubleRef<'a, T> { + type Inter = (*mut T, *mut T); + + fn start_projection(self) -> Self::Inter { + (self.first, self.second) + } +} + +impl<'a, T, F> SimultaneousProject for DoubleRef<'a, T> +where + F: Field, +{ + type Output = DoubleRef<'a, F::Type>; + + unsafe fn project(inter: Self::Inter) -> Self::Output { + DoubleRef { + first: unsafe { &mut *inter.0.project() }, + second: unsafe { &mut *inter.1.project() }, + } + } +} +``` + +A couple of important notes: +- we need to remove the `Project` implementation, since there is a blanket impl for types that + implement `SimultaneousProject`. +- the `Inter` type has to implement `Clone` and the compiler will clone it for every projection the + user requests. +- the compiler ensures that `project` will only be called at most once for each field via the + projection operator. + #### Containers The other kind of projections are that of *containers*, for example `UnsafeCell` or @@ -960,6 +1019,8 @@ concepts are: - projection operator `->`, - `Project` trait, - `Projectable` trait, + - `SimultaneousProject` trait, + - `SimultaneousProjectable` trait, To ease understanding, here is a short explanation of the interactions between these concepts. The following subsections explain them in more detail, so refer to them in cases of ambiguity. The @@ -970,6 +1031,9 @@ trait and the `field_of!` macro makes it possible to name the [field types]. Fin [`#[projecting]`](#projecting-attribute) attribute allows `repr(transparent)` structs to be ignored when looking for fields for projection. +The traits `SimultaneousProject` and `SimultaneousProjectable` exist to support simultaneous +projections. + ### Field Types [field type]: #field-types [field types]: #field-types @@ -1232,17 +1296,13 @@ The field projection operator `->` has the following syntax: [_Expression_]: https://doc.rust-lang.org/reference/expressions.html [TUPLE_INDEX]: https://doc.rust-lang.org/reference/tokens.html#tuple-index -#### `Projectable` & `Project` Traits +#### `[Simultaneous]Project[able]` Traits -The projection operator is governed by three traits added to `core::ops`: +The projection operator is governed by four traits added to `core::ops`: ```rust pub trait Projectable: Sized { type Inner: ?Sized; - // name-bikeshed needed - type Inter; - - fn start_projection(self) -> Self::Inter; } pub trait Project: Projectable @@ -1251,15 +1311,42 @@ where { type Output; - fn project_last(inter: Self::Inter) -> Self::Output; + fn project(self) -> Self::Output; } // name-bikeshed needed -pub unsafe trait SimultaneousProject: Project +pub trait SimultaneousProjectable: Projectable { + // name-bikeshed needed + type Inter: Clone; + + fn start_projection(self) -> Self::Inter; +} + +// name-bikeshed needed +pub trait SimultaneousProject: SimultaneousProjectable where F: Field, { - fn project(inter: Self::Inter) -> Self::Output; + type Output; + + /// # Safety + /// + /// This function may only be called once for each value of `Self::Inter` that is derived from + /// a value of `Self` via [`SimultaneousProjectable::start_projection`] or cloning such a value. + unsafe fn project(inter: Self::Inter) -> Self::Output; +} + +impl Project for T +where + T: SimultaneousProjectable, + F: Field, +{ + type Output = >::Output; + + fn project(self) -> Self::Output { + // SAFETY: we only call one project function from the derived value of `start_projection`. + unsafe { >::project(self.start_projection()) } + } } ``` @@ -1268,10 +1355,9 @@ type has any kind of projection and for which fields there could be projections. sees `x->y`, the type of `x` has to implement `Projectable` in order for the compiler to verify that the associated type `Inner` of that impl has a field named `y`. -`SimultaneousProject` can be implemented in addition to `Project` in order to allow projecting the -same expression for different fields at the same time. This is also the reason for the existence of -`Projectable::Inter`, since multiple values of `Self` "pointing" to the same value might not be -allowed to coexist (this is the case for `&mut T`). +`SimultaneousProject` can be implemented instead of `Project` in order to allow projecting the same +expression for different fields at the same time. The `Inter` associated type of +`SimultaneousProjectable` is cloned for each such simultaneous projection (except the last). #### Desugaring @@ -1313,15 +1399,17 @@ let _ = t->y; // becomes let __inter_t = Projectable::start_projection(t); -let _ = SimultaneousProject:: as Projectable>::Inner, field)>::project( - unsafe { ptr::read(&__inter_t) }, -); -let _ = SimultaneousProject:: as Projectable>::Inner, x)>::project( - unsafe { ptr::read(&__inter_t) }, -); -let _ = Project:: as Projectable>::Inner, y)>::project_last( - Projectable::start_projection(t), -); +let _ = unsafe { + SimultaneousProject:: as Projectable>::Inner, field)>::project( + __inter_t.clone() + ) +}; +let _ = unsafe { + SimultaneousProject:: as Projectable>::Inner, x)>::project(__inter_t.clone()) +}; +let _ = unsafe { + SimultaneousProject:: as Projectable>::Inner, y)>::project(__inter_t) +}; ``` Essentially, the compiler starts projecting the value and then re-uses the same `Inter` value for @@ -1351,6 +1439,9 @@ For example, `&T` would be implemented like this: ```rust impl<'a, T: ?Sized> Projectable for &'a T { type Inner = T; +} + +impl<'a, T: ?Sized> SimultaneousProjectable for &'a T { type Inter = *const T; fn start_projection(self) -> Self::Inter { @@ -1358,26 +1449,13 @@ impl<'a, T: ?Sized> Projectable for &'a T { } } -impl<'a, T: ?Sized, F> Project for &'a T -where - F: Field, - // Needed to be able to `.cast` below - F::Type: Sized + 'a, -{ - type Output = &'a F::Type; - - fn project_last(ptr: *const T) -> Self::Output { - Self::project(ptr) - } -} - unsafe impl<'a, T: ?Sized, F> SimultaneousProject for &'a T where F: Field, // Needed to be able to `.cast` below F::Type: Sized + 'a, { - fn project(ptr: *const T) -> Self::Output { + unsafe fn project(ptr: *const T) -> Self::Output { let ptr = ptr.cast::(); let ptr = unsafe { ptr.add(F::OFFSET) }; let ptr = ptr.cast::(); @@ -1467,31 +1545,22 @@ Now the only component that is left is an implementation of `Projectable` and `P ```rust impl<'a, T: ?Sized> Projectable for Pin<&'a mut T> { type Inner = T; +} + +impl<'a, T: ?Sized> SimultaneousProjectable for Pin<&'a mut T> { type Inter = *mut T; fn start_projection(self) -> Self::Inter { unsafe { Pin::into_inner_unchecked(self) } } } - -impl<'a, T, F> Project for Pin<&'a mut T> -where - F: UnalignedField + PinField, -{ - type Output = ::Projected<'a>; - - fn project_last(inter: Self::Inter) -> Self::Output { - let r: &mut F::Type = <&mut T as Project>::project(inter); - ::from_pinned_ref(r) - } -} - unsafe impl<'a, T, F> SimultaneousProject for Pin<&'a mut T> where F: UnalignedField + PinField, { - fn project(inter: Self::Inter) -> Self::Output { - let r: &mut F::Type = <&mut T as Project>::project(inter); + unsafe fn project(inter: Self::Inter) -> Self::Output { + let r: *mut F::Type = <*mut T as Project>::project(inter); + let r = unsafe { &mut *r }; ::from_pinned_ref(r) } } @@ -1757,6 +1826,9 @@ We can then make it have field projections: ```rust impl Projectable for ArcRef { type Inner = T; +} + +impl SimultaneousProjectable for ArcRef { type Inter = Self; fn start_projection(self) -> Self { @@ -1764,13 +1836,13 @@ impl Projectable for ArcRef { } } -impl Project for ArcRef +impl SimultaneousProject for ArcRef where F: Field { type Output = ArcRef; - fn project_last(inter: Self) -> Self::Output { + unsafe fn project(inter: Self) -> Self::Output { // We give the refcount to the output, so we have to forget `self`. let this = ManuallyDrop::new(inter); ArcRef { @@ -1779,17 +1851,6 @@ where } } } - -impl SimultaneousProject for ArcRef -where - F: Field -{ - fn project(inter: Self) -> Self::Output { - let this = inter.clone(); - mem::forget(inter); - >::project_last(this) - } -} ``` And to get an `ArcRef` from an `Arc`, we can also use field projections: @@ -1797,6 +1858,9 @@ And to get an `ArcRef` from an `Arc`, we can also use field projections: ```rust impl Projectable for Arc { type Inner = T; +} + +impl SimultaneousProjectable for Arc { type Inter = ArcRef; fn start_projection(self) -> ArcRef { @@ -1810,17 +1874,8 @@ where { type Output = ArcRef; - fn project_last(inter: Self::Inter) -> Self::Output { - as Project>::project_last(inter) - } -} - -impl SimultaneousProject for Arc -where - F: Field -{ - fn project(inter: Self::Inter) -> Self::Output { - as SimultaneousProject>::project(inter) + unsafe fn project(inter: Self::Inter) -> Self::Output { + unsafe { as SimultaneousProject>::project(inter) } } } ``` @@ -1895,7 +1950,7 @@ where { type Output = Cow<'a, F::Type>; - fn project_last(inter: Self) -> Self::Output { + fn project(inter: Self) -> Self::Output { match inter { Cow::Borrowed(this) => Cow::Borrowed(<&T as Project>::project(this)), Cow::Owned(this) => Cow::Owned(F::move_out(this)), @@ -1926,7 +1981,10 @@ projection: ```rust impl Projectable for Option { type Inner = ::Inner; - type Inter = Option<::Inter>; +} + +impl SimultaneousProjectable for Option { + type Inter = Option<::Inter>; fn start_projection(self) -> Self::Inter { self.map(T::start_projection) @@ -1941,19 +1999,21 @@ where { type Output = Option; - fn project_last(inter: Self::Inter) -> Self::Output { - inter.map(>::project_last) + fn project(inter: Self::Inter) -> Self::Output { + inter.map(>::project) } } +// This probably overlaps with the impl above, but if the compiler is smart enough, it should know +// that they don't actually overlap. impl SimultaneousProject for Option where T: SimultaneousProject, F: Field, F::Type: Sized, { - fn project(inter: Self::Inter) -> Self::Output { - inter.map(>::project) + unsafe fn project(inter: Self::Inter) -> Self::Output { + inter.map(|v| unsafe { >::project(v) }) } } ``` From 1fbd24715c41af1d43c358c964d254bde7cff52f Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 19 Mar 2025 12:01:56 +0100 Subject: [PATCH 35/36] add `PhantomPinned` to `Opaque` --- text/3735-field-projections.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index 91ec2efcfee..b36b35a6727 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -910,6 +910,9 @@ way: #[repr(transparent)] pub struct Opaque { inner: UnsafeCell>, + // should be replaced by wrapping `inner` in `UnsafePinned` from + // https://github.com/rust-lang/rfcs/pull/3467 + _phantom_pinned: PhantomPinned, } impl Opaque { From 70fc606d5bd37eb702910fbd2ef8ae117e9a59aa Mon Sep 17 00:00:00 2001 From: y86-dev Date: Wed, 19 Mar 2025 12:57:20 +0100 Subject: [PATCH 36/36] clarify `#[projecting]` and generics --- text/3735-field-projections.md | 54 +++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/text/3735-field-projections.md b/text/3735-field-projections.md index b36b35a6727..e2a3ee59aad 100644 --- a/text/3735-field-projections.md +++ b/text/3735-field-projections.md @@ -933,8 +933,9 @@ impl Opaque { Now `&Opaque` can represent a reference that points at data from other languages, such as C. The `#[projecting]` attribute changes the way the field types are generated for this type. Instead -of having a field type representing the `inner` field, this type will "project" through to the type -`T`, inheriting all fields that `T` has. So if we consider the type `Foo` from above: +of having a field type representing the `inner` field, this type will "project" through to the +generic parameter `T`, inheriting all fields that `T` has. So if we consider the type `Foo` from +above: ```rust struct Foo { @@ -1079,9 +1080,9 @@ implement the `Field` trait: pub unsafe trait Field: UnalignedField {} ``` -In addition to all fields of all structs and tuples, field types are generated for transparent, -[`#[projecting]`](#projecting-attribute) container types as follows: given a transparent, generic -type annotated with [`#[projecting]`](#projecting-attribute) and a struct contained in it: +In addition to all fields of all structs and tuples, field types are generated for +[`#[projecting]`](#projecting-attribute) container types as follows: given a type annotated with +[`#[projecting]`](#projecting-attribute) and a field contained in it that has itself field types: ```rust #[projecting] @@ -1113,8 +1114,8 @@ The implementation of the `UnalignedField` trait sets the associated types and c This can even be done for multiple levels: `Container>` also has a [field type] `bar` of type `Container>`. Mixing different container types is also possible. -Annotating a struct with [`#[projecting]`](#projecting-attribute) disables projection via that structs own fields. -Continuing the example from above: +Annotating a struct with [`#[projecting]`](#projecting-attribute) disables projection via that +structs own fields. Continuing the example from above: ```rust struct Bar {} @@ -1217,9 +1218,9 @@ pub mod bar { #### `#[projecting]` Attribute The `#[projecting]` attribute can be put on a struct or union declaration. It requires that the type -has at least one generic type parameter, here we call this parameter `T`. Additionally, the type -must be `#[repr(transparent)]` and either always zero-sized or there must be a unique non-zero-sized -field of type `T` (or a type that is `#[projecting]` over `T`). +is `#[repr(transparent)]` and there must be a unique non-zero-sized field (it is allowed to be +generic and thus not always non-zero-sized). Alternatively, it is allowed to be zero-sized, but then +must either have a single generic, or annotate the projected generic with `#[projecting]`. So for example: @@ -1246,21 +1247,39 @@ Some more examples: // This type is always zero-sized, but "contains" a field. #[projecting] #[repr(transparent)] -pub struct Container2 { +pub struct Container { _phantom: PhantomData, } #[projecting] #[repr(transparent)] -pub struct Container3 { - // nesting multiple containers is fine as long as all of them are `#[projecting]` over the same - // generic. +pub struct Container { + // nesting multiple containers is fine. ctr: Container>, // other zero-sized fields still are allowed: _variance: PhantomData, } + +// multiple generics, but still only one field that is not always zero-sized. +#[projecting] +#[repr(transparent)] +pub struct Container { + inner: T, + _phantom: PhantomData, +} + +// multiple generics and zero-sized +#[projecting] +#[repr(transparent)] +pub struct Container<#[projecting] T, U> { + inner: PhantomData, + other: PhantomData, +} ``` +In the last two examples, if we're given `&Container`, then the projection to `bar` has +the type `&Container`. + Here are some error examples: ```rust @@ -1281,6 +1300,13 @@ pub struct Container {} pub struct Container { foo: Foo, } + +// ERROR: ambiguous projection generic +#[projecting] +#[repr(transparent)] +pub struct Container { + _phantom: PhantomData<(T, U)>, +} ``` ### Field Projection Operator