From 10270f4b442207b23b28f776b8dda297bd03570a Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Wed, 11 May 2022 17:52:00 +0400 Subject: [PATCH 01/10] Add pointer masking convenience functions This commit adds the following functions all of which have a signature `pointer, usize -> pointer`: - `<*mut T>::mask` - `<*const T>::mask` - `intrinsics::ptr_mask` These functions are equivalent to `.map_addr(|a| a & mask)` but they utilize `llvm.ptrmask` llvm intrinsic. *masks your pointers* --- .../rustc_codegen_cranelift/src/intrinsics/mod.rs | 7 +++++++ compiler/rustc_codegen_llvm/src/context.rs | 4 ++++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 1 + compiler/rustc_span/src/symbol.rs | 1 + compiler/rustc_typeck/src/check/intrinsic.rs | 12 +++++++++++- library/core/src/intrinsics.rs | 11 +++++++++++ library/core/src/ptr/const_ptr.rs | 15 +++++++++++++++ library/core/src/ptr/mut_ptr.rs | 15 +++++++++++++++ 8 files changed, 65 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index b2a83e1d4ebc9..65e964c786b52 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -540,6 +540,13 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(res, base.layout())); } + sym::ptr_mask => { + intrinsic_args!(fx, args => (ptr, mask); intrinsic); + let ptr_val = ptr.load_scalar(fx); + + fx.bcx.ins().band(ptr_val, mask); + } + sym::transmute => { intrinsic_args!(fx, args => (from); intrinsic); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d4d8414723906..bbb6bacc452a9 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -644,6 +644,7 @@ impl<'ll> CodegenCx<'ll, '_> { let i8p = self.type_i8p(); let void = self.type_void(); + let voidp = self.type_ptr_to(void); let i1 = self.type_i1(); let t_i8 = self.type_i8(); let t_i16 = self.type_i16(); @@ -886,6 +887,9 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(voidp, t_isize) -> voidp); + None } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9f3647492877c..1a0e44fdc5bd2 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -71,6 +71,7 @@ fn get_simple_intrinsic<'ll>( sym::nearbyintf64 => "llvm.nearbyint.f64", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::ptr_mask => "llvm.ptrmask", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 156f53ac48626..19e136a4476f9 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1112,6 +1112,7 @@ symbols! { ptr, ptr_guaranteed_eq, ptr_guaranteed_ne, + ptr_mask, ptr_null, ptr_null_mut, ptr_offset_from, diff --git a/compiler/rustc_typeck/src/check/intrinsic.rs b/compiler/rustc_typeck/src/check/intrinsic.rs index 05686be5d4b3d..73dd7122e269a 100644 --- a/compiler/rustc_typeck/src/check/intrinsic.rs +++ b/compiler/rustc_typeck/src/check/intrinsic.rs @@ -105,7 +105,8 @@ pub fn intrinsic_operation_unsafety(intrinsic: Symbol) -> hir::Unsafety { | sym::type_name | sym::forget | sym::black_box - | sym::variant_count => hir::Unsafety::Normal, + | sym::variant_count + | sym::ptr_mask => hir::Unsafety::Normal, _ => hir::Unsafety::Unsafe, } } @@ -203,6 +204,15 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { ], tcx.mk_ptr(ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }), ), + sym::ptr_mask => ( + 1, + vec![ + tcx.mk_ptr(ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }), + tcx.types.usize, + ], + tcx.mk_ptr(ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }), + ), + sym::copy | sym::copy_nonoverlapping => ( 1, vec![ diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 15467e0191dbf..c95dae745f430 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1308,6 +1308,17 @@ extern "rust-intrinsic" { #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")] pub fn arith_offset(dst: *const T, offset: isize) -> *const T; + /// Masks out bits of the pointer according to a mask. + /// + /// Note that, unlike most intrinsics, this is safe to call; + /// it does not require an `unsafe` block. + /// Therefore, implementations must not require the user to uphold + /// any safety invariants. + /// + /// Consider using [`pointer::mask`] instead. + #[cfg(not(bootstrap))] + pub fn ptr_mask(ptr: *const T, mask: usize) -> *const T; + /// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with /// a size of `count` * `size_of::()` and an alignment of /// `min_align_of::()` diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index c25b159c533a1..624d283155ac1 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -559,6 +559,21 @@ impl *const T { from_raw_parts::(self.cast::().wrapping_offset(count).cast::<()>(), metadata(self)) } + /// Masks out bits of the pointer according to a mask. + /// + /// This is convenience for `ptr.map_addr(|a| a & mask)`. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[cfg(not(bootstrap))] + #[unstable(feature = "ptr_mask", issue = "none")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline(always)] + pub fn mask(self, mask: usize) -> *const T { + let this = intrinsics::ptr_mask(self.cast::<()>(), mask); + from_raw_parts::(this, metadata(self)) + } + /// Calculates the distance between two pointers. The returned value is in /// units of T: the distance in bytes divided by `mem::size_of::()`. /// diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index fff06b458c7c1..ea87bb88a1bc5 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -575,6 +575,21 @@ impl *mut T { ) } + /// Masks out bits of the pointer according to a mask. + /// + /// This is convenience for `ptr.map_addr(|a| a & mask)`. + /// + /// For non-`Sized` pointees this operation changes only the data pointer, + /// leaving the metadata untouched. + #[cfg(not(bootstrap))] + #[unstable(feature = "ptr_mask", issue = "none")] + #[must_use = "returns a new pointer rather than modifying its argument"] + #[inline(always)] + pub fn mask(self, mask: usize) -> *mut T { + let this = intrinsics::ptr_mask(self.cast::<()>(), mask) as *mut (); + from_raw_parts_mut::(this, metadata(self)) + } + /// Returns `None` if the pointer is null, or else returns a unique reference to /// the value wrapped in `Some`. If the value may be uninitialized, [`as_uninit_mut`] /// must be used instead. From 9af24df67af3e7f4ff6cc3dc2b2b8fd4f814f9af Mon Sep 17 00:00:00 2001 From: Waffle Maybe Date: Wed, 11 May 2022 21:43:13 +0400 Subject: [PATCH 02/10] use shorter `ptr_mask` impl in cg cranelift Co-authored-by: bjorn3 --- compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 65e964c786b52..35880bb3e22be 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -542,9 +542,7 @@ fn codegen_regular_intrinsic_call<'tcx>( sym::ptr_mask => { intrinsic_args!(fx, args => (ptr, mask); intrinsic); - let ptr_val = ptr.load_scalar(fx); - - fx.bcx.ins().band(ptr_val, mask); + fx.bcx.ins().band(ptr, mask); } sym::transmute => { From a6183c2f44309a32a54ce13da91ebd4ee49b51c9 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Wed, 11 May 2022 22:01:53 +0400 Subject: [PATCH 03/10] Implement `ptr_mask` intrinsic in cg gcc --- compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 5fbdedac0c45c..b0e7e5e1dd05c 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -309,6 +309,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { return; } + sym::ptr_mask => self.and(args[0].immediate(), args[1].immediate()), + _ if name_str.starts_with("simd_") => { match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, From 4c54973b65713dccfafd3035de2c60945945be57 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Fri, 13 May 2022 17:08:44 +0400 Subject: [PATCH 04/10] Fix `ptr_mask` impl in cg gcc --- compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index b0e7e5e1dd05c..beb9a80014421 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -309,8 +309,18 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { return; } - sym::ptr_mask => self.and(args[0].immediate(), args[1].immediate()), + sym::ptr_mask => { + let usize_type = self.context.new_type::(); + let void_ptr_type = self.context.new_type::<*const ()>(); + let ptr = args[0].immediate(); + let mask = args[1].immediate(); + + let addr = self.bitcast(ptr, usize_type); + let masked = self.and(addr, mask); + self.bitcast(masked, void_ptr_type) + }, + _ if name_str.starts_with("simd_") => { match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, From 168a83797566cffeb6e0ba6f4eb6724d06b6436c Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Mon, 20 Jun 2022 16:26:40 +0400 Subject: [PATCH 05/10] fill in tracking issue for `feature(ptr_mask)` --- library/core/src/ptr/const_ptr.rs | 2 +- library/core/src/ptr/mut_ptr.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 624d283155ac1..4912ba2f6cf5e 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -566,7 +566,7 @@ impl *const T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[cfg(not(bootstrap))] - #[unstable(feature = "ptr_mask", issue = "none")] + #[unstable(feature = "ptr_mask", issue = "98290")] #[must_use = "returns a new pointer rather than modifying its argument"] #[inline(always)] pub fn mask(self, mask: usize) -> *const T { diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index ea87bb88a1bc5..c446e1f907d9e 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -582,7 +582,7 @@ impl *mut T { /// For non-`Sized` pointees this operation changes only the data pointer, /// leaving the metadata untouched. #[cfg(not(bootstrap))] - #[unstable(feature = "ptr_mask", issue = "none")] + #[unstable(feature = "ptr_mask", issue = "98290")] #[must_use = "returns a new pointer rather than modifying its argument"] #[inline(always)] pub fn mask(self, mask: usize) -> *mut T { From 9c191203a9c38df4092caed1925fdab02deb8323 Mon Sep 17 00:00:00 2001 From: Waffle Maybe Date: Sun, 7 Aug 2022 22:32:26 +0400 Subject: [PATCH 06/10] fix cg cranelift Co-authored-by: bjorn3 <17426603+bjorn3@users.noreply.github.com> --- compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 35880bb3e22be..9fe43157d508a 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -542,6 +542,8 @@ fn codegen_regular_intrinsic_call<'tcx>( sym::ptr_mask => { intrinsic_args!(fx, args => (ptr, mask); intrinsic); + let ptr = ptr.load_scalar(fx); + let mask = mask.load_scalar(fx); fx.bcx.ins().band(ptr, mask); } From 553f79055649cb2697bb1be28053ae76613414a7 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Mon, 8 Aug 2022 18:00:45 +0400 Subject: [PATCH 07/10] Add codegen test for `intinsics::ptr_mask` --- src/test/codegen/intrinsics/mask.rs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/test/codegen/intrinsics/mask.rs diff --git a/src/test/codegen/intrinsics/mask.rs b/src/test/codegen/intrinsics/mask.rs new file mode 100644 index 0000000000000..a636fb20ecb73 --- /dev/null +++ b/src/test/codegen/intrinsics/mask.rs @@ -0,0 +1,7 @@ +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +pub fn mask_ptr(ptr: *const u8, mask: usize) -> *const u8 { + // CHECK: llvm.ptrmask + core::intrinsics::ptr_mask(ptr, mask) +} From 55ba58cadbd9d8e6c850491080ea7a77cd5e194d Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 9 Aug 2022 19:51:45 +0400 Subject: [PATCH 08/10] make `ptr_mask` codegen test more specific --- src/test/codegen/intrinsics/mask.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/codegen/intrinsics/mask.rs b/src/test/codegen/intrinsics/mask.rs index a636fb20ecb73..56e84a37e987f 100644 --- a/src/test/codegen/intrinsics/mask.rs +++ b/src/test/codegen/intrinsics/mask.rs @@ -1,7 +1,11 @@ #![crate_type = "lib"] #![feature(core_intrinsics)] +// CHECK-LABEL: @mask_ptr +// CHECK-SAME: [[WORD:i[0-9]+]] %mask +#[no_mangle] pub fn mask_ptr(ptr: *const u8, mask: usize) -> *const u8 { - // CHECK: llvm.ptrmask + // CHECK: call + // CHECK-SAME: @llvm.ptrmask.p0isVoid.[[WORD]]( core::intrinsics::ptr_mask(ptr, mask) } From 92b05db7619fb02f2265ffca42d2aa8d78646d7d Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Thu, 18 Aug 2022 22:21:53 +0400 Subject: [PATCH 09/10] Do not use void pointer for `ptr_mask` intrinsic I couldn't find where exactly it's documented, but apperantly pointers to void type are invalid in llvm - void is only allowed as a return type of functions. --- compiler/rustc_codegen_llvm/src/context.rs | 3 +-- src/test/codegen/intrinsics/mask.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index bbb6bacc452a9..67ffc7cb9511f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -644,7 +644,6 @@ impl<'ll> CodegenCx<'ll, '_> { let i8p = self.type_i8p(); let void = self.type_void(); - let voidp = self.type_ptr_to(void); let i1 = self.type_i1(); let t_i8 = self.type_i8(); let t_i16 = self.type_i16(); @@ -888,7 +887,7 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } - ifn!("llvm.ptrmask", fn(voidp, t_isize) -> voidp); + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); None } diff --git a/src/test/codegen/intrinsics/mask.rs b/src/test/codegen/intrinsics/mask.rs index 56e84a37e987f..c7432cc56c96b 100644 --- a/src/test/codegen/intrinsics/mask.rs +++ b/src/test/codegen/intrinsics/mask.rs @@ -4,8 +4,8 @@ // CHECK-LABEL: @mask_ptr // CHECK-SAME: [[WORD:i[0-9]+]] %mask #[no_mangle] -pub fn mask_ptr(ptr: *const u8, mask: usize) -> *const u8 { +pub fn mask_ptr(ptr: *const u16, mask: usize) -> *const u16 { // CHECK: call - // CHECK-SAME: @llvm.ptrmask.p0isVoid.[[WORD]]( + // CHECK-SAME: @llvm.ptrmask.p0i8.[[WORD]]( core::intrinsics::ptr_mask(ptr, mask) } From ca753124088c0d57ca6728ee093363cbc1db5274 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Sun, 21 Aug 2022 06:16:57 +0400 Subject: [PATCH 10/10] fix `ptr_mask` codegen test wrt llvm opaque pointers --- src/test/codegen/intrinsics/mask.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/codegen/intrinsics/mask.rs b/src/test/codegen/intrinsics/mask.rs index c7432cc56c96b..2e984db1be528 100644 --- a/src/test/codegen/intrinsics/mask.rs +++ b/src/test/codegen/intrinsics/mask.rs @@ -6,6 +6,6 @@ #[no_mangle] pub fn mask_ptr(ptr: *const u16, mask: usize) -> *const u16 { // CHECK: call - // CHECK-SAME: @llvm.ptrmask.p0i8.[[WORD]]( + // CHECK-SAME: @llvm.ptrmask.{{p0|p0i8}}.[[WORD]]({{ptr|i8\*}} {{%ptr|%0}}, [[WORD]] %mask) core::intrinsics::ptr_mask(ptr, mask) }