Skip to content

Commit 53c231b

Browse files
sayantnsayantn
committed
[TRY] Use LLVM AutoUpgrade to bypass autocasts
1 parent f797e42 commit 53c231b

File tree

7 files changed

+55
-165
lines changed

7 files changed

+55
-165
lines changed

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 33 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,8 @@ pub(crate) enum FunctionSignature<'ll> {
313313
/// This is an LLVM intrinsic, but the signature is just the Rust signature.
314314
/// FIXME: this should ideally not exist, we should be using the LLVM signature for all LLVM intrinsics
315315
RustSignature(llvm::Intrinsic, &'ll Type),
316+
/// FIXME: This shouldn't exist as well, but needed to get around autocast
317+
NonMatchingSignature { intrinsic: llvm::Intrinsic, llvm_ty: &'ll Type, rust_ty: &'ll Type },
316318
/// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
317319
MaybeInvalid(&'ll Type),
318320
/// Just the Rust signature
@@ -326,13 +328,15 @@ impl<'ll> FunctionSignature<'ll> {
326328
| FunctionSignature::RustSignature(_, fn_ty)
327329
| FunctionSignature::MaybeInvalid(fn_ty)
328330
| FunctionSignature::NotIntrinsic(fn_ty) => fn_ty,
331+
FunctionSignature::NonMatchingSignature { rust_ty, .. } => rust_ty,
329332
}
330333
}
331334

332335
pub(crate) fn intrinsic(&self) -> Option<llvm::Intrinsic> {
333336
match self {
334337
FunctionSignature::RustSignature(intrinsic, _)
335-
| FunctionSignature::LLVMSignature(intrinsic, _) => Some(*intrinsic),
338+
| FunctionSignature::LLVMSignature(intrinsic, _)
339+
| FunctionSignature::NonMatchingSignature { intrinsic, .. } => Some(*intrinsic),
336340
_ => None,
337341
}
338342
}
@@ -375,40 +379,26 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
375379
return true;
376380
}
377381

378-
match self.type_kind(llvm_ty) {
379-
TypeKind::BFloat => rust_ty == self.type_i16(),
380-
381-
// Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
382-
// due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
383-
// as, well, packed structs, so they won't match with those either)
384-
TypeKind::Struct if self.type_kind(rust_ty) == TypeKind::Struct => {
385-
let rust_element_tys = self.struct_element_types(rust_ty);
386-
let llvm_element_tys = self.struct_element_types(llvm_ty);
387-
388-
if rust_element_tys.len() != llvm_element_tys.len() {
389-
return false;
390-
}
382+
// Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
383+
// due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
384+
// as, well, packed structs, so they won't match with those either)
385+
if self.type_kind(llvm_ty) == TypeKind::Struct
386+
&& self.type_kind(rust_ty) == TypeKind::Struct
387+
{
388+
let rust_element_tys = self.struct_element_types(rust_ty);
389+
let llvm_element_tys = self.struct_element_types(llvm_ty);
391390

392-
iter::zip(rust_element_tys, llvm_element_tys).all(
393-
|(rust_element_ty, llvm_element_ty)| {
394-
self.equate_ty(rust_element_ty, llvm_element_ty)
395-
},
396-
)
397-
}
398-
TypeKind::Vector => {
399-
let element_count = self.vector_length(llvm_ty) as u64;
400-
let llvm_element_ty = self.element_type(llvm_ty);
401-
402-
if llvm_element_ty == self.type_bf16() {
403-
rust_ty == self.type_vector(self.type_i16(), element_count)
404-
} else if llvm_element_ty == self.type_i1() {
405-
let int_width = element_count.next_power_of_two().max(8);
406-
rust_ty == self.type_ix(int_width)
407-
} else {
408-
false
409-
}
391+
if rust_element_tys.len() != llvm_element_tys.len() {
392+
return false;
410393
}
411-
_ => false,
394+
395+
iter::zip(rust_element_tys, llvm_element_tys).all(
396+
|(rust_element_ty, llvm_element_ty)| {
397+
self.equate_ty(rust_element_ty, llvm_element_ty)
398+
},
399+
)
400+
} else {
401+
false
412402
}
413403
}
414404
}
@@ -520,7 +510,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
520510
if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
521511
if !intrinsic.is_overloaded() {
522512
// FIXME: also do this for overloaded intrinsics
523-
FunctionSignature::LLVMSignature(intrinsic, intrinsic.get_type(cx.llcx, &[]))
513+
let llvm_ty = intrinsic.get_type(cx.llcx, &[]);
514+
if self.verify_intrinsic_signature(cx, llvm_ty) {
515+
FunctionSignature::LLVMSignature(intrinsic, llvm_ty)
516+
} else {
517+
FunctionSignature::NonMatchingSignature {
518+
intrinsic,
519+
llvm_ty,
520+
rust_ty: self.rust_signature(cx),
521+
}
522+
}
524523
} else {
525524
FunctionSignature::RustSignature(intrinsic, self.rust_signature(cx))
526525
}

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 1 addition & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,46 +1680,6 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
16801680
}
16811681
}
16821682
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1683-
fn trunc_int_to_i1_vector(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1684-
let vector_length = self.vector_length(dest_ty) as u64;
1685-
let int_width = vector_length.next_power_of_two().max(8);
1686-
1687-
let bitcasted = self.bitcast(val, self.type_vector(self.type_i1(), int_width));
1688-
if vector_length == int_width {
1689-
bitcasted
1690-
} else {
1691-
let shuffle_mask =
1692-
(0..vector_length).map(|i| self.const_i32(i as i32)).collect::<Vec<_>>();
1693-
self.shuffle_vector(bitcasted, bitcasted, self.const_vector(&shuffle_mask))
1694-
}
1695-
}
1696-
1697-
fn zext_i1_vector_to_int(
1698-
&mut self,
1699-
mut val: &'ll Value,
1700-
src_ty: &'ll Type,
1701-
dest_ty: &'ll Type,
1702-
) -> &'ll Value {
1703-
let vector_length = self.vector_length(src_ty) as u64;
1704-
let int_width = vector_length.next_power_of_two().max(8);
1705-
1706-
if vector_length != int_width {
1707-
let shuffle_indices = match vector_length {
1708-
0 => unreachable!("zero length vectors are not allowed"),
1709-
1 => vec![0, 1, 1, 1, 1, 1, 1, 1],
1710-
2 => vec![0, 1, 2, 3, 2, 3, 2, 3],
1711-
3 => vec![0, 1, 2, 3, 4, 5, 3, 4],
1712-
4.. => (0..int_width as i32).collect(),
1713-
};
1714-
let shuffle_mask =
1715-
shuffle_indices.into_iter().map(|i| self.const_i32(i)).collect::<Vec<_>>();
1716-
val =
1717-
self.shuffle_vector(val, self.const_null(src_ty), self.const_vector(&shuffle_mask));
1718-
}
1719-
1720-
self.bitcast(val, dest_ty)
1721-
}
1722-
17231683
fn autocast(
17241684
&mut self,
17251685
llfn: &'ll Value,
@@ -1748,14 +1708,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
17481708
}
17491709
ret
17501710
}
1751-
TypeKind::Vector if self.element_type(llvm_ty) == self.type_i1() => {
1752-
if is_argument {
1753-
self.trunc_int_to_i1_vector(val, dest_ty)
1754-
} else {
1755-
self.zext_i1_vector_to_int(val, src_ty, dest_ty)
1756-
}
1757-
}
1758-
_ => self.bitcast(val, dest_ty), // for `bf16(xN)` <-> `u16(xN)`
1711+
_ => unreachable!(),
17591712
}
17601713
}
17611714

‎compiler/rustc_codegen_llvm/src/declare.rs

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -155,18 +155,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
155155

156156
let span = || instance.map(|instance| self.tcx.def_span(instance.def_id()));
157157

158-
if let FunctionSignature::LLVMSignature(_, llvm_fn_ty) = signature {
159-
// check if the intrinsic signatures match
160-
if !fn_abi.verify_intrinsic_signature(self, llvm_fn_ty) {
161-
self.tcx.dcx().emit_fatal(errors::IntrinsicSignatureMismatch {
162-
name,
163-
llvm_fn_ty: &format!("{llvm_fn_ty:?}"),
164-
rust_fn_ty: &format!("{:?}", fn_abi.rust_signature(self)),
165-
span: span(),
166-
});
167-
}
168-
}
169-
170158
// Function addresses in Rust are never significant, allowing functions to
171159
// be merged.
172160
let llfn = declare_raw_fn(
@@ -212,10 +200,20 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
212200
fn_abi.apply_attrs_llfn(self, llfn, instance);
213201
}
214202

203+
if let FunctionSignature::NonMatchingSignature { llvm_ty, rust_ty, .. } = signature {
204+
let (can_upgrade, _) = llvm::check_autoupgrade(llfn);
205+
if !can_upgrade {
206+
self.tcx.dcx().emit_fatal(errors::IntrinsicSignatureMismatch {
207+
name,
208+
llvm_fn_ty: &format!("{llvm_ty:?}"),
209+
rust_fn_ty: &format!("{rust_ty:?}"),
210+
span: span(),
211+
});
212+
}
213+
}
214+
215215
if let FunctionSignature::MaybeInvalid(..) = signature {
216-
let mut new_llfn = None;
217-
let can_upgrade =
218-
unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
216+
let (can_upgrade, new_llfn) = llvm::check_autoupgrade(llfn);
219217

220218
// we can emit diagnostics for local crates only
221219
if let Some(instance) = instance

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1101,9 +1101,6 @@ unsafe extern "C" {
11011101
pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type;
11021102
pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type;
11031103

1104-
// Operations on non-IEEE real types
1105-
pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type;
1106-
11071104
// Operations on function types
11081105
pub(crate) fn LLVMFunctionType<'a>(
11091106
ReturnType: &'a Type,

‎compiler/rustc_codegen_llvm/src/llvm/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,12 @@ impl Intrinsic {
327327
}
328328
}
329329

330+
pub(crate) fn check_autoupgrade(llfn: &Value) -> (bool, Option<&Value>) {
331+
let mut new_llfn = None;
332+
let can_upgrade = unsafe { LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
333+
(can_upgrade, new_llfn)
334+
}
335+
330336
/// Safe wrapper for `LLVMSetValueName2` from a byte slice
331337
pub(crate) fn set_value_name(value: &Value, name: &[u8]) {
332338
unsafe {

‎compiler/rustc_codegen_llvm/src/type_.rs

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,10 +176,6 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
176176
)
177177
}
178178
}
179-
180-
pub(crate) fn type_bf16(&self) -> &'ll Type {
181-
unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) }
182-
}
183179
}
184180

185181
impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
@@ -253,7 +249,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
253249

254250
fn float_width(&self, ty: &'ll Type) -> usize {
255251
match self.type_kind(ty) {
256-
TypeKind::Half | TypeKind::BFloat => 16,
252+
TypeKind::Half => 16,
257253
TypeKind::Float => 32,
258254
TypeKind::Double => 64,
259255
TypeKind::X86_FP80 => 80,

‎tests/codegen-llvm/inject-autocast.rs

Lines changed: 1 addition & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -4,51 +4,11 @@
44
#![feature(link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi, portable_simd, f16)]
55
#![crate_type = "lib"]
66

7-
use std::simd::{f32x4, i16x8, i64x2};
8-
9-
#[repr(simd)]
10-
pub struct Tile([i8; 1024]);
11-
7+
use std::simd::i64x2;
128
#[repr(C, packed)]
139
pub struct Bar(u32, i64x2, i64x2, i64x2, i64x2, i64x2, i64x2);
1410
// CHECK: %Bar = type <{ i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> }>
1511

16-
#[repr(simd)]
17-
pub struct f16x8([f16; 8]);
18-
19-
// CHECK-LABEL: @struct_with_i1_vector_autocast
20-
#[no_mangle]
21-
pub unsafe fn struct_with_i1_vector_autocast(a: i64x2, b: i64x2) -> (u8, u8) {
22-
extern "unadjusted" {
23-
#[link_name = "llvm.x86.avx512.vp2intersect.q.128"]
24-
fn foo(a: i64x2, b: i64x2) -> (u8, u8);
25-
}
26-
27-
// CHECK: %2 = call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> %0, <2 x i64> %1)
28-
// CHECK-NEXT: %3 = extractvalue { <2 x i1>, <2 x i1> } %2, 0
29-
// CHECK-NEXT: %4 = shufflevector <2 x i1> %3, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
30-
// CHECK-NEXT: %5 = bitcast <8 x i1> %4 to i8
31-
// CHECK-NEXT: %6 = insertvalue { i8, i8 } poison, i8 %5, 0
32-
// CHECK-NEXT: %7 = extractvalue { <2 x i1>, <2 x i1> } %2, 1
33-
// CHECK-NEXT: %8 = shufflevector <2 x i1> %7, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
34-
// CHECK-NEXT: %9 = bitcast <8 x i1> %8 to i8
35-
// CHECK-NEXT: %10 = insertvalue { i8, i8 } %6, i8 %9, 1
36-
foo(a, b)
37-
}
38-
39-
// CHECK-LABEL: @bf16_vector_autocast
40-
#[no_mangle]
41-
pub unsafe fn bf16_vector_autocast(a: f32x4) -> i16x8 {
42-
extern "unadjusted" {
43-
#[link_name = "llvm.x86.vcvtneps2bf16128"]
44-
fn foo(a: f32x4) -> i16x8;
45-
}
46-
47-
// CHECK: %1 = call <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float> %0)
48-
// CHECK-NEXT: %2 = bitcast <8 x bfloat> %1 to <8 x i16>
49-
foo(a)
50-
}
51-
5212
// CHECK-LABEL: @struct_autocast
5313
#[no_mangle]
5414
pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar {
@@ -75,23 +35,4 @@ pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar {
7535
foo(key_metadata, key)
7636
}
7737

78-
// CHECK-LABEL: @i1_vector_autocast
79-
#[no_mangle]
80-
pub unsafe fn i1_vector_autocast(a: f16x8) -> u8 {
81-
extern "unadjusted" {
82-
#[link_name = "llvm.x86.avx512fp16.fpclass.ph.128"]
83-
fn foo(a: f16x8, b: i32) -> u8;
84-
}
85-
86-
// CHECK: %1 = call <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half> %0, i32 1)
87-
// CHECK-NEXT: %_0 = bitcast <8 x i1> %1 to i8
88-
foo(a, 1)
89-
}
90-
91-
// CHECK: declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>)
92-
93-
// CHECK: declare <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float>)
94-
9538
// CHECK: declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>)
96-
97-
// CHECK: declare <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half>, i32 immarg)

0 commit comments

Comments
 (0)