CFI: Fix fn items, closures, and Fn trait objects

Fix casting between function items, closures, and Fn trait objects by
transforming function items, closures, and Fn trait objects into
function pointers for encoding.

Author: rcvalle

compiler/rustc_middle/src/ty/context.rs

@@ -1841,7 +1841,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// does not compute the full elaborated super-predicates but just the set of def-ids. It is used
     /// to identify which traits may define a given associated type to help avoid cycle errors.
     /// Returns a `DefId` iterator.
-    fn super_traits_of(self, trait_def_id: DefId) -> impl Iterator<Item = DefId> + 'tcx {
+    pub fn super_traits_of(self, trait_def_id: DefId) -> impl Iterator<Item = DefId> + 'tcx {
         let mut set = FxHashSet::default();
         let mut stack = vec![trait_def_id];
 

compiler/rustc_middle/src/ty/sty.rs

@@ -1836,6 +1836,11 @@ impl<'tcx> Ty<'tcx> {
         self.0.0.flags
     }
 
+    #[inline]
+    pub fn is_tuple(self) -> bool {
+        matches!(self.kind(), Tuple(..))
+    }
+
     #[inline]
     pub fn is_unit(self) -> bool {
         match self.kind() {
@@ -2208,6 +2213,11 @@ impl<'tcx> Ty<'tcx> {
         matches!(self.kind(), FnDef(..) | FnPtr(_))
     }
 
+    #[inline]
+    pub fn is_fn_def(self) -> bool {
+        matches!(self.kind(), FnDef(..))
+    }
+
     #[inline]
     pub fn is_fn_ptr(self) -> bool {
         matches!(self.kind(), FnPtr(_))

compiler/rustc_symbol_mangling/src/lib.rs

@@ -91,6 +91,8 @@
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
 #![allow(internal_features)]
+#![feature(iter_order_by)]
+#![feature(let_chains)]
 
 #[macro_use]
 extern crate rustc_middle;

compiler/rustc_symbol_mangling/src/typeid.rs

@@ -4,7 +4,7 @@
 /// For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
 /// see design document in the tracking issue #89653.
 use bitflags::bitflags;
-use rustc_middle::ty::{Instance, Ty, TyCtxt};
+use rustc_middle::ty::{Instance, List, Ty, TyCtxt};
 use rustc_target::abi::call::FnAbi;
 use std::hash::Hasher;
 use twox_hash::XxHash64;
@@ -27,7 +27,11 @@ pub fn typeid_for_fnabi<'tcx>(
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
     options: TypeIdOptions,
 ) -> String {
-    typeid_itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options)
+    typeid_itanium_cxx_abi::typeid_for_fnabi(
+        tcx,
+        &typeid_itanium_cxx_abi::transform_fnabi(tcx, &fn_abi, options, None),
+        options,
+    )
 }
 
 /// Returns a type metadata identifier for the specified Instance.
@@ -36,7 +40,16 @@ pub fn typeid_for_instance<'tcx>(
     instance: &Instance<'tcx>,
     options: TypeIdOptions,
 ) -> String {
-    typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options)
+    let fn_abi = tcx
+        .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((*instance, List::empty())))
+        .unwrap_or_else(|instance| {
+            bug!("typeid_for_instance: couldn't get fn_abi of instance {:?}", instance)
+        });
+    typeid_itanium_cxx_abi::typeid_for_fnabi(
+        tcx,
+        &typeid_itanium_cxx_abi::transform_fnabi(tcx, &fn_abi, options, Some(instance)),
+        options,
+    )
 }
 
 /// Returns a KCFI type metadata identifier for the specified FnAbi.
@@ -48,7 +61,14 @@ pub fn kcfi_typeid_for_fnabi<'tcx>(
     // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
     // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
     let mut hash: XxHash64 = Default::default();
-    hash.write(typeid_itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options).as_bytes());
+    hash.write(
+        typeid_itanium_cxx_abi::typeid_for_fnabi(
+            tcx,
+            &typeid_itanium_cxx_abi::transform_fnabi(tcx, &fn_abi, options, None),
+            options,
+        )
+        .as_bytes(),
+    );
     hash.finish() as u32
 }
 
@@ -58,9 +78,21 @@ pub fn kcfi_typeid_for_instance<'tcx>(
     instance: &Instance<'tcx>,
     options: TypeIdOptions,
 ) -> u32 {
+    let fn_abi = tcx
+        .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((*instance, List::empty())))
+        .unwrap_or_else(|instance| {
+            bug!("typeid_for_instance: couldn't get fn_abi of instance {:?}", instance)
+        });
     // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
     // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
     let mut hash: XxHash64 = Default::default();
-    hash.write(typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options).as_bytes());
+    hash.write(
+        typeid_itanium_cxx_abi::typeid_for_fnabi(
+            tcx,
+            &typeid_itanium_cxx_abi::transform_fnabi(tcx, &fn_abi, options, Some(instance)),
+            options,
+        )
+        .as_bytes(),
+    );
     hash.finish() as u32
 }