From 0aa49568f3f1488b43510c26e386e76c7c9e4d0c Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 29 Dec 2022 20:49:06 +0000 Subject: [PATCH] Use `#[thread_local]` instead of `thread_local!` This avoid monomorphizing `LocalKey::try_with` 5 times for each query. It has the downside that destructors for the thread-local are never run; but we don't depend on the destructors for ImplicitContext itself anywhere. --- compiler/rustc_middle/src/lib.rs | 3 + compiler/rustc_middle/src/ty/context.rs | 116 ++++++++++++++---------- 2 files changed, 73 insertions(+), 46 deletions(-) diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index 7e4063c2ffd78..bca98008991a5 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -33,6 +33,7 @@ #![feature(generators)] #![feature(get_mut_unchecked)] #![feature(if_let_guard)] +#![feature(inline_const)] #![feature(iter_from_generator)] #![feature(negative_impls)] #![feature(never_type)] @@ -47,6 +48,8 @@ #![feature(rustc_attrs)] #![feature(control_flow_enum)] #![feature(associated_type_defaults)] +#![feature(thread_local)] +#![feature(cfg_target_thread_local)] #![feature(trusted_step)] #![feature(try_blocks)] #![feature(try_reserve_kind)] diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 5de414077a2b1..a662f645a6eb1 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1215,11 +1215,77 @@ pub mod tls { use std::mem; use thin_vec::ThinVec; - #[cfg(not(parallel_compiler))] - use std::cell::Cell; - #[cfg(parallel_compiler)] - use rustc_rayon_core as rayon_core; + mod tls_impl { + use rustc_rayon_core as rayon_core; + + /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs + /// to `value` during the call to `f`. It is restored to its previous value after. + /// This is used to set the pointer to the new `ImplicitCtxt`. + #[inline] + pub(super) fn set_tlv R, R>(value: usize, f: F) -> R { + rayon_core::tlv::with(value, f) + } + + /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs. + /// This is used to get the pointer to the current `ImplicitCtxt`. + #[inline] + pub fn get_tlv() -> usize { + rayon_core::tlv::get() + } + } + + #[cfg(all(not(parallel_compiler), target_thread_local))] + mod tls_impl { + use std::cell::Cell; + + /// A thread local variable that stores a pointer to the current `ImplicitCtxt`. + #[thread_local] + static TLV: Cell = Cell::new(0); + + /// Sets TLV to `value` during the call to `f`. + /// It is restored to its previous value after. + /// This is used to set the pointer to the new `ImplicitCtxt`. + #[inline] + pub(super) fn set_tlv R, R>(value: usize, f: F) -> R { + let old = get_tlv(); + let _reset = rustc_data_structures::OnDrop(move || TLV.set(old)); + TLV.set(value); + f() + } + + /// Gets the pointer to the current `ImplicitCtxt`. + #[inline] + pub(super) fn get_tlv() -> usize { + TLV.get() + } + } + + #[cfg(all(not(parallel_compiler), not(target_thread_local)))] + /// Ideally this wouldn't be necessary, but `#[thread_local]` uses OS built-ins which don't work on every platform. + /// The reason we use `#[thread_local]` at all is to avoid monomorphizing `LocalKey::with` for each `F` passed to `set_tlv`. + mod tls_impl { + use std::cell::Cell; + + thread_local! { + static TLV: Cell = Cell::new(0); + } + + #[inline] + pub(super) fn set_tlv R, R>(value: usize, f: F) -> R { + let old = get_tlv(); + let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old))); + TLV.with(|tlv| tlv.set(value)); + f() + } + + #[inline] + pub(super) fn get_tlv() -> usize { + TLV.with(|tlv| tlv.get()) + } + } + + use tls_impl::{get_tlv, set_tlv}; /// This is the implicit state of rustc. It contains the current /// `TyCtxt` and query. It is updated when creating a local interner or @@ -1260,48 +1326,6 @@ pub mod tls { } } - /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs - /// to `value` during the call to `f`. It is restored to its previous value after. - /// This is used to set the pointer to the new `ImplicitCtxt`. - #[cfg(parallel_compiler)] - #[inline] - fn set_tlv R, R>(value: usize, f: F) -> R { - rayon_core::tlv::with(value, f) - } - - /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs. - /// This is used to get the pointer to the current `ImplicitCtxt`. - #[cfg(parallel_compiler)] - #[inline] - pub fn get_tlv() -> usize { - rayon_core::tlv::get() - } - - #[cfg(not(parallel_compiler))] - thread_local! { - /// A thread local variable that stores a pointer to the current `ImplicitCtxt`. - static TLV: Cell = const { Cell::new(0) }; - } - - /// Sets TLV to `value` during the call to `f`. - /// It is restored to its previous value after. - /// This is used to set the pointer to the new `ImplicitCtxt`. - #[cfg(not(parallel_compiler))] - #[inline] - fn set_tlv R, R>(value: usize, f: F) -> R { - let old = get_tlv(); - let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old))); - TLV.with(|tlv| tlv.set(value)); - f() - } - - /// Gets the pointer to the current `ImplicitCtxt`. - #[cfg(not(parallel_compiler))] - #[inline] - fn get_tlv() -> usize { - TLV.with(|tlv| tlv.get()) - } - /// Sets `context` as the new current `ImplicitCtxt` for the duration of the function `f`. #[inline] pub fn enter_context<'a, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'tcx>, f: F) -> R