Skip to content

Commit 1a4e2b6

Browse files
borsbors
committed
Auto merge of rust-lang#73180 - matthewjasper:predicate-cache, r=nikomatsakis
Cache flags and escaping vars for predicates With predicates becoming interned (rust-lang/compiler-team#285) this is now possible and could be a perf win. It would become an even larger win once we have recursive predicates. cc @lcnr @nikomatsakis r? @ghost
2 parents a8cf399 + 6e12272 commit 1a4e2b6

File tree

11 files changed

+321
-100
lines changed

11 files changed

+321
-100
lines changed

‎src/librustc_metadata/rmeta/decoder.rs

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,15 +294,36 @@ impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> {
294294

295295
let key = ty::CReaderCacheKey { cnum: self.cdata().cnum, pos: shorthand };
296296

297-
if let Some(&ty) = tcx.rcache.borrow().get(&key) {
297+
if let Some(&ty) = tcx.ty_rcache.borrow().get(&key) {
298298
return Ok(ty);
299299
}
300300

301301
let ty = or_insert_with(self)?;
302-
tcx.rcache.borrow_mut().insert(key, ty);
302+
tcx.ty_rcache.borrow_mut().insert(key, ty);
303303
Ok(ty)
304304
}
305305

306+
fn cached_predicate_for_shorthand<F>(
307+
&mut self,
308+
shorthand: usize,
309+
or_insert_with: F,
310+
) -> Result<ty::Predicate<'tcx>, Self::Error>
311+
where
312+
F: FnOnce(&mut Self) -> Result<ty::Predicate<'tcx>, Self::Error>,
313+
{
314+
let tcx = self.tcx();
315+
316+
let key = ty::CReaderCacheKey { cnum: self.cdata().cnum, pos: shorthand };
317+
318+
if let Some(&pred) = tcx.pred_rcache.borrow().get(&key) {
319+
return Ok(pred);
320+
}
321+
322+
let pred = or_insert_with(self)?;
323+
tcx.pred_rcache.borrow_mut().insert(key, pred);
324+
Ok(pred)
325+
}
326+
306327
fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
307328
where
308329
F: FnOnce(&mut Self) -> R,

‎src/librustc_metadata/rmeta/encoder.rs

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,17 @@ where
239239
}
240240
}
241241

242+
impl<'b, 'tcx> SpecializedEncoder<ty::Predicate<'b>> for EncodeContext<'tcx> {
243+
fn specialized_encode(&mut self, predicate: &ty::Predicate<'b>) -> Result<(), Self::Error> {
244+
debug_assert!(self.tcx.lift(predicate).is_some());
245+
let predicate =
246+
unsafe { std::mem::transmute::<&ty::Predicate<'b>, &ty::Predicate<'tcx>>(predicate) };
247+
ty_codec::encode_with_shorthand(self, predicate, |encoder| {
248+
&mut encoder.predicate_shorthands
249+
})
250+
}
251+
}
252+
242253
impl<'tcx> SpecializedEncoder<interpret::AllocId> for EncodeContext<'tcx> {
243254
fn specialized_encode(&mut self, alloc_id: &interpret::AllocId) -> Result<(), Self::Error> {
244255
use std::collections::hash_map::Entry;
@@ -256,22 +267,6 @@ impl<'tcx> SpecializedEncoder<interpret::AllocId> for EncodeContext<'tcx> {
256267
}
257268
}
258269

259-
impl<'a, 'b, 'tcx> SpecializedEncoder<&'a [(ty::Predicate<'b>, Span)]> for EncodeContext<'tcx> {
260-
fn specialized_encode(
261-
&mut self,
262-
predicates: &&'a [(ty::Predicate<'b>, Span)],
263-
) -> Result<(), Self::Error> {
264-
debug_assert!(self.tcx.lift(*predicates).is_some());
265-
let predicates = unsafe {
266-
std::mem::transmute::<
267-
&&'a [(ty::Predicate<'b>, Span)],
268-
&&'tcx [(ty::Predicate<'tcx>, Span)],
269-
>(predicates)
270-
};
271-
ty_codec::encode_spanned_predicates(self, &predicates, |ecx| &mut ecx.predicate_shorthands)
272-
}
273-
}
274-
275270
impl<'tcx> SpecializedEncoder<Fingerprint> for EncodeContext<'tcx> {
276271
fn specialized_encode(&mut self, f: &Fingerprint) -> Result<(), Self::Error> {
277272
f.encode_opaque(&mut self.opaque)

‎src/librustc_middle/arena.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ macro_rules! arena_types {
100100

101101
// Interned types
102102
[] tys: rustc_middle::ty::TyS<$tcx>, rustc_middle::ty::TyS<'_x>;
103+
[] predicates: rustc_middle::ty::PredicateInner<$tcx>, rustc_middle::ty::PredicateInner<'_x>;
103104

104105
// HIR query types
105106
[few] indexed_hir: rustc_middle::hir::map::IndexedHir<$tcx>, rustc_middle::hir::map::IndexedHir<'_x>;

‎src/librustc_middle/ty/codec.rs

Lines changed: 48 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use crate::arena::ArenaAllocatable;
1010
use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
1111
use crate::mir::{self, interpret::Allocation};
1212
use crate::ty::subst::SubstsRef;
13-
use crate::ty::{self, List, ToPredicate, Ty, TyCtxt};
13+
use crate::ty::{self, List, Ty, TyCtxt};
1414
use rustc_data_structures::fx::FxHashMap;
1515
use rustc_hir::def_id::{CrateNum, DefId};
1616
use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder};
@@ -95,23 +95,6 @@ where
9595
Ok(())
9696
}
9797

98-
pub fn encode_spanned_predicates<'tcx, E, C>(
99-
encoder: &mut E,
100-
predicates: &[(ty::Predicate<'tcx>, Span)],
101-
cache: C,
102-
) -> Result<(), E::Error>
103-
where
104-
E: TyEncoder,
105-
C: for<'b> Fn(&'b mut E) -> &'b mut FxHashMap<ty::Predicate<'tcx>, usize>,
106-
{
107-
predicates.len().encode(encoder)?;
108-
for (predicate, span) in predicates {
109-
encode_with_shorthand(encoder, predicate, &cache)?;
110-
span.encode(encoder)?;
111-
}
112-
Ok(())
113-
}
114-
11598
pub trait TyDecoder<'tcx>: Decoder {
11699
fn tcx(&self) -> TyCtxt<'tcx>;
117100

@@ -127,6 +110,14 @@ pub trait TyDecoder<'tcx>: Decoder {
127110
where
128111
F: FnOnce(&mut Self) -> Result<Ty<'tcx>, Self::Error>;
129112

113+
fn cached_predicate_for_shorthand<F>(
114+
&mut self,
115+
shorthand: usize,
116+
or_insert_with: F,
117+
) -> Result<ty::Predicate<'tcx>, Self::Error>
118+
where
119+
F: FnOnce(&mut Self) -> Result<ty::Predicate<'tcx>, Self::Error>;
120+
130121
fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
131122
where
132123
F: FnOnce(&mut Self) -> R;
@@ -188,6 +179,26 @@ where
188179
}
189180
}
190181

182+
#[inline]
183+
pub fn decode_predicate<D>(decoder: &mut D) -> Result<ty::Predicate<'tcx>, D::Error>
184+
where
185+
D: TyDecoder<'tcx>,
186+
{
187+
// Handle shorthands first, if we have an usize > 0x80.
188+
if decoder.positioned_at_shorthand() {
189+
let pos = decoder.read_usize()?;
190+
assert!(pos >= SHORTHAND_OFFSET);
191+
let shorthand = pos - SHORTHAND_OFFSET;
192+
193+
decoder.cached_predicate_for_shorthand(shorthand, |decoder| {
194+
decoder.with_position(shorthand, ty::Predicate::decode)
195+
})
196+
} else {
197+
let tcx = decoder.tcx();
198+
Ok(tcx.mk_predicate(ty::PredicateKind::decode(decoder)?))
199+
}
200+
}
201+
191202
#[inline]
192203
pub fn decode_spanned_predicates<D>(
193204
decoder: &mut D,
@@ -198,20 +209,7 @@ where
198209
let tcx = decoder.tcx();
199210
Ok(tcx.arena.alloc_from_iter(
200211
(0..decoder.read_usize()?)
201-
.map(|_| {
202-
// Handle shorthands first, if we have an usize > 0x80.
203-
let predicate_kind = if decoder.positioned_at_shorthand() {
204-
let pos = decoder.read_usize()?;
205-
assert!(pos >= SHORTHAND_OFFSET);
206-
let shorthand = pos - SHORTHAND_OFFSET;
207-
208-
decoder.with_position(shorthand, ty::PredicateKind::decode)
209-
} else {
210-
ty::PredicateKind::decode(decoder)
211-
}?;
212-
let predicate = predicate_kind.to_predicate(tcx);
213-
Ok((predicate, Decodable::decode(decoder)?))
214-
})
212+
.map(|_| Decodable::decode(decoder))
215213
.collect::<Result<Vec<_>, _>>()?,
216214
))
217215
}
@@ -421,7 +419,6 @@ macro_rules! implement_ty_decoder {
421419
// FIXME(#36588): These impls are horribly unsound as they allow
422420
// the caller to pick any lifetime for `'tcx`, including `'static`.
423421

424-
rustc_hir::arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx);
425422
arena_types!(impl_arena_allocatable_decoders, [$DecoderName [$($typaram),*]], 'tcx);
426423

427424
impl<$($typaram),*> SpecializedDecoder<CrateNum>
@@ -436,7 +433,24 @@ macro_rules! implement_ty_decoder {
436433
where &'_x ty::TyS<'_y>: UseSpecializedDecodable
437434
{
438435
fn specialized_decode(&mut self) -> Result<&'_x ty::TyS<'_y>, Self::Error> {
439-
unsafe { transmute::<Result<ty::Ty<'tcx>, Self::Error>, Result<&'_x ty::TyS<'_y>, Self::Error>>(decode_ty(self)) }
436+
unsafe {
437+
transmute::<
438+
Result<ty::Ty<'tcx>, Self::Error>,
439+
Result<&'_x ty::TyS<'_y>, Self::Error>,
440+
>(decode_ty(self))
441+
}
442+
}
443+
}
444+
445+
impl<'_x, $($typaram),*> SpecializedDecoder<ty::Predicate<'_x>>
446+
for $DecoderName<$($typaram),*> {
447+
fn specialized_decode(&mut self) -> Result<ty::Predicate<'_x>, Self::Error> {
448+
unsafe {
449+
transmute::<
450+
Result<ty::Predicate<'tcx>, Self::Error>,
451+
Result<ty::Predicate<'_x>, Self::Error>,
452+
>(decode_predicate(self))
453+
}
440454
}
441455
}
442456

‎src/librustc_middle/ty/context.rs

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ use crate::ty::TyKind::*;
1919
use crate::ty::{
2020
self, query, AdtDef, AdtKind, BindingMode, BoundVar, CanonicalPolyFnSig, Const, ConstVid,
2121
DefIdTree, ExistentialPredicate, FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy,
22-
IntVar, IntVid, List, ParamConst, ParamTy, PolyFnSig, Predicate, PredicateKind, ProjectionTy,
23-
Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, TyVid, TypeAndMut,
22+
IntVar, IntVid, List, ParamConst, ParamTy, PolyFnSig, Predicate, PredicateInner, PredicateKind,
23+
ProjectionTy, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar,
24+
TyVid, TypeAndMut,
2425
};
2526
use rustc_ast::ast;
2627
use rustc_ast::expand::allocator::AllocatorKind;
@@ -76,7 +77,7 @@ pub struct CtxtInterners<'tcx> {
7677
canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo>>,
7778
region: InternedSet<'tcx, RegionKind>,
7879
existential_predicates: InternedSet<'tcx, List<ExistentialPredicate<'tcx>>>,
79-
predicate_kind: InternedSet<'tcx, PredicateKind<'tcx>>,
80+
predicate: InternedSet<'tcx, PredicateInner<'tcx>>,
8081
predicates: InternedSet<'tcx, List<Predicate<'tcx>>>,
8182
projs: InternedSet<'tcx, List<ProjectionKind>>,
8283
place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
@@ -95,7 +96,7 @@ impl<'tcx> CtxtInterners<'tcx> {
9596
region: Default::default(),
9697
existential_predicates: Default::default(),
9798
canonical_var_infos: Default::default(),
98-
predicate_kind: Default::default(),
99+
predicate: Default::default(),
99100
predicates: Default::default(),
100101
projs: Default::default(),
101102
place_elems: Default::default(),
@@ -123,6 +124,23 @@ impl<'tcx> CtxtInterners<'tcx> {
123124
})
124125
.0
125126
}
127+
128+
#[inline(never)]
129+
fn intern_predicate(&self, kind: PredicateKind<'tcx>) -> &'tcx PredicateInner<'tcx> {
130+
self.predicate
131+
.intern(kind, |kind| {
132+
let flags = super::flags::FlagComputation::for_predicate(&kind);
133+
134+
let predicate_struct = PredicateInner {
135+
kind,
136+
flags: flags.flags,
137+
outer_exclusive_binder: flags.outer_exclusive_binder,
138+
};
139+
140+
Interned(self.arena.alloc(predicate_struct))
141+
})
142+
.0
143+
}
126144
}
127145

128146
pub struct CommonTypes<'tcx> {
@@ -938,8 +956,9 @@ pub struct GlobalCtxt<'tcx> {
938956
/// via `extern crate` item and not `--extern` option or compiler built-in.
939957
pub extern_prelude: FxHashMap<Symbol, bool>,
940958

941-
// Internal cache for metadata decoding. No need to track deps on this.
942-
pub rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
959+
// Internal caches for metadata decoding. No need to track deps on this.
960+
pub ty_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
961+
pub pred_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Predicate<'tcx>>>,
943962

944963
/// Caches the results of trait selection. This cache is used
945964
/// for things that do not have to do with the parameters in scope.
@@ -1128,7 +1147,8 @@ impl<'tcx> TyCtxt<'tcx> {
11281147
definitions,
11291148
def_path_hash_to_def_id,
11301149
queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache),
1131-
rcache: Default::default(),
1150+
ty_rcache: Default::default(),
1151+
pred_rcache: Default::default(),
11321152
selection_cache: Default::default(),
11331153
evaluation_cache: Default::default(),
11341154
crate_name: Symbol::intern(crate_name),
@@ -1625,7 +1645,7 @@ macro_rules! nop_list_lift {
16251645
nop_lift! {type_; Ty<'a> => Ty<'tcx>}
16261646
nop_lift! {region; Region<'a> => Region<'tcx>}
16271647
nop_lift! {const_; &'a Const<'a> => &'tcx Const<'tcx>}
1628-
nop_lift! {predicate_kind; &'a PredicateKind<'a> => &'tcx PredicateKind<'tcx>}
1648+
nop_lift! {predicate; &'a PredicateInner<'a> => &'tcx PredicateInner<'tcx>}
16291649

16301650
nop_list_lift! {type_list; Ty<'a> => Ty<'tcx>}
16311651
nop_list_lift! {existential_predicates; ExistentialPredicate<'a> => ExistentialPredicate<'tcx>}
@@ -1984,6 +2004,26 @@ impl<'tcx> Borrow<TyKind<'tcx>> for Interned<'tcx, TyS<'tcx>> {
19842004
&self.0.kind
19852005
}
19862006
}
2007+
// N.B., an `Interned<PredicateInner>` compares and hashes as a `PredicateKind`.
2008+
impl<'tcx> PartialEq for Interned<'tcx, PredicateInner<'tcx>> {
2009+
fn eq(&self, other: &Interned<'tcx, PredicateInner<'tcx>>) -> bool {
2010+
self.0.kind == other.0.kind
2011+
}
2012+
}
2013+
2014+
impl<'tcx> Eq for Interned<'tcx, PredicateInner<'tcx>> {}
2015+
2016+
impl<'tcx> Hash for Interned<'tcx, PredicateInner<'tcx>> {
2017+
fn hash<H: Hasher>(&self, s: &mut H) {
2018+
self.0.kind.hash(s)
2019+
}
2020+
}
2021+
2022+
impl<'tcx> Borrow<PredicateKind<'tcx>> for Interned<'tcx, PredicateInner<'tcx>> {
2023+
fn borrow<'a>(&'a self) -> &'a PredicateKind<'tcx> {
2024+
&self.0.kind
2025+
}
2026+
}
19872027

19882028
// N.B., an `Interned<List<T>>` compares and hashes as its elements.
19892029
impl<'tcx, T: PartialEq> PartialEq for Interned<'tcx, List<T>> {
@@ -2050,11 +2090,10 @@ macro_rules! direct_interners {
20502090
}
20512091
}
20522092

2053-
direct_interners!(
2093+
direct_interners! {
20542094
region: mk_region(RegionKind),
20552095
const_: mk_const(Const<'tcx>),
2056-
predicate_kind: intern_predicate_kind(PredicateKind<'tcx>),
2057-
);
2096+
}
20582097

20592098
macro_rules! slice_interners {
20602099
($($field:ident: $method:ident($ty:ty)),+) => (
@@ -2125,8 +2164,8 @@ impl<'tcx> TyCtxt<'tcx> {
21252164

21262165
#[inline]
21272166
pub fn mk_predicate(&self, kind: PredicateKind<'tcx>) -> Predicate<'tcx> {
2128-
let kind = self.intern_predicate_kind(kind);
2129-
Predicate { kind }
2167+
let inner = self.interners.intern_predicate(kind);
2168+
Predicate { inner }
21302169
}
21312170

21322171
pub fn mk_mach_int(self, tm: ast::IntTy) -> Ty<'tcx> {

0 commit comments

Comments
 (0)