Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 174810c

Browse files
committed
Auto merge of rust-lang#131650 - saethlin:post-mono-mir-opts, r=<try>
Add post-mono MIR passes to make mono-reachable analysis more accurate As of rust-lang#131650 (comment) I believe most of the incr overhead comes from re-computing, re-encoding, and loading a lot more MIR when all we're actually doing is traversing through it. I think that can be addressed by caching a query that looks up the mentioned/used items for an Instance. I think the full-build regressions are pretty much just the expense of cloning, then monomorphizing, then caching the MIR.
2 parents b8bb296 + 4ae3542 commit 174810c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+302
-386
lines changed

compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ pub(crate) fn codegen_fn<'tcx>(
4141
let symbol_name = tcx.symbol_name(instance).name.to_string();
4242
let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
4343

44-
let mir = tcx.instance_mir(instance.def);
44+
let mir = tcx.build_codegen_mir(instance);
4545
let _mir_guard = crate::PrintOnPanic(|| {
4646
let mut buf = Vec::new();
4747
with_no_trimmed_paths!({

compiler/rustc_codegen_ssa/src/base.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
416416
// release builds.
417417
info!("codegen_instance({})", instance);
418418

419-
mir::codegen_mir::<Bx>(cx, instance);
419+
mir::lower_mir::<Bx>(cx, instance);
420420
}
421421

422422
/// Creates the `main` function which will initialize the rust runtime and call

compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,15 +159,15 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
159159
///////////////////////////////////////////////////////////////////////////
160160

161161
#[instrument(level = "debug", skip(cx))]
162-
pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
162+
pub fn lower_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
163163
cx: &'a Bx::CodegenCx,
164164
instance: Instance<'tcx>,
165165
) {
166166
assert!(!instance.args.has_infer());
167167

168168
let llfn = cx.get_fn(instance);
169169

170-
let mir = cx.tcx().instance_mir(instance.def);
170+
let mir = cx.tcx().build_codegen_mir(instance);
171171

172172
let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
173173
debug!("fn_abi: {:?}", fn_abi);

compiler/rustc_middle/src/mir/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ impl RuntimePhase {
159159
"initial" => Self::Initial,
160160
"post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
161161
"optimized" => Self::Optimized,
162+
"codegen" => Self::Codegen,
162163
_ => bug!("Unknown runtime phase: '{}'", phase),
163164
}
164165
}

compiler/rustc_middle/src/mir/mono.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ impl<'tcx> MonoItem<'tcx> {
7373
InstanceKind::Item(..)
7474
| InstanceKind::DropGlue(..)
7575
| InstanceKind::AsyncDropGlueCtorShim(..) => {
76-
let mir = tcx.instance_mir(instance.def);
76+
let mir = tcx.build_codegen_mir(instance);
7777
mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum()
7878
}
7979
// Other compiler-generated shims size estimate: 1

compiler/rustc_middle/src/mir/syntax.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ impl MirPhase {
9999
MirPhase::Runtime(RuntimePhase::Initial) => "runtime",
100100
MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup",
101101
MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized",
102+
MirPhase::Runtime(RuntimePhase::Codegen) => "codegen",
102103
}
103104
}
104105

@@ -153,6 +154,7 @@ pub enum RuntimePhase {
153154
/// * [`ProjectionElem::Deref`] of `Box`
154155
PostCleanup = 1,
155156
Optimized = 2,
157+
Codegen = 3,
156158
}
157159

158160
///////////////////////////////////////////////////////////////////////////

compiler/rustc_middle/src/query/mod.rs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -555,14 +555,22 @@ rustc_queries! {
555555
desc { |tcx| "verify auto trait bounds for coroutine interior type `{}`", tcx.def_path_str(key) }
556556
}
557557

558-
/// MIR after our optimization passes have run. This is MIR that is ready
559-
/// for codegen. This is also the only query that can fetch non-local MIR, at present.
558+
/// Polymorphic MIR after our pre-mono optimization passes have run. This is the MIR that
559+
/// crates export.
560560
query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
561561
desc { |tcx| "optimizing MIR for `{}`", tcx.def_path_str(key) }
562562
cache_on_disk_if { key.is_local() }
563563
separate_provide_extern
564564
}
565565

566+
/// MIR for a specific Instance ready for codegen. This is `optimized_mir` but monomorphized
567+
/// and with extra transforms applied.
568+
query build_codegen_mir(key: ty::Instance<'tcx>) -> &'tcx mir::Body<'tcx> {
569+
desc { |tcx| "finalizing codegen MIR for `{}`", tcx.def_path_str_with_args(key.def_id(), key.args) }
570+
cache_on_disk_if { true }
571+
arena_cache
572+
}
573+
566574
/// Checks for the nearest `#[coverage(off)]` or `#[coverage(on)]` on
567575
/// this def and any enclosing defs, up to the crate root.
568576
///

compiler/rustc_middle/src/ty/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1718,6 +1718,14 @@ impl<'tcx> TyCtxt<'tcx> {
17181718
}
17191719
}
17201720

1721+
pub fn codegen_mir(self, instance: ty::Instance<'tcx>) -> &'tcx Body<'tcx> {
1722+
if self.sess.opts.incremental == None {
1723+
self.build_codegen_mir(instance)
1724+
} else {
1725+
self.instance_mir(instance.def)
1726+
}
1727+
}
1728+
17211729
/// Returns the possibly-auto-generated MIR of a [`ty::InstanceKind`].
17221730
#[instrument(skip(self), level = "debug")]
17231731
pub fn instance_mir(self, instance: ty::InstanceKind<'tcx>) -> &'tcx Body<'tcx> {

compiler/rustc_mir_transform/src/gvn.rs

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
use std::borrow::Cow;
8686

8787
use either::Either;
88+
use rustc_ast::attr;
8889
use rustc_const_eval::const_eval::DummyMachine;
8990
use rustc_const_eval::interpret::{
9091
ImmTy, Immediate, InterpCx, MemPlaceMeta, MemoryKind, OpTy, Projectable, Scalar,
@@ -101,17 +102,27 @@ use rustc_middle::mir::visit::*;
101102
use rustc_middle::mir::*;
102103
use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
103104
use rustc_middle::ty::{self, Ty, TyCtxt};
104-
use rustc_span::DUMMY_SP;
105105
use rustc_span::def_id::DefId;
106+
use rustc_span::{DUMMY_SP, sym};
106107
use rustc_target::abi::{self, Abi, FIRST_VARIANT, FieldIdx, Primitive, Size, VariantIdx};
107108
use smallvec::SmallVec;
108109
use tracing::{debug, instrument, trace};
109110

110111
use crate::ssa::{AssignedValue, SsaLocals};
111112

112-
pub(super) struct GVN;
113+
pub(super) enum GVN {
114+
Polymorphic,
115+
PostMono,
116+
}
113117

114118
impl<'tcx> crate::MirPass<'tcx> for GVN {
119+
fn name(&self) -> &'static str {
120+
match self {
121+
GVN::Polymorphic => "GVN",
122+
GVN::PostMono => "GVN-post-mono",
123+
}
124+
}
125+
115126
fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
116127
sess.mir_opt_level() >= 2
117128
}
@@ -125,7 +136,22 @@ impl<'tcx> crate::MirPass<'tcx> for GVN {
125136
// Clone dominators because we need them while mutating the body.
126137
let dominators = body.basic_blocks.dominators().clone();
127138

128-
let mut state = VnState::new(tcx, body, param_env, &ssa, dominators, &body.local_decls);
139+
let preserve_ub_checks = match self {
140+
GVN::Polymorphic => {
141+
attr::contains_name(tcx.hir().krate_attrs(), sym::rustc_preserve_ub_checks)
142+
}
143+
GVN::PostMono => false,
144+
};
145+
146+
let mut state = VnState::new(
147+
tcx,
148+
body,
149+
param_env,
150+
&ssa,
151+
dominators,
152+
&body.local_decls,
153+
preserve_ub_checks,
154+
);
129155
ssa.for_each_assignment_mut(
130156
body.basic_blocks.as_mut_preserves_cfg(),
131157
|local, value, location| {
@@ -260,6 +286,7 @@ struct VnState<'body, 'tcx> {
260286
ssa: &'body SsaLocals,
261287
dominators: Dominators<BasicBlock>,
262288
reused_locals: BitSet<Local>,
289+
preserve_ub_checks: bool,
263290
}
264291

265292
impl<'body, 'tcx> VnState<'body, 'tcx> {
@@ -270,6 +297,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
270297
ssa: &'body SsaLocals,
271298
dominators: Dominators<BasicBlock>,
272299
local_decls: &'body LocalDecls<'tcx>,
300+
preserve_ub_checks: bool,
273301
) -> Self {
274302
// Compute a rough estimate of the number of values in the body from the number of
275303
// statements. This is meant to reduce the number of allocations, but it's all right if
@@ -292,6 +320,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
292320
ssa,
293321
dominators,
294322
reused_locals: BitSet::new_empty(local_decls.len()),
323+
preserve_ub_checks,
295324
}
296325
}
297326

@@ -530,7 +559,14 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
530559
.tcx
531560
.offset_of_subfield(self.ecx.param_env(), layout, fields.iter())
532561
.bytes(),
533-
NullOp::UbChecks => return None,
562+
NullOp::UbChecks => {
563+
if self.preserve_ub_checks {
564+
return None;
565+
} else {
566+
let val = ImmTy::from_bool(self.tcx.sess.ub_checks(), self.tcx);
567+
return Some(val.into());
568+
}
569+
}
534570
};
535571
let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
536572
let imm = ImmTy::from_uint(val, usize_layout);

compiler/rustc_mir_transform/src/instsimplify.rs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,15 @@ use crate::take_array;
1616
pub(super) enum InstSimplify {
1717
BeforeInline,
1818
AfterSimplifyCfg,
19+
PostMono,
1920
}
2021

2122
impl<'tcx> crate::MirPass<'tcx> for InstSimplify {
2223
fn name(&self) -> &'static str {
2324
match self {
2425
InstSimplify::BeforeInline => "InstSimplify-before-inline",
2526
InstSimplify::AfterSimplifyCfg => "InstSimplify-after-simplifycfg",
27+
InstSimplify::PostMono => "InstSimplify-post-mono",
2628
}
2729
}
2830

@@ -51,6 +53,29 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify {
5153
ctx.simplify_ptr_aggregate(&statement.source_info, rvalue);
5254
ctx.simplify_cast(rvalue);
5355
}
56+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
57+
// UnreachablePropagation likes to generate this MIR:
58+
//
59+
// _1 = UbChecks();
60+
// assume(copy _1);
61+
// _2 = unreachable_unchecked::precondition_check() -> [return: bb2, unwind unreachable];
62+
//
63+
// Which is mind-bending but correct. When UbChecks is false, we
64+
// assume(false) which is unreachable, and we never hit the precondition
65+
// check. When UbChecks is true, we assume(true) and fall through to the
66+
// precondition check.
67+
//
68+
// So the branch on UbChecks is implicit, which is both clever and makes
69+
// the rest of MIR optimizations unable to delete this precondition check
70+
// call when UB checks are off.
71+
if let Some(ConstOperand { const_, .. }) = op.constant() {
72+
if let Some(false) = const_.try_to_bool() {
73+
block.statements.clear();
74+
block.terminator_mut().kind = TerminatorKind::Unreachable;
75+
break;
76+
}
77+
}
78+
}
5479
_ => {}
5580
}
5681
}

0 commit comments

Comments
 (0)