Skip to content

Commit 216f62c

Browse files
ZoxcZoxc
committed
Run linking and incremental saving / finalizing in parallel
1 parent 6db96de commit 216f62c

File tree

13 files changed

+250
-80
lines changed

13 files changed

+250
-80
lines changed

‎compiler/rustc_codegen_cranelift/src/lib.rs‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,16 @@ extern crate rustc_target;
3030
extern crate rustc_driver;
3131

3232
use std::any::Any;
33-
use std::cell::{Cell, RefCell};
33+
use std::cell::Cell;
3434
use std::sync::Arc;
35+
use std::sync::RwLock;
3536

3637
use cranelift_codegen::isa::TargetIsa;
3738
use cranelift_codegen::settings::{self, Configurable};
3839
use rustc_codegen_ssa::traits::CodegenBackend;
3940
use rustc_codegen_ssa::CodegenResults;
4041
use rustc_data_structures::profiling::SelfProfilerRef;
42+
use rustc_data_structures::sync::{downcast_box_any_dyn_send, DynSend};
4143
use rustc_errors::ErrorGuaranteed;
4244
use rustc_metadata::EncodedMetadata;
4345
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@@ -165,7 +167,7 @@ impl CodegenCx {
165167
}
166168

167169
pub struct CraneliftCodegenBackend {
168-
pub config: RefCell<Option<BackendConfig>>,
170+
pub config: RwLock<Option<BackendConfig>>,
169171
}
170172

171173
impl CodegenBackend for CraneliftCodegenBackend {
@@ -183,7 +185,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
183185
}
184186
}
185187

186-
let mut config = self.config.borrow_mut();
188+
let mut config = self.config.write().unwrap();
187189
if config.is_none() {
188190
let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
189191
.unwrap_or_else(|err| sess.dcx().fatal(err));
@@ -213,9 +215,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
213215
tcx: TyCtxt<'_>,
214216
metadata: EncodedMetadata,
215217
need_metadata_module: bool,
216-
) -> Box<dyn Any> {
218+
) -> Box<dyn Any + DynSend> {
217219
tcx.dcx().abort_if_errors();
218-
let config = self.config.borrow().clone().unwrap();
220+
let config = self.config.read().unwrap().clone().unwrap();
219221
match config.codegen_mode {
220222
CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
221223
CodegenMode::Jit | CodegenMode::JitLazy => {
@@ -230,14 +232,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
230232

231233
fn join_codegen(
232234
&self,
233-
ongoing_codegen: Box<dyn Any>,
235+
ongoing_codegen: Box<dyn Any + DynSend>,
234236
sess: &Session,
235237
_outputs: &OutputFilenames,
236238
) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
237-
ongoing_codegen
238-
.downcast::<driver::aot::OngoingCodegen>()
239+
downcast_box_any_dyn_send::<driver::aot::OngoingCodegen>(ongoing_codegen)
239240
.unwrap()
240-
.join(sess, self.config.borrow().as_ref().unwrap())
241+
.join(sess, self.config.read().unwrap().as_ref().unwrap())
241242
}
242243

243244
fn link(
@@ -350,5 +351,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar
350351
/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
351352
#[no_mangle]
352353
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
353-
Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
354+
Box::new(CraneliftCodegenBackend { config: RwLock::new(None) })
354355
}

‎compiler/rustc_codegen_gcc/src/lib.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ use rustc_codegen_ssa::base::codegen_crate;
9696
use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn};
9797
use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
9898
use rustc_data_structures::fx::FxIndexMap;
99-
use rustc_data_structures::sync::IntoDynSyncSend;
99+
use rustc_data_structures::sync::{downcast_box_any_dyn_send, DynSend, IntoDynSyncSend};
100100
use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ThinBufferMethods, WriteBackendMethods};
101101
use rustc_errors::{ErrorGuaranteed, DiagCtxt};
102102
use rustc_metadata::EncodedMetadata;
@@ -210,16 +210,15 @@ impl CodegenBackend for GccCodegenBackend {
210210
|tcx, ()| gcc_util::global_gcc_features(tcx.sess, true)
211211
}
212212

213-
fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>, metadata: EncodedMetadata, need_metadata_module: bool) -> Box<dyn Any> {
213+
fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>, metadata: EncodedMetadata, need_metadata_module: bool) -> Box<dyn Any + DynSend> {
214214
let target_cpu = target_cpu(tcx.sess);
215215
let res = codegen_crate(self.clone(), tcx, target_cpu.to_string(), metadata, need_metadata_module);
216216

217217
Box::new(res)
218218
}
219219

220-
fn join_codegen(&self, ongoing_codegen: Box<dyn Any>, sess: &Session, _outputs: &OutputFilenames) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
221-
ongoing_codegen
222-
.downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<GccCodegenBackend>>()
220+
fn join_codegen(&self, ongoing_codegen: Box<dyn Any + DynSend>, sess: &Session, _outputs: &OutputFilenames) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
221+
downcast_box_any_dyn_send::<rustc_codegen_ssa::back::write::OngoingCodegen<GccCodegenBackend>>( ongoing_codegen)
223222
.expect("Expected GccCodegenBackend's OngoingCodegen, found Box<Any>")
224223
.join(sess)
225224
}

‎compiler/rustc_codegen_llvm/src/lib.rs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use rustc_codegen_ssa::traits::*;
3434
use rustc_codegen_ssa::ModuleCodegen;
3535
use rustc_codegen_ssa::{CodegenResults, CompiledModule};
3636
use rustc_data_structures::fx::FxIndexMap;
37+
use rustc_data_structures::sync::{downcast_box_any_dyn_send, DynSend};
3738
use rustc_errors::{DiagCtxt, ErrorGuaranteed, FatalError};
3839
use rustc_metadata::EncodedMetadata;
3940
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@@ -250,9 +251,6 @@ impl WriteBackendMethods for LlvmCodegenBackend {
250251
}
251252
}
252253

253-
unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis
254-
unsafe impl Sync for LlvmCodegenBackend {}
255-
256254
impl LlvmCodegenBackend {
257255
pub fn new() -> Box<dyn CodegenBackend> {
258256
Box::new(LlvmCodegenBackend(()))
@@ -354,7 +352,7 @@ impl CodegenBackend for LlvmCodegenBackend {
354352
tcx: TyCtxt<'tcx>,
355353
metadata: EncodedMetadata,
356354
need_metadata_module: bool,
357-
) -> Box<dyn Any> {
355+
) -> Box<dyn Any + DynSend> {
358356
Box::new(rustc_codegen_ssa::base::codegen_crate(
359357
LlvmCodegenBackend(()),
360358
tcx,
@@ -366,14 +364,15 @@ impl CodegenBackend for LlvmCodegenBackend {
366364

367365
fn join_codegen(
368366
&self,
369-
ongoing_codegen: Box<dyn Any>,
367+
ongoing_codegen: Box<dyn Any + DynSend>,
370368
sess: &Session,
371369
outputs: &OutputFilenames,
372370
) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
373-
let (codegen_results, work_products) = ongoing_codegen
374-
.downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>()
375-
.expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>")
376-
.join(sess);
371+
let (codegen_results, work_products) = downcast_box_any_dyn_send::<
372+
rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>,
373+
>(ongoing_codegen)
374+
.expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<dyn Any + DynSend>")
375+
.join(sess);
377376

378377
if sess.opts.unstable_opts.llvm_time_trace {
379378
sess.time("llvm_dump_timing_file", || {

‎compiler/rustc_codegen_ssa/src/traits/backend.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ impl<'tcx, T> Backend<'tcx> for T where
5757
{
5858
}
5959

60-
pub trait CodegenBackend {
60+
pub trait CodegenBackend: DynSync + DynSend {
6161
/// Locale resources for diagnostic messages - a string the content of the Fluent resource.
6262
/// Called before `init` so that all other functions are able to emit translatable diagnostics.
6363
fn locale_resource(&self) -> &'static str;
@@ -90,16 +90,16 @@ pub trait CodegenBackend {
9090
tcx: TyCtxt<'tcx>,
9191
metadata: EncodedMetadata,
9292
need_metadata_module: bool,
93-
) -> Box<dyn Any>;
93+
) -> Box<dyn Any + DynSend>;
9494

95-
/// This is called on the returned `Box<dyn Any>` from `codegen_backend`
95+
/// This is called on the returned `Box<dyn Any + DynSend>` from `codegen_backend`
9696
///
9797
/// # Panics
9898
///
99-
/// Panics when the passed `Box<dyn Any>` was not returned by `codegen_backend`.
99+
/// Panics when the passed `Box<dyn Any + DynSend>` was not returned by `codegen_backend`.
100100
fn join_codegen(
101101
&self,
102-
ongoing_codegen: Box<dyn Any>,
102+
ongoing_codegen: Box<dyn Any + DynSend>,
103103
sess: &Session,
104104
outputs: &OutputFilenames,
105105
) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>);

‎compiler/rustc_data_structures/src/marker.rs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::any::Any;
2+
13
cfg_match! {
24
cfg(not(parallel_compiler)) => {
35
pub auto trait DynSend {}
@@ -67,6 +69,11 @@ cfg_match! {
6769
[std::io::Stderr]
6870
[std::io::Error]
6971
[std::fs::File]
72+
[std::sync::Condvar]
73+
[jobserver_crate::Client]
74+
[jobserver_crate::HelperThread]
75+
[jobserver_crate::Acquired]
76+
[Box<dyn Any + Send>]
7077
[rustc_arena::DroplessArena]
7178
[crate::memmap::Mmap]
7279
[crate::profiling::SelfProfiler]
@@ -81,10 +88,13 @@ cfg_match! {
8188

8289
impl_dyn_send!(
8390
[std::sync::atomic::AtomicPtr<T> where T]
84-
[std::sync::Mutex<T> where T: ?Sized+ DynSend]
91+
[std::sync::Mutex<T> where T: ?Sized + DynSend]
92+
[std::sync::RwLock<T> where T: ?Sized + DynSend]
8593
[std::sync::mpsc::Sender<T> where T: DynSend]
94+
[std::sync::mpsc::Receiver<T> where T: DynSend]
8695
[std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
8796
[std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
97+
[std::thread::JoinHandle<T> where T]
8898
[std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
8999
[std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
90100
[std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
@@ -139,6 +149,7 @@ cfg_match! {
139149
[std::sync::atomic::AtomicU8]
140150
[std::sync::atomic::AtomicU32]
141151
[std::backtrace::Backtrace]
152+
[std::sync::Condvar]
142153
[std::io::Error]
143154
[std::fs::File]
144155
[jobserver_crate::Client]
@@ -170,6 +181,7 @@ cfg_match! {
170181
[std::sync::OnceLock<T> where T: DynSend + DynSync]
171182
[std::sync::Mutex<T> where T: ?Sized + DynSend]
172183
[std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
184+
[std::sync::RwLock<T> where T: ?Sized + DynSend + DynSync]
173185
[std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
174186
[std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
175187
[std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
@@ -258,3 +270,15 @@ impl<T> std::ops::DerefMut for IntoDynSyncSend<T> {
258270
&mut self.0
259271
}
260272
}
273+
274+
#[inline]
275+
pub fn downcast_box_any_dyn_send<T: Any>(this: Box<dyn Any + DynSend>) -> Result<Box<T>, ()> {
276+
if <dyn Any>::is::<T>(&*this) {
277+
unsafe {
278+
let (raw, alloc): (*mut (dyn Any + DynSend), _) = Box::into_raw_with_allocator(this);
279+
Ok(Box::from_raw_in(raw as *mut T, alloc))
280+
}
281+
} else {
282+
Err(())
283+
}
284+
}

‎compiler/rustc_data_structures/src/sync.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ pub use crate::marker::*;
4444
use std::collections::HashMap;
4545
use std::hash::{BuildHasher, Hash};
4646

47+
mod task;
48+
pub use task::{task, Task};
49+
4750
mod lock;
4851
pub use lock::{Lock, LockGuard, Mode};
4952

‎compiler/rustc_data_structures/src/sync/task.rs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
use crate::jobserver;
2+
use crate::sync::{mode, DynSend, FromDyn, IntoDynSyncSend};
3+
use parking_lot::Condvar;
4+
use parking_lot::Mutex;
5+
use std::any::Any;
6+
use std::mem;
7+
use std::panic::{self, AssertUnwindSafe};
8+
use std::sync::Arc;
9+
10+
enum TaskState<T: DynSend + 'static> {
11+
Unexecuted(Box<dyn FnOnce() -> T + DynSend>),
12+
Running,
13+
Joined,
14+
Result(Result<T, IntoDynSyncSend<Box<dyn Any + Send + 'static>>>),
15+
}
16+
17+
struct TaskData<T: DynSend + 'static> {
18+
state: Mutex<TaskState<T>>,
19+
waiter: Condvar,
20+
}
21+
22+
#[must_use]
23+
pub struct Task<T: DynSend + 'static> {
24+
data: Arc<TaskData<T>>,
25+
}
26+
27+
/// This attempts to run a closure in a background thread. It returns a `Task` type which
28+
/// you must call `join` on to ensure that the closure runs.
29+
pub fn task<T: DynSend>(f: impl FnOnce() -> T + DynSend + 'static) -> Task<T> {
30+
let task = Task {
31+
data: Arc::new(TaskData {
32+
state: Mutex::new(TaskState::Unexecuted(Box::new(f))),
33+
waiter: Condvar::new(),
34+
}),
35+
};
36+
37+
#[cfg(parallel_compiler)]
38+
if mode::is_dyn_thread_safe() {
39+
let data = FromDyn::from(task.data.clone());
40+
41+
// Try to execute the task on a separate thread.
42+
rayon::spawn(move || {
43+
let data = data.into_inner();
44+
let mut state = data.state.lock();
45+
if matches!(*state, TaskState::Unexecuted(..)) {
46+
if let TaskState::Unexecuted(f) = mem::replace(&mut *state, TaskState::Running) {
47+
drop(state);
48+
let result = panic::catch_unwind(AssertUnwindSafe(f));
49+
50+
let unblock = {
51+
let mut state = data.state.lock();
52+
let unblock = matches!(*state, TaskState::Joined);
53+
*state = TaskState::Result(result.map_err(|e| IntoDynSyncSend(e)));
54+
unblock
55+
};
56+
57+
if unblock {
58+
rayon_core::mark_unblocked(&rayon_core::Registry::current());
59+
}
60+
61+
data.waiter.notify_one();
62+
}
63+
}
64+
});
65+
}
66+
67+
task
68+
}
69+
70+
#[inline]
71+
fn unwind<T>(result: Result<T, IntoDynSyncSend<Box<dyn Any + Send + 'static>>>) -> T {
72+
match result {
73+
Ok(r) => r,
74+
Err(err) => panic::resume_unwind(err.0),
75+
}
76+
}
77+
78+
impl<T: DynSend> Task<T> {
79+
pub fn join(self) -> T {
80+
let mut state_guard = self.data.state.lock();
81+
82+
match mem::replace(&mut *state_guard, TaskState::Joined) {
83+
TaskState::Unexecuted(f) => f(),
84+
TaskState::Result(result) => unwind(result),
85+
TaskState::Running => {
86+
#[cfg(parallel_compiler)]
87+
rayon_core::mark_blocked();
88+
jobserver::release_thread();
89+
90+
self.data.waiter.wait(&mut state_guard);
91+
92+
jobserver::acquire_thread();
93+
94+
match mem::replace(&mut *state_guard, TaskState::Joined) {
95+
TaskState::Result(result) => unwind(result),
96+
_ => panic!(),
97+
}
98+
}
99+
TaskState::Joined => panic!(),
100+
}
101+
}
102+
}

‎compiler/rustc_driver_impl/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -461,8 +461,8 @@ fn run_compiler(
461461
// Linking is done outside the `compiler.enter()` so that the
462462
// `GlobalCtxt` within `Queries` can be freed as early as possible.
463463
if let Some(linker) = linker {
464-
let _timer = sess.timer("link");
465-
linker.link(sess, codegen_backend)?
464+
let _timer = sess.timer("waiting for linking");
465+
linker.join()?;
466466
}
467467

468468
if sess.opts.unstable_opts.print_fuel.is_some() {

‎compiler/rustc_interface/src/interface.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ pub type Result<T> = result::Result<T, ErrorGuaranteed>;
3636
/// Can be used to run `rustc_interface` queries.
3737
/// Created by passing [`Config`] to [`run_compiler`].
3838
pub struct Compiler {
39-
pub sess: Session,
40-
pub codegen_backend: Box<dyn CodegenBackend>,
39+
pub sess: Lrc<Session>,
40+
pub codegen_backend: Lrc<dyn CodegenBackend>,
4141
pub(crate) override_queries: Option<fn(&Session, &mut Providers)>,
4242
}
4343

@@ -419,8 +419,11 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
419419
}
420420
sess.lint_store = Some(Lrc::new(lint_store));
421421

422-
let compiler =
423-
Compiler { sess, codegen_backend, override_queries: config.override_queries };
422+
let compiler = Compiler {
423+
sess: Lrc::new(sess),
424+
codegen_backend: Lrc::from(codegen_backend),
425+
override_queries: config.override_queries,
426+
};
424427

425428
rustc_span::set_source_map(compiler.sess.parse_sess.clone_source_map(), move || {
426429
// There are two paths out of `f`.

0 commit comments

Comments
 (0)