Rollup merge of rust-lang#64648 - alexreg:rush-parsing, r=Mark-Simulacrum

REPL, part 1: Added interpreter mode to compiler interface, interpreter parsing functionality

Summary:
* Adds "interpreter mode" to compiler interface. This will be used later in several places, including diagnostics.
* Adds "interpreter tag" to `ast::Local`s to track info like whether they came from a previous eval session or have been moved.
* Added interface for injecting a pre-parsed "user body" into the parsing pipeline. cf. `TyCtxt::get_interp_user_fn`, `expr.rs`
* Moved `Steal` data structure to `rustc_data_structures` crate since a) it was already used outside of `rustc::ty` crate, b) it's now used even a little more outside there.
* Made a few more things `pub` (as little as possible), so the interpreter can use them.

If you want the big picture of where this is going in terms of compiler changes (probably 2/3 more PRs needed), take a look at my [personal branch](https://github.com/alexreg/rust/tree/rush). I will also be publishing the REPL repo itself soon.

Also, sorry for the lack of commits; I basically just carved this out of an even bigger squashed commit after much, much hacking! (It might be a tad heavy on cosmetic stuff too, for the same reason. If it's okay, then great, otherwise I can try to revert certain areas that people really don't want.)

Maybe @Centril / @Zoxc for review? Not wholly sure.

CC @nikomatsakis @mw @eddyb

Author: Centril

src/librustc/arena.rs

@@ -23,9 +23,9 @@ macro_rules! arena_types {
             [] generics: rustc::ty::Generics,
             [] trait_def: rustc::ty::TraitDef,
             [] adt_def: rustc::ty::AdtDef,
-            [] steal_mir: rustc::ty::steal::Steal<rustc::mir::Body<$tcx>>,
+            [] steal_mir: rustc_data_structures::steal::Steal<rustc::mir::Body<$tcx>>,
             [] mir: rustc::mir::Body<$tcx>,
-            [] steal_promoted: rustc::ty::steal::Steal<
+            [] steal_promoted: rustc_data_structures::steal::Steal<
                 rustc_index::vec::IndexVec<
                     rustc::mir::Promoted,
                     rustc::mir::Body<$tcx>

src/librustc/hir/lowering.rs

@@ -2079,6 +2079,7 @@ impl<'a> LoweringContext<'a> {
             span: l.span,
             attrs: l.attrs.clone(),
             source: hir::LocalSource::Normal,
+            interp_tag: l.interp_tag.clone(),
         }, ids)
     }
 
@@ -3048,6 +3049,7 @@ impl<'a> LoweringContext<'a> {
             source,
             span,
             ty: None,
+            interp_tag: None,
         };
         self.stmt(span, hir::StmtKind::Local(P(local)))
     }

src/librustc/hir/mod.rs

@@ -1273,6 +1273,8 @@ pub struct Local {
     /// Can be `ForLoopDesugar` if the `let` statement is part of a `for` loop
     /// desugaring. Otherwise will be `Normal`.
     pub source: LocalSource,
+    /// See comment on `syntax::ast::Local`.
+    pub interp_tag: Option<ast::LocalInterpTag>,
 }
 
 /// Represents a single arm of a `match` expression, e.g.

src/librustc/ich/impls_hir.rs

@@ -392,3 +392,14 @@ impl<'hir> HashStable<StableHashingContext<'hir>> for attr::OptimizeAttr {
         mem::discriminant(self).hash_stable(hcx, hasher);
     }
 }
+
+impl_stable_hash_for!(enum ast::LocalInterpState {
+    Uninitialized,
+    Set,
+    Moved,
+});
+
+impl_stable_hash_for!(struct ast::LocalInterpTag {
+    id,
+    state,
+});

src/librustc/ich/impls_ty.rs

@@ -1,14 +1,15 @@
 //! This module contains `HashStable` implementations for various data types
-//! from rustc::ty in no particular order.
+//! from `rustc::ty` in no particular order.
 
 use crate::ich::{Fingerprint, StableHashingContext, NodeIdHashingMode};
+use crate::middle::region;
+use crate::mir;
+use crate::ty;
+
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, ToStableHashKey, StableHasher};
 use std::cell::RefCell;
 use std::mem;
-use crate::middle::region;
-use crate::ty;
-use crate::mir;
 
 impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for &'tcx ty::List<T>
 where
@@ -197,15 +198,6 @@ impl<'a> HashStable<StableHashingContext<'a>> for ty::FloatVid {
     }
 }
 
-impl<'a, T> HashStable<StableHashingContext<'a>> for ty::steal::Steal<T>
-where
-    T: HashStable<StableHashingContext<'a>>,
-{
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        self.borrow().hash_stable(hcx, hasher);
-    }
-}
-
 impl<'a> HashStable<StableHashingContext<'a>>
 for crate::middle::privacy::AccessLevels {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {

src/librustc/query/mod.rs

@@ -1120,8 +1120,7 @@ rustc_queries! {
         }
 
         // Get an estimate of the size of an InstanceDef based on its MIR for CGU partitioning.
-        query instance_def_size_estimate(def: ty::InstanceDef<'tcx>)
-            -> usize {
+        query instance_def_size_estimate(def: ty::InstanceDef<'tcx>) -> usize {
             no_force
             desc { |tcx| "estimating size for `{}`", tcx.def_path_str(def.def_id()) }
         }
@@ -1130,5 +1129,10 @@ rustc_queries! {
             eval_always
             desc { "looking up enabled feature gates" }
         }
+
+        query interp_user_fn(_: CrateNum) -> DefId {
+            eval_always
+            desc { "locating interpreter user fn in HIR" }
+        }
     }
 }

src/librustc/session/config.rs

@@ -391,6 +391,7 @@ top_level_options!(
         // can influence whether overflow checks are done or not.
         debug_assertions: bool [TRACKED],
         debuginfo: DebugInfo [TRACKED],
+        interp_mode: bool [TRACKED],
         lint_opts: Vec<(String, lint::Level)> [TRACKED],
         lint_cap: Option<lint::Level> [TRACKED],
         describe_lints: bool [UNTRACKED],
@@ -599,6 +600,7 @@ impl Default for Options {
             crate_types: Vec::new(),
             optimize: OptLevel::No,
             debuginfo: DebugInfo::None,
+            interp_mode: false,
             lint_opts: Vec::new(),
             lint_cap: None,
             describe_lints: false,
@@ -2467,6 +2469,7 @@ pub fn build_session_options_and_crate_config(
             crate_types,
             optimize: opt_level,
             debuginfo,
+            interp_mode: false,
             lint_opts,
             lint_cap,
             describe_lints,

src/librustc/ty/context.rs

@@ -38,7 +38,6 @@ use crate::ty::{InferConst, ParamConst};
 use crate::ty::GenericParamDefKind;
 use crate::ty::layout::{LayoutDetails, TargetDataLayout, VariantIdx};
 use crate::ty::query;
-use crate::ty::steal::Steal;
 use crate::ty::subst::{UserSubsts, GenericArgKind};
 use crate::ty::{BoundVar, BindingMode};
 use crate::ty::CanonicalPolyFnSig;
@@ -49,12 +48,13 @@ use crate::util::nodemap::{FxHashMap, FxHashSet};
 use errors::DiagnosticBuilder;
 use arena::SyncDroplessArena;
 use smallvec::SmallVec;
+use rustc_index::vec::{Idx, IndexVec};
 use rustc_data_structures::stable_hasher::{
     HashStable, StableHasher, StableVec, hash_stable_hashmap,
 };
-use rustc_index::vec::{Idx, IndexVec};
 use rustc_data_structures::sharded::ShardedHashMap;
 use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal};
+use rustc_data_structures::steal::Steal;
 use std::any::Any;
 use std::borrow::Borrow;
 use std::cmp::Ordering;
@@ -2894,6 +2894,44 @@ fn ptr_eq<T, U>(t: *const T, u: *const U) -> bool {
     t as *const () == u as *const ()
 }
 
+/// In interpreter mode, locates the `DefId` of the user fn (a closure marked by an attribute named
+/// `rustc_interp_user_fn`) by visiting the local crate's HIR.
+fn find_interp_user_fn(tcx: TyCtxt<'_>) -> DefId {
+    use hir::intravisit::{self, Visitor, NestedVisitorMap};
+
+    struct InterpUserFnVisitor<'tcx> {
+        tcx: TyCtxt<'tcx>,
+        def_id: Option<DefId>,
+    }
+
+    impl<'tcx> Visitor<'tcx> for InterpUserFnVisitor<'tcx> {
+        fn nested_visit_map<'this>(&'this mut self) -> NestedVisitorMap<'this, 'tcx> {
+            NestedVisitorMap::OnlyBodies(&self.tcx.hir())
+        }
+
+        fn visit_expr(&mut self, ex: &'tcx hir::Expr) {
+            if syntax::attr::contains_name(&ex.attrs, sym::rustc_interp_user_fn) {
+                self.def_id = Some(self.tcx.hir().local_def_id(ex.hir_id));
+                return;
+            }
+
+            intravisit::walk_expr(self, ex);
+        }
+    }
+
+    let mut visitor = InterpUserFnVisitor {
+        tcx,
+        def_id: None,
+    };
+    tcx.hir().krate().visit_all_item_likes(&mut visitor.as_deep_visitor());
+    visitor.def_id
+        .unwrap_or_else(|| tcx.sess.fatal(&format!(
+            "could not find interpreter user fn in HIR; it should be a closure expression \
+             marked with the `#[{}]` attribute",
+            sym::rustc_interp_user_fn
+        )))
+}
+
 pub fn provide(providers: &mut ty::query::Providers<'_>) {
     providers.in_scope_traits_map = |tcx, id| tcx.gcx.trait_map.get(&id);
     providers.module_exports = |tcx, id| tcx.gcx.export_map.get(&id).map(|v| &v[..]);
@@ -2971,4 +3009,8 @@ pub fn provide(providers: &mut ty::query::Providers<'_>) {
         assert_eq!(cnum, LOCAL_CRATE);
         attr::contains_name(tcx.hir().krate_attrs(), sym::compiler_builtins)
     };
+    providers.interp_user_fn = |tcx, cnum| {
+        assert_eq!(cnum, LOCAL_CRATE);
+        find_interp_user_fn(tcx)
+    };
 }

src/librustc/ty/mod.rs

@@ -105,7 +105,6 @@ pub mod outlives;
 pub mod print;
 pub mod query;
 pub mod relate;
-pub mod steal;
 pub mod subst;
 pub mod trait_def;
 pub mod walk;

src/librustc/ty/query/mod.rs

@@ -34,7 +34,7 @@ use crate::traits::query::outlives_bounds::OutlivesBound;
 use crate::traits::specialization_graph;
 use crate::traits::Clauses;
 use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, AdtSizedConstraint};
-use crate::ty::steal::Steal;
+use rustc_data_structures::steal::Steal;
 use crate::ty::util::NeedsDrop;
 use crate::ty::subst::SubstsRef;
 use crate::util::nodemap::{DefIdSet, DefIdMap, ItemLocalSet};

src/librustc_data_structures/lib.rs

@@ -94,6 +94,7 @@ pub use ena::unify;
 pub mod vec_linked_list;
 pub mod work_queue;
 pub mod fingerprint;
+pub mod steal;
 
 pub struct OnDrop<F: Fn()>(pub F);
 

src/librustc/ty/steal.rs renamed to src/librustc_data_structures/steal.rs

@@ -1,7 +1,8 @@
-use rustc_data_structures::sync::{RwLock, ReadGuard, MappedReadGuard};
+use crate::stable_hasher::{StableHasher, HashStable};
+use crate::sync::{RwLock, ReadGuard, MappedReadGuard};
 
 /// The `Steal` struct is intended to used as the value for a query.
-/// Specifically, we sometimes have queries (*cough* MIR *cough*)
+/// Specifically, we sometimes have queries (in particular, for MIR)
 /// where we create a large, complex value that we want to iteratively
 /// update (e.g., optimize). We could clone the value for each
 /// optimization, but that'd be expensive. And yet we don't just want
@@ -26,21 +27,28 @@ pub struct Steal<T> {
 
 impl<T> Steal<T> {
     pub fn new(value: T) -> Self {
-        Steal {
+        Self {
             value: RwLock::new(Some(value))
         }
     }
 
     pub fn borrow(&self) -> MappedReadGuard<'_, T> {
         ReadGuard::map(self.value.borrow(), |opt| match *opt {
-            None => bug!("attempted to read from stolen value"),
+            None => panic!("attempted to read from stolen value"),
             Some(ref v) => v
         })
     }
 
     pub fn steal(&self) -> T {
-        let value_ref = &mut *self.value.try_write().expect("stealing value which is locked");
+        let value_ref = &mut *self.value.try_write()
+            .expect("stealing value that is locked");
         let value = value_ref.take();
         value.expect("attempt to read from stolen value")
     }
 }
+
+impl<T: HashStable<CTX>, CTX> HashStable<CTX> for Steal<T> {
+    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
+        self.borrow().hash_stable(hcx, hasher);
+    }
+}

src/librustc_interface/interface.rs

@@ -9,13 +9,15 @@ use rustc::session::{DiagnosticOutput, Session};
 use rustc::util::common::ErrorReported;
 use rustc_codegen_utils::codegen_backend::CodegenBackend;
 use rustc_data_structures::OnDrop;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
-use rustc_data_structures::fx::{FxHashSet, FxHashMap};
+use rustc_data_structures::steal::Steal;
 use rustc_metadata::cstore::CStore;
 use std::path::PathBuf;
 use std::result;
 use std::sync::{Arc, Mutex};
 use syntax;
+use syntax::parse::parser::InterpUserFn;
 use syntax::source_map::{FileLoader, SourceMap};
 use syntax_pos::edition;
 
@@ -35,9 +37,19 @@ pub struct Compiler {
     pub(crate) queries: Queries,
     pub(crate) cstore: Lrc<CStore>,
     pub(crate) crate_name: Option<String>,
+    /// In interpreter mode, the user fn to use when parsing.
+    pub(crate) interp_user_fn: Option<Steal<InterpUserFn>>,
 }
 
 impl Compiler {
+    pub fn set_interp_user_fn(
+        &mut self,
+        interp_user_fn: Option<InterpUserFn>,
+    ) -> &mut Self {
+        self.interp_user_fn = interp_user_fn.map(|user_fn| Steal::new(user_fn));
+        self
+    }
+
     pub fn session(&self) -> &Lrc<Session> {
         &self.sess
     }
@@ -61,12 +73,12 @@ impl Compiler {
     }
 }
 
-/// The compiler configuration
+/// The compiler configuration.
 pub struct Config {
-    /// Command line options
+    /// The command-line options.
     pub opts: config::Options,
 
-    /// cfg! configuration in addition to the default ones
+    /// `cfg!` configuration in addition to the default ones.
     pub crate_cfg: FxHashSet<(String, Option<String>)>,
 
     pub input: Input,
@@ -76,17 +88,14 @@ pub struct Config {
     pub file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
     pub diagnostic_output: DiagnosticOutput,
 
-    /// Set to capture stderr output during compiler execution
+    /// `Some` to capture stderr output during compiler execution.
     pub stderr: Option<Arc<Mutex<Vec<u8>>>>,
 
     pub crate_name: Option<String>,
     pub lint_caps: FxHashMap<lint::LintId, lint::Level>,
 }
 
-pub fn run_compiler_in_existing_thread_pool<F, R>(config: Config, f: F) -> R
-where
-    F: FnOnce(&Compiler) -> R,
-{
+pub fn create_compiler(config: Config) -> Compiler {
     let (sess, codegen_backend, source_map) = util::create_session(
         config.opts,
         config.crate_cfg,
@@ -98,7 +107,7 @@ where
 
     let cstore = Lrc::new(CStore::new(codegen_backend.metadata_loader()));
 
-    let compiler = Compiler {
+    Compiler {
         sess,
         codegen_backend,
         source_map,
@@ -109,7 +118,15 @@ where
         output_file: config.output_file,
         queries: Default::default(),
         crate_name: config.crate_name,
-    };
+        interp_user_fn: None,
+    }
+}
+
+pub fn run_compiler_in_existing_thread_pool<R>(
+    config: Config,
+    f: impl FnOnce(&Compiler) -> R,
+) -> R {
+    let compiler = create_compiler(config);
 
     let _sess_abort_error = OnDrop(|| {
         compiler.sess.diagnostic().print_error_count(&util::diagnostics_registry());

src/librustc_interface/lib.rs

@@ -19,3 +19,4 @@ mod proc_macro_decls;
 mod profile;
 
 pub use interface::{run_compiler, Config};
+pub use passes::BoxedGlobalCtxt;