Skip to content

Commit 44a6662

Browse files
nnethercotennethercote
committed
Remove NtBlock, Nonterminal, and TokenKind::Interpolated.
`NtBlock` is the last remaining variant of `Nonterminal`, so once it is gone then `Nonterminal` can be removed as well.
1 parent 389351b commit 44a6662

File tree

17 files changed

+104
-366
lines changed

17 files changed

+104
-366
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

-14
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ use std::fmt;
66
use std::marker::PhantomData;
77

88
use crate::ptr::P;
9-
use crate::token::Nonterminal;
109
use crate::tokenstream::LazyAttrTokenStream;
1110
use crate::{
1211
Arm, AssocItem, AttrItem, AttrKind, AttrVec, Attribute, Block, Crate, Expr, ExprField,
@@ -196,19 +195,6 @@ impl HasTokens for Attribute {
196195
}
197196
}
198197

199-
impl HasTokens for Nonterminal {
200-
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
201-
match self {
202-
Nonterminal::NtBlock(block) => block.tokens(),
203-
}
204-
}
205-
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
206-
match self {
207-
Nonterminal::NtBlock(block) => block.tokens_mut(),
208-
}
209-
}
210-
}
211-
212198
/// A trait for AST nodes having (or not having) attributes.
213199
pub trait HasAttrs {
214200
/// This is `true` if this `HasAttrs` might support 'custom' (proc-macro) inner

‎compiler/rustc_ast/src/mut_visit.rs

+3-37
Original file line numberDiff line numberDiff line change
@@ -745,9 +745,9 @@ fn visit_lazy_tts<T: MutVisitor>(vis: &mut T, lazy_tts: &mut Option<LazyAttrToke
745745
visit_lazy_tts_opt_mut(vis, lazy_tts.as_mut());
746746
}
747747

748-
/// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
749-
/// In practice the ident part is not actually used by specific visitors right now,
750-
/// but there's a test below checking that it works.
748+
/// Applies ident visitor if it's an ident. In practice this is not actually
749+
/// used by specific visitors right now, but there's a test below checking that
750+
/// it works.
751751
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
752752
pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
753753
let Token { kind, span } = t;
@@ -765,45 +765,11 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
765765
token::NtLifetime(ident, _is_raw) => {
766766
vis.visit_ident(ident);
767767
}
768-
token::Interpolated(nt) => {
769-
let nt = Lrc::make_mut(nt);
770-
visit_nonterminal(vis, nt);
771-
}
772768
_ => {}
773769
}
774770
vis.visit_span(span);
775771
}
776772

777-
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
778-
/// Applies the visitor to elements of interpolated nodes.
779-
//
780-
// N.B., this can occur only when applying a visitor to partially expanded
781-
// code, where parsed pieces have gotten implanted ito *other* macro
782-
// invocations. This is relevant for macro hygiene, but possibly not elsewhere.
783-
//
784-
// One problem here occurs because the types for flat_map_item, flat_map_stmt,
785-
// etc., allow the visitor to return *multiple* items; this is a problem for the
786-
// nodes here, because they insist on having exactly one piece. One solution
787-
// would be to mangle the MutVisitor trait to include one-to-many and
788-
// one-to-one versions of these entry points, but that would probably confuse a
789-
// lot of people and help very few. Instead, I'm just going to put in dynamic
790-
// checks. I think the performance impact of this will be pretty much
791-
// nonexistent. The danger is that someone will apply a `MutVisitor` to a
792-
// partially expanded node, and will be confused by the fact that their
793-
// `flat_map_item` or `flat_map_stmt` isn't getting called on `NtItem` or `NtStmt`
794-
// nodes. Hopefully they'll wind up reading this comment, and doing something
795-
// appropriate.
796-
//
797-
// BTW, design choice: I considered just changing the type of, e.g., `NtItem` to
798-
// contain multiple items, but decided against it when I looked at
799-
// `parse_item_or_view_item` and tried to figure out what I would do with
800-
// multiple items there....
801-
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
802-
match nt {
803-
token::NtBlock(block) => vis.visit_block(block),
804-
}
805-
}
806-
807773
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
808774
fn visit_defaultness<T: MutVisitor>(vis: &mut T, defaultness: &mut Defaultness) {
809775
match defaultness {

‎compiler/rustc_ast/src/token.rs

+18-113
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
use std::borrow::Cow;
22
use std::fmt;
33

4-
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
5-
use rustc_data_structures::sync::Lrc;
64
use rustc_macros::{Decodable, Encodable, HashStable_Generic};
75
use rustc_span::edition::Edition;
86
use rustc_span::symbol::{kw, sym};
@@ -12,13 +10,11 @@ use rustc_span::symbol::{Ident, Symbol};
1210
use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP};
1311
pub use BinOpToken::*;
1412
pub use LitKind::*;
15-
pub use Nonterminal::*;
1613
pub use NtExprKind::*;
1714
pub use NtPatKind::*;
1815
pub use TokenKind::*;
1916

2017
use crate::ast;
21-
use crate::ptr::P;
2218
use crate::util::case::Case;
2319

2420
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -52,8 +48,8 @@ pub enum InvisibleOrigin {
5248
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
5349
ProcMacro,
5450

55-
// Converted from `TokenKind::Interpolated` in
56-
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
51+
// Converted from `TokenKind::NtLifetime` in `TokenStream::flatten_token`.
52+
// Treated similarly to `ProcMacro`.
5753
FlattenToken,
5854
}
5955

@@ -349,9 +345,7 @@ impl From<IdentIsRaw> for bool {
349345
}
350346
}
351347

352-
// SAFETY: due to the `Clone` impl below, all fields of all variants other than
353-
// `Interpolated` must impl `Copy`.
354-
#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
348+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
355349
pub enum TokenKind {
356350
/* Expression-operator symbols. */
357351
/// `=`
@@ -440,21 +434,6 @@ pub enum TokenKind {
440434
/// the `lifetime` metavariable in the macro's RHS.
441435
NtLifetime(Ident, IdentIsRaw),
442436

443-
/// An embedded AST node, as produced by a macro. This only exists for
444-
/// historical reasons. We'd like to get rid of it, for multiple reasons.
445-
/// - It's conceptually very strange. Saying a token can contain an AST
446-
/// node is like saying, in natural language, that a word can contain a
447-
/// sentence.
448-
/// - It requires special handling in a bunch of places in the parser.
449-
/// - It prevents `Token` from implementing `Copy`.
450-
/// It adds complexity and likely slows things down. Please don't add new
451-
/// occurrences of this token kind!
452-
///
453-
/// The span in the surrounding `Token` is that of the metavariable in the
454-
/// macro's RHS. The span within the Nonterminal is that of the fragment
455-
/// passed to the macro at the call site.
456-
Interpolated(Lrc<Nonterminal>),
457-
458437
/// A doc comment token.
459438
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
460439
/// similarly to symbols in string literal tokens.
@@ -464,19 +443,6 @@ pub enum TokenKind {
464443
Eof,
465444
}
466445

467-
impl Clone for TokenKind {
468-
fn clone(&self) -> Self {
469-
// `TokenKind` would impl `Copy` if it weren't for `Interpolated`. So
470-
// for all other variants, this implementation of `clone` is just like
471-
// a copy. This is faster than the `derive(Clone)` version which has a
472-
// separate path for every variant.
473-
match self {
474-
Interpolated(nt) => Interpolated(nt.clone()),
475-
_ => unsafe { std::ptr::read(self) },
476-
}
477-
}
478-
}
479-
480446
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
481447
pub struct Token {
482448
pub kind: TokenKind,
@@ -565,7 +531,6 @@ impl Token {
565531
pub fn uninterpolated_span(&self) -> Span {
566532
match self.kind {
567533
NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
568-
Interpolated(ref nt) => nt.use_span(),
569534
_ => self.span,
570535
}
571536
}
@@ -583,7 +548,7 @@ impl Token {
583548
}
584549

585550
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
586-
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
551+
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
587552
}
588553
}
589554

@@ -614,7 +579,6 @@ impl Token {
614579
PathSep | // global path
615580
Lifetime(..) | // labeled loop
616581
Pound => true, // expression attributes
617-
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
618582
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
619583
MetaVarKind::Block |
620584
MetaVarKind::Expr { .. } |
@@ -687,7 +651,6 @@ impl Token {
687651
match self.kind {
688652
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
689653
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
690-
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
691654
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
692655
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
693656
))) => true,
@@ -815,31 +778,20 @@ impl Token {
815778
/// Is this a pre-parsed expression dropped into the token stream
816779
/// (which happens while parsing the result of macro expansion)?
817780
pub fn is_metavar_expr(&self) -> bool {
818-
#[allow(irrefutable_let_patterns)] // FIXME: temporary
819-
if let Interpolated(nt) = &self.kind
820-
&& let NtBlock(_) = &**nt
821-
{
822-
true
823-
} else if matches!(
781+
matches!(
824782
self.is_metavar_seq(),
825-
Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
826-
) {
827-
true
828-
} else {
829-
false
830-
}
783+
Some(
784+
MetaVarKind::Expr { .. }
785+
| MetaVarKind::Literal
786+
| MetaVarKind::Path
787+
| MetaVarKind::Block
788+
)
789+
)
831790
}
832791

833-
/// Is the token an interpolated block (`$b:block`)?
834-
pub fn is_whole_block(&self) -> bool {
835-
#[allow(irrefutable_let_patterns)] // FIXME: temporary
836-
if let Interpolated(nt) = &self.kind
837-
&& let NtBlock(..) = &**nt
838-
{
839-
return true;
840-
}
841-
842-
false
792+
/// Are we at a block from a metavar (`$b:block`)?
793+
pub fn is_metavar_block(&self) -> bool {
794+
matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
843795
}
844796

845797
/// Returns `true` if the token is either the `mut` or `const` keyword.
@@ -864,7 +816,8 @@ impl Token {
864816
self.is_non_raw_ident_where(|id| id.name == kw)
865817
}
866818

867-
/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case.
819+
/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this
820+
/// token is an identifier equal to `kw` ignoring the case.
868821
pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
869822
self.is_keyword(kw)
870823
|| (case == Case::Insensitive
@@ -985,7 +938,7 @@ impl Token {
985938
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
986939
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
987940
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
988-
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
941+
| Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof => {
989942
return None;
990943
}
991944
};
@@ -1022,12 +975,6 @@ pub enum NtExprKind {
1022975
Expr2021 { inferred: bool },
1023976
}
1024977

1025-
#[derive(Clone, Encodable, Decodable)]
1026-
/// For interpolation during macro expansion.
1027-
pub enum Nonterminal {
1028-
NtBlock(P<ast::Block>),
1029-
}
1030-
1031978
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
1032979
pub enum NonterminalKind {
1033980
Item,
@@ -1111,47 +1058,6 @@ impl fmt::Display for NonterminalKind {
11111058
}
11121059
}
11131060

1114-
impl Nonterminal {
1115-
pub fn use_span(&self) -> Span {
1116-
match self {
1117-
NtBlock(block) => block.span,
1118-
}
1119-
}
1120-
1121-
pub fn descr(&self) -> &'static str {
1122-
match self {
1123-
NtBlock(..) => "block",
1124-
}
1125-
}
1126-
}
1127-
1128-
impl PartialEq for Nonterminal {
1129-
fn eq(&self, _rhs: &Self) -> bool {
1130-
// FIXME: Assume that all nonterminals are not equal, we can't compare them
1131-
// correctly based on data from AST. This will prevent them from matching each other
1132-
// in macros. The comparison will become possible only when each nonterminal has an
1133-
// attached token stream from which it was parsed.
1134-
false
1135-
}
1136-
}
1137-
1138-
impl fmt::Debug for Nonterminal {
1139-
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1140-
match *self {
1141-
NtBlock(..) => f.pad("NtBlock(..)"),
1142-
}
1143-
}
1144-
}
1145-
1146-
impl<CTX> HashStable<CTX> for Nonterminal
1147-
where
1148-
CTX: crate::HashStableContext,
1149-
{
1150-
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
1151-
panic!("interpolated tokens should not be present in the HIR")
1152-
}
1153-
}
1154-
11551061
// Some types are used a lot. Make sure they don't unintentionally get bigger.
11561062
#[cfg(target_pointer_width = "64")]
11571063
mod size_asserts {
@@ -1161,7 +1067,6 @@ mod size_asserts {
11611067
// tidy-alphabetical-start
11621068
static_assert_size!(Lit, 12);
11631069
static_assert_size!(LitKind, 2);
1164-
static_assert_size!(Nonterminal, 8);
11651070
static_assert_size!(Token, 24);
11661071
static_assert_size!(TokenKind, 16);
11671072
// tidy-alphabetical-end

‎compiler/rustc_ast/src/tokenstream.rs

+4-22
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP};
2424

2525
use crate::ast::AttrStyle;
2626
use crate::ast_traits::{HasAttrs, HasTokens};
27-
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
27+
use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind};
2828
use crate::{AttrVec, Attribute};
2929

3030
/// Part of a `TokenStream`.
@@ -288,11 +288,6 @@ pub struct AttrsTarget {
288288
}
289289

290290
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
291-
///
292-
/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
293-
/// instead of a representation of the abstract syntax tree.
294-
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
295-
/// backwards compatibility.
296291
#[derive(Clone, Debug, Default, Encodable, Decodable)]
297292
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
298293

@@ -459,12 +454,6 @@ impl TokenStream {
459454
TokenStream::new(tts)
460455
}
461456

462-
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
463-
match nt {
464-
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
465-
}
466-
}
467-
468457
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
469458
match token.kind {
470459
token::NtIdent(ident, is_raw) => {
@@ -476,12 +465,6 @@ impl TokenStream {
476465
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
477466
TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
478467
),
479-
token::Interpolated(ref nt) => TokenTree::Delimited(
480-
DelimSpan::from_single(token.span),
481-
DelimSpacing::new(Spacing::JointHidden, spacing),
482-
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
483-
TokenStream::from_nonterminal_ast(&nt).flattened(),
484-
),
485468
_ => TokenTree::Token(token.clone(), spacing),
486469
}
487470
}
@@ -499,10 +482,9 @@ impl TokenStream {
499482
pub fn flattened(&self) -> TokenStream {
500483
fn can_skip(stream: &TokenStream) -> bool {
501484
stream.trees().all(|tree| match tree {
502-
TokenTree::Token(token, _) => !matches!(
503-
token.kind,
504-
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
505-
),
485+
TokenTree::Token(token, _) => {
486+
!matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..))
487+
}
506488
TokenTree::Delimited(.., inner) => can_skip(inner),
507489
})
508490
}

0 commit comments

Comments
 (0)