Skip to content

Commit 5add3ce

Browse files
nnethercotennethercote
committed
XXX: Prepare for invisible delimiters.
Current places where `Interpolated` is used are going to change to instead use invisible delimiters. This prepares for that by adding panics in various places.
1 parent dcdbb88 commit 5add3ce

File tree

11 files changed

+138
-10
lines changed

11 files changed

+138
-10
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,12 @@ impl MetaItem {
347347
token::Nonterminal::NtPath(path) => (**path).clone(),
348348
_ => return None,
349349
},
350+
Some(TokenTree::Token(
351+
Token { kind: token::OpenDelim(Delimiter::Invisible(_)), .. },
352+
_,
353+
)) => {
354+
panic!("njn: invis-delim?");
355+
}
350356
_ => return None,
351357
};
352358
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,9 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
789789
vis.visit_span(sp);
790790
visit_nonterminal(nt, vis);
791791
}
792+
token::OpenDelim(token::Delimiter::Invisible(_)) => {
793+
panic!("njn: invis-delim?");
794+
}
792795
_ => {}
793796
}
794797
vis.visit_span(span);

‎compiler/rustc_ast/src/token.rs

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -513,13 +513,19 @@ impl Token {
513513
// DotDotDot is no longer supported, but we need some way to display the error
514514
DotDot | DotDotDot | DotDotEq | // range notation
515515
Lt | BinOp(Shl) | // associated path
516-
PathSep | // global path
516+
PathSep | // global path
517517
Lifetime(..) | // labeled loop
518518
Pound => true, // expression attributes
519519
Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
520520
NtExpr(..) |
521521
NtBlock(..) |
522522
NtPath(..)),
523+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
524+
NonterminalKind::Block |
525+
NonterminalKind::Expr |
526+
NonterminalKind::Literal |
527+
NonterminalKind::Path
528+
))) => true,
523529
_ => false,
524530
}
525531
}
@@ -539,11 +545,18 @@ impl Token {
539545
// DotDotDot is no longer supported
540546
| DotDot | DotDotDot | DotDotEq // ranges
541547
| Lt | BinOp(Shl) // associated path
542-
| PathSep => true, // global path
548+
| PathSep => true, // global path
543549
Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
544550
NtPat(..) |
545551
NtBlock(..) |
546552
NtPath(..)),
553+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
554+
NonterminalKind::Block |
555+
NonterminalKind::PatParam { .. } |
556+
NonterminalKind::PatWithOr |
557+
NonterminalKind::Path |
558+
NonterminalKind::Literal
559+
))) => true,
547560
_ => false,
548561
}
549562
}
@@ -564,6 +577,10 @@ impl Token {
564577
Lt | BinOp(Shl) | // associated path
565578
PathSep => true, // global path
566579
Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)),
580+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
581+
NonterminalKind::Ty |
582+
NonterminalKind::Path
583+
))) => true,
567584
// For anonymous structs or unions, which only appear in specific positions
568585
// (type of struct fields or union fields), we don't consider them as regular types
569586
_ => false,
@@ -575,6 +592,9 @@ impl Token {
575592
match self.kind {
576593
OpenDelim(Delimiter::Brace) => true,
577594
Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
595+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
596+
NonterminalKind::Expr | NonterminalKind::Block | NonterminalKind::Literal,
597+
))) => true,
578598
_ => self.can_begin_literal_maybe_minus(),
579599
}
580600
}
@@ -630,6 +650,10 @@ impl Token {
630650
},
631651
_ => false,
632652
},
653+
// njn: too simple compared to what's above?
654+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
655+
NonterminalKind::Literal | NonterminalKind::Expr,
656+
))) => true,
633657
_ => false,
634658
}
635659
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,9 @@ impl TokenStream {
500500
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
501501
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
502502
),
503+
token::OpenDelim(Delimiter::Invisible(_)) => {
504+
panic!("njn: invis-delim?");
505+
}
503506
_ => TokenTree::Token(token.clone(), spacing),
504507
}
505508
}
@@ -517,7 +520,15 @@ impl TokenStream {
517520
pub fn flattened(&self) -> TokenStream {
518521
fn can_skip(stream: &TokenStream) -> bool {
519522
stream.trees().all(|tree| match tree {
520-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
523+
TokenTree::Token(token, _) => {
524+
if let token::Interpolated(_) = token.kind {
525+
false
526+
} else if let token::OpenDelim(Delimiter::Invisible(_)) = token.kind {
527+
panic!("njn: invis-delim?");
528+
} else {
529+
true
530+
}
531+
}
521532
TokenTree::Delimited(.., inner) => can_skip(inner),
522533
})
523534
}

‎compiler/rustc_expand/src/config.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,11 @@ impl<'a> StripUnconfigured<'a> {
204204
{
205205
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
206206
}
207+
AttrTokenTree::Token(ref token, _)
208+
if let TokenKind::OpenDelim(Delimiter::Invisible(_)) = &token.kind =>
209+
{
210+
panic!("njn: invis-delim?");
211+
}
207212
AttrTokenTree::Token(token, spacing) => {
208213
Some(AttrTokenTree::Token(token, spacing)).into_iter()
209214
}

‎compiler/rustc_expand/src/mbe/diagnostics.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,13 @@ pub(super) fn failed_to_match_macro<'cx>(
8484
}
8585
}
8686

87+
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
88+
&& (matches!(expected_token.kind, TokenKind::OpenDelim(token::Delimiter::Invisible(_)))
89+
|| matches!(token.kind, TokenKind::OpenDelim(token::Delimiter::Invisible(_))))
90+
{
91+
panic!("njn: invis-delim?");
92+
}
93+
8794
// Check whether there's a missing comma in this macro call, like `println!("{}" a);`
8895
if let Some((arg, comma_span)) = arg.add_comma() {
8996
for lhs in lhses {

‎compiler/rustc_parse/src/lexer/tokentrees.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4242
let mut buf = Vec::new();
4343
loop {
4444
match self.token.kind {
45+
token::OpenDelim(Delimiter::Invisible(_)) => {
46+
panic!("njn: invis-delim?");
47+
}
4548
token::OpenDelim(delim) => {
4649
buf.push(match self.parse_token_tree_open_delim(delim) {
4750
Ok(val) => val,

‎compiler/rustc_parse/src/parser/diagnostics.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2369,6 +2369,9 @@ impl<'a> Parser<'a> {
23692369
// in a subsequent macro invocation (#71039).
23702370
let mut tok = self.token.clone();
23712371
let mut labels = vec![];
2372+
if let token::OpenDelim(Delimiter::Invisible(_)) = &tok.kind {
2373+
panic!("njn: invis-delim?");
2374+
}
23722375
while let TokenKind::Interpolated(node) = &tok.kind {
23732376
let tokens = node.0.tokens();
23742377
labels.push(node.clone());
@@ -2378,6 +2381,9 @@ impl<'a> Parser<'a> {
23782381
&& let [AttrTokenTree::Token(token, _)] = &tokens[..]
23792382
{
23802383
tok = token.clone();
2384+
if let token::OpenDelim(Delimiter::Invisible(_)) = &tok.kind {
2385+
panic!("njn: invis-delim?");
2386+
}
23812387
} else {
23822388
break;
23832389
}

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,9 @@ impl<'a> Parser<'a> {
711711
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
712712
match self.prev_token.kind {
713713
TokenKind::Interpolated(..) => self.prev_token.span,
714+
TokenKind::OpenDelim(Delimiter::Invisible(_)) => {
715+
panic!("njn: invis-delim?");
716+
}
714717
_ => expr.span,
715718
}
716719
}
@@ -3539,7 +3542,12 @@ impl<'a> Parser<'a> {
35393542
&& !self.token.is_reserved_ident()
35403543
&& self.look_ahead(1, |t| {
35413544
AssocOp::from_token(t).is_some()
3542-
|| matches!(t.kind, token::OpenDelim(_))
3545+
|| {
3546+
if let token::OpenDelim(Delimiter::Invisible(_)) = t.kind {
3547+
panic!("njn: invis-delim?");
3548+
}
3549+
matches!(t.kind, token::OpenDelim(_))
3550+
}
35433551
|| t.kind == token::Dot
35443552
})
35453553
{

‎compiler/rustc_parse/src/parser/nonterminal.rs

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use rustc_ast::ptr::P;
2-
use rustc_ast::token::{self, Delimiter, Nonterminal, Nonterminal::*, NonterminalKind, Token};
2+
use rustc_ast::token::{
3+
self, Delimiter, InvisibleOrigin, Nonterminal, Nonterminal::*, NonterminalKind, Token,
4+
};
35
use rustc_ast::HasTokens;
46
use rustc_ast_pretty::pprust;
57
use rustc_errors::PResult;
@@ -18,7 +20,30 @@ impl<'a> Parser<'a> {
1820
#[inline]
1921
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
2022
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
21-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
23+
fn may_be_ident(kind: NonterminalKind) -> bool {
24+
use NonterminalKind::*;
25+
match kind {
26+
Stmt
27+
| PatParam { .. }
28+
| PatWithOr
29+
| Expr
30+
| Ty
31+
| Ident
32+
| Literal // `true`, `false`
33+
| Meta
34+
| Path => true,
35+
36+
Item
37+
| Block
38+
| Vis
39+
| Lifetime => false,
40+
41+
TT => unreachable!(),
42+
}
43+
}
44+
45+
/// Old variant of `may_be_ident`. Being phased out.
46+
fn nt_may_be_ident(nt: &token::Nonterminal) -> bool {
2247
match nt {
2348
NtStmt(_)
2449
| NtPat(_)
@@ -48,8 +73,11 @@ impl<'a> Parser<'a> {
4873
NonterminalKind::Ident => get_macro_ident(token).is_some(),
4974
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
5075
NonterminalKind::Vis => match token.kind {
51-
// The follow-set of :vis + "priv" keyword + interpolated
52-
token::Comma | token::Ident(..) | token::Interpolated(_) => true,
76+
// The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion
77+
token::Comma
78+
| token::Ident(..)
79+
| token::Interpolated(..)
80+
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
5381
_ => token.can_begin_type(),
5482
},
5583
NonterminalKind::Block => match &token.kind {
@@ -59,11 +87,30 @@ impl<'a> Parser<'a> {
5987
NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_)
6088
| NtVis(_) => false,
6189
},
90+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
91+
NonterminalKind::Block
92+
| NonterminalKind::Lifetime
93+
| NonterminalKind::Stmt
94+
| NonterminalKind::Expr
95+
| NonterminalKind::Literal => true,
96+
NonterminalKind::Item
97+
| NonterminalKind::PatParam { .. }
98+
| NonterminalKind::PatWithOr
99+
| NonterminalKind::Ty
100+
| NonterminalKind::Ident
101+
| NonterminalKind::Meta
102+
| NonterminalKind::Path
103+
| NonterminalKind::Vis => false,
104+
NonterminalKind::TT => unreachable!(),
105+
},
62106
_ => false,
63107
},
64108
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
65109
token::PathSep | token::Ident(..) => true,
66-
token::Interpolated(nt) => may_be_ident(&nt.0),
110+
token::Interpolated(nt) => nt_may_be_ident(&nt.0),
111+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
112+
may_be_ident(*kind)
113+
}
67114
_ => false,
68115
},
69116
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
@@ -81,14 +128,20 @@ impl<'a> Parser<'a> {
81128
token::BinOp(token::Shl) => true, // path (double UFCS)
82129
// leading vert `|` or-pattern
83130
token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr),
84-
token::Interpolated(nt) => may_be_ident(&nt.0),
131+
token::Interpolated(nt) => nt_may_be_ident(&nt.0),
132+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
133+
may_be_ident(*kind)
134+
}
85135
_ => false,
86136
},
87137
NonterminalKind::Lifetime => match &token.kind {
88138
token::Lifetime(_) => true,
89139
token::Interpolated(nt) => {
90140
matches!(&nt.0, NtLifetime(_))
91141
}
142+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
143+
NonterminalKind::Lifetime,
144+
))) => true,
92145
_ => false,
93146
},
94147
NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => {

0 commit comments

Comments
 (0)