fix token issue

Author: Bryntet

compiler/rustc_ast/src/attr/mod.rs

@@ -323,18 +323,15 @@ impl Attribute {
         }
     }
 
-    pub fn token_trees(&self) -> Vec<TokenTree> {
+    pub fn token_trees(&self) -> Option<Vec<TokenTree>> {
         match self.kind {
-            AttrKind::Normal(ref normal) => normal
-                .tokens
-                .as_ref()
-                .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
-                .to_attr_token_stream()
-                .to_token_trees(),
-            AttrKind::DocComment(comment_kind, data) => vec![TokenTree::token_alone(
+            AttrKind::Normal(ref normal) => {
+                Some(normal.tokens.as_ref()?.to_attr_token_stream().to_token_trees())
+            }
+            AttrKind::DocComment(comment_kind, data) => Some(vec![TokenTree::token_alone(
                 token::DocComment(comment_kind, self.style, data),
                 self.span,
-            )],
+            )]),
         }
     }
 }

compiler/rustc_ast/src/tokenstream.rs

@@ -455,9 +455,7 @@ fn attrs_and_tokens_to_token_trees(
     let (outer_attrs, inner_attrs) = attrs.split_at(idx);
 
     // Add outer attribute tokens.
-    for attr in outer_attrs {
-        res.extend(attr.token_trees());
-    }
+    res.extend(outer_attrs.iter().filter_map(Attribute::token_trees).flatten());
 
     // Add target AST node tokens.
     res.extend(target_tokens.to_attr_token_stream().to_token_trees());
@@ -485,10 +483,12 @@ fn attrs_and_tokens_to_token_trees(
         for tree in tts.iter_mut().rev() {
             if let TokenTree::Delimited(span, spacing, Delimiter::Brace, stream) = tree {
                 // Found it: the rightmost, outermost braced group.
-                let mut tts = vec![];
-                for inner_attr in inner_attrs {
-                    tts.extend(inner_attr.token_trees());
-                }
+                let mut tts = inner_attrs
+                    .iter()
+                    .filter_map(Attribute::token_trees)
+                    .flatten()
+                    .collect::<Vec<_>>();
+
                 tts.extend(stream.0.iter().cloned());
                 let stream = TokenStream::new(tts);
                 *tree = TokenTree::Delimited(*span, *spacing, Delimiter::Brace, stream);

compiler/rustc_expand/src/config.rs

@@ -336,7 +336,10 @@ impl<'a> StripUnconfigured<'a> {
         // Convert `#[cfg_attr(pred, attr)]` to `#[attr]`.
 
         // Use the `#` from `#[cfg_attr(pred, attr)]` in the result `#[attr]`.
-        let mut orig_trees = cfg_attr.token_trees().into_iter();
+        let mut orig_trees = cfg_attr
+            .token_trees()
+            .unwrap_or_else(|| panic!("#[cfg_attr] should always have token_trees"))
+            .into_iter();
         let Some(TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _)) =
             orig_trees.next()
         else {