resolve: Block expansion of a derive container until all its derives are resolved

Also mark derive helpers as known as a part of the derive container's expansion instead of expansion of the derives themselves which may happen too late.

Author: petrochenkov

src/librustc_metadata/decoder.rs

@@ -538,9 +538,7 @@ impl<'a, 'tcx> CrateMetadata {
                     attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
                 (
                     trait_name,
-                    SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
-                        client, attrs: helper_attrs.clone()
-                    })),
+                    SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive { client })),
                     helper_attrs,
                 )
             }

src/librustc_resolve/macros.rs

@@ -13,7 +13,7 @@ use rustc::{ty, lint, span_bug};
 use syntax::ast::{self, NodeId, Ident};
 use syntax::attr::StabilityLevel;
 use syntax::edition::Edition;
-use syntax::ext::base::{self, Indeterminate, SpecialDerives};
+use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives};
 use syntax::ext::base::{MacroKind, SyntaxExtension};
 use syntax::ext::expand::{AstFragment, Invocation, InvocationKind};
 use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind};
@@ -142,7 +142,7 @@ impl<'a> base::Resolver for Resolver<'a> {
 
     fn resolve_macro_invocation(
         &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool
-    ) -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate> {
+    ) -> Result<InvocationRes, Indeterminate> {
         let invoc_id = invoc.expansion_data.id;
         let parent_scope = match self.invocation_parent_scopes.get(&invoc_id) {
             Some(parent_scope) => *parent_scope,
@@ -165,25 +165,24 @@ impl<'a> base::Resolver for Resolver<'a> {
             InvocationKind::Derive { ref path, .. } =>
                 (path, MacroKind::Derive, &[][..], false),
             InvocationKind::DeriveContainer { ref derives, .. } => {
-                // Block expansion of derives in the container until we know whether one of them
-                // is a built-in `Copy`. Skip the resolution if there's only one derive - either
-                // it's not a `Copy` and we don't need to do anything, or it's a `Copy` and it
-                // will automatically knows about itself.
-                let mut result = Ok(None);
-                if derives.len() > 1 {
-                    for path in derives {
-                        match self.resolve_macro_path(path, Some(MacroKind::Derive),
-                                                      &parent_scope, true, force) {
-                            Ok((Some(ref ext), _)) if ext.is_derive_copy => {
-                                self.add_derives(invoc_id, SpecialDerives::COPY);
-                                return Ok(None);
-                            }
-                            Err(Determinacy::Undetermined) => result = Err(Indeterminate),
-                            _ => {}
-                        }
-                    }
+                // Block expansion of the container until we resolve all derives in it.
+                // This is required for two reasons:
+                // - Derive helper attributes are in scope for the item to which the `#[derive]`
+                //   is applied, so they have to be produced by the container's expansion rather
+                //   than by individual derives.
+                // - Derives in the container need to know whether one of them is a built-in `Copy`.
+                // FIXME: Try to avoid repeated resolutions for derives here and in expansion.
+                let mut exts = Vec::new();
+                for path in derives {
+                    exts.push(match self.resolve_macro_path(
+                        path, Some(MacroKind::Derive), &parent_scope, true, force
+                    ) {
+                        Ok((Some(ext), _)) => ext,
+                        Ok(_) | Err(Determinacy::Determined) => self.dummy_ext(MacroKind::Derive),
+                        Err(Determinacy::Undetermined) => return Err(Indeterminate),
+                    })
                 }
-                return result;
+                return Ok(InvocationRes::DeriveContainer(exts));
             }
         };
 
@@ -203,7 +202,7 @@ impl<'a> base::Resolver for Resolver<'a> {
             self.definitions.add_parent_module_of_macro_def(invoc_id, normal_module_def_id);
         }
 
-        Ok(Some(ext))
+        Ok(InvocationRes::Single(ext))
     }
 
     fn check_unused_macros(&self) {

src/libsyntax/ext/base.rs

@@ -11,6 +11,7 @@ use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
 use crate::{ThinVec, MACRO_ARGUMENTS};
 use crate::tokenstream::{self, TokenStream, TokenTree};
+use crate::visit::Visitor;
 
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};
@@ -72,6 +73,17 @@ impl Annotatable {
         }
     }
 
+    pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
+        match self {
+            Annotatable::Item(item) => visitor.visit_item(item),
+            Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item),
+            Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item),
+            Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item),
+            Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt),
+            Annotatable::Expr(expr) => visitor.visit_expr(expr),
+        }
+    }
+
     pub fn expect_item(self) -> P<ast::Item> {
         match self {
             Annotatable::Item(i) => i,
@@ -637,6 +649,12 @@ impl SyntaxExtension {
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 
+/// Result of resolving a macro invocation.
+pub enum InvocationRes {
+    Single(Lrc<SyntaxExtension>),
+    DeriveContainer(Vec<Lrc<SyntaxExtension>>),
+}
+
 /// Error type that denotes indeterminacy.
 pub struct Indeterminate;
 
@@ -664,7 +682,7 @@ pub trait Resolver {
 
     fn resolve_macro_invocation(
         &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool
-    ) -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate>;
+    ) -> Result<InvocationRes, Indeterminate>;
 
     fn check_unused_macros(&self);
 

src/libsyntax/ext/expand.rs

@@ -4,7 +4,7 @@ use crate::attr::{self, HasAttrs};
 use crate::source_map::respan;
 use crate::config::StripUnconfigured;
 use crate::ext::base::*;
-use crate::ext::proc_macro::collect_derives;
+use crate::ext::proc_macro::{collect_derives, MarkAttrs};
 use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
 use crate::ext::tt::macro_rules::annotate_err_with_kind;
 use crate::ext::placeholders::{placeholder, PlaceholderExpander};
@@ -307,10 +307,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
 
             let eager_expansion_root =
                 if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id };
-            let ext = match self.cx.resolver.resolve_macro_invocation(
+            let res = match self.cx.resolver.resolve_macro_invocation(
                 &invoc, eager_expansion_root, force
             ) {
-                Ok(ext) => ext,
+                Ok(res) => res,
                 Err(Indeterminate) => {
                     undetermined_invocations.push(invoc);
                     continue
@@ -322,54 +322,72 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             self.cx.current_expansion = invoc.expansion_data.clone();
 
             // FIXME(jseyfried): Refactor out the following logic
-            let (expanded_fragment, new_invocations) = if let Some(ext) = ext {
-                let fragment = self.expand_invoc(invoc, &ext.kind);
-                self.collect_invocations(fragment, &[])
-            } else if let InvocationKind::DeriveContainer { derives: traits, item } = invoc.kind {
-                if !item.derive_allowed() {
-                    let attr = attr::find_by_name(item.attrs(), sym::derive)
-                        .expect("`derive` attribute should exist");
-                    let span = attr.span;
-                    let mut err = self.cx.mut_span_err(span,
-                                                        "`derive` may only be applied to \
-                                                        structs, enums and unions");
-                    if let ast::AttrStyle::Inner = attr.style {
-                        let trait_list = traits.iter()
-                            .map(|t| t.to_string()).collect::<Vec<_>>();
-                        let suggestion = format!("#[derive({})]", trait_list.join(", "));
-                        err.span_suggestion(
-                            span, "try an outer attribute", suggestion,
-                            // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
-                            Applicability::MaybeIncorrect
-                        );
-                    }
-                    err.emit();
+            let (expanded_fragment, new_invocations) = match res {
+                InvocationRes::Single(ext) => {
+                    let fragment = self.expand_invoc(invoc, &ext.kind);
+                    self.collect_invocations(fragment, &[])
                 }
+                InvocationRes::DeriveContainer(exts) => {
+                    let (derives, item) = match invoc.kind {
+                        InvocationKind::DeriveContainer { derives, item } => (derives, item),
+                        _ => unreachable!(),
+                    };
+                    if !item.derive_allowed() {
+                        let attr = attr::find_by_name(item.attrs(), sym::derive)
+                            .expect("`derive` attribute should exist");
+                        let span = attr.span;
+                        let mut err = self.cx.mut_span_err(span,
+                            "`derive` may only be applied to structs, enums and unions");
+                        if let ast::AttrStyle::Inner = attr.style {
+                            let trait_list = derives.iter()
+                                .map(|t| t.to_string()).collect::<Vec<_>>();
+                            let suggestion = format!("#[derive({})]", trait_list.join(", "));
+                            err.span_suggestion(
+                                span, "try an outer attribute", suggestion,
+                                // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
+                                Applicability::MaybeIncorrect
+                            );
+                        }
+                        err.emit();
+                    }
 
-                let mut item = self.fully_configure(item);
-                item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
-                let derive_placeholders =
-                    all_derive_placeholders.entry(invoc.expansion_data.id).or_default();
-
-                derive_placeholders.reserve(traits.len());
-                invocations.reserve(traits.len());
-                for path in traits {
-                    let expn_id = ExpnId::fresh(None);
-                    derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
-                    invocations.push(Invocation {
-                        kind: InvocationKind::Derive { path, item: item.clone() },
-                        fragment_kind: invoc.fragment_kind,
-                        expansion_data: ExpansionData {
-                            id: expn_id,
-                            ..invoc.expansion_data.clone()
-                        },
-                    });
+                    let mut item = self.fully_configure(item);
+                    item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
+                    let mut helper_attrs = Vec::new();
+                    let mut has_copy = false;
+                    for ext in exts {
+                        helper_attrs.extend(&ext.helper_attrs);
+                        has_copy |= ext.is_derive_copy;
+                    }
+                    // Mark derive helpers inside this item as known and used.
+                    // FIXME: This is a hack, derive helpers should be integrated with regular name
+                    // resolution instead. For example, helpers introduced by a derive container
+                    // can be in scope for all code produced by that container's expansion.
+                    item.visit_with(&mut MarkAttrs(&helper_attrs));
+                    if has_copy {
+                        self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY);
+                    }
+
+                    let derive_placeholders =
+                        all_derive_placeholders.entry(invoc.expansion_data.id).or_default();
+                    derive_placeholders.reserve(derives.len());
+                    invocations.reserve(derives.len());
+                    for path in derives {
+                        let expn_id = ExpnId::fresh(None);
+                        derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
+                        invocations.push(Invocation {
+                            kind: InvocationKind::Derive { path, item: item.clone() },
+                            fragment_kind: invoc.fragment_kind,
+                            expansion_data: ExpansionData {
+                                id: expn_id,
+                                ..invoc.expansion_data.clone()
+                            },
+                        });
+                    }
+                    let fragment = invoc.fragment_kind
+                        .expect_from_annotatables(::std::iter::once(item));
+                    self.collect_invocations(fragment, derive_placeholders)
                 }
-                let fragment = invoc.fragment_kind
-                    .expect_from_annotatables(::std::iter::once(item));
-                self.collect_invocations(fragment, derive_placeholders)
-            } else {
-                unreachable!()
             };
 
             if expanded_fragments.len() < depth {

src/libsyntax/ext/proc_macro.rs

@@ -78,7 +78,6 @@ pub struct ProcMacroDerive {
     pub client: proc_macro::bridge::client::Client<
         fn(proc_macro::TokenStream) -> proc_macro::TokenStream,
     >,
-    pub attrs: Vec<ast::Name>,
 }
 
 impl MultiItemModifier for ProcMacroDerive {
@@ -111,9 +110,6 @@ impl MultiItemModifier for ProcMacroDerive {
             }
         }
 
-        // Mark attributes as known, and used.
-        MarkAttrs(&self.attrs).visit_item(&item);
-
         let token = token::Interpolated(Lrc::new(token::NtItem(item)));
         let input = tokenstream::TokenTree::token(token, DUMMY_SP).into();
 
@@ -164,7 +160,7 @@ impl MultiItemModifier for ProcMacroDerive {
     }
 }
 
-struct MarkAttrs<'a>(&'a [ast::Name]);
+crate struct MarkAttrs<'a>(crate &'a [ast::Name]);
 
 impl<'a> Visitor<'a> for MarkAttrs<'a> {
     fn visit_attribute(&mut self, attr: &Attribute) {

src/test/ui/derives/deriving-bounds.stderr

@@ -1,15 +1,3 @@
-error: cannot find derive macro `Send` in this scope
-  --> $DIR/deriving-bounds.rs:1:10
-   |
-LL | #[derive(Send)]
-   |          ^^^^
-   |
-note: unsafe traits like `Send` should be implemented explicitly
-  --> $DIR/deriving-bounds.rs:1:10
-   |
-LL | #[derive(Send)]
-   |          ^^^^
-
 error: cannot find derive macro `Sync` in this scope
   --> $DIR/deriving-bounds.rs:5:10
    |
@@ -22,5 +10,17 @@ note: unsafe traits like `Sync` should be implemented explicitly
 LL | #[derive(Sync)]
    |          ^^^^
 
+error: cannot find derive macro `Send` in this scope
+  --> $DIR/deriving-bounds.rs:1:10
+   |
+LL | #[derive(Send)]
+   |          ^^^^
+   |
+note: unsafe traits like `Send` should be implemented explicitly
+  --> $DIR/deriving-bounds.rs:1:10
+   |
+LL | #[derive(Send)]
+   |          ^^^^
+
 error: aborting due to 2 previous errors
 

src/test/ui/feature-gate/issue-43106-gating-of-derive-2.stderr

@@ -1,5 +1,5 @@
 error: cannot find derive macro `x3300` in this scope
-  --> $DIR/issue-43106-gating-of-derive-2.rs:4:14
+  --> $DIR/issue-43106-gating-of-derive-2.rs:12:14
    |
 LL |     #[derive(x3300)]
    |              ^^^^^
@@ -11,7 +11,7 @@ LL |     #[derive(x3300)]
    |              ^^^^^
 
 error: cannot find derive macro `x3300` in this scope
-  --> $DIR/issue-43106-gating-of-derive-2.rs:12:14
+  --> $DIR/issue-43106-gating-of-derive-2.rs:4:14
    |
 LL |     #[derive(x3300)]
    |              ^^^^^

src/test/ui/feature-gate/issue-43106-gating-of-derive.rs

@@ -1,9 +1,6 @@
 // `#![derive]` raises errors when it occurs at contexts other than ADT
 // definitions.
 
-#![derive(Debug)]
-//~^ ERROR `derive` may only be applied to structs, enums and unions
-
 #[derive(Debug)]
 //~^ ERROR `derive` may only be applied to structs, enums and unions
 mod derive {

src/test/ui/feature-gate/issue-43106-gating-of-derive.stderr

@@ -1,38 +1,32 @@
 error: `derive` may only be applied to structs, enums and unions
   --> $DIR/issue-43106-gating-of-derive.rs:4:1
    |
-LL | #![derive(Debug)]
-   | ^^^^^^^^^^^^^^^^^ help: try an outer attribute: `#[derive(Debug)]`
-
-error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:7:1
-   |
 LL | #[derive(Debug)]
    | ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:10:17
+  --> $DIR/issue-43106-gating-of-derive.rs:7:17
    |
 LL |     mod inner { #![derive(Debug)] }
    |                 ^^^^^^^^^^^^^^^^^ help: try an outer attribute: `#[derive(Debug)]`
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:13:5
+  --> $DIR/issue-43106-gating-of-derive.rs:10:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:26:5
+  --> $DIR/issue-43106-gating-of-derive.rs:23:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:30:5
+  --> $DIR/issue-43106-gating-of-derive.rs:27:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
-error: aborting due to 6 previous errors
+error: aborting due to 5 previous errors