Support bundling all transitive crates in one directory

Transitive crate dependencies are passed to rustc by specifying the
folder they live in as a library search path (an `-Ldependency=path`
flag). Given that bazel builds all libraries into their own folders,
this means passing potentially hundreds of paths to rustc.

For a number of transitive dependencies greater than 400 we've observed
that rustc runs into some odd `LoadLibraryExW` error and fails to find
some transitive crates and thus fails to build.

For this reason, this patch can optionally createc symlinks to all the
transitive crates into a single directory that it then sets as the
`-Ldependency=` flag to rustc. We only do this when building on Windows
and when the number of transitive crates is greater than 400, which
happens for a handful of top-level crates in the dropbox_core project.

Author: ddeville

rust/private/clippy.bzl

@@ -83,7 +83,7 @@ def _clippy_aspect_impl(target, ctx):
         build_info,
     )
 
-    args, env = construct_arguments(
+    args, env, extra_link_inputs = construct_arguments(
         ctx = ctx,
         attr = ctx.rule.attr,
         file = ctx.file,
@@ -102,6 +102,7 @@ def _clippy_aspect_impl(target, ctx):
         build_flags_files = build_flags_files,
         emit = ["dep-info", "metadata"],
     )
+    compile_inputs = depset(extra_link_inputs, transitive = [compile_inputs])
 
     if crate_info.is_test:
         args.rustc_flags.add("--test")

rust/private/rustc.bzl

@@ -631,6 +631,7 @@ def construct_arguments(
                 - all (list): A list of all `Args` objects in the order listed above.
                     This is to be passed to the `arguments` parameter of actions
             - (dict): Common rustc environment variables
+            - (list): Extra input files for the compile action
     """
     output_dir = getattr(crate_info.output, "dirname", None)
     linker_script = getattr(file, "linker_script", None)
@@ -764,7 +765,7 @@ def construct_arguments(
         _add_native_link_flags(rustc_flags, dep_info, linkstamp_outs, ambiguous_libs, crate_info.type, toolchain, cc_toolchain, feature_configuration)
 
     # These always need to be added, even if not linking this crate.
-    add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct)
+    extra_link_inputs = add_crate_link_flags(ctx, toolchain, rustc_flags, crate_info, dep_info, force_all_deps_direct)
 
     needs_extern_proc_macro_flag = "proc-macro" in [crate_info.type, crate_info.wrapped_crate_type] and \
                                    crate_info.edition != "2015"
@@ -810,7 +811,7 @@ def construct_arguments(
         all = [process_wrapper_flags, rustc_path, rustc_flags],
     )
 
-    return args, env
+    return args, env, extra_link_inputs
 
 def rustc_compile_action(
         ctx,
@@ -867,7 +868,7 @@ def rustc_compile_action(
         stamp = stamp,
     )
 
-    args, env_from_args = construct_arguments(
+    args, env_from_args, extra_link_inputs = construct_arguments(
         ctx = ctx,
         attr = attr,
         file = ctx.file,
@@ -887,6 +888,7 @@ def rustc_compile_action(
         force_all_deps_direct = force_all_deps_direct,
         stamp = stamp,
     )
+    compile_inputs = depset(extra_link_inputs, transitive = [compile_inputs])
 
     env = dict(ctx.configuration.default_shell_env)
     env.update(env_from_args)
@@ -1179,16 +1181,71 @@ def _get_dir_names(files):
         dirs[f.dirname] = None
     return dirs.keys()
 
-def add_crate_link_flags(args, dep_info, force_all_deps_direct = False):
+def _symlink_transitive_crates_if_needed(ctx, toolchain, crate_info, dep_info):
+    """Collect and symlink the transitive crates into a single directory.
+
+    The reason for this is that when passing a large (350+) number of `-Ldependency=` arguments to rustc on
+    Windows, we seem to be getting some obscure errors such as "failure to call `LoadLibraryExW` that results
+    on some transitive dependencies not being picked up and the build to fail.
+
+    If we detect a large number of transitive dependencies we symlink them all into a single directory that
+    we can pass to rustc in a single `-Ldependency=` argument.
+
+    Args:
+        ctx (ctx): The rule's context object
+        toolchain (rust_toolchain): The current `rust_toolchain`
+        crate_info (CrateInfo): The CrateInfo provider of the target crate
+        dep_info (DepInfo): The current target's dependency info
+
+    Returns:
+        tuple: A tuple of the following items
+            - (File): Optional - the output directory containing all transitive crates that this crate depends on.
+            - (list): The list of transitive crates files that should be used as input to the build action.
+    """
+    if toolchain.os != "windows":
+        return None, []
+
+    deps = dep_info.transitive_crates.to_list()
+
+    # Only symlink if we are about to pass a large (350+) number of transitive dependencies to rustc.
+    if not deps or len(deps) < 350:
+        return None, []
+
+    output_dirname = crate_info.output.basename + ".transitive_crates"
+    links = []
+
+    # Keep a list of the crates that were currently added so that we can uniquify them.
+    names = {}
+
+    for dep in deps:
+        name = dep.output.basename
+        if name in names:
+            continue
+
+        link = ctx.actions.declare_file(output_dirname + "/" + name)
+        ctx.actions.symlink(output = link, target_file = dep.output, is_executable = True)
+
+        names[name] = True
+        links.append(link)
+
+    return links[0].dirname, links
+
+def add_crate_link_flags(ctx, toolchain, args, crate_info, dep_info, force_all_deps_direct = False):
     """Adds link flags to an Args object reference
 
     Args:
+        ctx (ctx): The rule's context object
+        toolchain (rust_toolchain): The current `rust_toolchain`
         args (Args): An arguments object reference
+        crate_info (CrateInfo): The CrateInfo provider for the current target.
         dep_info (DepInfo): The current target's dependency info
+            If this argument is set, only it will be added as a `-Ldependency=` flag, otherwise all rlibs will be set individually.
         force_all_deps_direct (bool, optional): Whether to pass the transitive rlibs with --extern
             to the commandline as opposed to -L.
-    """
 
+    Returns:
+        - (list): A list of extra inputs that should be added to the rustc compile action.
+    """
     if force_all_deps_direct:
         args.add_all(
             depset(
@@ -1203,12 +1260,21 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False):
     else:
         # nb. Direct crates are linked via --extern regardless of their crate_type
         args.add_all(dep_info.direct_crates, map_each = _crate_to_link_flag)
-    args.add_all(
-        dep_info.transitive_crates,
-        map_each = _get_crate_dirname,
-        uniquify = True,
-        format_each = "-Ldependency=%s",
-    )
+
+    transitive_crates_dir, transitive_crates_links = _symlink_transitive_crates_if_needed(ctx, toolchain, crate_info, dep_info)
+
+    # If transitive rlibs have been collected into this single directory, only set this directory
+    if transitive_crates_dir:
+        args.add("-Ldependency={}".format(transitive_crates_dir))
+    else:
+        args.add_all(
+            dep_info.transitive_crates,
+            map_each = _get_crate_dirname,
+            uniquify = True,
+            format_each = "-Ldependency=%s",
+        )
+
+    return transitive_crates_links
 
 def _crate_to_link_flag(crate):
     """A helper macro used by `add_crate_link_flags` for adding crate link flags to a Arg object

rust/private/rustdoc.bzl

@@ -98,7 +98,7 @@ def rustdoc_compile_action(
     # arguments expecting to do so.
     rustdoc_crate_info = _strip_crate_info_output(crate_info)
 
-    args, env = construct_arguments(
+    args, env, extra_link_inputs = construct_arguments(
         ctx = ctx,
         attr = ctx.attr,
         file = ctx.file,
@@ -119,6 +119,7 @@ def rustdoc_compile_action(
         remap_path_prefix = None,
         force_link = True,
     )
+    compile_inputs = depset(extra_link_inputs, transitive = [compile_inputs])
 
     # Because rustdoc tests compile tests outside of the sandbox, the sysroot
     # must be updated to the `short_path` equivilant as it will now be