Support target collections

Author: Chad Norvell

refresh.template.py

@@ -18,6 +18,7 @@
 # Similarly, when upgrading, please search for that MIN_PY= tag.
 
 
+import collections
 import concurrent.futures
 import enum
 import functools  # MIN_PY=3.9: Replace `functools.lru_cache(maxsize=None)` with `functools.cache`.
@@ -1405,30 +1406,26 @@ def main():
         # End:   template filled by Bazel
     ]
 
-    target_file_names = {
+    target_collections = {
         # Begin: template filled by Bazel
-        {target_file_names}
+        {target_collections}
         # End:   template filled by Bazel
     }
 
-    # Associates compilation database file names with lists of compile commands.
-    # __all__ is a special case: It's the "catch all" for any compile commands that aren't
-    # assigned to a specific file. If no targets are assigned to specific files (i.e., if
-    # target_file_names is empty), all compile commands will go into __all__ and end up
-    # in one file.
-    compile_command_sets = {
-        '__all__': []
-    }
+    # Associates lists of compile commands with compile command "collections".
+    # __all__ is a special case: It contains all generated compile commands.
+    # Any other collections defined in `target_collections` will contain only
+    # the compile commands for the targets defined for those collections.
+    compile_command_sets = collections.defaultdict(list)
 
     for (target, flags) in target_flag_pairs:
-        # If the target has a specific file name assigned, put the compile commands in their
-        # own set, to be written to their own file.
-        if target in target_file_names:
-            target_name = target_file_names[target]
-            compile_command_sets[target_name] = list(_get_commands(target, flags))
-        # Otherwise, put them into the main file.
-        else:
-            compile_command_sets['__all__'].extend(_get_commands(target, flags))
+        commands = list(_get_commands(target, flags))
+        # If the target is assigned to any collections, put the compile commands in the compile command sets for those collections.
+        collections_for_target = [collection_name for target_name, collection_name in target_collections.items() if target == target_name]
+        for collection_name in collections_for_target:
+            compile_command_sets[collection_name].extend(commands)
+        # Also put them into the main file.
+        compile_command_sets['__all__'].extend(commands)
 
     if len(compile_command_sets) <= 1 and len(compile_command_sets['__all__']) == 0:
         log_error(""">>> Not (over)writing compile_commands.json, since no commands were extracted and an empty file is of no use.

refresh_compile_commands.bzl

@@ -71,42 +71,67 @@ load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
 def refresh_compile_commands(
         name,
         targets = None,
-        target_file_names = None,
+        target_collections = None,
         out_dir = None,
         exclude_headers = None,
         exclude_external_sources = False,
         **kwargs):  # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
+
+    if not target_collections:
+        target_collections = {}
+    
+    target_collections = {
+        _make_label_absolute(target): collection_name for target, collection_name in target_collections.items()
+    }
+
+    # We want a list of targets specified in `target_collections`, and we want a list of all of the target collections.
+    # There's a many-to-many relationship between targets and target collections, there can be duplicates in both of those lists.
+    # We want to de-duplicate those lists, but Starlark doesn't have a set class like Python does, so we do it manually.
+    target_collection_names = []
+    target_collection_targets = []
+
+    for target, collection_name in target_collections.items():
+        if not target in target_collection_targets:
+            target_collection_targets.append(target)
+
+        if not collection_name in target_collection_names:
+            target_collection_names.append(collection_name)
+    
+
+
     # Convert the various, acceptable target shorthands into the dictionary format
     # In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
-    if not targets:  # Default to all targets in main workspace
+    if not targets and len(target_collection_targets) == 0:  # Default to all targets in main workspace
         targets = {"@//...": ""}
+    if not targets:  # In this case, targets were defined only in `target_collections`
+        targets = {target: "" for target in target_collection_targets}
     elif type(targets) == "select":  # Allow select: https://bazel.build/reference/be/functions#select
         # Pass select() to _expand_template to make it work
         # see https://bazel.build/docs/configurable-attributes#faq-select-macro
         pass
     elif type(targets) == "list":  # Allow specifying a list of targets w/o arguments
-        targets = {target: "" for target in targets}
+        # The reason we want the list of target collection targets is that if you specify a target in `target_collections`, you don't have to redundantly specify it in `targets`, *unless* you want to specify build flags too.
+        # So we want to cull the list of target collection targets down to only those that *aren't* specified in `targets`.
+        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
+        targets = {target: "" for target in (targets + unique_target_collection_targets)}
     elif type(targets) != "dict":  # Assume they've supplied a single string/label and wrap it
-        targets = {targets: ""}
+        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
+        targets = {target: "" for target in ([targets] + unique_target_collection_targets)}
+    else:  # Assume that they've provided a dict of targets with flags
+        unique_target_collection_targets = [target for target in target_collection_targets if target not in targets]
+        targets = targets | {target: "" for target in target_collection_targets}
 
     targets = {
         _make_label_absolute(target): flags for target, flags in targets.items()
     }
 
-    if not target_file_names:
-        target_file_names = {}
-    
-    target_file_names = {
-        _make_label_absolute(target): file_names for target, file_names in target_file_names.items()
-    }
-
     # Create a wrapper script that prints a helpful error message if the python version is too old, generated from check_python_version.template.py
     version_checker_script_name = name + ".check_python_version.py"
     _check_python_version(name = version_checker_script_name, to_run = name)
 
     # Generate the core, runnable python script from refresh.template.py
     script_name = name + ".py"
-    _expand_template(name = script_name, labels_to_flags = targets, labels_to_file_names = target_file_names, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
+    _expand_template(name = script_name, labels_to_flags = targets, labels_to_collections = target_collections, out_dir = out_dir, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
 
     # Combine them so the wrapper calls the main script
     native.py_binary(
@@ -132,7 +157,7 @@ def _expand_template_impl(ctx):
         substitutions = {
             # Note, don't delete whitespace. Correctly doing multiline indenting.
             "        {target_flag_pairs}": "\n".join(["        {},".format(pair) for pair in ctx.attr.labels_to_flags.items()]),
-            "        {target_file_names}": "\n".join(["        '{}': '{}',".format(target, file_name) for (target, file_name) in ctx.attr.labels_to_file_names.items()]),
+            "        {target_collections}": "\n".join(["        '{}': '{}',".format(target, collection_name) for (target, collection_name) in ctx.attr.labels_to_collections.items()]),
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
             "{out_dir}": repr(ctx.attr.out_dir),
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
@@ -145,7 +170,7 @@ def _expand_template_impl(ctx):
 _expand_template = rule(
     attrs = {
         "labels_to_flags": attr.string_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
-        "labels_to_file_names": attr.string_dict(),
+        "labels_to_collections": attr.string_dict(),
         "out_dir": attr.string(default = "."),
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0