Create jdeps output from classpath entries (#1376)

* Create jdeps output from classpath entries

Motivation: jdeps output is used by Intellij Bazel plugin to construct project model. Current implementation adds all entries as explicit source deps, does not do distinguish between implicit or unused deps.

Author: liucijus

scala/private/common_outputs.bzl

@@ -6,4 +6,5 @@ common_outputs = {
     "manifest": "%{name}_MANIFEST.MF",
     "statsfile": "%{name}.statsfile",
     "diagnosticsfile": "%{name}.diagnosticsproto",
+    "jdeps": "%{name}.jdep",
 }

scala/private/phases/phase_compile.bzl

@@ -193,6 +193,7 @@ def _compile_or_empty(
             manifest,
             ctx.outputs.statsfile,
             ctx.outputs.diagnosticsfile,
+            ctx.outputs.jdeps,
             sources,
             jars,
             all_srcjars,
@@ -269,6 +270,7 @@ rm -f {jar_output}
 # ensures that empty src targets still emit a statsfile and a diagnosticsfile
 touch {statsfile}
 touch {diagnosticsfile}
+touch {jdepsfile}
 """ + ijar_cmd
 
     cmd = cmd.format(
@@ -277,9 +279,10 @@ touch {diagnosticsfile}
         zipper = ctx.executable._zipper.path,
         statsfile = ctx.outputs.statsfile.path,
         diagnosticsfile = ctx.outputs.diagnosticsfile.path,
+        jdepsfile = ctx.outputs.jdeps.path,
     )
 
-    outs = [ctx.outputs.jar, ctx.outputs.statsfile, ctx.outputs.diagnosticsfile]
+    outs = [ctx.outputs.jar, ctx.outputs.statsfile, ctx.outputs.diagnosticsfile, ctx.outputs.jdeps]
     inputs = ctx.files.resources + [ctx.outputs.manifest]
 
     ctx.actions.run_shell(
@@ -303,6 +306,7 @@ def _create_scala_compilation_provider(ctx, ijar, source_jar, deps_providers):
         compile_jar = ijar,
         source_jar = source_jar,
         deps = deps_providers,
+        jdeps = ctx.outputs.jdeps,
         exports = exports,
         runtime_deps = runtime_deps,
         neverlink = ctx.attr.neverlink,

scala/private/rule_impls.bzl

@@ -39,6 +39,7 @@ def compile_scala(
         manifest,
         statsfile,
         diagnosticsfile,
+        jdepsPath,
         sources,
         cjars,
         all_srcjars,
@@ -79,6 +80,7 @@ def compile_scala(
     args.add("--Manifest", manifest)
     args.add("--PrintCompileTime", print_compile_time)
     args.add("--ExpectJavaOutput", expect_java_output)
+    args.add("--JDepsFilePath", jdepsPath)
     args.add("--StrictDepsMode", dependency_info.strict_deps_mode)
     args.add("--UnusedDependencyCheckerMode", dependency_info.unused_deps_mode)
     args.add("--DependencyTrackingMethod", dependency_info.dependency_tracking_method)
@@ -109,7 +111,7 @@ def compile_scala(
     if dependency_info.unused_deps_mode != "off":
         args.add_all("--UnusedDepsIgnoredTargets", unused_dependency_checker_ignored_targets)
 
-    outs = [output, statsfile, diagnosticsfile]
+    outs = [output, statsfile, diagnosticsfile, jdepsPath]
 
     ins = depset(
         direct = [manifest] + sources + classpath_resources + resources + resource_jars,

scala_proto/private/scala_proto_aspect.bzl

@@ -89,6 +89,7 @@ def _compile_sources(ctx, toolchain, proto, src_jars, deps, stamp_label):
     write_manifest_file(ctx.actions, manifest, None)
     statsfile = ctx.actions.declare_file(ctx.label.name + "_scalac.statsfile")
     diagnosticsfile = ctx.actions.declare_file(ctx.label.name + "_scalac.diagnosticsproto")
+    jdepsfile = ctx.actions.declare_file(ctx.label.name + ".jdeps")
     compile_deps = deps + _compile_deps(ctx, toolchain)
     merged_deps = java_common.merge(compile_deps)
 
@@ -103,6 +104,7 @@ def _compile_sources(ctx, toolchain, proto, src_jars, deps, stamp_label):
         manifest,
         statsfile,
         diagnosticsfile,
+        jdepsfile,
         sources = [],
         cjars = merged_deps.compile_jars,
         all_srcjars = depset(src_jars),
@@ -127,6 +129,7 @@ def _compile_sources(ctx, toolchain, proto, src_jars, deps, stamp_label):
         output_jar = output,
         compile_jar = output,
         deps = compile_deps,
+        jdeps = jdepsfile,
         exports = compile_deps,
         runtime_deps = compile_deps,
     )

src/java/io/bazel/rulesscala/jdeps/BUILD

@@ -0,0 +1,11 @@
+load("@rules_java//java:defs.bzl", "java_library")
+
+java_library(
+    name = "jdeps",
+    srcs = ["JdepsWriter.java"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@bazel_tools//src/main/protobuf:deps_java_proto",
+        "@com_google_protobuf//:protobuf_java",
+    ],
+)

src/java/io/bazel/rulesscala/jdeps/JdepsWriter.java

@@ -0,0 +1,33 @@
+package io.bazel.rulesscala.jdeps;
+
+import com.google.devtools.build.lib.view.proto.Deps;
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public final class JdepsWriter {
+
+  private JdepsWriter() {
+  }
+
+  public static void write(String jdpesPath, String currentTarget, String[] classpath)
+      throws IOException {
+
+    Deps.Dependencies.Builder builder = Deps.Dependencies.newBuilder();
+    builder.setSuccess(true);
+    builder.setRuleLabel(currentTarget);
+
+    for (String jar : classpath) {
+      Deps.Dependency.Builder dependencyBuilder = Deps.Dependency.newBuilder();
+      dependencyBuilder.setKind(Deps.Dependency.Kind.EXPLICIT);
+      dependencyBuilder.setPath(jar);
+      builder.addDependency(dependencyBuilder.build());
+    }
+
+    try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(jdpesPath))) {
+      builder.build().writeTo(outputStream);
+    }
+  }
+
+}

src/java/io/bazel/rulesscala/scalac/BUILD

@@ -15,6 +15,7 @@ java_binary(
     deps = [
         "//scala/private/toolchain_deps:scala_compile_classpath",
         "//src/java/io/bazel/rulesscala/io_utils",
+        "//src/java/io/bazel/rulesscala/jdeps",
         "//third_party/bazel/src/main/protobuf:worker_protocol_java_proto",
         "@io_bazel_rules_scala//src/java/io/bazel/rulesscala/jar",
         "@io_bazel_rules_scala//src/java/io/bazel/rulesscala/worker",

src/java/io/bazel/rulesscala/scalac/CompileOptions.java

@@ -24,6 +24,7 @@ public class CompileOptions {
   public final String[] unusedDepsIgnoredTargets;
   public final String[] indirectJars;
   public final String[] indirectTargets;
+  public final String jdepsFilePath;
   public final String strictDepsMode;
   public final String unusedDependencyCheckerMode;
   public final String currentTarget;
@@ -60,6 +61,7 @@ public CompileOptions(String[] lines) {
     indirectJars = args.getOrEmpty("IndirectJars");
     indirectTargets = args.getOrEmpty("IndirectTargets");
 
+    jdepsFilePath = args.getSingleOrError("JDepsFilePath");
     strictDepsMode = args.getSingleOrError("StrictDepsMode");
     unusedDependencyCheckerMode = args.getSingleOrError("UnusedDependencyCheckerMode");
     currentTarget = args.getSingleOrError("CurrentTarget");
@@ -75,7 +77,7 @@ public CompileOptions(String[] lines) {
 
   static final class Args {
 
-    private static final String[] EMPTY = new String[] {};
+    private static final String[] EMPTY = new String[]{};
     private final Map<String, String[]> index = new LinkedHashMap<>();
 
     Args(String[] lines) {

src/java/io/bazel/rulesscala/scalac/ScalacWorker.java

@@ -4,9 +4,18 @@
 
 import io.bazel.rulesscala.io_utils.StreamCopy;
 import io.bazel.rulesscala.jar.JarCreator;
+import io.bazel.rulesscala.jdeps.JdepsWriter;
 import io.bazel.rulesscala.worker.Worker;
-import java.io.*;
-import java.nio.file.*;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -61,6 +70,8 @@ public void work(String[] args) throws Exception {
       compileScalaSources(ops, scalaSources, classes);
     }
 
+    JdepsWriter.write(ops.jdepsFilePath, ops.currentTarget, ops.classpath);
+
     /** Copy the resources */
     copyResources(ops.resourceSources, ops.resourceTargets, classes);
 
@@ -95,7 +106,7 @@ private static String[] collectSrcJarSources(
   }
 
   private static List<File> filterFilesByExtension(List<File> files, String extension) {
-    List<File> filtered = new ArrayList<File>();
+    List<File> filtered = new ArrayList<>();
     for (File f : files) {
       if (f.toString().endsWith(extension)) {
         filtered.add(f);
@@ -238,7 +249,7 @@ private static String[] getPluginParamsFrom(CompileOptions ops) {
   }
 
   private static void compileScalaSources(CompileOptions ops, String[] scalaSources, Path classes)
-      throws IllegalAccessException, IOException {
+      throws IOException {
 
     String[] pluginArgs = buildPluginArgs(ops.plugins);
     String[] pluginParams = getPluginParamsFrom(ops);

src/java/io/bazel/rulesscala/scalac/ScalacWorker3.java

@@ -1,6 +1,8 @@
 package io.bazel.rulesscala.scalac;
 
 import static java.io.File.pathSeparator;
+
+import io.bazel.rulesscala.jdeps.JdepsWriter;
 import scala.Tuple2;
 import io.bazel.rulesscala.io_utils.StreamCopy;
 import io.bazel.rulesscala.jar.JarCreator;
@@ -76,6 +78,8 @@ public void work(String[] args) throws Exception {
         compileScalaSources(ops, scalaSources, tmpPath);
       }
 
+      JdepsWriter.write(ops.jdepsFilePath, ops.currentTarget, ops.classpath);
+
       /** Copy the resources */
       copyResources(ops.resourceSources, ops.resourceTargets, tmpPath);
 

test/src/main/java/rulesscala/test/jdeps/BUILD

@@ -0,0 +1,13 @@
+load("@rules_java//java:defs.bzl", "java_test")
+
+java_test(
+    name = "jdeps",
+    srcs = ["JdepsWriterTest.java"],
+    test_class = "jdeps.JdepsWriterTest",
+    deps = [
+        "//src/java/io/bazel/rulesscala/jdeps",
+        "@bazel_tools//src/main/protobuf:deps_java_proto",
+        "@com_google_protobuf//:protobuf_java",
+        "@io_bazel_rules_scala_junit_junit",
+    ],
+)

test/src/main/java/rulesscala/test/jdeps/JdepsWriterTest.java

@@ -0,0 +1,45 @@
+package jdeps;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import com.google.devtools.build.lib.view.proto.Deps;
+import com.google.devtools.build.lib.view.proto.Deps.Dependency;
+import io.bazel.rulesscala.jdeps.JdepsWriter;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.junit.Test;
+
+public class JdepsWriterTest {
+
+  @Test
+  public void writesJarListToJdepsFile() throws IOException {
+    File jdepsPath = Files
+        .createTempDirectory("jdeps-test-tempdir-")
+        .resolve("some-target.jdeps")
+        .toFile();
+
+    String[] classpath = {"foo.jar", "baz/zoo.class"};
+
+    JdepsWriter.write(
+        jdepsPath.getPath(),
+        "some-target",
+        classpath
+    );
+
+    assertArrayEquals(readJarsFromJdepsFile(jdepsPath), classpath);
+  }
+
+  private String[] readJarsFromJdepsFile(File jdepsPath) throws IOException {
+    Deps.Dependencies dependencies = Deps.Dependencies
+        .parseFrom(new BufferedInputStream(new FileInputStream(jdepsPath)));
+
+    return dependencies
+        .getDependencyList()
+        .stream()
+        .map(Dependency::getPath)
+        .toArray(String[]::new);
+  }
+}

twitter_scrooge/twitter_scrooge.bzl

@@ -248,6 +248,12 @@ def _compile_generated_scala(
         label.name + "_scalac.diagnosticsproto",
         sibling = scrooge_jar,
     )
+
+    jdepsFile = ctx.actions.declare_file(
+        label.name + ".jdeps",
+        sibling = scrooge_jar,
+    )
+
     all_deps = _concat_lists(deps_java_info, implicit_deps)
     merged_deps = java_common.merge(all_deps)
 
@@ -262,6 +268,7 @@ def _compile_generated_scala(
         manifest,
         statsfile,
         diagnosticsfile,
+        jdepsFile,
         sources = [],
         cjars = merged_deps.transitive_compile_time_jars,
         all_srcjars = depset([scrooge_jar]),