Fix issues causing the Rustc process wrapper to be built non-determ… (#2216)

This change aims to ensure the process wrapper is built
determinisitically, thus reducing churn in cases where multiple
developers share a bazel remote cache. The shared cache is notable as
the binaries can be deterministically produced for a single user for a
single checkout as some host information is embedded in the compiled
binaries. The host information is stored in the `debug-info` section of
the executable. This can be resolved by stripping it. The other issue of
determinism is `rust_binary_without_process_wrapper` does not resolve
`${pwd}` like the process wrapper does, thus stripping sandbox paths
from rustc outputs. To solve for this a small bash and batch script was
added to account for this behavior. The intent of these scripts is to
use __no__ host executables and instead rely on pure bash and batch to
accomplish this and keep any actions which use this as hermetic as
possible.


The following tables were produced on macOS. Note that `rules_rust_2` is
simply another checkout of `rules_rust` on the same commit

Before the changes in this PR:

| path | rules_rust sha256 | rules_rust_2 sha256 |
| --- | --- | --- |
|
bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/external/rules_rust_tinyjson/libtinyjson-4031717389.rlib
| 60db194b3e98b67cc0702c6b6c48c5bc8fcf7d723f3ece6a7a24a53888158c7e |
3eac0b443ba160e3a1bde3b023f4e953bb9fc9722e430b5ded6dbb723bc2b532 |
|
bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/util/process_wrapper/process_wrapper
| 32e7840602c977b76bba0bc39da768d35db3e7a17dbf96e455727be2b2f0151f |
0d248279bbc9b17be5914b41a66759f939ef4da77f8a40a5ce6fa3bf339648ad |

After the changes:

| path | rules_rust sha256 | rules_rust_2 sha256 |
| --- | --- | --- |
|
bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/external/rules_rust_tinyjson/libtinyjson-4031717389.rlib
| 07432f5d207da854266ccde212243c67c29907f81a7619434ce9f608b1658d96 |
07432f5d207da854266ccde212243c67c29907f81a7619434ce9f608b1658d96 |
|
bazel-out/darwin_arm64-opt-exec-2B5CBBC6/bin/util/process_wrapper/process_wrapper
| cbf17392338aabfc942d975f95a49a67b731c5e597a7d27e3d9cf4d4a06b8f2c |
cbf17392338aabfc942d975f95a49a67b731c5e597a7d27e3d9cf4d4a06b8f2c |

closes #2092

Author: UebelAndre

rust/private/rust.bzl

@@ -1060,6 +1060,13 @@ def _common_attrs_for_binary_without_process_wrapper(attrs):
         cfg = "exec",
     )
 
+    new_attr["_bootstrap_process_wrapper"] = attr.label(
+        default = Label("//util/process_wrapper:bootstrap_process_wrapper"),
+        executable = True,
+        allow_single_file = True,
+        cfg = "exec",
+    )
+
     # fix stamp = 0
     new_attr["stamp"] = attr.int(
         doc = dedent("""\

rust/private/rustc.bzl

@@ -1300,16 +1300,16 @@ def rustc_compile_action(
                 ),
                 toolchain = "@rules_rust//rust:toolchain_type",
             )
-    else:
+    elif hasattr(ctx.executable, "_bootstrap_process_wrapper"):
         # Run without process_wrapper
         if build_env_files or build_flags_files or stamp or build_metadata:
             fail("build_env_files, build_flags_files, stamp, build_metadata are not supported when building without process_wrapper")
         ctx.actions.run(
-            executable = toolchain.rustc,
+            executable = ctx.executable._bootstrap_process_wrapper,
             inputs = compile_inputs,
             outputs = action_outputs,
             env = env,
-            arguments = [args.rustc_flags],
+            arguments = [args.rustc_path, args.rustc_flags],
             mnemonic = "Rustc",
             progress_message = "Compiling Rust (without process_wrapper) {} {}{} ({} files)".format(
                 crate_info.type,
@@ -1319,6 +1319,8 @@ def rustc_compile_action(
             ),
             toolchain = "@rules_rust//rust:toolchain_type",
         )
+    else:
+        fail("No process wrapper was defined for {}".format(ctx.label))
 
     if experimental_use_cc_common_link:
         # Wrap the main `.o` file into a compilation output suitable for

util/process_wrapper/BUILD.bazel

@@ -1,12 +1,45 @@
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
 load("//rust:defs.bzl", "rust_test")
 
 # buildifier: disable=bzl-visibility
 load("//rust/private:rust.bzl", "rust_binary_without_process_wrapper")
 
+config_setting(
+    name = "compilation_mode_opt",
+    values = {"compilation_mode": "opt"},
+)
+
+selects.config_setting_group(
+    name = "opt_linux",
+    match_all = [
+        ":compilation_mode_opt",
+        "@platforms//os:linux",
+    ],
+    visibility = ["@rules_rust_tinyjson//:__pkg__"],
+)
+
+selects.config_setting_group(
+    name = "opt_macos",
+    match_all = [
+        ":compilation_mode_opt",
+        "@platforms//os:macos",
+    ],
+    visibility = ["@rules_rust_tinyjson//:__pkg__"],
+)
+
 rust_binary_without_process_wrapper(
     name = "process_wrapper",
     srcs = glob(["*.rs"]),
     edition = "2018",
+    # To ensure the process wrapper is produced deterministically
+    # debug info, which is known to sometimes have host specific
+    # paths embedded in this section, is stripped out.
+    rustc_flags = select({
+        ":opt_linux": ["-Cstrip=debuginfo"],
+        ":opt_macos": ["-Cstrip=debuginfo"],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
     deps = [
         "@rules_rust_tinyjson//:tinyjson",
@@ -18,3 +51,16 @@ rust_test(
     crate = ":process_wrapper",
     edition = "2018",
 )
+
+native_binary(
+    name = "bootstrap_process_wrapper",
+    src = select({
+        "@platforms//os:windows": "process_wrapper.bat",
+        "//conditions:default": "process_wrapper.sh",
+    }),
+    out = select({
+        "@platforms//os:windows": "process_wrapper.bat",
+        "//conditions:default": "process_wrapper.sh",
+    }),
+    visibility = ["//visibility:public"],
+)

util/process_wrapper/BUILD.tinyjson.bazel

@@ -5,5 +5,13 @@ rust_library_without_process_wrapper(
     name = "tinyjson",
     srcs = glob(["src/*.rs"]),
     edition = "2018",
+    # To ensure the process wrapper is produced deterministically
+    # debug info, which is known to sometimes have host specific
+    # paths embedded in this section, is stripped out.
+    rustc_flags = select({
+        "@rules_rust//util/process_wrapper:opt_linux": ["-Cstrip=debuginfo"],
+        "@rules_rust//util/process_wrapper:opt_macos": ["-Cstrip=debuginfo"],
+        "//conditions:default": [],
+    }),
     visibility = ["@rules_rust//util/process_wrapper:__pkg__"],
 )

util/process_wrapper/process_wrapper.bat

@@ -0,0 +1,31 @@
+@ECHO OFF
+SETLOCAL enabledelayedexpansion
+
+SET command=%*
+
+:: Resolve the `${pwd}` placeholders
+SET command=!command:${pwd}=%CD%!
+
+:: Strip out the leading `--` argument.
+SET command=!command:~3!
+
+:: Find the rustc.exe argument and sanitize it's path
+for %%A in (%*) do (
+    SET arg=%%~A
+    if "!arg:~-9!"=="rustc.exe" (
+        SET sanitized=!arg:/=\!
+
+        SET command=!sanitized! !command:%%~A=!
+        goto :break
+    )
+)
+
+:break
+
+%command%
+
+:: Capture the exit code of rustc.exe
+SET exit_code=!errorlevel!
+
+:: Exit with the same exit code
+EXIT /b %exit_code%

util/process_wrapper/process_wrapper.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Skip the first argument which is expected to be `--`
+shift
+
+args=()
+
+for arg in "$@"; do
+    # Check if the argument contains "${PWD}" and replace it with the actual value of PWD
+    if [[ "${arg}" == *'${pwd}'* ]]; then
+        arg="${arg//\$\{pwd\}/$PWD}"
+    fi
+    args+=("${arg}")
+done
+
+exec "${args[@]}"