Auto merge of rust-lang#140064 - EnzymeAD:enable-autodiff-in-ci, r=<try>

[DO NOT MERGE] start building Enzyme in CI

My goal is to put this in CI on April 26, to have a week to land some of the outstanding PRs (removed # in front of it to avoid spamming them every time I do a try build in this PR):
139700 [Landed]
139308 [Landed]
140104 [Landed]
140030 [Landed]
140049 [Almost ready]
140244 [Almost ready]
The autodiff flags PR should land first, but otherwise they don't overlap and are mostly ready, so it shouldn't be too hard to land them. In the meantime, I'll experiment here with some builders.

A fix for 140137 (Apple CI) would be nice, but I'm also happy to start with the working linux (x86-64 + aarch) as the most common target, and expand from there.

r? `@oli-obk`

Tracking:

- 124509

**Linux**
succeeded try with plugins and without plugins & with llvm-enzyme enabled:  dist-x86_64-linux
succeeded try with plugins and without plugins & with llvm-enzyme enabled:  dist-aarch64-linux

**MacOS**
failed try with plugins & llvm-enzyme enabled: dist-x86_64-apple (rust-lang#140064 (comment))
failed try with plugins & llvm-enzyme enabled: dist-apple-various (rust-lang#140064 (comment))
failed try with plugins & llvm-enzyme enabled: dist-aarch64-apple (rust-lang#140064 (comment))

**Windows**
failed try with plugins & llvm-enzyme enabled: dist-x86_64-mingw (rust-lang#140064 (comment))

try-job: dist-aarch64-apple

Author: bors

src/bootstrap/src/core/build_steps/llvm.rs

@@ -970,7 +970,7 @@ impl Step for Enzyme {
             .env("LLVM_CONFIG_REAL", &llvm_config)
             .define("LLVM_ENABLE_ASSERTIONS", "ON")
             .define("ENZYME_EXTERNAL_SHARED_LIB", "ON")
-            .define("LLVM_DIR", builder.llvm_out(target));
+            .define("LLVM_DIR", builder.llvm_out(target).join("build"));
 
         cfg.build();
 

src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile

@@ -89,6 +89,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set llvm.thin-lto=true \
       --set llvm.libzstd=true \
       --set llvm.ninja=false \
+      --set llvm.enzyme=true \
       --set rust.debug-assertions=false \
       --set rust.jemalloc \
       --set rust.use-lld=true \

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -90,6 +90,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
+      --set llvm.enzyme=true \
       --set llvm.libzstd=true \
       --set rust.jemalloc \
       --set rust.use-lld=true \

src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile

@@ -30,4 +30,5 @@ ENV RUST_CONFIGURE_ARGS \
  --enable-profiler \
  --enable-compiler-docs \
  --set llvm.libzstd=true
+ --set llvm.enzyme=true \
 ENV SCRIPT python3 ../x.py --stage 2 test

src/ci/github-actions/jobs.yml

@@ -382,7 +382,7 @@ auto:
   - name: dist-x86_64-apple
     env:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
-      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
+      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set llvm.enzyme=true --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       # Ensure that host tooling is built to support our minimum support macOS version.
       MACOSX_DEPLOYMENT_TARGET: 10.12
@@ -400,7 +400,7 @@ auto:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim,aarch64-apple-ios-macabi,x86_64-apple-ios-macabi
       # Mac Catalyst cannot currently compile the sanitizer:
       # https://github.com/rust-lang/rust/issues/129069
-      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false
+      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.enzyme=true --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       # Ensure that host tooling is built to support our minimum support macOS version.
       # FIXME(madsmtm): This might be redundant, as we're not building host tooling here (?)
@@ -432,6 +432,7 @@ auto:
         --enable-profiler
         --set rust.jemalloc
         --set llvm.ninja=false
+        --set llvm.enzyme=true
         --set rust.lto=thin
         --set rust.codegen-units=1
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
@@ -614,6 +615,7 @@ auto:
       SCRIPT: python x.py dist bootstrap --include-default-paths
       RUST_CONFIGURE_ARGS: >-
         --build=x86_64-pc-windows-gnu
+        --set llvm.enzyme=true
         --enable-full-tools
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift