Enable inline stack probes on X86 with LLVM 16

cuviper · cuviper · commit ed9e6f2ad82a · 2022-09-29T19:49:23.000-07:00
diff --git a/compiler/rustc_target/src/spec/i386_apple_ios.rs b/compiler/rustc_target/src/spec/i386_apple_ios.rs
@@ -14,8 +14,7 @@ pub fn target() -> Target {
         arch: "x86".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-            stack_probes: StackProbeType::Call,
+            stack_probes: StackProbeType::X86,
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/i686_apple_darwin.rs b/compiler/rustc_target/src/spec/i686_apple_darwin.rs
@@ -7,8 +7,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.link_env_remove.to_mut().extend(super::apple_base::macos_link_env_remove());
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.frame_pointer = FramePointer::Always;
 
     // Clang automatically chooses a more specific target based on
diff --git a/compiler/rustc_target/src/spec/i686_linux_android.rs b/compiler/rustc_target/src/spec/i686_linux_android.rs
@@ -11,8 +11,7 @@ pub fn target() -> Target {
     // https://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".into();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".into();
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-linux-android".into(),
diff --git a/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs b/compiler/rustc_target/src/spec/i686_unknown_freebsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-Wl,-znotext"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-freebsd".into(),
diff --git a/compiler/rustc_target/src/spec/i686_unknown_haiku.rs b/compiler/rustc_target/src/spec/i686_unknown_haiku.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-haiku".into(),
diff --git a/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/i686_unknown_linux_gnu.rs
@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.supported_sanitizers = SanitizerSet::ADDRESS;
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/i686_unknown_linux_musl.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-Wl,-melf_i386"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
     // implementation, apparently relies on frame pointers existing... somehow.
diff --git a/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs b/compiler/rustc_target/src/spec/i686_unknown_netbsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-netbsdelf".into(),
diff --git a/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs b/compiler/rustc_target/src/spec/i686_unknown_openbsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "-fuse-ld=lld"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-openbsd".into(),
diff --git a/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs b/compiler/rustc_target/src/spec/i686_wrs_vxworks.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/linux_kernel_base.rs b/compiler/rustc_target/src/spec/linux_kernel_base.rs
@@ -6,8 +6,7 @@ pub fn opts() -> TargetOptions {
         env: "gnu".into(),
         disable_redzone: true,
         panic_strategy: PanicStrategy::Abort,
-        // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-        stack_probes: StackProbeType::Call,
+        stack_probes: StackProbeType::X86,
         frame_pointer: FramePointer::Always,
         position_independent_executables: true,
         needs_plt: true,
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
@@ -635,6 +635,10 @@ pub enum StackProbeType {
 }
 
 impl StackProbeType {
+    // LLVM X86 targets (ix86 and x86_64) can use inline-asm stack probes starting with LLVM 16.
+    // Notable past issues were rust#83139 (fixed in 14) and rust#84667 (fixed in 16).
+    const X86: Self = Self::InlineOrCall { min_llvm_version_for_inline: (16, 0, 0) };
+
     fn from_json(json: &Json) -> Result<Self, String> {
         let object = json.as_object().ok_or_else(|| "expected a JSON object")?;
         let kind = object
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs
@@ -9,8 +9,7 @@ pub fn target() -> Target {
     base.frame_pointer = FramePointer::Always;
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.link_env_remove.to_mut().extend(super::apple_base::macos_link_env_remove());
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_ios.rs b/compiler/rustc_target/src/spec/x86_64_apple_ios.rs
@@ -13,8 +13,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-            stack_probes: StackProbeType::Call,
+            stack_probes: StackProbeType::X86,
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs
@@ -15,8 +15,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-            stack_probes: StackProbeType::Call,
+            stack_probes: StackProbeType::X86,
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_tvos.rs b/compiler/rustc_target/src/spec/x86_64_apple_tvos.rs
@@ -10,8 +10,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-            stack_probes: StackProbeType::Call,
+            stack_probes: StackProbeType::X86,
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/x86_64_apple_watchos_sim.rs
@@ -15,8 +15,7 @@ pub fn target() -> Target {
         arch: "x86_64".into(),
         options: TargetOptions {
             max_atomic_width: Some(64),
-            // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-            stack_probes: StackProbeType::Call,
+            stack_probes: StackProbeType::X86,
             forces_embed_bitcode: true,
             // Taken from a clang build on Xcode 11.4.1.
             // These arguments are not actually invoked - they just have
diff --git a/compiler/rustc_target/src/spec/x86_64_fuchsia.rs b/compiler/rustc_target/src/spec/x86_64_fuchsia.rs
@@ -4,8 +4,7 @@ pub fn target() -> Target {
     let mut base = super::fuchsia_base::opts();
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_linux_android.rs b/compiler/rustc_target/src/spec/x86_64_linux_android.rs
@@ -7,8 +7,7 @@ pub fn target() -> Target {
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-linux-android".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_pc_solaris.rs b/compiler/rustc_target/src/spec/x86_64_pc_solaris.rs
@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.vendor = "pc".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/x86_64_sun_solaris.rs b/compiler/rustc_target/src/spec/x86_64_sun_solaris.rs
@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.vendor = "sun".into();
     base.max_atomic_width = Some(64);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-pc-solaris".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs b/compiler/rustc_target/src/spec/x86_64_unknown_dragonfly.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-unknown-dragonfly".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_haiku.rs b/compiler/rustc_target/src/spec/x86_64_unknown_haiku.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     // This option is required to build executables on Haiku x86_64
     base.position_independent_executables = true;
 
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_hermit.rs b/compiler/rustc_target/src/spec/x86_64_unknown_hermit.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.features = "+rdrnd,+rdseed".into();
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-unknown-hermit".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnux32.rs
@@ -6,8 +6,7 @@ pub fn target() -> Target {
     base.abi = "x32".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mx32"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.has_thread_local = false;
     // BUG(GabrielMajeri): disabling the PLT on x86_64 Linux with x32 ABI
     // breaks code gen. See LLVM bug 36743
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
         | SanitizerSet::LEAK
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_none.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none.rs
@@ -11,8 +11,7 @@ pub fn target() -> Target {
     let opts = TargetOptions {
         cpu: "x86-64".into(),
         max_atomic_width: Some(64),
-        // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-        stack_probes: StackProbeType::Call,
+        stack_probes: StackProbeType::X86,
         position_independent_executables: true,
         static_position_independent_executables: true,
         relro_level: RelroLevel::Full,
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-unknown-openbsd".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs b/compiler/rustc_target/src/spec/x86_64_unknown_redox.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
 
     Target {
         llvm_target: "x86_64-unknown-redox".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/x86_64_wrs_vxworks.rs
@@ -5,8 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
-    // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved
-    base.stack_probes = StackProbeType::Call;
+    base.stack_probes = StackProbeType::X86;
     base.disable_redzone = true;
 
     Target {
diff --git a/src/test/assembly/x86-stack-probes.rs b/src/test/assembly/x86-stack-probes.rs
@@ -0,0 +1,42 @@
+// min-llvm-version: 16
+// revisions: x86_64 i686
+// assembly-output: emit-asm
+//[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//[x86_64] needs-llvm-components: x86
+//[i686] compile-flags: --target i686-unknown-linux-gnu
+//[i686] needs-llvm-components: x86
+// compile-flags: -C llvm-args=-x86-asm-syntax=intel
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+#[lang = "copy"]
+trait Copy {}
+
+impl Copy for u8 {}
+
+// Check that inline-asm stack probes are generated correctly.
+// To avoid making this test fragile to slight asm changes,
+// we only check that the stack pointer is decremented by a page at a time,
+// instead of matching the whole probe sequence.
+
+// CHECK-LABEL: small_stack_probe:
+#[no_mangle]
+pub fn small_stack_probe(x: u8, f: fn(&mut [u8; 8192])) {
+    // CHECK-NOT: __rust_probestack
+    // x86_64: sub rsp, 4096
+    // i686: sub esp, 4096
+    f(&mut [x; 8192]);
+}
+
+// CHECK-LABEL: big_stack_probe:
+#[no_mangle]
+pub fn big_stack_probe(x: u8, f: fn(&[u8; 65536])) {
+    // CHECK-NOT: __rust_probestack
+    // x86_64: sub rsp, 4096
+    // i686: sub esp, 4096
+    f(&mut [x; 65536]);
+}
diff --git a/src/test/codegen/stack-probes-call.rs b/src/test/codegen/stack-probes-call.rs
@@ -5,8 +5,10 @@
 // revisions: i686 x86_64
 //[i686] compile-flags: --target i686-unknown-linux-gnu
 //[i686] needs-llvm-components: x86
+//[i686] ignore-llvm-version: 16 - 99
 //[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
 //[x86_64] needs-llvm-components: x86
+//[x86_64] ignore-llvm-version: 16 - 99
 
 #![crate_type = "rlib"]
 #![feature(no_core, lang_items)]
diff --git a/src/test/codegen/stack-probes-inline.rs b/src/test/codegen/stack-probes-inline.rs
