[X86] Don't use rbp when it's reserved

[brandtbucher]

This fixes the x86 backend to properly reserve rbp when the frame-pointer=reserved option is used. Currently, this option is ignored.

Disassembly of the new test case before:

pushq %rbp
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %ebp
callq bar@PLT
movl %ebp, %edi
movl %ebx, %esi
callq bar@PLT
addq $8, %rsp
popq %rbx
popq %rbp
retq

...and after (r14 is used as scratch space in place of rbp):

pushq %r14
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %r14d
callq bar@PLT
movl %r14d, %edi
movl %ebx, %esi
callq bar@PLT
addq $8, %rsp
popq %rbx
popq %r14
retq

Fixes #117178.

[llvmbot]

@llvm/pr-subscribers-backend-x86

Author: Brandt Bucher (brandtbucher)

Changes

This fixes the x86 backend to properly reserve rbp when the frame-pointer=reserved option is used. Currently, this option is ignored.

Disassembly of the new test case before:

pushq %rbp
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %ebp
callq bar@<!-- -->PLT
movl %ebp, %edi
movl %ebx, %esi
callq bar@<!-- -->PLT
addq $8, %rsp
popq %rbx
popq %rbp
retq

...and after (r14 is used as scratch space in place of rbp):

pushq %r14
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %r14d
callq bar@<!-- -->PLT
movl %r14d, %edi
movl %ebx, %esi
callq bar@<!-- -->PLT
addq $8, %rsp
popq %rbx
popq %r14
retq

Fixes #117178.

---

Full diff: https://github.com/llvm/llvm-project/pull/146638.diff

2 Files Affected:

• (modified) llvm/lib/Target/X86/X86RegisterInfo.cpp (+1-1)
• (added) llvm/test/CodeGen/X86/frame-pointer-reserved.ll (+25)

diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 9adac067aace8..83b11eede829e 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -562,7 +562,7 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(SubReg);
 
   // Set the frame-pointer register and its aliases as reserved if needed.
-  if (TFI->hasFP(MF)) {
+  if (TFI->hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF)) {
     if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
       MF.getContext().reportError(
           SMLoc(),
diff --git a/llvm/test/CodeGen/X86/frame-pointer-reserved.ll b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
new file mode 100644
index 0000000000000..bf5887fe50508
--- /dev/null
+++ b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @bar(i32, i32)
+
+define void @foo(i32 %0, i32 %1) nounwind "frame-pointer"="reserved" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+  call void @bar(i32 %0, i32 %1)
+  call void @bar(i32 %0, i32 %1)
+  ret void
+}

[phoebewang]

LGTM.