Skip to content

Commit 5f1813e

Browse files
rampitecrampitec
authored
[AMDGPU] Support true16 spill restore with sram-ecc (#165320)
1 parent 032900e commit 5f1813e

File tree

4 files changed

+348
-1
lines changed

4 files changed

+348
-1
lines changed

‎llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1823,6 +1823,16 @@ void SIRegisterInfo::buildSpillLoadStore(
18231823
}
18241824
}
18251825

1826+
Register FinalValueReg = ValueReg;
1827+
if (LoadStoreOp == AMDGPU::SCRATCH_LOAD_USHORT_SADDR) {
1828+
// If we are loading 16-bit value with SRAMECC endabled we need a temp
1829+
// 32-bit VGPR to load and extract 16-bits into the final register.
1830+
ValueReg =
1831+
RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0);
1832+
SubReg = ValueReg;
1833+
IsKill = false;
1834+
}
1835+
18261836
MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(RegOffset);
18271837
MachineMemOperand *NewMMO =
18281838
MF->getMachineMemOperand(PInfo, MMO->getFlags(), RemEltSize,
@@ -1863,6 +1873,17 @@ void SIRegisterInfo::buildSpillLoadStore(
18631873
MIB.addImm(0); // swz
18641874
MIB.addMemOperand(NewMMO);
18651875

1876+
if (FinalValueReg != ValueReg) {
1877+
// Extract 16-bit from the loaded 32-bit value.
1878+
ValueReg = getSubReg(ValueReg, AMDGPU::lo16);
1879+
MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B16_t16_e64))
1880+
.addReg(FinalValueReg, getDefRegState(true))
1881+
.addImm(0)
1882+
.addReg(ValueReg, getKillRegState(true))
1883+
.addImm(0);
1884+
ValueReg = FinalValueReg;
1885+
}
1886+
18661887
if (!IsAGPR && NeedSuperRegDef)
18671888
MIB.addReg(ValueReg, RegState::ImplicitDefine);
18681889

@@ -2505,7 +2526,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
25052526
unsigned Opc;
25062527
if (MI->getOpcode() == AMDGPU::SI_SPILL_V16_RESTORE) {
25072528
assert(ST.enableFlatScratch() && "Flat Scratch is not enabled!");
2508-
Opc = AMDGPU::SCRATCH_LOAD_SHORT_D16_SADDR_t16;
2529+
Opc = ST.d16PreservesUnusedBits()
2530+
? AMDGPU::SCRATCH_LOAD_SHORT_D16_SADDR_t16
2531+
: AMDGPU::SCRATCH_LOAD_USHORT_SADDR;
25092532
} else {
25102533
Opc = MI->getOpcode() == AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE
25112534
? AMDGPU::SCRATCH_LOAD_BLOCK_SADDR

‎llvm/test/CodeGen/AMDGPU/spill_kill_v16.mir‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
22
# RUN: llc -march=amdgcn -verify-machineinstrs -mcpu=gfx1100 -mattr=+real-true16 -run-pass=prologepilog -o - %s | FileCheck -check-prefix=EXPANDED %s
3+
# RUN: llc -march=amdgcn -verify-machineinstrs -mcpu=gfx1250 -mattr=+real-true16 -run-pass=prologepilog -o - %s | FileCheck -check-prefix=SRAMECC-EXPANDED %s
34

45
---
56
name: spill_restore_vgpr16
@@ -31,6 +32,28 @@ body: |
3132
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
3233
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
3334
; EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
35+
;
36+
; SRAMECC-EXPANDED-LABEL: name: spill_restore_vgpr16
37+
; SRAMECC-EXPANDED: bb.0:
38+
; SRAMECC-EXPANDED-NEXT: successors: %bb.1(0x80000000)
39+
; SRAMECC-EXPANDED-NEXT: {{ $}}
40+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
41+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
42+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
43+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
44+
; SRAMECC-EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
45+
; SRAMECC-EXPANDED-NEXT: {{ $}}
46+
; SRAMECC-EXPANDED-NEXT: bb.1:
47+
; SRAMECC-EXPANDED-NEXT: successors: %bb.2(0x80000000)
48+
; SRAMECC-EXPANDED-NEXT: {{ $}}
49+
; SRAMECC-EXPANDED-NEXT: S_NOP 1
50+
; SRAMECC-EXPANDED-NEXT: {{ $}}
51+
; SRAMECC-EXPANDED-NEXT: bb.2:
52+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
53+
; SRAMECC-EXPANDED-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
54+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
55+
; SRAMECC-EXPANDED-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
56+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
3457
bb.0:
3558
successors: %bb.1(0x80000000)
3659
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
@@ -78,6 +101,29 @@ body: |
78101
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
79102
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
80103
; EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
104+
;
105+
; SRAMECC-EXPANDED-LABEL: name: spill_restore_vgpr16_middle_of_block
106+
; SRAMECC-EXPANDED: bb.0:
107+
; SRAMECC-EXPANDED-NEXT: successors: %bb.1(0x80000000)
108+
; SRAMECC-EXPANDED-NEXT: {{ $}}
109+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
110+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
111+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
112+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
113+
; SRAMECC-EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
114+
; SRAMECC-EXPANDED-NEXT: {{ $}}
115+
; SRAMECC-EXPANDED-NEXT: bb.1:
116+
; SRAMECC-EXPANDED-NEXT: successors: %bb.2(0x80000000)
117+
; SRAMECC-EXPANDED-NEXT: {{ $}}
118+
; SRAMECC-EXPANDED-NEXT: S_NOP 1
119+
; SRAMECC-EXPANDED-NEXT: {{ $}}
120+
; SRAMECC-EXPANDED-NEXT: bb.2:
121+
; SRAMECC-EXPANDED-NEXT: S_NOP 1
122+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
123+
; SRAMECC-EXPANDED-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
124+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
125+
; SRAMECC-EXPANDED-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
126+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
81127
bb.0:
82128
successors: %bb.1(0x80000000)
83129
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
@@ -124,6 +170,27 @@ body: |
124170
; EXPANDED-NEXT: bb.2:
125171
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
126172
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
173+
;
174+
; SRAMECC-EXPANDED-LABEL: name: spill_restore_vgpr16_end_of_block
175+
; SRAMECC-EXPANDED: bb.0:
176+
; SRAMECC-EXPANDED-NEXT: successors: %bb.1(0x80000000)
177+
; SRAMECC-EXPANDED-NEXT: {{ $}}
178+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
179+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
180+
; SRAMECC-EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
181+
; SRAMECC-EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
182+
; SRAMECC-EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
183+
; SRAMECC-EXPANDED-NEXT: {{ $}}
184+
; SRAMECC-EXPANDED-NEXT: bb.1:
185+
; SRAMECC-EXPANDED-NEXT: successors: %bb.2(0x80000000)
186+
; SRAMECC-EXPANDED-NEXT: {{ $}}
187+
; SRAMECC-EXPANDED-NEXT: S_NOP 1
188+
; SRAMECC-EXPANDED-NEXT: {{ $}}
189+
; SRAMECC-EXPANDED-NEXT: bb.2:
190+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
191+
; SRAMECC-EXPANDED-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
192+
; SRAMECC-EXPANDED-NEXT: $vgpr1 = SCRATCH_LOAD_USHORT_SADDR $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
193+
; SRAMECC-EXPANDED-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e64 0, killed $vgpr1_lo16, 0, implicit $exec
127194
bb.0:
128195
successors: %bb.1(0x80000000)
129196
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16

0 commit comments

Comments
 (0)