Skip to content

[InstCombine] Merge one-use GEP offsets during expansion #147263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+98 −38
Open

[InstCombine] Merge one-use GEP offsets during expansion #147263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eebf5a8
[InstCombine] Merge one-use GEP offsets during expansion
nikic Jul 7, 2025
5530e1f
Fix insert point handling
nikic Jul 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 55 additions & 9 deletions llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,18 +219,64 @@ Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
Value *InstCombinerImpl::EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs,
GEPNoWrapFlags NW, Type *IdxTy,
bool RewriteGEPs) {
Value *Sum = nullptr;
for (GEPOperator *GEP : reverse(GEPs)) {
Value *Offset = EmitGEPOffset(GEP, RewriteGEPs);
if (Offset->getType() != IdxTy)
Offset = Builder.CreateVectorSplat(
cast<VectorType>(IdxTy)->getElementCount(), Offset);
auto Add = [&](Value *Sum, Value *Offset) -> Value * {
if (Sum)
Sum = Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
NW.isInBounds());
return Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
NW.isInBounds());
else
Sum = Offset;
return Offset;
};

Value *Sum = nullptr;
Value *OneUseSum = nullptr;
Value *OneUseBase = nullptr;
GEPNoWrapFlags OneUseFlags = GEPNoWrapFlags::all();
for (GEPOperator *GEP : reverse(GEPs)) {
Value *Offset;
{
// Expand the offset at the point of the previous GEP to enable rewriting.
// However, use the original insertion point for calculating Sum.
IRBuilderBase::InsertPointGuard Guard(Builder);
auto *Inst = dyn_cast<Instruction>(GEP);
if (RewriteGEPs && Inst)
Builder.SetInsertPoint(Inst);

Offset = llvm::emitGEPOffset(&Builder, DL, GEP);
if (Offset->getType() != IdxTy)
Offset = Builder.CreateVectorSplat(
cast<VectorType>(IdxTy)->getElementCount(), Offset);
if (GEP->hasOneUse()) {
// Offsets of one-use GEPs will be merged into the next multi-use GEP.
OneUseSum = Add(OneUseSum, Offset);
OneUseFlags = OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags());
if (!OneUseBase)
OneUseBase = GEP->getPointerOperand();
continue;
}

if (OneUseSum)
Offset = Add(OneUseSum, Offset);

// Rewrite the GEP to reuse the computed offset. This also includes
// offsets from preceding one-use GEPs.
if (RewriteGEPs && Inst &&
!(GEP->getSourceElementType()->isIntegerTy(8) &&
GEP->getOperand(1) == Offset)) {
replaceInstUsesWith(
*Inst,
Builder.CreatePtrAdd(
OneUseBase ? OneUseBase : GEP->getPointerOperand(), Offset, "",
OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags())));
eraseInstFromFunction(*Inst);
}
}

Sum = Add(Sum, Offset);
OneUseSum = OneUseBase = nullptr;
OneUseFlags = GEPNoWrapFlags::all();
}
if (OneUseSum)
Sum = Add(Sum, OneUseSum);
if (!Sum)
return Constant::getNullValue(IdxTy);
return Sum;
Expand Down
6 changes: 3 additions & 3 deletions llvm/test/Transforms/InstCombine/getelementptr.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -682,15 +682,15 @@ define i32 @test28() nounwind {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ORIENTATIONS:%.*]] = alloca [1 x [1 x %struct.x]], align 8
; CHECK-NEXT: [[T3:%.*]] = call i32 @puts(ptr noundef nonnull dereferenceable(1) @.str) #[[ATTR0]]
; CHECK-NEXT: [[T45:%.*]] = getelementptr inbounds nuw i8, ptr [[ORIENTATIONS]], i64 1
; CHECK-NEXT: br label [[BB10:%.*]]
; CHECK: bb10:
; CHECK-NEXT: [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
; CHECK-NEXT: [[T12_REC:%.*]] = xor i32 [[INDVAR]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[T12_REC]] to i64
; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i8, ptr [[T45]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i8, ptr [[ORIENTATIONS]], i64 [[TMP1]]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This illustrates a regression from this approach: We pull an addition into the loop.

(I think it we allowed folding of multi-use x+c==c2 this particular case would ultimately fold away.)

; CHECK-NEXT: [[T16:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str1, ptr nonnull [[T12]]) #[[ATTR0]]
; CHECK-NEXT: [[T84:%.*]] = icmp eq i32 [[INDVAR]], 0
; CHECK-NEXT: [[T84:%.*]] = icmp eq i64 [[TMP1]], 0
; CHECK-NEXT: [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
; CHECK-NEXT: br i1 [[T84]], label [[BB17:%.*]], label [[BB10]]
; CHECK: bb17:
Expand Down
66 changes: 40 additions & 26 deletions llvm/test/Transforms/InstCombine/sub-gep.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -945,19 +945,15 @@ define i64 @multiple_geps_two_chains_gep_base(ptr %base, i64 %base.idx, i64 %idx

define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4) {
; CHECK-LABEL: @multiple_geps_two_chains_multi_use(
; CHECK-NEXT: [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
; CHECK-NEXT: [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
; CHECK-NEXT: [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P3_IDX]]
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[P4_IDX1]]
; CHECK-NEXT: [[P1_IDX1:%.*]] = add i64 [[IDX1:%.*]], [[IDX2:%.*]]
; CHECK-NEXT: [[P4_IDX:%.*]] = shl i64 [[P1_IDX1]], 2
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 [[P4_IDX]]
; CHECK-NEXT: [[P3_IDX2:%.*]] = add i64 [[IDX3:%.*]], [[IDX4:%.*]]
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl i64 [[P3_IDX2]], 2
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX1]]
; CHECK-NEXT: call void @use(ptr [[P5]])
; CHECK-NEXT: call void @use(ptr [[P4]])
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[P3_IDX]], [[P4_IDX1]]
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[P4_IDX]], [[P4_IDX1]]
; CHECK-NEXT: ret i64 [[GEPDIFF]]
;
%p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
Expand All @@ -974,23 +970,18 @@ define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2,

define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4, i64 %idx5, i64 %idx6) {
; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use(
; CHECK-NEXT: [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
; CHECK-NEXT: [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
; CHECK-NEXT: [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl nsw i64 [[IDX7:%.*]], 2
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P4_IDX1]]
; CHECK-NEXT: [[P5_IDX:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P5]], i64 [[P5_IDX]]
; CHECK-NEXT: [[P6_IDX:%.*]] = shl nsw i64 [[IDX6:%.*]], 2
; CHECK-NEXT: [[P1_IDX1:%.*]] = add i64 [[IDX1:%.*]], [[IDX2:%.*]]
; CHECK-NEXT: [[P4_IDX:%.*]] = shl i64 [[P1_IDX1]], 2
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 [[P4_IDX]]
; CHECK-NEXT: [[P4_IDX2:%.*]] = add i64 [[IDX4:%.*]], [[IDX5:%.*]]
; CHECK-NEXT: [[P5_IDX:%.*]] = shl i64 [[P4_IDX2]], 2
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P5_IDX]]
; CHECK-NEXT: call void @use(ptr [[P3]])
; CHECK-NEXT: call void @use(ptr [[P4]])
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[TMP1]], [[P3_IDX]]
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[P4_IDX1]], [[P5_IDX]]
; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[P6_IDX]]
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[TMP2]], [[TMP4]]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[P1_IDX1]], [[IDX3:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[P4_IDX2]], [[IDX6:%.*]]
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[GEPDIFF:%.*]] = shl i64 [[TMP5]], 2
; CHECK-NEXT: ret i64 [[GEPDIFF]]
;
%p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
Expand All @@ -1007,6 +998,29 @@ define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64
ret i64 %d
}

define i64 @multiple_geps_two_chains_partial_multi_use_insert_point(ptr %p, i64 %idx1, i64 %idx2, i64 %idx3) {
; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use_insert_point(
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 8
; CHECK-NEXT: call void @use(ptr [[GEP2]])
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[IDX2:%.*]], [[IDX3:%.*]]
; CHECK-NEXT: [[GEP4:%.*]] = getelementptr i8, ptr [[GEP2]], i64 [[TMP1]]
; CHECK-NEXT: call void @use(ptr [[GEP4]])
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 8
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub i64 [[IDX1:%.*]], [[TMP2]]
; CHECK-NEXT: ret i64 [[GEPDIFF]]
;
%gep1 = getelementptr i8, ptr %p, i64 %idx1
%gep2 = getelementptr i8, ptr %p, i64 8
call void @use(ptr %gep2)
%gep3 = getelementptr i8, ptr %gep2, i64 %idx2
%gep4 = getelementptr i8, ptr %gep3, i64 %idx3
call void @use(ptr %gep4)
%gep1.int = ptrtoint ptr %gep1 to i64
%gep4.int = ptrtoint ptr %gep4 to i64
%sub = sub i64 %gep1.int, %gep4.int
ret i64 %sub
}

define i64 @multiple_geps_inbounds(ptr %base, i64 %idx, i64 %idx2) {
; CHECK-LABEL: @multiple_geps_inbounds(
; CHECK-NEXT: [[D:%.*]] = add nsw i64 [[IDX:%.*]], [[IDX2:%.*]]
Expand Down
Loading