diff --git a/llvm/lib/Target/EVM/CMakeLists.txt b/llvm/lib/Target/EVM/CMakeLists.txt index 7925a6fd9385..18c208f185c1 100644 --- a/llvm/lib/Target/EVM/CMakeLists.txt +++ b/llvm/lib/Target/EVM/CMakeLists.txt @@ -23,7 +23,6 @@ add_llvm_target(EVMCodeGen EVMBackwardPropagationStackification.cpp EVMCodegenPrepare.cpp EVMFrameLowering.cpp - EVMHelperUtilities.cpp EVMISelDAGToDAG.cpp EVMISelLowering.cpp EVMInstrInfo.cpp diff --git a/llvm/lib/Target/EVM/EVMBackwardPropagationStackification.cpp b/llvm/lib/Target/EVM/EVMBackwardPropagationStackification.cpp index 3c26c080b2db..265cc0a5eecd 100644 --- a/llvm/lib/Target/EVM/EVMBackwardPropagationStackification.cpp +++ b/llvm/lib/Target/EVM/EVMBackwardPropagationStackification.cpp @@ -94,7 +94,7 @@ bool EVMBPStackification::runOnMachineFunction(MachineFunction &MF) { EVMMachineCFGInfo CFGInfo(MF, MLI); EVMStackModel StackModel(MF, LIS); std::unique_ptr Layout = - EVMStackLayoutGenerator(MF, StackModel, CFGInfo).run(); + EVMStackLayoutGenerator(MF, MLI, StackModel, CFGInfo).run(); EVMStackifyCodeEmitter(*Layout, StackModel, CFGInfo, MF).run(); return true; } diff --git a/llvm/lib/Target/EVM/EVMHelperUtilities.cpp b/llvm/lib/Target/EVM/EVMHelperUtilities.cpp deleted file mode 100644 index 46cfac9b5588..000000000000 --- a/llvm/lib/Target/EVM/EVMHelperUtilities.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//===----------- EVMHelperUtilities.cpp - Helper utilities ------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// -//===----------------------------------------------------------------------===// - -#include "EVMHelperUtilities.h" -#include "MCTargetDesc/EVMMCTargetDesc.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/IR/Function.h" - -using namespace llvm; - -// Return whether the function of the call instruction will return. -bool EVMUtils::callWillReturn(const MachineInstr *Call) { - assert(Call->getOpcode() == EVM::FCALL && "Unexpected call instruction"); - const MachineOperand *FuncOp = Call->explicit_uses().begin(); - assert(FuncOp->isGlobal() && "Expected a global value"); - const auto *Func = dyn_cast(FuncOp->getGlobal()); - assert(Func && "Expected a function"); - return !Func->hasFnAttribute(Attribute::NoReturn); -} diff --git a/llvm/lib/Target/EVM/EVMHelperUtilities.h b/llvm/lib/Target/EVM/EVMHelperUtilities.h deleted file mode 100644 index 27c9ff54999d..000000000000 --- a/llvm/lib/Target/EVM/EVMHelperUtilities.h +++ /dev/null @@ -1,110 +0,0 @@ -//===----------- EVMHelperUtilities.h - Helper utilities --------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines different utility templates. In particular, for a partial -// emulation of the C++ range-V3 library functionality; BFS graph traversal; -// the overload pattern support. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H -#define LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H - -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/iterator_range.h" -#include -#include -#include - -namespace llvm { - -class MachineInstr; - -template struct Overload : Ts... { - using Ts::operator()...; -}; -template Overload(Ts...) -> Overload; - -namespace EVMUtils { -bool callWillReturn(const MachineInstr *Call); - -/// Return a vector, containing sequentially increasing values from \p Begin -/// to \p End. -template static inline SmallVector iota(T Begin, T End) { - SmallVector R(End - Begin); - std::iota(R.begin(), R.end(), Begin); - return R; -} - -/// Return the number of hops from the beginning of the \p RangeOrContainer -/// to the \p Item. If no \p Item is found in the \p RangeOrContainer, -/// std::nullopt is returned. -template -std::optional offset(T &&RangeOrContainer, V &&Item) { - auto It = find(RangeOrContainer, Item); - return (It == adl_end(RangeOrContainer)) - ? std::nullopt - : std::optional(std::distance(adl_begin(RangeOrContainer), It)); -} - -/// Return a range covering the last N elements of \p RangeOrContainer. -template auto take_back(T &&RangeOrContainer, size_t N = 1) { - return make_range(std::prev(adl_end(RangeOrContainer), N), - adl_end(RangeOrContainer)); -} - -/// Generic breadth first search. -/// -/// Note that V needs to be a comparable value type or a pointer. -/// -/// Example: Gather all (recursive) children in a graph starting at (and -/// including) ``root``: -/// -/// Node const* root = ...; -/// std::set allNodes = BreadthFirstSearch{{root}}.run([](Node const* _node, auto&& _addChild) { -/// // Potentially process ``_node``. -/// for (Node const& _child: _node->children()) -/// // Potentially filter the children to be visited. -/// _addChild(&_child); -/// }).visited; -/// -template struct BreadthFirstSearch { - /// Runs the breadth first search. The verticesToTraverse member of the struct - /// needs to be initialized. - /// @param _forEachChild is a callable of the form [...](V const& _node, - /// auto&& _addChild) { ... } that is called for each visited node and is - /// supposed to call _addChild(childNode) for every child node of _node. - template - BreadthFirstSearch &run(ForEachChild &&forEachChild) { - while (!verticesToTraverse.empty()) { - V v = std::move(verticesToTraverse.front()); - verticesToTraverse.pop_front(); - - if (!visited.insert(v).second) - continue; - - forEachChild(v, [this](V vertex) { - verticesToTraverse.emplace_back(std::move(vertex)); - }); - } - return *this; - } - - void abort() { verticesToTraverse.clear(); } - - std::deque verticesToTraverse; - DenseSet visited{}; -}; - -} // namespace EVMUtils -} // namespace llvm - -#endif // LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H diff --git a/llvm/lib/Target/EVM/EVMMachineCFGInfo.cpp b/llvm/lib/Target/EVM/EVMMachineCFGInfo.cpp index c6f45aa65909..8a2ad98da818 100644 --- a/llvm/lib/Target/EVM/EVMMachineCFGInfo.cpp +++ b/llvm/lib/Target/EVM/EVMMachineCFGInfo.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "EVMMachineCFGInfo.h" -#include "EVMHelperUtilities.h" #include "EVMMachineFunctionInfo.h" #include "EVMSubtarget.h" #include "MCTargetDesc/EVMMCTargetDesc.h" @@ -98,9 +97,6 @@ void EVMMachineCFGInfo::collectTerminatorsInfo(const TargetInstrInfo *TII, return; } - bool IsLatch = false; - if (MachineLoop *ML = MLI->getLoopFor(&MBB)) - IsLatch = ML->isLoopLatch(&MBB); auto [CondBr, UncondBr] = getBranchInstructions(MBB); if (!TBB || (TBB && Cond.empty())) { // Fall through, or unconditional jump. @@ -111,10 +107,10 @@ void EVMMachineCFGInfo::collectTerminatorsInfo(const TargetInstrInfo *TII, TBB = MBB.getFallThrough(); assert(TBB); } + Info->ExitType = MBBExitType::UnconditionalBranch; - Info->BranchInfo.Unconditional = {TBB, UncondBr, IsLatch}; + Info->BranchInfo.Unconditional = {TBB, UncondBr}; } else if (TBB && !Cond.empty()) { - assert(!IsLatch); // Conditional jump + fallthrough, or // conditional jump followed by unconditional jump). if (!FBB) { diff --git a/llvm/lib/Target/EVM/EVMMachineCFGInfo.h b/llvm/lib/Target/EVM/EVMMachineCFGInfo.h index 6792765ee8ff..fdb97ae1785e 100644 --- a/llvm/lib/Target/EVM/EVMMachineCFGInfo.h +++ b/llvm/lib/Target/EVM/EVMMachineCFGInfo.h @@ -51,7 +51,6 @@ class EVMMBBTerminatorsInfo { struct { MachineBasicBlock *TargetBB; MachineInstr *Br; - bool IsLatch; } Unconditional; } BranchInfo; @@ -73,11 +72,10 @@ class EVMMBBTerminatorsInfo { BranchInfo.Conditional.Condition}; } - std::tuple + std::pair getUnconditionalBranch() const { assert(ExitType == MBBExitType::UnconditionalBranch); - return {BranchInfo.Unconditional.Br, BranchInfo.Unconditional.TargetBB, - BranchInfo.Unconditional.IsLatch}; + return {BranchInfo.Unconditional.Br, BranchInfo.Unconditional.TargetBB}; } MachineInstr *getFunctionReturn() const { diff --git a/llvm/lib/Target/EVM/EVMStackDebug.cpp b/llvm/lib/Target/EVM/EVMStackDebug.cpp index 0752c768c9df..6e13e145fb52 100644 --- a/llvm/lib/Target/EVM/EVMStackDebug.cpp +++ b/llvm/lib/Target/EVM/EVMStackDebug.cpp @@ -12,7 +12,6 @@ //===----------------------------------------------------------------------===// #include "EVMStackDebug.h" -#include "EVMHelperUtilities.h" #include "EVMStackLayoutGenerator.h" #include "EVMSubtarget.h" #include "MCTargetDesc/EVMMCTargetDesc.h" @@ -20,6 +19,11 @@ using namespace llvm; +template struct Overload : Ts... { + using Ts::operator()...; +}; +template Overload(Ts...) -> Overload; + static std::string getInstName(const MachineInstr *MI) { const MachineFunction *MF = MI->getParent()->getParent(); const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); @@ -140,11 +144,9 @@ void StackLayoutPrinter::printBlock(MachineBasicBlock const &Block) { const EVMMBBTerminatorsInfo *TermInfo = CFGInfo.getTerminatorsInfo(&Block); MBBExitType ExitType = TermInfo->getExitType(); if (ExitType == MBBExitType::UnconditionalBranch) { - auto [BranchInstr, Target, IsLatch] = TermInfo->getUnconditionalBranch(); + auto [BranchInstr, Target] = TermInfo->getUnconditionalBranch(); OS << "Block" << getBlockId(Block) << "Exit [label=\""; OS << "Jump\"];\n"; - if (IsLatch) - OS << "Backwards"; OS << "Block" << getBlockId(Block) << "Exit -> Block" << getBlockId(*Target) << ";\n"; } else if (ExitType == MBBExitType::ConditionalBranch) { diff --git a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp index b5e5961e65da..26afd5114633 100644 --- a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp +++ b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp @@ -16,11 +16,11 @@ //===----------------------------------------------------------------------===// #include "EVMStackLayoutGenerator.h" -#include "EVMHelperUtilities.h" #include "EVMRegisterInfo.h" #include "EVMStackDebug.h" #include "EVMStackShuffler.h" #include "MCTargetDesc/EVMMCTargetDesc.h" +#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/IR/DebugLoc.h" @@ -32,6 +32,28 @@ using namespace llvm; #define DEBUG_TYPE "evm-stack-layout-generator" namespace { +template struct Overload : Ts... { + using Ts::operator()...; +}; +template Overload(Ts...) -> Overload; + +/// Return the number of hops from the beginning of the \p RangeOrContainer +/// to the \p Item. If no \p Item is found in the \p RangeOrContainer, +/// std::nullopt is returned. +template +std::optional offset(T &&RangeOrContainer, V &&Item) { + auto It = find(RangeOrContainer, Item); + return (It == adl_end(RangeOrContainer)) + ? std::nullopt + : std::optional(std::distance(adl_begin(RangeOrContainer), It)); +} + +/// Return a range covering the last N elements of \p RangeOrContainer. +template auto take_back(T &&RangeOrContainer, size_t N = 1) { + return make_range(std::prev(adl_end(RangeOrContainer), N), + adl_end(RangeOrContainer)); +} + /// Returns all stack too deep errors that would occur when shuffling \p Source /// to \p Target. SmallVector @@ -53,18 +75,17 @@ findStackTooDeep(Stack const &Source, Stack const &Target) { [&](unsigned I) { if (I > 16) Errors.emplace_back(EVMStackLayoutGenerator::StackTooDeep{ - I - 16, - getVariableChoices(EVMUtils::take_back(CurrentStack, I + 1))}); + I - 16, getVariableChoices(take_back(CurrentStack, I + 1))}); }, [&](StackSlot const &Slot) { if (isRematerializable(Slot)) return; - if (auto Depth = EVMUtils::offset(reverse(CurrentStack), Slot); + if (auto Depth = offset(reverse(CurrentStack), Slot); Depth && *Depth >= 16) Errors.emplace_back(EVMStackLayoutGenerator::StackTooDeep{ - *Depth - 15, getVariableChoices( - EVMUtils::take_back(CurrentStack, *Depth + 1))}); + *Depth - 15, + getVariableChoices(take_back(CurrentStack, *Depth + 1))}); }, [&]() {}); return Errors; @@ -223,12 +244,12 @@ Stack createIdealLayout(const Stack &OperationOutput, const Stack &Post, } // end anonymous namespace EVMStackLayoutGenerator::EVMStackLayoutGenerator( - const MachineFunction &MF, const EVMStackModel &StackModel, - const EVMMachineCFGInfo &CFGInfo) - : MF(MF), StackModel(StackModel), CFGInfo(CFGInfo) {} + const MachineFunction &MF, const MachineLoopInfo *MLI, + const EVMStackModel &StackModel, const EVMMachineCFGInfo &CFGInfo) + : MF(MF), MLI(MLI), StackModel(StackModel), CFGInfo(CFGInfo) {} std::unique_ptr EVMStackLayoutGenerator::run() { - processEntryPoint(&MF.front()); + runPropagation(); auto Layout = std::make_unique( MBBEntryLayoutMap, MBBExitLayoutMap, OperationEntryLayoutMap); @@ -283,8 +304,8 @@ Stack EVMStackLayoutGenerator::propagateStackThroughOperation( while (!IdealStack.empty()) { if (isRematerializable(IdealStack.back())) IdealStack.pop_back(); - else if (auto Offset = EVMUtils::offset(drop_begin(reverse(IdealStack), 1), - IdealStack.back())) { + else if (auto Offset = offset(drop_begin(reverse(IdealStack), 1), + IdealStack.back())) { if (*Offset + 2 < 16) IdealStack.pop_back(); else @@ -312,15 +333,49 @@ Stack EVMStackLayoutGenerator::propagateStackThroughBlock( return CurrentStack; } -void EVMStackLayoutGenerator::processEntryPoint( - const MachineBasicBlock *Entry) { - std::list ToVisit{Entry}; +// Returns the number of junk slots to be prepended to \p TargetLayout for +// an optimal transition from \p EntryLayout to \p TargetLayout. +static size_t getOptimalNumberOfJunks(const Stack &EntryLayout, + const Stack &TargetLayout) { + size_t BestCost = EvaluateStackTransform(EntryLayout, TargetLayout); + size_t BestNumJunk = 0; + size_t MaxJunk = EntryLayout.size(); + for (size_t NumJunk = 1; NumJunk <= MaxJunk; ++NumJunk) { + Stack JunkedTarget(NumJunk, JunkSlot{}); + JunkedTarget.append(TargetLayout); + size_t Cost = EvaluateStackTransform(EntryLayout, JunkedTarget); + if (Cost < BestCost) { + BestCost = Cost; + BestNumJunk = NumJunk; + } + } + return BestNumJunk; +} + +void EVMStackLayoutGenerator::runPropagation() { + std::deque ToVisit{&MF.front()}; DenseSet Visited; - // TODO: check whether visiting only a subset of these in the outer iteration - // below is enough. - std::list> - BackwardsJumps = collectBackwardsJumps(Entry); + // Collect all the backedges in the MF. + // TODO: CPR-1847. Consider changing CFG before the stackification such that + // every loop has only one backedge. + SmallVector, + 64> + Backedges; + for (const MachineLoop *TopLevelLoop : *MLI) { + // TODO: CPR-1847. Investigate in which order it's better to traverse + // loops. + for (const MachineLoop *L : depth_first(TopLevelLoop)) { + SmallVector Latches; + L->getLoopLatches(Latches); + const MachineBasicBlock *Header = L->getHeader(); + transform(Latches, std::back_inserter(Backedges), + [Header](const MachineBasicBlock *MBB) { + return std::make_pair(MBB, Header); + }); + } + } + while (!ToVisit.empty()) { // First calculate stack layouts without walking backwards jumps, i.e. // assuming the current preliminary entry layout of the backwards jump @@ -328,7 +383,6 @@ void EVMStackLayoutGenerator::processEntryPoint( while (!ToVisit.empty()) { const MachineBasicBlock *Block = *ToVisit.begin(); ToVisit.pop_front(); - if (Visited.count(Block)) continue; @@ -343,52 +397,90 @@ void EVMStackLayoutGenerator::processEntryPoint( } } - // Determine which backwards jumps still require fixing and stage revisits - // of appropriate nodes. - for (auto [JumpingBlock, Target] : BackwardsJumps) { - // This block jumps backwards, but does not provide all slots required by - // the jump target on exit. Therefore we need to visit the subgraph - // between 'Target' and 'JumpingBlock' again. - const Stack &EntryLayout = MBBEntryLayoutMap[Target]; - auto B = EntryLayout.begin(), E = EntryLayout.end(); - const Stack &ExitLayout = MBBExitLayoutMap[JumpingBlock]; - if (std::any_of(B, E, [ExitLayout](const StackSlot &Slot) { - return find(ExitLayout, Slot) == ExitLayout.end(); - })) { - // In particular we can visit backwards starting from 'JumpingBlock' - // and mark all entries to-be-visited again until we hit 'Target'. - ToVisit.emplace_front(JumpingBlock); - // Since we are likely to permute the entry layout of 'Target', we - // also visit its entries again. This is not required for correctness, - // since the set of stack slots will match, but it may move some - // required stack shuffling from the loop condition to outside the loop. - for (const MachineBasicBlock *Pred : Target->predecessors()) - Visited.erase(Pred); - - EVMUtils::BreadthFirstSearch{{JumpingBlock}} - .run([&Visited, Target = Target](const MachineBasicBlock *Block, - auto VisitPred) { - Visited.erase(Block); - if (Block == Target) - return; - for (auto const *Pred : Block->predecessors()) - VisitPred(Pred); - }); - // While the shuffled layout for 'Target' will be compatible, it can - // be worthwhile propagating it further up once more. This would mean - // not stopping at Block == Target above, resp. even doing - // Visited.clear() here, revisiting the entire graph. This is a tradeoff - // between the runtime of this process and the optimality of the result. - // Also note that while visiting the entire graph again *can* be - // helpful, it can also be detrimental. + // Check which latch blocks still require fixing of their exit layouts. + // Revisit these blocks again. + for (auto [Latch, Header] : Backedges) { + const Stack &HeaderEntryLayout = MBBEntryLayoutMap[Header]; + const Stack &LatchExitLayout = MBBExitLayoutMap[Latch]; + if (all_of(HeaderEntryLayout, [LatchExitLayout](const StackSlot &Slot) { + return is_contained(LatchExitLayout, Slot); + })) + continue; + + // The latch block does not provide all slots required by the loop + // header. Therefore we need to visit the subgraph between the latch + // and header again. We will visit blocks backwards starting from latch + // and mark all MBBs to-be-visited again until we reach the header. + + ToVisit.emplace_back(Latch); + + // Since we are likely to permute the entry layout of 'Header', we + // also visit its entries again. This is not required for correctness, + // since the set of stack slots will match, but it may move some + // required stack shuffling from the loop condition to outside the loop. + for (const MachineBasicBlock *Pred : Header->predecessors()) + Visited.erase(Pred); + + // DFS upwards traversal from latch to the header. + for (auto I = idf_begin(Latch), E = idf_end(Latch); I != E;) { + const MachineBasicBlock *MBB = *I; + Visited.erase(MBB); + if (MBB == Header) { + I.skipChildren(); + continue; + } + ++I; } + // TODO: Consider revisiting the entire graph to propagate the optimal + // layout above the loop. } } - stitchConditionalJumps(Entry); - fillInJunk(Entry); + // At this point layouts at conditional jumps are merely + // compatible, i.e. the exit layout of the jumping block is a superset of the + // entry layout of the target block. We need to modify the entry layouts + // of conditional jump targets, s.t., the entry layout of target blocks match + // the exit layout of the jumping block exactly, except that slots not + // required after the jump are marked as 'JunkSlot'. + for (const MachineBasicBlock &MBB : MF) { + const EVMMBBTerminatorsInfo *TermInfo = CFGInfo.getTerminatorsInfo(&MBB); + MBBExitType ExitType = TermInfo->getExitType(); + if (ExitType != MBBExitType::ConditionalBranch) + continue; + + Stack ExitLayout = MBBExitLayoutMap.at(&MBB); + +#ifndef NDEBUG + // The last block must have produced the condition at the stack top. + auto [CondBr, UncondBr, TrueBB, FalseBB, Condition] = + TermInfo->getConditionalBranch(); + assert(ExitLayout.back() == StackModel.getStackSlot(*Condition)); +#endif // NDEBUG + + // The condition is consumed by the conditional jump. + ExitLayout.pop_back(); + for (const MachineBasicBlock *Succ : MBB.successors()) { + const Stack &SuccEntryLayout = MBBEntryLayoutMap.at(Succ); + Stack NewSuccEntryLayout = ExitLayout; + // Whatever the block being jumped to does not actually require, + // can be marked as junk. + for (StackSlot &Slot : NewSuccEntryLayout) + if (!is_contained(SuccEntryLayout, Slot)) + Slot = JunkSlot{}; + +#ifndef NDEBUG + // Make sure everything the block being jumped to requires is + // actually present or can be generated. + for (const StackSlot &Slot : SuccEntryLayout) + assert(isRematerializable(Slot) || + is_contained(NewSuccEntryLayout, Slot)); +#endif // NDEBUG + + MBBEntryLayoutMap[Succ] = NewSuccEntryLayout; + } + } - // Create function entry layout. + // Create the function entry layout. Stack EntryStack; bool IsNoReturn = MF.getFunction().hasFnAttribute(Attribute::NoReturn); if (!IsNoReturn) @@ -399,18 +491,42 @@ void EVMStackLayoutGenerator::processEntryPoint( EntryStack.append(StackModel.getFunctionParameters()); std::reverse(IsNoReturn ? EntryStack.begin() : std::next(EntryStack.begin()), EntryStack.end()); - MBBEntryLayoutMap[Entry] = std::move(EntryStack); + MBBEntryLayoutMap[&MF.front()] = std::move(EntryStack); + + // Traverse the CFG and at each block that allows junk, i.e. that is a + // cut-vertex that never leads to a function return, checks if adding junk + // reduces the shuffling cost upon entering and if so recursively adds junk + // to the spanned subgraph. This is needed only for optimization purposes, + // not for correctness. + for (const MachineBasicBlock &MBB : MF) { + if (!CFGInfo.isCutVertex(&MBB) || CFGInfo.isOnPathToFuncReturn(&MBB)) + continue; + + const Stack EntryLayout = MBBEntryLayoutMap.at(&MBB); + const Stack &ExitLayout = MBBExitLayoutMap.at(&MBB); + const SmallVector &Ops = StackModel.getOperations(&MBB); + Stack const &NextLayout = + Ops.empty() ? ExitLayout : OperationEntryLayoutMap.at(&Ops.front()); + if (EntryLayout != NextLayout) { + size_t OptimalNumJunks = getOptimalNumberOfJunks(EntryLayout, NextLayout); + if (OptimalNumJunks > 0) { + addJunksToStackBottom(&MBB, OptimalNumJunks); + MBBEntryLayoutMap[&MBB] = EntryLayout; + } + } + } } std::optional EVMStackLayoutGenerator::getExitLayoutOrStageDependencies( const MachineBasicBlock *Block, const DenseSet &Visited, - std::list &ToVisit) const { + std::deque &ToVisit) const { const EVMMBBTerminatorsInfo *TermInfo = CFGInfo.getTerminatorsInfo(Block); MBBExitType ExitType = TermInfo->getExitType(); if (ExitType == MBBExitType::UnconditionalBranch) { - auto [_, Target, IsLatch] = TermInfo->getUnconditionalBranch(); - if (IsLatch) { + auto [_, Target] = TermInfo->getUnconditionalBranch(); + if (MachineLoop *ML = MLI->getLoopFor(Block); + ML && ML->isLoopLatch(Block)) { // Choose the best currently known entry layout of the jump target // as initial exit. Note that this may not yet be the final // layout. @@ -461,78 +577,6 @@ std::optional EVMStackLayoutGenerator::getExitLayoutOrStageDependencies( return Stack{}; } -std::list> -EVMStackLayoutGenerator::collectBackwardsJumps( - const MachineBasicBlock *Entry) const { - std::list> - BackwardsJumps; - EVMUtils::BreadthFirstSearch{{Entry}}.run( - [&](const MachineBasicBlock *Block, auto VisitSucc) { - const EVMMBBTerminatorsInfo *TermInfo = - CFGInfo.getTerminatorsInfo(Block); - MBBExitType ExitType = TermInfo->getExitType(); - if (ExitType == MBBExitType::UnconditionalBranch) { - auto [_, Target, IsLatch] = TermInfo->getUnconditionalBranch(); - if (IsLatch) - BackwardsJumps.emplace_back(Block, Target); - } - for (const MachineBasicBlock *Succ : Block->successors()) - VisitSucc(Succ); - }); - return BackwardsJumps; -} - -void EVMStackLayoutGenerator::stitchConditionalJumps( - const MachineBasicBlock *Block) { - EVMUtils::BreadthFirstSearch BFS{{Block}}; - BFS.run([&](const MachineBasicBlock *Block, auto VisitSucc) { - const EVMMBBTerminatorsInfo *TermInfo = CFGInfo.getTerminatorsInfo(Block); - MBBExitType ExitType = TermInfo->getExitType(); - if (ExitType == MBBExitType::UnconditionalBranch) { - auto [_, Target, IsLatch] = TermInfo->getUnconditionalBranch(); - if (!IsLatch) - VisitSucc(Target); - return; - } - if (ExitType == MBBExitType::ConditionalBranch) { - auto [CondBr, UncondBr, TrueBB, FalseBB, Condition] = - TermInfo->getConditionalBranch(); - Stack ExitLayout = MBBExitLayoutMap.at(Block); - // The last block must have produced the condition at the stack - // top. - assert(!ExitLayout.empty()); - assert(ExitLayout.back() == StackModel.getStackSlot(*Condition)); - // The condition is consumed by the jump. - ExitLayout.pop_back(); - - auto FixJumpTargetEntry = [&](const Stack &OriginalEntryLayout) -> Stack { - Stack NewEntryLayout = ExitLayout; - // Whatever the block being jumped to does not actually require, - // can be marked as junk. - for (StackSlot &Slot : NewEntryLayout) { - if (find(OriginalEntryLayout, Slot) == OriginalEntryLayout.end()) - Slot = JunkSlot{}; - } -#ifndef NDEBUG - // Make sure everything the block being jumped to requires is - // actually present or can be generated. - for (StackSlot const &Slot : OriginalEntryLayout) - assert(isRematerializable(Slot) || - find(NewEntryLayout, Slot) != NewEntryLayout.end()); -#endif // NDEBUG - return NewEntryLayout; - }; - - const Stack &ZeroTargetEntryLayout = MBBEntryLayoutMap.at(FalseBB); - MBBEntryLayoutMap[FalseBB] = FixJumpTargetEntry(ZeroTargetEntryLayout); - const Stack &NonZeroTargetEntryLayout = MBBEntryLayoutMap.at(TrueBB); - MBBEntryLayoutMap[TrueBB] = FixJumpTargetEntry(NonZeroTargetEntryLayout); - VisitSucc(FalseBB); - VisitSucc(TrueBB); - } - }); -} - Stack EVMStackLayoutGenerator::combineStack(Stack const &Stack1, Stack const &Stack2) { // TODO: it would be nicer to replace this by a constructive algorithm. @@ -599,7 +643,7 @@ Stack EVMStackLayoutGenerator::combineStack(Stack const &Stack1, Stack Tmp = CommonPrefix; Tmp.append(TestStack); - auto Depth = EVMUtils::offset(reverse(Tmp), Slot); + auto Depth = offset(reverse(Tmp), Slot); if (Depth && *Depth >= 16) NumOps += 1000; }; @@ -661,8 +705,8 @@ Stack EVMStackLayoutGenerator::compressStack(Stack CurStack) { break; } - if (auto DupDepth = EVMUtils::offset( - drop_begin(reverse(CurStack), Depth + 1), Slot)) { + if (auto DupDepth = + offset(drop_begin(reverse(CurStack), Depth + 1), Slot)) { if (Depth + *DupDepth <= 16) { FirstDupOffset = CurStack.size() - Depth - 1; break; @@ -688,7 +732,7 @@ size_t llvm::EvaluateStackTransform(Stack Source, Stack const &Target) { if (isRematerializable(Slot)) OpGas += 3; else { - auto Depth = EVMUtils::offset(reverse(Source), Slot); + auto Depth = offset(reverse(Source), Slot); if (!Depth) llvm_unreachable("No slot in the stack"); @@ -704,83 +748,21 @@ size_t llvm::EvaluateStackTransform(Stack Source, Stack const &Target) { return OpGas; } -void EVMStackLayoutGenerator::fillInJunk(const MachineBasicBlock *Block) { - /// Recursively adds junk to the subgraph starting on \p Entry. - /// Since it is only called on cut-vertices, the full subgraph retains proper - /// stack balance. - auto AddJunkRecursive = [&](const MachineBasicBlock *Entry, size_t NumJunk) { - EVMUtils::BreadthFirstSearch BFS{{Entry}}; - - BFS.run([&](const MachineBasicBlock *Block, auto VisitSucc) { - Stack EntryTmp(NumJunk, JunkSlot{}); - EntryTmp.append(MBBEntryLayoutMap.at(Block)); - MBBEntryLayoutMap[Block] = std::move(EntryTmp); - - for (const Operation &Operation : StackModel.getOperations(Block)) { - Stack OpEntryTmp(NumJunk, JunkSlot{}); - OpEntryTmp.append(OperationEntryLayoutMap.at(&Operation)); - OperationEntryLayoutMap[&Operation] = std::move(OpEntryTmp); - } - - Stack ExitTmp(NumJunk, JunkSlot{}); - ExitTmp.append(MBBExitLayoutMap.at(Block)); - MBBExitLayoutMap[Block] = std::move(ExitTmp); - - for (const MachineBasicBlock *Succ : Block->successors()) - VisitSucc(Succ); - }); - }; - - /// Returns the number of junk slots to be prepended to \p TargetLayout for - /// an optimal transition from \p EntryLayout to \p TargetLayout. - auto GetBestNumJunk = [&](const Stack &EntryLayout, - const Stack &TargetLayout) -> size_t { - size_t BestCost = EvaluateStackTransform(EntryLayout, TargetLayout); - size_t BestNumJunk = 0; - size_t MaxJunk = EntryLayout.size(); - for (size_t NumJunk = 1; NumJunk <= MaxJunk; ++NumJunk) { - Stack JunkedTarget(NumJunk, JunkSlot{}); - JunkedTarget.append(TargetLayout); - size_t Cost = EvaluateStackTransform(EntryLayout, JunkedTarget); - if (Cost < BestCost) { - BestCost = Cost; - BestNumJunk = NumJunk; - } +void EVMStackLayoutGenerator::addJunksToStackBottom( + const MachineBasicBlock *Entry, size_t NumJunk) { + for (const MachineBasicBlock *MBB : depth_first(Entry)) { + Stack EntryTmp(NumJunk, JunkSlot{}); + EntryTmp.append(MBBEntryLayoutMap.at(MBB)); + MBBEntryLayoutMap[MBB] = std::move(EntryTmp); + + for (const Operation &Operation : StackModel.getOperations(MBB)) { + Stack OpEntryTmp(NumJunk, JunkSlot{}); + OpEntryTmp.append(OperationEntryLayoutMap.at(&Operation)); + OperationEntryLayoutMap[&Operation] = std::move(OpEntryTmp); } - return BestNumJunk; - }; - if (MF.getFunction().hasFnAttribute(Attribute::NoReturn) && - (&MF.front() == Block)) { - Stack Params = StackModel.getFunctionParameters(); - std::reverse(Params.begin(), Params.end()); - size_t BestNumJunk = GetBestNumJunk(Params, MBBEntryLayoutMap.at(Block)); - if (BestNumJunk > 0) - AddJunkRecursive(Block, BestNumJunk); + Stack ExitTmp(NumJunk, JunkSlot{}); + ExitTmp.append(MBBExitLayoutMap.at(MBB)); + MBBExitLayoutMap[MBB] = std::move(ExitTmp); } - /// Traverses the CFG and at each block that allows junk, i.e. that is a - /// cut-vertex that never leads to a function return, checks if adding junk - /// reduces the shuffling cost upon entering and if so recursively adds junk - /// to the spanned subgraph. - EVMUtils::BreadthFirstSearch{{Block}}.run( - [&](MachineBasicBlock const *Block, auto VisitSucc) { - if (CFGInfo.isCutVertex(Block) && - !CFGInfo.isOnPathToFuncReturn(Block)) { - const Stack EntryLayout = MBBEntryLayoutMap.at(Block); - const Stack &ExitLayout = MBBExitLayoutMap.at(Block); - const SmallVector &Ops = StackModel.getOperations(Block); - Stack const &NextLayout = - Ops.empty() ? ExitLayout - : OperationEntryLayoutMap.at(&Ops.front()); - if (EntryLayout != NextLayout) { - size_t BestNumJunk = GetBestNumJunk(EntryLayout, NextLayout); - if (BestNumJunk > 0) { - AddJunkRecursive(Block, BestNumJunk); - MBBEntryLayoutMap[Block] = EntryLayout; - } - } - } - for (const MachineBasicBlock *Succ : Block->successors()) - VisitSucc(Succ); - }); } diff --git a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h index 8230e2f1bb16..280d7c1e0fe3 100644 --- a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h +++ b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h @@ -22,6 +22,8 @@ #include "EVMStackModel.h" #include "llvm/ADT/DenseMap.h" +#include + namespace llvm { /// Returns the number of operations required to transform stack \p Source to @@ -69,7 +71,7 @@ class EVMStackLayoutGenerator { SmallVector variableChoices; }; - EVMStackLayoutGenerator(const MachineFunction &MF, + EVMStackLayoutGenerator(const MachineFunction &MF, const MachineLoopInfo *MLI, const EVMStackModel &StackModel, const EVMMachineCFGInfo &CFGInfo); @@ -93,7 +95,11 @@ class EVMStackLayoutGenerator { /// Main algorithm walking the graph from entry to exit and propagating back /// the stack layouts to the entries. Iteratively reruns itself along /// backwards jumps until the layout is stabilized. - void processEntryPoint(const MachineBasicBlock *Entry); + void runPropagation(); + + /// Adds junks to the subgraph starting at \p Entry. It should only be + /// called on cut-vertices, so the full subgraph retains proper stack balance. + void addJunksToStackBottom(const MachineBasicBlock *Entry, size_t NumJunk); /// Returns the best known exit layout of \p Block, if all dependencies are /// already \p Visited. If not, adds the dependencies to \p DependencyList and @@ -101,20 +107,7 @@ class EVMStackLayoutGenerator { std::optional getExitLayoutOrStageDependencies( const MachineBasicBlock *Block, const DenseSet &Visited, - std::list &DependencyList) const; - - /// Returns a pair of '{jumpingBlock, targetBlock}' for each backwards jump - /// in the graph starting at \p Entry. - std::list> - collectBackwardsJumps(const MachineBasicBlock *Entry) const; - - /// After the main algorithms, layouts at conditional jumps are merely - /// compatible, i.e. the exit layout of the jumping block is a superset of the - /// entry layout of the target block. This function modifies the entry layouts - /// of conditional jump targets, s.t., the entry layout of target blocks match - /// the exit layout of the jumping block exactly, except that slots not - /// required after the jump are marked as 'JunkSlot's. - void stitchConditionalJumps(const MachineBasicBlock *Block); + std::deque &DependencyList) const; /// Calculates the ideal stack layout, s.t., both \p Stack1 and \p Stack2 can /// be achieved with minimal stack shuffling when starting from the returned @@ -131,11 +124,8 @@ class EVMStackLayoutGenerator { /// amount of operations to reconstruct the original stack \p Stack. static Stack compressStack(Stack Stack); - /// Fills in junk when entering branches that do not need a clean stack in - /// case the result is cheaper. - void fillInJunk(const MachineBasicBlock *Block); - const MachineFunction &MF; + const MachineLoopInfo *MLI; const EVMStackModel &StackModel; const EVMMachineCFGInfo &CFGInfo; diff --git a/llvm/lib/Target/EVM/EVMStackShuffler.h b/llvm/lib/Target/EVM/EVMStackShuffler.h index 6db0184bce9a..e91a640727b5 100644 --- a/llvm/lib/Target/EVM/EVMStackShuffler.h +++ b/llvm/lib/Target/EVM/EVMStackShuffler.h @@ -16,7 +16,6 @@ #ifndef LLVM_LIB_TARGET_EVM_EVMSTACKSHUFFLER_H #define LLVM_LIB_TARGET_EVM_EVMSTACKSHUFFLER_H -#include "EVMHelperUtilities.h" #include "EVMStackModel.h" #include #include @@ -152,7 +151,7 @@ class Shuffler { // If this slot occurs again later, we skip this occurrence. // TODO: use C++ 20 ranges::views::iota if (const auto &R = - EVMUtils::iota(SourceOffset + 1, Ops.sourceSize()); + llvm::seq(SourceOffset + 1, Ops.sourceSize()); std::any_of(R.begin(), R.end(), [&](size_t Offset) { return Ops.sourceIsSame(SourceOffset, Offset); })) @@ -214,7 +213,7 @@ class Shuffler { ShuffleOperations Ops{std::forward(args)...}; // All source slots are final. - if (const auto &R = EVMUtils::iota(0u, Ops.sourceSize()); + if (const auto &R = llvm::seq(0u, Ops.sourceSize()); std::all_of(R.begin(), R.end(), [&](size_t Index) { return Ops.isCompatible(Index, Index); })) { @@ -335,7 +334,7 @@ class Shuffler { assert(Ops.isCompatible(SourceTop, SourceTop)); const auto &SwappableOffsets = - EVMUtils::iota(Size > 17 ? Size - 17 : 0u, Size); + llvm::seq(Size > 17 ? Size - 17 : 0u, Size); // If we find a lower slot that is out of position, but also compatible with // the top, swap that up. diff --git a/llvm/lib/Target/EVM/EVMStackifyCodeEmitter.cpp b/llvm/lib/Target/EVM/EVMStackifyCodeEmitter.cpp index 94f265587924..bbf6d822fb95 100644 --- a/llvm/lib/Target/EVM/EVMStackifyCodeEmitter.cpp +++ b/llvm/lib/Target/EVM/EVMStackifyCodeEmitter.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "EVMStackifyCodeEmitter.h" -#include "EVMHelperUtilities.h" #include "EVMMachineFunctionInfo.h" #include "EVMStackDebug.h" #include "EVMStackShuffler.h" @@ -22,6 +21,21 @@ using namespace llvm; #define DEBUG_TYPE "evm-stackify-code-emitter" +template struct Overload : Ts... { + using Ts::operator()...; +}; +template Overload(Ts...) -> Overload; + +// Return whether the function of the call instruction will return. +bool callWillReturn(const MachineInstr *Call) { + assert(Call->getOpcode() == EVM::FCALL && "Unexpected call instruction"); + const MachineOperand *FuncOp = Call->explicit_uses().begin(); + assert(FuncOp->isGlobal() && "Expected a global value"); + const auto *Func = cast(FuncOp->getGlobal()); + assert(Func && "Expected a function"); + return !Func->hasFnAttribute(Attribute::NoReturn); +} + // Return the number of input arguments of the call instruction. static size_t getCallArgCount(const MachineInstr *Call) { assert(Call->getOpcode() == EVM::FCALL && "Unexpected call instruction"); @@ -33,7 +47,7 @@ static size_t getCallArgCount(const MachineInstr *Call) { // The first operand is a function, so don't count it. If function // will return, we need to account for the return label. constexpr size_t NumFuncOp = 1; - return NumExplicitInputs - NumFuncOp + EVMUtils::callWillReturn(Call); + return NumExplicitInputs - NumFuncOp + callWillReturn(Call); } size_t EVMStackifyCodeEmitter::CodeEmitter::stackHeight() const { @@ -149,7 +163,7 @@ void EVMStackifyCodeEmitter::CodeEmitter::emitFuncCall(const MachineInstr *MI) { // If this function returns, add a return label so we can emit it together // with JUMPDEST. This is taken care in the AsmPrinter. - if (EVMUtils::callWillReturn(MI)) + if (callWillReturn(MI)) NewMI.addSym(CallReturnSyms.at(MI)); verify(NewMI); } @@ -232,7 +246,7 @@ void EVMStackifyCodeEmitter::visitCall(const FunctionCall &Call) { assert(CurrentStack.size() >= NumArgs); // Assert that we got the correct return label on stack. - if (EVMUtils::callWillReturn(Call.MI)) { + if (callWillReturn(Call.MI)) { [[maybe_unused]] const auto *returnLabelSlot = std::get_if( &CurrentStack[CurrentStack.size() - NumArgs]); @@ -479,7 +493,7 @@ void EVMStackifyCodeEmitter::run() { const EVMMBBTerminatorsInfo *TermInfo = CFGInfo.getTerminatorsInfo(Block); MBBExitType ExitType = TermInfo->getExitType(); if (ExitType == MBBExitType::UnconditionalBranch) { - auto [UncondBr, Target, _] = TermInfo->getUnconditionalBranch(); + auto [UncondBr, Target] = TermInfo->getUnconditionalBranch(); // Create the stack expected at the jump target. createStackLayout(Layout.getMBBEntryLayout(Target));