diff --git a/llvm/lib/Target/EVM/CMakeLists.txt b/llvm/lib/Target/EVM/CMakeLists.txt index ceed4bf575e0..5a523008aea2 100644 --- a/llvm/lib/Target/EVM/CMakeLists.txt +++ b/llvm/lib/Target/EVM/CMakeLists.txt @@ -20,7 +20,9 @@ add_llvm_target(EVMCodeGen EVMAllocaHoisting.cpp EVMArgumentMove.cpp EVMAsmPrinter.cpp + EVMAssembly.cpp EVMCodegenPrepare.cpp + EVMControlFlowGraphBuilder.cpp EVMFrameLowering.cpp EVMISelDAGToDAG.cpp EVMISelLowering.cpp @@ -30,10 +32,15 @@ add_llvm_target(EVMCodeGen EVMMachineFunctionInfo.cpp EVMMCInstLower.cpp EVMOptimizeLiveIntervals.cpp + EVMOptimizedCodeTransform.cpp EVMRegColoring.cpp EVMRegisterInfo.cpp EVMSingleUseExpression.cpp + EVMSplitCriticalEdges.cpp + EVMStackDebug.cpp + EVMStackLayoutGenerator.cpp EVMStackify.cpp + EVMStackifyEF.cpp EVMSubtarget.cpp EVMTargetMachine.cpp EVMTargetTransformInfo.cpp diff --git a/llvm/lib/Target/EVM/EVM.h b/llvm/lib/Target/EVM/EVM.h index 2ff8488a5089..f9846fa615e5 100644 --- a/llvm/lib/Target/EVM/EVM.h +++ b/llvm/lib/Target/EVM/EVM.h @@ -50,7 +50,9 @@ ModulePass *createEVMLinkRuntimePass(); FunctionPass *createEVMOptimizeLiveIntervals(); FunctionPass *createEVMRegColoring(); FunctionPass *createEVMSingleUseExpression(); +FunctionPass *createEVMSplitCriticalEdges(); FunctionPass *createEVMStackify(); +FunctionPass *createEVMStackifyEF(); // PassRegistry initialization declarations. void initializeEVMCodegenPreparePass(PassRegistry &); @@ -61,7 +63,9 @@ void initializeEVMLinkRuntimePass(PassRegistry &); void initializeEVMOptimizeLiveIntervalsPass(PassRegistry &); void initializeEVMRegColoringPass(PassRegistry &); void initializeEVMSingleUseExpressionPass(PassRegistry &); +void initializeEVMSplitCriticalEdgesPass(PassRegistry &); void initializeEVMStackifyPass(PassRegistry &); +void initializeEVMStackifyEFPass(PassRegistry &); struct EVMLinkRuntimePass : PassInfoMixin { EVMLinkRuntimePass() = default; diff --git a/llvm/lib/Target/EVM/EVMArgumentMove.cpp b/llvm/lib/Target/EVM/EVMArgumentMove.cpp index b76122b5336f..5da6aeb6caf2 100644 --- a/llvm/lib/Target/EVM/EVMArgumentMove.cpp +++ b/llvm/lib/Target/EVM/EVMArgumentMove.cpp @@ -6,7 +6,8 @@ // //===----------------------------------------------------------------------===// // -// This file moves ARGUMENT instructions after ScheduleDAG scheduling. +// This file moves and orders ARGUMENT instructions after ScheduleDAG +// scheduling. // // Arguments are really live-in registers, however, since we use virtual // registers and LLVM doesn't support live-in virtual registers, we're @@ -67,21 +68,24 @@ bool EVMArgumentMove::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; MachineBasicBlock &EntryMBB = MF.front(); + SmallVector Args; + for (MachineInstr &MI : EntryMBB) { + if (EVM::ARGUMENT == MI.getOpcode()) + Args.push_back(&MI); + } - // Look for the first NonArg instruction. - const auto InsertPt = - std::find_if_not(EntryMBB.begin(), EntryMBB.end(), [](auto &MI) { - return EVM::ARGUMENT == MI.getOpcode(); - }); + // Sort ARGUMENT instructions in ascending order of their arguments. + std::sort(Args.begin(), Args.end(), + [](const MachineInstr *MI1, const MachineInstr *MI2) { + int64_t Arg1Idx = MI1->getOperand(1).getImm(); + int64_t Arg2Idx = MI2->getOperand(1).getImm(); + return Arg1Idx < Arg2Idx; + }); - // Now move any argument instructions later in the block - // to before our first NonArg instruction. - for (MachineInstr &MI : llvm::make_range(InsertPt, EntryMBB.end())) { - if (EVM::ARGUMENT == MI.getOpcode()) { - EntryMBB.insert(InsertPt, MI.removeFromParent()); - Changed = true; - } + for (MachineInstr *MI : reverse(Args)) { + MachineInstr *Arg = MI->removeFromParent(); + EntryMBB.insert(EntryMBB.begin(), Arg); + Changed = true; } - return Changed; } diff --git a/llvm/lib/Target/EVM/EVMAssembly.cpp b/llvm/lib/Target/EVM/EVMAssembly.cpp new file mode 100644 index 000000000000..4a63578058d1 --- /dev/null +++ b/llvm/lib/Target/EVM/EVMAssembly.cpp @@ -0,0 +1,281 @@ +//===----------- EVMAssembly.cpp - EVM Assembly generator -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the EVMAssembly class that generates machine IR +// with all the required stack manipulation instructions. +// Resulting machine instructions still have explicit operands, but some of the +// auxiliary instructions (ARGUMENT, RET, EVM::CONST_I256, COPY_I256 +// FCALLARGUMENT) are removed after this step, beaking use-def chains. So, the +// resulting Machine IR breaks the MachineVerifier checks. +// +//===----------------------------------------------------------------------===// + +#include "EVMAssembly.h" +#include "EVM.h" +#include "EVMMachineFunctionInfo.h" +#include "EVMSubtarget.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include "TargetInfo/EVMTargetInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/MC/MCContext.h" + +using namespace llvm; + +#define DEBUG_TYPE "evm-assembly" + +#ifndef NDEBUG +void EVMAssembly::dumpInst(const MachineInstr *MI) const { + LLVM_DEBUG(dbgs() << "Adding: " << *MI << "stack height: " << StackHeight + << "\n"); +} +#endif // NDEBUG + +int EVMAssembly::getStackHeight() const { return StackHeight; } + +void EVMAssembly::setStackHeight(int Height) { + StackHeight = Height; + LLVM_DEBUG(dbgs() << "Set stack height: " << StackHeight << "\n"); +} + +void EVMAssembly::setCurrentLocation(MachineBasicBlock *MBB) { + CurMBB = MBB; + CurMIIt = MBB->begin(); + LLVM_DEBUG(dbgs() << "Setting current location to: " << MBB->getNumber() + << "." << MBB->getName() << "\n"); +} + +void EVMAssembly::appendInstruction(MachineInstr *MI) { + unsigned Opc = MI->getOpcode(); + assert(Opc != EVM::JUMP && Opc != EVM::JUMPI && Opc != EVM::ARGUMENT && + Opc != EVM::RET && Opc != EVM::CONST_I256 && Opc != EVM::COPY_I256 && + Opc != EVM::FCALL); + + auto Ret = AssemblyInstrs.insert(MI); + assert(Ret.second); + int StackAdj = (2 * static_cast(MI->getNumExplicitDefs())) - + static_cast(MI->getNumExplicitOperands()); + StackHeight += StackAdj; + LLVM_DEBUG(dumpInst(MI)); + CurMIIt = std::next(MIIter(MI)); +} + +void EVMAssembly::appendSWAPInstruction(unsigned Depth) { + unsigned Opc = EVM::getSWAPOpcode(Depth); + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc)); + AssemblyInstrs.insert(&*CurMIIt); + dumpInst(&*CurMIIt); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendDUPInstruction(unsigned Depth) { + unsigned Opc = EVM::getDUPOpcode(Depth); + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc)); + StackHeight += 1; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendPOPInstruction() { + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::POP)); + assert(StackHeight > 0); + StackHeight -= 1; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendConstant(const APInt &Val) { + unsigned Opc = EVM::getPUSHOpcode(Val); + MachineInstrBuilder Builder = + BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc)); + if (Opc != EVM::PUSH0) { + LLVMContext &Ctx = MF->getFunction().getContext(); + Builder.addCImm(ConstantInt::get(Ctx, Val)); + } + StackHeight += 1; + CurMIIt = Builder; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendSymbol(MCSymbol *Symbol) { + // This is codegen-only instruction, that will be converted into PUSH4. + CurMIIt = + BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::DATA)).addSym(Symbol); + StackHeight += 1; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendConstant(uint64_t Val) { + appendConstant(APInt(256, Val)); +} + +void EVMAssembly::appendLabel() { + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::JUMPDEST)); + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendLabelReference(MCSymbol *Label) { + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::PUSH_LABEL)) + .addSym(Label); + StackHeight += 1; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendMBBReference(MachineBasicBlock *MBB) { + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::PUSH_LABEL)) + .addMBB(MBB); + StackHeight += 1; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +MCSymbol *EVMAssembly::createFuncRetSymbol() { + return MF->getContext().createTempSymbol("FUNC_RET", true); +} + +void EVMAssembly::appendFuncCall(const MachineInstr *MI, + const llvm::GlobalValue *Func, int StackAdj, + MCSymbol *RetSym) { + // Push the function label + assert(CurMBB == MI->getParent()); + CurMIIt = + BuildMI(*CurMBB, CurMIIt, MI->getDebugLoc(), TII->get(EVM::PUSH_LABEL)) + .addGlobalAddress(Func); + // PUSH_LABEL technically increases the stack height on 1, but we don't + // increase it explicitly here, as the label will be consumed by the following + // JUMP. + AssemblyInstrs.insert(&*CurMIIt); + StackHeight += StackAdj; + LLVM_DEBUG(dumpInst(&*CurMIIt)); + + CurMIIt = std::next(CurMIIt); + // Create jump to the callee. Note, we don't add the 'target' operand to JUMP. + // This should be fine, unless we run MachineVerifier after this step + CurMIIt = BuildMI(*CurMBB, CurMIIt, MI->getDebugLoc(), TII->get(EVM::JUMP)); + if (RetSym) + CurMIIt->setPostInstrSymbol(*MF, RetSym); + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendJump(int StackAdj) { + CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::JUMP)); + StackHeight += StackAdj; + AssemblyInstrs.insert(&*CurMIIt); + LLVM_DEBUG(dumpInst(&*CurMIIt)); + CurMIIt = std::next(CurMIIt); +} + +void EVMAssembly::appendUncondJump(MachineInstr *MI, + MachineBasicBlock *Target) { + assert(MI->getOpcode() == EVM::JUMP); + appendMBBReference(Target); + [[maybe_unused]] auto It = AssemblyInstrs.insert(MI); + assert(It.second && StackHeight > 0); + StackHeight -= 1; + LLVM_DEBUG(dumpInst(MI)); + CurMIIt = std::next(MIIter(MI)); +} + +void EVMAssembly::appendCondJump(MachineInstr *MI, MachineBasicBlock *Target) { + assert(MI->getOpcode() == EVM::JUMPI); + appendMBBReference(Target); + [[maybe_unused]] auto It = AssemblyInstrs.insert(MI); + assert(It.second && StackHeight > 1); + StackHeight -= 2; + LLVM_DEBUG(dumpInst(MI)); + CurMIIt = std::next(MIIter(MI)); +} + +// Remove all registers operands of the \p MI and repaces the opcode with +// the stack variant variant. +void EVMAssembly::stackifyInstruction(MachineInstr *MI) { + if (MI->isDebugInstr() || MI->isLabel() || MI->isInlineAsm()) + return; + + unsigned RegOpcode = MI->getOpcode(); + if (RegOpcode == EVM::PUSH_LABEL) + return; + + // Remove register operands. + for (unsigned I = MI->getNumOperands(); I > 0; --I) { + auto &MO = MI->getOperand(I - 1); + if (MO.isReg()) { + MI->removeOperand(I - 1); + } + } + + // Transform 'register' instruction to the 'stack' one. + unsigned StackOpcode = EVM::getStackOpcode(RegOpcode); + MI->setDesc(TII->get(StackOpcode)); +} + +void EVMAssembly::finalize() { + // Collect and erase instructions that are not required in + // a stackified code. These are auxiliary codegn-only instructions. + SmallVector ToRemove; + for (MachineBasicBlock &MBB : *MF) { + for (MachineInstr &MI : MBB) { + if (!AssemblyInstrs.count(&MI) && MI.getOpcode() != EVM::JUMP && + MI.getOpcode() != EVM::JUMPI) + ToRemove.emplace_back(&MI); + } + } + + for (MachineInstr *MI : ToRemove) + MI->eraseFromParent(); + + // Remove register operands and replace instruction opcode with 'stack' one. + for (MachineBasicBlock &MBB : *MF) + for (MachineInstr &MI : MBB) + stackifyInstruction(&MI); + + auto *MFI = MF->getInfo(); + MFI->setIsStackified(); + + // In a stackified code register liveness has no meaning. + MachineRegisterInfo &MRI = MF->getRegInfo(); + MRI.invalidateLiveness(); + + // In EVM architecture jump target is set up using one of PUSH* instructions + // that come right before the jump instruction. + // For example: + + // PUSH_LABEL %bb.10 + // JUMPI_S + // PUSH_LABEL %bb.9 + // JUMP_S + // + // The problem here is that such MIR is not valid. There should not be + // non-terminator (PUSH) instructions between terminator (JUMP) ones. + // To overcome this issue, we bundle adjacent instructions + // together and unbundle them in the AsmPrinter. + for (MachineBasicBlock &MBB : *MF) { + MachineBasicBlock::instr_iterator I = MBB.instr_begin(), + E = MBB.instr_end(); + for (; I != E; ++I) { + if (I->isBranch()) { + auto P = std::next(I); + if (P != E && P->getOpcode() == EVM::PUSH_LABEL) + I->bundleWithPred(); + } + } + } +} diff --git a/llvm/lib/Target/EVM/EVMAssembly.h b/llvm/lib/Target/EVM/EVMAssembly.h new file mode 100644 index 000000000000..2e74d31ed8d2 --- /dev/null +++ b/llvm/lib/Target/EVM/EVMAssembly.h @@ -0,0 +1,101 @@ +//===------------- EVMAssembly.h - EVM Assembly generator -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the EVMAssembly class that generates machine IR +// with all the required stack manipulation instructions. +// Resulting machine instructions still have explicit operands, but some of the +// auxiliary instructions (ARGUMENT, RET, EVM::CONST_I256, COPY_I256 +// FCALLARGUMENT) are removed after this step, beaking use-def chains. So, the +// resulting Machine IR breaks the MachineVerifier checks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMASSEMBLY_H +#define LLVM_LIB_TARGET_EVM_EVMASSEMBLY_H + +#include "EVM.h" + +#include "EVMSubtarget.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include "llvm/CodeGen/MachineFunction.h" + +namespace llvm { + +class MachineInstr; +class MCSymbol; + +class EVMAssembly { +private: + using MIIter = MachineBasicBlock::iterator; + + MachineFunction *MF; + const EVMInstrInfo *TII; + + int StackHeight = 0; + MIIter CurMIIt; + MachineBasicBlock *CurMBB; + DenseSet AssemblyInstrs; + +public: + EVMAssembly(MachineFunction *MF, const EVMInstrInfo *TII) + : MF(MF), TII(TII) {} + + // Retrieve the current height of the stack. + // This does not have to be zero at the beginning. + int getStackHeight() const; + + void setStackHeight(int Height); + + void setCurrentLocation(MachineBasicBlock *MBB); + + void appendInstruction(MachineInstr *MI); + + void appendSWAPInstruction(unsigned Depth); + + void appendDUPInstruction(unsigned Depth); + + void appendPOPInstruction(); + + void appendConstant(const APInt &Val); + + void appendSymbol(MCSymbol *Symbol); + + void appendConstant(uint64_t Val); + + void appendLabel(); + + void appendFuncCall(const MachineInstr *MI, const llvm::GlobalValue *Func, + int stackAdj, MCSymbol *RetSym = nullptr); + + void appendJump(int stackAdj); + + void appendCondJump(MachineInstr *MI, MachineBasicBlock *Target); + + void appendUncondJump(MachineInstr *MI, MachineBasicBlock *Target); + + void appendLabelReference(MCSymbol *Label); + + void appendMBBReference(MachineBasicBlock *MBB); + + MCSymbol *createFuncRetSymbol(); + + // Removes unused codegen-only instructions and + // stackifies remaining ones. + void finalize(); + +private: + void stackifyInstruction(MachineInstr *MI); + +#ifndef NDEBUG + void dumpInst(const MachineInstr *MI) const; +#endif // NDEBUG +}; + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMASSEMBLY_H diff --git a/llvm/lib/Target/EVM/EVMControlFlowGraph.h b/llvm/lib/Target/EVM/EVMControlFlowGraph.h new file mode 100644 index 000000000000..b4e8a1905d8f --- /dev/null +++ b/llvm/lib/Target/EVM/EVMControlFlowGraph.h @@ -0,0 +1,274 @@ +//===----- EVMControlFlowGraph.h - CFG for stackification -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines Control Flow Graph used for the stackification algorithm. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPH_H +#define LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPH_H + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/Register.h" +#include "llvm/MC/MCSymbol.h" + +#include +#include +#include + +namespace llvm { + +class MachineFunction; +class MachineBasicBlock; +class MachineInstr; +/// The following structs describe different kinds of stack slots. +/// Each stack slot is equality- and less-than-comparable and +/// specifies an attribute ``canBeFreelyGenerated`` that is true, +/// if a slot of this kind always has a known value at compile time and +/// therefore can safely be removed from the stack at any time and then +/// regenerated later. + +/// The label pushed as return label before a function call, i.e. the label the +/// call is supposed to return to. +struct FunctionCallReturnLabelSlot { + const MachineInstr *Call = nullptr; + static constexpr bool canBeFreelyGenerated = true; + + bool operator==(FunctionCallReturnLabelSlot const &Rhs) const { + return Call == Rhs.Call; + } + + bool operator<(FunctionCallReturnLabelSlot const &Rhs) const { + return Call < Rhs.Call; + } +}; + +/// The return jump target of a function while generating the code of the +/// function body. I.e. the caller of a function pushes a +/// ``FunctionCallReturnLabelSlot`` (see above) before jumping to the function +/// and this very slot is viewed as ``FunctionReturnLabelSlot`` inside the +/// function body and jumped to when returning from the function. +struct FunctionReturnLabelSlot { + const MachineFunction *MF = nullptr; + static constexpr bool canBeFreelyGenerated = false; + + bool operator==(FunctionReturnLabelSlot const &Rhs) const { + // There can never be return label slots of different functions on stack + // simultaneously. + assert(MF == Rhs.MF); + return true; + } + + bool operator<(FunctionReturnLabelSlot const &Rhs) const { + // There can never be return label slots of different functions on stack + // simultaneously. + assert(MF == Rhs.MF); + return false; + } +}; + +/// A slot containing the current value of a particular variable. +struct VariableSlot { + Register VirtualReg; + static constexpr bool canBeFreelyGenerated = false; + + bool operator==(VariableSlot const &Rhs) const { + return VirtualReg == Rhs.VirtualReg; + } + + bool operator<(VariableSlot const &Rhs) const { + return VirtualReg < Rhs.VirtualReg; + } +}; + +/// A slot containing a literal value. +struct LiteralSlot { + APInt Value; + static constexpr bool canBeFreelyGenerated = true; + + bool operator==(LiteralSlot const &Rhs) const { return Value == Rhs.Value; } + + bool operator<(LiteralSlot const &Rhs) const { return Value.ult(Rhs.Value); } +}; + +/// A slot containing a Symbol. +struct SymbolSlot { + MCSymbol *Symbol; + static constexpr bool canBeFreelyGenerated = true; + + bool operator==(SymbolSlot const &Rhs) const { return Symbol == Rhs.Symbol; } + + bool operator<(SymbolSlot const &Rhs) const { return Symbol < Rhs.Symbol; } +}; + +/// A slot containing the index-th return value of a previous call. +struct TemporarySlot { + /// The call that returned this slot. + const MachineInstr *MI = nullptr; + + Register VirtualReg; + /// Specifies to which of the values returned by the call this slot refers. + /// index == 0 refers to the slot deepest in the stack after the call. + size_t Index = 0; + static constexpr bool canBeFreelyGenerated = false; + + bool operator==(TemporarySlot const &Rhs) const { + return MI == Rhs.MI && Index == Rhs.Index; + } + + bool operator<(TemporarySlot const &Rhs) const { + return std::make_pair(MI, Index) < std::make_pair(Rhs.MI, Rhs.Index); + } +}; + +/// A slot containing an arbitrary value that is always eventually popped and +/// never used. Used to maintain stack balance on control flow joins. +struct JunkSlot { + static constexpr bool canBeFreelyGenerated = true; + + bool operator==(JunkSlot const &) const { return true; } + + bool operator<(JunkSlot const &) const { return false; } +}; + +using StackSlot = + std::variant; + +/// The stack top is usually the last element of the vector. +using Stack = std::vector; + +/// Returns true if Slot can be generated on the stack at any time. +inline bool canBeFreelyGenerated(StackSlot const &Slot) { + return std::visit( + [](auto const &TypedSlot) { + return std::decay_t::canBeFreelyGenerated; + }, + Slot); +} + +/// Control flow graph consisting of ``CFG::BasicBlock``s connected by control +/// flow. +struct CFG { + explicit CFG() {} + CFG(CFG const &) = delete; + CFG(CFG &&) = delete; + CFG &operator=(CFG const &) = delete; + CFG &operator=(CFG &&) = delete; + ~CFG() = default; + + struct BuiltinCall { + MachineInstr *Builtin = nullptr; + bool TerminatesOrReverts = false; + }; + + struct FunctionCall { + const MachineInstr *Call; + /// True, if the call can return. + bool CanContinue = true; + size_t NumArguments = 0; + }; + + struct Assignment { + /// The variables being assigned to also occur as ``output`` in the + /// ``Operation`` containing the assignment, but are also stored here for + /// convenience. + std::vector Variables; + }; + + struct Operation { + /// Stack slots this operation expects at the top of the stack and consumes. + Stack Input; + /// Stack slots this operation leaves on the stack as output. + Stack Output; + std::variant Operation; + }; + + struct FunctionInfo; + /// A basic control flow block containing ``Operation``s acting on the stack. + /// Maintains a list of entry blocks and a typed exit. + struct BasicBlock { + struct InvalidExit {}; + + struct ConditionalJump { + StackSlot Condition; + BasicBlock *NonZero = nullptr; + BasicBlock *Zero = nullptr; + bool FallThrough = false; + MachineInstr *CondJump = nullptr; + MachineInstr *UncondJump = nullptr; + }; + + struct Jump { + BasicBlock *Target = nullptr; + bool FallThrough = false; + bool Backwards = false; + MachineInstr *UncondJump = nullptr; + }; + + struct FunctionReturn { + Stack RetValues; + CFG::FunctionInfo *Info = nullptr; + }; + + struct Terminated {}; + + MachineBasicBlock *MBB; + std::vector Entries; + std::vector Operations; + /// True, if the block is the beginning of a disconnected subgraph. That is, + /// if no block that is reachable from this block is an ancestor of this + /// block. In other words, this is true, if this block is the target of a + /// cut-edge/bridge in the CFG or if the block itself terminates. + bool IsStartOfSubGraph = false; + /// True, if there is a path from this block to a function return. + bool NeedsCleanStack = false; + /// If the block starts a sub-graph and does not lead to a function return, + /// we are free to add junk to it. + bool AllowsJunk() const { return IsStartOfSubGraph && !NeedsCleanStack; } + + std::variant + Exit = InvalidExit{}; + }; + + struct FunctionInfo { + MachineFunction *MF = nullptr; + BasicBlock *Entry = nullptr; + std::vector Parameters; + std::vector Exits; + bool CanContinue = true; + }; + + FunctionInfo FuncInfo; + + /// Container for blocks for explicit ownership. + std::list Blocks; + DenseMap MachineBBToBB; + + BasicBlock &getBlock(const MachineBasicBlock *MBB) { + auto It = MachineBBToBB.find(MBB); + assert(It != MachineBBToBB.end()); + return *It->second; + } + + void createBlock(MachineBasicBlock *MBB) { + auto It = MachineBBToBB.find(MBB); + if (It == MachineBBToBB.end()) { + BasicBlock &Block = Blocks.emplace_back(BasicBlock{MBB, {}, {}}); + MachineBBToBB[MBB] = &Block; + } + } +}; + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPH_H diff --git a/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.cpp b/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.cpp new file mode 100644 index 000000000000..2b768fcfbb5b --- /dev/null +++ b/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.cpp @@ -0,0 +1,389 @@ +//===----- EVMControlFlowGraphBuilder.CPP - CFG builder ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file builds the Control Flow Graph used for the stackification +// algorithm. +// +//===----------------------------------------------------------------------===// + +#include "EVM.h" + +#include "EVMControlFlowGraphBuilder.h" +#include "EVMHelperUtilities.h" +#include "EVMStackDebug.h" +#include "EVMSubtarget.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include "llvm/CodeGen/MachineFunction.h" + +#include +#include + +using namespace llvm; + +#define DEBUG_TYPE "evm-control-flow-graph-builder" + +/// Marks each block that needs to maintain a clean stack. That is each block +/// that has an outgoing path to a function return. +static void markNeedsCleanStack(CFG &Cfg) { + for (CFG::BasicBlock *Exit : Cfg.FuncInfo.Exits) + EVMUtils::BreadthFirstSearch{{Exit}}.run( + [&](CFG::BasicBlock *Block, auto AddChild) { + Block->NeedsCleanStack = true; + // TODO: it seems this is not needed, as the return block has + // no childs. + for (CFG::BasicBlock *Entry : Block->Entries) + AddChild(Entry); + }); +} + +/// Marks each cut-vertex in the CFG, i.e. each block that begins a disconnected +/// sub-graph of the CFG. Entering such a block means that control flow will +/// never return to a previously visited block. +static void markStartsOfSubGraphs(CFG &Cfg) { + CFG::BasicBlock *Entry = Cfg.FuncInfo.Entry; + /** + * Detect bridges following Algorithm 1 in + * https://arxiv.org/pdf/2108.07346.pdf and mark the bridge targets as starts + * of sub-graphs. + */ + std::set Visited; + std::map Disc; + std::map Low; + std::map Parent; + size_t Time = 0; + auto Dfs = [&](CFG::BasicBlock *U, auto Recurse) -> void { + Visited.insert(U); + Disc[U] = Low[U] = Time; + Time++; + + std::vector Children = U->Entries; + std::visit(Overload{[&](CFG::BasicBlock::Jump const &Jump) { + Children.emplace_back(Jump.Target); + }, + [&](CFG::BasicBlock::ConditionalJump const &Jump) { + Children.emplace_back(Jump.Zero); + Children.emplace_back(Jump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &) {}, + [&](CFG::BasicBlock::Terminated const &) { + U->IsStartOfSubGraph = true; + }, + [&](CFG::BasicBlock::InvalidExit const &) { + llvm_unreachable("Unexpected BB terminator"); + }}, + U->Exit); + assert(!EVMUtils::contains(Children, U)); + + for (CFG::BasicBlock *V : Children) { + if (!Visited.count(V)) { + Parent[V] = U; + Recurse(V, Recurse); + Low[U] = std::min(Low[U], Low[V]); + if (Low[V] > Disc[U]) { + // U <-> v is a cut edge in the undirected graph + bool EdgeVtoU = EVMUtils::contains(U->Entries, V); + bool EdgeUtoV = EVMUtils::contains(V->Entries, U); + if (EdgeVtoU && !EdgeUtoV) + // Cut edge V -> U + U->IsStartOfSubGraph = true; + else if (EdgeUtoV && !EdgeVtoU) + // Cut edge U -> v + V->IsStartOfSubGraph = true; + } + } else if (V != Parent[U]) + Low[U] = std::min(Low[U], Disc[V]); + } + }; + Dfs(Entry, Dfs); +} + +std::unique_ptr ControlFlowGraphBuilder::build(MachineFunction &MF, + const LiveIntervals &LIS, + MachineLoopInfo *MLI) { + auto Result = std::make_unique(); + ControlFlowGraphBuilder Builder(*Result, LIS, MLI); + + for (MachineBasicBlock &MBB : MF) + Result->createBlock(&MBB); + + Result->FuncInfo.MF = &MF; + Result->FuncInfo.Entry = &Result->getBlock(&MF.front()); + const Function &F = MF.getFunction(); + if (F.hasFnAttribute(Attribute::NoReturn)) + Result->FuncInfo.CanContinue = false; + + for (MachineBasicBlock &MBB : MF) + Builder.handleBasicBlock(MBB); + + for (MachineBasicBlock &MBB : MF) + Builder.handleBasicBlockSuccessors(MBB); + + markStartsOfSubGraphs(*Result); + markNeedsCleanStack(*Result); + + LLVM_DEBUG({ + dbgs() << "************* CFG *************\n"; + ControlFlowGraphPrinter P(dbgs()); + P(*Result); + }); + + return Result; +} + +void ControlFlowGraphBuilder::handleBasicBlock(MachineBasicBlock &MBB) { + CurrentBlock = &Cfg.getBlock(&MBB); + for (MachineInstr &MI : MBB) + handleMachineInstr(MI); +} + +void ControlFlowGraphBuilder::collectInstrOperands(const MachineInstr &MI, + Stack &Input, + Stack &Output) { + for (const auto &MO : reverse(MI.explicit_uses())) { + if (!MO.isReg()) { + if (MO.isMCSymbol()) + Input.push_back(SymbolSlot{MO.getMCSymbol()}); + continue; + } + + const Register Reg = MO.getReg(); + // SP is not used anyhow. + if (Reg == EVM::SP) + continue; + + StackSlot Slot = VariableSlot{Reg}; + SlotIndex Idx = LIS.getInstructionIndex(MI); + const LiveInterval *LI = &LIS.getInterval(Reg); + LiveQueryResult LRQ = LI->Query(Idx); + const VNInfo *VNI = LRQ.valueIn(); + assert(VNI && "Use of non-existing value"); + if (LI->containsOneValue()) { + assert(!VNI->isPHIDef()); + const MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def); + assert(DefMI && "Dead valno in interval"); + if (DefMI->getOpcode() == EVM::CONST_I256) { + const APInt Imm = DefMI->getOperand(1).getCImm()->getValue(); + Slot = LiteralSlot{std::move(Imm)}; + } + } + Input.push_back(Slot); + } + + unsigned ArgsNumber = 0; + for (const auto &MO : MI.defs()) { + Output.push_back(TemporarySlot{&MI, MO.getReg(), ArgsNumber++}); + } +} + +void ControlFlowGraphBuilder::handleMachineInstr(MachineInstr &MI) { + bool TerminatesOrReverts = false; + unsigned Opc = MI.getOpcode(); + switch (Opc) { + case EVM::ARGUMENT: + Cfg.FuncInfo.Parameters.emplace_back( + VariableSlot{MI.getOperand(0).getReg()}); + return; + case EVM::FCALL: + handleFunctionCall(MI); + break; + case EVM::RET: + handleReturn(MI); + return; + case EVM::JUMP: + [[fallthrough]]; + case EVM::JUMPI: + // Branch instructions are handled separetly. + return; + case EVM::COPY_I256: + case EVM::DATA: + // The copy/data instructions just represnt an assignment. This case is + // handled below. + break; + case EVM::CONST_I256: { + const LiveInterval *LI = &LIS.getInterval(MI.getOperand(0).getReg()); + // We can ignore this instruction, as we will directly create the literal + // slot from the immediate value; + if (LI->containsOneValue()) + return; + } break; + case EVM::REVERT: + [[fallthrough]]; + case EVM::RETURN: + [[fallthrough]]; + case EVM::STOP: + [[fallthrough]]; + case EVM::INVALID: + CurrentBlock->Exit = CFG::BasicBlock::Terminated{}; + TerminatesOrReverts = true; + [[fallthrough]]; + default: { + Stack Input, Output; + collectInstrOperands(MI, Input, Output); + CurrentBlock->Operations.emplace_back( + CFG::Operation{std::move(Input), std::move(Output), + CFG::BuiltinCall{&MI, TerminatesOrReverts}}); + } break; + } + + // Cretae CFG::Assignment object for the MI. + Stack Input, Output; + std::vector Variables; + switch (MI.getOpcode()) { + case EVM::CONST_I256: { + const Register DefReg = MI.getOperand(0).getReg(); + const APInt Imm = MI.getOperand(1).getCImm()->getValue(); + Input.push_back(LiteralSlot{std::move(Imm)}); + Output.push_back(VariableSlot{DefReg}); + Variables.push_back(VariableSlot{DefReg}); + } break; + case EVM::DATA: { + const Register DefReg = MI.getOperand(0).getReg(); + MCSymbol *Sym = MI.getOperand(1).getMCSymbol(); + Input.push_back(SymbolSlot{Sym}); + Output.push_back(VariableSlot{DefReg}); + Variables.push_back(VariableSlot{DefReg}); + } break; + case EVM::COPY_I256: { + // Copy instruction corresponds to the assignment operator, so + // we do not need to create intermediate TmpSlots. + Stack In, Out; + collectInstrOperands(MI, In, Out); + Input = In; + const Register DefReg = MI.getOperand(0).getReg(); + Output.push_back(VariableSlot{DefReg}); + Variables.push_back(VariableSlot{DefReg}); + } break; + default: { + unsigned ArgsNumber = 0; + for (const auto &MO : MI.defs()) { + assert(MO.isReg()); + const Register Reg = MO.getReg(); + Input.push_back(TemporarySlot{&MI, Reg, ArgsNumber++}); + Output.push_back(VariableSlot{Reg}); + Variables.push_back(VariableSlot{Reg}); + } + } break; + } + // We don't need an assignment part of the instructions that do not write + // results. + if (!Input.empty() || !Output.empty()) + CurrentBlock->Operations.emplace_back( + CFG::Operation{std::move(Input), std::move(Output), + CFG::Assignment{std::move(Variables)}}); +} + +void ControlFlowGraphBuilder::handleFunctionCall(const MachineInstr &MI) { + Stack Input, Output; + const Function *Called = getCalledFunction(MI); + bool IsNoReturn = Called->hasFnAttribute(Attribute::NoReturn); + if (IsNoReturn) + CurrentBlock->Exit = CFG::BasicBlock::Terminated{}; + else + Input.push_back(FunctionCallReturnLabelSlot{&MI}); + collectInstrOperands(MI, Input, Output); + CurrentBlock->Operations.emplace_back( + CFG::Operation{Input, Output, + CFG::FunctionCall{&MI, !IsNoReturn, + Input.size() - (IsNoReturn ? 0 : 1)}}); +} + +void ControlFlowGraphBuilder::handleReturn(const MachineInstr &MI) { + Cfg.FuncInfo.Exits.emplace_back(CurrentBlock); + Stack Input, Output; + collectInstrOperands(MI, Input, Output); + CurrentBlock->Exit = + CFG::BasicBlock::FunctionReturn{std::move(Input), &Cfg.FuncInfo}; +} + +static std::pair +getMBBJumps(MachineBasicBlock &MBB) { + MachineInstr *CondJump = nullptr; + MachineInstr *UncondJump = nullptr; + MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend(); + while (I != E) { + if (I->isUnconditionalBranch()) + UncondJump = &*I; + else if (I->isConditionalBranch()) + CondJump = &*I; + ++I; + } + return std::make_pair(CondJump, UncondJump); +} + +void ControlFlowGraphBuilder::handleBasicBlockSuccessors( + MachineBasicBlock &MBB) { + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; + SmallVector Cond; + const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); + CurrentBlock = &Cfg.getBlock(&MBB); + if (TII->analyzeBranch(MBB, TBB, FBB, Cond)) { + if (!std::holds_alternative( + CurrentBlock->Exit) && + !std::holds_alternative( + CurrentBlock->Exit)) + llvm_unreachable("Unexpected MBB termination"); + return; + } + + // This corresponds to a noreturn functions at the end of the MBB. + if (std::holds_alternative(CurrentBlock->Exit)) { +#ifndef NDEBUG + CFG::FunctionCall *Call = std::get_if( + &CurrentBlock->Operations.back().Operation); + assert(Call && !Call->CanContinue); +#endif // NDEBUG + return; + } + + CurrentBlock = &Cfg.getBlock(&MBB); + bool IsLatch = false; + MachineLoop *ML = MLI->getLoopFor(&MBB); + if (ML) { + SmallVector Latches; + ML->getLoopLatches(Latches); + IsLatch = std::any_of( + Latches.begin(), Latches.end(), + [&MBB](const MachineBasicBlock *Latch) { return &MBB == Latch; }); + } + + std::pair MBBJumps = getMBBJumps(MBB); + if (!TBB || (TBB && Cond.empty())) { + // Fall through, or unconditional jump. + bool FallThrough = !TBB; + if (!TBB) { + assert(MBB.getSingleSuccessor()); + TBB = MBB.getFallThrough(); + assert(TBB); + } + CFG::BasicBlock &Target = Cfg.getBlock(TBB); + assert(!MBBJumps.first); + assert((FallThrough && !MBBJumps.second) || !FallThrough); + CurrentBlock->Exit = + CFG::BasicBlock::Jump{&Target, FallThrough, IsLatch, MBBJumps.second}; + EVMUtils::emplace_back_unique(Target.Entries, CurrentBlock); + } else if (TBB && !Cond.empty()) { + assert(!IsLatch); + // Conditional jump + fallthrough or unconditional jump. + bool FallThrough = !FBB; + if (!FBB) { + FBB = MBB.getFallThrough(); + assert(FBB); + } + + CFG::BasicBlock &NonZeroTarget = Cfg.getBlock(TBB); + CFG::BasicBlock &ZeroTarget = Cfg.getBlock(FBB); + assert(Cond[0].isReg()); + auto CondSlot = VariableSlot{Cond[0].getReg()}; + CurrentBlock->Exit = CFG::BasicBlock::ConditionalJump{ + std::move(CondSlot), &NonZeroTarget, &ZeroTarget, + FallThrough, MBBJumps.first, MBBJumps.second}; + + EVMUtils::emplace_back_unique(NonZeroTarget.Entries, CurrentBlock); + EVMUtils::emplace_back_unique(ZeroTarget.Entries, CurrentBlock); + } +} diff --git a/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.h b/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.h new file mode 100644 index 000000000000..ffa501f4fc22 --- /dev/null +++ b/llvm/lib/Target/EVM/EVMControlFlowGraphBuilder.h @@ -0,0 +1,55 @@ +//===----- EVMControlFlowGraphBuilder.h - CFG builder -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file builds the Control Flow Graph used for the stackification +// algorithm. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPHBUILDER_H +#define LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPHBUILDER_H + +#include "EVMControlFlowGraph.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/MachineLoopInfo.h" + +namespace llvm { + +class MachineFunction; +class MachineBasicBlock; + +class ControlFlowGraphBuilder { +public: + ControlFlowGraphBuilder(ControlFlowGraphBuilder const &) = delete; + ControlFlowGraphBuilder &operator=(ControlFlowGraphBuilder const &) = delete; + static std::unique_ptr + build(MachineFunction &MF, const LiveIntervals &LIS, MachineLoopInfo *MLI); + +private: + ControlFlowGraphBuilder(CFG &Cfg, const LiveIntervals &LIS, + MachineLoopInfo *MLI) + : Cfg(Cfg), LIS(LIS), MLI(MLI) {} + + void handleBasicBlock(MachineBasicBlock &MBB); + void handleMachineInstr(MachineInstr &MI); + void handleFunctionCall(const MachineInstr &MI); + void handleReturn(const MachineInstr &MI); + void handleBasicBlockSuccessors(MachineBasicBlock &MBB); + void collectInstrOperands(const MachineInstr &MI, Stack &Input, + Stack &Output); + + CFG &Cfg; + const LiveIntervals &LIS; + MachineLoopInfo *MLI = nullptr; + CFG::BasicBlock *CurrentBlock = nullptr; +}; + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMCONTROLFLOWGRAPH_H diff --git a/llvm/lib/Target/EVM/EVMHelperUtilities.h b/llvm/lib/Target/EVM/EVMHelperUtilities.h new file mode 100644 index 000000000000..6f137535cbcd --- /dev/null +++ b/llvm/lib/Target/EVM/EVMHelperUtilities.h @@ -0,0 +1,182 @@ +//===----- EVMHelperUtilities.h - CFG for stackification --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines different utility templates. In particular, for a partial +// emulation of the C++ range-V3 library functionality; BFS graph traversal; +// the overload pattern support. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H +#define LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H + +#include "llvm/ADT/iterator_range.h" +#include +#include +#include +#include +#include +#include + +namespace llvm { + +template struct Overload : Ts... { + using Ts::operator()...; +}; +template Overload(Ts...) -> Overload; + +template static inline SmallVector iota17(T begin, T end) { + SmallVector R(end - begin); + std::iota(R.begin(), R.end(), begin); + return R; +} + +/// Concatenate the contents of a container into a vector +template +std::vector &operator+=(std::vector &lhs, U &rhs) { + for (auto const &elm : rhs) + lhs.push_back(T(elm)); + return lhs; +} + +template +std::vector &operator+=(std::vector &lhs, U &&rhs) { + std::move(rhs.begin(), rhs.end(), std::back_inserter(lhs)); + return lhs; +} + +/// Concatenate two vectors of elements. +template +inline std::vector operator+(std::vector const &lhs, + std::vector const &rhs) { + std::vector result(lhs); + result += rhs; + return result; +} + +/// Concatenate two vectors of elements, moving them. +template +inline std::vector operator+(std::vector &&lhs, std::vector &&rhs) { + std::vector result(std::move(lhs)); + assert(&lhs != &rhs); + result += std::move(rhs); + return result; +} + +namespace EVMUtils { + +template bool contains(const T &t, const V &v) { + return std::end(t) != std::find(std::begin(t), std::end(t), v); +} + +template +std::optional findOffset(T &&t, V &&v) { + auto begin = std::begin(t); + auto end = std::end(t); + auto it = std::find(begin, end, std::forward(v)); + if (it == end) + return std::nullopt; + return std::distance(begin, it); +} + +template +auto take_last(T &&t, size_t N) -> iterator_range { + auto it = t.end(); + std::advance(it, -N); + return make_range(it, t.end()); +} + +template +auto drop_first(T &&t, size_t N) -> iterator_range { + auto it = t.begin(); + std::advance(it, N); + return make_range(it, t.end()); +} + +template +iterator_range get_reverse(const T &t) { + return llvm::make_range(t.rbegin(), t.rend()); +} + +// Returns a pointer to the entry of map \p m at the key \p k, +// if there is one, and nullptr otherwise. +template +decltype(auto) valueOrNullptr(M &&m, K const &k) { + auto it = m.find(k); + return (it == m.end()) ? nullptr : &it->second; +} + +template auto to_vector(R &&r) { + std::vector v; + v.assign(r.begin(), r.end()); + return v; +} + +/// RAII utility class whose destructor calls a given function. +class ScopeGuard { +public: + explicit ScopeGuard(std::function Func) : Func(std::move(Func)) {} + ~ScopeGuard() { Func(); } + +private: + std::function Func; +}; + +template void emplace_back_unique(T &t, V &&v) { + if (t.end() == std::find(t.begin(), t.end(), v)) + t.emplace_back(v); +} + +/// Generic breadth first search. +/// +/// Note that V needs to be a comparable value type or a pointer. +/// +/// Example: Gather all (recursive) children in a graph starting at (and +/// including) ``root``: +/// +/// Node const* root = ...; +/// std::set allNodes = BreadthFirstSearch{{root}}.run([](Node const* _node, auto&& _addChild) { +/// // Potentially process ``_node``. +/// for (Node const& _child: _node->children()) +/// // Potentially filter the children to be visited. +/// _addChild(&_child); +/// }).visited; +/// +template struct BreadthFirstSearch { + /// Runs the breadth first search. The verticesToTraverse member of the struct + /// needs to be initialized. + /// @param _forEachChild is a callable of the form [...](V const& _node, + /// auto&& _addChild) { ... } that is called for each visited node and is + /// supposed to call _addChild(childNode) for every child node of _node. + template + BreadthFirstSearch &run(ForEachChild &&forEachChild) { + while (!verticesToTraverse.empty()) { + V v = std::move(verticesToTraverse.front()); + verticesToTraverse.pop_front(); + + if (!visited.insert(v).second) + continue; + + forEachChild(v, [this](V vertex) { + verticesToTraverse.emplace_back(std::move(vertex)); + }); + } + return *this; + } + + void abort() { verticesToTraverse.clear(); } + + std::list verticesToTraverse; + std::set visited{}; +}; + +} // namespace EVMUtils +} // namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMHELPERUTILITIES_H diff --git a/llvm/lib/Target/EVM/EVMInstrFormats.td b/llvm/lib/Target/EVM/EVMInstrFormats.td index bd2e6b8b7b0b..ed163bf650f7 100644 --- a/llvm/lib/Target/EVM/EVMInstrFormats.td +++ b/llvm/lib/Target/EVM/EVMInstrFormats.td @@ -40,6 +40,7 @@ class EVMInst let Pattern = []; let AsmString = asmstr; let TSFlags{0} = stack; + let Defs = [ARGUMENTS]; } // Normal instructions. Default instantiation of a EVMInst. diff --git a/llvm/lib/Target/EVM/EVMInstrInfo.cpp b/llvm/lib/Target/EVM/EVMInstrInfo.cpp index caa9e68cb011..9ebbed388599 100644 --- a/llvm/lib/Target/EVM/EVMInstrInfo.cpp +++ b/llvm/lib/Target/EVM/EVMInstrInfo.cpp @@ -173,7 +173,7 @@ unsigned EVMInstrInfo::insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *FBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded) const { - assert(!BytesAdded && "Code is size not handled"); + assert(!BytesAdded && "Code size not handled"); // The number of instructions inserted. unsigned InstrCount = 0; diff --git a/llvm/lib/Target/EVM/EVMInstrInfo.td b/llvm/lib/Target/EVM/EVMInstrInfo.td index 8700834e45d8..8038cb0cd7d2 100644 --- a/llvm/lib/Target/EVM/EVMInstrInfo.td +++ b/llvm/lib/Target/EVM/EVMInstrInfo.td @@ -200,8 +200,7 @@ def ADJCALLSTACKUP [(EVMcallseq_end timm:$amt1, timm:$amt2)]>; } -let isCodeGenOnly = 1 in { -let hasSideEffects = 1 in +let hasSideEffects = 1, Defs = [], Uses = [ARGUMENTS] in def ARGUMENT : NRI<(outs GPR:$res), (ins i256imm:$argno), [(set GPR:$res, (EVMargument timm:$argno))], @@ -219,7 +218,6 @@ def CONST_I256 let isAsCheapAsAMove = 1 in def COPY_I256 : NRI<(outs GPR:$res), (ins GPR:$src), [], "COPY_I256 $res, $src">; -} def : Pat<(i256 (EVMTargetAddrWrapper tglobaladdr:$addr)), (CONST_I256 tglobaladdr:$addr)>; diff --git a/llvm/lib/Target/EVM/EVMLinkRuntime.cpp b/llvm/lib/Target/EVM/EVMLinkRuntime.cpp index a758131d9c7e..6cdf1314621e 100644 --- a/llvm/lib/Target/EVM/EVMLinkRuntime.cpp +++ b/llvm/lib/Target/EVM/EVMLinkRuntime.cpp @@ -81,6 +81,8 @@ static bool EVMLinkRuntimeImpl(Module &M, const char *ModuleToLink) { exit(1); } + // TODO: remove this after ensuring the stackification + // algorithm can deal with a high register pressure. for (auto &F : M.functions()) { if (!F.isDeclaration()) { F.addFnAttr(Attribute::NoInline); diff --git a/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.cpp b/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.cpp new file mode 100644 index 000000000000..ac1c3f0df1af --- /dev/null +++ b/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.cpp @@ -0,0 +1,432 @@ +//===--- EVMOptimizedCodeTransform.h - Stack layout generator ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file transforms the stack layout back into the Machine IR instructions +// in 'stackified' form using the EVMAssembly class. +// +//===----------------------------------------------------------------------===// + +#include "EVMOptimizedCodeTransform.h" +#include "EVMControlFlowGraphBuilder.h" +#include "EVMHelperUtilities.h" +#include "EVMStackDebug.h" +#include "EVMStackLayoutGenerator.h" +#include "EVMStackShuffler.h" +#include "llvm/Support/ErrorHandling.h" + +#include + +using namespace llvm; + +#define DEBUG_TYPE "evm-optimized-code-transform" + +void EVMOptimizedCodeTransform::run(EVMAssembly &Assembly, MachineFunction &MF, + const LiveIntervals &LIS, + MachineLoopInfo *MLI) { + std::unique_ptr Cfg = ControlFlowGraphBuilder::build(MF, LIS, MLI); + StackLayout Layout = StackLayoutGenerator::run(*Cfg); + EVMOptimizedCodeTransform optimizedCodeTransform(Assembly, *Cfg, Layout, MF); + optimizedCodeTransform(); +} + +void EVMOptimizedCodeTransform::operator()(CFG::FunctionCall const &Call) { + // Validate stack. + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); + assert(CurrentStack.size() >= Call.NumArguments + (Call.CanContinue ? 1 : 0)); + + // Assert that we got the correct return label on stack. + if (Call.CanContinue) { + auto const *returnLabelSlot = std::get_if( + &CurrentStack.at(CurrentStack.size() - Call.NumArguments - 1)); + assert(returnLabelSlot && returnLabelSlot->Call == Call.Call); + } + + // Emit code. + const MachineOperand *CalleeOp = Call.Call->explicit_uses().begin(); + assert(CalleeOp->isGlobal()); + Assembly.appendFuncCall(Call.Call, CalleeOp->getGlobal(), + Call.Call->getNumExplicitDefs() - Call.NumArguments - + (Call.CanContinue ? 1 : 0), + Call.CanContinue ? CallToReturnMCSymbol.at(Call.Call) + : nullptr); + if (Call.CanContinue) + Assembly.appendLabel(); + + // Update stack. + // Remove arguments and return label from CurrentStack. + for (size_t I = 0; I < Call.NumArguments + (Call.CanContinue ? 1 : 0); ++I) + CurrentStack.pop_back(); + + // Push return values to CurrentStack. + unsigned Idx = 0; + for (const auto &MO : Call.Call->defs()) { + assert(MO.isReg()); + CurrentStack.emplace_back(TemporarySlot{Call.Call, MO.getReg(), Idx++}); + } + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); +} + +void EVMOptimizedCodeTransform::operator()(CFG::BuiltinCall const &Call) { + size_t NumArgs = Call.Builtin->getNumExplicitOperands() - + Call.Builtin->getNumExplicitDefs(); + // Validate stack. + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); + assert(CurrentStack.size() >= NumArgs); + // TODO: assert that we got a correct stack for the call. + + // Emit code. + Assembly.appendInstruction(Call.Builtin); + + // Update stack. + // Remove arguments from CurrentStack. + for (size_t i = 0; i < NumArgs; ++i) + CurrentStack.pop_back(); + + // Push return values to CurrentStack. + unsigned Idx = 0; + for (const auto &MO : Call.Builtin->defs()) { + assert(MO.isReg()); + CurrentStack.emplace_back(TemporarySlot{Call.Builtin, MO.getReg(), Idx++}); + } + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); +} + +void EVMOptimizedCodeTransform::operator()(CFG::Assignment const &Assignment) { + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); + + // Invalidate occurrences of the assigned variables. + for (auto &CurrentSlot : CurrentStack) + if (VariableSlot const *VarSlot = std::get_if(&CurrentSlot)) + if (EVMUtils::contains(Assignment.Variables, *VarSlot)) + CurrentSlot = JunkSlot{}; + + // Assign variables to current stack top. + assert(CurrentStack.size() >= Assignment.Variables.size()); + auto StackRange = + EVMUtils::take_last(CurrentStack, Assignment.Variables.size()); + auto RangeIt = StackRange.begin(), RangeE = StackRange.end(); + auto VarsIt = Assignment.Variables.begin(), + VarsE = Assignment.Variables.end(); + for (; (RangeIt != RangeE) && (VarsIt != VarsE); ++RangeIt, ++VarsIt) + *RangeIt = *VarsIt; +} + +EVMOptimizedCodeTransform::EVMOptimizedCodeTransform(EVMAssembly &Assembly, + CFG const &Cfg, + StackLayout const &Layout, + MachineFunction &MF) + : Assembly(Assembly), Layout(Layout), FuncInfo(&Cfg.FuncInfo), MF(MF) {} + +void EVMOptimizedCodeTransform::assertLayoutCompatibility( + Stack const &SourceStack, Stack const &TargetStack) { + assert(SourceStack.size() == TargetStack.size()); + for (unsigned Idx = 0; Idx < SourceStack.size(); ++Idx) + assert(std::holds_alternative(TargetStack[Idx]) || + SourceStack[Idx] == TargetStack[Idx]); +} + +void EVMOptimizedCodeTransform::createStackLayout(Stack TargetStack) { + auto SlotVariableName = [](StackSlot const &Slot) { + return std::visit( + Overload{ + [&](VariableSlot const &Var) { + SmallString<1024> StrBuf; + llvm::raw_svector_ostream OS(StrBuf); + OS << printReg(Var.VirtualReg, nullptr, 0, nullptr); + return std::string(StrBuf.c_str()); + }, + [&](const FunctionCallReturnLabelSlot &) { return std::string(); }, + [&](const FunctionReturnLabelSlot &) { return std::string(); }, + [&](const LiteralSlot &) { return std::string(); }, + [&](const SymbolSlot &) { return std::string(); }, + [&](const TemporarySlot &) { return std::string(); }, + [&](const JunkSlot &) { return std::string(); }}, + Slot); + }; + + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); + // ::createStackLayout asserts that it has successfully achieved the target + // layout. + ::createStackLayout( + CurrentStack, TargetStack, + // Swap callback. + [&](unsigned I) { + assert(static_cast(CurrentStack.size()) == + Assembly.getStackHeight()); + assert(I > 0 && I < CurrentStack.size()); + if (I <= 16) { + Assembly.appendSWAPInstruction(I); + } else { + int Deficit = static_cast(I) - 16; + StackSlot const &DeepSlot = + CurrentStack.at(CurrentStack.size() - I - 1); + std::string VarNameDeep = SlotVariableName(DeepSlot); + std::string VarNameTop = SlotVariableName(CurrentStack.back()); + std::string Msg = + (Twine("Cannot swap ") + + (VarNameDeep.empty() ? ("Slot " + stackSlotToString(DeepSlot)) + : (Twine("Variable ") + VarNameDeep)) + + " with " + + (VarNameTop.empty() + ? ("Slot " + stackSlotToString(CurrentStack.back())) + : (Twine("Variable ") + VarNameTop)) + + ": too deep in the stack by " + std::to_string(Deficit) + + " slots in " + stackToString(CurrentStack)) + .str(); + + report_fatal_error(FuncInfo->MF->getName() + Twine(": ") + Msg); + } + }, + // Push or dup callback. + [&](StackSlot const &Slot) { + assert(static_cast(CurrentStack.size()) == + Assembly.getStackHeight()); + // Dup the slot, if already on stack and reachable. + if (auto Depth = EVMUtils::findOffset( + EVMUtils::get_reverse(CurrentStack), Slot)) { + if (*Depth < 16) { + Assembly.appendDUPInstruction(static_cast(*Depth + 1)); + return; + } else if (!canBeFreelyGenerated(Slot)) { + std::string VarName = SlotVariableName(Slot); + Twine Msg = + ((VarName.empty() ? "Slot " + stackSlotToString(Slot) + : Twine("Variable ") + VarName) + + " is " + std::to_string(*Depth - 15) + + " too deep in the stack " + stackToString(CurrentStack)) + .str(); + + report_fatal_error(MF.getName() + ": " + Msg); + return; + } + // else: the slot is too deep in stack, but can be freely generated, + // we fall through to push it again. + } + + // The slot can be freely generated or is an unassigned return variable. + // Push it. + std::visit( + Overload{[&](LiteralSlot const &Literal) { + Assembly.appendConstant(Literal.Value); + }, + [&](SymbolSlot const &Symbol) { + Assembly.appendSymbol(Symbol.Symbol); + }, + [&](FunctionReturnLabelSlot const &) { + llvm_unreachable("Cannot produce function return label"); + }, + [&](const FunctionCallReturnLabelSlot &ReturnLabel) { + if (!CallToReturnMCSymbol.count(ReturnLabel.Call)) + CallToReturnMCSymbol[ReturnLabel.Call] = + Assembly.createFuncRetSymbol(); + + Assembly.appendLabelReference( + CallToReturnMCSymbol[ReturnLabel.Call]); + }, + [&](const VariableSlot &Variable) { + llvm_unreachable("Variable not found on stack"); + }, + [&](const TemporarySlot &) { + llvm_unreachable("Function call result requested, but " + "not found on stack."); + }, + [&](const JunkSlot &) { + // Note: this will always be popped, so we can push + // anything. + Assembly.appendConstant(0); + }}, + Slot); + }, + // Pop callback. + [&]() { Assembly.appendPOPInstruction(); }); + + assert(Assembly.getStackHeight() == static_cast(CurrentStack.size())); +} + +void EVMOptimizedCodeTransform::operator()(CFG::BasicBlock const &Block) { + // Current location for the entry BB was set up in operator()(). + if (&Block != FuncInfo->Entry) + Assembly.setCurrentLocation(Block.MBB); + + // Assert that this is the first visit of the block and mark as generated. + auto It = GeneratedBlocks.insert(&Block); + assert(It.second); + + auto const &BlockInfo = Layout.blockInfos.at(&Block); + + // Assert that the stack is valid for entering the block. + assertLayoutCompatibility(CurrentStack, BlockInfo.entryLayout); + + // Might set some slots to junk, if not required by the block. + CurrentStack = BlockInfo.entryLayout; + assert(static_cast(CurrentStack.size()) == Assembly.getStackHeight()); + + // Emit jumpdest, if required. + if (EVMUtils::valueOrNullptr(BlockLabels, &Block)) + Assembly.appendLabel(); + + for (auto const &Operation : Block.Operations) { + // Create required layout for entering the Operation. + createStackLayout(Layout.operationEntryLayout.at(&Operation)); + + // Assert that we have the inputs of the Operation on stack top. + assert(static_cast(CurrentStack.size()) == Assembly.getStackHeight()); + assert(CurrentStack.size() >= Operation.Input.size()); + size_t BaseHeight = CurrentStack.size() - Operation.Input.size(); + assertLayoutCompatibility(EVMUtils::to_vector(EVMUtils::take_last( + CurrentStack, Operation.Input.size())), + Operation.Input); + + // Perform the Operation. + std::visit(*this, Operation.Operation); + + // Assert that the Operation produced its proclaimed output. + assert(static_cast(CurrentStack.size()) == Assembly.getStackHeight()); + assert(CurrentStack.size() == BaseHeight + Operation.Output.size()); + assert(CurrentStack.size() >= Operation.Output.size()); + assertLayoutCompatibility(EVMUtils::to_vector(EVMUtils::take_last( + CurrentStack, Operation.Output.size())), + Operation.Output); + } + + // Exit the block. + std::visit( + Overload{ + [&](const CFG::BasicBlock::InvalidExit &) { + llvm_unreachable("Unexpected BB terminator"); + }, + [&](const CFG::BasicBlock::Jump &Jump) { + // Create the stack expected at the jump target. + createStackLayout(Layout.blockInfos.at(Jump.Target).entryLayout); + + // If this is the only jump to the block which is a fallthrought + // we can directly continue with the target block. + if (Jump.Target->Entries.size() == 1 && Jump.FallThrough) + assert(!Jump.Backwards && !BlockLabels.count(Jump.Target)); + + if (!BlockLabels.count(Jump.Target)) + BlockLabels[Jump.Target] = Jump.Target->MBB->getSymbol(); + + if (Jump.UncondJump) + Assembly.appendUncondJump(Jump.UncondJump, Jump.Target->MBB); + + if (!GeneratedBlocks.count(Jump.Target)) + (*this)(*Jump.Target); + }, + [&](CFG::BasicBlock::ConditionalJump const &CondJump) { + // Create the shared entry layout of the jump targets, which is + // stored as exit layout of the current block. + createStackLayout(BlockInfo.exitLayout); + + // Create labels for the targets, if not already present. + if (!BlockLabels.count(CondJump.NonZero)) + BlockLabels[CondJump.NonZero] = + CondJump.NonZero->MBB->getSymbol(); + + if (!BlockLabels.count(CondJump.Zero)) + BlockLabels[CondJump.Zero] = CondJump.Zero->MBB->getSymbol(); + + // Assert that we have the correct condition on stack. + assert(!CurrentStack.empty()); + assert(CurrentStack.back() == CondJump.Condition); + + // Emit the conditional jump to the non-zero label and update the + // stored stack. + assert(CondJump.CondJump); + Assembly.appendCondJump(CondJump.CondJump, CondJump.NonZero->MBB); + CurrentStack.pop_back(); + + // Assert that we have a valid stack for both jump targets. + assertLayoutCompatibility( + CurrentStack, + Layout.blockInfos.at(CondJump.NonZero).entryLayout); + assertLayoutCompatibility( + CurrentStack, Layout.blockInfos.at(CondJump.Zero).entryLayout); + + { + // Restore the stack afterwards for the non-zero case below. + EVMUtils::ScopeGuard stackRestore([storedStack = CurrentStack, + this]() { + CurrentStack = std::move(storedStack); + Assembly.setStackHeight(static_cast(CurrentStack.size())); + }); + + // If we have already generated the zero case, jump to it, + // otherwise generate it in place. + if (CondJump.UncondJump) + Assembly.appendUncondJump(CondJump.UncondJump, + CondJump.Zero->MBB); + + if (!GeneratedBlocks.count(CondJump.Zero)) + (*this)(*CondJump.Zero); + } + // Note that each block visit terminates control flow, so we cannot + // fall through from the zero case. + + // Generate the non-zero block, if not done already. + if (!GeneratedBlocks.count(CondJump.NonZero)) + (*this)(*CondJump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &FuncReturn) { + assert(FuncInfo->CanContinue); + + // Construct the function return layout, which is fully determined + // by the function signature. + Stack ExitStack = FuncReturn.RetValues; + + ExitStack.emplace_back( + FunctionReturnLabelSlot{FuncReturn.Info->MF}); + + // Create the function return layout and jump. + createStackLayout(ExitStack); + Assembly.appendJump(0); + }, + [&](CFG::BasicBlock::Terminated const &) { + assert(!Block.Operations.empty()); + if (const CFG::BuiltinCall *BuiltinCall = + std::get_if( + &Block.Operations.back().Operation)) + assert(BuiltinCall->TerminatesOrReverts); + else if (CFG::FunctionCall const *FunctionCall = + std::get_if( + &Block.Operations.back().Operation)) + assert(!FunctionCall->CanContinue); + else + llvm_unreachable("Unexpected BB terminator"); + }}, + Block.Exit); + + // TODO: We could assert that the last emitted assembly item terminated or was + // an (unconditional) jump. + CurrentStack.clear(); + Assembly.setStackHeight(0); +} + +void EVMOptimizedCodeTransform::operator()() { + assert(CurrentStack.empty() && Assembly.getStackHeight() == 0); + Assembly.setCurrentLocation(FuncInfo->Entry->MBB); + + assert(!BlockLabels.count(FuncInfo->Entry)); + + // Create function entry layout in CurrentStack. + if (FuncInfo->CanContinue) + CurrentStack.emplace_back(FunctionReturnLabelSlot{FuncInfo->MF}); + + for (auto const &Param : reverse(FuncInfo->Parameters)) + CurrentStack.emplace_back(Param); + + Assembly.setStackHeight(static_cast(CurrentStack.size())); + Assembly.appendLabel(); + + // Create the entry layout of the function body block and visit. + createStackLayout(Layout.blockInfos.at(FuncInfo->Entry).entryLayout); + + (*this)(*FuncInfo->Entry); + + Assembly.finalize(); +} diff --git a/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.h b/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.h new file mode 100644 index 000000000000..dce8819454bc --- /dev/null +++ b/llvm/lib/Target/EVM/EVMOptimizedCodeTransform.h @@ -0,0 +1,87 @@ +//===--- EVMOptimizedCodeTransform.h - Stack layout generator ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file transforms the stack layout back into the Machine IR instructions +// in 'stackified' form using the EVMAssembly class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMOPTIMIZEDCODETRANSFORM_H +#define LLVM_LIB_TARGET_EVM_EVMOPTIMIZEDCODETRANSFORM_H + +#include "EVMAssembly.h" +#include "EVMControlFlowGraph.h" +#include "EVMStackLayoutGenerator.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/MachineLoopInfo.h" + +#include +#include + +namespace llvm { + +class MachineInstr; +class MCSymbol; + +class EVMOptimizedCodeTransform { +public: + static void run(EVMAssembly &Assembly, MachineFunction &MF, + const LiveIntervals &LIS, MachineLoopInfo *MLI); + + /// Generate code for the function call \p Call. Only public for using with + /// std::visit. + void operator()(CFG::FunctionCall const &Call); + /// Generate code for the builtin call \p Call. Only public for using with + /// std::visit. + void operator()(CFG::BuiltinCall const &Call); + /// Generate code for the assignment \p Assignment. Only public for using + /// with std::visit. + void operator()(CFG::Assignment const &Assignment); + +private: + EVMOptimizedCodeTransform(EVMAssembly &Assembly, const CFG &Cfg, + const StackLayout &Layout, MachineFunction &MF); + + /// Assert that it is valid to transition from \p SourceStack to \p + /// TargetStack. That is \p SourceStack matches each slot in \p + /// TargetStack that is not a JunkSlot exactly. + static void assertLayoutCompatibility(Stack const &SourceStack, + Stack const &TargetStack); + + /// Shuffles CurrentStack to the desired \p TargetStack while emitting the + /// shuffling code to Assembly. + void createStackLayout(Stack TargetStack); + + /// Generate code for the given block \p Block. + /// Expects the current stack layout 'CurrentStack' to be a stack layout that + /// is compatible with the entry layout expected by the block. Recursively + /// generates code for blocks that are jumped to. The last emitted assembly + /// instruction is always an unconditional jump or terminating. Always exits + /// with an empty stack layout. + void operator()(CFG::BasicBlock const &Block); + + /// Generate code for the given function. + /// Resets CurrentStack. + void operator()(); + + EVMAssembly &Assembly; + StackLayout const &Layout; + CFG::FunctionInfo const *FuncInfo = nullptr; + MachineFunction &MF; + + Stack CurrentStack; + DenseMap CallToReturnMCSymbol; + DenseMap BlockLabels; + /// Set of blocks already generated. If any of the contained blocks is ever + /// jumped to, BlockLabels should contain a jump label for it. + std::set GeneratedBlocks; +}; + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMOPTIMIZEDCODETRANSFORM_H diff --git a/llvm/lib/Target/EVM/EVMSingleUseExpression.cpp b/llvm/lib/Target/EVM/EVMSingleUseExpression.cpp index 8337dc7875d1..f31a49b53033 100644 --- a/llvm/lib/Target/EVM/EVMSingleUseExpression.cpp +++ b/llvm/lib/Target/EVM/EVMSingleUseExpression.cpp @@ -311,6 +311,10 @@ static bool isSafeToMove(const MachineOperand *Def, const MachineOperand *Use, if (NextI == Insert) return true; + // Don't move ARGUMENT instructions, as stackification pass relies on this. + if (DefI->getOpcode() == EVM::ARGUMENT) + return false; + // Check for register dependencies. SmallVector MutableRegisters; for (const MachineOperand &MO : DefI->operands()) { diff --git a/llvm/lib/Target/EVM/EVMSplitCriticalEdges.cpp b/llvm/lib/Target/EVM/EVMSplitCriticalEdges.cpp new file mode 100644 index 000000000000..1d4c55ccbc72 --- /dev/null +++ b/llvm/lib/Target/EVM/EVMSplitCriticalEdges.cpp @@ -0,0 +1,225 @@ +//===----- EVMSplitCriticalEdges.cpp - Split Critical Edges ------*- C++ +//-*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file performs spliting of critical edges. +// +//===----------------------------------------------------------------------===// + +#include "EVM.h" +#include "EVMMachineFunctionInfo.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +#define DEBUG_TYPE "evm-split-critical-edges" + +namespace { +class EVMSplitCriticalEdges final : public MachineFunctionPass { +public: + static char ID; // Pass identification, replacement for typeid + + EVMSplitCriticalEdges() : MachineFunctionPass(ID) {} + + StringRef getPassName() const override { return "EVM spliting latch blocks"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + +private: + /* + bool splitLatch(MachineBasicBlock *Latch, MachineBasicBlock *Header); + + MachineBasicBlock *createNewBlockAfter(MachineBasicBlock &OrigMBB); + + void splitLatchBlockBeforeInstr(MachineInstr *MI, + MachineBasicBlock *LoopHeader); + + void createNewLatchBlock(MachineInstr *CondBr, MachineBasicBlock *Latch, + MachineBasicBlock *LoopHeader); + */ + bool splitCriticalEdges(); + + MachineFunction *MF = nullptr; + const TargetInstrInfo *TII = nullptr; +}; +} // end anonymous namespace + +char EVMSplitCriticalEdges::ID = 0; + +INITIALIZE_PASS_BEGIN(EVMSplitCriticalEdges, DEBUG_TYPE, + "Split Latch/exiting mbb", false, false) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_END(EVMSplitCriticalEdges, DEBUG_TYPE, + "Split Latch/exiting mbb", false, false) + +FunctionPass *llvm::createEVMSplitCriticalEdges() { + return new EVMSplitCriticalEdges(); +} + +/* +/// Insert a new empty MachineBasicBlock after \p OrigMBB +MachineBasicBlock * +EVMSplitCriticalEdges::createNewBlockAfter(MachineBasicBlock &OrigMBB) { + // Create a new MBB for the code after the OrigBB. + MachineBasicBlock *NewBB = + MF->CreateMachineBasicBlock(OrigMBB.getBasicBlock()); + MF->insert(++OrigMBB.getIterator(), NewBB); + return NewBB; +} + +void EVMSplitCriticalEdges::createNewLatchBlock(MachineInstr *CondBr, + MachineBasicBlock *Latch, + MachineBasicBlock *LoopHeader) { + MachineBasicBlock *NewLatch = createNewBlockAfter(*Latch); + // Update branch target of the conditional branch. + CondBr->getOperand(0).setMBB(NewLatch); + + // Insert unconditional jump to the new block. + TII->insertUnconditionalBranch(*NewLatch, LoopHeader, CondBr->getDebugLoc()); + + // Update the succesor lists according to the transformation. + Latch->replaceSuccessor(LoopHeader, NewLatch); + NewLatch->addSuccessor(LoopHeader); + + // Cleanup potential unconditional branch to successor block. + Latch->updateTerminator(NewLatch); +} + +/// Split the latch basic block containing MI into two blocks, which are joined +/// by an unconditional branch. +void EVMSplitCriticalEdges::splitLatchBlockBeforeInstr( + MachineInstr *MI, MachineBasicBlock *LoopHeader) { + MachineBasicBlock *Latch = MI->getParent(); + + // Create a new MBB after the OrigBB. + MachineBasicBlock *NewLatch = + MF->CreateMachineBasicBlock(Latch->getBasicBlock()); + MF->insert(++Latch->getIterator(), NewLatch); + + // Splice the instructions starting with MI over to NewBB. + NewLatch->splice(NewLatch->end(), Latch, MI->getIterator(), Latch->end()); + + // Add an unconditional branch from OrigBB to NewBB. + TII->insertUnconditionalBranch(*Latch, NewLatch, DebugLoc()); + + Latch->replaceSuccessor(LoopHeader, NewLatch); + NewLatch->addSuccessor(LoopHeader); + + // Cleanup potential unconditional branch to successor block. + Latch->updateTerminator(NewLatch); +} + +bool EVMSplitCriticalEdges::splitLatch(MachineBasicBlock *Latch, + MachineBasicBlock *Header) { + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; + SmallVector Cond; + const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); + if (TII->analyzeBranch(*Latch, TBB, FBB, Cond)) + llvm_unreachable("Unexpected Latch terminator"); + + // Return if this is an unconditional branch + if (!TBB || Cond.empty()) + return false; + + MachineInstr *CondJump = Cond[0].getParent(); + if (!FBB) { + MachineBasicBlock *FH = Latch->getFallThrough(); + assert(FH); + // Insert an unconditional jump to the Latch, because "false" block isn't + // fallthrough anymore. + TII->insertUnconditionalBranch(*Latch, FH, CondJump->getDebugLoc()); + assert(TBB = Header); + createNewLatchBlock(CondJump, Latch, Header); + } else { + // There are both conditional and unconditional branches at the BB end. + // We need to figure out which branch targets the loop header. + MachineInstr *JumpToHeader = nullptr; + MachineBasicBlock::iterator I = Latch->getFirstTerminator(), + E = Latch->end(); + for (; I != E; ++I) { + if (I->isConditionalBranch() || I->isUnconditionalBranch()) + if (I->getOperand(0).getMBB() == Header) { + JumpToHeader = &*I; + break; + } + } + if (JumpToHeader->isUnconditionalBranch()) + splitLatchBlockBeforeInstr(JumpToHeader, Header); + else + createNewLatchBlock(JumpToHeader, Latch, Header); + } + + return true; +} +*/ + +bool EVMSplitCriticalEdges::splitCriticalEdges() { + SetVector> ToSplit; + for (MachineBasicBlock &MBB : *MF) { + if (MBB.pred_size() > 1) { + for (MachineBasicBlock *Pred : MBB.predecessors()) { + if (Pred->succ_size() > 1) + ToSplit.insert(std::make_pair(Pred, &MBB)); + } + } + } + + bool Changed = false; + for (const auto &Pair : ToSplit) { + auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *this); + if (NewSucc != nullptr) { + Pair.first->updateTerminator(NewSucc); + NewSucc->updateTerminator(Pair.second); + LLVM_DEBUG(dbgs() << " *** Splitting critical edge: " + << printMBBReference(*Pair.first) << " -- " + << printMBBReference(*NewSucc) << " -- " + << printMBBReference(*Pair.second) << '\n'); + Changed = true; + } else { + llvm_unreachable(" Cannot break critical edge"); + } + } + return Changed; +} + +bool EVMSplitCriticalEdges::runOnMachineFunction(MachineFunction &Mf) { + MF = &Mf; + LLVM_DEBUG({ + dbgs() << "********** Spliting Latch/exiting MBB **********\n" + << "********** Function: " << Mf.getName() << '\n'; + }); + + const TargetSubtargetInfo &ST = MF->getSubtarget(); + TII = ST.getInstrInfo(); + + bool Changed = splitCriticalEdges(); + /* + MachineLoopInfo *MLI = &getAnalysis(); + SmallVector Worklist(MLI->begin(), MLI->end()); + while (!Worklist.empty()) { + MachineLoop *ML = Worklist.pop_back_val(); + SmallVector Latches; + ML->getLoopLatches(Latches); + Worklist.append(ML->begin(), ML->end()); + for (MachineBasicBlock *Latch : Latches) + Changed |= splitLatch(Latch, ML->getHeader()); + } +*/ + return Changed; +} diff --git a/llvm/lib/Target/EVM/EVMStackDebug.cpp b/llvm/lib/Target/EVM/EVMStackDebug.cpp new file mode 100644 index 000000000000..2c2b199322fd --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackDebug.cpp @@ -0,0 +1,297 @@ +//===-- EVMStackDebug.cpp - Debugging of the stackification -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares classes for dumping a state of stackifcation related data +// structures and algorithms. +// +//===----------------------------------------------------------------------===// + +#include "EVMStackDebug.h" +#include "EVMHelperUtilities.h" +#include "EVMStackLayoutGenerator.h" +#include "EVMSubtarget.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include + +using namespace llvm; + +#ifndef NDEBUG +static StringRef getInstName(const MachineInstr *MI) { + const MachineFunction *MF = MI->getParent()->getParent(); + const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); + return TII->getName(MI->getOpcode()); +} + +const Function *llvm::getCalledFunction(const MachineInstr &MI) { + for (const MachineOperand &MO : MI.operands()) { + if (!MO.isGlobal()) + continue; + const Function *Func = dyn_cast(MO.getGlobal()); + if (Func != nullptr) + return Func; + } + return nullptr; +} + +std::string llvm::stackToString(const Stack &S) { + std::string Result("[ "); + for (auto const &Slot : S) + Result += stackSlotToString(Slot) + ' '; + Result += ']'; + return Result; +} + +std::string llvm::stackSlotToString(const StackSlot &Slot) { + return std::visit( + Overload{ + [](const FunctionCallReturnLabelSlot &Ret) -> std::string { + return "RET[" + + std::string(getCalledFunction(*Ret.Call)->getName()) + "]"; + }, + [](const FunctionReturnLabelSlot &) -> std::string { return "RET"; }, + [](const VariableSlot &Var) -> std::string { + SmallString<64> S; + raw_svector_ostream OS(S); + OS << printReg(Var.VirtualReg, nullptr, 0, nullptr); + return std::string(S); + ; + }, + [](const LiteralSlot &Literal) -> std::string { + SmallString<64> S; + Literal.Value.toStringSigned(S); + return std::string(S); + }, + [](const SymbolSlot &Symbol) -> std::string { + return std::string(Symbol.Symbol->getName()); + }, + [](const TemporarySlot &Tmp) -> std::string { + SmallString<128> S; + raw_svector_ostream OS(S); + OS << "TMP[" << getInstName(Tmp.MI) << ", "; + OS << std::to_string(Tmp.Index) + "]"; + return std::string(S); + }, + [](const JunkSlot &Junk) -> std::string { return "JUNK"; }}, + Slot); + ; +} + +void ControlFlowGraphPrinter::operator()(const CFG &Cfg) { + (*this)(Cfg.FuncInfo); + for (const auto &Block : Cfg.Blocks) + printBlock(Block); +} + +void ControlFlowGraphPrinter::operator()(CFG::FunctionInfo const &Info) { + OS << "Function: " << Info.MF->getName() << '\n'; + OS << "Entry block: " << getBlockId(*Info.Entry) << ";\n"; +} + +std::string ControlFlowGraphPrinter::getBlockId(CFG::BasicBlock const &Block) { + return std::to_string(Block.MBB->getNumber()) + "." + + std::string(Block.MBB->getName()); +} + +void ControlFlowGraphPrinter::printBlock(CFG::BasicBlock const &Block) { + OS << "Block" << getBlockId(Block) << " [\n"; + + // Verify that the entries of this block exit into this block. + for (auto const &Entry : Block.Entries) { + std::visit( + Overload{ + [&](CFG::BasicBlock::Jump &Jump) { + assert((Jump.Target == &Block) && "Invalid control flow graph"); + }, + [&](CFG::BasicBlock::ConditionalJump &CondJump) { + assert((CondJump.Zero == &Block || CondJump.NonZero == &Block) && + "Invalid control flow graph"); + }, + [&](const auto &) { assert(0 && "Invalid control flow graph."); }}, + Entry->Exit); + } + + for (auto const &Op : Block.Operations) { + std::visit(Overload{[&](const CFG::FunctionCall &FuncCall) { + const MachineOperand *Callee = + FuncCall.Call->explicit_uses().begin(); + OS << Callee->getGlobal()->getName() << ": "; + }, + [&](const CFG::BuiltinCall &BuiltinCall) { + OS << getInstName(BuiltinCall.Builtin) << ": "; + }, + [&](const CFG::Assignment &Assignment) { + OS << "Assignment("; + for (const auto &Var : Assignment.Variables) + OS << printReg(Var.VirtualReg, nullptr, 0, nullptr) + << ", "; + OS << "): "; + }}, + Op.Operation); + OS << stackToString(Op.Input) << " => " << stackToString(Op.Output) << '\n'; + } + OS << "];\n"; + + std::visit( + Overload{[&](const CFG::BasicBlock::Jump &Jump) { + OS << "Block" << getBlockId(Block) << "Exit [label=\""; + OS << "Jump\" FallThrough:" << Jump.FallThrough << "];\n"; + if (Jump.Backwards) + OS << "Backwards"; + OS << "Block" << getBlockId(Block) << "Exit -> Block" + << getBlockId(*Jump.Target) << ";\n"; + }, + [&](const CFG::BasicBlock::ConditionalJump &CondJump) { + OS << "Block" << getBlockId(Block) << "Exit [label=\"{ "; + OS << stackSlotToString(CondJump.Condition); + OS << "| { <0> Zero | <1> NonZero }}\" FallThrough:"; + OS << CondJump.FallThrough << "];\n"; + OS << "Block" << getBlockId(Block); + OS << "Exit:0 -> Block" << getBlockId(*CondJump.Zero) << ";\n"; + OS << "Block" << getBlockId(Block); + OS << "Exit:1 -> Block" << getBlockId(*CondJump.NonZero) + << ";\n"; + }, + [&](const CFG::BasicBlock::FunctionReturn &Return) { + OS << "Block" << getBlockId(Block) + << "Exit [label=\"FunctionReturn[" + << Return.Info->MF->getName() << "]\"];\n"; + OS << "Return values: " << stackToString(Return.RetValues); + }, + [&](const CFG::BasicBlock::Terminated &) { + OS << "Block" << getBlockId(Block) + << "Exit [label=\"Terminated\"];\n"; + }, + [&](const CFG::BasicBlock::InvalidExit &) { + assert(0 && "Invalid basic block exit"); + }}, + Block.Exit); + OS << "\n"; +} + +void StackLayoutPrinter::operator()(CFG::BasicBlock const &Block, + bool IsMainEntry) { + if (IsMainEntry) { + OS << "Entry [label=\"Entry\"];\n"; + OS << "Entry -> Block" << getBlockId(Block) << ";\n"; + } + while (!BlocksToPrint.empty()) { + CFG::BasicBlock const *block = *BlocksToPrint.begin(); + BlocksToPrint.erase(BlocksToPrint.begin()); + printBlock(*block); + } +} + +void StackLayoutPrinter::operator()(CFG::FunctionInfo const &Info) { + OS << "Function: " << Info.MF->getName() << "("; + for (const auto &Param : Info.Parameters) + OS << printReg(Param.VirtualReg, nullptr, 0, nullptr); + OS << ");\n"; + OS << "FunctionEntry " << " -> Block" << getBlockId(*Info.Entry) << ";\n"; + (*this)(*Info.Entry, false); +} + +void StackLayoutPrinter::printBlock(CFG::BasicBlock const &Block) { + OS << "Block" << getBlockId(Block) << " [\n"; + // Verify that the entries of this block exit into this block. + for (auto const &entry : Block.Entries) { + std::visit( + Overload{[&](CFG::BasicBlock::Jump const &Jump) { + assert(Jump.Target == &Block); + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + assert(ConditionalJump.Zero == &Block || + ConditionalJump.NonZero == &Block); + }, + [&](auto const &) { + llvm_unreachable("Invalid control flow graph"); + }}, + entry->Exit); + } + + auto const &BlockInfo = Layout.blockInfos.at(&Block); + OS << stackToString(BlockInfo.entryLayout) << "\n"; + for (auto const &Operation : Block.Operations) { + OS << "\n"; + auto EntryLayout = Layout.operationEntryLayout.at(&Operation); + OS << stackToString(EntryLayout) << "\n"; + std::visit(Overload{[&](CFG::FunctionCall const &Call) { + const MachineOperand *Callee = + Call.Call->explicit_uses().begin(); + OS << Callee->getGlobal()->getName(); + }, + [&](CFG::BuiltinCall const &Call) { + OS << getInstName(Call.Builtin); + }, + [&](CFG::Assignment const &Assignment) { + OS << "Assignment("; + for (const auto &Var : Assignment.Variables) + OS << printReg(Var.VirtualReg, nullptr, 0, nullptr) + << ", "; + OS << ")"; + }}, + Operation.Operation); + OS << "\n"; + + assert(Operation.Input.size() <= EntryLayout.size()); + for (size_t i = 0; i < Operation.Input.size(); ++i) + EntryLayout.pop_back(); + EntryLayout += Operation.Output; + OS << stackToString(EntryLayout) << "\n"; + } + OS << "\n"; + OS << stackToString(BlockInfo.exitLayout) << "\n"; + OS << "];\n"; + + std::visit( + Overload{[&](CFG::BasicBlock::InvalidExit const &) { + assert(0 && "Invalid basic block exit"); + }, + [&](CFG::BasicBlock::Jump const &Jump) { + OS << "Block" << getBlockId(Block) << "Exit [label=\""; + if (Jump.Backwards) + OS << "Backwards"; + OS << "Jump\"];\n"; + OS << "Block" << getBlockId(Block) << "Exit -> Block" + << getBlockId(*Jump.Target) << ";\n"; + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + OS << "Block" << getBlockId(Block) << "Exit [label=\"{ "; + OS << stackSlotToString(ConditionalJump.Condition); + OS << "| { <0> Zero | <1> NonZero }}\"];\n"; + OS << "Block" << getBlockId(Block); + OS << "Exit:0 -> Block" << getBlockId(*ConditionalJump.Zero) + << ";\n"; + OS << "Block" << getBlockId(Block); + OS << "Exit:1 -> Block" << getBlockId(*ConditionalJump.NonZero) + << ";\n"; + }, + [&](CFG::BasicBlock::FunctionReturn const &Return) { + OS << "Block" << getBlockId(Block) + << "Exit [label=\"FunctionReturn[" + << Return.Info->MF->getName() << "]\"];\n"; + }, + [&](CFG::BasicBlock::Terminated const &) { + OS << "Block" << getBlockId(Block) + << "Exit [label=\"Terminated\"];\n"; + }}, + Block.Exit); + OS << "\n"; +} + +std::string StackLayoutPrinter::getBlockId(CFG::BasicBlock const &Block) { + std::string Name = std::to_string(Block.MBB->getNumber()) + "." + + std::string(Block.MBB->getName()); + if (auto It = BlockIds.find(&Block); It != BlockIds.end()) + return std::to_string(It->second) + "(" + Name + ")"; + + size_t Id = BlockIds[&Block] = BlockCount++; + BlocksToPrint.emplace_back(&Block); + return std::to_string(Id) + "(" + Name + ")"; +} + +#endif // NDEBUG diff --git a/llvm/lib/Target/EVM/EVMStackDebug.h b/llvm/lib/Target/EVM/EVMStackDebug.h new file mode 100644 index 000000000000..e18ce24e6e9b --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackDebug.h @@ -0,0 +1,69 @@ +//===---- EVMStackDebug.h - Debugging of the stackification -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares classes for dumping a state of stackifcation related data +// structures and algorithms. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMSTACKDEBUG_H +#define LLVM_LIB_TARGET_EVM_EVMSTACKDEBUG_H + +#include "EVMControlFlowGraph.h" +#include "llvm/CodeGen/MachineFunction.h" +#include +#include +#include +#include +#include + +namespace llvm { + +struct StackLayout; + +#ifndef NDEBUG +const Function *getCalledFunction(const MachineInstr &MI); +std::string stackSlotToString(const StackSlot &Slot); +std::string stackToString(Stack const &S); + +class ControlFlowGraphPrinter { +public: + ControlFlowGraphPrinter(raw_ostream &OS) : OS(OS) {} + void operator()(const CFG &Cfg); + +private: + void operator()(const CFG::FunctionInfo &Info); + std::string getBlockId(const CFG::BasicBlock &Block); + void printBlock(const CFG::BasicBlock &Block); + + raw_ostream &OS; +}; + +class StackLayoutPrinter { +public: + StackLayoutPrinter(raw_ostream &OS, const StackLayout &StackLayout) + : OS(OS), Layout(StackLayout) {} + + void operator()(CFG::BasicBlock const &Block, bool IsMainEntry = true); + void operator()(CFG::FunctionInfo const &Info); + +private: + void printBlock(CFG::BasicBlock const &Block); + std::string getBlockId(CFG::BasicBlock const &Block); + + raw_ostream &OS; + const StackLayout &Layout; + std::map BlockIds; + size_t BlockCount = 0; + std::list BlocksToPrint; +}; +#endif // NDEBUG + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMSTACKDEBUG_H diff --git a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp new file mode 100644 index 000000000000..8748af41222d --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.cpp @@ -0,0 +1,857 @@ +//===---- EVMStackLayoutGenerator.h - Stack layout generator ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the stack layout generator which for each operation +// finds complete stack layout that: +// - has the slots required for the operation at the stack top. +// - will have the operation result in a layout that makes it easy to achieve +// the next desired layout. +// It also finds an entering/exiting stack layout for each block. +// +//===----------------------------------------------------------------------===// + +#include "EVMStackLayoutGenerator.h" +#include "EVMHelperUtilities.h" +#include "EVMRegisterInfo.h" +#include "EVMStackDebug.h" +#include "EVMStackShuffler.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +#define DEBUG_TYPE "evm-stack-layout-generator" + +StackLayout StackLayoutGenerator::run(const CFG &Cfg) { + StackLayout Layout; + StackLayoutGenerator LayoutGenerator{Layout, &Cfg.FuncInfo}; + LayoutGenerator.processEntryPoint(*Cfg.FuncInfo.Entry, &Cfg.FuncInfo); + + LLVM_DEBUG({ + dbgs() << "************* Stack Layout *************\n"; + StackLayoutPrinter P(dbgs(), Layout); + P(Cfg.FuncInfo); + }); + + return Layout; +} + +StackLayoutGenerator::StackLayoutGenerator( + StackLayout &Layout, CFG::FunctionInfo const *FunctionInfo) + : Layout(Layout), CurrentFunctionInfo(FunctionInfo) {} + +namespace { + +/// Returns all stack too deep errors that would occur when shuffling \p Source +/// to \p Target. +std::vector +findStackTooDeep(Stack const &Source, Stack const &Target) { + Stack CurrentStack = Source; + std::vector Errors; + + auto getVariableChoices = [](auto &&SlotRange) { + std::vector Result; + for (auto const &Slot : SlotRange) + if (auto const *VarSlot = std::get_if(&Slot)) + if (!EVMUtils::contains(Result, VarSlot->VirtualReg)) + Result.push_back(VarSlot->VirtualReg); + return Result; + }; + + ::createStackLayout( + CurrentStack, Target, + [&](unsigned I) { + if (I > 16) + Errors.emplace_back(StackLayoutGenerator::StackTooDeep{ + I - 16, + getVariableChoices(EVMUtils::take_last(CurrentStack, I + 1))}); + }, + [&](StackSlot const &Slot) { + if (canBeFreelyGenerated(Slot)) + return; + + if (auto depth = + EVMUtils::findOffset(EVMUtils::get_reverse(CurrentStack), Slot); + depth && *depth >= 16) + Errors.emplace_back(StackLayoutGenerator::StackTooDeep{ + *depth - 15, getVariableChoices( + EVMUtils::take_last(CurrentStack, *depth + 1))}); + }, + [&]() {}); + return Errors; +} + +/// Returns the ideal stack to have before executing an operation that outputs +/// \p OperationOutput, s.t. shuffling to \p Post is cheap (excluding the +/// input of the operation itself). If \p GenerateSlotOnTheFly returns true for +/// a slot, this slot should not occur in the ideal stack, but rather be +/// generated on the fly during shuffling. +template +Stack createIdealLayout(const Stack &OperationOutput, const Stack &Post, + Callable GenerateSlotOnTheFly) { + struct PreviousSlot { + size_t slot; + }; + using LayoutT = std::vector>; + + // Determine the number of slots that have to be on stack before executing the + // operation (excluding the inputs of the operation itself). That is slots + // that should not be generated on the fly and are not outputs of the + // operation. + size_t PreOperationLayoutSize = Post.size(); + for (auto const &Slot : Post) + if (EVMUtils::contains(OperationOutput, Slot) || GenerateSlotOnTheFly(Slot)) + --PreOperationLayoutSize; + + // The symbolic layout directly after the operation has the form + // PreviousSlot{0}, ..., PreviousSlot{n}, [output<0>], ..., [output] + LayoutT Layout; + for (size_t Index = 0; Index < PreOperationLayoutSize; ++Index) + Layout.emplace_back(PreviousSlot{Index}); + Layout += OperationOutput; + + // Shortcut for trivial case. + if (Layout.empty()) + return Stack{}; + + // Next we will shuffle the layout to the Post stack using ShuffleOperations + // that are aware of PreviousSlot's. + struct ShuffleOperations { + LayoutT &Layout; + const Stack &Post; + std::set Outputs; + Multiplicity Mult; + Callable GenerateSlotOnTheFly; + ShuffleOperations(LayoutT &Layout, Stack const &Post, + Callable GenerateSlotOnTheFly) + : Layout(Layout), Post(Post), + GenerateSlotOnTheFly(GenerateSlotOnTheFly) { + for (auto const &LayoutSlot : Layout) + if (const StackSlot *Slot = std::get_if(&LayoutSlot)) + Outputs.insert(*Slot); + + for (auto const &LayoutSlot : Layout) + if (const StackSlot *Slot = std::get_if(&LayoutSlot)) + --Mult[*Slot]; + + for (auto &&Slot : Post) + if (Outputs.count(Slot) || GenerateSlotOnTheFly(Slot)) + ++Mult[Slot]; + } + + bool isCompatible(size_t Source, size_t Target) { + return Source < Layout.size() && Target < Post.size() && + (std::holds_alternative(Post.at(Target)) || + std::visit(Overload{[&](const PreviousSlot &) { + return !Outputs.count(Post.at(Target)) && + !GenerateSlotOnTheFly( + Post.at(Target)); + }, + [&](const StackSlot &S) { + return S == Post.at(Target); + }}, + Layout.at(Source))); + } + + bool sourceIsSame(size_t Lhs, size_t Rhs) { + return std::visit( + Overload{ + [&](PreviousSlot const &, PreviousSlot const &) { return true; }, + [&](StackSlot const &Lhs, StackSlot const &Rhs) { + return Lhs == Rhs; + }, + [&](auto const &, auto const &) { return false; }}, + Layout.at(Lhs), Layout.at(Rhs)); + } + + int sourceMultiplicity(size_t Offset) { + return std::visit( + Overload{[&](PreviousSlot const &) { return 0; }, + [&](StackSlot const &S) { return Mult.at(S); }}, + Layout.at(Offset)); + } + + int targetMultiplicity(size_t Offset) { + if (!Outputs.count(Post.at(Offset)) && + !GenerateSlotOnTheFly(Post.at(Offset))) + return 0; + return Mult.at(Post.at(Offset)); + } + + bool targetIsArbitrary(size_t Offset) { + return Offset < Post.size() && + std::holds_alternative(Post.at(Offset)); + } + + void swap(size_t I) { + assert(!std::holds_alternative( + Layout.at(Layout.size() - I - 1)) || + !std::holds_alternative(Layout.back())); + std::swap(Layout.at(Layout.size() - I - 1), Layout.back()); + } + + size_t sourceSize() { return Layout.size(); } + + size_t targetSize() { return Post.size(); } + + void pop() { Layout.pop_back(); } + + void pushOrDupTarget(size_t Offset) { Layout.push_back(Post.at(Offset)); } + }; + + Shuffler::shuffle(Layout, Post, GenerateSlotOnTheFly); + + // Now we can construct the ideal layout before the operation. + // "layout" has shuffled the PreviousSlot{x} to new places using minimal + // operations to move the operation output in place. The resulting permutation + // of the PreviousSlot yields the ideal positions of slots before the + // operation, i.e. if PreviousSlot{2} is at a position at which Post contains + // VariableSlot{"tmp"}, then we want the variable tmp in the slot at offset 2 + // in the layout before the operation. + assert(Layout.size() == Post.size()); + std::vector> IdealLayout(Post.size(), std::nullopt); + for (unsigned Idx = 0; Idx < std::min(Layout.size(), Post.size()); ++Idx) { + auto &Slot = Post[Idx]; + auto &IdealPosition = Layout[Idx]; + if (PreviousSlot *PrevSlot = std::get_if(&IdealPosition)) + IdealLayout.at(PrevSlot->slot) = Slot; + } + + // The tail of layout must have contained the operation outputs and will not + // have been assigned slots in the last loop. + while (!IdealLayout.empty() && !IdealLayout.back()) + IdealLayout.pop_back(); + + assert(IdealLayout.size() == PreOperationLayoutSize); + + Stack Result; + for (const auto &Item : IdealLayout) { + assert(Item); + Result.emplace_back(*Item); + } + + return Result; +} + +} // end anonymous namespace + +Stack StackLayoutGenerator::propagateStackThroughOperation( + Stack ExitStack, const CFG::Operation &Operation, + bool AggressiveStackCompression) { + // Enable aggressive stack compression for recursive calls. + if (auto const *functionCall = + std::get_if(&Operation.Operation)) + // TODO: compress stack for recursive functions. + AggressiveStackCompression = false; + + // This is a huge tradeoff between code size, gas cost and stack size. + auto generateSlotOnTheFly = [&](StackSlot const &Slot) { + return AggressiveStackCompression && canBeFreelyGenerated(Slot); + }; + + // Determine the ideal permutation of the slots in ExitLayout that are not + // operation outputs (and not to be generated on the fly), s.t. shuffling the + // `IdealStack + Operation.output` to ExitLayout is cheap. + Stack IdealStack = + createIdealLayout(Operation.Output, ExitStack, generateSlotOnTheFly); + + // Make sure the resulting previous slots do not overlap with any assignmed + // variables. + if (auto const *Assignment = + std::get_if(&Operation.Operation)) + for (auto &StackSlot : IdealStack) + if (auto const *VarSlot = std::get_if(&StackSlot)) + assert(!EVMUtils::contains(Assignment->Variables, *VarSlot)); + + // Since stack+Operation.output can be easily shuffled to _exitLayout, the + // desired layout before the operation is stack+Operation.input; + IdealStack += Operation.Input; + + // Store the exact desired operation entry layout. The stored layout will be + // recreated by the code transform before executing the operation. However, + // this recreation can produce slots that can be freely generated or are + // duplicated, i.e. we can compress the stack afterwards without causing + // problems for code generation later. + Layout.operationEntryLayout[&Operation] = IdealStack; + + // Remove anything from the stack top that can be freely generated or dupped + // from deeper on the stack. + while (!IdealStack.empty()) { + if (canBeFreelyGenerated(IdealStack.back())) + IdealStack.pop_back(); + else if (auto Offset = EVMUtils::findOffset( + EVMUtils::drop_first(EVMUtils::get_reverse(IdealStack), 1), + IdealStack.back())) { + if (*Offset + 2 < 16) + IdealStack.pop_back(); + else + break; + } else + break; + } + + return IdealStack; +} + +Stack StackLayoutGenerator::propagateStackThroughBlock( + Stack ExitStack, CFG::BasicBlock const &Block, + bool AggressiveStackCompression) { + Stack CurrentStack = ExitStack; + for (auto &Operation : EVMUtils::get_reverse(Block.Operations)) { + Stack NewStack = propagateStackThroughOperation(CurrentStack, Operation, + AggressiveStackCompression); + if (!AggressiveStackCompression && + !findStackTooDeep(NewStack, CurrentStack).empty()) + // If we had stack errors, run again with aggressive stack compression. + return propagateStackThroughBlock(std::move(ExitStack), Block, true); + CurrentStack = std::move(NewStack); + } + return CurrentStack; +} + +void StackLayoutGenerator::processEntryPoint( + CFG::BasicBlock const &Entry, CFG::FunctionInfo const *FunctionInfo) { + std::list ToVisit{&Entry}; + std::set Visited; + + // TODO: check whether visiting only a subset of these in the outer iteration + // below is enough. + std::list> + BackwardsJumps = collectBackwardsJumps(Entry); + while (!ToVisit.empty()) { + // First calculate stack layouts without walking backwards jumps, i.e. + // assuming the current preliminary entry layout of the backwards jump + // target as the initial exit layout of the backwards-jumping block. + while (!ToVisit.empty()) { + CFG::BasicBlock const *Block = *ToVisit.begin(); + ToVisit.pop_front(); + + if (Visited.count(Block)) + continue; + + if (std::optional ExitLayout = + getExitLayoutOrStageDependencies(*Block, Visited, ToVisit)) { + Visited.emplace(Block); + auto &Info = Layout.blockInfos[Block]; + Info.exitLayout = *ExitLayout; + Info.entryLayout = propagateStackThroughBlock(Info.exitLayout, *Block); + for (auto Entry : Block->Entries) + ToVisit.emplace_back(Entry); + } + } + + // Determine which backwards jumps still require fixing and stage revisits + // of appropriate nodes. + for (auto [JumpingBlock, Target] : BackwardsJumps) { + // This block jumps backwards, but does not provide all slots required by + // the jump target on exit. Therefore we need to visit the subgraph + // between ``Target`` and ``JumpingBlock`` again. + auto StartIt = std::begin(Layout.blockInfos[Target].entryLayout); + auto EndIt = std::end(Layout.blockInfos[Target].entryLayout); + if (std::any_of(StartIt, EndIt, + [exitLayout = Layout.blockInfos[JumpingBlock].exitLayout]( + StackSlot const &Slot) { + return !EVMUtils::contains(exitLayout, Slot); + })) { + // In particular we can visit backwards starting from ``JumpingBlock`` + // and mark all entries to-be-visited again until we hit ``Target``. + ToVisit.emplace_front(JumpingBlock); + // Since we are likely to permute the entry layout of ``Target``, we + // also visit its entries again. This is not required for correctness, + // since the set of stack slots will match, but it may move some + // required stack shuffling from the loop condition to outside the loop. + for (CFG::BasicBlock const *Entry : Target->Entries) + Visited.erase(Entry); + + EVMUtils::BreadthFirstSearch{{JumpingBlock}} + .run([&Visited, Target = Target](const CFG::BasicBlock *Block, + auto AddChild) { + Visited.erase(Block); + if (Block == Target) + return; + for (auto const *Entry : Block->Entries) + AddChild(Entry); + }); + // While the shuffled layout for ``Target`` will be compatible, it can + // be worthwhile propagating it further up once more. This would mean + // not stopping at Block == Target above, resp. even doing + // Visited.clear() here, revisiting the entire graph. This is a tradeoff + // between the runtime of this process and the optimality of the result. + // Also note that while visiting the entire graph again *can* be + // helpful, it can also be detrimental. + } + } + } + + stitchConditionalJumps(Entry); + fillInJunk(Entry, FunctionInfo); +} + +std::optional StackLayoutGenerator::getExitLayoutOrStageDependencies( + const CFG::BasicBlock &Block, + const std::set &Visited, + std::list &ToVisit) const { + return std::visit( + Overload{ + [&](CFG::BasicBlock::Jump const &Jump) -> std::optional { + if (Jump.Backwards) { + // Choose the best currently known entry layout of the jump target + // as initial exit. Note that this may not yet be the final + // layout. + auto It = Layout.blockInfos.find(Jump.Target); + if (It == Layout.blockInfos.end()) + return Stack{}; + + return It->second.entryLayout; + } + // If the current iteration has already visited the jump target, + // start from its entry layout. + if (Visited.count(Jump.Target)) + return Layout.blockInfos.at(Jump.Target).entryLayout; + // Otherwise stage the jump target for visit and defer the current + // block. + ToVisit.emplace_front(Jump.Target); + return std::nullopt; + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) + -> std::optional { + bool ZeroVisited = Visited.count(ConditionalJump.Zero); + bool NonZeroVisited = Visited.count(ConditionalJump.NonZero); + + if (ZeroVisited && NonZeroVisited) { + // If the current iteration has already Visited both jump targets, + // start from its entry layout. + Stack CombonedStack = combineStack( + Layout.blockInfos.at(ConditionalJump.Zero).entryLayout, + Layout.blockInfos.at(ConditionalJump.NonZero).entryLayout); + // Additionally, the jump condition has to be at the stack top at + // exit. + CombonedStack.emplace_back(ConditionalJump.Condition); + return CombonedStack; + } + + // If one of the jump targets has not been visited, stage it for + // visit and defer the current block. + if (!ZeroVisited) + ToVisit.emplace_front(ConditionalJump.Zero); + + if (!NonZeroVisited) + ToVisit.emplace_front(ConditionalJump.NonZero); + + return std::nullopt; + }, + [&](CFG::BasicBlock::FunctionReturn const &FunctionReturn) + -> std::optional { + // A function return needs the return variables and the function + // return label slot on stack. + assert(FunctionReturn.Info); + Stack ReturnStack = FunctionReturn.RetValues; + ReturnStack.emplace_back( + FunctionReturnLabelSlot{FunctionReturn.Info->MF}); + return ReturnStack; + }, + [&](CFG::BasicBlock::Terminated const &) -> std::optional { + // A terminating block can have an empty stack on exit. + return Stack{}; + }, + [](CFG::BasicBlock::InvalidExit const &) -> std::optional { + llvm_unreachable("Unexpected BB terminator"); + }}, + Block.Exit); +} + +std::list> +StackLayoutGenerator::collectBackwardsJumps( + CFG::BasicBlock const &Entry) const { + std::list> + BackwardsJumps; + EVMUtils::BreadthFirstSearch{{&Entry}}.run( + [&](CFG::BasicBlock const *Block, auto AddChild) { + std::visit( + Overload{ + [&](CFG::BasicBlock::InvalidExit const &) { + llvm_unreachable("Unexpected BB terminator"); + }, + [&](CFG::BasicBlock::Jump const &Jump) { + if (Jump.Backwards) + BackwardsJumps.emplace_back(Block, Jump.Target); + AddChild(Jump.Target); + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + AddChild(ConditionalJump.Zero); + AddChild(ConditionalJump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &) {}, + [&](CFG::BasicBlock::Terminated const &) {}, + }, + Block->Exit); + }); + return BackwardsJumps; +} + +void StackLayoutGenerator::stitchConditionalJumps( + CFG::BasicBlock const &Block) { + EVMUtils::BreadthFirstSearch BFS{{&Block}}; + BFS.run([&](CFG::BasicBlock const *Block, auto AddChild) { + auto &Info = Layout.blockInfos.at(Block); + std::visit( + Overload{ + [&](CFG::BasicBlock::InvalidExit const &) { + llvm_unreachable("Unexpected BB terminator"); + }, + [&](CFG::BasicBlock::Jump const &Jump) { + if (!Jump.Backwards) + AddChild(Jump.Target); + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + auto &ZeroTargetInfo = Layout.blockInfos.at(ConditionalJump.Zero); + auto &NonZeroTargetInfo = + Layout.blockInfos.at(ConditionalJump.NonZero); + Stack ExitLayout = Info.exitLayout; + + // The last block must have produced the condition at the stack + // top. + assert(!ExitLayout.empty()); + assert(ExitLayout.back() == ConditionalJump.Condition); + // The condition is consumed by the jump. + ExitLayout.pop_back(); + + auto FixJumpTargetEntry = + [&](Stack const &OriginalEntryLayout) -> Stack { + Stack NewEntryLayout = ExitLayout; + // Whatever the block being jumped to does not actually require, + // can be marked as junk. + for (auto &Slot : NewEntryLayout) + if (!EVMUtils::contains(OriginalEntryLayout, Slot)) + Slot = JunkSlot{}; + // Make sure everything the block being jumped to requires is + // actually present or can be generated. + for (auto const &Slot : OriginalEntryLayout) + assert(canBeFreelyGenerated(Slot) || + EVMUtils::contains(NewEntryLayout, Slot)); + return NewEntryLayout; + }; + + ZeroTargetInfo.entryLayout = + FixJumpTargetEntry(ZeroTargetInfo.entryLayout); + NonZeroTargetInfo.entryLayout = + FixJumpTargetEntry(NonZeroTargetInfo.entryLayout); + AddChild(ConditionalJump.Zero); + AddChild(ConditionalJump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &) {}, + [&](CFG::BasicBlock::Terminated const &) {}, + }, + Block->Exit); + }); +} + +Stack StackLayoutGenerator::combineStack(Stack const &Stack1, + Stack const &Stack2) { + // TODO: it would be nicer to replace this by a constructive algorithm. + // Currently it uses a reduced version of the Heap Algorithm to partly + // brute-force, which seems to work decently well. + + Stack CommonPrefix; + for (unsigned Idx = 0; Idx < std::min(Stack1.size(), Stack2.size()); ++Idx) { + const StackSlot &Slot1 = Stack1[Idx]; + const StackSlot &Slot2 = Stack2[Idx]; + if (!(Slot1 == Slot2)) + break; + CommonPrefix.emplace_back(Slot1); + } + + Stack Stack1Tail, Stack2Tail; + for (auto Slot : EVMUtils::drop_first(Stack1, CommonPrefix.size())) + Stack1Tail.emplace_back(Slot); + + for (auto Slot : EVMUtils::drop_first(Stack2, CommonPrefix.size())) + Stack2Tail.emplace_back(Slot); + + if (Stack1Tail.empty()) + return CommonPrefix + compressStack(Stack2Tail); + + if (Stack2Tail.empty()) + return CommonPrefix + compressStack(Stack1Tail); + + Stack Candidate; + for (auto Slot : Stack1Tail) + if (!EVMUtils::contains(Candidate, Slot)) + Candidate.emplace_back(Slot); + + for (auto Slot : Stack2Tail) + if (!EVMUtils::contains(Candidate, Slot)) + Candidate.emplace_back(Slot); + + { + auto RemIt = std::remove_if( + Candidate.begin(), Candidate.end(), [](StackSlot const &Slot) { + return std::holds_alternative(Slot) || + std::holds_alternative(Slot) || + std::holds_alternative(Slot); + }); + Candidate.erase(RemIt, Candidate.end()); + } + + auto evaluate = [&](Stack const &Candidate) -> size_t { + size_t NumOps = 0; + Stack TestStack = Candidate; + auto Swap = [&](unsigned SwapDepth) { + ++NumOps; + if (SwapDepth > 16) + NumOps += 1000; + }; + + auto DupOrPush = [&](StackSlot const &Slot) { + if (canBeFreelyGenerated(Slot)) + return; + + Stack Tmp = CommonPrefix; + Tmp += TestStack; + + auto Depth = EVMUtils::findOffset(EVMUtils::get_reverse(Tmp), Slot); + if (Depth && *Depth >= 16) + NumOps += 1000; + }; + createStackLayout(TestStack, Stack1Tail, Swap, DupOrPush, [&]() {}); + TestStack = Candidate; + createStackLayout(TestStack, Stack2Tail, Swap, DupOrPush, [&]() {}); + return NumOps; + }; + + // See https://en.wikipedia.org/wiki/Heap's_algorithm + size_t N = Candidate.size(); + Stack BestCandidate = Candidate; + size_t BestCost = evaluate(Candidate); + std::vector C(N, 0); + size_t I = 1; + while (I < N) { + if (C[I] < I) { + if (I & 1) + std::swap(Candidate.front(), Candidate[I]); + else + std::swap(Candidate[C[I]], Candidate[I]); + + size_t Cost = evaluate(Candidate); + if (Cost < BestCost) { + BestCost = Cost; + BestCandidate = Candidate; + } + ++C[I]; + // Note that for a proper implementation of the Heap algorithm this would + // need to revert back to ``I = 1.`` However, the incorrect implementation + // produces decent result and the proper version would have N! complexity + // and is thereby not feasible. + ++I; + } else { + C[I] = 0; + ++I; + } + } + + return CommonPrefix + BestCandidate; +} + +Stack StackLayoutGenerator::compressStack(Stack CurStack) { + std::optional FirstDupOffset; + do { + if (FirstDupOffset) { + if (*FirstDupOffset != (CurStack.size() - 1)) + std::swap(CurStack.at(*FirstDupOffset), CurStack.back()); + CurStack.pop_back(); + FirstDupOffset.reset(); + } + + auto I = CurStack.rbegin(), E = CurStack.rend(); + for (size_t Depth = 0; I < E; ++I, ++Depth) { + StackSlot &Slot = *I; + if (canBeFreelyGenerated(Slot)) { + FirstDupOffset = CurStack.size() - Depth - 1; + break; + } + + if (auto DupDepth = EVMUtils::findOffset( + EVMUtils::drop_first(EVMUtils::get_reverse(CurStack), Depth + 1), + Slot)) { + if (Depth + *DupDepth <= 16) { + FirstDupOffset = CurStack.size() - Depth - 1; + break; + } + } + } + } while (FirstDupOffset); + return CurStack; +} + +void StackLayoutGenerator::fillInJunk(CFG::BasicBlock const &Block, + CFG::FunctionInfo const *FunctionInfo) { + /// Recursively adds junk to the subgraph starting on \p Entry. + /// Since it is only called on cut-vertices, the full subgraph retains proper + /// stack balance. + auto AddJunkRecursive = [&](CFG::BasicBlock const *Entry, size_t NumJunk) { + EVMUtils::BreadthFirstSearch BFS{{Entry}}; + + BFS.run([&](CFG::BasicBlock const *Block, auto AddChild) { + auto &BlockInfo = Layout.blockInfos.at(Block); + BlockInfo.entryLayout = + Stack{NumJunk, JunkSlot{}} + std::move(BlockInfo.entryLayout); + + for (auto const &Operation : Block->Operations) { + auto &OpEntryLayout = Layout.operationEntryLayout.at(&Operation); + OpEntryLayout = Stack{NumJunk, JunkSlot{}} + std::move(OpEntryLayout); + } + + BlockInfo.exitLayout = + Stack{NumJunk, JunkSlot{}} + std::move(BlockInfo.exitLayout); + + std::visit( + Overload{ + [&](CFG::BasicBlock::InvalidExit const &) { + llvm_unreachable("Unexpected BB terminator"); + }, + [&](CFG::BasicBlock::Jump const &Jump) { AddChild(Jump.Target); }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + AddChild(ConditionalJump.Zero); + AddChild(ConditionalJump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &) { + llvm_unreachable("FunctionReturn : unexpected BB terminator"); + }, + [&](CFG::BasicBlock::Terminated const &) {}, + }, + Block->Exit); + }); + }; + + /// Returns the number of operations required to transform \p Source to \p + /// Target. + auto EvaluateTransform = [&](Stack Source, Stack const &Target) -> size_t { + size_t OpGas = 0; + auto Swap = [&](unsigned SwapDepth) { + if (SwapDepth > 16) + OpGas += 1000; + else + OpGas += 3; // SWAP* gas price; + }; + + auto DupOrPush = [&](StackSlot const &Slot) { + if (canBeFreelyGenerated(Slot)) + OpGas += 3; + else { + if (auto Depth = + EVMUtils::findOffset(EVMUtils::get_reverse(Source), Slot)) { + if (*Depth < 16) + OpGas += 3; // gas price for DUP + else + OpGas += 1000; + } else { + // This has to be a previously unassigned return variable. + // We at least sanity-check that it is among the return variables at + // all. +#ifndef NDEBUG + bool VarExists = false; + assert(std::holds_alternative(Slot)); + for (CFG::BasicBlock *Exit : FunctionInfo->Exits) { + const Stack &RetValues = + std::get(Exit->Exit).RetValues; + + for (const StackSlot &Val : RetValues) { + if (const VariableSlot *VarSlot = std::get_if(&Val)) + if (*VarSlot == std::get(Slot)) + VarExists = true; + } + } + assert(VarExists); +#endif // NDEBUG + // Strictly speaking the cost of the + // PUSH0 depends on the targeted EVM version, but the difference will + // not matter here. + OpGas += 2; + } + } + }; + + auto Pop = [&]() { OpGas += 2; }; + + createStackLayout(Source, Target, Swap, DupOrPush, Pop); + return OpGas; + }; + + /// Returns the number of junk slots to be prepended to \p TargetLayout for + /// an optimal transition from \p EntryLayout to \p TargetLayout. + auto GetBestNumJunk = [&](Stack const &EntryLayout, + Stack const &TargetLayout) -> size_t { + size_t BestCost = EvaluateTransform(EntryLayout, TargetLayout); + size_t BestNumJunk = 0; + size_t MaxJunk = EntryLayout.size(); + for (size_t NumJunk = 1; NumJunk <= MaxJunk; ++NumJunk) { + size_t Cost = EvaluateTransform(EntryLayout, Stack{NumJunk, JunkSlot{}} + + TargetLayout); + if (Cost < BestCost) { + BestCost = Cost; + BestNumJunk = NumJunk; + } + } + return BestNumJunk; + }; + + if (FunctionInfo && !FunctionInfo->CanContinue && Block.AllowsJunk()) { + Stack Params; + for (const auto &Param : FunctionInfo->Parameters) + Params.emplace_back(Param); + std::reverse(Params.begin(), Params.end()); + size_t BestNumJunk = + GetBestNumJunk(Params, Layout.blockInfos.at(&Block).entryLayout); + if (BestNumJunk > 0) + AddJunkRecursive(&Block, BestNumJunk); + } + + /// Traverses the CFG and at each block that allows junk, i.e. that is a + /// cut-vertex that never leads to a function return, checks if adding junk + /// reduces the shuffling cost upon entering and if so recursively adds junk + /// to the spanned subgraph. + EVMUtils::BreadthFirstSearch{{&Block}}.run( + [&](CFG::BasicBlock const *Block, auto AddChild) { + if (Block->AllowsJunk()) { + auto &BlockInfo = Layout.blockInfos.at(Block); + Stack EntryLayout = BlockInfo.entryLayout; + Stack const &NextLayout = + Block->Operations.empty() + ? BlockInfo.exitLayout + : Layout.operationEntryLayout.at(&Block->Operations.front()); + if (EntryLayout != NextLayout) { + size_t BestNumJunk = GetBestNumJunk(EntryLayout, NextLayout); + if (BestNumJunk > 0) { + AddJunkRecursive(Block, BestNumJunk); + BlockInfo.entryLayout = EntryLayout; + } + } + } + + std::visit( + Overload{ + [&](CFG::BasicBlock::InvalidExit const &) { + llvm_unreachable("Invalid BB terminator"); + }, + [&](CFG::BasicBlock::Jump const &Jump) { + AddChild(Jump.Target); + }, + [&](CFG::BasicBlock::ConditionalJump const &ConditionalJump) { + AddChild(ConditionalJump.Zero); + AddChild(ConditionalJump.NonZero); + }, + [&](CFG::BasicBlock::FunctionReturn const &) {}, + [&](CFG::BasicBlock::Terminated const &) {}, + }, + Block->Exit); + }); +} diff --git a/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h new file mode 100644 index 000000000000..e7dff86228bf --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackLayoutGenerator.h @@ -0,0 +1,126 @@ +//===---- EVMStackLayoutGenerator.h - Stack layout generator ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the stack layout generator which for each operation +// finds complete stack layout that: +// - has the slots required for the operation at the stack top. +// - will have the operation result in a layout that makes it easy to achieve +// the next desired layout. +// It also finds an entering/exiting stack layout for each block. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMSTACKLAYOUTGENERATOR_H +#define LLVM_LIB_TARGET_EVM_EVMSTACKLAYOUTGENERATOR_H + +#include "EVMControlFlowGraph.h" +#include +#include + +namespace llvm { + +struct StackLayout { + struct BlockInfo { + /// Complete stack layout that is required for entering a block. + Stack entryLayout; + /// The resulting stack layout after executing the block. + Stack exitLayout; + }; + std::map blockInfos; + /// For each operation the complete stack layout that: + /// - has the slots required for the operation at the stack top. + /// - will have the operation result in a layout that makes it easy to achieve + /// the next desired layout. + std::map operationEntryLayout; +}; + +class StackLayoutGenerator { +public: + struct StackTooDeep { + /// Number of slots that need to be saved. + size_t deficit = 0; + /// Set of variables, eliminating which would decrease the stack deficit. + std::vector variableChoices; + }; + + static StackLayout run(CFG const &Cfg); + /// Returns all stack too deep errors for the given CFG. + static std::vector reportStackTooDeep(CFG const &Cfg); + +private: + StackLayoutGenerator(StackLayout &Context, + CFG::FunctionInfo const *FunctionInfo); + + /// Returns the optimal entry stack layout, s.t. \p Operation can be applied + /// to it and the result can be transformed to \p ExitStack with minimal stack + /// shuffling. Simultaneously stores the entry layout required for executing + /// the operation in Layout. + Stack propagateStackThroughOperation(Stack ExitStack, + CFG::Operation const &Operation, + bool AggressiveStackCompression = false); + + /// Returns the desired stack layout at the entry of \p Block, assuming the + /// layout after executing the block should be \p ExitStack. + Stack propagateStackThroughBlock(Stack ExitStack, + CFG::BasicBlock const &block, + bool AggressiveStackCompression = false); + + /// Main algorithm walking the graph from entry to exit and propagating back + /// the stack layouts to the entries. Iteratively reruns itself along + /// backwards jumps until the layout is stabilized. + void processEntryPoint(CFG::BasicBlock const &Entry, + CFG::FunctionInfo const *FunctionInfo = nullptr); + + /// Returns the best known exit layout of \p Block, if all dependencies are + /// already \p Visited. If not, adds the dependencies to \p DependencyList and + /// returns std::nullopt. + std::optional getExitLayoutOrStageDependencies( + CFG::BasicBlock const &Block, + std::set const &Visited, + std::list &DependencyList) const; + + /// Returns a pair of ``{jumpingBlock, targetBlock}`` for each backwards jump + /// in the graph starting at \p Eentry. + std::list> + collectBackwardsJumps(CFG::BasicBlock const &Entry) const; + + /// After the main algorithms, layouts at conditional jumps are merely + /// compatible, i.e. the exit layout of the jumping block is a superset of the + /// entry layout of the target block. This function modifies the entry layouts + /// of conditional jump targets, s.t. the entry layout of target blocks match + /// the exit layout of the jumping block exactly, except that slots not + /// required after the jump are marked as `JunkSlot`s. + void stitchConditionalJumps(CFG::BasicBlock const &Block); + + /// Calculates the ideal stack layout, s.t. both \p Stack1 and \p Stack2 can + /// be achieved with minimal stack shuffling when starting from the returned + /// layout. + static Stack combineStack(Stack const &Stack1, Stack const &Stack2); + + /// Walks through the CFG and reports any stack too deep errors that would + /// occur when generating code for it without countermeasures. + std::vector + reportStackTooDeep(CFG::BasicBlock const &Entry) const; + + /// Returns a copy of \p Stack stripped of all duplicates and slots that can + /// be freely generated. Attempts to create a layout that requires a minimal + /// amount of operations to reconstruct the original stack \p Stack. + static Stack compressStack(Stack Stack); + + //// Fills in junk when entering branches that do not need a clean stack in + ///case the result is cheaper. + void fillInJunk(CFG::BasicBlock const &Block, + CFG::FunctionInfo const *FunctionInfo = nullptr); + + StackLayout &Layout; + CFG::FunctionInfo const *CurrentFunctionInfo = nullptr; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_EVM_EVMSTACKLAYOUTGENERATOR_H diff --git a/llvm/lib/Target/EVM/EVMStackShuffler.h b/llvm/lib/Target/EVM/EVMStackShuffler.h new file mode 100644 index 000000000000..9968972f8b13 --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackShuffler.h @@ -0,0 +1,549 @@ +//===-- EVMStackShuffler.h - Implementation of stack shuffling ---*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares template algorithms to find optimal (cheapest) transition +// between two stack layouts using three shuffling primitives: `swap`, `dup`, +// and `pop`. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_EVM_EVMSTACKSHUFFLER_H +#define LLVM_LIB_TARGET_EVM_EVMSTACKSHUFFLER_H + +#include "EVMControlFlowGraph.h" +#include "EVMHelperUtilities.h" +#include "llvm/CodeGen/MachineFunction.h" +#include +#include +#include +#include + +namespace llvm { + +/* +template struct Overload : Ts... { +using Ts::operator()...; +}; +template Overload(Ts...) -> Overload; + +template static inline SmallVector iota17(T begin, T end) { +SmallVector R(end - begin); +std::iota(R.begin(), R.end(), begin); +return R; +} +*/ +// Abstraction of stack shuffling operations. Can be defined as actual concept +// once we switch to C++20. Used as an interface for the stack shuffler below. +// The shuffle operation class is expected to internally keep track of a current +// stack layout (the "source layout") that the shuffler is supposed to shuffle +// to a fixed target stack layout. The shuffler works iteratively. At each +// iteration it instantiates an instance of the shuffle operations and queries +// it for various information about the current source stack layout and the +// target layout, as described in the interface below. Based on that information +// the shuffler decides which is the next optimal operation to perform on the +// stack and calls the corresponding entry point in the shuffling operations +// (swap, pushOrDupTarget or pop). +/* +template +concept ShuffleOperationConcept = + requires(ShuffleOperations ops, size_t sourceOffset, + size_t targetOffset, size_t depth) { + + // Returns true, iff the current slot at sourceOffset in source layout + // is a suitable slot at targetOffset. + { ops.isCompatible(sourceOffset, targetOffset) } + -> std::convertible_to; + + // Returns true, iff the slots at the two given source offsets are identical. + { ops.sourceIsSame(sourceOffset, sourceOffset) } -> + std::convertible_to; + + // Returns a positive integer n, if the slot at the given source offset + // needs n more copies. Returns a negative integer -n, if the slot at the + // given source offsets occurs n times too many. Returns zero if the amount + // of occurrences, in the current source layout, of the slot at the given + // source offset matches the desired amount of occurrences in the target. + { ops.sourceMultiplicity(sourceOffset) } -> std::convertible_to; + + // Returns a positive integer n, if the slot at the given target offset + // needs n more copies. Returns a negative integer -n, if the slot at the + // given target offsets occurs n times too many. Returns zero if the amount + // of occurrences, in the current source layout, of the slot at the given + // target offset matches the desired amount of occurrences in the target. + { ops.targetMultiplicity(targetOffset) } -> std::convertible_to; + + // Returns true, iff any slot is compatible with the given target offset. + { ops.targetIsArbitrary(targetOffset) } -> std::convertible_to; + + // Returns the number of slots in the source layout. + { ops.sourceSize() } -> std::convertible_to; + + // Returns the number of slots in the target layout. + { ops.targetSize() } -> std::convertible_to; + + // Swaps the top most slot in the source with the slot `depth` slots below + // the top. In terms of EVM opcodes this is supposed to be a `SWAP`. + // In terms of vectors this is supposed to be + //`std::swap(source.at(source.size() - depth - 1, source.top))`. + { ops.swap(depth) }; + + // Pops the top most slot in the source, i.e. the slot at offset + // ops.sourceSize() - 1. In terms of EVM opcodes this is `POP`. + // In terms of vectors this is `source.pop();`. + { ops.pop() }; + + // Dups or pushes the slot that is supposed to end up at the given + // target offset. + { ops.pushOrDupTarget(targetOffset) }; +}; +*/ + +/// Helper class that can perform shuffling of a source stack layout to a target +/// stack layout via abstracted shuffle operations. +template +class Shuffler { +public: + /// Executes the stack shuffling operations. Instantiates an instance of + /// ShuffleOperations in each iteration. Each iteration performs exactly one + /// operation that modifies the stack. After `shuffle`, source and target have + /// the same size and all slots in the source layout are compatible with the + /// slots at the same target offset. + template static void shuffle(Args &&...args) { + bool NeedsMoreShuffling = true; + // The shuffling algorithm should always terminate in polynomial time, but + // we provide a limit in case it does not terminate due to a bug. + size_t IterationCount = 0; + while (IterationCount < 1000 && + (NeedsMoreShuffling = shuffleStep(std::forward(args)...))) + ++IterationCount; + + if (NeedsMoreShuffling) + llvm_unreachable("Could not create stack layout after 1000 iterations."); + } + +private: + // If dupping an ideal slot causes a slot that will still be required to + // become unreachable, then dup the latter slot first. + // Returns true, if it performed a dup. + static bool dupDeepSlotIfRequired(ShuffleOperations &Ops) { + // Check if the stack is large enough for anything to potentially become + // unreachable. + if (Ops.sourceSize() < 15) + return false; + // Check whether any deep slot might still be needed later (i.e. we still + // need to reach it with a DUP or SWAP). + for (size_t SourceOffset = 0; SourceOffset < (Ops.sourceSize() - 15); + ++SourceOffset) { + // This slot needs to be moved. + if (!Ops.isCompatible(SourceOffset, SourceOffset)) { + // If the current top fixes the slot, swap it down now. + if (Ops.isCompatible(Ops.sourceSize() - 1, SourceOffset)) { + Ops.swap(Ops.sourceSize() - SourceOffset - 1); + return true; + } + // Bring up a slot to fix this now, if possible. + if (bringUpTargetSlot(Ops, SourceOffset)) + return true; + // Otherwise swap up the slot that will fix the offending slot. + for (auto offset = SourceOffset + 1; offset < Ops.sourceSize(); + ++offset) + if (Ops.isCompatible(offset, SourceOffset)) { + Ops.swap(Ops.sourceSize() - offset - 1); + return true; + } + // Otherwise give up - we will need stack compression or stack limit + // evasion. + } + // We need another copy of this slot. + else if (Ops.sourceMultiplicity(SourceOffset) > 0) { + // If this slot occurs again later, we skip this occurrence. + // TODO: use C++ 20 ranges::views::iota + if (const auto &R = iota17(SourceOffset + 1, Ops.sourceSize()); + std::any_of(R.begin(), R.end(), [&](size_t Offset) { + return Ops.sourceIsSame(SourceOffset, Offset); + })) + continue; + + // Bring up the target slot that would otherwise become unreachable. + for (size_t TargetOffset = 0; TargetOffset < Ops.targetSize(); + ++TargetOffset) + if (!Ops.targetIsArbitrary(TargetOffset) && + Ops.isCompatible(SourceOffset, TargetOffset)) { + Ops.pushOrDupTarget(TargetOffset); + return true; + } + } + } + return false; + } + + /// Finds a slot to dup or push with the aim of eventually fixing \p + /// TargetOffset in the target. In the simplest case, the slot at \p + /// TargetOffset has a multiplicity > 0, i.e. it can directly be dupped or + /// pushed and the next iteration will fix \p TargetOffset. But, in general, + /// there may already be enough copies of the slot that is supposed to end up + /// at \p TargetOffset on stack, s.t. it cannot be dupped again. In that case + /// there has to be a copy of the desired slot on stack already elsewhere that + /// is not yet in place (`nextOffset` below). The fact that ``nextOffset`` is + /// not in place means that we can (recursively) try bringing up the slot that + /// is supposed to end up at ``nextOffset`` in the *target*. When the target + /// slot at ``nextOffset`` is fixed, the current source slot at ``nextOffset`` + /// will be at the stack top, which is the slot required at \p TargetOffset. + static bool bringUpTargetSlot(ShuffleOperations &Ops, size_t TargetOffset) { + std::list ToVisit{TargetOffset}; + std::set Visited; + + while (!ToVisit.empty()) { + size_t Offset = *ToVisit.begin(); + ToVisit.erase(ToVisit.begin()); + Visited.emplace(Offset); + if (Ops.targetMultiplicity(Offset) > 0) { + Ops.pushOrDupTarget(Offset); + return true; + } + // There must be another slot we can dup/push that will lead to the target + // slot at ``offset`` to be fixed. + for (size_t NextOffset = 0; + NextOffset < std::min(Ops.sourceSize(), Ops.targetSize()); + ++NextOffset) + if (!Ops.isCompatible(NextOffset, NextOffset) && + Ops.isCompatible(NextOffset, Offset)) + if (!Visited.count(NextOffset)) + ToVisit.emplace_back(NextOffset); + } + return false; + } + + /// Performs a single stack operation, transforming the source layout closer + /// to the target layout. + template static bool shuffleStep(Args &&...args) { + ShuffleOperations Ops{std::forward(args)...}; + + // All source slots are final. + if (const auto &R = iota17(0u, Ops.sourceSize()); + std::all_of(R.begin(), R.end(), [&](size_t Index) { + return Ops.isCompatible(Index, Index); + })) { + // Bring up all remaining target slots, if any, or terminate otherwise. + if (Ops.sourceSize() < Ops.targetSize()) { + if (!dupDeepSlotIfRequired(Ops)) + assert(bringUpTargetSlot(Ops, Ops.sourceSize())); + return true; + } + return false; + } + + size_t SourceTop = Ops.sourceSize() - 1; + // If we no longer need the current stack top, we pop it, unless we need an + // arbitrary slot at this position in the target. + if (Ops.sourceMultiplicity(SourceTop) < 0 && + !Ops.targetIsArbitrary(SourceTop)) { + Ops.pop(); + return true; + } + + assert(Ops.targetSize() > 0); + + // If the top is not supposed to be exactly what is on top right now, try to + // find a lower position to swap it to. + if (!Ops.isCompatible(SourceTop, SourceTop) || + Ops.targetIsArbitrary(SourceTop)) + for (size_t Offset = 0; + Offset < std::min(Ops.sourceSize(), Ops.targetSize()); ++Offset) + // It makes sense to swap to a lower position, if + if (!Ops.isCompatible( + Offset, Offset) && // The lower slot is not already in position. + !Ops.sourceIsSame( + Offset, SourceTop) && // We would not just swap identical slots. + Ops.isCompatible( + SourceTop, + Offset)) { // The lower position wants to have this slot. + // We cannot swap that deep. + if (Ops.sourceSize() - Offset - 1 > 16) { + // If there is a reachable slot to be removed, park the current top + // there. + for (size_t SwapDepth = 16; SwapDepth > 0; --SwapDepth) + if (Ops.sourceMultiplicity(Ops.sourceSize() - 1 - SwapDepth) < + 0) { + Ops.swap(SwapDepth); + if (Ops.targetIsArbitrary(SourceTop)) + // Usually we keep a slot that is to-be-removed, if the + // current top is arbitrary. However, since we are in a + // stack-too-deep situation, pop it immediately to compress + // the stack (we can always push back junk in the end). + Ops.pop(); + return true; + } + // TODO: otherwise we rely on stack compression or stack-to-memory. + } + Ops.swap(Ops.sourceSize() - Offset - 1); + return true; + } + + // Ops.sourceSize() > Ops.targetSize() cannot be true anymore, since if the + // source top is no longer required, we already popped it, and if it is + // required, we already swapped it down to a suitable target position. + assert(Ops.sourceSize() <= Ops.targetSize()); + + // If a lower slot should be removed, try to bring up the slot that should + // end up there and bring it up. Note that after the cases above, there will + // always be a target slot to duplicate in this case. + for (size_t Offset = 0; Offset < Ops.sourceSize(); ++Offset) + if (!Ops.isCompatible( + Offset, Offset) && // The lower slot is not already in position. + Ops.sourceMultiplicity(Offset) < + 0 && // We have too many copies of this slot. + Offset <= + Ops.targetSize() && // There is a target slot at this position. + !Ops.targetIsArbitrary( + Offset)) { // And that target slot is not arbitrary. + if (!dupDeepSlotIfRequired(Ops)) + assert(bringUpTargetSlot(Ops, Offset)); + return true; + } + + // At this point we want to keep all slots. + for (size_t i = 0; i < Ops.sourceSize(); ++i) + assert(Ops.sourceMultiplicity(i) >= 0); + assert(Ops.sourceSize() <= Ops.targetSize()); + + // If the top is not in position, try to find a slot that wants to be at the + // top and swap it up. + if (!Ops.isCompatible(SourceTop, SourceTop)) + for (size_t sourceOffset = 0; sourceOffset < Ops.sourceSize(); + ++sourceOffset) + if (!Ops.isCompatible(sourceOffset, sourceOffset) && + Ops.isCompatible(sourceOffset, SourceTop)) { + Ops.swap(Ops.sourceSize() - sourceOffset - 1); + return true; + } + + // If we still need more slots, produce a suitable one. + if (Ops.sourceSize() < Ops.targetSize()) { + if (!dupDeepSlotIfRequired(Ops)) + assert(bringUpTargetSlot(Ops, Ops.sourceSize())); + return true; + } + + // The stack has the correct size, each slot has the correct number of + // copies and the top is in position. + assert(Ops.sourceSize() == Ops.targetSize()); + size_t Size = Ops.sourceSize(); + for (size_t I = 0; I < Ops.sourceSize(); ++I) + assert(Ops.sourceMultiplicity(I) == 0 && + (Ops.targetIsArbitrary(I) || Ops.targetMultiplicity(I) == 0)); + assert(Ops.isCompatible(SourceTop, SourceTop)); + + const auto &SwappableOffsets = iota17(Size > 17 ? Size - 17 : 0u, Size); + + // If we find a lower slot that is out of position, but also compatible with + // the top, swap that up. + for (size_t Offset : SwappableOffsets) + if (!Ops.isCompatible(Offset, Offset) && + Ops.isCompatible(SourceTop, Offset)) { + Ops.swap(Size - Offset - 1); + return true; + } + + // Swap up any reachable slot that is still out of position. + for (size_t Offset : SwappableOffsets) + if (!Ops.isCompatible(Offset, Offset) && + !Ops.sourceIsSame(Offset, SourceTop)) { + Ops.swap(Size - Offset - 1); + return true; + } + + // We are in a stack-too-deep situation and try to reduce the stack size. + // If the current top is merely kept since the target slot is arbitrary, pop + // it. + if (Ops.targetIsArbitrary(SourceTop) && + Ops.sourceMultiplicity(SourceTop) <= 0) { + Ops.pop(); + return true; + } + + // If any reachable slot is merely kept, since the target slot is arbitrary, + // swap it up and pop it. + for (size_t Offset : SwappableOffsets) + if (Ops.targetIsArbitrary(Offset) && + Ops.sourceMultiplicity(Offset) <= 0) { + Ops.swap(Size - Offset - 1); + Ops.pop(); + return true; + } + + // We cannot avoid a stack-too-deep error. Repeat the above without + // restricting to reachable slots. + for (size_t Offset = 0; Offset < Size; ++Offset) + if (!Ops.isCompatible(Offset, Offset) && + Ops.isCompatible(SourceTop, Offset)) { + Ops.swap(Size - Offset - 1); + return true; + } + + for (size_t Offset = 0; Offset < Size; ++Offset) + if (!Ops.isCompatible(Offset, Offset) && + !Ops.sourceIsSame(Offset, SourceTop)) { + Ops.swap(Size - Offset - 1); + return true; + } + + llvm_unreachable("Unexpected state"); + } +}; + +/// A simple optimized map for mapping StackSlot to ints. +class Multiplicity { +public: + int &operator[](StackSlot const &Slot) { + if (auto *p = std::get_if(&Slot)) + return FunctionCallReturnLabelSlotMultiplicity[*p]; + if (std::holds_alternative(Slot)) + return FunctionReturnLabelSlotMultiplicity; + if (auto *p = std::get_if(&Slot)) + return VariableSlotMultiplicity[*p]; + if (auto *p = std::get_if(&Slot)) + return LiteralSlotMultiplicity[*p]; + if (auto *p = std::get_if(&Slot)) + return SymbolSlotMultiplicity[*p]; + if (auto *p = std::get_if(&Slot)) + return TemporarySlotMultiplicity[*p]; + + assert(std::holds_alternative(Slot)); + return JunkSlotMultiplicity; + } + + int at(StackSlot const &Slot) const { + if (auto *p = std::get_if(&Slot)) + return FunctionCallReturnLabelSlotMultiplicity.at(*p); + if (std::holds_alternative(Slot)) + return FunctionReturnLabelSlotMultiplicity; + if (auto *p = std::get_if(&Slot)) + return VariableSlotMultiplicity.at(*p); + if (auto *p = std::get_if(&Slot)) + return LiteralSlotMultiplicity.at(*p); + if (auto *p = std::get_if(&Slot)) + return SymbolSlotMultiplicity.at(*p); + if (auto *p = std::get_if(&Slot)) + return TemporarySlotMultiplicity.at(*p); + + assert(std::holds_alternative(Slot)); + return JunkSlotMultiplicity; + } + +private: + std::map + FunctionCallReturnLabelSlotMultiplicity; + int FunctionReturnLabelSlotMultiplicity = 0; + std::map VariableSlotMultiplicity; + std::map LiteralSlotMultiplicity; + std::map SymbolSlotMultiplicity; + std::map TemporarySlotMultiplicity; + int JunkSlotMultiplicity = 0; +}; + +/// Transforms \p CurrentStack to \p TargetStack, invoking the provided +/// shuffling operations. Modifies `CurrentStack` itself after each invocation +/// of the shuffling operations. +/// \p Swap is a function with signature void(unsigned) that is called when the +/// top most slot is swapped with the slot `depth` slots below the top. In terms +/// of EVM opcodes this is supposed to be a `SWAP`. +/// \p PushOrDup is a function with signature void(StackSlot const&) that is +/// called to push or dup the slot given as its argument to the stack top. +/// \p Pop is a function with signature void() that is called when the top most +/// slot is popped. +template +void createStackLayout(Stack &CurrentStack, Stack const &TargetStack, + SwapT Swap, PushOrDupT PushOrDup, PopT Pop) { + struct ShuffleOperations { + Stack ¤tStack; + Stack const &targetStack; + SwapT swapCallback; + PushOrDupT pushOrDupCallback; + PopT popCallback; + Multiplicity multiplicity; + + ShuffleOperations(Stack &CurrentStack, Stack const &TargetStack, SwapT Swap, + PushOrDupT PushOrDup, PopT Pop) + : currentStack(CurrentStack), targetStack(TargetStack), + swapCallback(Swap), pushOrDupCallback(PushOrDup), popCallback(Pop) { + for (auto const &slot : currentStack) + --multiplicity[slot]; + + for (unsigned Offset = 0; Offset < targetStack.size(); ++Offset) { + auto &Slot = targetStack[Offset]; + if (std::holds_alternative(Slot) && + Offset < currentStack.size()) + ++multiplicity[currentStack.at(Offset)]; + else + ++multiplicity[Slot]; + } + } + + bool isCompatible(size_t Source, size_t Target) { + return Source < currentStack.size() && Target < targetStack.size() && + (std::holds_alternative(targetStack.at(Target)) || + currentStack.at(Source) == targetStack.at(Target)); + } + + bool sourceIsSame(size_t Lhs, size_t Rhs) { + return currentStack.at(Lhs) == currentStack.at(Rhs); + } + + int sourceMultiplicity(size_t Offset) { + return multiplicity.at(currentStack.at(Offset)); + } + + int targetMultiplicity(size_t Offset) { + return multiplicity.at(targetStack.at(Offset)); + } + + bool targetIsArbitrary(size_t Offset) { + return Offset < targetStack.size() && + std::holds_alternative(targetStack.at(Offset)); + } + + void swap(size_t I) { + swapCallback(static_cast(I)); + std::swap(currentStack.at(currentStack.size() - I - 1), + currentStack.back()); + } + + size_t sourceSize() { return currentStack.size(); } + + size_t targetSize() { return targetStack.size(); } + + void pop() { + popCallback(); + currentStack.pop_back(); + } + + void pushOrDupTarget(size_t Offset) { + auto const &targetSlot = targetStack.at(Offset); + pushOrDupCallback(targetSlot); + currentStack.push_back(targetSlot); + } + }; + + Shuffler::shuffle(CurrentStack, TargetStack, Swap, + PushOrDup, Pop); + + assert(CurrentStack.size() == TargetStack.size()); + for (unsigned I = 0; I < CurrentStack.size(); ++I) { + auto &Current = CurrentStack[I]; + auto &Target = TargetStack[I]; + if (std::holds_alternative(Target)) + Current = JunkSlot{}; + else + assert(Current == Target); + } +} + +} // end namespace llvm +#endif // LLVM_LIB_TARGET_EVM_EVMSTACKSHUFFLER_H diff --git a/llvm/lib/Target/EVM/EVMStackifyEF.cpp b/llvm/lib/Target/EVM/EVMStackifyEF.cpp new file mode 100644 index 000000000000..31f2bc3bf4bf --- /dev/null +++ b/llvm/lib/Target/EVM/EVMStackifyEF.cpp @@ -0,0 +1,84 @@ +//===----- EVMStackifyEF.cpp - Split Critical Edges ------*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file performs spliting of critical edges. +// +//===----------------------------------------------------------------------===// + +#include "EVM.h" +#include "EVMAssembly.h" +#include "EVMOptimizedCodeTransform.h" +#include "EVMSubtarget.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +#define DEBUG_TYPE "evm-ethereum-stackify" + +namespace { +class EVMStackifyEF final : public MachineFunctionPass { +public: + static char ID; // Pass identification, replacement for typeid + + EVMStackifyEF() : MachineFunctionPass(ID) {} + +private: + StringRef getPassName() const override { return "EVM spliting latch blocks"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + MachineFunctionProperties getRequiredProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::TracksLiveness); + } +}; +} // end anonymous namespace + +char EVMStackifyEF::ID = 0; + +INITIALIZE_PASS_BEGIN(EVMStackifyEF, DEBUG_TYPE, "Ethereum stackification", + false, false) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_END(EVMStackifyEF, DEBUG_TYPE, "Ethereum stackification", false, + false) + +FunctionPass *llvm::createEVMStackifyEF() { return new EVMStackifyEF(); } + +bool EVMStackifyEF::runOnMachineFunction(MachineFunction &MF) { + LLVM_DEBUG({ + dbgs() << "********** Ethereum stackification **********\n" + << "********** Function: " << MF.getName() << '\n'; + }); + + MachineRegisterInfo &MRI = MF.getRegInfo(); + const EVMInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + LiveIntervals &LIS = getAnalysis(); + MachineLoopInfo *MLI = &getAnalysis(); + + // We don't preserve SSA form. + MRI.leaveSSA(); + + assert(MRI.tracksLiveness() && "Stackify expects liveness"); + + EVMAssembly Assembly(&MF, TII); + EVMOptimizedCodeTransform::run(Assembly, MF, LIS, MLI); + return true; +} diff --git a/llvm/lib/Target/EVM/EVMTargetMachine.cpp b/llvm/lib/Target/EVM/EVMTargetMachine.cpp index eaf0892de5ff..306f4d4db1cb 100644 --- a/llvm/lib/Target/EVM/EVMTargetMachine.cpp +++ b/llvm/lib/Target/EVM/EVMTargetMachine.cpp @@ -38,6 +38,10 @@ cl::opt cl::desc("EVM: output stack registers in" " instruction output for test purposes only."), cl::init(false)); +cl::opt + EVMUseLocalStakify("evm-use-local-stackify", cl::Hidden, + cl::desc("EVM: use the local stackification algorithm"), + cl::init(false)); extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeEVMTarget() { // Register the target. @@ -50,7 +54,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeEVMTarget() { initializeEVMOptimizeLiveIntervalsPass(PR); initializeEVMRegColoringPass(PR); initializeEVMSingleUseExpressionPass(PR); + initializeEVMSplitCriticalEdgesPass(PR); initializeEVMStackifyPass(PR); + initializeEVMStackifyEFPass(PR); } static std::string computeDataLayout() { @@ -74,7 +80,6 @@ EVMTargetMachine::EVMTargetMachine(const Target &T, const Triple &TT, getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique()), Subtarget(TT, std::string(CPU), std::string(FS), *this) { - setRequiresStructuredCFG(true); initAsmInfo(); } @@ -194,11 +199,16 @@ void EVMPassConfig::addPreEmitPass() { // FIXME: enable all the passes below, but the Stackify with EVMKeepRegisters. if (!EVMKeepRegisters) { + addPass(createEVMSplitCriticalEdges()); + addPass(&MachineBlockPlacementID); addPass(createEVMOptimizeLiveIntervals()); addPass(createEVMSingleUseExpression()); // Run the register coloring pass to reduce the total number of registers. addPass(createEVMRegColoring()); - addPass(createEVMStackify()); + if (EVMUseLocalStakify) + addPass(createEVMStackify()); + else + addPass(createEVMStackifyEF()); } } diff --git a/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.cpp b/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.cpp index d9f73e7a1de7..cb4ae74f2672 100644 --- a/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.cpp +++ b/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.cpp @@ -122,3 +122,81 @@ unsigned llvm::EVM::getPUSHOpcode(const APInt &Imm) { llvm_unreachable("Unexpected stack depth"); } } + +unsigned llvm::EVM::getDUPOpcode(unsigned Depth) { + switch (Depth) { + case 1: + return EVM::DUP1; + case 2: + return EVM::DUP2; + case 3: + return EVM::DUP3; + case 4: + return EVM::DUP4; + case 5: + return EVM::DUP5; + case 6: + return EVM::DUP6; + case 7: + return EVM::DUP7; + case 8: + return EVM::DUP8; + case 9: + return EVM::DUP9; + case 10: + return EVM::DUP10; + case 11: + return EVM::DUP11; + case 12: + return EVM::DUP12; + case 13: + return EVM::DUP13; + case 14: + return EVM::DUP14; + case 15: + return EVM::DUP15; + case 16: + return EVM::DUP16; + default: + llvm_unreachable("Unexpected stack depth"); + } +} + +unsigned llvm::EVM::getSWAPOpcode(unsigned Depth) { + switch (Depth) { + case 1: + return EVM::SWAP1; + case 2: + return EVM::SWAP2; + case 3: + return EVM::SWAP3; + case 4: + return EVM::SWAP4; + case 5: + return EVM::SWAP5; + case 6: + return EVM::SWAP6; + case 7: + return EVM::SWAP7; + case 8: + return EVM::SWAP8; + case 9: + return EVM::SWAP9; + case 10: + return EVM::SWAP10; + case 11: + return EVM::SWAP11; + case 12: + return EVM::SWAP12; + case 13: + return EVM::SWAP13; + case 14: + return EVM::SWAP14; + case 15: + return EVM::SWAP15; + case 16: + return EVM::SWAP16; + default: + llvm_unreachable("Unexpected stack depth"); + } +} diff --git a/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.h b/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.h index 5b4abc84d6cc..e0c6998bbb5a 100644 --- a/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.h +++ b/llvm/lib/Target/EVM/TargetInfo/EVMTargetInfo.h @@ -25,6 +25,8 @@ namespace EVM { unsigned getStackOpcode(unsigned Opcode); unsigned getRegisterOpcode(unsigned Opcode); unsigned getPUSHOpcode(const APInt &Imm); +unsigned getDUPOpcode(unsigned Depth); +unsigned getSWAPOpcode(unsigned Depth); } // namespace EVM diff --git a/llvm/unittests/Target/EVM/CMakeLists.txt b/llvm/unittests/Target/EVM/CMakeLists.txt new file mode 100644 index 000000000000..e259d70e745d --- /dev/null +++ b/llvm/unittests/Target/EVM/CMakeLists.txt @@ -0,0 +1,23 @@ +include_directories( + ${LLVM_MAIN_SRC_DIR}/lib/Target/EVM + ${LLVM_BINARY_DIR}/lib/Target/EVM + ) + +set(LLVM_LINK_COMPONENTS + EVM + EVMDesc + EVMInfo + CodeGen + Core + MC + MIRParser + SelectionDAG + Support + TargetParser +) + +add_llvm_target_unittest(EVMTests + StackShuffler.cpp + ) + +set_property(TARGET EVMTests PROPERTY FOLDER "Tests/UnitTests/TargetTests") diff --git a/llvm/unittests/Target/EVM/StackShuffler.cpp b/llvm/unittests/Target/EVM/StackShuffler.cpp new file mode 100644 index 000000000000..82bfde1b5e6a --- /dev/null +++ b/llvm/unittests/Target/EVM/StackShuffler.cpp @@ -0,0 +1,196 @@ +//===---------- llvm/unittests/MC/AssemblerTest.cpp -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "EVMControlFlowGraph.h" +#include "EVMRegisterInfo.h" +#include "EVMStackDebug.h" +#include "EVMStackShuffler.h" +#include "EVMTargetMachine.h" +#include "MCTargetDesc/EVMMCTargetDesc.h" +#include "llvm-c/Core.h" +#include "llvm-c/IRReader.h" +#include "llvm-c/TargetMachine.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Target/CodeGenCWrappers.h" +#include "llvm/Target/TargetMachine.h" +#include "gtest/gtest.h" +#include +#include + +using namespace llvm; + +#include +static TargetMachine *unwrap(LLVMTargetMachineRef P) { + return reinterpret_cast(P); +} + +class LLDCTest : public testing::Test { + void SetUp() override { + LLVMInitializeEVMTargetInfo(); + LLVMInitializeEVMTarget(); + LLVMInitializeEVMTargetMC(); + + LLVMTargetRef Target = 0; + const char *Triple = "evm"; + char *ErrMsg = 0; + if (LLVMGetTargetFromTriple(Triple, &Target, &ErrMsg)) { + FAIL() << "Failed to create target from the triple (" << Triple + << "): " << ErrMsg; + return; + } + ASSERT_TRUE(Target); + + // Construct a TargetMachine. + TM = + LLVMCreateTargetMachine(Target, Triple, "", "", LLVMCodeGenLevelDefault, + LLVMRelocDefault, LLVMCodeModelDefault); + Context = std::make_unique(); + Mod = std::make_unique("TestModule", *Context); + Mod->setDataLayout(unwrap(TM)->createDataLayout()); + const LLVMTargetMachine &LLVMTM = + static_cast(*unwrap(TM)); + MMIWP = std::make_unique(&LLVMTM); + + Type *const ReturnType = Type::getVoidTy(Mod->getContext()); + FunctionType *FunctionType = FunctionType::get(ReturnType, false); + Function *const F = Function::Create( + FunctionType, GlobalValue::InternalLinkage, "TestFunction", Mod.get()); + MF = &MMIWP->getMMI().getOrCreateMachineFunction(*F); + } + + void TearDown() override { LLVMDisposeTargetMachine(TM); } + +public: + LLVMTargetMachineRef TM; + MachineFunction *MF = nullptr; + std::unique_ptr Context; + std::unique_ptr Mod; + std::unique_ptr MMIWP; +}; + +TEST_F(LLDCTest, Basic) { + Stack SourceStack; + Stack TargetStack; + + MachineBasicBlock *MBB = MF->CreateMachineBasicBlock(nullptr); + MF->push_back(MBB); + + const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); + const MCInstrDesc &MCID = TII->get(EVM::SELFBALANCE); + MachineRegisterInfo &MRI = MF->getRegInfo(); + + auto CreateInstr = [&]() { + Register Reg = MRI.createVirtualRegister(&EVM::GPRRegClass); + MachineInstr *MI = BuildMI(MBB, DebugLoc(), MCID, Reg); + return std::pair(MI, Reg); + }; + SmallVector> Instrs; + for (unsigned I = 0; I < 17; ++I) + Instrs.emplace_back(CreateInstr()); + + // Create the source stack layout: + // [ %0 %1 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET RET %5 ] + SourceStack.emplace_back(VariableSlot{Instrs[0].second}); + SourceStack.emplace_back(VariableSlot{Instrs[1].second}); + SourceStack.emplace_back(VariableSlot{Instrs[2].second}); + SourceStack.emplace_back(VariableSlot{Instrs[3].second}); + SourceStack.emplace_back(VariableSlot{Instrs[4].second}); + SourceStack.emplace_back(VariableSlot{Instrs[5].second}); + SourceStack.emplace_back(VariableSlot{Instrs[6].second}); + SourceStack.emplace_back(VariableSlot{Instrs[7].second}); + SourceStack.emplace_back(VariableSlot{Instrs[9].second}); + SourceStack.emplace_back(VariableSlot{Instrs[10].second}); + SourceStack.emplace_back(VariableSlot{Instrs[11].second}); + SourceStack.emplace_back(VariableSlot{Instrs[12].second}); + SourceStack.emplace_back(VariableSlot{Instrs[13].second}); + SourceStack.emplace_back(VariableSlot{Instrs[14].second}); + SourceStack.emplace_back(VariableSlot{Instrs[15].second}); + SourceStack.emplace_back(VariableSlot{Instrs[16].second}); + SourceStack.emplace_back(FunctionReturnLabelSlot{MBB->getParent()}); + SourceStack.emplace_back(FunctionReturnLabelSlot{MBB->getParent()}); + SourceStack.emplace_back(VariableSlot{Instrs[5].second}); + + // [ %1 %0 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET JUNK JUNK ] + TargetStack.emplace_back(VariableSlot{Instrs[1].second}); + TargetStack.emplace_back(VariableSlot{Instrs[0].second}); + TargetStack.emplace_back(VariableSlot{Instrs[2].second}); + TargetStack.emplace_back(VariableSlot{Instrs[3].second}); + TargetStack.emplace_back(VariableSlot{Instrs[4].second}); + TargetStack.emplace_back(VariableSlot{Instrs[5].second}); + TargetStack.emplace_back(VariableSlot{Instrs[6].second}); + TargetStack.emplace_back(VariableSlot{Instrs[7].second}); + TargetStack.emplace_back(VariableSlot{Instrs[9].second}); + TargetStack.emplace_back(VariableSlot{Instrs[10].second}); + TargetStack.emplace_back(VariableSlot{Instrs[11].second}); + TargetStack.emplace_back(VariableSlot{Instrs[12].second}); + TargetStack.emplace_back(VariableSlot{Instrs[13].second}); + TargetStack.emplace_back(VariableSlot{Instrs[14].second}); + TargetStack.emplace_back(VariableSlot{Instrs[15].second}); + TargetStack.emplace_back(VariableSlot{Instrs[16].second}); + TargetStack.emplace_back(FunctionReturnLabelSlot{MBB->getParent()}); + TargetStack.emplace_back(JunkSlot{}); + TargetStack.emplace_back(JunkSlot{}); + + StringRef Reference("\ +[ %0 %1 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET RET %5 ]\n\ +POP\n\ +[ %0 %1 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET RET ]\n\ +SWAP16\n\ +[ %0 RET %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET %1 ]\n\ +SWAP16\n\ +[ %0 %1 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET RET ]\n\ +POP\n\ +[ %0 %1 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET ]\n\ +SWAP15\n\ +[ %0 RET %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 %1 ]\n\ +SWAP16\n\ +[ %1 RET %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 %0 ]\n\ +SWAP15\n\ +[ %1 %0 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET ]\n\ +PUSH JUNK\n\ +[ %1 %0 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET JUNK ]\n\ +PUSH JUNK\n\ +[ %1 %0 %2 %3 %4 %5 %6 %7 %9 %10 %11 %12 %13 %14 %15 %16 RET JUNK JUNK ]\n"); + + std::ostringstream Output; + createStackLayout( + SourceStack, TargetStack, + [&](unsigned SwapDepth) { // swap + Output << stackToString(SourceStack) << '\n'; + Output << "SWAP" << SwapDepth << '\n'; + }, + [&](StackSlot const &Slot) { // dupOrPush + Output << stackToString(SourceStack) << '\n'; + if (canBeFreelyGenerated(Slot)) + Output << "PUSH " << stackSlotToString(Slot) << '\n'; + else { + Stack TmpStack = SourceStack; + std::reverse(TmpStack.begin(), TmpStack.end()); + auto It = std::find(TmpStack.begin(), TmpStack.end(), Slot); + if (It == TmpStack.end()) + FAIL() << "Invalid DUP operation."; + + auto Depth = std::distance(TmpStack.begin(), It); + Output << "DUP" << Depth + 1 << '\n'; + } + }, + [&]() { // pop + Output << stackToString(SourceStack) << '\n'; + Output << "POP" << '\n'; + }); + + Output << stackToString(SourceStack) << '\n'; + std::cerr << Output.str(); + EXPECT_TRUE(Reference == Output.str()); +}