[EVM] Add Ethereum stackification.

This includes:
 - splitting of critical edges
 - CFG implementation.
 - stack shuffleri, with unittest
 - stack layout generator
 - optimized code transform
 - proper ordering of ARGUMENT instructions

llvm/lib/Target/EVM/CMakeLists.txt

@@ -20,7 +20,9 @@ add_llvm_target(EVMCodeGen
   EVMAllocaHoisting.cpp
   EVMArgumentMove.cpp
   EVMAsmPrinter.cpp
+  EVMAssembly.cpp
   EVMCodegenPrepare.cpp
+  EVMControlFlowGraphBuilder.cpp
   EVMFrameLowering.cpp
   EVMISelDAGToDAG.cpp
   EVMISelLowering.cpp
@@ -30,10 +32,15 @@ add_llvm_target(EVMCodeGen
   EVMMachineFunctionInfo.cpp
   EVMMCInstLower.cpp
   EVMOptimizeLiveIntervals.cpp
+  EVMOptimizedCodeTransform.cpp
   EVMRegColoring.cpp
   EVMRegisterInfo.cpp
   EVMSingleUseExpression.cpp
+  EVMSplitCriticalEdges.cpp
+  EVMStackDebug.cpp
+  EVMStackLayoutGenerator.cpp
   EVMStackify.cpp
+  EVMStackifyEF.cpp
   EVMSubtarget.cpp
   EVMTargetMachine.cpp
   EVMTargetTransformInfo.cpp

llvm/lib/Target/EVM/EVM.h

@@ -50,7 +50,9 @@ ModulePass *createEVMLinkRuntimePass();
 FunctionPass *createEVMOptimizeLiveIntervals();
 FunctionPass *createEVMRegColoring();
 FunctionPass *createEVMSingleUseExpression();
+FunctionPass *createEVMSplitCriticalEdges();
 FunctionPass *createEVMStackify();
+FunctionPass *createEVMStackifyEF();
 
 // PassRegistry initialization declarations.
 void initializeEVMCodegenPreparePass(PassRegistry &);
@@ -61,7 +63,9 @@ void initializeEVMLinkRuntimePass(PassRegistry &);
 void initializeEVMOptimizeLiveIntervalsPass(PassRegistry &);
 void initializeEVMRegColoringPass(PassRegistry &);
 void initializeEVMSingleUseExpressionPass(PassRegistry &);
+void initializeEVMSplitCriticalEdgesPass(PassRegistry &);
 void initializeEVMStackifyPass(PassRegistry &);
+void initializeEVMStackifyEFPass(PassRegistry &);
 
 struct EVMLinkRuntimePass : PassInfoMixin<EVMLinkRuntimePass> {
   EVMLinkRuntimePass() = default;

llvm/lib/Target/EVM/EVMArgumentMove.cpp

@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file moves ARGUMENT instructions after ScheduleDAG scheduling.
+// This file moves and orders ARGUMENT instructions after ScheduleDAG
+// scheduling.
 //
 // Arguments are really live-in registers, however, since we use virtual
 // registers and LLVM doesn't support live-in virtual registers, we're
@@ -67,21 +68,24 @@ bool EVMArgumentMove::runOnMachineFunction(MachineFunction &MF) {
 
   bool Changed = false;
   MachineBasicBlock &EntryMBB = MF.front();
+  SmallVector<MachineInstr *> Args;
+  for (MachineInstr &MI : EntryMBB) {
+    if (EVM::ARGUMENT == MI.getOpcode())
+      Args.push_back(&MI);
+  }
 
-  // Look for the first NonArg instruction.
-  const auto InsertPt =
-      std::find_if_not(EntryMBB.begin(), EntryMBB.end(), [](auto &MI) {
-        return EVM::ARGUMENT == MI.getOpcode();
-      });
+  // Sort ARGUMENT instructions in ascending order of their arguments.
+  std::sort(Args.begin(), Args.end(),
+            [](const MachineInstr *MI1, const MachineInstr *MI2) {
+              int64_t Arg1Idx = MI1->getOperand(1).getImm();
+              int64_t Arg2Idx = MI2->getOperand(1).getImm();
+              return Arg1Idx < Arg2Idx;
+            });
 
-  // Now move any argument instructions later in the block
-  // to before our first NonArg instruction.
-  for (MachineInstr &MI : llvm::make_range(InsertPt, EntryMBB.end())) {
-    if (EVM::ARGUMENT == MI.getOpcode()) {
-      EntryMBB.insert(InsertPt, MI.removeFromParent());
-      Changed = true;
-    }
+  for (MachineInstr *MI : reverse(Args)) {
+    MachineInstr *Arg = MI->removeFromParent();
+    EntryMBB.insert(EntryMBB.begin(), Arg);
+    Changed = true;
   }
-
   return Changed;
 }

llvm/lib/Target/EVM/EVMAssembly.cpp

@@ -0,0 +1,281 @@
+//===----------- EVMAssembly.cpp - EVM Assembly generator -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the EVMAssembly class that generates machine IR
+// with all the required stack manipulation instructions.
+// Resulting machine instructions still have explicit operands, but some of the
+// auxiliary instructions (ARGUMENT, RET, EVM::CONST_I256, COPY_I256
+// FCALLARGUMENT) are removed after this step, beaking use-def chains. So, the
+// resulting Machine IR breaks the MachineVerifier checks.
+//
+//===----------------------------------------------------------------------===//
+
+#include "EVMAssembly.h"
+#include "EVM.h"
+#include "EVMMachineFunctionInfo.h"
+#include "EVMSubtarget.h"
+#include "MCTargetDesc/EVMMCTargetDesc.h"
+#include "TargetInfo/EVMTargetInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/MC/MCContext.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "evm-assembly"
+
+#ifndef NDEBUG
+void EVMAssembly::dumpInst(const MachineInstr *MI) const {
+  LLVM_DEBUG(dbgs() << "Adding: " << *MI << "stack height: " << StackHeight
+                    << "\n");
+}
+#endif // NDEBUG
+
+int EVMAssembly::getStackHeight() const { return StackHeight; }
+
+void EVMAssembly::setStackHeight(int Height) {
+  StackHeight = Height;
+  LLVM_DEBUG(dbgs() << "Set stack height: " << StackHeight << "\n");
+}
+
+void EVMAssembly::setCurrentLocation(MachineBasicBlock *MBB) {
+  CurMBB = MBB;
+  CurMIIt = MBB->begin();
+  LLVM_DEBUG(dbgs() << "Setting current location to: " << MBB->getNumber()
+                    << "." << MBB->getName() << "\n");
+}
+
+void EVMAssembly::appendInstruction(MachineInstr *MI) {
+  unsigned Opc = MI->getOpcode();
+  assert(Opc != EVM::JUMP && Opc != EVM::JUMPI && Opc != EVM::ARGUMENT &&
+         Opc != EVM::RET && Opc != EVM::CONST_I256 && Opc != EVM::COPY_I256 &&
+         Opc != EVM::FCALL);
+
+  auto Ret = AssemblyInstrs.insert(MI);
+  assert(Ret.second);
+  int StackAdj = (2 * static_cast<int>(MI->getNumExplicitDefs())) -
+                 static_cast<int>(MI->getNumExplicitOperands());
+  StackHeight += StackAdj;
+  LLVM_DEBUG(dumpInst(MI));
+  CurMIIt = std::next(MIIter(MI));
+}
+
+void EVMAssembly::appendSWAPInstruction(unsigned Depth) {
+  unsigned Opc = EVM::getSWAPOpcode(Depth);
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc));
+  AssemblyInstrs.insert(&*CurMIIt);
+  dumpInst(&*CurMIIt);
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendDUPInstruction(unsigned Depth) {
+  unsigned Opc = EVM::getDUPOpcode(Depth);
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc));
+  StackHeight += 1;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendPOPInstruction() {
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::POP));
+  assert(StackHeight > 0);
+  StackHeight -= 1;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendConstant(const APInt &Val) {
+  unsigned Opc = EVM::getPUSHOpcode(Val);
+  MachineInstrBuilder Builder =
+      BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(Opc));
+  if (Opc != EVM::PUSH0) {
+    LLVMContext &Ctx = MF->getFunction().getContext();
+    Builder.addCImm(ConstantInt::get(Ctx, Val));
+  }
+  StackHeight += 1;
+  CurMIIt = Builder;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendSymbol(MCSymbol *Symbol) {
+  // This is codegen-only instruction, that will be converted into PUSH4.
+  CurMIIt =
+      BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::DATA)).addSym(Symbol);
+  StackHeight += 1;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendConstant(uint64_t Val) {
+  appendConstant(APInt(256, Val));
+}
+
+void EVMAssembly::appendLabel() {
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::JUMPDEST));
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendLabelReference(MCSymbol *Label) {
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::PUSH_LABEL))
+                .addSym(Label);
+  StackHeight += 1;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendMBBReference(MachineBasicBlock *MBB) {
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::PUSH_LABEL))
+                .addMBB(MBB);
+  StackHeight += 1;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+MCSymbol *EVMAssembly::createFuncRetSymbol() {
+  return MF->getContext().createTempSymbol("FUNC_RET", true);
+}
+
+void EVMAssembly::appendFuncCall(const MachineInstr *MI,
+                                 const llvm::GlobalValue *Func, int StackAdj,
+                                 MCSymbol *RetSym) {
+  // Push the function label
+  assert(CurMBB == MI->getParent());
+  CurMIIt =
+      BuildMI(*CurMBB, CurMIIt, MI->getDebugLoc(), TII->get(EVM::PUSH_LABEL))
+          .addGlobalAddress(Func);
+  // PUSH_LABEL technically increases the stack height on 1, but we don't
+  // increase it explicitly here, as the label will be consumed by the following
+  // JUMP.
+  AssemblyInstrs.insert(&*CurMIIt);
+  StackHeight += StackAdj;
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+
+  CurMIIt = std::next(CurMIIt);
+  // Create jump to the callee. Note, we don't add the 'target' operand to JUMP.
+  // This should be fine, unless we run MachineVerifier after this step
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, MI->getDebugLoc(), TII->get(EVM::JUMP));
+  if (RetSym)
+    CurMIIt->setPostInstrSymbol(*MF, RetSym);
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendJump(int StackAdj) {
+  CurMIIt = BuildMI(*CurMBB, CurMIIt, DebugLoc(), TII->get(EVM::JUMP));
+  StackHeight += StackAdj;
+  AssemblyInstrs.insert(&*CurMIIt);
+  LLVM_DEBUG(dumpInst(&*CurMIIt));
+  CurMIIt = std::next(CurMIIt);
+}
+
+void EVMAssembly::appendUncondJump(MachineInstr *MI,
+                                   MachineBasicBlock *Target) {
+  assert(MI->getOpcode() == EVM::JUMP);
+  appendMBBReference(Target);
+  [[maybe_unused]] auto It = AssemblyInstrs.insert(MI);
+  assert(It.second && StackHeight > 0);
+  StackHeight -= 1;
+  LLVM_DEBUG(dumpInst(MI));
+  CurMIIt = std::next(MIIter(MI));
+}
+
+void EVMAssembly::appendCondJump(MachineInstr *MI, MachineBasicBlock *Target) {
+  assert(MI->getOpcode() == EVM::JUMPI);
+  appendMBBReference(Target);
+  [[maybe_unused]] auto It = AssemblyInstrs.insert(MI);
+  assert(It.second && StackHeight > 1);
+  StackHeight -= 2;
+  LLVM_DEBUG(dumpInst(MI));
+  CurMIIt = std::next(MIIter(MI));
+}
+
+// Remove all registers operands of the \p MI and repaces the opcode with
+// the stack variant variant.
+void EVMAssembly::stackifyInstruction(MachineInstr *MI) {
+  if (MI->isDebugInstr() || MI->isLabel() || MI->isInlineAsm())
+    return;
+
+  unsigned RegOpcode = MI->getOpcode();
+  if (RegOpcode == EVM::PUSH_LABEL)
+    return;
+
+  // Remove register operands.
+  for (unsigned I = MI->getNumOperands(); I > 0; --I) {
+    auto &MO = MI->getOperand(I - 1);
+    if (MO.isReg()) {
+      MI->removeOperand(I - 1);
+    }
+  }
+
+  // Transform 'register' instruction to the 'stack' one.
+  unsigned StackOpcode = EVM::getStackOpcode(RegOpcode);
+  MI->setDesc(TII->get(StackOpcode));
+}
+
+void EVMAssembly::finalize() {
+  // Collect and erase instructions that are not required in
+  // a stackified code. These are auxiliary codegn-only instructions.
+  SmallVector<MachineInstr *, 128> ToRemove;
+  for (MachineBasicBlock &MBB : *MF) {
+    for (MachineInstr &MI : MBB) {
+      if (!AssemblyInstrs.count(&MI) && MI.getOpcode() != EVM::JUMP &&
+          MI.getOpcode() != EVM::JUMPI)
+        ToRemove.emplace_back(&MI);
+    }
+  }
+
+  for (MachineInstr *MI : ToRemove)
+    MI->eraseFromParent();
+
+  // Remove register operands and replace instruction opcode with 'stack' one.
+  for (MachineBasicBlock &MBB : *MF)
+    for (MachineInstr &MI : MBB)
+      stackifyInstruction(&MI);
+
+  auto *MFI = MF->getInfo<EVMMachineFunctionInfo>();
+  MFI->setIsStackified();
+
+  // In a stackified code register liveness has no meaning.
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  MRI.invalidateLiveness();
+
+  // In EVM architecture jump target is set up using one of PUSH* instructions
+  // that come right before the jump instruction.
+  // For example:
+
+  //   PUSH_LABEL %bb.10
+  //   JUMPI_S
+  //   PUSH_LABEL %bb.9
+  //   JUMP_S
+  //
+  // The problem here is that such MIR is not valid. There should not be
+  // non-terminator (PUSH) instructions between terminator (JUMP) ones.
+  // To overcome this issue, we bundle adjacent <PUSH_LABEL, JUMP> instructions
+  // together and unbundle them in the AsmPrinter.
+  for (MachineBasicBlock &MBB : *MF) {
+    MachineBasicBlock::instr_iterator I = MBB.instr_begin(),
+                                      E = MBB.instr_end();
+    for (; I != E; ++I) {
+      if (I->isBranch()) {
+        auto P = std::next(I);
+        if (P != E && P->getOpcode() == EVM::PUSH_LABEL)
+          I->bundleWithPred();
+      }
+    }
+  }
+}