[Autobackout][FuncReg]Revert of change: 89ca3bd

Switch TPM to SVM entirely

Author: zuban32

IGC/VectorCompiler/include/vc/GenXCodeGen/GenXInternalMetadata.h

@@ -38,14 +38,6 @@ namespace FunctionMD {
 inline constexpr const char GenXKernelInternal[] = "genx.kernel.internal";
 }
 
-namespace InstMD {
-inline constexpr const char SVMBlockType[] = "SVMBlockType";
-}
-
-namespace ModuleMD {
-inline constexpr const char UseSVMStack[] = "genx.useGlobalMem";
-}
-
 namespace internal {
 
 namespace KernelMDOp {

IGC/VectorCompiler/lib/GenXCodeGen/GenXCisaBuilder.cpp

@@ -3640,10 +3640,7 @@ void GenXKernelBuilder::buildIntrinsic(CallInst *CI, unsigned IntrinID,
     Value *V = CI;
     if (!AI.isRet())
       V = CI->getArgOperand(AI.getArgIdx());
-    auto *EltType = V->getType()->getScalarType();
-    if (auto *MDType = CI->getMetadata(InstMD::SVMBlockType))
-      EltType = cast<ValueAsMetadata>(MDType->getOperand(0).get())->getType();
-    unsigned ElBytes = getResultedTypeSize(EltType, DL);
+    unsigned ElBytes = getResultedTypeSize(V->getType()->getScalarType(), DL);
     switch (ElBytes) {
       // For N = 2 byte data type, use block size 1 and block count 2.
       // Otherwise, use block size N and block count 1.

IGC/VectorCompiler/lib/GenXCodeGen/GenXThreadPrivateMemory.cpp

@@ -35,7 +35,6 @@ IN THE SOFTWARE.
 #include "GenXTargetMachine.h"
 #include "GenXUtil.h"
 #include "GenXVisa.h"
-#include "vc/GenXCodeGen/GenXInternalMetadata.h"
 
 #include "Probe/Assertion.h"
 #include "llvmWrapper/IR/DerivedTypes.h"
@@ -209,7 +208,6 @@ std::pair<Value *, unsigned>
 GenXThreadPrivateMemory::NormalizeVector(Value *From, Type *To,
                                          Instruction *Inst) {
   Type *I32Ty = Type::getInt32Ty(Inst->getContext());
-  Type *I64Ty = Type::getInt64Ty(Inst->getContext());
   Value *Res = From;
   Type *FromTy = From->getType();
   IGC_ASSERT(isa<VectorType>(FromTy));
@@ -236,22 +234,22 @@ GenXThreadPrivateMemory::NormalizeVector(Value *From, Type *To,
     To = IGCLLVM::FixedVectorType::get(I32Ty, NumElts);
     EltSz = I32Ty->getPrimitiveSizeInBits() / genx::ByteBits;
     Res = CastInst::Create(Instruction::BitCast, Res, To, "", Inst);
-  } else if (m_DL->getTypeSizeInBits(cast<VectorType>(To)->getElementType()) <
-             genx::DWordBits) {
+  } else if (cast<VectorType>(To)->getElementType()->getPrimitiveSizeInBits() <
+                 genx::DWordBits
+             // this is required for correct generation of svm.gather/scatter
+             // of data of type which size is < i32 because these intrinsics
+             // infer their block size from the type of the data they handle
+             && !m_useGlobalMem) {
     To = IGCLLVM::FixedVectorType::get(I32Ty, NumElts);
-    Res = CastInst::CreateZExtOrBitCast(From, To, "", Inst);
-  } else if (!m_useGlobalMem &&
-             m_DL->getTypeSizeInBits(cast<VectorType>(To)->getElementType()) ==
-                 genx::QWordBits) {
-    if (From->getType()->getScalarType()->isPointerTy()) {
-      auto *NewType = IGCLLVM::FixedVectorType::get(I64Ty, NumElts);
-      From = CastInst::Create(CastInst::PtrToInt, From, NewType, "", Inst);
-    }
+
+    Res = CastInst::Create(Instruction::ZExt, From, To, "", Inst);
+  } else if (cast<VectorType>(To)->getElementType()->getPrimitiveSizeInBits() ==
+             genx::QWordBits) {
     NumElts *= 2;
     EltSz = I32Ty->getPrimitiveSizeInBits() / genx::ByteBits;
     To = IGCLLVM::FixedVectorType::get(I32Ty, NumElts);
 
-    Res = CastInst::CreateBitOrPointerCast(From, To, "", Inst);
+    Res = CastInst::Create(Instruction::BitCast, From, To, "", Inst);
   }
 
   return std::make_pair(Res, EltSz);
@@ -260,8 +258,6 @@ GenXThreadPrivateMemory::NormalizeVector(Value *From, Type *To,
 Instruction *
 GenXThreadPrivateMemory::RestoreVectorAfterNormalization(Instruction *From,
                                                          Type *To) {
-  if (From->getType() == To)
-    return From;
   Instruction *Restored = From;
   unsigned EltSz = m_DL->getTypeSizeInBits(To->getScalarType());
   IGC_ASSERT(EltSz > 0);
@@ -523,19 +519,35 @@ bool GenXThreadPrivateMemory::replaceLoad(LoadInst *LdI) {
     LdTy = IGCLLVM::FixedVectorType::get(LdTy, 1);
 
   unsigned NumEltsToLoad = cast<VectorType>(LdTy)->getNumElements();
-  unsigned ValueEltSz = m_DL->getTypeSizeInBits(LdEltTy) / genx::ByteBits;
+  unsigned LdEltTySz = m_DL->getTypeSizeInBits(LdEltTy);
+  if (!(m_useGlobalMem && LdEltTy->isIntegerTy(64)) &&
+      LdEltTySz == genx::QWordBits)
+    NumEltsToLoad *= 2;
 
   Value *PredVal = ConstantInt::get(Type::getInt1Ty(*m_ctx), 1);
   Value *Pred = Builder.CreateVectorSplat(NumEltsToLoad, PredVal);
 
   Type *I32Ty = Type::getInt32Ty(*m_ctx);
   Type *I64Ty = Type::getInt64Ty(*m_ctx);
+  Type *TyToLoad = (m_useGlobalMem && LdEltTy->isIntegerTy(64)) ? I64Ty : I32Ty;
+  if (LdEltTy->isFloatTy())
+    TyToLoad = LdEltTy;
+  Type *RealTyToLoad = LdEltTy;
+  if (!(m_useGlobalMem && LdEltTy->isIntegerTy(64)) &&
+      m_DL->getTypeSizeInBits(RealTyToLoad) == genx::QWordBits)
+    RealTyToLoad = I32Ty;
+  unsigned RealTyToLoadSz =
+      m_DL->getTypeSizeInBits(RealTyToLoad) / genx::ByteBits;
+  // we don't want to use improper block sizes for loads of i8/i16
+  // to make sure we comply with alignment rules for gathers
+  bool NoExtToDword =
+      m_useGlobalMem &&
+      !(LdI->getType()->isAggregateType() || LdI->getType()->isVectorTy()) &&
+      m_DL->getTypeSizeInBits(LdI->getType()) < genx::DWordBits;
+  if (NoExtToDword)
+    TyToLoad = LdI->getType();
   Value *OldValOfTheDataRead =
-      Builder.CreateVectorSplat(NumEltsToLoad, UndefValue::get(LdEltTy));
-  std::tie(OldValOfTheDataRead, ValueEltSz) =
-      NormalizeVector(OldValOfTheDataRead, LdTy, LdI);
-  NumEltsToLoad =
-      cast<VectorType>(OldValOfTheDataRead->getType())->getNumElements();
+      Builder.CreateVectorSplat(NumEltsToLoad, UndefValue::get(TyToLoad));
 
   Value *PointerOp = LdI->getPointerOperand();
   Value *Offset = lookForPtrReplacement(PointerOp);
@@ -545,13 +557,10 @@ bool GenXThreadPrivateMemory::replaceLoad(LoadInst *LdI) {
                  ? llvm::GenXIntrinsic::genx_svm_gather
                  : llvm::GenXIntrinsic::genx_gather_scaled;
 
-  Value *EltsOffset = FormEltsOffsetVector(NumEltsToLoad, ValueEltSz, LdI);
+  Value *EltsOffset = FormEltsOffsetVector(NumEltsToLoad, RealTyToLoadSz, LdI);
 
-  unsigned NumBlocks = m_DL->getTypeSizeInBits(LdEltTy) / genx::ByteBits;
-  // This logic is aligned with the on in CisaBuilder and GenXLowering
-  // The reason behind check for == 2 is that svm intrinsics don't support
-  // BlockSize of 2, so for ops with i16s we have to use BlockSize == 1 and NumBlocks == 2
-  Value *logNumBlocks = ConstantInt::get(I32Ty, genx::log2(NumBlocks == 2 ? NumBlocks : 1));
+  unsigned SrcSize = genx::log2(RealTyToLoadSz);
+  Value *logNumBlocks = ConstantInt::get(I32Ty, m_useGlobalMem ? 0 : SrcSize);
   Value *Scale = ConstantInt::get(Type::getInt16Ty(*m_ctx), 0);
   Value *Surface = ConstantInt::get(I32Ty,
                                     visa::getReservedSurfaceIndex(m_stack));
@@ -592,10 +601,6 @@ bool GenXThreadPrivateMemory::replaceLoad(LoadInst *LdI) {
     ProperGather = LdVal;
   }
 
-  Gather->setMetadata(InstMD::SVMBlockType,
-                      MDNode::get(*m_ctx, llvm::ValueAsMetadata::get(
-                                              UndefValue::get(LdEltTy))));
-
   LLVM_DEBUG(dbgs() << *Gather << "\n");
   LdI->replaceAllUsesWith(ProperGather);
   LdI->eraseFromParent();
@@ -642,9 +647,7 @@ bool GenXThreadPrivateMemory::replaceStore(StoreInst *StI) {
       {Pred->getType(),
        (m_useGlobalMem ? Offset : EltsOffset)->getType(),
        ValueOp->getType()});
-  unsigned NumBlocks = m_DL->getTypeSizeInBits(ValueOpTy->getScalarType()) / genx::ByteBits;
-  // see the comment in replaceLoad above
-  Value *logNumBlocks = ConstantInt::get(I32Ty, genx::log2(NumBlocks == 2 ? NumBlocks : 1));
+  Value *logNumBlocks = ConstantInt::get(I32Ty, m_useGlobalMem ? 0 : genx::log2(ValueEltSz));
   Value *Scale = ConstantInt::get(Type::getInt16Ty(*m_ctx), 0);
   Value *Surface = ConstantInt::get(I32Ty,
                                     visa::getReservedSurfaceIndex(m_stack));
@@ -659,11 +662,6 @@ bool GenXThreadPrivateMemory::replaceStore(StoreInst *StI) {
   Scatter->insertAfter(StI);
   StI->eraseFromParent();
 
-  Scatter->setMetadata(
-      InstMD::SVMBlockType,
-      MDNode::get(*m_ctx, llvm::ValueAsMetadata::get(
-                              UndefValue::get(ValueOpTy->getScalarType()))));
-
   LLVM_DEBUG(dbgs() << *Scatter << "\n");
   m_scatter.push_back(Scatter);
 
@@ -1096,12 +1094,6 @@ void SplitScatter(CallInst *CI) {
   }
   IGC_ASSERT(FirstScatter && SecondScatter);
 
-  auto *MD = CI->getMetadata(InstMD::SVMBlockType);
-  if (MD) {
-    FirstScatter->setMetadata(InstMD::SVMBlockType, MD);
-    SecondScatter->setMetadata(InstMD::SVMBlockType, MD);
-  }
-
   FirstScatter->insertAfter(CI);
   SecondScatter->insertAfter(FirstScatter);
 
@@ -1171,12 +1163,6 @@ void SplitGather(CallInst *CI) {
   }
   IGC_ASSERT(FirstGather && SecondGather);
 
-  auto *MD = CI->getMetadata(InstMD::SVMBlockType);
-  if (MD) {
-    FirstGather->setMetadata(InstMD::SVMBlockType, MD);
-    SecondGather->setMetadata(InstMD::SVMBlockType, MD);
-  }
-
   FirstGather->insertAfter(CI);
   SecondGather->insertAfter(FirstGather);
 
@@ -1294,16 +1280,14 @@ bool GenXThreadPrivateMemory::runOnModule(Module &M) {
   m_ST = &getAnalysis<TargetPassConfig>()
               .getTM<GenXTargetMachine>()
               .getGenXSubtarget();
-  if (!m_ST->isOCLRuntime())
-    m_useGlobalMem = false;
   for (auto &F : M)
     visit(F);
-  if (m_useGlobalMem ||
-      (m_ST->isOCLRuntime() && std::find_if(m_alloca.begin(), m_alloca.end(),
-                                            SVMChecker()) != m_alloca.end())) {
+  if (!m_useGlobalMem &&
+      std::find_if(m_alloca.begin(), m_alloca.end(), SVMChecker()) !=
+          m_alloca.end()) {
     LLVM_DEBUG(dbgs() << "Switching TPM to SVM\n");
     // TODO: move the name string to vc-intrinsics *MD::useGlobalMem
-    M.addModuleFlag(Module::ModFlagBehavior::Error, ModuleMD::UseSVMStack, 1);
+    M.addModuleFlag(Module::ModFlagBehavior::Error, "genx.useGlobalMem", 1);
     m_useGlobalMem = true;
   }
   bool Result = false;