|
35 | 35 | #include "llvm/IR/MatrixBuilder.h" |
36 | 36 | #include "llvm/Support/ConvertUTF.h" |
37 | 37 | #include "llvm/Support/ScopedPrinter.h" |
| 38 | +#include "llvm/TargetParser/AArch64TargetParser.h" |
| 39 | +#include "llvm/TargetParser/X86TargetParser.h" |
| 40 | +#include <algorithm> |
38 | 41 | #include <optional> |
39 | 42 | #include <utility> |
40 | 43 |
|
@@ -2592,6 +2595,205 @@ static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, |
2592 | 2595 | return RValue::get(CGF->Builder.CreateCall(UBF, Args)); |
2593 | 2596 | } |
2594 | 2597 |
|
| 2598 | +template <class T> |
| 2599 | +void RecursivelyClearPaddingImpl(CodeGenFunction &CGF, Value *Ptr, QualType Ty, |
| 2600 | + size_t CurrentStartOffset, |
| 2601 | + size_t &RunningOffset, T &&WriteZeroAtOffset, |
| 2602 | + bool VisitVirtualBase); |
| 2603 | + |
| 2604 | +template <class T> |
| 2605 | +void ClearPaddingStruct(CodeGenFunction &CGF, Value *Ptr, QualType Ty, |
| 2606 | + StructType *ST, size_t CurrentStartOffset, |
| 2607 | + size_t &RunningOffset, T &&WriteZeroAtOffset, |
| 2608 | + bool VisitVirtualBase) { |
| 2609 | + llvm::dbgs() << "clear padding struct: " << ST->getName().data() << '\n'; |
| 2610 | + const auto &DL = CGF.CGM.getModule().getDataLayout(); |
| 2611 | + auto *SL = DL.getStructLayout(ST); |
| 2612 | + auto *R = dyn_cast<CXXRecordDecl>(Ty->getAsRecordDecl()); |
| 2613 | + if (!R) { |
| 2614 | + llvm::dbgs() << "Not a CXXRecordDecl\n"; |
| 2615 | + return; |
| 2616 | + } |
| 2617 | + const ASTRecordLayout &ASTLayout = CGF.getContext().getASTRecordLayout(R); |
| 2618 | + if (ASTLayout.hasOwnVFPtr()) { |
| 2619 | + llvm::dbgs() << "vtable ptr. Incrementing RunningOffset from " |
| 2620 | + << RunningOffset << " to " |
| 2621 | + << RunningOffset + DL.getPointerSizeInBits() / 8 << '\n'; |
| 2622 | + RunningOffset += DL.getPointerSizeInBits() / 8; |
| 2623 | + } |
| 2624 | + std::vector<std::pair<size_t, CXXBaseSpecifier>> Bases; |
| 2625 | + Bases.reserve(R->getNumBases()); |
| 2626 | + // todo get vbases |
| 2627 | + for (auto Base : R->bases()) { |
| 2628 | + auto *BaseRecord = cast<CXXRecordDecl>(Base.getType()->getAsRecordDecl()); |
| 2629 | + if (!Base.isVirtual()) { |
| 2630 | + auto Offset = static_cast<size_t>( |
| 2631 | + ASTLayout.getBaseClassOffset(BaseRecord).getQuantity()); |
| 2632 | + Bases.emplace_back(Offset, Base); |
| 2633 | + } |
| 2634 | + } |
| 2635 | + |
| 2636 | + auto VisitBases = |
| 2637 | + [&](std::vector<std::pair<size_t, CXXBaseSpecifier>> &BasesToVisit) { |
| 2638 | + std::sort( |
| 2639 | + BasesToVisit.begin(), BasesToVisit.end(), |
| 2640 | + [](const auto &P1, const auto &P2) { return P1.first < P2.first; }); |
| 2641 | + for (const auto &Pair : BasesToVisit) { |
| 2642 | + // is it OK to use structured binding in clang? what is the language |
| 2643 | + // version? |
| 2644 | + auto Offset = Pair.first; |
| 2645 | + auto Base = Pair.second; |
| 2646 | + |
| 2647 | + llvm::dbgs() << "visiting base at offset " << Offset << '\n'; |
| 2648 | + // Recursively zero out base classes. |
| 2649 | + auto Index = SL->getElementContainingOffset(Offset); |
| 2650 | + Value *Idx = CGF.Builder.getSize(Index); |
| 2651 | + llvm::Type *CurrentBaseType = CGF.ConvertTypeForMem(Base.getType()); |
| 2652 | + Value *BaseElement = CGF.Builder.CreateGEP(CurrentBaseType, Ptr, Idx); |
| 2653 | + RecursivelyClearPaddingImpl(CGF, BaseElement, Base.getType(), |
| 2654 | + CurrentStartOffset + Offset, |
| 2655 | + RunningOffset, WriteZeroAtOffset, false); |
| 2656 | + } |
| 2657 | + }; |
| 2658 | + |
| 2659 | + VisitBases(Bases); |
| 2660 | + |
| 2661 | + size_t NumFields = std::distance(R->field_begin(), R->field_end()); |
| 2662 | + std::vector<size_t> FieldOffsets; |
| 2663 | + FieldOffsets.reserve(NumFields); |
| 2664 | + auto CurrentField = R->field_begin(); |
| 2665 | + for (size_t I = 0; I < NumFields; ++I, ++CurrentField) { |
| 2666 | + // Size needs to be in bytes so we can compare it later. |
| 2667 | + auto Offset = ASTLayout.getFieldOffset(I) / 8; |
| 2668 | + llvm::dbgs() << "visiting field at offset " << Offset << '\n'; |
| 2669 | + auto Index = SL->getElementContainingOffset(Offset); |
| 2670 | + Value *Idx = CGF.Builder.getSize(Index); |
| 2671 | + llvm::Type *CurrentFieldType = |
| 2672 | + CGF.ConvertTypeForMem(CurrentField->getType()); |
| 2673 | + Value *Element = CGF.Builder.CreateGEP(CurrentFieldType, Ptr, Idx); |
| 2674 | + RecursivelyClearPaddingImpl(CGF, Element, CurrentField->getType(), |
| 2675 | + CurrentStartOffset + Offset, RunningOffset, |
| 2676 | + WriteZeroAtOffset, true); |
| 2677 | + } |
| 2678 | + |
| 2679 | + if (VisitVirtualBase) { |
| 2680 | + |
| 2681 | + std::vector<std::pair<size_t, CXXBaseSpecifier>> VBases; |
| 2682 | + VBases.reserve(R->getNumVBases()); |
| 2683 | + for (auto VBase : R->vbases()) { |
| 2684 | + auto *BaseRecord = |
| 2685 | + cast<CXXRecordDecl>(VBase.getType()->getAsRecordDecl()); |
| 2686 | + auto Offset = static_cast<size_t>( |
| 2687 | + ASTLayout.getVBaseClassOffset(BaseRecord).getQuantity()); |
| 2688 | + VBases.emplace_back(Offset, VBase); |
| 2689 | + } |
| 2690 | + |
| 2691 | + VisitBases(VBases); |
| 2692 | + } |
| 2693 | +} |
| 2694 | + |
| 2695 | +template <class T> |
| 2696 | +void ClearPaddingConstantArray(CodeGenFunction &CGF, Value *Ptr, |
| 2697 | + llvm::Type *Type, ConstantArrayType const *AT, |
| 2698 | + size_t CurrentStartOffset, size_t &RunningOffset, |
| 2699 | + T &&WriteZeroAtOffset) { |
| 2700 | + llvm::dbgs() << "clear padding constant array\n"; |
| 2701 | + for (size_t ArrIndex = 0; ArrIndex < AT->getSize().getLimitedValue(); |
| 2702 | + ++ArrIndex) { |
| 2703 | + |
| 2704 | + QualType ElementQualType = AT->getElementType(); |
| 2705 | + |
| 2706 | + auto *ElementRecord = ElementQualType->getAsRecordDecl(); |
| 2707 | + if (!ElementRecord) { |
| 2708 | + llvm::dbgs() << "null!\n"; |
| 2709 | + } |
| 2710 | + auto ElementAlign = |
| 2711 | + ElementRecord |
| 2712 | + ? CGF.getContext().getASTRecordLayout(ElementRecord).getAlignment() |
| 2713 | + : CGF.getContext().getTypeAlignInChars(ElementQualType); |
| 2714 | + |
| 2715 | + Address FieldElementAddr{Ptr, Type, ElementAlign}; |
| 2716 | + |
| 2717 | + auto Element = CGF.Builder.CreateConstArrayGEP(FieldElementAddr, ArrIndex); |
| 2718 | + auto *ElementType = CGF.ConvertTypeForMem(ElementQualType); |
| 2719 | + auto AllocSize = |
| 2720 | + CGF.CGM.getModule().getDataLayout().getTypeAllocSize(ElementType); |
| 2721 | + llvm::dbgs() << "clearing array index! " << ArrIndex << '\n'; |
| 2722 | + RecursivelyClearPaddingImpl(CGF, Element.getBasePointer(), ElementQualType, |
| 2723 | + CurrentStartOffset + |
| 2724 | + ArrIndex * AllocSize.getKnownMinValue(), |
| 2725 | + RunningOffset, WriteZeroAtOffset, true); |
| 2726 | + } |
| 2727 | +} |
| 2728 | + |
| 2729 | +template <class T> |
| 2730 | +void RecursivelyClearPaddingImpl(CodeGenFunction &CGF, Value *Ptr, QualType Ty, |
| 2731 | + size_t CurrentStartOffset, |
| 2732 | + size_t &RunningOffset, T &&WriteZeroAtOffset, |
| 2733 | + bool VisitVirtualBase) { |
| 2734 | + |
| 2735 | + llvm::dbgs() << "clear padding before current [" << RunningOffset << ", " |
| 2736 | + << CurrentStartOffset << ")\n"; |
| 2737 | + for (; RunningOffset < CurrentStartOffset; ++RunningOffset) { |
| 2738 | + WriteZeroAtOffset(RunningOffset); |
| 2739 | + } |
| 2740 | + auto *Type = CGF.ConvertTypeForMem(Ty); |
| 2741 | + auto Size = CGF.CGM.getModule() |
| 2742 | + .getDataLayout() |
| 2743 | + .getTypeSizeInBits(Type) |
| 2744 | + .getKnownMinValue() / |
| 2745 | + 8; |
| 2746 | + |
| 2747 | + if (auto *AT = dyn_cast<ConstantArrayType>(Ty)) { |
| 2748 | + ClearPaddingConstantArray(CGF, Ptr, Type, AT, CurrentStartOffset, |
| 2749 | + RunningOffset, WriteZeroAtOffset); |
| 2750 | + } else if (auto *ST = dyn_cast<StructType>(Type); ST && Ty->isRecordType()) { |
| 2751 | + ClearPaddingStruct(CGF, Ptr, Ty, ST, CurrentStartOffset, RunningOffset, |
| 2752 | + WriteZeroAtOffset, VisitVirtualBase); |
| 2753 | + } else if (Ty->isAtomicType()) { |
| 2754 | + RecursivelyClearPaddingImpl(CGF, Ptr, Ty.getAtomicUnqualifiedType(), |
| 2755 | + CurrentStartOffset, RunningOffset, |
| 2756 | + WriteZeroAtOffset, true); |
| 2757 | + } else { |
| 2758 | + llvm::dbgs() << "increment running offset from: " << RunningOffset << " to " |
| 2759 | + << RunningOffset + Size << '\n'; |
| 2760 | + RunningOffset = |
| 2761 | + std::max(RunningOffset, CurrentStartOffset + static_cast<size_t>(Size)); |
| 2762 | + } |
| 2763 | +} |
| 2764 | + |
| 2765 | +static void RecursivelyClearPadding(CodeGenFunction &CGF, Value *Ptr, |
| 2766 | + QualType Ty) { |
| 2767 | + auto *I8Ptr = CGF.Builder.CreateBitCast(Ptr, CGF.Int8PtrTy); |
| 2768 | + auto *Zero = ConstantInt::get(CGF.Int8Ty, 0); |
| 2769 | + auto WriteZeroAtOffset = [&](uint64_t Offset) { |
| 2770 | + auto *Index = ConstantInt::get(CGF.IntTy, Offset); |
| 2771 | + auto *Element = CGF.Builder.CreateGEP(CGF.Int8Ty, I8Ptr, Index); |
| 2772 | + CGF.Builder.CreateAlignedStore( |
| 2773 | + Zero, Element, |
| 2774 | + CharUnits::One().alignmentAtOffset(CharUnits::fromQuantity(Offset))); |
| 2775 | + }; |
| 2776 | + |
| 2777 | + size_t RunningOffset = 0; |
| 2778 | + |
| 2779 | + RecursivelyClearPaddingImpl(CGF, Ptr, Ty, 0, RunningOffset, WriteZeroAtOffset, |
| 2780 | + true); |
| 2781 | + |
| 2782 | + // Clear tail padding |
| 2783 | + auto *Type = CGF.ConvertTypeForMem(Ty); |
| 2784 | + |
| 2785 | + auto Size = CGF.CGM.getModule() |
| 2786 | + .getDataLayout() |
| 2787 | + .getTypeAllocSize(Type) |
| 2788 | + .getKnownMinValue(); |
| 2789 | + |
| 2790 | + llvm::dbgs() << "clear tail padding [" << RunningOffset << ", " << Size |
| 2791 | + << ")\n"; |
| 2792 | + for (; RunningOffset < Size; ++RunningOffset) { |
| 2793 | + WriteZeroAtOffset(RunningOffset); |
| 2794 | + } |
| 2795 | +} |
| 2796 | + |
2595 | 2797 | RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, |
2596 | 2798 | const CallExpr *E, |
2597 | 2799 | ReturnValueSlot ReturnValue) { |
@@ -4839,6 +5041,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, |
4839 | 5041 |
|
4840 | 5042 | return RValue::get(Ptr); |
4841 | 5043 | } |
| 5044 | + case Builtin::BI__builtin_clear_padding: { |
| 5045 | + const Expr *Op = E->getArg(0); |
| 5046 | + Value *Address = EmitScalarExpr(Op); |
| 5047 | + auto PointeeTy = Op->getType()->getPointeeType(); |
| 5048 | + RecursivelyClearPadding(*this, Address, PointeeTy); |
| 5049 | + return RValue::get(nullptr); |
| 5050 | + } |
4842 | 5051 | case Builtin::BI__sync_fetch_and_add: |
4843 | 5052 | case Builtin::BI__sync_fetch_and_sub: |
4844 | 5053 | case Builtin::BI__sync_fetch_and_or: |
|
0 commit comments