[SHT_LLVM_BB_ADDR_MAP] Emit callsite offsets in the SHT_LLVM_BB_ADDR_MAP section.

llvm/docs/Extensions.rst

@@ -405,31 +405,72 @@ This section is emitted with ``-basic-block-address-map`` and will contain
 a BB address map table for every function.
 
 The ``SHT_LLVM_BB_ADDR_MAP`` type provides backward compatibility to allow
-reading older versions of the BB address map generated by older compilers. Each
-function entry starts with a version byte which specifies the encoding version
-to use. The following versioning schemes are currently supported.
+reading older versions of the BB address map generated by older compilers (up to
+two years old). Each function entry starts with a version byte which specifies
+the encoding version to use. This follows by a feature byte which specifies the
+features specific to this particular entry. The function base address is stored
+as a full address. Other addresses in the entry (block begin and end addresses
+and callsite addresses) are stored in a running-offset fashion, as offset
+relative to the prior address.
 
-Version 1 (newest): basic block address offsets are computed relative to the end
-of previous blocks.
+The following versioning schemes are currently supported (newer versions support
+features of the older versions).
+
+Version 3 (newest): Capable of encoding callsite offsets. Enabled by the 6th bit
+of the feature byte.
 
 Example:
 
 .. code-block:: gas
 
   .section  ".llvm_bb_addr_map","",@llvm_bb_addr_map
-  .byte     1                             # version number
-  .byte     0                             # feature byte (reserved for future use)
+  .byte     3                             # version number
+  .byte     32                            # feature byte
   .quad     .Lfunc_begin0                 # address of the function
   .byte     2                             # number of basic blocks
   # BB record for BB_0
-   .uleb128  .Lfunc_beign0-.Lfunc_begin0  # BB_0 offset relative to function entry (always zero)
+   .byte     0                            # BB_0 ID
+   .uleb128  .Lfunc_begin0-.Lfunc_begin0  # BB_0 offset relative to function entry (always zero)
+   .byte     0                            # number of callsites in this block
    .uleb128  .LBB_END0_0-.Lfunc_begin0    # BB_0 size
    .byte     x                            # BB_0 metadata
   # BB record for BB_1
+   .byte     1                            # BB_1 ID
    .uleb128  .LBB0_1-.LBB_END0_0          # BB_1 offset relative to the end of last block (BB_0).
-   .uleb128  .LBB_END0_1-.LBB0_1          # BB_1 size
+   .byte     2                            # number of callsites in this block
+   .uleb128  .LBB0_1_CS0-.LBB0_1          # offset of callsite relative to the previous offset (.LBB0_1)
+   .uleb128  .LBB0_1_CS1-.LBB0_1_CS0      # offset of callsite relative to the previous offset (.LBB0_1_CS0)
+   .uleb128  .LBB_END0_1-.LBB0_1_CS1      # BB_1 size offset (Offset of the block end relative to the previous offset).
    .byte     y                            # BB_1 metadata
 
+Version 2: Capable of encoding split functions. Enabled by the 4th bit of the
+feature byte. The base address of each split range is stored as a full address.
+The first range corresponds to the function entry.
+
+Example:
+
+.. code-block:: gas
+  .section  ".llvm_bb_addr_map","",@llvm_bb_addr_map
+  .byte     2                             # version number
+  .byte     8                             # feature byte
+  .byte     2                             # number of basic block ranges
+  # 1st BB range (corresponding to the function entry)
+   .quad     .Lfunc_begin0                 # base address
+   .byte     1                             # number of basic blocks in this range
+    # BB record for BB_0
+    .byte     0                            # BB_0 ID
+    .uleb128  .Lfunc_begin0-.Lfunc_begin0  # BB_0 offset relative to function entry (always zero)
+    .uleb128  .LBB_END0_0-.Lfunc_begin0    # BB_0 size
+    .byte     x                            # BB_0 metadata
+  # 2nd BB range
+   .quad     func.part.1
+   .byte     1                             # number of basic blocks in this range
+    # BB record for BB_1
+    .byte     1                            # BB_1 ID
+    .uleb128  func.part.1-func.part.1      # BB_1 offset relative to the range begin (always zero)
+    .uleb128  .LBB_END0_1-func.part.1      # BB_1 size
+    .byte     1                            # BB_1 metadata
+
 PGO Analysis Map
 """"""""""""""""
 

llvm/include/llvm/CodeGen/AsmPrinter.h

@@ -173,6 +173,8 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass {
   // function. This is used to calculate the size of the BB section.
   MCSymbol *CurrentSectionBeginSym = nullptr;
 
+  bool HasAnyCallsitesForBBAddrMap = false;
+
   /// This map keeps track of which symbol is being used for the specified basic
   /// block's address of label.
   std::unique_ptr<AddrLabelMap> AddrLabelSymbols;
@@ -426,7 +428,7 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass {
 
   void emitStackUsage(const MachineFunction &MF);
 
-  void emitBBAddrMapSection(const MachineFunction &MF);
+  void emitBBAddrMapSection(const MachineFunction &MF, bool HasCalls);
 
   void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol);
   virtual void emitKCFITypeId(const MachineFunction &MF);

llvm/include/llvm/CodeGen/MachineBasicBlock.h

@@ -242,6 +242,10 @@ class MachineBasicBlock
   /// calculate the size of the basic block, or the BB section ending with it.
   mutable MCSymbol *CachedEndMCSymbol = nullptr;
 
+  /// Vector of symbols marking the position of callsites in the basic
+  /// block, stored in the order they appear in the basic block.
+  mutable SmallVector<MCSymbol *, 1> CallsiteSymbols;
+
   // Intrusive list support
   MachineBasicBlock() = default;
 
@@ -325,6 +329,12 @@ class MachineBasicBlock
   /// its label be emitted.
   void setLabelMustBeEmitted() { LabelMustBeEmitted = true; }
 
+  /// Returns the symbols marking callsites in the order they appear in the
+  /// basic block.
+  const SmallVectorImpl<MCSymbol *> &getCallsiteSymbols() const {
+    return CallsiteSymbols;
+  }
+
   /// Return the MachineFunction containing this basic block.
   const MachineFunction *getParent() const { return xParent; }
   MachineFunction *getParent() { return xParent; }
@@ -725,6 +735,10 @@ class MachineBasicBlock
   /// Returns the MCSymbol marking the end of this basic block.
   LLVM_ABI MCSymbol *getEndSymbol() const;
 
+  /// Returns a temporary MCSymbol marking the beginning of a callsite, and
+  /// appends it to `CallsiteSymbols`.
+  MCSymbol *createCallsiteSymbol() const;
+
   /// Returns true if this block may have an INLINEASM_BR (overestimate, by
   /// checking if any of the successors are indirect targets of any inlineasm_br
   /// in the function).

llvm/include/llvm/MC/MCContext.h

@@ -175,8 +175,8 @@ class MCContext {
   /// for the LocalLabelVal and adds it to the map if needed.
   unsigned GetInstance(unsigned LocalLabelVal);
 
-  /// LLVM_BB_ADDR_MAP version to emit.
-  uint8_t BBAddrMapVersion = 2;
+  /// SHT_LLVM_BB_ADDR_MAP version to emit.
+  uint8_t BBAddrMapVersion = 3;
 
   /// The file name of the log file from the environment variable
   /// AS_SECURE_LOG_FILE.  Which must be set before the .secure_log_unique

llvm/include/llvm/Object/ELFTypes.h

@@ -917,8 +917,8 @@ struct BBAddrMap {
     uint32_t Size = 0;   // Size of the basic block.
     Metadata MD = {false, false, false, false,
                    false}; // Metdata for this basic block.
-    // Offsets of callsites (end of call instructions), relative to the basic
-    // block start.
+    // Offsets of callsites (beginning of call instructions), relative to the
+    // basic block start.
     SmallVector<uint32_t, 1> CallsiteOffsets;
 
     BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD,

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -1391,7 +1391,8 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
 }
 
 static llvm::object::BBAddrMap::Features
-getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
+getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
+                    bool HasCalls) {
   // Ensure that the user has not passed in additional options while also
   // specifying all or none.
   if ((PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None) ||
@@ -1424,10 +1425,11 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
           BrProbEnabled,
           MF.hasBBSections() && NumMBBSectionRanges > 1,
           static_cast<bool>(BBAddrMapSkipEmitBBEntries),
-          false};
+          HasCalls};
 }
 
-void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
+void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF,
+                                      bool HasCalls) {
   MCSection *BBAddrMapSection =
       getObjFileLowering().getBBAddrMapSection(*MF.getSection());
   assert(BBAddrMapSection && ".llvm_bb_addr_map section is not initialized.");
@@ -1440,7 +1442,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
   uint8_t BBAddrMapVersion = OutStreamer->getContext().getBBAddrMapVersion();
   OutStreamer->emitInt8(BBAddrMapVersion);
   OutStreamer->AddComment("feature");
-  auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size());
+  auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls);
   OutStreamer->emitInt8(Features.encode());
   // Emit BB Information for each basic block in the function.
   if (Features.MultiBBRange) {
@@ -1493,13 +1495,24 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
       // Emit the basic block offset relative to the end of the previous block.
       // This is zero unless the block is padded due to alignment.
       emitLabelDifferenceAsULEB128(MBBSymbol, PrevMBBEndSymbol);
-      // Emit the basic block size. When BBs have alignments, their size cannot
-      // always be computed from their offsets.
-      emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), MBBSymbol);
+      const MCSymbol *CurrentLabel = MBBSymbol;
+      if (HasCalls) {
+        const SmallVectorImpl<MCSymbol *> &CallsiteSymbols =
+            MBB.getCallsiteSymbols();
+        OutStreamer->AddComment("number of callsites");
+        OutStreamer->emitULEB128IntValue(CallsiteSymbols.size());
+        for (const MCSymbol *CallsiteSymbol : CallsiteSymbols) {
+          // Emit the callsite offset.
+          emitLabelDifferenceAsULEB128(CallsiteSymbol, CurrentLabel);
+          CurrentLabel = CallsiteSymbol;
+        }
+      }
+      // Emit the offset to the end of the block, which can be used to compute
+      // the total block size.
+      emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), CurrentLabel);
       // Emit the Metadata.
       OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB));
     }
-
     PrevMBBEndSymbol = MBB.getEndSymbol();
   }
 
@@ -1802,6 +1815,7 @@ void AsmPrinter::emitFunctionBody() {
 
   // Print out code for the function.
   bool HasAnyRealCode = false;
+  bool HasCalls = false;
   int NumInstsInFunction = 0;
   bool IsEHa = MMI->getModule()->getModuleFlag("eh-asynch");
 
@@ -1828,6 +1842,11 @@ void AsmPrinter::emitFunctionBody() {
           !MI.isDebugInstr()) {
         HasAnyRealCode = true;
       }
+      if (MI.isCall()) {
+        HasCalls = true;
+        if (MF->getTarget().Options.BBAddrMap)
+          OutStreamer->emitLabel(MBB.createCallsiteSymbol());
+      }
 
       // If there is a pre-instruction symbol, emit a label for it here.
       if (MCSymbol *S = MI.getPreInstrSymbol())
@@ -2114,7 +2133,7 @@ void AsmPrinter::emitFunctionBody() {
   // BB labels are requested for this function. Skip empty functions.
   if (HasAnyRealCode) {
     if (MF->getTarget().Options.BBAddrMap)
-      emitBBAddrMapSection(*MF);
+      emitBBAddrMapSection(*MF, HasCalls);
     else if (PgoAnalysisMapFeatures.getBits() != 0)
       MF->getContext().reportWarning(
           SMLoc(), "pgo-analysis-map is enabled for function " + MF->getName() +

llvm/lib/CodeGen/MachineBasicBlock.cpp

@@ -112,6 +112,15 @@ MCSymbol *MachineBasicBlock::getEndSymbol() const {
   return CachedEndMCSymbol;
 }
 
+MCSymbol *MachineBasicBlock::createCallsiteSymbol() const {
+  const MachineFunction *MF = getParent();
+  MCContext &Ctx = MF->getContext();
+  CallsiteSymbols.push_back(
+      Ctx.createTempSymbol("BB" + Twine(MF->getFunctionNumber()) + "_" +
+                           Twine(getNumber()) + "_CS"));
+  return CallsiteSymbols.back();
+}
+
 raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineBasicBlock &MBB) {
   MBB.print(OS);
   return OS;

llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll

@@ -19,7 +19,7 @@ entry:
 ; CHECK:	func:
 ; CHECK:	.Lfunc_begin1:
 ; CHECK:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text{{$}}
-; CHECK-NEXT:		.byte 2			# version
+; CHECK-NEXT:		.byte 3			# version
 ; BASIC-NEXT:		.byte 0			# feature
 ; PGO-NEXT:		.byte 3			# feature
 ; CHECK-NEXT:		.quad	.Lfunc_begin1	# function address

llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll

@@ -10,7 +10,7 @@ define dso_local i32 @_Z3barv() {
 ; CHECK-LABEL:	_Z3barv:
 ; CHECK-NEXT:	[[BAR_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3barv{{$}}
-; CHECK-NEXT:		.byte 2			# version
+; CHECK-NEXT:		.byte 3			# version
 ; CHECK-NEXT:		.byte 0			# feature
 ; CHECK-NEXT:		.quad [[BAR_BEGIN]]	# function address
 
@@ -23,8 +23,8 @@ define dso_local i32 @_Z3foov() {
 ; CHECK-LABEL:	_Z3foov:
 ; CHECK-NEXT:	[[FOO_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section  .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3foov{{$}}
-; CHECK-NEXT:		.byte 2			# version
-; CHECK-NEXT:		.byte 0			# feature
+; CHECK-NEXT:		.byte 3			# version
+; CHECK-NEXT:		.byte 32                # feature
 ; CHECK-NEXT:		.quad [[FOO_BEGIN]]	# function address
 
 
@@ -36,6 +36,6 @@ define linkonce_odr dso_local i32 @_Z4fooTIiET_v() comdat {
 ; CHECK-LABEL:	_Z4fooTIiET_v:
 ; CHECK-NEXT:	[[FOOCOMDAT_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section .llvm_bb_addr_map,"oG",@llvm_bb_addr_map,.text._Z4fooTIiET_v,_Z4fooTIiET_v,comdat{{$}}
-; CHECK-NEXT:		.byte 2				# version
+; CHECK-NEXT:		.byte 3				# version
 ; CHECK-NEXT:		.byte 0				# feature
 ; CHECK-NEXT:		.quad [[FOOCOMDAT_BEGIN]]	# function address

llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll

@@ -69,36 +69,44 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-LABEL:	.Lfunc_end0:
 
 ; CHECK: 	.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}}
-; CHECK-NEXT:	.byte	2		# version
-; BASIC-NEXT:	.byte	0		# feature
-; PGO-ALL-NEXT:	.byte	7		# feature
-; FEC-ONLY-NEXT:.byte	1		# feature
-; BBF-ONLY-NEXT:.byte	2		# feature
-; BRP-ONLY-NEXT:.byte	4		# feature
+; CHECK-NEXT:	.byte	3		# version
+; BASIC-NEXT:	.byte	32		# feature
+; PGO-ALL-NEXT:	.byte	39		# feature
+; FEC-ONLY-NEXT:.byte	33		# feature
+; BBF-ONLY-NEXT:.byte	34		# feature
+; BRP-ONLY-NEXT:.byte	36		# feature
 ; CHECK-NEXT:	.quad	.Lfunc_begin0	# function address
 ; CHECK-NEXT:	.byte	6		# number of basic blocks
 ; CHECK-NEXT:	.byte	0		# BB id
 ; CHECK-NEXT:	.uleb128 .Lfunc_begin0-.Lfunc_begin0
+; CHECK-NEXT:   .byte   0               # number of callsites
 ; CHECK-NEXT:	.uleb128 .LBB_END0_0-.Lfunc_begin0
 ; CHECK-NEXT:	.byte	8
 ; CHECK-NEXT:	.byte	1		# BB id
 ; CHECK-NEXT:	.uleb128 .LBB0_1-.LBB_END0_0
-; CHECK-NEXT:	.uleb128 .LBB_END0_1-.LBB0_1
+; CHECK-NEXT:   .byte   1               # number of callsites
+; CHECK-NEXT:	.uleb128 .LBB0_1_CS0-.LBB0_1
+; CHECK-NEXT:	.uleb128 .LBB_END0_1-.LBB0_1_CS0
 ; CHECK-NEXT:	.byte	8
 ; CHECK-NEXT:	.byte	3		# BB id
 ; CHECK-NEXT:	.uleb128 .LBB0_2-.LBB_END0_1
-; CHECK-NEXT:	.uleb128 .LBB_END0_2-.LBB0_2
+; CHECK-NEXT:   .byte   1               # number of callsites
+; CHECK-NEXT:	.uleb128 .LBB0_2_CS0-.LBB0_2
+; CHECK-NEXT:	.uleb128 .LBB_END0_2-.LBB0_2_CS0
 ; CHECK-NEXT:	.byte	8
 ; CHECK-NEXT:	.byte	5		# BB id
 ; CHECK-NEXT:	.uleb128 .LBB0_3-.LBB_END0_2
+; CHECK-NEXT:   .byte   0               # number of callsites
 ; CHECK-NEXT:	.uleb128 .LBB_END0_3-.LBB0_3
 ; CHECK-NEXT:	.byte	1
 ; CHECK-NEXT:	.byte	4		# BB id
 ; CHECK-NEXT:	.uleb128 .LBB0_4-.LBB_END0_3
+; CHECK-NEXT:   .byte   0               # number of callsites
 ; CHECK-NEXT:	.uleb128 .LBB_END0_4-.LBB0_4
 ; CHECK-NEXT:	.byte	16
 ; CHECK-NEXT:	.byte	2		# BB id
 ; CHECK-NEXT:	.uleb128 .LBB0_5-.LBB_END0_4
+; CHECK-NEXT:   .byte   0               # number of callsites
 ; CHECK-NEXT:	.uleb128 .LBB_END0_5-.LBB0_5
 ; CHECK-NEXT:	.byte	4
 
@@ -138,7 +146,7 @@ declare i32 @__gxx_personality_v0(...)
 ; PGO-BRP-NEXT:	.byte	5		# successor BB ID
 ; PGO-BRP-NEXT:	.ascii	"\200\200\200\200\b"	# successor branch probability
 
-; SKIP-BB-ENTRIES:      .byte	17                              # feature
+; SKIP-BB-ENTRIES:      .byte	49                              # feature
 ; SKIP-BB-ENTRIES-NEXT:	.quad	.Lfunc_begin0                   # function address
 ; SKIP-BB-ENTRIES-NEXT:	.byte	6                               # number of basic blocks
 ; SKIP-BB-ENTRIES-NEXT:	.byte	100                             # function entry count