Add Cross Translation Unit support library

This patch introduces a class that can help to build tools that require cross
translation unit facilities. This class allows function definitions to be loaded
from external AST files based on an index. In order to use this functionality an
index is required. The index format is a flat text file but it might be
replaced with a different solution in the near future. USRs are used as names to
look up the functions definitions. This class also does caching to avoid
redundant loading of AST files.

Right now only function defnitions can be loaded using this API because this is
what the in progress cross translation unit feature of the Static Analyzer
requires. In to future this might be extended to classes, types etc.

Differential Revision: https://reviews.llvm.org/D34512


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@313975 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Basic/AllDiagnostics.h

@@ -18,6 +18,7 @@
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/AST/CommentDiagnostic.h"
 #include "clang/Analysis/AnalysisDiagnostic.h"
+#include "clang/CrossTU/CrossTUDiagnostic.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Lex/LexDiagnostic.h"

include/clang/Basic/CMakeLists.txt

@@ -9,6 +9,7 @@ clang_diag_gen(Analysis)
 clang_diag_gen(AST)
 clang_diag_gen(Comment)
 clang_diag_gen(Common)
+clang_diag_gen(CrossTU)
 clang_diag_gen(Driver)
 clang_diag_gen(Frontend)
 clang_diag_gen(Lex)

include/clang/Basic/Diagnostic.td

@@ -133,6 +133,7 @@ include "DiagnosticASTKinds.td"
 include "DiagnosticAnalysisKinds.td"
 include "DiagnosticCommentKinds.td"
 include "DiagnosticCommonKinds.td"
+include "DiagnosticCrossTUKinds.td"
 include "DiagnosticDriverKinds.td"
 include "DiagnosticFrontendKinds.td"
 include "DiagnosticLexKinds.td"

include/clang/Basic/DiagnosticCrossTUKinds.td

@@ -0,0 +1,18 @@
+//==--- DiagnosticCrossTUKinds.td - Cross Translation Unit diagnostics ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+let Component = "CrossTU" in {
+
+def err_fnmap_parsing : Error<
+  "error parsing index file: '%0' line: %1 'UniqueID filename' format "
+  "expected">;
+
+def err_multiple_def_index : Error<
+  "multiple definitions are found for the same key in index ">;
+}

include/clang/Basic/DiagnosticIDs.h

@@ -36,6 +36,7 @@ namespace clang {
       DIAG_SIZE_PARSE         =  500,
       DIAG_SIZE_AST           =  110,
       DIAG_SIZE_COMMENT       =  100,
+      DIAG_SIZE_CROSSTU       =  100,
       DIAG_SIZE_SEMA          = 3500,
       DIAG_SIZE_ANALYSIS      =  100
     };
@@ -49,7 +50,8 @@ namespace clang {
       DIAG_START_PARSE         = DIAG_START_LEX           + DIAG_SIZE_LEX,
       DIAG_START_AST           = DIAG_START_PARSE         + DIAG_SIZE_PARSE,
       DIAG_START_COMMENT       = DIAG_START_AST           + DIAG_SIZE_AST,
-      DIAG_START_SEMA          = DIAG_START_COMMENT       + DIAG_SIZE_COMMENT,
+      DIAG_START_CROSSTU       = DIAG_START_COMMENT       + DIAG_SIZE_CROSSTU,
+      DIAG_START_SEMA          = DIAG_START_CROSSTU       + DIAG_SIZE_COMMENT,
       DIAG_START_ANALYSIS      = DIAG_START_SEMA          + DIAG_SIZE_SEMA,
       DIAG_UPPER_LIMIT         = DIAG_START_ANALYSIS      + DIAG_SIZE_ANALYSIS
     };

include/clang/CrossTU/CrossTUDiagnostic.h

@@ -0,0 +1,29 @@
+//===--- CrossTUDiagnostic.h - Diagnostics for Cross TU ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_CROSSTU_CROSSTUDIAGNOSTIC_H
+#define LLVM_CLANG_CROSSTU_CROSSTUDIAGNOSTIC_H
+
+#include "clang/Basic/Diagnostic.h"
+
+namespace clang {
+namespace diag {
+enum {
+#define DIAG(ENUM, FLAGS, DEFAULT_MAPPING, DESC, GROUP, SFINAE, NOWERROR,      \
+             SHOWINSYSHEADER, CATEGORY)                                        \
+  ENUM,
+#define CROSSTUSTART
+#include "clang/Basic/DiagnosticCrossTUKinds.inc"
+#undef DIAG
+  NUM_BUILTIN_CROSSTU_DIAGNOSTICS
+};
+} // end namespace diag
+} // end namespace clang
+
+#endif // LLVM_CLANG_FRONTEND_FRONTENDDIAGNOSTIC_H

include/clang/CrossTU/CrossTranslationUnit.h

@@ -0,0 +1,159 @@
+//===--- CrossTranslationUnit.h - -------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file provides an interface to load binary AST dumps on demand. This
+//  feature can be utilized for tools that require cross translation unit
+//  support.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H
+#define LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H
+
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+class CompilerInstance;
+class ASTContext;
+class ASTImporter;
+class ASTUnit;
+class DeclContext;
+class FunctionDecl;
+class NamedDecl;
+class TranslationUnitDecl;
+
+namespace cross_tu {
+
+enum class index_error_code {
+  unspecified = 1,
+  missing_index_file,
+  invalid_index_format,
+  multiple_definitions,
+  missing_definition,
+  failed_import,
+  failed_to_get_external_ast,
+  failed_to_generate_usr
+};
+
+class IndexError : public llvm::ErrorInfo<IndexError> {
+public:
+  static char ID;
+  IndexError(index_error_code C) : Code(C), LineNo(0) {}
+  IndexError(index_error_code C, std::string FileName, int LineNo = 0)
+      : Code(C), FileName(std::move(FileName)), LineNo(LineNo) {}
+  void log(raw_ostream &OS) const override;
+  std::error_code convertToErrorCode() const override;
+  index_error_code getCode() const { return Code; }
+  int getLineNum() const { return LineNo; }
+  std::string getFileName() const { return FileName; }
+
+private:
+  index_error_code Code;
+  std::string FileName;
+  int LineNo;
+};
+
+/// \brief This function parses an index file that determines which
+///        translation unit contains which definition.
+///
+/// The index file format is the following:
+/// each line consists of an USR and a filepath separated by a space.
+///
+/// \return Returns a map where the USR is the key and the filepath is the value
+///         or an error.
+llvm::Expected<llvm::StringMap<std::string>>
+parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir);
+
+std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
+
+/// \brief This class is used for tools that requires cross translation
+///        unit capability.
+///
+/// This class can load function definitions from external AST files.
+/// The loaded definition will be merged back to the original AST using the
+/// AST Importer.
+/// In order to use this class, an index file is required that describes
+/// the locations of the AST files for each function definition.
+///
+/// Note that this class also implements caching.
+class CrossTranslationUnitContext {
+public:
+  CrossTranslationUnitContext(CompilerInstance &CI);
+  ~CrossTranslationUnitContext();
+
+  /// \brief This function loads a function definition from an external AST
+  ///        file and merge it into the original AST.
+  ///
+  /// This method should only be used on functions that have no definitions in
+  /// the current translation unit. A function definition with the same
+  /// declaration will be looked up in the index file which should be in the
+  /// \p CrossTUDir directory, called \p IndexName. In case the declaration is
+  /// found in the index the corresponding AST file will be loaded and the
+  /// definition of the function will be merged into the original AST using
+  /// the AST Importer.
+  ///
+  /// \return The declaration with the definition will be returned.
+  /// If no suitable definition is found in the index file or multiple
+  /// definitions found error will be returned.
+  ///
+  /// Note that the AST files should also be in the \p CrossTUDir.
+  llvm::Expected<const FunctionDecl *>
+  getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir,
+                       StringRef IndexName);
+
+  /// \brief This function loads a function definition from an external AST
+  ///        file.
+  ///
+  /// A function definition with the same declaration will be looked up in the
+  /// index file which should be in the \p CrossTUDir directory, called
+  /// \p IndexName. In case the declaration is found in the index the
+  /// corresponding AST file will be loaded.
+  ///
+  /// \return Returns an ASTUnit that contains the definition of the looked up
+  /// function.
+  ///
+  /// Note that the AST files should also be in the \p CrossTUDir.
+  llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName,
+                                            StringRef CrossTUDir,
+                                            StringRef IndexName);
+
+  /// \brief This function merges a definition from a separate AST Unit into
+  ///        the current one which was created by the compiler instance that
+  ///        was passed to the constructor.
+  ///
+  /// \return Returns the resulting definition or an error.
+  llvm::Expected<const FunctionDecl *> importDefinition(const FunctionDecl *FD);
+
+  /// \brief Get a name to identify a function.
+  static std::string getLookupName(const NamedDecl *ND);
+
+  /// \brief Emit diagnostics for the user for potential configuration errors.
+  void emitCrossTUDiagnostics(const IndexError &IE);
+
+private:
+  ASTImporter &getOrCreateASTImporter(ASTContext &From);
+  const FunctionDecl *findFunctionInDeclContext(const DeclContext *DC,
+                                                StringRef LookupFnName);
+
+  llvm::StringMap<std::unique_ptr<clang::ASTUnit>> FileASTUnitMap;
+  llvm::StringMap<clang::ASTUnit *> FunctionASTUnitMap;
+  llvm::StringMap<std::string> FunctionFileMap;
+  llvm::DenseMap<TranslationUnitDecl *, std::unique_ptr<ASTImporter>>
+      ASTUnitImporterMap;
+  CompilerInstance &CI;
+  ASTContext &Context;
+};
+
+} // namespace cross_tu
+} // namespace clang
+
+#endif // LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H

lib/AST/ASTImporter.cpp

@@ -1846,6 +1846,8 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   if (ToD)
     return ToD;
 
+  const FunctionDecl *FoundWithoutBody = nullptr;
+
   // Try to find a function in our own ("to") context with the same name, same
   // type, and in the same context as the function we're importing.
   if (!LexicalDC->isFunctionOrMethod()) {
@@ -1863,6 +1865,13 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
           if (Importer.IsStructurallyEquivalent(D->getType(), 
                                                 FoundFunction->getType())) {
             // FIXME: Actually try to merge the body and other attributes.
+            const FunctionDecl *FromBodyDecl = nullptr;
+            D->hasBody(FromBodyDecl);
+            if (D == FromBodyDecl && !FoundFunction->hasBody()) {
+              // This function is needed to merge completely.
+              FoundWithoutBody = FoundFunction;
+              break;
+            }
             return Importer.Imported(D, FoundFunction);
           }
 
@@ -2013,6 +2022,12 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   }
   ToFunction->setParams(Parameters);
 
+  if (FoundWithoutBody) {
+    auto *Recent = const_cast<FunctionDecl *>(
+          FoundWithoutBody->getMostRecentDecl());
+    ToFunction->setPreviousDecl(Recent);
+  }
+
   if (usedDifferentExceptionSpec) {
     // Update FunctionProtoType::ExtProtoInfo.
     QualType T = Importer.Import(D->getType());

lib/Basic/DiagnosticIDs.cpp

@@ -109,6 +109,7 @@ static const StaticDiagInfoRec StaticDiagInfo[] = {
 #include "clang/Basic/DiagnosticParseKinds.inc"
 #include "clang/Basic/DiagnosticASTKinds.inc"
 #include "clang/Basic/DiagnosticCommentKinds.inc"
+#include "clang/Basic/DiagnosticCrossTUKinds.inc"
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
 #undef DIAG
@@ -146,7 +147,8 @@ CATEGORY(LEX, SERIALIZATION)
 CATEGORY(PARSE, LEX)
 CATEGORY(AST, PARSE)
 CATEGORY(COMMENT, AST)
-CATEGORY(SEMA, COMMENT)
+CATEGORY(CROSSTU, COMMENT)
+CATEGORY(SEMA, CROSSTU)
 CATEGORY(ANALYSIS, SEMA)
 #undef CATEGORY
 

lib/CMakeLists.txt

@@ -4,6 +4,7 @@ add_subdirectory(Lex)
 add_subdirectory(Parse)
 add_subdirectory(AST)
 add_subdirectory(ASTMatchers)
+add_subdirectory(CrossTU)
 add_subdirectory(Sema)
 add_subdirectory(CodeGen)
 add_subdirectory(Analysis)

lib/CrossTU/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_clang_library(clangCrossTU
+  CrossTranslationUnit.cpp
+
+  LINK_LIBS
+  clangAST
+  clangBasic
+  clangFrontend
+  clangIndex
+  )