clang-xform: Clang Libtooling for large-scale c++ code refactoring

 On branch master

 Initial commit

 Changes to be committed:
	new file:   .gitignore
	new file:   CMakeLists.txt
	new file:   include/ApplyReplacements.hpp
	new file:   include/Logger.hpp
	new file:   include/MatcherFactory.hpp
	new file:   include/MyASTMatchers.hpp
	new file:   include/MyFrontendAction.hpp
	new file:   include/MyMatchCallback.hpp
	new file:   include/MyReplacementsYaml.hpp
	new file:   include/ProgramOptions.hpp
	new file:   include/TestingUtil.hpp
	new file:   include/ToolingUtil.hpp
	new file:   include/cxxopts.hpp
	new file:   scripts/gen-matcher.py
	new file:   scripts/gen-test.py
	new file:   scripts/generate_compdb.py
	new file:   scripts/template/MatcherTemplate.cpp
	new file:   scripts/template/TestTemplate.cpp
	new file:   scripts/template/compile_commands.json
	new file:   scripts/update-compile-commands.py
	new file:   src/ApplyReplacements.cpp
	new file:   src/MyFrontendAction.cpp
	new file:   src/ProgramOptions.cpp
	new file:   src/TestingUtil.cpp
	new file:   src/ToolingUtil.cpp
	new file:   src/main.cpp
	new file:   src/matchers/rename/RenameFcn.cpp
	new file:   test/.gitignore
	new file:   test/CMakeLists.txt
	new file:   test/CMakeLists.txt.in
	new file:   test/rename/RenameFcn/compile_commands.json
	new file:   test/rename/RenameFcn/example.cpp
	new file:   test/rename/RenameFcn/example.cpp.gold
	new file:   test/rename/RenameFcn/example.cpp.refactored
	new file:   test/rename/RenameFcn/sbcodexform.log
	new file:   test/rename/RenameFcn/sbcodexform.log.gold
	new file:   test/rename/RenameFcn/tRenameFcn.cpp

.gitignore

@@ -0,0 +1,9 @@
+*~
+CMakeCache.txt
+CMakeFiles/
+CTestTestfile.cmake
+Makefile
+Testing/
+bin/
+cmake_install.cmake
+lib/

CMakeLists.txt

@@ -0,0 +1,52 @@
+cmake_minimum_required(VERSION 2.8)
+
+# tool name
+set(TOOL clang-xform)
+
+if (NOT DEFINED LLVM_ROOT)
+  message(FATAL_ERROR "Clang 8.0.0 required. Please provide LLVM root path.\n"
+  "Usage: cmake <dir> -DLLVM_ROOT=<LLVM root path> \n"
+  "Clang prebuilt binaries are available at http://releases.llvm.org/download.html")
+endif (NOT DEFINED LLVM_ROOT)
+
+find_package(Clang REQUIRED CONFIG
+  HINTS "${LLVM_ROOT}/lib/cmake/clang")
+
+message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+
+# source files
+file(GLOB_RECURSE SRC_CPP
+    src/*.cpp
+)
+
+# include/link path
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
+link_directories(${LLVM_LIBRARY_DIRS})
+
+# set compile_options
+if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+  add_definitions(${LLVM_DEFINITIONS} -DCXXOPTS_NO_RTTI)
+  add_compile_options(-std=c++14 -fno-rtti)
+else ()
+  add_definitions(${LLVM_DEFINITIONS} /DCXXOPTS_NO_RTTI)
+  add_compile_options(/std:c++14 /GR-)
+endif()
+
+# clang libs to link
+set(CLANG_LIBS clangTooling clangToolingCore clangFrontendTool clangFrontend clangDriver clangBasic)
+set(CLANG_LIBS ${CLANG_LIBS} clangSerialization clangParse clangSema clangAnalysis clangEdit)
+set(CLANG_LIBS ${CLANG_LIBS} clangRewrite clangRewriteFrontend clangAST clangASTMatchers clangLex)
+set(CLANG_LIBS ${CLANG_LIBS} clangToolingRefactor clangFormat clangToolingInclusions)
+
+# enable testing
+enable_testing()
+option(BUILD_TESTS "Set to ON to build tests" OFF)
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+
+add_executable(${TOOL} ${SRC_CPP})
+target_link_libraries(${TOOL} ${CLANG_LIBS})
+
+add_subdirectory(test)

include/ApplyReplacements.hpp

@@ -0,0 +1,119 @@
+//===-- ApplyReplacements.hpp - Deduplicate and apply replacements -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides the interface for deduplicating, detecting
+/// conflicts in, and applying collections of Replacements.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_APPLYREPLACEMENTS_HPP
+#define LLVM_CLANG_APPLYREPLACEMENTS_HPP
+
+#include "clang/Tooling/Core/Diagnostic.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Refactoring/AtomicChange.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include <string>
+#include <system_error>
+#include <vector>
+
+namespace clang {
+
+class DiagnosticsEngine;
+class Rewriter;
+
+namespace replace {
+
+/// \brief Collection of TranslationUnitReplacements.
+typedef std::vector<clang::tooling::TranslationUnitReplacements> TUReplacements;
+
+/// \brief Collection of TranslationUnitReplacement files.
+typedef std::vector<std::string> TUReplacementFiles;
+
+/// \brief Collection of TranslationUniDiagnostics.
+typedef std::vector<clang::tooling::TranslationUnitDiagnostics> TUDiagnostics;
+
+/// \brief Map mapping file name to a set of AtomicChange targeting that file.
+typedef llvm::DenseMap<const clang::FileEntry *,
+                       std::vector<tooling::AtomicChange>>
+    FileToChangesMap;
+
+/// \brief Attempts to deserialize the given yaml file as
+/// TranslationUnitReplacements. All docs that successfully deserialize are
+/// added to \p TUs.
+///
+/// \param[in] FilePath File path to read for serialized
+/// TranslationUnitReplacements.
+/// \param[out] TUs Collection of all found and deserialized
+/// TranslationUnitReplacements or TranslationUnitDiagnostics.
+/// \param[in] Diagnostics DiagnosticsEngine used for error output.
+///
+/// \returns A boolean indicating success or failure in navigating the
+/// directory structure. true for success and false for failure
+bool collectReplacementsFromFile(
+    const llvm::StringRef FilePath, TUReplacements &TUs,
+    clang::DiagnosticsEngine &Diagnostics);
+
+bool collectReplacementsFromFile(
+    const llvm::StringRef FilePath, TUDiagnostics &TUs,
+    clang::DiagnosticsEngine &Diagnostics);
+
+/// \brief Deduplicate, check for conflicts, and extract all Replacements stored
+/// in \c TUs. Conflicting replacements are skipped.
+///
+/// \post For all (key,value) in FileChanges, value[i].getOffset() <=
+/// value[i+1].getOffset().
+///
+/// \param[in] TUs Collection of TranslationUnitReplacements or
+/// TranslationUnitDiagnostics to merge, deduplicate, and test for conflicts.
+/// \param[out] FileChanges Container grouping all changes by the
+/// file they target. Only non conflicting replacements are kept into
+/// FileChanges.
+/// \param[in] SM SourceManager required for conflict reporting.
+///
+/// \returns \parblock
+///          \li true If all changes were converted successfully.
+///          \li false If there were conflicts.
+bool mergeAndDeduplicate(const TUReplacements &TUs, const TUDiagnostics &TUDs,
+                         FileToChangesMap &FileChanges,
+                         clang::SourceManager &SM);
+
+/// \brief Apply \c AtomicChange on File and rewrite it.
+///
+/// \param[in] File Path of the file where to apply AtomicChange.
+/// \param[in] Changes to apply.
+/// \param[in] Spec For code cleanup and formatting.
+/// \param[in] Diagnostics DiagnosticsEngine used for error output.
+///
+/// \returns The changed code if all changes are applied successfully;
+/// otherwise, an llvm::Error carrying llvm::StringError or an error_code.
+llvm::Expected<std::string>
+applyChanges(StringRef File, const std::vector<tooling::AtomicChange> &Changes,
+             const tooling::ApplyChangesSpec &Spec,
+             DiagnosticsEngine &Diagnostics);
+
+/// \brief Delete the replacement file.
+///
+/// \param[in] File Replacement file to delete.
+/// \param[in] Diagnostics DiagnosticsEngine used for error output.
+///
+/// \returns \parblock
+///          \li true If all files have been deleted successfully.
+///          \li false If at least one or more failures occur when deleting
+/// files.
+bool deleteReplacementFile(const llvm::StringRef File,
+                           clang::DiagnosticsEngine &Diagnostics);
+
+bool applyReplacements(const llvm::StringRef File, const llvm::StringRef Output = "");
+
+} // end namespace replace
+} // end namespace clang
+
+#endif // LLVM_CLANG_APPLYREPLACEMENTS_HPP

include/Logger.hpp

@@ -0,0 +1,207 @@
+#ifndef LOGGER_HPP
+#define LOGGER_HPP
+
+#include <string>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <ctime>
+#include <thread>
+#include <mutex>
+
+// a simple logging class
+
+enum severity {trace, debug, info, warning, error, fatal};
+
+enum verbosity {quiet, minimal, normal, verbose};
+
+namespace detail{
+const std::string severity_string[6] = {"trace",
+                                        "debug",
+                                        "info",
+                                        "warning",
+                                        "error",
+                                        "fatal"};
+} // end namespace detail
+
+template <typename OStream, typename... Attributes>
+class Log
+{
+  public:
+    Log() = default;
+    Log(const Log&) = delete;
+    Log& operator =(const Log&) = delete;
+    virtual ~Log() {
+        if (Verbosity() != verbosity::quiet) {
+            OStream::Output(msg_.str());
+        }
+    }
+    std::ostringstream& Get(severity level = severity::info) {
+        switch (Verbosity()) {
+        case verbosity::quiet:
+            break;
+        case verbosity::minimal:
+            break;
+        case verbosity::normal:
+            OutputAttributes<Attributes...>(msg_);
+            break;
+        case verbosity::verbose:
+            OutputAttributes<Attributes...>(msg_);
+            msg_ << std::setw(7) <<  detail::severity_string[level] << " | ";
+            break;
+        }
+        return msg_;
+    }
+
+    static severity& Severity() {
+        static severity level = severity::info;
+        return level;
+    }
+
+    static verbosity& Verbosity() {
+        static verbosity level = verbosity::normal;
+        return level;
+    }
+
+  protected:
+    template <typename Attrib>
+    std::ostream& OutputAttributes(std::ostream& os) {
+        Attrib::Output(os);
+        os << " | ";
+        return os;
+    }
+
+    template <typename Attrib1, typename... Attribs,
+              typename = std::enable_if_t<(sizeof...(Attribs) > 0)> >
+    std::ostream& OutputAttributes(std::ostream& os) {
+        Attrib1::Output(os);
+        os << " | ";
+        return OutputAttributes<Attribs...>(os);
+    }
+
+    template <typename... Attribs,
+              typename = std::enable_if_t<sizeof...(Attribs) == 0> >
+    std::ostream& OutputAttributes(std::ostream& os) {
+        return os;
+    }
+
+    std::ostringstream msg_;
+};
+
+// attributes
+class Counter {
+  public:
+    static std::ostream& Output(std::ostream& os) {
+        return os << "No. " << ++Count();
+    }
+  private:
+    static int& Count() {
+        static int n = 0;
+        return n;
+    }
+};
+
+class ThreadID {
+  public:
+    static std::ostream& Output(std::ostream& os) {
+        return os << "T." << std::this_thread::get_id();
+    }
+};
+
+class TimeStamp {
+  public:
+    static std::ostream& Output(std::ostream& os) {
+        std::time_t current_time = std::time(nullptr);
+        std::string current_time_string = std::ctime(&current_time);
+        return os << current_time_string.substr(0, current_time_string.length() - 1);
+    }
+};
+
+// ostreams
+class FileStream {
+  public:
+    static void SetStream(std::ofstream& stream) {
+        std::lock_guard<std::mutex> guard(Mutex());
+        GetStream() = &stream;
+    }
+    static void Output(const std::string& msg) {
+        std::lock_guard<std::mutex> guard(Mutex());
+        std::ofstream* stream = GetStream();
+        if (!stream || !stream->is_open())
+            return;
+
+        int tmp = msg.length();
+        stream->write(msg.c_str(), tmp);
+        stream->flush();
+    }
+  private:
+    static std::ofstream*& GetStream() {
+        static std::ofstream* stream = nullptr;
+        return stream;
+    }
+    static std::mutex& Mutex() {
+        static std::mutex m;
+        return m;
+    }
+};
+
+class STDCStream {
+  public:
+    static void SetStream(std::ostream& stream) {
+        std::lock_guard<std::mutex> guard(Mutex());
+        GetStream() = &stream;
+    }
+    static void Output(const std::string& msg) {
+        std::lock_guard<std::mutex> guard(Mutex());
+        std::ostream* stream = GetStream();
+
+        stream->write(msg.c_str(), msg.length());
+        stream->flush();
+    }
+  private:
+    static std::ostream*& GetStream() {
+        static std::ostream* stream = &std::cout;
+        return stream;
+    }
+    static std::mutex& Mutex() {
+        static std::mutex m;
+        return m;
+    }
+};
+
+// helper class to set output file
+class RegisterLogFile {
+  public:
+    RegisterLogFile(const std::string& name)
+        : ofs_(name)
+    {
+        FileStream::SetStream(ofs_);
+        // set file in compilation mode in emacs
+        ofs_ << "-*- compilation-minor -*-" << "\n\n";
+    }
+    ~RegisterLogFile() {
+        Close();
+    }
+    void Close() {
+        if (ofs_.is_open()) {
+            ofs_.close();
+        }
+    }
+  private:
+    std::ofstream ofs_;
+};
+
+
+using FileLog = Log<FileStream, Counter, ThreadID, TimeStamp>;
+using TrivialLog = Log<STDCStream, ThreadID, TimeStamp>;
+
+#define FILE_LOG(level) \
+    if (level < FileLog::Severity()); \
+    else FileLog().Get(level)
+
+#define TRIVIAL_LOG(level) \
+    if (level < TrivialLog::Severity()); \
+    else TrivialLog().Get(level)
+
+#endif