Skip to content

[6.2][Caching] Swift Caching Dependency Scanning Improvements #81720

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion include/swift/AST/DiagnosticsFrontend.def
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,7 @@ REMARK(matching_output_produced,none,

// Caching related diagnostics
ERROR(error_caching_no_cas_fs, none,
"caching is enabled without -cas-fs option, input is not immutable", ())
"caching is enabled without CAS file-system options, input is not immutable", ())
ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef))

REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef))
Expand All @@ -534,6 +534,7 @@ ERROR(error_cache_key_creation, none, "cannot create cache key for compilation %
ERROR(error_cas_file_ref, none, "cannot load file %0 from CAS filesystem", (StringRef))
ERROR(error_cas_conflict_options, none, "cannot setup CAS due to conflicting '-cas-*' options", ())
ERROR(error_cas_initialization, none, "CAS cannot be initialized from the specified '-cas-*' options: %0", (StringRef))
ERROR(error_cas_malformed_input, none, "CAS input '%0' is malformed: %1", (StringRef, StringRef))
WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef))

ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef))
Expand Down
23 changes: 9 additions & 14 deletions include/swift/AST/ModuleDependencies.h
Original file line number Diff line number Diff line change
Expand Up @@ -1000,7 +1000,7 @@ using ModuleDependenciesKindMap =
/// Track swift dependency
class SwiftDependencyTracker {
public:
SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
SwiftDependencyTracker(std::shared_ptr<llvm::cas::ObjectStore> CAS,
llvm::PrefixMapper *Mapper,
const CompilerInvocation &CI);

Expand All @@ -1009,7 +1009,8 @@ class SwiftDependencyTracker {
llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();

private:
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
std::shared_ptr<llvm::cas::ObjectStore> CAS;
llvm::PrefixMapper *Mapper;

struct FileEntry {
Expand All @@ -1035,10 +1036,7 @@ class SwiftDependencyScanningService {
ClangScanningService;

/// CachingOnDiskFileSystem for dependency tracking.
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;

/// If use clang include tree.
bool UseClangIncludeTree = false;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> CacheFS;

/// CAS Instance.
std::shared_ptr<llvm::cas::ObjectStore> CAS;
Expand Down Expand Up @@ -1076,8 +1074,8 @@ class SwiftDependencyScanningService {
return *SharedFilesystemCache;
}

llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
assert(CacheFS && "Expect CachingOnDiskFileSystem");
llvm::vfs::FileSystem &getSharedCachingFS() const {
assert(CacheFS && "Expect a CASFileSystem");
return *CacheFS;
}

Expand All @@ -1088,20 +1086,17 @@ class SwiftDependencyScanningService {

std::optional<SwiftDependencyTracker>
createSwiftDependencyTracker(const CompilerInvocation &CI) {
if (!CacheFS)
if (!CAS)
return std::nullopt;

return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI);
return SwiftDependencyTracker(CAS, Mapper.get(), CI);
}

llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
if (UseClangIncludeTree)
if (CAS)
return llvm::cas::createCASProvidingFileSystem(
CAS, llvm::vfs::createPhysicalFileSystem());

if (CacheFS)
return CacheFS->createProxyFS();

return llvm::vfs::createPhysicalFileSystem();
}

Expand Down
12 changes: 4 additions & 8 deletions include/swift/Basic/CASOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,11 @@ class CASOptions final {
/// CASOptions
clang::CASOptions CASOpts;

/// CASFS Root.
std::vector<std::string> CASFSRootIDs;

/// Clang Include Trees.
std::vector<std::string> ClangIncludeTrees;
std::string ClangIncludeTree;

/// Clang Include Tree FileList.
std::vector<std::string> ClangIncludeTreeFileList;
std::string ClangIncludeTreeFileList;

/// CacheKey for input file.
std::string InputFileKey;
Expand All @@ -62,9 +59,8 @@ class CASOptions final {
/// Check to see if a CASFileSystem is required.
bool requireCASFS() const {
return EnableCaching &&
(!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() ||
!ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
!BridgingHeaderPCHCacheKey.empty());
(!ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() ||
!InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty());
}

/// Return a hash code of any components from these options that should
Expand Down
6 changes: 0 additions & 6 deletions include/swift/Basic/LangOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -1094,12 +1094,6 @@ namespace swift {
/// built and provided to the compiler invocation.
bool DisableImplicitClangModules = false;

/// Enable ClangIncludeTree for explicit module builds scanning.
bool UseClangIncludeTree = false;

/// Using ClangIncludeTreeRoot for compilation.
bool HasClangIncludeTreeRoot = false;

/// Whether the dependency scanner should construct all swift-frontend
/// invocations directly from clang cc1 args.
bool ClangImporterDirectCC1Scan = false;
Expand Down
5 changes: 0 additions & 5 deletions include/swift/ClangImporter/ClangImporter.h
Original file line number Diff line number Diff line change
Expand Up @@ -472,11 +472,6 @@ class ClangImporter final : public ClangModuleLoader {
/// Reads the original source file name from PCH.
std::string getOriginalSourceFile(StringRef PCHFilename);

/// Add clang dependency file names.
///
/// \param files The list of file to append dependencies to.
void addClangInvovcationDependencies(std::vector<std::string> &files);

/// Makes a temporary replica of the ClangImporter's CompilerInstance, reads a
/// module map into the replica and emits a PCM file for one of the modules it
/// declares. Delegates to clang for everything except construction of the
Expand Down
6 changes: 3 additions & 3 deletions include/swift/Frontend/CachingUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
llvm::StringRef Filename = "");

llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
ArrayRef<std::string> IncludeTreeRoots,
ArrayRef<std::string> IncludeTreeFileList);
createCASFileSystem(llvm::cas::ObjectStore &CAS,
const std::string &IncludeTreeRoot,
const std::string &IncludeTreeFileList);

std::vector<std::string> remapPathsFromCommandLine(
ArrayRef<std::string> Args,
Expand Down
3 changes: 0 additions & 3 deletions include/swift/Option/FrontendOptions.td
Original file line number Diff line number Diff line change
Expand Up @@ -1454,9 +1454,6 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
def no_clang_include_tree: Flag<["-"], "no-clang-include-tree">,
HelpText<"Do not use clang include tree, fallback to use CAS filesystem to build clang modules">;

def cas_fs: Separate<["-"], "cas-fs">,
HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;

def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">,
Expand Down
35 changes: 13 additions & 22 deletions lib/AST/ModuleDependencies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#include "swift/Strings.h"
#include "clang/CAS/IncludeTree.h"
#include "llvm/CAS/CASProvidingFileSystem.h"
#include "llvm/CAS/CachingOnDiskFileSystem.h"
#include "llvm/Config/config.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
Expand Down Expand Up @@ -602,13 +601,16 @@ swift::dependencies::registerBackDeployLibraries(
}

SwiftDependencyTracker::SwiftDependencyTracker(
llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper,
std::shared_ptr<llvm::cas::ObjectStore> CAS, llvm::PrefixMapper *Mapper,
const CompilerInvocation &CI)
: FS(FS.createProxyFS()), Mapper(Mapper) {
: CAS(CAS), Mapper(Mapper) {
auto &SearchPathOpts = CI.getSearchPathOptions();

FS = llvm::cas::createCASProvidingFileSystem(
CAS, llvm::vfs::createPhysicalFileSystem());

auto addCommonFile = [&](StringRef path) {
auto file = FS.openFileForRead(path);
auto file = FS->openFileForRead(path);
if (!file)
return;
auto status = (*file)->status();
Expand Down Expand Up @@ -679,7 +681,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
llvm::SmallVector<clang::cas::IncludeTree::FileList::FileEntry> Files;
for (auto &file : TrackedFiles) {
auto includeTreeFile = clang::cas::IncludeTree::File::create(
FS->getCAS(), file.first, file.second.FileRef);
*CAS, file.first, file.second.FileRef);
if (!includeTreeFile) {
return llvm::createStringError("CASFS createTree failed for " +
file.first + ": " +
Expand All @@ -691,7 +693,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
}

auto includeTreeList =
clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {});
clang::cas::IncludeTree::FileList::create(*CAS, Files, {});
if (!includeTreeList)
return llvm::createStringError("casfs include-tree filelist error: " +
toString(includeTreeList.takeError()));
Expand Down Expand Up @@ -719,14 +721,8 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
CAS = Instance.getSharedCASInstance();
ActionCache = Instance.getSharedCacheInstance();

auto CachingFS =
llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
if (!CachingFS) {
Instance.getDiags().diagnose(SourceLoc(), diag::error_cas_fs_creation,
toString(CachingFS.takeError()));
return true;
}
CacheFS = std::move(*CachingFS);
CacheFS = llvm::cas::createCASProvidingFileSystem(
CAS, llvm::vfs::createPhysicalFileSystem());

// Setup prefix mapping.
auto &ScannerPrefixMapper =
Expand All @@ -744,19 +740,14 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
Mapper->sort();
}

UseClangIncludeTree =
Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
UseClangIncludeTree
? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree
: clang::tooling::dependencies::ScanningOutputFormat::FullTree;
clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree;

ClangScanningService.emplace(
clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
ClangScanningFormat,
Instance.getInvocation().getCASOptions().CASOpts,
ClangScanningFormat, Instance.getInvocation().getCASOptions().CASOpts,
Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(),
UseClangIncludeTree ? nullptr : CacheFS,
/*CachingOnDiskFileSystem=*/nullptr,
// The current working directory optimization (off by default)
// should not impact CAS. We set the optization to all to be
// consistent with the non-CAS case.
Expand Down
4 changes: 2 additions & 2 deletions lib/AST/PluginLoader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {

static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
getPluginLoadingFS(ASTContext &Ctx) {
// If there is a clang include tree FS, using real file system to load plugin
// If there is an immutable file system, using real file system to load plugin
// as the FS in SourceMgr doesn't support directory iterator.
if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
if (Ctx.CASOpts.HasImmutableFileSystem)
return llvm::vfs::getRealFileSystem();
return Ctx.SourceMgr.getFileSystem();
}
Expand Down
47 changes: 12 additions & 35 deletions lib/ClangImporter/ClangImporter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/STLExtras.h"
Expand Down Expand Up @@ -1062,27 +1063,6 @@ std::string ClangImporter::getOriginalSourceFile(StringRef PCHFilename) {
Impl.Instance->getPCHContainerReader(), Impl.Instance->getDiagnostics());
}

void ClangImporter::addClangInvovcationDependencies(
std::vector<std::string> &files) {
auto addFiles = [&files](const auto &F) {
files.insert(files.end(), F.begin(), F.end());
};
auto &invocation = *Impl.Invocation;
// FIXME: Add file dependencies that are not accounted. The long term solution
// is to do a dependency scanning for clang importer and use that directly.
SmallVector<std::string, 4> HeaderMapFileNames;
Impl.Instance->getPreprocessor().getHeaderSearchInfo().getHeaderMapFileNames(
HeaderMapFileNames);
addFiles(HeaderMapFileNames);
addFiles(invocation.getHeaderSearchOpts().VFSOverlayFiles);
// FIXME: Should not depend on working directory. Build system/swift driver
// should not pass working directory here but if that option is passed,
// repect that and add that into CASFS.
auto CWD = invocation.getFileSystemOpts().WorkingDir;
if (!CWD.empty())
files.push_back(CWD);
}

std::optional<std::string>
ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions,
StringRef SwiftPCHHash, bool &isExplicit) {
Expand Down Expand Up @@ -1210,9 +1190,8 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
clang::frontend::ActionKind::GenerateModule ||
CI->getFrontendOpts().ProgramAction ==
clang::frontend::ActionKind::GeneratePCH) &&
ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
CI->getFrontendOpts().CASIncludeTreeID =
ctx.CASOpts.ClangIncludeTrees.back();
!ctx.CASOpts.ClangIncludeTree.empty()) {
CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree;
CI->getFrontendOpts().Inputs.clear();
}
}
Expand Down Expand Up @@ -1271,7 +1250,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(

std::vector<std::string> FilteredModuleMapFiles;
for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) {
if (ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
if (ctx.CASOpts.HasImmutableFileSystem) {
// There is no need to add any module map file here. Issue a warning and
// drop the option.
Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile);
Expand Down Expand Up @@ -1351,7 +1330,7 @@ ClangImporter::create(ASTContext &ctx,
fileMapping.requiresBuiltinHeadersInSystemModules;

// Avoid creating indirect file system when using include tree.
if (!ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
if (!ctx.CASOpts.HasImmutableFileSystem) {
// Wrap Swift's FS to allow Clang to override the working directory
VFS = llvm::vfs::RedirectingFileSystem::create(
fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem());
Expand Down Expand Up @@ -2756,7 +2735,6 @@ ClangImporter::Implementation::Implementation(
!ctx.ClangImporterOpts.BridgingHeader.empty()),
DisableOverlayModules(ctx.ClangImporterOpts.DisableOverlayModules),
EnableClangSPI(ctx.ClangImporterOpts.EnableClangSPI),
UseClangIncludeTree(ctx.ClangImporterOpts.UseClangIncludeTree),
importSymbolicCXXDecls(
ctx.LangOpts.hasFeature(Feature::ImportSymbolicCXXDecls)),
IsReadingBridgingPCH(false),
Expand Down Expand Up @@ -4281,15 +4259,14 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
PPOpts.MacroIncludes.clear();
PPOpts.Includes.clear();

// CodeGenOptions.
auto &CGOpts = instance.getCodeGenOpts();
CGOpts.DebugCompilationDir.clear();
// Clear benign CodeGenOptions.
clang::tooling::dependencies::resetBenignCodeGenOptions(
clang::frontend::ActionKind::GenerateModule, instance.getLangOpts(),
instance.getCodeGenOpts());

if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
// FileSystemOptions.
auto &FSOpts = instance.getFileSystemOpts();
FSOpts.WorkingDir.clear();
}
// FileSystemOptions.
auto &FSOpts = instance.getFileSystemOpts();
FSOpts.WorkingDir.clear();

if (!Impl.SwiftContext.SearchPathOpts.ScannerPrefixMapper.empty()) {
// Remap all the paths if requested.
Expand Down
Loading