From 4670acf7f91ccef5b3dac873636df6bfce0814dc Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 23 Apr 2025 13:45:55 -0700 Subject: [PATCH 1/5] [Caching] Reduce the number of cas ID passed on frontend commandline Using IncludeTree::FileList to concat the include tree file systems that are passed on the command-line. This significantly reduce the command-line size, and also makes the cache key computation a lot faster. rdar://148752988 (cherry picked from commit 201e4faea720bfaaa39a2f5bc6d25c3c84efa788) --- include/swift/AST/DiagnosticsFrontend.def | 1 + include/swift/Basic/CASOptions.h | 6 +- include/swift/Frontend/CachingUtils.h | 4 +- lib/AST/PluginLoader.cpp | 4 +- lib/ClangImporter/ClangImporter.cpp | 7 +- lib/DependencyScan/ScanDependencies.cpp | 91 ++++++++++++++++------- lib/Frontend/CachingUtils.cpp | 59 +++++---------- lib/Frontend/CompilerInvocation.cpp | 10 +-- lib/Frontend/Frontend.cpp | 4 +- lib/Sema/TypeCheckMacros.cpp | 2 +- test/CAS/module_deps_include_tree.swift | 1 - 11 files changed, 101 insertions(+), 88 deletions(-) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 4271af1edf4d2..77beea192f272 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -534,6 +534,7 @@ ERROR(error_cache_key_creation, none, "cannot create cache key for compilation % ERROR(error_cas_file_ref, none, "cannot load file %0 from CAS filesystem", (StringRef)) ERROR(error_cas_conflict_options, none, "cannot setup CAS due to conflicting '-cas-*' options", ()) ERROR(error_cas_initialization, none, "CAS cannot be initialized from the specified '-cas-*' options: %0", (StringRef)) +ERROR(error_cas_malformed_input, none, "CAS input '%0' is malformed: %1", (StringRef, StringRef)) WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef)) ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef)) diff --git a/include/swift/Basic/CASOptions.h b/include/swift/Basic/CASOptions.h index fb63938e1f3fb..4f01df225ace7 100644 --- a/include/swift/Basic/CASOptions.h +++ b/include/swift/Basic/CASOptions.h @@ -41,10 +41,10 @@ class CASOptions final { std::vector CASFSRootIDs; /// Clang Include Trees. - std::vector ClangIncludeTrees; + std::string ClangIncludeTree; /// Clang Include Tree FileList. - std::vector ClangIncludeTreeFileList; + std::string ClangIncludeTreeFileList; /// CacheKey for input file. std::string InputFileKey; @@ -62,7 +62,7 @@ class CASOptions final { /// Check to see if a CASFileSystem is required. bool requireCASFS() const { return EnableCaching && - (!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() || + (!CASFSRootIDs.empty() || !ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty()); } diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index e478c22daf56c..8f249cbd4159b 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -71,8 +71,8 @@ std::unique_ptr loadCachedCompileResultFromCacheKey( llvm::Expected> createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots, - ArrayRef IncludeTreeRoots, - ArrayRef IncludeTreeFileList); + const std::string &IncludeTreeRoot, + const std::string &IncludeTreeFileList); std::vector remapPathsFromCommandLine( ArrayRef Args, diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 271c484fc99b8..3fb2f6a5ec04d 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -64,9 +64,9 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) { static llvm::IntrusiveRefCntPtr getPluginLoadingFS(ASTContext &Ctx) { - // If there is a clang include tree FS, using real file system to load plugin + // If there is an immutable file system, using real file system to load plugin // as the FS in SourceMgr doesn't support directory iterator. - if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot) + if (Ctx.CASOpts.HasImmutableFileSystem) return llvm::vfs::getRealFileSystem(); return Ctx.SourceMgr.getFileSystem(); } diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index a2b173f8265a1..d130bd18554cd 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1211,8 +1211,7 @@ std::optional> ClangImporter::getClangCC1Arguments( CI->getFrontendOpts().ProgramAction == clang::frontend::ActionKind::GeneratePCH) && ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { - CI->getFrontendOpts().CASIncludeTreeID = - ctx.CASOpts.ClangIncludeTrees.back(); + CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree; CI->getFrontendOpts().Inputs.clear(); } } @@ -1271,7 +1270,7 @@ std::optional> ClangImporter::getClangCC1Arguments( std::vector FilteredModuleMapFiles; for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) { - if (ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { + if (ctx.ClangImporterOpts.UseClangIncludeTree) { // There is no need to add any module map file here. Issue a warning and // drop the option. Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile); @@ -1351,7 +1350,7 @@ ClangImporter::create(ASTContext &ctx, fileMapping.requiresBuiltinHeadersInSystemModules; // Avoid creating indirect file system when using include tree. - if (!ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { + if (!ctx.CASOpts.HasImmutableFileSystem) { // Wrap Swift's FS to allow Clang to override the working directory VFS = llvm::vfs::RedirectingFileSystem::create( fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem()); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 8547191c93df1..615b0e79265fe 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -45,7 +45,7 @@ #include "swift/Frontend/FrontendOptions.h" #include "swift/Frontend/ModuleInterfaceLoader.h" #include "swift/Strings.h" -#include "clang/Basic/Module.h" +#include "clang/CAS/IncludeTree.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" @@ -69,7 +69,6 @@ #include #include #include -#include using namespace swift; using namespace swift::dependencies; @@ -101,9 +100,6 @@ class ExplicitModuleDependencyResolver { if (resolvingDepInfo.isFinalized()) return false; - if (auto ID = resolvingDepInfo.getClangIncludeTree()) - includeTrees.push_back(*ID); - for (const auto &depModuleID : dependencies) { const auto &depInfo = cache.findKnownDependency(depModuleID); switch (depModuleID.Kind) { @@ -322,8 +318,10 @@ class ExplicitModuleDependencyResolver { // Collect CAS deppendencies from clang modules. if (!clangDepDetails.CASFileSystemRootID.empty()) rootIDs.push_back(clangDepDetails.CASFileSystemRootID); - if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) - includeTrees.push_back(clangDepDetails.CASClangIncludeTreeRootID); + if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) { + if (addIncludeTree(clangDepDetails.CASClangIncludeTreeRootID)) + return true; + } collectUsedVFSOverlay(clangDepDetails); @@ -358,12 +356,14 @@ class ExplicitModuleDependencyResolver { auto bridgeRoot = tracker->createTreeFromDependencies(); if (!bridgeRoot) return diagnoseCASFSCreationError(bridgeRoot.takeError()); - fileListIDs.push_back(bridgeRoot->getID().toString()); + + fileListRefs.push_back(bridgeRoot->getRef()); } } - } else - includeTrees.push_back(sourceDepDetails.textualModuleDetails - .CASBridgingHeaderIncludeTreeRootID); + } else if (addIncludeTree(sourceDepDetails.textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID)) + return true; + return false; }; @@ -499,9 +499,7 @@ class ExplicitModuleDependencyResolver { auto root = tracker->createTreeFromDependencies(); if (!root) return diagnoseCASFSCreationError(root.takeError()); - auto rootID = root->getID().toString(); - dependencyInfoCopy.updateCASFileSystemRootID(rootID); - fileListIDs.push_back(rootID); + fileListRefs.push_back(root->getRef()); } else if (auto *textualDep = resolvingDepInfo.getAsSwiftInterfaceModule()) { tracker->startTracking(); @@ -516,9 +514,7 @@ class ExplicitModuleDependencyResolver { auto root = tracker->createTreeFromDependencies(); if (!root) return diagnoseCASFSCreationError(root.takeError()); - auto rootID = root->getID().toString(); - dependencyInfoCopy.updateCASFileSystemRootID(rootID); - fileListIDs.push_back(rootID); + fileListRefs.push_back(root->getRef()); } // Update build command line. @@ -530,15 +526,8 @@ class ExplicitModuleDependencyResolver { commandline.push_back(rootID); } - for (auto tree : includeTrees) { - commandline.push_back("-clang-include-tree-root"); - commandline.push_back(tree); - } - - for (auto list : fileListIDs) { - commandline.push_back("-clang-include-tree-filelist"); - commandline.push_back(list); - } + if (computeCASFileSystem(dependencyInfoCopy)) + return true; } // Compute and update module cache key. @@ -636,6 +625,53 @@ class ExplicitModuleDependencyResolver { cmd.push_back("-cache-disable-replay"); } + bool addIncludeTree(StringRef includeTree) { + auto &db = cache.getScanService().getCAS(); + auto casID = db.parseID(includeTree); + if (!casID) { + instance.getDiags().diagnose(SourceLoc(), diag::error_invalid_cas_id, + includeTree, toString(casID.takeError())); + return true; + } + auto ref = db.getReference(*casID); + if (!ref) { + instance.getDiags().diagnose(SourceLoc(), diag::error_load_input_from_cas, + includeTree); + return true; + } + + auto root = clang::cas::IncludeTreeRoot::get(db, *ref); + if (!root) { + instance.getDiags().diagnose(SourceLoc(), diag::error_cas_malformed_input, + includeTree, toString(root.takeError())); + return true; + } + + fileListRefs.push_back(root->getFileListRef()); + return false; + } + + bool computeCASFileSystem(ModuleDependencyInfo &dependencyInfoCopy) { + if (fileListRefs.empty()) + return false; + + auto &db = cache.getScanService().getCAS(); + auto casFS = + clang::cas::IncludeTree::FileList::create(db, {}, fileListRefs); + if (!casFS) { + instance.getDiags().diagnose(SourceLoc(), diag::error_cas, + "CAS IncludeTree FileList creation", + toString(casFS.takeError())); + return true; + } + + auto casID = casFS->getID().toString(); + dependencyInfoCopy.updateCASFileSystemRootID(casID); + commandline.push_back("-clang-include-tree-filelist"); + commandline.push_back(casID); + return false; + } + private: const ModuleDependencyID &moduleID; ModuleDependenciesCache &cache; @@ -644,8 +680,7 @@ class ExplicitModuleDependencyResolver { std::optional tracker; std::vector rootIDs; - std::vector includeTrees; - std::vector fileListIDs; + std::vector fileListRefs; std::vector commandline; std::vector bridgingHeaderBuildCmd; llvm::StringMap macros; diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index 7bc73326332b5..c181854f7708d 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -472,16 +472,10 @@ static Expected mergeCASFileSystem(ObjectStore &CAS, Expected> createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, - ArrayRef IncludeTrees, - ArrayRef IncludeTreeFileList) { - assert(!FSRoots.empty() || !IncludeTrees.empty() || + const std::string &IncludeTree, + const std::string &IncludeTreeFileList) { + assert(!FSRoots.empty() || !IncludeTree.empty() || !IncludeTreeFileList.empty() && "no root ID provided"); - if (FSRoots.size() == 1 && IncludeTrees.empty()) { - auto ID = CAS.parseID(FSRoots.front()); - if (!ID) - return ID.takeError(); - return createCASFileSystem(CAS, *ID); - } auto NewRoot = mergeCASFileSystem(CAS, FSRoots); if (!NewRoot) @@ -492,10 +486,9 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, return FS.takeError(); auto CASFS = makeIntrusiveRefCnt(std::move(*FS)); - std::vector Files; - // Push all Include File System onto overlay. - for (auto &Tree : IncludeTrees) { - auto ID = CAS.parseID(Tree); + + if (!IncludeTree.empty()) { + auto ID = CAS.parseID(IncludeTree); if (!ID) return ID.takeError(); @@ -510,46 +503,32 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, if (!ITF) return ITF.takeError(); - auto Err = ITF->forEachFile( - [&](clang::cas::IncludeTree::File File, - clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { - Files.push_back({File.getRef(), Size}); - return llvm::Error::success(); - }); + auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF); + if (!ITFS) + return ITFS.takeError(); - if (Err) - return std::move(Err); + CASFS->pushOverlay(*ITFS); } - for (auto &List: IncludeTreeFileList) { - auto ID = CAS.parseID(List); + if (!IncludeTreeFileList.empty()) { + auto ID = CAS.parseID(IncludeTreeFileList); if (!ID) return ID.takeError(); auto Ref = CAS.getReference(*ID); if (!Ref) return createCASObjectNotFoundError(*ID); - auto IT = clang::cas::IncludeTree::FileList::get(CAS, *Ref); - if (!IT) - return IT.takeError(); + auto ITF = clang::cas::IncludeTree::FileList::get(CAS, *Ref); + if (!ITF) + return ITF.takeError(); - auto Err = IT->forEachFile( - [&](clang::cas::IncludeTree::File File, - clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { - Files.push_back({File.getRef(), Size}); - return llvm::Error::success(); - }); + auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF); + if (!ITFS) + return ITFS.takeError(); - if (Err) - return std::move(Err); + CASFS->pushOverlay(std::move(*ITFS)); } - auto ITFS = clang::cas::createIncludeTreeFileSystem(CAS, Files); - if (!ITFS) - return ITFS.takeError(); - - CASFS->pushOverlay(std::move(*ITFS)); - return CASFS; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 22256cb84ba1b..4d26cb8edbd0e 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -763,10 +763,10 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, for (const auto &A : Args.getAllArgValues(OPT_cas_fs)) Opts.CASFSRootIDs.emplace_back(A); - for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_root)) - Opts.ClangIncludeTrees.emplace_back(A); - for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_filelist)) - Opts.ClangIncludeTreeFileList.emplace_back(A); + if (auto *A = Args.getLastArg(OPT_clang_include_tree_root)) + Opts.ClangIncludeTree = A->getValue(); + if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist)) + Opts.ClangIncludeTreeFileList = A->getValue(); if (const Arg *A = Args.getLastArg(OPT_input_file_key)) Opts.InputFileKey = A->getValue(); @@ -774,7 +774,7 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, if (const Arg*A = Args.getLastArg(OPT_bridging_header_pch_key)) Opts.BridgingHeaderPCHCacheKey = A->getValue(); - if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTrees.empty() || + if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTree.empty() || !Opts.ClangIncludeTreeFileList.empty()) Opts.HasImmutableFileSystem = true; diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index ff160adffaa9b..40149a5ddf03f 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -630,11 +630,11 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() { } if (Invocation.getCASOptions().requireCASFS()) { - if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTrees.empty() || + if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTree.empty() || !CASOpts.ClangIncludeTreeFileList.empty()) { // Set up CASFS as BaseFS. auto FS = createCASFileSystem(*CAS, CASOpts.CASFSRootIDs, - CASOpts.ClangIncludeTrees, + CASOpts.ClangIncludeTree, CASOpts.ClangIncludeTreeFileList); if (!FS) { Diagnostics.diagnose(SourceLoc(), diag::error_cas_fs_creation, diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 2d5c237044671..521af16790613 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -282,7 +282,7 @@ initializePlugin(ASTContext &ctx, CompilerPlugin *plugin, StringRef libraryPath, if (!libraryPath.empty()) { #if SWIFT_BUILD_SWIFT_SYNTAX llvm::SmallString<128> resolvedLibraryPath; - auto fs = ctx.ClangImporterOpts.HasClangIncludeTreeRoot + auto fs = ctx.CASOpts.HasImmutableFileSystem ? llvm::vfs::getRealFileSystem() : ctx.SourceMgr.getFileSystem(); if (auto err = fs->getRealPath(libraryPath, resolvedLibraryPath)) { diff --git a/test/CAS/module_deps_include_tree.swift b/test/CAS/module_deps_include_tree.swift index 711d84a680864..5ae9d06981603 100644 --- a/test/CAS/module_deps_include_tree.swift +++ b/test/CAS/module_deps_include_tree.swift @@ -41,7 +41,6 @@ // INCLUDE_TREE_F-NEXT: CHeaders/F.h // MAIN_CMD: -direct-clang-cc1-module-build -// MAIN_CMD: -clang-include-tree-root // MAIN_CMD: -clang-include-tree-filelist import C From 559734c6d06fe9f0f0ed68ac25eeb368b30d84c1 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 24 Apr 2025 10:13:52 -0700 Subject: [PATCH 2/5] [Caching] Remove CASFS clang module implemenation Remove the CASFS based clang module implemenation as it is not longer used. (cherry picked from commit 3c81c1ca9fc5d5ebbeef525f084015d9ff5c65e6) --- include/swift/AST/DiagnosticsFrontend.def | 2 +- include/swift/AST/ModuleDependencies.h | 23 +- include/swift/Basic/CASOptions.h | 8 +- include/swift/Basic/LangOptions.h | 6 - include/swift/Frontend/CachingUtils.h | 2 +- include/swift/Option/FrontendOptions.td | 3 - lib/AST/ModuleDependencies.cpp | 35 ++-- lib/ClangImporter/ClangImporter.cpp | 11 +- .../ClangModuleDependencyScanner.cpp | 15 +- lib/ClangImporter/ImporterImpl.h | 1 - lib/DependencyScan/ScanDependencies.cpp | 8 - lib/Frontend/CachingUtils.cpp | 42 +--- lib/Frontend/CompilerInvocation.cpp | 9 +- lib/Frontend/Frontend.cpp | 14 +- lib/Frontend/ModuleInterfaceLoader.cpp | 10 +- test/CAS/cas_fs.swift | 28 --- test/CAS/cas_output_backend.swift | 2 +- test/CAS/module_deps.swift | 197 ------------------ 18 files changed, 46 insertions(+), 370 deletions(-) delete mode 100644 test/CAS/cas_fs.swift delete mode 100644 test/CAS/module_deps.swift diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 77beea192f272..2405f6228fe33 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -520,7 +520,7 @@ REMARK(matching_output_produced,none, // Caching related diagnostics ERROR(error_caching_no_cas_fs, none, - "caching is enabled without -cas-fs option, input is not immutable", ()) + "caching is enabled without CAS file-system options, input is not immutable", ()) ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef)) REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef)) diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 998a57650f637..47e69d1bed37b 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -1000,7 +1000,7 @@ using ModuleDependenciesKindMap = /// Track swift dependency class SwiftDependencyTracker { public: - SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS, + SwiftDependencyTracker(std::shared_ptr CAS, llvm::PrefixMapper *Mapper, const CompilerInvocation &CI); @@ -1009,7 +1009,8 @@ class SwiftDependencyTracker { llvm::Expected createTreeFromDependencies(); private: - llvm::IntrusiveRefCntPtr FS; + llvm::IntrusiveRefCntPtr FS; + std::shared_ptr CAS; llvm::PrefixMapper *Mapper; struct FileEntry { @@ -1035,10 +1036,7 @@ class SwiftDependencyScanningService { ClangScanningService; /// CachingOnDiskFileSystem for dependency tracking. - llvm::IntrusiveRefCntPtr CacheFS; - - /// If use clang include tree. - bool UseClangIncludeTree = false; + llvm::IntrusiveRefCntPtr CacheFS; /// CAS Instance. std::shared_ptr CAS; @@ -1076,8 +1074,8 @@ class SwiftDependencyScanningService { return *SharedFilesystemCache; } - llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const { - assert(CacheFS && "Expect CachingOnDiskFileSystem"); + llvm::vfs::FileSystem &getSharedCachingFS() const { + assert(CacheFS && "Expect a CASFileSystem"); return *CacheFS; } @@ -1088,20 +1086,17 @@ class SwiftDependencyScanningService { std::optional createSwiftDependencyTracker(const CompilerInvocation &CI) { - if (!CacheFS) + if (!CAS) return std::nullopt; - return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI); + return SwiftDependencyTracker(CAS, Mapper.get(), CI); } llvm::IntrusiveRefCntPtr getClangScanningFS() const { - if (UseClangIncludeTree) + if (CAS) return llvm::cas::createCASProvidingFileSystem( CAS, llvm::vfs::createPhysicalFileSystem()); - if (CacheFS) - return CacheFS->createProxyFS(); - return llvm::vfs::createPhysicalFileSystem(); } diff --git a/include/swift/Basic/CASOptions.h b/include/swift/Basic/CASOptions.h index 4f01df225ace7..82f7731b88dd4 100644 --- a/include/swift/Basic/CASOptions.h +++ b/include/swift/Basic/CASOptions.h @@ -37,9 +37,6 @@ class CASOptions final { /// CASOptions clang::CASOptions CASOpts; - /// CASFS Root. - std::vector CASFSRootIDs; - /// Clang Include Trees. std::string ClangIncludeTree; @@ -62,9 +59,8 @@ class CASOptions final { /// Check to see if a CASFileSystem is required. bool requireCASFS() const { return EnableCaching && - (!CASFSRootIDs.empty() || !ClangIncludeTree.empty() || - !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() || - !BridgingHeaderPCHCacheKey.empty()); + (!ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() || + !InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty()); } /// Return a hash code of any components from these options that should diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h index 4613509989473..9e557da32c21a 100644 --- a/include/swift/Basic/LangOptions.h +++ b/include/swift/Basic/LangOptions.h @@ -1094,12 +1094,6 @@ namespace swift { /// built and provided to the compiler invocation. bool DisableImplicitClangModules = false; - /// Enable ClangIncludeTree for explicit module builds scanning. - bool UseClangIncludeTree = false; - - /// Using ClangIncludeTreeRoot for compilation. - bool HasClangIncludeTreeRoot = false; - /// Whether the dependency scanner should construct all swift-frontend /// invocations directly from clang cc1 args. bool ClangImporterDirectCC1Scan = false; diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index 8f249cbd4159b..d676ccbcaa4e6 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -70,7 +70,7 @@ std::unique_ptr loadCachedCompileResultFromCacheKey( llvm::StringRef Filename = ""); llvm::Expected> -createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots, +createCASFileSystem(llvm::cas::ObjectStore &CAS, const std::string &IncludeTreeRoot, const std::string &IncludeTreeFileList); diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index c73c48622d235..cef1ba7eadbf0 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1454,9 +1454,6 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">, def no_clang_include_tree: Flag<["-"], "no-clang-include-tree">, HelpText<"Do not use clang include tree, fallback to use CAS filesystem to build clang modules">; -def cas_fs: Separate<["-"], "cas-fs">, - HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"">; - def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">, HelpText<"Clang Include Tree CASID">, MetaVarName<"">; def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">, diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 4fa8bb918322c..149a85e514e4e 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -25,7 +25,6 @@ #include "swift/Strings.h" #include "clang/CAS/IncludeTree.h" #include "llvm/CAS/CASProvidingFileSystem.h" -#include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Config/config.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -602,13 +601,16 @@ swift::dependencies::registerBackDeployLibraries( } SwiftDependencyTracker::SwiftDependencyTracker( - llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper, + std::shared_ptr CAS, llvm::PrefixMapper *Mapper, const CompilerInvocation &CI) - : FS(FS.createProxyFS()), Mapper(Mapper) { + : CAS(CAS), Mapper(Mapper) { auto &SearchPathOpts = CI.getSearchPathOptions(); + FS = llvm::cas::createCASProvidingFileSystem( + CAS, llvm::vfs::createPhysicalFileSystem()); + auto addCommonFile = [&](StringRef path) { - auto file = FS.openFileForRead(path); + auto file = FS->openFileForRead(path); if (!file) return; auto status = (*file)->status(); @@ -679,7 +681,7 @@ SwiftDependencyTracker::createTreeFromDependencies() { llvm::SmallVector Files; for (auto &file : TrackedFiles) { auto includeTreeFile = clang::cas::IncludeTree::File::create( - FS->getCAS(), file.first, file.second.FileRef); + *CAS, file.first, file.second.FileRef); if (!includeTreeFile) { return llvm::createStringError("CASFS createTree failed for " + file.first + ": " + @@ -691,7 +693,7 @@ SwiftDependencyTracker::createTreeFromDependencies() { } auto includeTreeList = - clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {}); + clang::cas::IncludeTree::FileList::create(*CAS, Files, {}); if (!includeTreeList) return llvm::createStringError("casfs include-tree filelist error: " + toString(includeTreeList.takeError())); @@ -719,14 +721,8 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService( CAS = Instance.getSharedCASInstance(); ActionCache = Instance.getSharedCacheInstance(); - auto CachingFS = - llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore()); - if (!CachingFS) { - Instance.getDiags().diagnose(SourceLoc(), diag::error_cas_fs_creation, - toString(CachingFS.takeError())); - return true; - } - CacheFS = std::move(*CachingFS); + CacheFS = llvm::cas::createCASProvidingFileSystem( + CAS, llvm::vfs::createPhysicalFileSystem()); // Setup prefix mapping. auto &ScannerPrefixMapper = @@ -744,19 +740,14 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService( Mapper->sort(); } - UseClangIncludeTree = - Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree; const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat = - UseClangIncludeTree - ? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree - : clang::tooling::dependencies::ScanningOutputFormat::FullTree; + clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree; ClangScanningService.emplace( clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, - ClangScanningFormat, - Instance.getInvocation().getCASOptions().CASOpts, + ClangScanningFormat, Instance.getInvocation().getCASOptions().CASOpts, Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(), - UseClangIncludeTree ? nullptr : CacheFS, + /*CachingOnDiskFileSystem=*/nullptr, // The current working directory optimization (off by default) // should not impact CAS. We set the optization to all to be // consistent with the non-CAS case. diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index d130bd18554cd..c8f204989d815 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1210,7 +1210,7 @@ std::optional> ClangImporter::getClangCC1Arguments( clang::frontend::ActionKind::GenerateModule || CI->getFrontendOpts().ProgramAction == clang::frontend::ActionKind::GeneratePCH) && - ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { + !ctx.CASOpts.ClangIncludeTree.empty()) { CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree; CI->getFrontendOpts().Inputs.clear(); } @@ -1270,7 +1270,7 @@ std::optional> ClangImporter::getClangCC1Arguments( std::vector FilteredModuleMapFiles; for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) { - if (ctx.ClangImporterOpts.UseClangIncludeTree) { + if (ctx.CASOpts.HasImmutableFileSystem) { // There is no need to add any module map file here. Issue a warning and // drop the option. Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile); @@ -2755,7 +2755,6 @@ ClangImporter::Implementation::Implementation( !ctx.ClangImporterOpts.BridgingHeader.empty()), DisableOverlayModules(ctx.ClangImporterOpts.DisableOverlayModules), EnableClangSPI(ctx.ClangImporterOpts.EnableClangSPI), - UseClangIncludeTree(ctx.ClangImporterOpts.UseClangIncludeTree), importSymbolicCXXDecls( ctx.LangOpts.hasFeature(Feature::ImportSymbolicCXXDecls)), IsReadingBridgingPCH(false), @@ -4284,11 +4283,9 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const { auto &CGOpts = instance.getCodeGenOpts(); CGOpts.DebugCompilationDir.clear(); - if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) { // FileSystemOptions. - auto &FSOpts = instance.getFileSystemOpts(); - FSOpts.WorkingDir.clear(); - } + auto &FSOpts = instance.getFileSystemOpts(); + FSOpts.WorkingDir.clear(); if (!Impl.SwiftContext.SearchPathOpts.ScannerPrefixMapper.empty()) { // Remap all the paths if requested. diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index be86dc11b4a87..ac15ed23c0bf5 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -147,8 +147,8 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( // Swift frontend option for input file path (Foo.modulemap). swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile)); - // Handle VFSOverlay. If include tree is used, there is no need for overlay. - if (!ctx.ClangImporterOpts.UseClangIncludeTree) { + // Handle VFSOverlay. If caching is enabled, there is no need for overlay. + if (!ctx.CASOpts.EnableCaching) { for (auto &overlay : ctx.SearchPathOpts.VFSOverlayFiles) { swiftArgs.push_back("-vfsoverlay"); swiftArgs.push_back(remapPath(overlay)); @@ -217,12 +217,6 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( ctx.CASOpts.enumerateCASConfigurationFlags( [&](StringRef Arg) { swiftArgs.push_back(Arg.str()); }); - if (!RootID.empty()) { - swiftArgs.push_back("-no-clang-include-tree"); - swiftArgs.push_back("-cas-fs"); - swiftArgs.push_back(RootID); - } - if (!IncludeTree.empty()) { swiftArgs.push_back("-clang-include-tree-root"); swiftArgs.push_back(IncludeTree); @@ -326,11 +320,6 @@ void ClangImporter::getBridgingHeaderOptions( swiftArgs.push_back("-clang-include-tree-root"); swiftArgs.push_back(*Tree); } - if (auto CASFS = deps.CASFileSystemRootID) { - swiftArgs.push_back("-no-clang-include-tree"); - swiftArgs.push_back("-cas-fs"); - swiftArgs.push_back(*CASFS); - } } ModuleDependencyVector diff --git a/lib/ClangImporter/ImporterImpl.h b/lib/ClangImporter/ImporterImpl.h index 5a4029caa530e..b718cc679b0e2 100644 --- a/lib/ClangImporter/ImporterImpl.h +++ b/lib/ClangImporter/ImporterImpl.h @@ -474,7 +474,6 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation const bool BridgingHeaderExplicitlyRequested; const bool DisableOverlayModules; const bool EnableClangSPI; - const bool UseClangIncludeTree; bool importSymbolicCXXDecls; bool IsReadingBridgingPCH; diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 615b0e79265fe..36520fd6f8d49 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -316,8 +316,6 @@ class ExplicitModuleDependencyResolver { } // Collect CAS deppendencies from clang modules. - if (!clangDepDetails.CASFileSystemRootID.empty()) - rootIDs.push_back(clangDepDetails.CASFileSystemRootID); if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) { if (addIncludeTree(clangDepDetails.CASClangIncludeTreeRootID)) return true; @@ -521,11 +519,6 @@ class ExplicitModuleDependencyResolver { if (resolvingDepInfo.isSwiftInterfaceModule() || resolvingDepInfo.isSwiftSourceModule()) { // Update with casfs option. - for (auto rootID : rootIDs) { - commandline.push_back("-cas-fs"); - commandline.push_back(rootID); - } - if (computeCASFileSystem(dependencyInfoCopy)) return true; } @@ -679,7 +672,6 @@ class ExplicitModuleDependencyResolver { const ModuleDependencyInfo &resolvingDepInfo; std::optional tracker; - std::vector rootIDs; std::vector fileListRefs; std::vector commandline; std::vector bridgingHeaderBuildCmd; diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index c181854f7708d..151db67c8f282 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -449,44 +449,12 @@ static llvm::Error createCASObjectNotFoundError(const llvm::cas::CASID &ID) { "CASID missing from Object Store " + ID.toString()); } -static Expected mergeCASFileSystem(ObjectStore &CAS, - ArrayRef FSRoots) { - llvm::cas::HierarchicalTreeBuilder Builder; - for (auto &Root : FSRoots) { - auto ID = CAS.parseID(Root); - if (!ID) - return ID.takeError(); - - auto Ref = CAS.getReference(*ID); - if (!Ref) - return createCASObjectNotFoundError(*ID); - Builder.pushTreeContent(*Ref, ""); - } - - auto NewRoot = Builder.create(CAS); - if (!NewRoot) - return NewRoot.takeError(); - - return NewRoot->getRef(); -} - Expected> -createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, - const std::string &IncludeTree, +createCASFileSystem(ObjectStore &CAS, const std::string &IncludeTree, const std::string &IncludeTreeFileList) { - assert(!FSRoots.empty() || !IncludeTree.empty() || + assert(!IncludeTree.empty() || !IncludeTreeFileList.empty() && "no root ID provided"); - auto NewRoot = mergeCASFileSystem(CAS, FSRoots); - if (!NewRoot) - return NewRoot.takeError(); - - auto FS = createCASFileSystem(CAS, CAS.getID(*NewRoot)); - if (!FS) - return FS.takeError(); - - auto CASFS = makeIntrusiveRefCnt(std::move(*FS)); - if (!IncludeTree.empty()) { auto ID = CAS.parseID(IncludeTree); if (!ID) @@ -507,7 +475,7 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, if (!ITFS) return ITFS.takeError(); - CASFS->pushOverlay(*ITFS); + return *ITFS; } if (!IncludeTreeFileList.empty()) { @@ -526,10 +494,10 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, if (!ITFS) return ITFS.takeError(); - CASFS->pushOverlay(std::move(*ITFS)); + return *ITFS; } - return CASFS; + return nullptr; } std::vector remapPathsFromCommandLine( diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 4d26cb8edbd0e..d081a3b685254 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -761,8 +761,6 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, std::string(Value)); } - for (const auto &A : Args.getAllArgValues(OPT_cas_fs)) - Opts.CASFSRootIDs.emplace_back(A); if (auto *A = Args.getLastArg(OPT_clang_include_tree_root)) Opts.ClangIncludeTree = A->getValue(); if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist)) @@ -774,8 +772,7 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, if (const Arg*A = Args.getLastArg(OPT_bridging_header_pch_key)) Opts.BridgingHeaderPCHCacheKey = A->getValue(); - if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTree.empty() || - !Opts.ClangIncludeTreeFileList.empty()) + if (!Opts.ClangIncludeTree.empty() || !Opts.ClangIncludeTreeFileList.empty()) Opts.HasImmutableFileSystem = true; return false; @@ -2131,10 +2128,6 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args, // Forward the FrontendOptions to clang importer option so it can be // accessed when creating clang module compilation invocation. if (CASOpts.EnableCaching) { - // Only set UseClangIncludeTree when caching is enabled since it is not - // useful in non-caching context. - Opts.UseClangIncludeTree |= !Args.hasArg(OPT_no_clang_include_tree); - Opts.HasClangIncludeTreeRoot |= Args.hasArg(OPT_clang_include_tree_root); // Caching requires direct clang import cc1 scanning. Opts.ClangImporterDirectCC1Scan = true; } diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 40149a5ddf03f..9eb9cc11c0fe7 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -630,11 +630,9 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() { } if (Invocation.getCASOptions().requireCASFS()) { - if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTree.empty() || - !CASOpts.ClangIncludeTreeFileList.empty()) { + if (Invocation.getCASOptions().HasImmutableFileSystem) { // Set up CASFS as BaseFS. - auto FS = createCASFileSystem(*CAS, CASOpts.CASFSRootIDs, - CASOpts.ClangIncludeTree, + auto FS = createCASFileSystem(*CAS, CASOpts.ClangIncludeTree, CASOpts.ClangIncludeTreeFileList); if (!FS) { Diagnostics.diagnose(SourceLoc(), diag::error_cas_fs_creation, @@ -682,8 +680,8 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() { } } llvm::IntrusiveRefCntPtr OverlayVFS = - new llvm::vfs::OverlayFileSystem(SourceMgr.getFileSystem()); - OverlayVFS->pushOverlay(MemFS); + new llvm::vfs::OverlayFileSystem(MemFS); + OverlayVFS->pushOverlay(SourceMgr.getFileSystem()); SourceMgr.setFileSystem(std::move(OverlayVFS)); } @@ -967,10 +965,10 @@ std::string CompilerInstance::getBridgingHeaderPath() const { } bool CompilerInstance::setUpInputs() { - // There is no input file when building PCM using ClangIncludeTree. + // There is no input file when building PCM using Caching. if (Invocation.getFrontendOptions().RequestedAction == FrontendOptions::ActionType::EmitPCM && - Invocation.getClangImporterOptions().HasClangIncludeTreeRoot) + Invocation.getCASOptions().EnableCaching) return false; // Adds to InputSourceCodeBufferIDs, so may need to happen before the diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 4c85d49f18d69..002918eb63c6d 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1815,13 +1815,6 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( casOpts.HasImmutableFileSystem; casOpts.enumerateCASConfigurationFlags( [&](StringRef Arg) { GenericArgs.push_back(ArgSaver.save(Arg)); }); - // ClangIncludeTree is default on when caching is enabled. - genericSubInvocation.getClangImporterOptions().UseClangIncludeTree = true; - } - - if (!clangImporterOpts.UseClangIncludeTree) { - genericSubInvocation.getClangImporterOptions().UseClangIncludeTree = false; - GenericArgs.push_back("-no-clang-include-tree"); } } @@ -2521,8 +2514,7 @@ struct ExplicitCASModuleLoader::Implementation { }; for (auto &entry : ExplicitClangModuleMap) { const auto &moduleMapPath = entry.getValue().moduleMapPath; - if (!moduleMapPath.empty() && - !Ctx.ClangImporterOpts.UseClangIncludeTree && + if (!moduleMapPath.empty() && !Ctx.CASOpts.EnableCaching && moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) { moduleMapsSeen.insert(moduleMapPath); extraClangArgs.push_back( diff --git a/test/CAS/cas_fs.swift b/test/CAS/cas_fs.swift deleted file mode 100644 index a4b0e2cab6ea5..0000000000000 --- a/test/CAS/cas_fs.swift +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: %empty-directory(%t) -// RUN: mkdir -p %t/empty -// RUN: mkdir -p %t/cas - -// RUN: llvm-cas --cas %t/cas --ingest %t/empty > %t/empty.casid -// RUN: not %target-swift-frontend -typecheck -cache-compile-job -cas-fs @%t/empty.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-INPUTS -// NO-INPUTS: error: error opening input file - -// RUN: llvm-cas --cas %t/cas --ingest %s > %t/source.casid -// RUN: not %target-swift-frontend -typecheck -cache-compile-job -cas-fs @%t/source.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-RESOURCES -// NO-RESOURCES: error: unable to load standard library - -/// Ingest the resource directory to satisfy the file system requirement. Also switch CWD to resource dir. -// RUN: llvm-cas --cas %t/cas --merge @%t/source.casid %test-resource-dir > %t/full.casid -// RUN: cd %test-resource-dir -// RUN: %target-swift-frontend -typecheck -cache-compile-job -cas-fs @%t/full.casid -cas-path %t/cas %s - -/// Try clang importer. -// RUN: not %target-swift-frontend -typecheck -cache-compile-job -cas-fs @%t/full.casid -cas-path %t/cas %s -import-objc-header %S/Inputs/objc.h 2>&1 | %FileCheck %s --check-prefix NO-BRIDGING-HEADER -// NO-BRIDGING-HEADER: error: bridging header - -// RUN: llvm-cas --cas %t/cas --merge @%t/full.casid %S/Inputs/objc.h > %t/bridging_header.casid -// RUN: %target-swift-frontend -typecheck -cache-compile-job -cas-fs @%t/bridging_header.casid -cas-path %t/cas %s -import-objc-header %S/Inputs/objc.h - -/// Clean the CAS to save space. -// RUN: %empty-directory(%t) - -func testFunc() {} diff --git a/test/CAS/cas_output_backend.swift b/test/CAS/cas_output_backend.swift index 603bc5d58adbb..992c4b6425f7c 100644 --- a/test/CAS/cas_output_backend.swift +++ b/test/CAS/cas_output_backend.swift @@ -2,7 +2,7 @@ // RUN: mkdir -p %t/cas // RUN: not %target-swift-frontend -c -cache-compile-job -cas-path %t/cas %s -o %t/test.o 2>&1 | %FileCheck %s --check-prefix=NO-CASFS -// NO-CASFS: caching is enabled without -cas-fs option +// NO-CASFS: caching is enabled without CAS file-system options // RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \ // RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ diff --git a/test/CAS/module_deps.swift b/test/CAS/module_deps.swift deleted file mode 100644 index 509f3d2984b3b..0000000000000 --- a/test/CAS/module_deps.swift +++ /dev/null @@ -1,197 +0,0 @@ -// REQUIRES: objc_interop - -// RUN: %empty-directory(%t) -// RUN: mkdir -p %t/clang-module-cache -// RUN: mkdir -p %t/cas - -// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cross-import-overlays -cache-compile-job -cas-path %t/cas -no-clang-include-tree -scanner-output-dir %t -auto-bridging-header-chaining -// Check the contents of the JSON output -// RUN: %validate-json %t/deps.json &>/dev/null -// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json - -// Check the contents of the JSON output -// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json - -// Check the make-style dependencies file -// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d - -// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cross-import-overlays -cache-compile-job -cas-path %t/cas -no-clang-include-tree -scanner-output-dir %t -auto-bridging-header-chaining -// RUN: %validate-json %t/deps.json &>/dev/null -// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json - -// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples. -// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/clang-module-cache %s -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cross-import-overlays -clang-target %target-cpu-apple-macosx10.14 -cache-compile-job -cas-path %t/cas -no-clang-include-tree - -/// check cas-fs content -// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json E casFSRootID > %t/E_fs.casid -// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/E_fs.casid | %FileCheck %s -check-prefix FS_ROOT_E -// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid -// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F - -// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps commandLine > %t/deps.cmd -// RUN: %FileCheck %s -check-prefix MAIN_CMD -input-file=%t/deps.cmd - -// FS_ROOT_E-DAG: E.swiftinterface -// FS_ROOT_E-DAG: SDKSettings.json - -// FS_ROOT_F: CHeaders/A.h -// FS_ROOT_F: CHeaders/B.h -// FS_ROOT_F: CHeaders/C.h -// FS_ROOT_F: CHeaders/D.h -// FS_ROOT_F: CHeaders/F.h -// FS_ROOT_F: CHeaders/G.h -// FS_ROOT_F: CHeaders/H.h -// FS_ROOT_F: CHeaders/I.h -// FS_ROOT_F: CHeaders/X.h -// FS_ROOT_F: CHeaders/module.modulemap - -// MAIN_CMD: -direct-clang-cc1-module-build -// MAIN_CMD: -cas-fs -// MAIN_CMD-NOT: -clang-include-tree-root - -import C -import E -import G -import SubE - -// CHECK: "mainModuleName": "deps" - -/// --------Main module -// CHECK-LABEL: "modulePath": "deps.swiftmodule", -// CHECK-NEXT: sourceFiles -// CHECK-NEXT: module_deps.swift -// CHECK-NEXT: ], -// CHECK-NEXT: "directDependencies": [ -// CHECK-DAG: "clang": "C" -// CHECK-DAG: "swift": "E" -// CHECK-DAG: "swift": "G" -// CHECK-DAG: "swift": "SubE" -// CHECK-DAG: "swift": "Swift" -// CHECK-DAG: "swift": "SwiftOnoneSupport" -// CHECK-DAG: "swift": "_Concurrency" -// CHECK-DAG: "swift": "_cross_import_E" -// CHECK: ], -// CHECK: "commandLine": -// CHECK: "casFSRootID": -// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from" -// CHECK: "bridgingHeader": -// CHECK-NEXT: "path": -// CHECK-SAME: Bridging.h - -// CHECK-NEXT: "sourceFiles": -// CHECK-NEXT: ChainedBridgingHeader.h -// CHECK-NEXT: Bridging.h -// CHECK-NEXT: BridgingOther.h - -// CHECK: "moduleDependencies": [ -// CHECK-NEXT: "F" -// CHECK-NEXT: ] - -// CHECK: "swiftOverlayDependencies": [ -// CHECK-DAG: "swift": "A" -// CHECK-DAG: "swift": "F" - -/// --------Clang module C -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm", - -// CHECK: "sourceFiles": [ -// CHECK-DAG: module.modulemap -// CHECK-DAG: C.h - -// CHECK: directDependencies -// CHECK-NEXT: { -// CHECK-NEXT: "clang": "B" - -// CHECK: "moduleMapPath" -// CHECK-SAME: module.modulemap - -// CHECK: "contextHash" -// CHECK-SAME: "{{.*}}" - -/// --------Clang module B -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm", -// CHECK: "contextHash": "[[B_CONTEXT:.*]]", -// CHECK: "-o" -// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm - -// Check make-style dependencies -// CHECK-MAKE-DEPS: module_deps.swift -// CHECK-MAKE-DEPS-SAME: A.swiftinterface -// CHECK-MAKE-DEPS-SAME: G.swiftinterface -// CHECK-MAKE-DEPS-SAME: B.h -// CHECK-MAKE-DEPS-SAME: Bridging.h -// CHECK-MAKE-DEPS-SAME: BridgingOther.h -// CHECK-MAKE-DEPS-SAME: module.modulemap - -/// --------Swift module F -// CHECK: "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule", -// CHECK-NEXT: "sourceFiles": [ -// CHECK-NEXT: ], -// CHECK-NEXT: "directDependencies": [ -// CHECK-NEXT: { -// CHECK-DAG: "clang": "F" -// CHECK-DAG: "swift": "Swift" -// CHECK-DAG: "swift": "SwiftOnoneSupport" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK: "details": -// CHECK: "moduleCacheKey": - -/// --------Swift module A -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule", - -// CHECK: directDependencies -// CHECK-NEXT: { -// CHECK-DAG: "clang": "A" -// CHECK-DAG: "swift": "Swift" -// CHECK-DAG: "swift": "SwiftOnoneSupport" -// CHECK-NEXT: } -// CHECK: "details": -// CHECK: "moduleCacheKey": - -/// --------Swift module G -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule" -// CHECK: "directDependencies" -// CHECK-NEXT: { -// CHECK-DAG: "clang": "G" -// CHECK-DAG: "swift": "Swift" -// CHECK-DAG: "swift": "SwiftOnoneSupport" -// CHECK: ], -// CHECK-NEXT: "linkLibraries": [ -// CHECK: "details": { - -// CHECK: "commandLine": [ -// CHECK: "-compile-module-from-interface" -// CHECK: "-target" -// CHECK: "-cache-compile-job" -// CHECK: "-cas-path" -// CHECK: "-module-name" -// CHECK: "G" -// CHECK: "-swift-version" -// CHECK: "5" -// CHECK: ], -// CHECK: "contextHash": "{{.*}}", - -/// --------Swift module E -// CHECK: "swift": "E" -// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule" -// CHECK: "directDependencies" -// CHECK-NEXT: { -// CHECK-NEXT: "swift": "Swift" - -// CHECK: "moduleInterfacePath" -// CHECK-SAME: E.swiftinterface - -/// --------Swift module Swift -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule", - -// CHECK: directDependencies -// CHECK-NEXT: { -// CHECK-NEXT: "clang": "SwiftShims" - -/// --------Clang module SwiftShims -// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm", -// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]", -// CHECK: "-o" -// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm -// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path" From ed4e035f97790082baa5c50b97d09c5cdf175f46 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 8 May 2025 13:03:12 -0700 Subject: [PATCH 3/5] [Caching] Remove capture clang extra files After removing the CASFS implementation for clang modules, there is no need to capture clang extra file that sets up the VFS for the clang modules since all content imported by ClangImporter is dependency scanned and available via include-tree. This saves more ClangImporter instance when caching is enabled. Update the test to check that clang content found via `-Xcc` VFS options can currently work without capture the headermaps and vfs overlays. (cherry picked from commit 1506a0d495b4f40a1489cccfed09ed45a460fd18) --- include/swift/ClangImporter/ClangImporter.h | 5 -- lib/ClangImporter/ClangImporter.cpp | 21 ----- .../ModuleDependencyScanner.cpp | 8 -- lib/Frontend/Frontend.cpp | 9 -- lib/Serialization/ScanningLoaders.cpp | 10 --- test/CAS/module_deps_clang_extras.swift | 85 +++++++++++++++---- 6 files changed, 69 insertions(+), 69 deletions(-) diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 1745ca146d0d8..a3d0374e40e5d 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -472,11 +472,6 @@ class ClangImporter final : public ClangModuleLoader { /// Reads the original source file name from PCH. std::string getOriginalSourceFile(StringRef PCHFilename); - /// Add clang dependency file names. - /// - /// \param files The list of file to append dependencies to. - void addClangInvovcationDependencies(std::vector &files); - /// Makes a temporary replica of the ClangImporter's CompilerInstance, reads a /// module map into the replica and emits a PCM file for one of the modules it /// declares. Delegates to clang for everything except construction of the diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index c8f204989d815..67499d93f2e44 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1062,27 +1062,6 @@ std::string ClangImporter::getOriginalSourceFile(StringRef PCHFilename) { Impl.Instance->getPCHContainerReader(), Impl.Instance->getDiagnostics()); } -void ClangImporter::addClangInvovcationDependencies( - std::vector &files) { - auto addFiles = [&files](const auto &F) { - files.insert(files.end(), F.begin(), F.end()); - }; - auto &invocation = *Impl.Invocation; - // FIXME: Add file dependencies that are not accounted. The long term solution - // is to do a dependency scanning for clang importer and use that directly. - SmallVector HeaderMapFileNames; - Impl.Instance->getPreprocessor().getHeaderSearchInfo().getHeaderMapFileNames( - HeaderMapFileNames); - addFiles(HeaderMapFileNames); - addFiles(invocation.getHeaderSearchOpts().VFSOverlayFiles); - // FIXME: Should not depend on working directory. Build system/swift driver - // should not pass working directory here but if that option is passed, - // repect that and add that into CASFS. - auto CWD = invocation.getFileSystemOpts().WorkingDir; - if (!CWD.empty()) - files.push_back(CWD); -} - std::optional ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash, bool &isExplicit) { diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 358f4904ecfd7..5f8cae82a6f9e 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -595,14 +595,6 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) { auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule( {}, buildCommands, {}, {}, {}); - if (ScanASTContext.CASOpts.EnableCaching) { - std::vector clangDependencyFiles; - clangImporter->addClangInvovcationDependencies(clangDependencyFiles); - llvm::for_each(clangDependencyFiles, [&](std::string &file) { - mainDependencies.addAuxiliaryFile(file); - }); - } - llvm::StringSet<> alreadyAddedModules; // Compute Implicit dependencies of the main module { diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 9eb9cc11c0fe7..17527cffd3a20 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -783,15 +783,6 @@ bool CompilerInstance::setUpModuleLoaders() { this->DefaultSerializedLoader = ISML.get(); Context->addModuleLoader(std::move(ISML)); - // When caching is enabled, we rely on ClangImporter for - // 'addClangInvovcationDependencies' - if (Invocation.getCASOptions().EnableCaching) { - std::unique_ptr clangImporter = - ClangImporter::create(*Context, Invocation.getPCHHash(), - getDependencyTracker()); - Context->addModuleLoader(std::move(clangImporter), /*isClang*/ true); - } - return false; } diff --git a/lib/Serialization/ScanningLoaders.cpp b/lib/Serialization/ScanningLoaders.cpp index 516b834f580b7..2b90236952336 100644 --- a/lib/Serialization/ScanningLoaders.cpp +++ b/lib/Serialization/ScanningLoaders.cpp @@ -231,16 +231,6 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath, InPath, compiledCandidatesRefs, ArgsRefs, {}, {}, linkLibraries, isFramework, isStatic, {}, /*module-cache-key*/ "", UserModVer); - if (Ctx.CASOpts.EnableCaching) { - std::vector clangDependencyFiles; - auto clangImporter = - static_cast(Ctx.getClangModuleLoader()); - clangImporter->addClangInvovcationDependencies(clangDependencyFiles); - llvm::for_each(clangDependencyFiles, [&](std::string &file) { - Result->addAuxiliaryFile(file); - }); - } - // Walk the source file to find the import declarations. llvm::StringSet<> alreadyAddedModules; Result->addModuleImports(*sourceFile, alreadyAddedModules, diff --git a/test/CAS/module_deps_clang_extras.swift b/test/CAS/module_deps_clang_extras.swift index b0950f6e23502..e11709da3b839 100644 --- a/test/CAS/module_deps_clang_extras.swift +++ b/test/CAS/module_deps_clang_extras.swift @@ -4,46 +4,99 @@ // RUN: mkdir -p %t/clang-module-cache // RUN: mkdir -p %t/cas // RUN: split-file %s %t -// RUN: %hmaptool write %t/hmap.json %t/empty.hmap +// RUN: sed "s|DIR|%/t|g" %t/hmap.json.template > %t/hmap.json +// RUN: sed "s|DIR|%/t|g" %t/test.yaml.template > %t/test.yaml +// RUN: %hmaptool write %t/hmap.json %t/test.hmap // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache \ -// RUN: %t/Test.swift -o %t/deps.json -cache-compile-job -cas-path %t/cas \ -// RUN: -Xcc -fmodule-map-file=%t/module.modulemap -Xcc -ivfsoverlay -Xcc %t/empty.yaml \ -// RUN: -Xcc -I%t/empty.hmap +// RUN: %t/Test.swift -module-name Test -o %t/deps.json -cache-compile-job -cas-path %t/cas \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -Xcc -fmodule-map-file=%t/include/module.modulemap -Xcc -ivfsoverlay -Xcc %t/test.yaml \ +// RUN: -Xcc -I%t/test.hmap -module-load-mode prefer-serialized -scanner-output-dir %t \ +// RUN: -import-objc-header %t/Bridge.h -auto-bridging-header-chaining // RUN: %validate-json %t/deps.json &>/dev/null -// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps casFSRootID > %t/fs.casid +// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Test casFSRootID > %t/fs.casid // RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/fs.casid | %FileCheck %s -DDIR=%basename_t -check-prefix FS_ROOT -// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:Dummy clangIncludeTree > %t/tree.casid -// RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/tree.casid | %FileCheck %s -DDIR=%basename_t -check-prefix INCLUDE_TREE -// FS_ROOT: [[DIR]].tmp/empty.hmap -// FS_ROOT: [[DIR]].tmp/empty.yaml +// FS_ROOT: [[DIR]].tmp/hidden/Dummy.h +// FS_ROOT: [[DIR]].tmp/hidden/a.h +// FS_ROOT: [[DIR]].tmp/hidden/b.h -// INCLUDE_TREE: [[DIR]].tmp/Dummy.h +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd +// RUN: %swift_frontend_plain @%t/shim.cmd +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd +// RUN: %swift_frontend_plain @%t/dummy.cmd + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json bridgingHeader > %t/header.cmd +// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/objc.pch +// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-output-keys -- \ +// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/objc.pch > %t/keys.json +// RUN: %{python} %S/Inputs/ExtractOutputKey.py %t/keys.json > %t/key + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd +// RUN: echo "\"-import-objc-header\"" >> %t/MyApp.cmd +// RUN: echo "\"%t/objc.pch\"" >> %t/MyApp.cmd +// RUN: echo "\"-bridging-header-pch-key\"" >> %t/MyApp.cmd +// RUN: echo "\"@%t/key\"" >> %t/MyApp.cmd +// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd +// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd + +// RUN: %target-swift-frontend -cache-compile-job -module-name Test -O -cas-path %t/cas @%t/MyApp.cmd %t/Test.swift \ +// RUN: -emit-module -o %t/test.swiftmodule //--- Test.swift import Dummy func test() {} -//--- module.modulemap +//--- Bridge.h +#include "b.h" + +//--- hidden/module.modulemap module Dummy { umbrella header "Dummy.h" } -//--- Dummy.h +//--- hidden/Dummy.h +#include "a.h" void dummy(void); -//--- hmap.json +//--- hidden/a.h +/* empty file */ + +//--- hidden/b.h +/* empty file */ + +//--- hmap.json.template { - "mappings": {} + "mappings": { + "a.h": "DIR/hidden/a.h", + "b.h": "DIR/hidden/b.h" + } } -//--- empty.yaml +//--- test.yaml.template { "version": 0, "case-sensitive": "false", "use-external-names": true, - "roots": [] + "roots": [ + { + "type": "file", + "name": "DIR/include/module.modulemap", + "external-contents": "DIR/hidden/module.modulemap" + }, + { + "type": "file", + "name": "DIR/include/Dummy.h", + "external-contents": "DIR/hidden/Dummy.h" + }, + ] } From 19c0919cf7cdf296e1ed9331ddc71d436b40014f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 15 May 2025 11:41:21 -0700 Subject: [PATCH 4/5] [Caching] Clear benign CodeGen options when building modules Use `resetBenignCodeGenOptions()` from clang dependency scanner to clear the swift explicit module build cc1 arguments. This fixes the problem that CurrentWorkingDirectory is leaking through `-fcoverage-compilation-dir` that can cause extra module variants when caching is enabled. This also avoid the duplicating the logics for clearing CodeGen options inside Swift. rdar://151395300 (cherry picked from commit 531227d4b99d7df1591809e312b8e7dcfa6784cf) --- lib/ClangImporter/ClangImporter.cpp | 10 ++++++---- test/CAS/coverage-dir.swift | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 test/CAS/coverage-dir.swift diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 67499d93f2e44..b54fd10c7ebe5 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -90,6 +90,7 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" @@ -4258,11 +4259,12 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const { PPOpts.MacroIncludes.clear(); PPOpts.Includes.clear(); - // CodeGenOptions. - auto &CGOpts = instance.getCodeGenOpts(); - CGOpts.DebugCompilationDir.clear(); + // Clear benign CodeGenOptions. + clang::tooling::dependencies::resetBenignCodeGenOptions( + clang::frontend::ActionKind::GenerateModule, instance.getLangOpts(), + instance.getCodeGenOpts()); - // FileSystemOptions. + // FileSystemOptions. auto &FSOpts = instance.getFileSystemOpts(); FSOpts.WorkingDir.clear(); diff --git a/test/CAS/coverage-dir.swift b/test/CAS/coverage-dir.swift new file mode 100644 index 0000000000000..69d79c94831fb --- /dev/null +++ b/test/CAS/coverage-dir.swift @@ -0,0 +1,28 @@ +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -I %t/include \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/main.swift -o %t/deps.json -cache-compile-job -cas-path %t/cas -profile-coverage-mapping -profile-generate + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd +// RUN: %swift_frontend_plain @%t/shim.cmd +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json A > %t/A.cmd +// RUN: %swift_frontend_plain @%t/A.cmd + +// RUN: %FileCheck %s --input-file=%t/shim.cmd +// RUN: %FileCheck %s --input-file=%t/A.cmd + +// CHECK: -direct-clang-cc1-module-build +// CHECK-NOT: -fcoverage-compilation-dir + +//--- main.swift +import A +func test() { + a(); +} + +//--- include/A.swiftinterface +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A -O -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -user-module-version 1.0 +public func a() { } From db85088493886165aee1572b0a4b0504d9f87feb Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 12 May 2025 11:31:33 -0700 Subject: [PATCH 5/5] [DependencyScanning] Avoid extra cc1 arg round-trip to speed up bridging Use the underlying compiler invocation inside module scanning result to speed up the clang module dependency bridging. This avoids converting cc1 arguments to compiler invocation and back, just to modify the cc1 arguments needed for building PCM using swift-frontend. rdar://151705822 (cherry picked from commit 391446d4b3bef0026da66816b0f54ceaea0b9819) --- .../ClangModuleDependencyScanner.cpp | 91 ++++--------------- test/ScanDependencies/preserve_used_vfs.swift | 44 +++++++-- 2 files changed, 55 insertions(+), 80 deletions(-) diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index ac15ed23c0bf5..cc054f3de524a 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -81,25 +81,6 @@ std::vector ClangImporter::getClangDepScanningInvocationArguments( return commandLineArgs; } -static std::unique_ptr -getClangPrefixMapper(DependencyScanningTool &clangScanningTool, - ModuleDeps &clangModuleDep, - clang::CompilerInvocation &depsInvocation) { - std::unique_ptr Mapper; - if (clangModuleDep.IncludeTreeID) { - Mapper = std::make_unique(); - } else if (clangModuleDep.CASFileSystemRootID) { - assert(clangScanningTool.getCachingFileSystem()); - Mapper = std::make_unique( - clangScanningTool.getCachingFileSystem()); - } - - if (Mapper) - DepscanPrefixMapping::configurePrefixMapper(depsInvocation, *Mapper); - - return Mapper; -} - ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( const ASTContext &ctx, clang::tooling::dependencies::DependencyScanningTool &clangScanningTool, @@ -147,63 +128,30 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( // Swift frontend option for input file path (Foo.modulemap). swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile)); - // Handle VFSOverlay. If caching is enabled, there is no need for overlay. - if (!ctx.CASOpts.EnableCaching) { - for (auto &overlay : ctx.SearchPathOpts.VFSOverlayFiles) { - swiftArgs.push_back("-vfsoverlay"); - swiftArgs.push_back(remapPath(overlay)); - } - } - - // Add args reported by the scanner. - - // Round-trip clang args to canonicalize and clear the options that swift - // compiler doesn't need. - clang::CompilerInvocation depsInvocation; - clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(), - new clang::DiagnosticOptions(), - new clang::IgnoringDiagConsumer()); - - llvm::SmallVector clangArgs; - llvm::for_each( - clangModuleDep.getBuildArguments(), - [&](const std::string &Arg) { clangArgs.push_back(Arg.c_str()); }); - - bool success = clang::CompilerInvocation::CreateFromArgs( - depsInvocation, clangArgs, clangDiags); - (void)success; - assert(success && "clang option from dep scanner round trip failed"); - - // Create a prefix mapper that matches clang's configuration. - auto Mapper = - getClangPrefixMapper(clangScanningTool, clangModuleDep, depsInvocation); - - // Clear the cache key for module. The module key is computed from clang - // invocation, not swift invocation. - depsInvocation.getFrontendOpts().ModuleCacheKeys.clear(); - depsInvocation.getFrontendOpts().PathPrefixMappings.clear(); - depsInvocation.getFrontendOpts().OutputFile.clear(); + auto invocation = clangModuleDep.getUnderlyingCompilerInvocation(); + // Clear some options from clang scanner. + invocation.getMutFrontendOpts().ModuleCacheKeys.clear(); + invocation.getMutFrontendOpts().PathPrefixMappings.clear(); + invocation.getMutFrontendOpts().OutputFile.clear(); // Reset CASOptions since that should be coming from swift. - depsInvocation.getCASOpts() = clang::CASOptions(); - depsInvocation.getFrontendOpts().CASIncludeTreeID.clear(); + invocation.getMutCASOpts() = clang::CASOptions(); + invocation.getMutFrontendOpts().CASIncludeTreeID.clear(); // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option // from clang scanner and pass to swift. - for (auto overlay : depsInvocation.getHeaderSearchOpts().VFSOverlayFiles) { - if (llvm::is_contained(ctx.SearchPathOpts.VFSOverlayFiles, overlay)) - continue; - swiftArgs.push_back("-vfsoverlay"); - swiftArgs.push_back(overlay); + if (!ctx.CASOpts.EnableCaching) { + auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles; + for (auto overlay : overlayFiles) { + swiftArgs.push_back("-vfsoverlay"); + swiftArgs.push_back(overlay); + } + // Clear overlay files since they are forwarded from swift to clang. + overlayFiles.clear(); } - llvm::BumpPtrAllocator allocator; - llvm::StringSaver saver(allocator); - clangArgs.clear(); - depsInvocation.generateCC1CommandLine( - clangArgs, - [&saver](const llvm::Twine &T) { return saver.save(T).data(); }); - + // Add args reported by the scanner. + auto clangArgs = invocation.getCC1CommandLine(); llvm::for_each(clangArgs, addClangArg); // CASFileSystemRootID. @@ -221,10 +169,7 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( swiftArgs.push_back("-clang-include-tree-root"); swiftArgs.push_back(IncludeTree); } - - std::string mappedPCMPath = pcmPath; - if (Mapper) - Mapper->mapInPlace(mappedPCMPath); + std::string mappedPCMPath = remapPath(pcmPath); std::vector LinkLibraries; for (const auto &ll : clangModuleDep.LinkLibraries) diff --git a/test/ScanDependencies/preserve_used_vfs.swift b/test/ScanDependencies/preserve_used_vfs.swift index 4aef2c36d45b9..745c7918ef987 100644 --- a/test/ScanDependencies/preserve_used_vfs.swift +++ b/test/ScanDependencies/preserve_used_vfs.swift @@ -1,17 +1,45 @@ // REQUIRES: objc_interop // RUN: %empty-directory(%t) // RUN: %empty-directory(%t/module-cache) -// RUN: %empty-directory(%t/redirects) +// RUN: %empty-directory(%t/inputs) // RUN: split-file %s %t -// RUN: sed -e "s|OUT_DIR|%t/redirects|g" -e "s|IN_DIR|%S/Inputs/CHeaders|g" %t/overlay_template.yaml > %t/overlay.yaml +// RUN: sed -e "s|OUT_DIR|%t/redirects|g" -e "s|IN_DIR|%t/inputs|g" %t/overlay_template.yaml > %t/overlay.yaml -// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-cache-path %t/module-cache %t/test.swift -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -Xcc -ivfsoverlay -Xcc %t/overlay.yaml +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-cache-path %t/module-cache %t/test.swift -o %t/deps.json -I %t/inputs -I %S/Inputs/Swift -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -Xcc -ivfsoverlay -Xcc %t/overlay.yaml // RUN: %validate-json %t/deps.json | %FileCheck %s +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd +// RUN: %swift_frontend_plain @%t/shim.cmd +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json Swift > %t/swift.cmd +// RUN: %swift_frontend_plain @%t/swift.cmd +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json SwiftOnoneSupport > %t/onone.cmd +// RUN: %swift_frontend_plain @%t/onone.cmd +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:F > %t/F.cmd +// RUN: %swift_frontend_plain @%t/F.cmd +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json F > %t/SwiftF.cmd +// RUN: %swift_frontend_plain @%t/SwiftF.cmd + +// RUN: %{python} %S/../CAS/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd +// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd +// RUN: echo "\"%t/map.json\"" >> %t/MyApp.cmd + +// RUN: %target-swift-frontend @%t/MyApp.cmd %t/test.swift -Xcc -ivfsoverlay -Xcc %t/overlay.yaml \ +// RUN: -emit-module -o %t/Test.swiftmodule + //--- redirects/RedirectedF.h void funcRedirectedF(void); +//--- redirects/modulemap +module F { + header "F_2.h" + export * +} + //--- overlay_template.yaml { 'version': 0, @@ -20,8 +48,11 @@ void funcRedirectedF(void); { 'name': 'IN_DIR', 'type': 'directory', 'contents': [ - { 'name': 'F.h', 'type': 'file', + { 'name': 'F_2.h', 'type': 'file', 'external-contents': 'OUT_DIR/RedirectedF.h' + }, + { 'name': 'module.modulemap', 'type': 'file', + 'external-contents': 'OUT_DIR/modulemap' } ] }, @@ -31,6 +62,8 @@ void funcRedirectedF(void); //--- test.swift import F +func testF() { funcRedirectedF() } + // CHECK: "mainModuleName": "deps" /// --------Main module // CHECK-LABEL: "modulePath": "deps.swiftmodule", @@ -68,7 +101,4 @@ import F // CHECK: "commandLine": [ // CHECK: "-vfsoverlay", // CHECK-NEXT: "{{.*}}{{/|\\}}preserve_used_vfs.swift.tmp{{/|\\}}overlay.yaml", -// CHECK: "-ivfsoverlay", -// CHECK-NEXT: "-Xcc", -// CHECK-NEXT: "{{.*}}{{/|\\}}preserve_used_vfs.swift.tmp{{/|\\}}overlay.yaml", // CHECK: ],