diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index ef5b3e40d07a2..827b1bef3ed4b 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -585,10 +585,98 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); CINDEX_LINKAGE CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); -CINDEX_LINKAGE -CXCStringArray - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( - CXDependencyScannerService); +/** + * The kind of scanning file system cache out-of-date entries. + */ +typedef enum { + /** + * The entry is negatively stat cached (which indicates the file did not exist + * the first time it was looked up during scanning), but the cached file + * exists on the underlying file system. + */ + NegativelyCached, + + /** + * The entry indicates that for the cached file, its cached size + * is different from its size reported by the underlying file system. + */ + SizeChanged +} CXDepScanFSCacheOutOfDateKind; + +/** + * The opaque object that contains the scanning file system cache's out-of-date + * entires. + */ +typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet; + +/** + * The opaque object that represents a single scanning file system cache's out- + * of-date entry. + */ +typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry; + +/** + * Returns all the file system cache out-of-date entries given a + * \c CXDependencyScannerService . + * + * This function is intended to be called when the build has finished, + * and the \c CXDependencyScannerService instance is about to be disposed. + * + * The \c CXDependencyScannerService instance owns the strings used + * by the out-of-date entries and should be disposed after the + * out-of-date entries are used and disposed. + */ +CXDepScanFSOutOfDateEntrySet +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + CXDependencyScannerService S); + +/** + * Returns the number of out-of-date entries contained in a + * \c CXDepScanFSOutOfDateEntrySet . + */ +size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + CXDepScanFSOutOfDateEntrySet Entries); + +/** + * Returns the out-of-date entry at offset \p Idx of the \c + * CXDepScanFSOutOfDateEntrySet instance. + */ +CXDepScanFSOutOfDateEntry +clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry( + CXDepScanFSOutOfDateEntrySet Entries, size_t Idx); + +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind. + */ +CXDepScanFSCacheOutOfDateKind +clang_experimental_DepScanFSCacheOutOfDateEntry_getKind( + CXDepScanFSOutOfDateEntry Entry); + +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path. + */ +CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath( + CXDepScanFSOutOfDateEntry Entry); + +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, + * returns the cached size. + */ +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize( + CXDepScanFSOutOfDateEntry Entry); + +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, + * returns the actual size on the underlying file system. + */ +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize( + CXDepScanFSOutOfDateEntry Entry); + +/** + * Dispose the \c CXDepScanFSOutOfDateEntrySet instance. + */ +void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( + CXDepScanFSOutOfDateEntrySet Entries); /** * Options used to generate a reproducer. diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 06e5633401b84..7c836f211a004 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -131,7 +131,11 @@ DependencyScanningFilesystemSharedCache::getOutOfDateEntries( // later. The cache entry is not invalidated (as we have no good // way to do it now), which may lead to missing file build errors. InvalidDiagInfo.emplace_back(Path.data()); - } else { + } else if (Status->getType() == + llvm::sys::fs::file_type::regular_file) { + // We only check regular files. Directory files sizes could change due + // to content changes, and reporting directory size changes can lead + // to false positives. llvm::vfs::Status CachedStatus = Entry->getStatus(); uint64_t CachedSize = CachedStatus.getSize(); uint64_t ActualSize = Status->getSize(); diff --git a/clang/test/ClangScanDeps/error-c-api.cpp b/clang/test/ClangScanDeps/error-c-api.cpp index cfd1bcbd3a75c..a9e9b3831efce 100644 --- a/clang/test/ClangScanDeps/error-c-api.cpp +++ b/clang/test/ClangScanDeps/error-c-api.cpp @@ -4,4 +4,4 @@ // CHECK: error: failed to get dependencies // CHECK-NEXT: 'missing.h' file not found -// CHECK-NEXT: number of invalid negatively stat cached paths: 0 +// CHECK-NEXT: number of out of date file system cache entries: 0 diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 60d62cc5f3ca4..0d24df3a67955 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -916,12 +916,18 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_disposeDiagnostic(Diag); } - CXCStringArray InvalidNegativeStatCachedPaths = - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( + CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet = + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( Service); - llvm::errs() << "note: number of invalid negatively stat cached paths: " - << InvalidNegativeStatCachedPaths.Count << "\n"; + llvm::errs() + << "note: number of out of date file system cache entries: " + << clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + OutOfDateEntrySet) + << "\n"; + + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( + OutOfDateEntrySet); return 1; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 8a7dcb2246929..c701fced9e7bb 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -81,17 +81,11 @@ struct CStringsManager { return createCStringsRef(*OwnedStdStr.back()); } }; - -struct DependencyScannerService { - DependencyScanningService Service; - CStringsManager StrMgr{}; -}; } // end anonymous namespace DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions, CXDependencyScannerServiceOptions) - -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerService, +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService, CXDependencyScannerService) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker, CXDependencyScannerWorker) @@ -172,9 +166,9 @@ clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) { // FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now. CASOptions CASOpts; IntrusiveRefCntPtr FS; - return wrap(new DependencyScannerService{DependencyScanningService( + return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, unwrap(Format), CASOpts, - /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)}); + /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)); } ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { @@ -210,10 +204,10 @@ clang_experimental_DependencyScannerService_create_v1( FS = llvm::cantFail( llvm::cas::createCachingOnDiskFileSystem(CAS)); } - return wrap(new DependencyScannerService{DependencyScanningService( + return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), - unwrap(Opts)->OptimizeArgs)}); + unwrap(Opts)->OptimizeArgs)); } void clang_experimental_DependencyScannerService_dispose_v0( @@ -223,16 +217,16 @@ void clang_experimental_DependencyScannerService_dispose_v0( CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( CXDependencyScannerService S) { - ScanningOutputFormat Format = unwrap(S)->Service.getFormat(); + ScanningOutputFormat Format = unwrap(S)->getFormat(); bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree || Format == ScanningOutputFormat::FullIncludeTree; llvm::IntrusiveRefCntPtr FS = llvm::vfs::createPhysicalFileSystem(); if (IsIncludeTreeOutput) - FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->Service.getCAS(), + FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->getCAS(), std::move(FS)); - return wrap(new DependencyScanningWorker(unwrap(S)->Service, FS)); + return wrap(new DependencyScanningWorker(*unwrap(S), FS)); } void clang_experimental_DependencyScannerWorker_dispose_v0( @@ -566,43 +560,6 @@ CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { return unwrap(Graph)->getDiagnosticSet(); } -CXCStringArray -clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( - CXDependencyScannerService S) { - DependencyScanningService &Service = unwrap(S)->Service; - CStringsManager &StrMgr = unwrap(S)->StrMgr; - - // FIXME: CAS currently does not use the shared cache, and cannot produce - // the same diagnostics. We should add such a diagnostics to CAS as well. - if (Service.useCASFS()) - return {nullptr, 0}; - - DependencyScanningFilesystemSharedCache &SharedCache = - Service.getSharedCache(); - - // Note that it is critical that this FS is the same as the default virtual - // file system we pass to the DependencyScanningWorkers. - llvm::IntrusiveRefCntPtr FS = - llvm::vfs::createPhysicalFileSystem(); - - auto OutOfDateEntries = SharedCache.getOutOfDateEntries(*FS); - - // FIXME: replace this code with proper APIs that handles the - // OutOfDateEntries. - std::vector OutOfDatePaths; - for (const auto &E : OutOfDateEntries) - OutOfDatePaths.emplace_back(E.Path); - - // FIXME: This code here creates copies of strings from - // InvaidNegStatCachedPaths. It is acceptable because this C-API is expected - // to be called only at the end of a CXDependencyScannerService's lifetime. - // In other words, it is called very infrequently. We can change - // CStringsManager's interface to accommodate handling arbitrary StringRefs - // (which may not be null terminated) if we want to avoid copying. - return StrMgr.createCStringsOwned( - {OutOfDatePaths.begin(), OutOfDatePaths.end()}); -} - static std::string lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind MOK, void *MLOContext, std::variantsecond; } +namespace { +typedef std::vector + DependencyScannerFSOutOfDateEntrySet; + +typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry + DependencyScannerFSOutOfDateEntry; +} // namespace + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet, + CXDepScanFSOutOfDateEntrySet) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry, + CXDepScanFSOutOfDateEntry) + +CXDepScanFSOutOfDateEntrySet +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + CXDependencyScannerService S) { + DependencyScanningService &Service = *unwrap(S); + + // FIXME: CAS FS currently does not use the shared cache, and cannot produce + // the same diagnostics. We should add such a diagnostics to CAS as well. + if (Service.useCASFS()) + return nullptr; + + // Note that it is critical that this FS is the same as the default virtual + // file system we pass to the DependencyScanningWorkers. + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + + DependencyScannerFSOutOfDateEntrySet *OODEntrySet = + new DependencyScannerFSOutOfDateEntrySet(); + *OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS); + + return wrap(OODEntrySet); +} + +size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + CXDepScanFSOutOfDateEntrySet Entries) { + return unwrap(Entries)->size(); +} + +CXDepScanFSOutOfDateEntry +clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry( + CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) { + DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries); + return wrap(&(*EntSet)[Idx]); +} + +CXDepScanFSCacheOutOfDateKind +clang_experimental_DepScanFSCacheOutOfDateEntry_getKind( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + auto &Info = E->Info; + return std::visit( + llvm::makeVisitor( + [](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo + &Info) { return NegativelyCached; }, + [](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) { + return SizeChanged; + }), + Info); +} + +CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath( + CXDepScanFSOutOfDateEntry Entry) { + return cxstring::createRef(unwrap(Entry)->Path); +} + +static DependencyScannerFSOutOfDateEntry::SizeChangedInfo * +getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) { + auto *SizeInfo = + std::get_if(&E->Info); + assert(SizeInfo && "Wrong entry kind to get size changed info!"); + return SizeInfo; +} + +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + return getOutOfDateEntrySizeChangedInfo(E)->CachedSize; +} + +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + return getOutOfDateEntrySizeChangedInfo(E)->ActualSize; +} + +void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( + CXDepScanFSOutOfDateEntrySet Entries) { + delete unwrap(Entries); +} + namespace { struct DependencyScannerReproducerOptions { std::vector BuildArgs; diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 058eb50cde124..21a61c9d9fa28 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -577,7 +577,6 @@ LLVM_21 { clang_experimental_DepGraphModule_isCWDIgnored; clang_experimental_DepGraphModule_isInStableDirs; clang_getFullyQualifiedName; - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths; clang_experimental_DependencyScannerReproducerOptions_create; clang_experimental_DependencyScannerReproducerOptions_dispose; clang_experimental_DependencyScanner_generateReproducer; @@ -590,6 +589,14 @@ LLVM_21 { clang_Cursor_getGCCAssemblyNumClobbers; clang_Cursor_getGCCAssemblyClobber; clang_Cursor_isGCCAssemblyVolatile; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry; + clang_experimental_DepScanFSCacheOutOfDateEntry_getKind; + clang_experimental_DepScanFSCacheOutOfDateEntry_getPath; + clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize; + clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet; }; # Example of how to add a new symbol version entry. If you do add a new symbol diff --git a/clang/unittests/libclang/CMakeLists.txt b/clang/unittests/libclang/CMakeLists.txt index 4ef0cb12c7c07..9a041bced8a3d 100644 --- a/clang/unittests/libclang/CMakeLists.txt +++ b/clang/unittests/libclang/CMakeLists.txt @@ -1,6 +1,7 @@ add_clang_unittest(libclangTests LibclangTest.cpp DriverTest.cpp + DependencyScanningCAPITests.cpp LINK_LIBS libclang ) diff --git a/clang/unittests/libclang/DependencyScanningCAPITests.cpp b/clang/unittests/libclang/DependencyScanningCAPITests.cpp new file mode 100644 index 0000000000000..93b569aace6eb --- /dev/null +++ b/clang/unittests/libclang/DependencyScanningCAPITests.cpp @@ -0,0 +1,152 @@ +//===- unittests/libclang/DependencyScanningCAPITests.cpp ---------------- ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang-c/Dependencies.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "gtest/gtest.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { + // This test is setup to have two out-of-date file system cache entries, + // one is negatively stat cached, the other has its size changed. + // - `include/b.h` is negatively stat cached. + // - `include` (the directory) has its size changed, because `b.h` is added + // to it. + + CXDependencyScannerServiceOptions ServiceOptions = + clang_experimental_DependencyScannerServiceOptions_create(); + CXDependencyScannerService Service = + clang_experimental_DependencyScannerService_create_v1(ServiceOptions); + CXDependencyScannerWorker Worker = + clang_experimental_DependencyScannerWorker_create_v0(Service); + + // Set up the directory structure before scanning. + // - `/tmp/include/a.h` + // - `/tmp/include2/b.h` + llvm::SmallString<128> Dir; + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("tmp", Dir)); + + llvm::SmallString<128> Include = Dir; + llvm::sys::path::append(Include, "include"); + ASSERT_FALSE(llvm::sys::fs::create_directories(Include)); + + llvm::SmallString<128> Include2 = Dir; + llvm::sys::path::append(Include2, "include2"); + ASSERT_FALSE(llvm::sys::fs::create_directories(Include2)); + + llvm::SmallString<128> HeaderA = Include; + llvm::sys::path::append(HeaderA, "a.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderAFile(HeaderA, EC); + ASSERT_FALSE(EC); + } + + // Initially, we keep include/b.h missing and only create include2/b.h. + llvm::SmallString<128> HeaderB2 = Include2; + llvm::sys::path::append(HeaderB2, "b.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderB2File(HeaderB2, EC); + ASSERT_FALSE(EC); + } + + llvm::SmallString<128> TU = Dir; + llvm::sys::path::append(TU, "tu.c"); + { + std::error_code EC; + llvm::raw_fd_ostream TUFile(TU, EC); + ASSERT_FALSE(EC); + TUFile << R"( + #include "a.h" + #include "b.h" + ")"; + } + + const char *Argv[] = {"-c", TU.c_str(), "-I", Include.c_str(), + "-I", Include2.c_str()}; + size_t Argc = std::size(Argv); + CXDependencyScannerWorkerScanSettings ScanSettings = + clang_experimental_DependencyScannerWorkerScanSettings_create( + Argc, Argv, /*ModuleName=*/nullptr, /*WorkingDirectory=*/Dir.c_str(), + /*MLOContext=*/nullptr, /*MLO=*/nullptr); + + CXDepGraph *Graph = new CXDepGraph; + CXErrorCode ScanResult = + clang_experimental_DependencyScannerWorker_getDepGraph( + Worker, ScanSettings, Graph); + ASSERT_EQ(ScanResult, CXError_Success); + + // Change the size of include/a.h. + { + std::error_code EC; + llvm::raw_fd_ostream HeaderAFile(HeaderA, EC); + ASSERT_FALSE(EC); + HeaderAFile << "// New content!\n"; + } + + // Populate include/b.h. + llvm::SmallString<128> HeaderB = Include; + llvm::sys::path::append(HeaderB, "b.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderBFile(HeaderB, EC); + ASSERT_FALSE(EC); + } + + // Directory structure after the change. + // - `/tmp/include/` + // - `a.h` ==> size has changed. + // - `b.h` + // - `/tmp/include2/b.h` + + CXDepScanFSOutOfDateEntrySet Entries = + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + Service); + + size_t NumEntries = + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + Entries); + EXPECT_EQ(NumEntries, 2u); + + bool CheckedNegativelyCached = false; + bool CheckedSizeChanged = false; + for (size_t Idx = 0; Idx < NumEntries; Idx++) { + CXDepScanFSOutOfDateEntry Entry = + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(Entries, + Idx); + CXDepScanFSCacheOutOfDateKind Kind = + clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(Entry); + CXString Path = + clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(Entry); + ASSERT_TRUE(Kind == NegativelyCached || Kind == SizeChanged); + switch (Kind) { + case NegativelyCached: + EXPECT_STREQ(clang_getCString(Path), HeaderB.c_str()); + CheckedNegativelyCached = true; + break; + case SizeChanged: + EXPECT_STREQ(clang_getCString(Path), HeaderA.c_str()); + EXPECT_EQ( + clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(Entry), + 0u); + EXPECT_EQ( + clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(Entry), + 16u); + CheckedSizeChanged = true; + break; + } + } + + EXPECT_TRUE(CheckedNegativelyCached && CheckedSizeChanged); + + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(Entries); +}