From bbd2aaa15ada625e516093c6a8469e18abf8e474 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Tue, 24 Jun 2025 15:19:58 -0700 Subject: [PATCH 01/10] Initial implementation of C-APIs for out of date scanning FS cache entry diagnostics. --- clang/include/clang-c/Dependencies.h | 39 +++++++- clang/tools/c-index-test/core_main.cpp | 8 +- clang/tools/libclang/CDependencies.cpp | 131 ++++++++++++++++++------- clang/tools/libclang/libclang.map | 9 +- 4 files changed, 139 insertions(+), 48 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index ef5b3e40d07a2..b7d5fcff8b9cb 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -585,10 +585,41 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); CINDEX_LINKAGE CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); -CINDEX_LINKAGE -CXCStringArray - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( - CXDependencyScannerService); +typedef enum { NegativelyCached, SizeChanged } CXDepScanFSCacheOutOfDateKind; + +typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet; +typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry; + +CXDepScanFSOutOfDateEntrySet +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + CXDependencyScannerService S); + +size_t +clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( + CXDepScanFSOutOfDateEntrySet Entries); + +CXDepScanFSOutOfDateEntry +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry( + CXDepScanFSOutOfDateEntrySet Entries, size_t Idx); + +CXDepScanFSCacheOutOfDateKind +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( + CXDepScanFSOutOfDateEntry Entry); + +CXString +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath( + CXDepScanFSOutOfDateEntry Entry); + +uint64_t +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize( + CXDepScanFSOutOfDateEntry Entry); + +uint64_t +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize( + CXDepScanFSOutOfDateEntry Entry); + +void clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet( + CXDepScanFSOutOfDateEntrySet Entries); /** * Options used to generate a reproducer. diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 60d62cc5f3ca4..415fd8b3622ae 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -916,12 +916,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_disposeDiagnostic(Diag); } - CXCStringArray InvalidNegativeStatCachedPaths = - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( - Service); - - llvm::errs() << "note: number of invalid negatively stat cached paths: " - << InvalidNegativeStatCachedPaths.Count << "\n"; + llvm::errs() << "note: number of invalid negatively stat cached paths: " << 0 + << "\n"; return 1; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 8a7dcb2246929..340614f852b54 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -566,43 +566,6 @@ CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { return unwrap(Graph)->getDiagnosticSet(); } -CXCStringArray -clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths( - CXDependencyScannerService S) { - DependencyScanningService &Service = unwrap(S)->Service; - CStringsManager &StrMgr = unwrap(S)->StrMgr; - - // FIXME: CAS currently does not use the shared cache, and cannot produce - // the same diagnostics. We should add such a diagnostics to CAS as well. - if (Service.useCASFS()) - return {nullptr, 0}; - - DependencyScanningFilesystemSharedCache &SharedCache = - Service.getSharedCache(); - - // Note that it is critical that this FS is the same as the default virtual - // file system we pass to the DependencyScanningWorkers. - llvm::IntrusiveRefCntPtr FS = - llvm::vfs::createPhysicalFileSystem(); - - auto OutOfDateEntries = SharedCache.getOutOfDateEntries(*FS); - - // FIXME: replace this code with proper APIs that handles the - // OutOfDateEntries. - std::vector OutOfDatePaths; - for (const auto &E : OutOfDateEntries) - OutOfDatePaths.emplace_back(E.Path); - - // FIXME: This code here creates copies of strings from - // InvaidNegStatCachedPaths. It is acceptable because this C-API is expected - // to be called only at the end of a CXDependencyScannerService's lifetime. - // In other words, it is called very infrequently. We can change - // CStringsManager's interface to accommodate handling arbitrary StringRefs - // (which may not be null terminated) if we want to avoid copying. - return StrMgr.createCStringsOwned( - {OutOfDatePaths.begin(), OutOfDatePaths.end()}); -} - static std::string lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind MOK, void *MLOContext, std::variantsecond; } +namespace { +typedef std::vector + DependencyScannerFSOutOfDateEntrySet; + +typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry + DependencyScannerFSOutOfDateEntry; +} // namespace + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet, + CXDepScanFSOutOfDateEntrySet) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry, + CXDepScanFSOutOfDateEntry) + +CXDepScanFSOutOfDateEntrySet +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + CXDependencyScannerService S) { + DependencyScanningService &Service = unwrap(S)->Service; + + if (Service.useCASFS()) + return nullptr; + + // Note that it is critical that this FS is the same as the default virtual + // file system we pass to the DependencyScanningWorkers. + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + + DependencyScannerFSOutOfDateEntrySet *OODEntrySet = + new DependencyScannerFSOutOfDateEntrySet(); + *OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS); + + return wrap(OODEntrySet); +} + +size_t +clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( + CXDepScanFSOutOfDateEntrySet Entries) { + return unwrap(Entries)->size(); +} + +CXDepScanFSOutOfDateEntry +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry( + CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) { + DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries); + return wrap(&(*EntSet)[Idx]); +} + +CXDepScanFSCacheOutOfDateKind +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + auto &Info = E->Info; + return std::visit( + llvm::makeVisitor( + [](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo + &Info) { return NegativelyCached; }, + [](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) { + return SizeChanged; + }), + Info); +} + +CXString +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath( + CXDepScanFSOutOfDateEntry Entry) { + return cxstring::createRef(unwrap(Entry)->Path); +} + +static DependencyScannerFSOutOfDateEntry::SizeChangedInfo * +getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) { + auto *SizeInfo = + std::get_if(&E->Info); + assert(SizeInfo && "Wrong entry kind to get the original size!"); + return SizeInfo; +} + +uint64_t +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + return getOutOfDateEntrySizeChangedInfo(E)->CachedSize; +} + +uint64_t +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize( + CXDepScanFSOutOfDateEntry Entry) { + DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); + return getOutOfDateEntrySizeChangedInfo(E)->ActualSize; +} + +void clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet( + CXDepScanFSOutOfDateEntrySet Entries) { + delete unwrap(Entries); +} + namespace { struct DependencyScannerReproducerOptions { std::vector BuildArgs; diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 058eb50cde124..4bd6b60eb3200 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -577,7 +577,6 @@ LLVM_21 { clang_experimental_DepGraphModule_isCWDIgnored; clang_experimental_DepGraphModule_isInStableDirs; clang_getFullyQualifiedName; - clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths; clang_experimental_DependencyScannerReproducerOptions_create; clang_experimental_DependencyScannerReproducerOptions_dispose; clang_experimental_DependencyScanner_generateReproducer; @@ -590,6 +589,14 @@ LLVM_21 { clang_Cursor_getGCCAssemblyNumClobbers; clang_Cursor_getGCCAssemblyClobber; clang_Cursor_isGCCAssemblyVolatile; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet; + clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize; + clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet; }; # Example of how to add a new symbol version entry. If you do add a new symbol From ab2de70642f81ee56e99d4cc091aa64c45067b18 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Mon, 30 Jun 2025 10:08:16 -0700 Subject: [PATCH 02/10] Adding documentations for the C-API. --- clang/include/clang-c/Dependencies.h | 63 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index b7d5fcff8b9cb..2b1062b71b715 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -585,39 +585,100 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); CINDEX_LINKAGE CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); -typedef enum { NegativelyCached, SizeChanged } CXDepScanFSCacheOutOfDateKind; +/** + * The kind of scanning file system cache out-of-date entries. + */ +typedef enum { + /** + * The entry is negatively stat cached (which indicates the file did not exist + * the first time it was looked up during scanning), but the cached file + * exists on the underlying file system. + */ + NegativelyCached, + + /** + * The entry indicates that for the cached file, its cached size + * is different from its size reported by the underlying file system. + */ + SizeChanged +} CXDepScanFSCacheOutOfDateKind; +/** + * The opaque object that contains the scanning file system cache's out-of-date + * entires. + */ typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet; + +/** + * The opaque object that represents a single scanning file system cache's out- + * of-date entry. + */ typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry; +/** + * Returns all the file system cache out-of-date entries given a + * \c CXDependencyScannerService . + * + * This function is intended to be called when the build has finished, + * and the \c CXDependencyScannerService instance is about to be disposed. + * + * The \c CXDependencyScannerService instance owns the strings used + * by the out-of-date entries and should be disposed after the + * out-of-date entries are used and disposed. + */ CXDepScanFSOutOfDateEntrySet clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( CXDependencyScannerService S); +/** + * Returns the number of out-of-date entries contained in a + * \c CXDepScanFSOutOfDateEntrySet . + */ size_t clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( CXDepScanFSOutOfDateEntrySet Entries); +/** + * Returns the out-of-date entry at offset \p Idx of the \c + * CXDepScanFSOutOfDateEntrySet instance. + */ CXDepScanFSOutOfDateEntry clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry( CXDepScanFSOutOfDateEntrySet Entries, size_t Idx); +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind. + */ CXDepScanFSCacheOutOfDateKind clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( CXDepScanFSOutOfDateEntry Entry); +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path. + */ CXString clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath( CXDepScanFSOutOfDateEntry Entry); +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, + * returns the cached size. + */ uint64_t clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize( CXDepScanFSOutOfDateEntry Entry); +/** + * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, + * returns the actual size on the underlying file system. + */ uint64_t clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize( CXDepScanFSOutOfDateEntry Entry); +/** + * Dispose the \c CXDepScanFSOutOfDateEntrySet instance. + */ void clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet( CXDepScanFSOutOfDateEntrySet Entries); From 2ed8acba6b3bdf67f30e369a1893d5b6610a7a94 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Mon, 30 Jun 2025 14:37:14 -0700 Subject: [PATCH 03/10] Address code review - rename the functions so the names are more indicative and concise. --- clang/include/clang-c/Dependencies.h | 20 ++++++++------------ clang/tools/libclang/CDependencies.cpp | 20 ++++++++------------ clang/tools/libclang/libclang.map | 16 ++++++++-------- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 2b1062b71b715..6442b79d2826e 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -627,15 +627,14 @@ typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry; * out-of-date entries are used and disposed. */ CXDepScanFSOutOfDateEntrySet -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( +clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( CXDependencyScannerService S); /** * Returns the number of out-of-date entries contained in a * \c CXDepScanFSOutOfDateEntrySet . */ -size_t -clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( +size_t clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries( CXDepScanFSOutOfDateEntrySet Entries); /** @@ -643,43 +642,40 @@ clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( * CXDepScanFSOutOfDateEntrySet instance. */ CXDepScanFSOutOfDateEntry -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry( +clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry( CXDepScanFSOutOfDateEntrySet Entries, size_t Idx); /** * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind. */ CXDepScanFSCacheOutOfDateKind -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( +clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path. */ -CXString -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath( +CXString clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, * returns the cached size. */ -uint64_t -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize( +uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, * returns the actual size on the underlying file system. */ -uint64_t -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize( +uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( CXDepScanFSOutOfDateEntry Entry); /** * Dispose the \c CXDepScanFSOutOfDateEntrySet instance. */ -void clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet( +void clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet( CXDepScanFSOutOfDateEntrySet Entries); /** diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 340614f852b54..6154ee405bf9c 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -623,7 +623,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry, CXDepScanFSOutOfDateEntry) CXDepScanFSOutOfDateEntrySet -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( +clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( CXDependencyScannerService S) { DependencyScanningService &Service = unwrap(S)->Service; @@ -642,21 +642,20 @@ clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( return wrap(OODEntrySet); } -size_t -clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries( +size_t clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries( CXDepScanFSOutOfDateEntrySet Entries) { return unwrap(Entries)->size(); } CXDepScanFSOutOfDateEntry -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry( +clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry( CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) { DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries); return wrap(&(*EntSet)[Idx]); } CXDepScanFSCacheOutOfDateKind -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( +clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); auto &Info = E->Info; @@ -670,8 +669,7 @@ clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind( Info); } -CXString -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath( +CXString clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath( CXDepScanFSOutOfDateEntry Entry) { return cxstring::createRef(unwrap(Entry)->Path); } @@ -684,21 +682,19 @@ getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) { return SizeInfo; } -uint64_t -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize( +uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); return getOutOfDateEntrySizeChangedInfo(E)->CachedSize; } -uint64_t -clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize( +uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); return getOutOfDateEntrySizeChangedInfo(E)->ActualSize; } -void clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet( +void clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet( CXDepScanFSOutOfDateEntrySet Entries) { delete unwrap(Entries); } diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 4bd6b60eb3200..dabaf4711803f 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -589,14 +589,14 @@ LLVM_21 { clang_Cursor_getGCCAssemblyNumClobbers; clang_Cursor_getGCCAssemblyClobber; clang_Cursor_isGCCAssemblyVolatile; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet; - clang_experimental_DependencyScannerService_getNumOfFSCacheOutOfDateEntries; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntry; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryKind; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryPath; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryCachedSize; - clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntryActualSize; - clang_experimental_DependencyScannerService_disposeFSCacheOutOfDateEntrySet; + clang_experimental_DepScanFSCacheOutOfEntrySet_getSet; + clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries; + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry; + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind; + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath; + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize; + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize; + clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet; }; # Example of how to add a new symbol version entry. If you do add a new symbol From a00fdd5b5a8b2fa7be0daea32633316c9adf8c38 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Wed, 2 Jul 2025 11:14:52 -0700 Subject: [PATCH 04/10] Adding a test case, and cleaning up the APIs' implementations. --- clang/test/ClangScanDeps/error-c-api.cpp | 1 - clang/tools/c-index-test/core_main.cpp | 3 - clang/tools/libclang/CDependencies.cpp | 26 +++--- clang/unittests/libclang/CMakeLists.txt | 1 + ...ependencyScanningFSCacheOutOfDateTests.cpp | 85 +++++++++++++++++++ 5 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp diff --git a/clang/test/ClangScanDeps/error-c-api.cpp b/clang/test/ClangScanDeps/error-c-api.cpp index cfd1bcbd3a75c..b40319fcb6313 100644 --- a/clang/test/ClangScanDeps/error-c-api.cpp +++ b/clang/test/ClangScanDeps/error-c-api.cpp @@ -4,4 +4,3 @@ // CHECK: error: failed to get dependencies // CHECK-NEXT: 'missing.h' file not found -// CHECK-NEXT: number of invalid negatively stat cached paths: 0 diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 415fd8b3622ae..a04ba036efcbd 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -916,9 +916,6 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_disposeDiagnostic(Diag); } - llvm::errs() << "note: number of invalid negatively stat cached paths: " << 0 - << "\n"; - return 1; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 6154ee405bf9c..1b6769a3e3dfd 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -81,17 +81,11 @@ struct CStringsManager { return createCStringsRef(*OwnedStdStr.back()); } }; - -struct DependencyScannerService { - DependencyScanningService Service; - CStringsManager StrMgr{}; -}; } // end anonymous namespace DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions, CXDependencyScannerServiceOptions) - -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerService, +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService, CXDependencyScannerService) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker, CXDependencyScannerWorker) @@ -172,9 +166,9 @@ clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) { // FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now. CASOptions CASOpts; IntrusiveRefCntPtr FS; - return wrap(new DependencyScannerService{DependencyScanningService( + return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, unwrap(Format), CASOpts, - /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)}); + /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)); } ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { @@ -210,10 +204,10 @@ clang_experimental_DependencyScannerService_create_v1( FS = llvm::cantFail( llvm::cas::createCachingOnDiskFileSystem(CAS)); } - return wrap(new DependencyScannerService{DependencyScanningService( + return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), - unwrap(Opts)->OptimizeArgs)}); + unwrap(Opts)->OptimizeArgs)); } void clang_experimental_DependencyScannerService_dispose_v0( @@ -223,16 +217,16 @@ void clang_experimental_DependencyScannerService_dispose_v0( CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( CXDependencyScannerService S) { - ScanningOutputFormat Format = unwrap(S)->Service.getFormat(); + ScanningOutputFormat Format = unwrap(S)->getFormat(); bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree || Format == ScanningOutputFormat::FullIncludeTree; llvm::IntrusiveRefCntPtr FS = llvm::vfs::createPhysicalFileSystem(); if (IsIncludeTreeOutput) - FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->Service.getCAS(), + FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->getCAS(), std::move(FS)); - return wrap(new DependencyScanningWorker(unwrap(S)->Service, FS)); + return wrap(new DependencyScanningWorker(*unwrap(S), FS)); } void clang_experimental_DependencyScannerWorker_dispose_v0( @@ -625,7 +619,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry, CXDepScanFSOutOfDateEntrySet clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( CXDependencyScannerService S) { - DependencyScanningService &Service = unwrap(S)->Service; + DependencyScanningService &Service = *unwrap(S); if (Service.useCASFS()) return nullptr; @@ -678,7 +672,7 @@ static DependencyScannerFSOutOfDateEntry::SizeChangedInfo * getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) { auto *SizeInfo = std::get_if(&E->Info); - assert(SizeInfo && "Wrong entry kind to get the original size!"); + assert(SizeInfo && "Wrong entry kind to get size changed info!"); return SizeInfo; } diff --git a/clang/unittests/libclang/CMakeLists.txt b/clang/unittests/libclang/CMakeLists.txt index 4ef0cb12c7c07..a91b4b2a31714 100644 --- a/clang/unittests/libclang/CMakeLists.txt +++ b/clang/unittests/libclang/CMakeLists.txt @@ -1,6 +1,7 @@ add_clang_unittest(libclangTests LibclangTest.cpp DriverTest.cpp + DependencyScanningFSCacheOutOfDateTests.cpp LINK_LIBS libclang ) diff --git a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp new file mode 100644 index 0000000000000..d82ad053d1aeb --- /dev/null +++ b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp @@ -0,0 +1,85 @@ +//===- unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp ---- ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang-c/Dependencies.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "gtest/gtest.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +TEST(DependencyScanningFSCacheOutOfDate, Basic) { + DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, + ScanningOutputFormat::Full, CASOptions(), + nullptr, nullptr, nullptr); + + auto &SharedCache = Service.getSharedCache(); + + auto InMemoryFS = llvm::makeIntrusiveRefCnt(); + DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); + + // file1 is negatively stat cached. + // file2 has different sizes in the cache and on the physical file system. + // The test creates file1 and file2 on the physical file system. + // Service's cache is setup to use an in-memory file system so we + // can leave out file1 and have a file2 with size 8 (instead of 0). + int fd; + llvm::SmallString<128> File1Path; + llvm::SmallString<128> File2Path; + auto EC = llvm::sys::fs::createTemporaryFile("file1", "h", fd, File1Path); + ASSERT_FALSE(EC); + EC = llvm::sys::fs::createTemporaryFile("file2", "h", fd, File2Path); + ASSERT_FALSE(EC); + + // Trigger negative stat caching on DepFS. + bool PathExists = DepFS.exists(File1Path); + ASSERT_FALSE(PathExists); + + // Add file2 to the in memory FS with non-zero size. + InMemoryFS->addFile(File2Path, 1, + llvm::MemoryBuffer::getMemBuffer(" ")); + PathExists = DepFS.exists(File2Path); + ASSERT_TRUE(PathExists); + + CXDepScanFSOutOfDateEntrySet Entries = + clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( + reinterpret_cast(&Service)); + + size_t NumEntries = + clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries(Entries); + EXPECT_EQ(NumEntries, 2u); + + for (size_t Idx = 0; Idx < NumEntries; Idx++) { + CXDepScanFSOutOfDateEntry Entry = + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry(Entries, 0); + CXDepScanFSCacheOutOfDateKind Kind = + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind(Entry); + CXString Path = + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath(Entry); + ASSERT_TRUE(Kind == NegativelyCached || Kind == SizeChanged); + switch (Kind) { + case NegativelyCached: + ASSERT_STREQ(clang_getCString(Path), File1Path.c_str()); + break; + case SizeChanged: + ASSERT_STREQ(clang_getCString(Path), File2Path.c_str()); + ASSERT_EQ( + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( + Entry), + 8u); + ASSERT_EQ( + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( + Entry), + 0u); + break; + } + } + + clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet(Entries); +} \ No newline at end of file From a7e37a810f4f40195138d32d6b87bf92669ee7c8 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Wed, 2 Jul 2025 11:31:45 -0700 Subject: [PATCH 05/10] Fix a typo in the test case. --- .../libclang/DependencyScanningFSCacheOutOfDateTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp index d82ad053d1aeb..e193f784f3ae7 100644 --- a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp +++ b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp @@ -57,7 +57,7 @@ TEST(DependencyScanningFSCacheOutOfDate, Basic) { for (size_t Idx = 0; Idx < NumEntries; Idx++) { CXDepScanFSOutOfDateEntry Entry = - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry(Entries, 0); + clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry(Entries, Idx); CXDepScanFSCacheOutOfDateKind Kind = clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind(Entry); CXString Path = From e495bba3e410d090ff6c1b3d25ca0da4f9851509 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Mon, 7 Jul 2025 12:14:45 -0700 Subject: [PATCH 06/10] Address code review. --- clang/include/clang-c/Dependencies.h | 16 +++++++------- clang/tools/libclang/CDependencies.cpp | 18 ++++++++------- clang/tools/libclang/libclang.map | 16 +++++++------- ...ependencyScanningFSCacheOutOfDateTests.cpp | 22 +++++++++---------- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 6442b79d2826e..827b1bef3ed4b 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -627,14 +627,14 @@ typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry; * out-of-date entries are used and disposed. */ CXDepScanFSOutOfDateEntrySet -clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( CXDependencyScannerService S); /** * Returns the number of out-of-date entries contained in a * \c CXDepScanFSOutOfDateEntrySet . */ -size_t clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries( +size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( CXDepScanFSOutOfDateEntrySet Entries); /** @@ -642,40 +642,40 @@ size_t clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries( * CXDepScanFSOutOfDateEntrySet instance. */ CXDepScanFSOutOfDateEntry -clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry( +clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry( CXDepScanFSOutOfDateEntrySet Entries, size_t Idx); /** * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind. */ CXDepScanFSCacheOutOfDateKind -clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind( +clang_experimental_DepScanFSCacheOutOfDateEntry_getKind( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path. */ -CXString clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath( +CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, * returns the cached size. */ -uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize( CXDepScanFSOutOfDateEntry Entry); /** * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged, * returns the actual size on the underlying file system. */ -uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize( CXDepScanFSOutOfDateEntry Entry); /** * Dispose the \c CXDepScanFSOutOfDateEntrySet instance. */ -void clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet( +void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( CXDepScanFSOutOfDateEntrySet Entries); /** diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 1b6769a3e3dfd..c701fced9e7bb 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -617,10 +617,12 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry, CXDepScanFSOutOfDateEntry) CXDepScanFSOutOfDateEntrySet -clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( +clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( CXDependencyScannerService S) { DependencyScanningService &Service = *unwrap(S); + // FIXME: CAS FS currently does not use the shared cache, and cannot produce + // the same diagnostics. We should add such a diagnostics to CAS as well. if (Service.useCASFS()) return nullptr; @@ -636,20 +638,20 @@ clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( return wrap(OODEntrySet); } -size_t clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries( +size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( CXDepScanFSOutOfDateEntrySet Entries) { return unwrap(Entries)->size(); } CXDepScanFSOutOfDateEntry -clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry( +clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry( CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) { DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries); return wrap(&(*EntSet)[Idx]); } CXDepScanFSCacheOutOfDateKind -clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind( +clang_experimental_DepScanFSCacheOutOfDateEntry_getKind( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); auto &Info = E->Info; @@ -663,7 +665,7 @@ clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind( Info); } -CXString clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath( +CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath( CXDepScanFSOutOfDateEntry Entry) { return cxstring::createRef(unwrap(Entry)->Path); } @@ -676,19 +678,19 @@ getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) { return SizeInfo; } -uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); return getOutOfDateEntrySizeChangedInfo(E)->CachedSize; } -uint64_t clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( +uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize( CXDepScanFSOutOfDateEntry Entry) { DependencyScannerFSOutOfDateEntry *E = unwrap(Entry); return getOutOfDateEntrySizeChangedInfo(E)->ActualSize; } -void clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet( +void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( CXDepScanFSOutOfDateEntrySet Entries) { delete unwrap(Entries); } diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index dabaf4711803f..21a61c9d9fa28 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -589,14 +589,14 @@ LLVM_21 { clang_Cursor_getGCCAssemblyNumClobbers; clang_Cursor_getGCCAssemblyClobber; clang_Cursor_isGCCAssemblyVolatile; - clang_experimental_DepScanFSCacheOutOfEntrySet_getSet; - clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries; - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry; - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind; - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath; - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize; - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize; - clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet; + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry; + clang_experimental_DepScanFSCacheOutOfDateEntry_getKind; + clang_experimental_DepScanFSCacheOutOfDateEntry_getPath; + clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize; + clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize; + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet; }; # Example of how to add a new symbol version entry. If you do add a new symbol diff --git a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp index e193f784f3ae7..dd247f07dc4c2 100644 --- a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp +++ b/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp @@ -48,20 +48,22 @@ TEST(DependencyScanningFSCacheOutOfDate, Basic) { ASSERT_TRUE(PathExists); CXDepScanFSOutOfDateEntrySet Entries = - clang_experimental_DepScanFSCacheOutOfEntrySet_getSet( + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( reinterpret_cast(&Service)); size_t NumEntries = - clang_experimental_DepScanFSCacheOutOfEntrySet_getNumOfEntries(Entries); + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + Entries); EXPECT_EQ(NumEntries, 2u); for (size_t Idx = 0; Idx < NumEntries; Idx++) { CXDepScanFSOutOfDateEntry Entry = - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntry(Entries, Idx); + clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(Entries, + Idx); CXDepScanFSCacheOutOfDateKind Kind = - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryKind(Entry); + clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(Entry); CXString Path = - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryPath(Entry); + clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(Entry); ASSERT_TRUE(Kind == NegativelyCached || Kind == SizeChanged); switch (Kind) { case NegativelyCached: @@ -70,16 +72,14 @@ TEST(DependencyScanningFSCacheOutOfDate, Basic) { case SizeChanged: ASSERT_STREQ(clang_getCString(Path), File2Path.c_str()); ASSERT_EQ( - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryCachedSize( - Entry), + clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(Entry), 8u); ASSERT_EQ( - clang_experimental_DepScanFSCacheOutOfEntrySet_getEntryActualSize( - Entry), + clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(Entry), 0u); break; } } - clang_experimental_DepScanFSCacheOutOfEntrySet_disposeSet(Entries); -} \ No newline at end of file + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(Entries); +} From 2266ba7f2beb96b32295fd53c199da912fb1f182 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Mon, 7 Jul 2025 12:39:55 -0700 Subject: [PATCH 07/10] Re-add the c-index test. --- clang/test/ClangScanDeps/error-c-api.cpp | 1 + clang/tools/c-index-test/core_main.cpp | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/clang/test/ClangScanDeps/error-c-api.cpp b/clang/test/ClangScanDeps/error-c-api.cpp index b40319fcb6313..a9e9b3831efce 100644 --- a/clang/test/ClangScanDeps/error-c-api.cpp +++ b/clang/test/ClangScanDeps/error-c-api.cpp @@ -4,3 +4,4 @@ // CHECK: error: failed to get dependencies // CHECK-NEXT: 'missing.h' file not found +// CHECK-NEXT: number of out of date file system cache entries: 0 diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index a04ba036efcbd..0d24df3a67955 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -916,6 +916,19 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_disposeDiagnostic(Diag); } + CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet = + clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( + Service); + + llvm::errs() + << "note: number of out of date file system cache entries: " + << clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( + OutOfDateEntrySet) + << "\n"; + + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet( + OutOfDateEntrySet); + return 1; } From 8ae7a57fba23a9f3d225aafc57f7491d9dcb1f46 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Mon, 7 Jul 2025 15:15:22 -0700 Subject: [PATCH 08/10] Rename the new unit test file. --- clang/unittests/libclang/CMakeLists.txt | 2 +- ...acheOutOfDateTests.cpp => DependencyScanningCAPITests.cpp} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename clang/unittests/libclang/{DependencyScanningFSCacheOutOfDateTests.cpp => DependencyScanningCAPITests.cpp} (95%) diff --git a/clang/unittests/libclang/CMakeLists.txt b/clang/unittests/libclang/CMakeLists.txt index a91b4b2a31714..9a041bced8a3d 100644 --- a/clang/unittests/libclang/CMakeLists.txt +++ b/clang/unittests/libclang/CMakeLists.txt @@ -1,7 +1,7 @@ add_clang_unittest(libclangTests LibclangTest.cpp DriverTest.cpp - DependencyScanningFSCacheOutOfDateTests.cpp + DependencyScanningCAPITests.cpp LINK_LIBS libclang ) diff --git a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp b/clang/unittests/libclang/DependencyScanningCAPITests.cpp similarity index 95% rename from clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp rename to clang/unittests/libclang/DependencyScanningCAPITests.cpp index dd247f07dc4c2..7762526f646ab 100644 --- a/clang/unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp +++ b/clang/unittests/libclang/DependencyScanningCAPITests.cpp @@ -1,4 +1,4 @@ -//===- unittests/libclang/DependencyScanningFSCacheOutOfDateTests.cpp ---- ===// +//===- unittests/libclang/DependencyScanningCAPITests.cpp ---------------- ===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -14,7 +14,7 @@ using namespace clang; using namespace tooling; using namespace dependencies; -TEST(DependencyScanningFSCacheOutOfDate, Basic) { +TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Full, CASOptions(), nullptr, nullptr, nullptr); From 2b16e180ccf28236f344d260a711e0c5aaf7d09f Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Tue, 8 Jul 2025 09:31:30 -0700 Subject: [PATCH 09/10] Revise the unit test. --- .../libclang/DependencyScanningCAPITests.cpp | 116 ++++++++++++------ 1 file changed, 78 insertions(+), 38 deletions(-) diff --git a/clang/unittests/libclang/DependencyScanningCAPITests.cpp b/clang/unittests/libclang/DependencyScanningCAPITests.cpp index 7762526f646ab..801c468df3b42 100644 --- a/clang/unittests/libclang/DependencyScanningCAPITests.cpp +++ b/clang/unittests/libclang/DependencyScanningCAPITests.cpp @@ -15,41 +15,81 @@ using namespace tooling; using namespace dependencies; TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Full, CASOptions(), - nullptr, nullptr, nullptr); - - auto &SharedCache = Service.getSharedCache(); - - auto InMemoryFS = llvm::makeIntrusiveRefCnt(); - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); - - // file1 is negatively stat cached. - // file2 has different sizes in the cache and on the physical file system. - // The test creates file1 and file2 on the physical file system. - // Service's cache is setup to use an in-memory file system so we - // can leave out file1 and have a file2 with size 8 (instead of 0). - int fd; - llvm::SmallString<128> File1Path; - llvm::SmallString<128> File2Path; - auto EC = llvm::sys::fs::createTemporaryFile("file1", "h", fd, File1Path); - ASSERT_FALSE(EC); - EC = llvm::sys::fs::createTemporaryFile("file2", "h", fd, File2Path); - ASSERT_FALSE(EC); - - // Trigger negative stat caching on DepFS. - bool PathExists = DepFS.exists(File1Path); - ASSERT_FALSE(PathExists); - - // Add file2 to the in memory FS with non-zero size. - InMemoryFS->addFile(File2Path, 1, - llvm::MemoryBuffer::getMemBuffer(" ")); - PathExists = DepFS.exists(File2Path); - ASSERT_TRUE(PathExists); + // This test is setup to have two out-of-date file system cache entries, + // one is negatively stat cached, the other has its size changed. + // - `include/b.h` is negatively stat cached. + // - `include` (the directory) has its size changed, because `b.h` is added + // to it. + + CXDependencyScannerServiceOptions ServiceOptions = + clang_experimental_DependencyScannerServiceOptions_create(); + CXDependencyScannerService Service = + clang_experimental_DependencyScannerService_create_v1(ServiceOptions); + CXDependencyScannerWorker Worker = + clang_experimental_DependencyScannerWorker_create_v0(Service); + + // Set up the directory structure before scanning. + // - `/tmp/include/` + // - `/tmp/include2/b.h` + llvm::SmallString<128> Dir; + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("tmp", Dir)); + + llvm::SmallString<128> Include = Dir; + llvm::sys::path::append(Include, "include"); + ASSERT_FALSE(llvm::sys::fs::create_directories(Include)); + + llvm::SmallString<128> Include2 = Dir; + llvm::sys::path::append(Include2, "include2"); + ASSERT_FALSE(llvm::sys::fs::create_directories(Include2)); + + // Initially, we keep include/b.h missing and only create include2/b.h. + llvm::SmallString<128> HeaderB2 = Include2; + llvm::sys::path::append(HeaderB2, "b.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderB2File(HeaderB2, EC); + ASSERT_FALSE(EC); + } + + llvm::SmallString<128> TU = Dir; + llvm::sys::path::append(TU, "tu.c"); + { + std::error_code EC; + llvm::raw_fd_ostream TUFile(TU, EC); + ASSERT_FALSE(EC); + TUFile << R"( + #include "b.h" + ")"; + } + + const char *Argv[] = {"-c", TU.c_str(), "-I", Include.c_str(), + "-I", Include2.c_str()}; + size_t Argc = std::size(Argv); + CXDependencyScannerWorkerScanSettings ScanSettings = + clang_experimental_DependencyScannerWorkerScanSettings_create( + Argc, Argv, /*ModuleName=*/nullptr, /*WorkingDirectory=*/Dir.c_str(), + /*MLOContext=*/nullptr, /*MLO=*/nullptr); + + CXDepGraph *Graph = new CXDepGraph; + CXErrorCode ScanResult = + clang_experimental_DependencyScannerWorker_getDepGraph( + Worker, ScanSettings, Graph); + ASSERT_EQ(ScanResult, CXError_Success); + + // Now, we populate include/b.h. We have + // - `/tmp/include/b.h` + // - `/tmp/include2/b.h` + llvm::SmallString<128> HeaderB = Include; + llvm::sys::path::append(HeaderB, "b.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderBFile(HeaderB, EC); + ASSERT_FALSE(EC); + } CXDepScanFSOutOfDateEntrySet Entries = clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( - reinterpret_cast(&Service)); + Service); size_t NumEntries = clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries( @@ -67,16 +107,16 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { ASSERT_TRUE(Kind == NegativelyCached || Kind == SizeChanged); switch (Kind) { case NegativelyCached: - ASSERT_STREQ(clang_getCString(Path), File1Path.c_str()); + EXPECT_STREQ(clang_getCString(Path), HeaderB.c_str()); break; case SizeChanged: - ASSERT_STREQ(clang_getCString(Path), File2Path.c_str()); - ASSERT_EQ( + EXPECT_STREQ(clang_getCString(Path), Include.c_str()); + EXPECT_EQ( clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(Entry), - 8u); - ASSERT_EQ( + 64u); + EXPECT_EQ( clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(Entry), - 0u); + 96u); break; } } From 0805f8ef210efd078e0b09674dd94f7c87c98116 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Tue, 8 Jul 2025 10:26:04 -0700 Subject: [PATCH 10/10] Revise the implementation that looks for out-of-date entries to not look at directories for size changes. --- .../DependencyScanningFilesystem.cpp | 6 ++- .../libclang/DependencyScanningCAPITests.cpp | 41 +++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 06e5633401b84..7c836f211a004 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -131,7 +131,11 @@ DependencyScanningFilesystemSharedCache::getOutOfDateEntries( // later. The cache entry is not invalidated (as we have no good // way to do it now), which may lead to missing file build errors. InvalidDiagInfo.emplace_back(Path.data()); - } else { + } else if (Status->getType() == + llvm::sys::fs::file_type::regular_file) { + // We only check regular files. Directory files sizes could change due + // to content changes, and reporting directory size changes can lead + // to false positives. llvm::vfs::Status CachedStatus = Entry->getStatus(); uint64_t CachedSize = CachedStatus.getSize(); uint64_t ActualSize = Status->getSize(); diff --git a/clang/unittests/libclang/DependencyScanningCAPITests.cpp b/clang/unittests/libclang/DependencyScanningCAPITests.cpp index 801c468df3b42..93b569aace6eb 100644 --- a/clang/unittests/libclang/DependencyScanningCAPITests.cpp +++ b/clang/unittests/libclang/DependencyScanningCAPITests.cpp @@ -29,7 +29,7 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { clang_experimental_DependencyScannerWorker_create_v0(Service); // Set up the directory structure before scanning. - // - `/tmp/include/` + // - `/tmp/include/a.h` // - `/tmp/include2/b.h` llvm::SmallString<128> Dir; ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("tmp", Dir)); @@ -42,6 +42,14 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { llvm::sys::path::append(Include2, "include2"); ASSERT_FALSE(llvm::sys::fs::create_directories(Include2)); + llvm::SmallString<128> HeaderA = Include; + llvm::sys::path::append(HeaderA, "a.h"); + { + std::error_code EC; + llvm::raw_fd_ostream HeaderAFile(HeaderA, EC); + ASSERT_FALSE(EC); + } + // Initially, we keep include/b.h missing and only create include2/b.h. llvm::SmallString<128> HeaderB2 = Include2; llvm::sys::path::append(HeaderB2, "b.h"); @@ -58,6 +66,7 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { llvm::raw_fd_ostream TUFile(TU, EC); ASSERT_FALSE(EC); TUFile << R"( + #include "a.h" #include "b.h" ")"; } @@ -76,9 +85,15 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { Worker, ScanSettings, Graph); ASSERT_EQ(ScanResult, CXError_Success); - // Now, we populate include/b.h. We have - // - `/tmp/include/b.h` - // - `/tmp/include2/b.h` + // Change the size of include/a.h. + { + std::error_code EC; + llvm::raw_fd_ostream HeaderAFile(HeaderA, EC); + ASSERT_FALSE(EC); + HeaderAFile << "// New content!\n"; + } + + // Populate include/b.h. llvm::SmallString<128> HeaderB = Include; llvm::sys::path::append(HeaderB, "b.h"); { @@ -87,6 +102,12 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { ASSERT_FALSE(EC); } + // Directory structure after the change. + // - `/tmp/include/` + // - `a.h` ==> size has changed. + // - `b.h` + // - `/tmp/include2/b.h` + CXDepScanFSOutOfDateEntrySet Entries = clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet( Service); @@ -96,6 +117,8 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { Entries); EXPECT_EQ(NumEntries, 2u); + bool CheckedNegativelyCached = false; + bool CheckedSizeChanged = false; for (size_t Idx = 0; Idx < NumEntries; Idx++) { CXDepScanFSOutOfDateEntry Entry = clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(Entries, @@ -108,18 +131,22 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { switch (Kind) { case NegativelyCached: EXPECT_STREQ(clang_getCString(Path), HeaderB.c_str()); + CheckedNegativelyCached = true; break; case SizeChanged: - EXPECT_STREQ(clang_getCString(Path), Include.c_str()); + EXPECT_STREQ(clang_getCString(Path), HeaderA.c_str()); EXPECT_EQ( clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(Entry), - 64u); + 0u); EXPECT_EQ( clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(Entry), - 96u); + 16u); + CheckedSizeChanged = true; break; } } + EXPECT_TRUE(CheckedNegativelyCached && CheckedSizeChanged); + clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(Entries); }