Skip to content

[clang Dependency Scanning] Out-of-Date Scanning File System Cache Entry Reporting C-APIs #10927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: next
Choose a base branch
from
Open
96 changes: 92 additions & 4 deletions clang/include/clang-c/Dependencies.h
Original file line number Diff line number Diff line change
Expand Up @@ -585,10 +585,98 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph);
CINDEX_LINKAGE
CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph);

CINDEX_LINKAGE
CXCStringArray
clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
CXDependencyScannerService);
/**
* The kind of scanning file system cache out-of-date entries.
*/
typedef enum {
/**
* The entry is negatively stat cached (which indicates the file did not exist
* the first time it was looked up during scanning), but the cached file
* exists on the underlying file system.
*/
NegativelyCached,

/**
* The entry indicates that for the cached file, its cached size
* is different from its size reported by the underlying file system.
*/
SizeChanged
} CXDepScanFSCacheOutOfDateKind;

/**
* The opaque object that contains the scanning file system cache's out-of-date
* entires.
*/
typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet;

/**
* The opaque object that represents a single scanning file system cache's out-
* of-date entry.
*/
typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry;

/**
* Returns all the file system cache out-of-date entries given a
* \c CXDependencyScannerService .
*
* This function is intended to be called when the build has finished,
* and the \c CXDependencyScannerService instance is about to be disposed.
*
* The \c CXDependencyScannerService instance owns the strings used
* by the out-of-date entries and should be disposed after the
* out-of-date entries are used and disposed.
*/
CXDepScanFSOutOfDateEntrySet
clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
CXDependencyScannerService S);

/**
* Returns the number of out-of-date entries contained in a
* \c CXDepScanFSOutOfDateEntrySet .
*/
size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
CXDepScanFSOutOfDateEntrySet Entries);

/**
* Returns the out-of-date entry at offset \p Idx of the \c
* CXDepScanFSOutOfDateEntrySet instance.
*/
CXDepScanFSOutOfDateEntry
clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
CXDepScanFSOutOfDateEntrySet Entries, size_t Idx);

/**
* Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind.
*/
CXDepScanFSCacheOutOfDateKind
clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
CXDepScanFSOutOfDateEntry Entry);

/**
* Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path.
*/
CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
CXDepScanFSOutOfDateEntry Entry);

/**
* Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
* returns the cached size.
*/
uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
CXDepScanFSOutOfDateEntry Entry);

/**
* Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
* returns the actual size on the underlying file system.
*/
uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
CXDepScanFSOutOfDateEntry Entry);

/**
* Dispose the \c CXDepScanFSOutOfDateEntrySet instance.
*/
void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
CXDepScanFSOutOfDateEntrySet Entries);

/**
* Options used to generate a reproducer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,11 @@ DependencyScanningFilesystemSharedCache::getOutOfDateEntries(
// later. The cache entry is not invalidated (as we have no good
// way to do it now), which may lead to missing file build errors.
InvalidDiagInfo.emplace_back(Path.data());
} else {
} else if (Status->getType() ==
llvm::sys::fs::file_type::regular_file) {
// We only check regular files. Directory files sizes could change due
// to content changes, and reporting directory size changes can lead
// to false positives.
llvm::vfs::Status CachedStatus = Entry->getStatus();
uint64_t CachedSize = CachedStatus.getSize();
uint64_t ActualSize = Status->getSize();
Expand Down
2 changes: 1 addition & 1 deletion clang/test/ClangScanDeps/error-c-api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@

// CHECK: error: failed to get dependencies
// CHECK-NEXT: 'missing.h' file not found
// CHECK-NEXT: number of invalid negatively stat cached paths: 0
// CHECK-NEXT: number of out of date file system cache entries: 0
14 changes: 10 additions & 4 deletions clang/tools/c-index-test/core_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -916,12 +916,18 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
clang_disposeDiagnostic(Diag);
}

CXCStringArray InvalidNegativeStatCachedPaths =
clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet =
clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
Service);

llvm::errs() << "note: number of invalid negatively stat cached paths: "
<< InvalidNegativeStatCachedPaths.Count << "\n";
llvm::errs()
<< "note: number of out of date file system cache entries: "
<< clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
OutOfDateEntrySet)
<< "\n";

clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
OutOfDateEntrySet);

return 1;
}
Expand Down
151 changes: 100 additions & 51 deletions clang/tools/libclang/CDependencies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,11 @@ struct CStringsManager {
return createCStringsRef(*OwnedStdStr.back());
}
};

struct DependencyScannerService {
DependencyScanningService Service;
CStringsManager StrMgr{};
};
} // end anonymous namespace

DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions,
CXDependencyScannerServiceOptions)

DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerService,
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService,
CXDependencyScannerService)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker,
CXDependencyScannerWorker)
Expand Down Expand Up @@ -172,9 +166,9 @@ clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) {
// FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now.
CASOptions CASOpts;
IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
return wrap(new DependencyScannerService{DependencyScanningService(
return wrap(new DependencyScanningService(
ScanningMode::DependencyDirectivesScan, unwrap(Format), CASOpts,
/*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)});
/*CAS=*/nullptr, /*ActionCache=*/nullptr, FS));
}

ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const {
Expand Down Expand Up @@ -210,10 +204,10 @@ clang_experimental_DependencyScannerService_create_v1(
FS = llvm::cantFail(
llvm::cas::createCachingOnDiskFileSystem(CAS));
}
return wrap(new DependencyScannerService{DependencyScanningService(
return wrap(new DependencyScanningService(
ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts,
std::move(CAS), std::move(Cache), std::move(FS),
unwrap(Opts)->OptimizeArgs)});
unwrap(Opts)->OptimizeArgs));
}

void clang_experimental_DependencyScannerService_dispose_v0(
Expand All @@ -223,16 +217,16 @@ void clang_experimental_DependencyScannerService_dispose_v0(

CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0(
CXDependencyScannerService S) {
ScanningOutputFormat Format = unwrap(S)->Service.getFormat();
ScanningOutputFormat Format = unwrap(S)->getFormat();
bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree ||
Format == ScanningOutputFormat::FullIncludeTree;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
llvm::vfs::createPhysicalFileSystem();
if (IsIncludeTreeOutput)
FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->Service.getCAS(),
FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->getCAS(),
std::move(FS));

return wrap(new DependencyScanningWorker(unwrap(S)->Service, FS));
return wrap(new DependencyScanningWorker(*unwrap(S), FS));
}

void clang_experimental_DependencyScannerWorker_dispose_v0(
Expand Down Expand Up @@ -566,43 +560,6 @@ CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) {
return unwrap(Graph)->getDiagnosticSet();
}

CXCStringArray
clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
CXDependencyScannerService S) {
DependencyScanningService &Service = unwrap(S)->Service;
CStringsManager &StrMgr = unwrap(S)->StrMgr;

// FIXME: CAS currently does not use the shared cache, and cannot produce
// the same diagnostics. We should add such a diagnostics to CAS as well.
if (Service.useCASFS())
return {nullptr, 0};

DependencyScanningFilesystemSharedCache &SharedCache =
Service.getSharedCache();

// Note that it is critical that this FS is the same as the default virtual
// file system we pass to the DependencyScanningWorkers.
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
llvm::vfs::createPhysicalFileSystem();

auto OutOfDateEntries = SharedCache.getOutOfDateEntries(*FS);

// FIXME: replace this code with proper APIs that handles the
// OutOfDateEntries.
std::vector<const char *> OutOfDatePaths;
for (const auto &E : OutOfDateEntries)
OutOfDatePaths.emplace_back(E.Path);

// FIXME: This code here creates copies of strings from
// InvaidNegStatCachedPaths. It is acceptable because this C-API is expected
// to be called only at the end of a CXDependencyScannerService's lifetime.
// In other words, it is called very infrequently. We can change
// CStringsManager's interface to accommodate handling arbitrary StringRefs
// (which may not be null terminated) if we want to avoid copying.
return StrMgr.createCStringsOwned(
{OutOfDatePaths.begin(), OutOfDatePaths.end()});
}

static std::string
lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind MOK, void *MLOContext,
std::variant<CXModuleLookupOutputCallback *,
Expand Down Expand Up @@ -646,6 +603,98 @@ std::string OutputLookup::lookupModuleOutput(const ModuleDeps &MD,
return PCMPath.first->second;
}

namespace {
typedef std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
DependencyScannerFSOutOfDateEntrySet;

typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry
DependencyScannerFSOutOfDateEntry;
} // namespace

DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet,
CXDepScanFSOutOfDateEntrySet)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry,
CXDepScanFSOutOfDateEntry)

CXDepScanFSOutOfDateEntrySet
clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
CXDependencyScannerService S) {
DependencyScanningService &Service = *unwrap(S);

// FIXME: CAS FS currently does not use the shared cache, and cannot produce
// the same diagnostics. We should add such a diagnostics to CAS as well.
if (Service.useCASFS())
return nullptr;

// Note that it is critical that this FS is the same as the default virtual
// file system we pass to the DependencyScanningWorkers.
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
llvm::vfs::createPhysicalFileSystem();

DependencyScannerFSOutOfDateEntrySet *OODEntrySet =
new DependencyScannerFSOutOfDateEntrySet();
*OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS);

return wrap(OODEntrySet);
}

size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
CXDepScanFSOutOfDateEntrySet Entries) {
return unwrap(Entries)->size();
}

CXDepScanFSOutOfDateEntry
clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) {
DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries);
return wrap(&(*EntSet)[Idx]);
}

CXDepScanFSCacheOutOfDateKind
clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
CXDepScanFSOutOfDateEntry Entry) {
DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
auto &Info = E->Info;
return std::visit(
llvm::makeVisitor(
[](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo
&Info) { return NegativelyCached; },
[](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) {
return SizeChanged;
}),
Info);
}

CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
CXDepScanFSOutOfDateEntry Entry) {
return cxstring::createRef(unwrap(Entry)->Path);
}

static DependencyScannerFSOutOfDateEntry::SizeChangedInfo *
getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) {
auto *SizeInfo =
std::get_if<DependencyScannerFSOutOfDateEntry::SizeChangedInfo>(&E->Info);
assert(SizeInfo && "Wrong entry kind to get size changed info!");
return SizeInfo;
}

uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
CXDepScanFSOutOfDateEntry Entry) {
DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
return getOutOfDateEntrySizeChangedInfo(E)->CachedSize;
}

uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
CXDepScanFSOutOfDateEntry Entry) {
DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
return getOutOfDateEntrySizeChangedInfo(E)->ActualSize;
}

void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
CXDepScanFSOutOfDateEntrySet Entries) {
delete unwrap(Entries);
}

namespace {
struct DependencyScannerReproducerOptions {
std::vector<std::string> BuildArgs;
Expand Down
9 changes: 8 additions & 1 deletion clang/tools/libclang/libclang.map
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,6 @@ LLVM_21 {
clang_experimental_DepGraphModule_isCWDIgnored;
clang_experimental_DepGraphModule_isInStableDirs;
clang_getFullyQualifiedName;
clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths;
clang_experimental_DependencyScannerReproducerOptions_create;
clang_experimental_DependencyScannerReproducerOptions_dispose;
clang_experimental_DependencyScanner_generateReproducer;
Expand All @@ -590,6 +589,14 @@ LLVM_21 {
clang_Cursor_getGCCAssemblyNumClobbers;
clang_Cursor_getGCCAssemblyClobber;
clang_Cursor_isGCCAssemblyVolatile;
clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet;
clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries;
clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry;
clang_experimental_DepScanFSCacheOutOfDateEntry_getKind;
clang_experimental_DepScanFSCacheOutOfDateEntry_getPath;
clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize;
clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize;
clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet;
};

# Example of how to add a new symbol version entry. If you do add a new symbol
Expand Down
1 change: 1 addition & 0 deletions clang/unittests/libclang/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
add_clang_unittest(libclangTests
LibclangTest.cpp
DriverTest.cpp
DependencyScanningCAPITests.cpp
LINK_LIBS
libclang
)
Expand Down
Loading