diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 511f7ba837afa..3c05347bf17a5 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -161,6 +161,17 @@ CINDEX_DEPRECATED CINDEX_LINKAGE void clang_experimental_DependencyScannerServiceOptions_setActionCache( CXDependencyScannerServiceOptions Opts, CXCASActionCache Cache); +/** + * Set if negative stat caching is enabled or disabled. If this is not called + * the default value is used. + * + * The default is false unless the environment variable + * `CLANG_SCAN_CACHE_NEGATIVE_STATS` is set to empty or "1". + */ +CINDEX_LINKAGE void +clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats( + CXDependencyScannerServiceOptions Opts, bool CacheNegativeStats); + /** * Create a \c CXDependencyScannerService object. * Must be disposed with \c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h index 62d7e16e1251c..b27d2c0ce9615 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h @@ -38,9 +38,8 @@ class DependencyScanningCASFilesystem public: static const char ID; - DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS, - llvm::cas::ActionCache &Cache); + DependencyScanningCASFilesystem(DependencyScanningService &Service, + IntrusiveRefCntPtr WorkerFS); ~DependencyScanningCASFilesystem(); @@ -109,6 +108,8 @@ class DependencyScanningCASFilesystem llvm::cas::ObjectStore &CAS; llvm::cas::ActionCache &Cache; + /// The service associated with this VFS. + DependencyScanningService &Service; std::optional ClangFullVersionID; std::optional DepDirectivesID; std::optional EmptyBlobID; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index c452398c1b637..dc161401f0add 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -25,6 +25,8 @@ namespace clang { namespace tooling { namespace dependencies { +class DependencyScanningService; + using DependencyDirectivesTy = SmallVector; @@ -387,7 +389,7 @@ class DependencyScanningWorkerFilesystem static const char ID; DependencyScanningWorkerFilesystem( - DependencyScanningFilesystemSharedCache &SharedCache, + DependencyScanningService &Service, IntrusiveRefCntPtr FS); llvm::ErrorOr status(const Twine &Path) override; @@ -476,10 +478,7 @@ class DependencyScanningWorkerFilesystem /// Returns entry associated with the unique ID in the shared cache or nullptr /// if none is found. const CachedFileSystemEntry * - findSharedEntryByUID(llvm::vfs::Status Stat) const { - return SharedCache.getShardForUID(Stat.getUniqueID()) - .findEntryByUID(Stat.getUniqueID()); - } + findSharedEntryByUID(llvm::vfs::Status Stat) const; /// Associates the given entry with the filename in the local cache and /// returns it. @@ -493,20 +492,14 @@ class DependencyScanningWorkerFilesystem /// some. Otherwise, constructs new one with the given error code, associates /// it with the filename and returns the result. const CachedFileSystemEntry & - getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) { - return SharedCache.getShardForFilename(Filename) - .getOrEmplaceEntryForFilename(Filename, EC); - } + getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC); /// Returns entry associated with the filename in the shared cache if there is /// some. Otherwise, associates the given entry with the filename and returns /// it. const CachedFileSystemEntry & getOrInsertSharedEntryForFilename(StringRef Filename, - const CachedFileSystemEntry &Entry) { - return SharedCache.getShardForFilename(Filename) - .getOrInsertEntryForFilename(Filename, Entry); - } + const CachedFileSystemEntry &Entry); void printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const override { @@ -519,8 +512,9 @@ class DependencyScanningWorkerFilesystem /// VFS. bool shouldBypass(StringRef Path) const; - /// The global cache shared between worker threads. - DependencyScanningFilesystemSharedCache &SharedCache; + /// The service associated with this VFS. + DependencyScanningService &Service; + /// The local cache is used by the worker thread to cache file system queries /// locally instead of querying the global cache every time. DependencyScanningFilesystemLocalCache LocalCache; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index 5c86186a466df..c995d7d8ebad7 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -97,6 +97,12 @@ enum class ScanningOptimizations { #undef DSS_LAST_BITMASK_ENUM +bool shouldCacheNegativeStatsDefault(); + +/// \return true if failed stats for files with this name should be cached, +/// false otherwise. +bool shouldCacheNegativeStatsForPath(StringRef Path); + /// The dependency scanning service contains shared configuration and state that /// is used by the individual dependency scanning workers. class DependencyScanningService { @@ -109,7 +115,8 @@ class DependencyScanningService { ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default, bool EagerLoadModules = false, bool TraceVFS = false, std::time_t BuildSessionTimestamp = - llvm::sys::toTimeT(std::chrono::system_clock::now())); + llvm::sys::toTimeT(std::chrono::system_clock::now()), + bool CacheNegativeStats = shouldCacheNegativeStatsDefault()); ScanningMode getMode() const { return Mode; } @@ -121,6 +128,8 @@ class DependencyScanningService { bool shouldTraceVFS() const { return TraceVFS; } + bool shouldCacheNegativeStats() const { return CacheNegativeStats; } + DependencyScanningFilesystemSharedCache &getSharedCache() { assert(!SharedFS && "Expected not to have a CASFS"); assert(SharedCache && "Expected a shared cache"); @@ -152,6 +161,7 @@ class DependencyScanningService { const bool EagerLoadModules; /// Whether to trace VFS accesses. const bool TraceVFS; + const bool CacheNegativeStats; /// Shared CachingOnDiskFileSystem. Set to nullptr to not use CAS dependency /// scanning. IntrusiveRefCntPtr SharedFS; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp index a32afcc55c715..8741fb1a15ddd 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Basic/Version.h" #include "clang/Lex/DependencyDirectivesScanner.h" #include "llvm/CAS/ActionCache.h" @@ -35,10 +36,10 @@ static void reportAsFatalIfError(llvm::Error E) { using llvm::Error; DependencyScanningCASFilesystem::DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS, - llvm::cas::ActionCache &Cache) - : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) { -} + DependencyScanningService &Service, + IntrusiveRefCntPtr WorkerFS) + : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), + Cache(*Service.getCache()), Service(Service) {} const char DependencyScanningCASFilesystem::ID = 0; DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default; @@ -201,13 +202,6 @@ DependencyScanningCASFilesystem::getOriginal(cas::CASID InputDataID) { return Blob.takeError(); } -static bool shouldCacheStatFailures(StringRef Filename) { - StringRef Ext = llvm::sys::path::extension(Filename); - if (Ext.empty()) - return false; // This may be the module cache directory. - return true; -} - llvm::cas::CachingOnDiskFileSystem & DependencyScanningCASFilesystem::getCachingFS() { return static_cast(*FS); @@ -233,7 +227,8 @@ DependencyScanningCASFilesystem::lookupPath(const Twine &Path) { llvm::ErrorOr MaybeStatus = getCachingFS().statusAndFileID(PathRef, FileID); if (!MaybeStatus) { - if (shouldCacheStatFailures(PathRef)) + if (Service.shouldCacheNegativeStats() && + shouldCacheNegativeStatsForPath(PathRef)) Entries[PathRef].EC = MaybeStatus.getError(); return LookupPathResult{nullptr, MaybeStatus.getError()}; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index f606b9c3e990e..034b3530626c7 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Threading.h" @@ -255,19 +256,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const { } DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( - DependencyScanningFilesystemSharedCache &SharedCache, + DependencyScanningService &Service, IntrusiveRefCntPtr FS) : llvm::RTTIExtends(std::move(FS)), - SharedCache(SharedCache), - WorkingDirForCacheLookup(llvm::errc::invalid_argument) { + Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) { updateWorkingDirForCacheLookup(); } const CachedFileSystemEntry & DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID( TentativeEntry TEntry) { - auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID()); + auto &Shard = + Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID()); return Shard.getOrEmplaceEntryForUID( TEntry.Status.getUniqueID(), std::move(TEntry.Status), std::move(TEntry.Contents), std::move(TEntry.CASContents)); @@ -278,18 +279,46 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough( StringRef Filename) { if (const auto *Entry = LocalCache.findEntryByFilename(Filename)) return Entry; - auto &Shard = SharedCache.getShardForFilename(Filename); + auto &Shard = Service.getSharedCache().getShardForFilename(Filename); if (const auto *Entry = Shard.findEntryByFilename(Filename)) return &LocalCache.insertEntryForFilename(Filename, *Entry); return nullptr; } +const CachedFileSystemEntry * +DependencyScanningWorkerFilesystem::findSharedEntryByUID( + llvm::vfs::Status Stat) const { + return Service.getSharedCache() + .getShardForUID(Stat.getUniqueID()) + .findEntryByUID(Stat.getUniqueID()); +} + +const CachedFileSystemEntry & +DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename( + StringRef Filename, std::error_code EC) { + return Service.getSharedCache() + .getShardForFilename(Filename) + .getOrEmplaceEntryForFilename(Filename, EC); +} + +const CachedFileSystemEntry & +DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename( + StringRef Filename, const CachedFileSystemEntry &Entry) { + return Service.getSharedCache() + .getShardForFilename(Filename) + .getOrInsertEntryForFilename(Filename, Entry); +} + llvm::ErrorOr DependencyScanningWorkerFilesystem::computeAndStoreResult( StringRef OriginalFilename, StringRef FilenameForLookup) { llvm::ErrorOr Stat = getUnderlyingFS().status(OriginalFilename); if (!Stat) { + if (!Service.shouldCacheNegativeStats() || + !shouldCacheNegativeStatsForPath(OriginalFilename)) + return Stat.getError(); + const auto &Entry = getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError()); return insertLocalEntryForFilename(FilenameForLookup, Entry); @@ -478,7 +507,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path, return HandleCachedRealPath(*RealPath); // If we have the result in the shared cache, cache it locally. - auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup); + auto &Shard = + Service.getSharedCache().getShardForFilename(*FilenameForLookup); if (const auto *ShardRealPath = Shard.findRealPathByFilename(*FilenameForLookup)) { const auto &RealPath = LocalCache.insertRealPathForFilename( diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp index 846b0dd6460d3..a157f10a899a7 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -11,21 +11,49 @@ #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/Process.h" using namespace clang; using namespace tooling; using namespace dependencies; +bool clang::tooling::dependencies::shouldCacheNegativeStatsDefault() { + if (std::optional MaybeNegStats = + llvm::sys::Process::GetEnv("CLANG_SCAN_CACHE_NEGATIVE_STATS")) { + if (MaybeNegStats->empty()) + return true; + return llvm::StringSwitch(*MaybeNegStats) + .Case("1", true) + .Case("0", false) + .Default(false); + } + return false; +} + +// rdar://148027982 +// rdar://127079541 +// Negative caching directories can cause build failures due to incorrectly +// configured projects. +bool dependencies::shouldCacheNegativeStatsForPath(StringRef Path) { + StringRef Ext = llvm::sys::path::extension(Path); + if (Ext.empty()) + return false; + if (Ext == ".framework") + return false; + return true; +} + DependencyScanningService::DependencyScanningService( ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts, std::shared_ptr CAS, std::shared_ptr Cache, IntrusiveRefCntPtr SharedFS, ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS, - std::time_t BuildSessionTimestamp) + std::time_t BuildSessionTimestamp, bool CacheNegativeStats) : Mode(Mode), Format(Format), CASOpts(std::move(CASOpts)), CAS(std::move(CAS)), Cache(std::move(Cache)), OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS), + CacheNegativeStats(CacheNegativeStats), SharedFS(std::move(SharedFS)), BuildSessionTimestamp(BuildSessionTimestamp) { if (!this->SharedFS) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index ac5fff1d8ecb7..45ab928d9cf1c 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -34,15 +34,14 @@ DependencyScanningWorker::DependencyScanningWorker( if (Service.useCASFS()) { CacheFS = Service.getSharedFS().createProxyFS(); - DepCASFS = new DependencyScanningCASFilesystem(CacheFS, *Service.getCache()); + DepCASFS = new DependencyScanningCASFilesystem(Service, CacheFS); BaseFS = DepCASFS; return; } switch (Service.getMode()) { case ScanningMode::DependencyDirectivesScan: - DepFS = - new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS); + DepFS = new DependencyScanningWorkerFilesystem(Service, FS); BaseFS = DepFS; break; case ScanningMode::CanonicalPreprocessing: diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 2b32424fe5e89..b1cbef3a5d18c 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -92,6 +92,7 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make; static ScanningOptimizations OptimizeArgs; static std::string ModuleFilesDir; static bool EagerLoadModules; +static bool CacheNegativeStats; static unsigned NumThreads = 0; static std::string CompilationDB; static std::optional ModuleName; @@ -212,6 +213,8 @@ static void ParseArgs(int argc, char **argv) { EagerLoadModules = Args.hasArg(OPT_eager_load_pcm); + CacheNegativeStats = Args.hasArg(OPT_cache_negative_stats); + if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) { StringRef S{A->getValue()}; if (!llvm::to_integer(S, NumThreads, 0)) { @@ -1496,9 +1499,10 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { }); }; - DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, - EagerLoadModules, - /*TraceVFS=*/Verbose); + DependencyScanningService Service( + ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules, + /*TraceVFS=*/Verbose, + llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats); llvm::Timer T; T.startTimer(); diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index efcf950780f73..6f4655c0a1b77 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir", def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">; def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">; +def cache_negative_stats : F<"cache-negative-stats", "Cache stat failures">; def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">; diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 26c8b92435e48..090e74bb136e6 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -38,6 +38,7 @@ struct DependencyScannerServiceOptions { ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default; std::shared_ptr CAS; std::shared_ptr Cache; + std::optional CacheNegativeStats; ScanningOutputFormat getFormat() const; }; @@ -161,6 +162,11 @@ void clang_experimental_DependencyScannerServiceOptions_setActionCache( unwrap(Opts)->CASOpts.CASPath = cas::unwrap(Cache)->CachePath; } +void clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats( + CXDependencyScannerServiceOptions Opts, bool CacheNegativeStats) { + unwrap(Opts)->CacheNegativeStats = CacheNegativeStats; +} + CXDependencyScannerService clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) { // FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now. @@ -207,7 +213,10 @@ clang_experimental_DependencyScannerService_create_v1( return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), - unwrap(Opts)->OptimizeArgs)); + unwrap(Opts)->OptimizeArgs, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + unwrap(Opts)->CacheNegativeStats ? *unwrap(Opts)->CacheNegativeStats + : shouldCacheNegativeStatsDefault())); } void clang_experimental_DependencyScannerService_dispose_v0( diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 46cedc116b6dc..703d7107f6402 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -609,6 +609,11 @@ LLVM_21 { clang_experimental_DepGraphModuleLinkLibrary_isFramework; }; +LLVM_22 { + global: + clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats; +}; + # Example of how to add a new symbol version entry. If you do add a new symbol # version, please update the example to depend on the version you added. # LLVM_X { diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp index 272ae80588b7d..663f0a48c9628 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp @@ -263,8 +263,7 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { ScanningOutputFormat::Make, CASOptions(), nullptr, nullptr, nullptr); { - DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), - std::move(CASFS)); + DependencyScanningWorkerFilesystem DepFS(Service, std::move(CASFS)); llvm::ErrorOr> File = DepFS.openFileForRead(Path); ASSERT_TRUE(File); @@ -286,7 +285,7 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { // Check that even though we pass a new InMemoryFileSystem instance here the // DependencyScanningService's SharedCache cached the file's buffer and // cas::ObjectRef and will be able to provide it. - DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), + DependencyScanningWorkerFilesystem DepFS(Service, new llvm::vfs::InMemoryFileSystem); DepFS.setCurrentWorkingDirectory("/root"); llvm::ErrorOr> File = @@ -446,3 +445,116 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { EXPECT_TRUE(DiagConsumer.Finished); } } + +TEST(DependencyScanner, NoNegativeCache) { + StringRef CWD = "/root"; + + auto VFS = new llvm::vfs::InMemoryFileSystem(); + VFS->setCurrentWorkingDirectory(CWD); + auto Sept = llvm::sys::path::get_separator(); + std::string HeaderPath = + std::string(llvm::formatv("{0}root{0}header.h", Sept)); + std::string Test0Path = + std::string(llvm::formatv("{0}root{0}test0.cpp", Sept)); + std::string Test1Path = + std::string(llvm::formatv("{0}root{0}test1.cpp", Sept)); + + VFS->addFile(Test0Path, 0, + llvm::MemoryBuffer::getMemBuffer( + "#if __has_include(\"header.h\")\n#endif")); + VFS->addFile(Test1Path, 0, + llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"")); + + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::All, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/false); + DependencyScanningTool ScanTool(Service, VFS); + + std::vector CommandLine0 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test0.cpp", + "-o" + "test0.cpp.o"}; + std::vector CommandLine1 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test1.cpp", + "-o" + "test1.cpp.o"}; + + std::string Result; + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result), + llvm::Succeeded()); + + VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result), + llvm::Succeeded()); +} + +TEST(DependencyScanner, NoNegativeCacheCAS) { + StringRef CWD = "/root"; + + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); + auto VFS = llvm::makeIntrusiveRefCnt(); + VFS->setCurrentWorkingDirectory(CWD); + std::unique_ptr CASFS = + llvm::cas::createCASProvidingFileSystem(DB, VFS); + + auto Sept = llvm::sys::path::get_separator(); + std::string HeaderPath = + std::string(llvm::formatv("{0}root{0}header.h", Sept)); + std::string Test0Path = + std::string(llvm::formatv("{0}root{0}test0.cpp", Sept)); + std::string Test1Path = + std::string(llvm::formatv("{0}root{0}test1.cpp", Sept)); + + VFS->addFile(Test0Path, 0, + llvm::MemoryBuffer::getMemBuffer( + "#if __has_include(\"header.h\")\n#endif")); + VFS->addFile(Test1Path, 0, + llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"")); + + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::FullIncludeTree, + CASOptions(), DB, Cache, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/false); + DependencyScanningTool ScanTool(Service, VFS); + + std::vector CommandLine0 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test0.cpp", + "-o" + "test0.cpp.o"}; + std::vector CommandLine1 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test1.cpp", + "-o" + "test1.cpp.o"}; + + std::string Result; + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result), + llvm::Succeeded()); + + VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result), + llvm::Succeeded()); +} diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp index 7988e31c76cf8..52bf06d6d5a18 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp +++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp @@ -7,9 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/CAS/CASOptions.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/ADT/SmallString.h" +#include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/VirtualFileSystem.h" #include "gtest/gtest.h" @@ -21,9 +23,14 @@ TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); - DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); + DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS); DepFS.status("/foo.c"); EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u); @@ -47,9 +54,14 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); - DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); + DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS); { llvm::SmallString<128> Result; @@ -82,8 +94,10 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) { InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer("")); InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer("")); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS); // Success. { @@ -135,8 +149,13 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) { InMemoryFS->setCurrentWorkingDirectory("/"); InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer("")); InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer("")); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); DepFS.status("/foo"); DepFS.status("/foo"); @@ -158,8 +177,13 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); DepFS.status("/dir"); DepFS.status("/dir"); @@ -185,8 +209,13 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) { auto InMemoryFS = llvm::makeIntrusiveRefCnt(); InMemoryFS->setCurrentWorkingDirectory("/"); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS); bool Path1Exists = DepFS.exists("/path1.suffix"); ASSERT_EQ(Path1Exists, false); @@ -198,7 +227,8 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) { // DepFS's eyes. ASSERT_EQ(Path1Exists, false); - auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS); + auto InvalidEntries = + Service.getSharedCache().getOutOfDateEntries(*InMemoryFS); EXPECT_EQ(InvalidEntries.size(), 1u); ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path); @@ -210,8 +240,13 @@ TEST(DependencyScanningFilesystem, DiagnoseCachedFileSizeChange) { InMemoryFS1->setCurrentWorkingDirectory("/"); InMemoryFS2->setCurrentWorkingDirectory("/"); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS1); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/true); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS1); InMemoryFS1->addFile("/path1.suffix", 0, llvm::MemoryBuffer::getMemBuffer("")); @@ -224,7 +259,8 @@ TEST(DependencyScanningFilesystem, DiagnoseCachedFileSizeChange) { // Check against the new file system. InMemoryFS2 could be the underlying // physical system in the real world. - auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS2); + auto InvalidEntries = + Service.getSharedCache().getOutOfDateEntries(*InMemoryFS2); ASSERT_EQ(InvalidEntries.size(), 1u); ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path); @@ -243,8 +279,10 @@ TEST(DependencyScanningFilesystem, DoNotDiagnoseDirSizeChange) { llvm::IntrusiveRefCntPtr FS = llvm::vfs::createPhysicalFileSystem(); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, FS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr); + DependencyScanningWorkerFilesystem DepFS(Service, FS); // Trigger the file system cache. ASSERT_EQ(DepFS.exists(Dir), true); @@ -263,6 +301,6 @@ TEST(DependencyScanningFilesystem, DoNotDiagnoseDirSizeChange) { } // We do not report directory size changes. - auto InvalidEntries = SharedCache.getOutOfDateEntries(*FS); + auto InvalidEntries = Service.getSharedCache().getOutOfDateEntries(*FS); EXPECT_EQ(InvalidEntries.size(), 0u); } diff --git a/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp index de2e308407020..ec585d2a31564 100644 --- a/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp +++ b/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" @@ -32,10 +33,13 @@ TEST(DependencyScanningCASFilesystem, FilenameSpelling) { TempLink TestLink(TestDir.path("File.h"), TestDir.path("SymFile.h")); #endif - std::unique_ptr CAS = llvm::cas::createInMemoryCAS(); - std::unique_ptr Cache = llvm::cas::createInMemoryActionCache(); + std::shared_ptr CAS = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); auto CacheFS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); - DependencyScanningCASFilesystem FS(CacheFS, *Cache); + DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, + ScanningOutputFormat::Make, + clang::CASOptions(), CAS, Cache, nullptr); + DependencyScanningCASFilesystem FS(Service, CacheFS); EXPECT_EQ(FS.status(TestFile.path()).getError(), std::error_code()); auto Directives = FS.getDirectiveTokens(TestFile.path()); @@ -53,10 +57,13 @@ TEST(DependencyScanningCASFilesystem, DirectiveScanFailure) { TempDir TestDir("DependencyScanningCASFilesystemTest", /*Unique=*/true); TempFile TestFile(TestDir.path("python"), "", "import sys\n"); - std::unique_ptr CAS = llvm::cas::createInMemoryCAS(); - std::unique_ptr Cache = llvm::cas::createInMemoryActionCache(); + std::shared_ptr CAS = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); auto CacheFS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); - DependencyScanningCASFilesystem FS(CacheFS, *Cache); + DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, + ScanningOutputFormat::Make, + clang::CASOptions(), CAS, Cache, nullptr); + DependencyScanningCASFilesystem FS(Service, CacheFS); EXPECT_EQ(FS.status(TestFile.path()).getError(), std::error_code()); auto Directives = FS.getDirectiveTokens(TestFile.path()); @@ -64,7 +71,7 @@ TEST(DependencyScanningCASFilesystem, DirectiveScanFailure) { // Check the cached failure in the action cache. { - DependencyScanningCASFilesystem NewFS(CacheFS, *Cache); + DependencyScanningCASFilesystem NewFS(Service, CacheFS); EXPECT_EQ(NewFS.status(TestFile.path()).getError(), std::error_code()); auto Directives = NewFS.getDirectiveTokens(TestFile.path()); ASSERT_FALSE(Directives); diff --git a/clang/unittests/libclang/DependencyScanningCAPITests.cpp b/clang/unittests/libclang/DependencyScanningCAPITests.cpp index 43c7a9f4b2a56..c33f55aa9603f 100644 --- a/clang/unittests/libclang/DependencyScanningCAPITests.cpp +++ b/clang/unittests/libclang/DependencyScanningCAPITests.cpp @@ -22,6 +22,8 @@ TEST(DependencyScanningCAPITests, DependencyScanningFSCacheOutOfDate) { CXDependencyScannerServiceOptions ServiceOptions = clang_experimental_DependencyScannerServiceOptions_create(); + clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats( + ServiceOptions, true); CXDependencyScannerService Service = clang_experimental_DependencyScannerService_create_v1(ServiceOptions); CXDependencyScannerWorker Worker =