From c4b7042d67bfef30d4db9ac737b36ac411d4299b Mon Sep 17 00:00:00 2001 From: Volodymyr Sapsai Date: Mon, 8 Dec 2025 17:51:28 -0800 Subject: [PATCH 1/3] [libclang][reproducers] Add compilation caching support to reproducers for explicitly-built modules. Add API to provide CAS options for reproducer generation and use these options to generate compilation commands. Instead of copying relevant files and providing them through VFS, create a new CAS object storage and copy relevant CAS trees there. rdar://158780270 --- clang/include/clang-c/Dependencies.h | 8 ++ .../reproducer-with-module-dependencies.c | 12 ++ clang/tools/c-index-test/core_main.cpp | 7 +- clang/tools/libclang/CDependencies.cpp | 132 ++++++++++++++---- clang/tools/libclang/libclang.map | 1 + 5 files changed, 134 insertions(+), 26 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 3c05347bf17a5..70f19034ac5eb 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -726,6 +726,14 @@ CINDEX_LINKAGE void clang_experimental_DependencyScannerReproducerOptions_dispose( CXDependencyScannerReproducerOptions); +/** + * Specify the object store and action cache databases, and CAS options for + * generating a reproducer. Should be used if the original compilation uses CAS. + */ +CINDEX_LINKAGE void +clang_experimental_DependencyScannerReproducerOptions_setCASOptions( + CXDependencyScannerReproducerOptions, CXCASDatabases, CXCASOptions); + /** * Generates a reproducer to compile a requested file with required modules. * diff --git a/clang/test/Modules/reproducer-with-module-dependencies.c b/clang/test/Modules/reproducer-with-module-dependencies.c index 906e55e5acd9b..bb5deae650d90 100644 --- a/clang/test/Modules/reproducer-with-module-dependencies.c +++ b/clang/test/Modules/reproducer-with-module-dependencies.c @@ -24,6 +24,13 @@ // RUN: -MMD -MT dependencies -MF %t/deps.d // RUN: FileCheck %t/script-expectations.txt --input-file %t/repro-content/reproducer.sh +// Test the content of a reproducer script with CAS enabled. +// RUN: c-index-test core -gen-deps-reproducer -working-dir %t -cas-path %t/cas -o %t/repro-cas-content \ +// RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ +// RUN: -fmodules -fmodules-cache-path=%t.cas \ +// RUN: -I %t/include +// RUN: FileCheck %t/cas-script-expectations.txt --input-file %t/repro-cas-content/reproducer.sh + //--- include/modular-header.h void fn_in_modular_header(void); @@ -76,3 +83,8 @@ CHECK: -ivfsoverlay "reproducer.cache/vfs/vfs.yaml" CHECK: "-ivfsoverlay" "{{.*}}/existing.yaml" CHECK: MACRO=\$foo CHECK: "-dependency-file" "reproducer.cache/deps.d" + +//--- cas-script-expectations.txt +CHECK: -fcas-path "reproducer.cache/cas" +CHECK: "-fcas-include-tree" "llvmcas:// +CHECKL "-fmodule-file-cache-key" "reproducer.cache/explicitly-built-modules/Test-{{.*}}.pcm" "llvmcas:// diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 19d3fad09dd29..9dd1e3f98a523 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -955,7 +955,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, static int generateDepsReproducer(ArrayRef Args, std::string WorkingDirectory, - std::string ReproLocation) { + std::string ReproLocation, + CXCASDatabases DBs) { CXDependencyScannerReproducerOptions Opts = clang_experimental_DependencyScannerReproducerOptions_create( Args.size(), Args.data(), /*ModuleName=*/nullptr, @@ -965,6 +966,8 @@ static int generateDepsReproducer(ArrayRef Args, auto DisposeOpts = llvm::make_scope_exit([&] { clang_experimental_DependencyScannerReproducerOptions_dispose(Opts); }); + clang_experimental_DependencyScannerReproducerOptions_setCASOptions(Opts, DBs, + nullptr); CXString MessageString; auto DisposeMessageString = llvm::make_scope_exit([&]() { clang_disposeString(MessageString); @@ -1611,7 +1614,7 @@ int indextest_core_main(int argc, const char **argv) { return 1; } return generateDepsReproducer(CompArgs, options::WorkingDir, - options::OutputFile); + options::OutputFile, DBs); } if (options::Action == ActionType::UploadCachedJob) { diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 4c5168a550477..7eca9c9351666 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -694,6 +694,9 @@ struct DependencyScannerReproducerOptions { std::optional WorkingDirectory; std::optional ReproducerLocation; bool UseUniqueReproducerName; + CASOptions CASOpts; + std::shared_ptr CAS; + std::shared_ptr Cache; DependencyScannerReproducerOptions(int argc, const char *const *argv, const char *ModuleName, @@ -750,6 +753,20 @@ clang_experimental_DependencyScannerReproducerOptions_create( UseUniqueReproducerName}); } +void clang_experimental_DependencyScannerReproducerOptions_setCASOptions( + CXDependencyScannerReproducerOptions CXOptions, CXCASDatabases CDBs, + CXCASOptions CASOpts) { + DependencyScannerReproducerOptions &Opts = *unwrap(CXOptions); + if (CDBs) { + cas::WrappedCASDatabases &DBs = *cas::unwrap(CDBs); + Opts.CASOpts = DBs.CASOpts; + Opts.CAS = DBs.CAS; + Opts.Cache = DBs.Cache; + } + if (CASOpts) + Opts.CASOpts = *cas::unwrap(CASOpts); +} + void clang_experimental_DependencyScannerReproducerOptions_dispose( CXDependencyScannerReproducerOptions Options) { delete unwrap(Options); @@ -773,11 +790,18 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( return Report(CXError_InvalidArguments) << "non-unique reproducer is allowed only in a custom location"; - CASOptions CASOpts; + std::shared_ptr UpstreamCAS = Opts.CAS; + bool IsReproducerCASBased(UpstreamCAS); DependencyScanningService DepsService( - ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Full, - CASOpts, /*CAS=*/nullptr, /*ActionCache=*/nullptr); - DependencyScanningTool DepsTool(DepsService); + ScanningMode::DependencyDirectivesScan, + IsReproducerCASBased ? ScanningOutputFormat::FullIncludeTree + : ScanningOutputFormat::Full, + Opts.CASOpts, UpstreamCAS, Opts.Cache); + IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + if (UpstreamCAS) + FS = llvm::cas::createCASProvidingFileSystem(UpstreamCAS, std::move(FS)); + DependencyScanningTool DepsTool(DepsService, FS); llvm::SmallString<128> ReproScriptPath; int ScriptFD; @@ -844,7 +868,7 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( // it is easier to run the reproducer with a different compiler and to // simplify running an individual command manually. std::string ReproExecutable = "\"${CLANG:-" + Opts.BuildArgs.front() + "}\""; - auto PrintArguments = [&ReproExecutable, &FileCacheName, + auto PrintArguments = [IsReproducerCASBased, &ReproExecutable, &FileCacheName, &ClangOpts](llvm::raw_fd_ostream &OS, ArrayRef Arguments, bool RedirectOutput) { @@ -856,7 +880,8 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( ClangOpts.ParseArgs(CharArgs, MissingArgIndex, MissingArgCount, llvm::opt::Visibility(options::CC1Option)); - bool DidAddVFSOverlay = false; + // CAS-based reproducer doesn't use VFS overlays. + bool DidAddVFSOverlay = IsReproducerCASBased; OS << ReproExecutable; for (const llvm::opt::Arg *Arg : ParsedArgs) { const llvm::opt::Option &Opt = Arg->getOption(); @@ -866,6 +891,10 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( DidAddVFSOverlay = true; } } + if (Opt.matches(options::OPT_fcas_path)) { + OS << " -fcas-path \"" << FileCacheName << "/cas\""; + continue; + } bool IsOutputArg = Opt.matches(options::OPT_o) || Opt.matches(options::OPT_dependency_file); llvm::opt::ArgStringList OutArgs; @@ -899,26 +928,81 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( auto RealFS = llvm::vfs::getRealFileSystem(); RealFS->setCurrentWorkingDirectory(*Opts.WorkingDirectory); - SmallString<128> VFSCachePath = FileCachePath; - llvm::sys::path::append(VFSCachePath, "vfs"); - std::string VFSCachePathStr = VFSCachePath.str().str(); - llvm::FileCollector FileCollector(VFSCachePathStr, - /*OverlayRoot=*/VFSCachePathStr, RealFS); - for (const auto &FileDep : TU.FileDeps) { - FileCollector.addFile(FileDep); - } - for (ModuleDeps &ModuleDep : TU.ModuleGraph) { - ModuleDep.forEachFileDep([&FileCollector](StringRef FileDep) { + if (IsReproducerCASBased) { + SmallString<128> CASPath = FileCachePath; + llvm::sys::path::append(CASPath, "cas"); + clang::CASOptions ReproducerCASOpts; + ReproducerCASOpts.CASPath = CASPath.str(); + ReproducerCASOpts.PluginPath = Opts.CASOpts.PluginPath; + ReproducerCASOpts.PluginOptions = Opts.CASOpts.PluginOptions; + auto DBsOrErr = ReproducerCASOpts.getOrCreateDatabases(); + if (!DBsOrErr) + return ReportFailure() << "failed to create a CAS database\n" + << toString(DBsOrErr.takeError()); + std::shared_ptr ReproCAS = DBsOrErr->first; + + auto transplantCASIncludeTree = + [UpstreamCAS, ReproCAS]( + const std::optional &IncludeTreeID) -> llvm::Error { + if (!IncludeTreeID.has_value()) + // Missing `IncludeTreeID` likely indicates a problem but ignore it, so + // can capture enough data to reproduce it later. + return llvm::Error::success(); + auto IDOrErr = UpstreamCAS->parseID(*IncludeTreeID); + if (!IDOrErr) + return llvm::make_error( + "failure to parse include tree id '" + *IncludeTreeID + + "':" + toString(IDOrErr.takeError()), + llvm::inconvertibleErrorCode()); + std::optional UpstreamRef = + UpstreamCAS->getReference(*IDOrErr); + if (!UpstreamRef.has_value()) + return llvm::make_error( + "missing include tree with ID '" + *IncludeTreeID + + "' in the provided CAS object storage", + llvm::inconvertibleErrorCode()); + auto ReproRefOrErr = ReproCAS->importObject(*UpstreamCAS, *UpstreamRef); + if (!ReproRefOrErr) + return llvm::make_error( + "failure to import an include tree with id '" + *IncludeTreeID + + "':" + toString(ReproRefOrErr.takeError()), + llvm::inconvertibleErrorCode()); + return llvm::Error::success(); + }; + + if (auto Err = transplantCASIncludeTree(TU.IncludeTreeID)) + return ReportFailure() + << "failed to transplant a translation unit include tree due to " + << toString(std::move(Err)); + for (const ModuleDeps &ModuleDep : TU.ModuleGraph) { + if (auto Err = transplantCASIncludeTree(ModuleDep.IncludeTreeID)) + return ReportFailure() + << "failed to transplant a module '" + ModuleDep.ID.ModuleName + + "' include tree due to " + << toString(std::move(Err)); + } + } else { + SmallString<128> VFSCachePath = FileCachePath; + llvm::sys::path::append(VFSCachePath, "vfs"); + std::string VFSCachePathStr = VFSCachePath.str().str(); + llvm::FileCollector FileCollector(VFSCachePathStr, + /*OverlayRoot=*/VFSCachePathStr); + for (const auto &FileDep : TU.FileDeps) { FileCollector.addFile(FileDep); - }); + } + for (ModuleDeps &ModuleDep : TU.ModuleGraph) { + ModuleDep.forEachFileDep([&FileCollector](StringRef FileDep) { + FileCollector.addFile(FileDep); + }); + } + if (FileCollector.copyFiles(/*StopOnError=*/true)) + return ReportFailure() + << "failed to copy the files used for the compilation"; + SmallString<128> VFSOverlayPath = VFSCachePath; + llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); + if (FileCollector.writeMapping(VFSOverlayPath)) + return ReportFailure() << "failed to write a VFS overlay mapping"; } - if (FileCollector.copyFiles(/*StopOnError=*/true)) - return ReportFailure() - << "failed to copy the files used for the compilation"; - SmallString<128> VFSOverlayPath = VFSCachePath; - llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); - if (FileCollector.writeMapping(VFSOverlayPath)) - return ReportFailure() << "failed to write a VFS overlay mapping"; return Report(CXError_Success) << "Created a reproducer. Sources and associated run script(s) are " diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 703d7107f6402..d3121dd11d2d9 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -612,6 +612,7 @@ LLVM_21 { LLVM_22 { global: clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats; + clang_experimental_DependencyScannerReproducerOptions_setCASOptions; }; # Example of how to add a new symbol version entry. If you do add a new symbol From df2e1178ecf689bfadad6858a4b37ea385df8e01 Mon Sep 17 00:00:00 2001 From: Volodymyr Sapsai Date: Wed, 10 Dec 2025 17:08:02 -0800 Subject: [PATCH 2/3] Unindented previous code (with VFS overlay) and noticed my mistake. --- clang/tools/libclang/CDependencies.cpp | 40 +++++++++++++------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 7eca9c9351666..c280b9b740a8d 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -981,28 +981,28 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( "' include tree due to " << toString(std::move(Err)); } - } else { - SmallString<128> VFSCachePath = FileCachePath; - llvm::sys::path::append(VFSCachePath, "vfs"); - std::string VFSCachePathStr = VFSCachePath.str().str(); - llvm::FileCollector FileCollector(VFSCachePathStr, - /*OverlayRoot=*/VFSCachePathStr); - for (const auto &FileDep : TU.FileDeps) { +} else { + SmallString<128> VFSCachePath = FileCachePath; + llvm::sys::path::append(VFSCachePath, "vfs"); + std::string VFSCachePathStr = VFSCachePath.str().str(); + llvm::FileCollector FileCollector(VFSCachePathStr, + /*OverlayRoot=*/VFSCachePathStr, RealFS); + for (const auto &FileDep : TU.FileDeps) { + FileCollector.addFile(FileDep); + } + for (ModuleDeps &ModuleDep : TU.ModuleGraph) { + ModuleDep.forEachFileDep([&FileCollector](StringRef FileDep) { FileCollector.addFile(FileDep); - } - for (ModuleDeps &ModuleDep : TU.ModuleGraph) { - ModuleDep.forEachFileDep([&FileCollector](StringRef FileDep) { - FileCollector.addFile(FileDep); - }); - } - if (FileCollector.copyFiles(/*StopOnError=*/true)) - return ReportFailure() - << "failed to copy the files used for the compilation"; - SmallString<128> VFSOverlayPath = VFSCachePath; - llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); - if (FileCollector.writeMapping(VFSOverlayPath)) - return ReportFailure() << "failed to write a VFS overlay mapping"; + }); } + if (FileCollector.copyFiles(/*StopOnError=*/true)) + return ReportFailure() + << "failed to copy the files used for the compilation"; + SmallString<128> VFSOverlayPath = VFSCachePath; + llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); + if (FileCollector.writeMapping(VFSOverlayPath)) + return ReportFailure() << "failed to write a VFS overlay mapping"; +} return Report(CXError_Success) << "Created a reproducer. Sources and associated run script(s) are " From 69a6b1b26437b592864fcc397459f15f6f204b45 Mon Sep 17 00:00:00 2001 From: Volodymyr Sapsai Date: Wed, 10 Dec 2025 17:10:23 -0800 Subject: [PATCH 3/3] Fix tlpo in the test. Actually run the reproducers to make sure they work. --- .../reproducer-with-module-dependencies.c | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/clang/test/Modules/reproducer-with-module-dependencies.c b/clang/test/Modules/reproducer-with-module-dependencies.c index bb5deae650d90..84f394a63e3e9 100644 --- a/clang/test/Modules/reproducer-with-module-dependencies.c +++ b/clang/test/Modules/reproducer-with-module-dependencies.c @@ -1,5 +1,6 @@ // Test generating a reproducer for a modular build where required modules are // built explicitly as separate steps. +// REQUIRES: shell // RUN: rm -rf %t // RUN: split-file %s %t @@ -7,18 +8,18 @@ // // RUN: c-index-test core -gen-deps-reproducer -working-dir %t \ // RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache \ // RUN: -ivfsoverlay %t/existing.yaml -I /virtual | FileCheck %t/reproducer.c // Test a failed attempt at generating a reproducer. // RUN: not c-index-test core -gen-deps-reproducer -working-dir %t \ // RUN: -- clang-executable -c %t/failed-reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t 2>&1 | FileCheck %t/failed-reproducer.c +// RUN: -fmodules -fmodules-cache-path=%t.modulecache 2>&1 | FileCheck %t/failed-reproducer.c // Test the content of a reproducer script. // RUN: c-index-test core -gen-deps-reproducer -working-dir %t -o %t/repro-content \ // RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache \ // RUN: -DMACRO="\$foo" \ // RUN: -ivfsoverlay %t/existing.yaml -I /virtual \ // RUN: -MMD -MT dependencies -MF %t/deps.d @@ -26,11 +27,21 @@ // Test the content of a reproducer script with CAS enabled. // RUN: c-index-test core -gen-deps-reproducer -working-dir %t -cas-path %t/cas -o %t/repro-cas-content \ -// RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t.cas \ +// RUN: -- %clang -c %t/reproducer.c -o %t/reproducer.o \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache.cas \ // RUN: -I %t/include // RUN: FileCheck %t/cas-script-expectations.txt --input-file %t/repro-cas-content/reproducer.sh +// Verify can reproduce the original compilations with the files captured by reproducers. +// RUN: rm -rf %t.modulecache +// RUN: rm -rf %t.modulecache.cas +// RUN: rm -rf %t/cas +// RUN: rm %t/include/modular-header.h +// RUN: cd %t/repro-content +// RUN: env CLANG=%clang bash ./reproducer.sh +// RUN: cd %t/repro-cas-content +// RUN: bash ./reproducer.sh + //--- include/modular-header.h void fn_in_modular_header(void); @@ -87,4 +98,4 @@ CHECK: "-dependency-file" "reproducer.cache/deps.d" //--- cas-script-expectations.txt CHECK: -fcas-path "reproducer.cache/cas" CHECK: "-fcas-include-tree" "llvmcas:// -CHECKL "-fmodule-file-cache-key" "reproducer.cache/explicitly-built-modules/Test-{{.*}}.pcm" "llvmcas:// +CHECK: "-fmodule-file-cache-key" "Test-{{.*}}.pcm" "llvmcas://