From 578e772e6ba177d75dc9e212e9d551b664fa4893 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 8 Oct 2026 10:03:47 -0700 Subject: [PATCH 1/2] [makeOptions] Add the ArgumentIsCASID option attribute Support the new `ArgumentIsCASID` flag from swift options, which labels the options whose argument is a CAS ID when compilation caching is enabled, and update the options that have the flag. rdar://189417668 --- Sources/SwiftOptions/Option.swift | 1 + Sources/SwiftOptions/Options.swift | 16 ++++++++-------- Sources/makeOptions/makeOptions.cpp | 1 + 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Sources/SwiftOptions/Option.swift b/Sources/SwiftOptions/Option.swift index af4f86004..52951dcdc 100644 --- a/Sources/SwiftOptions/Option.swift +++ b/Sources/SwiftOptions/Option.swift @@ -33,6 +33,7 @@ public struct OptionAttributes: OptionSet, Hashable { public static let supplementaryOutput = OptionAttributes(rawValue: 0x800) public static let argumentIsFileList = OptionAttributes(rawValue: 0x1000) public static let cacheInvariant = OptionAttributes(rawValue: 0x2000) + public static let argumentIsCASID = OptionAttributes(rawValue: 0x4000) } /// Describes a command-line option. diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index a833b6f9c..d382162a7 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -64,7 +64,7 @@ extension Option { public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver, .argumentIsPath], metaVar: "", helpText: "The path to a blocklist configuration file") public static let breakageAllowlistPath: Option = Option("-breakage-allowlist-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "An allowlist of breakages to not complain about") public static let bridgingHeaderDirectoryForPrint: Option = Option("-bridging-header-directory-for-print", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Directory for bridging header to be printed in compatibility header") - public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for bridging header pch") + public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], helpText: "Cache Key for bridging header pch") public static let bsdk: Option = Option("-bsdk", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "path to the baseline SDK to import frameworks") public static let buildIdEQ: Option = Option("-build-id=", .joined, alias: Option.buildId, attributes: []) public static let buildId: Option = Option("-build-id", .joinedOrSeparate, attributes: [], metaVar: "", helpText: "Specify the build ID argument passed to the linker") @@ -88,8 +88,8 @@ extension Option { public static let clangBuildSessionFile: Option = Option("-clang-build-session-file", .separate, attributes: [.frontend, .argumentIsPath, .cacheInvariant], helpText: "Use the last modification time of as the underlying Clang build session timestamp") public static let clangHeaderExposeDecls: Option = Option("-clang-header-expose-decls=", .joined, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "all-public|has-expose-attr", helpText: "Which declarations should be exposed in the generated clang header.") public static let clangHeaderExposeModule: Option = Option("-clang-header-expose-module", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "=", helpText: "Allow the compiler to assume that APIs from the specified module are exposed to C/C++/Objective-C in another generated header, so that APIs in the current module that depend on declarations from the specified module can be exposed in the generated header.") - public static let clangIncludeTreeFilelist: Option = Option("-clang-include-tree-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Clang Include Tree FileList CASID") - public static let clangIncludeTreeRoot: Option = Option("-clang-include-tree-root", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Clang Include Tree CASID") + public static let clangIncludeTreeFilelist: Option = Option("-clang-include-tree-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], metaVar: "", helpText: "Clang Include Tree FileList CASID") + public static let clangIncludeTreeRoot: Option = Option("-clang-include-tree-root", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], metaVar: "", helpText: "Clang Include Tree CASID") public static let clangScannerModuleCachePath: Option = Option("-clang-scanner-module-cache-path", .separate, attributes: [.frontend, .doesNotAffectIncrementalBuild, .argumentIsPath, .cacheInvariant], helpText: "Specifies the Clang dependency scanner module cache path") public static let clangTargetVariant: Option = Option("-clang-target-variant", .separate, attributes: [.frontend, .synthesizeInterface], helpText: "Separately set the target we should use for internal Clang instance for the 'zippered' code for macCatalyst") public static let clangTarget: Option = Option("-clang-target", .separate, attributes: [.frontend, .synthesizeInterface], helpText: "Separately set the target we should use for internal Clang instance") @@ -139,8 +139,8 @@ extension Option { public static let debugInverseRequirements: Option = Option("-debug-inverse-requirements", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print real requirements in -debug-generic-signatures output") public static let debugMapping: Option = Option("-debug-mapping", .flag, attributes: [.noDriver], helpText: "Dumping information for debug purposes") public static let debugMapping_: Option = Option("--debug-mapping", .flag, alias: Option.debugMapping, attributes: [.noDriver], helpText: "Dumping information for debug purposes") - public static let debugModulePathEQ: Option = Option("-debug-module-path=", .joined, alias: Option.debugModulePath, attributes: [.frontend, .argumentIsPath]) - public static let debugModulePath: Option = Option("-debug-module-path", .separate, attributes: [.frontend, .argumentIsPath], helpText: "Path or cache key to this module's binary swiftmodule artifact (required by debug info)") + public static let debugModulePathEQ: Option = Option("-debug-module-path=", .joined, alias: Option.debugModulePath, attributes: [.frontend, .argumentIsPath, .argumentIsCASID]) + public static let debugModulePath: Option = Option("-debug-module-path", .separate, attributes: [.frontend, .argumentIsPath, .argumentIsCASID], helpText: "Path or cache key to this module's binary swiftmodule artifact (required by debug info)") public static let debugModuleSelfKey: Option = Option("-debug-module-self-key", .flag, attributes: [.frontend], helpText: "Embeded the key of current job in debug info generation (required by debug info)") public static let debugPrefixMap: Option = Option("-debug-prefix-map", .separate, attributes: [.frontend], metaVar: "", helpText: "Remap source paths in debug info") public static let debugRequirementMachine: Option = Option("-debug-requirement-machine=", .joined, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Fine-grained debug output from the generics implementation") @@ -593,7 +593,7 @@ extension Option { public static let explicitDependencyGraphFormat: Option = Option("-explicit-dependency-graph-format=", .joined, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Specify the explicit dependency graph output format to either 'json' or 'dot'") public static let explicitInterfaceModuleBuild: Option = Option("-explicit-interface-module-build", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Use the specified command-line to build the module from interface, instead of flags specified in the interface") public static let driverExplicitModuleBuild: Option = Option("-explicit-module-build", .flag, attributes: [], helpText: "Prebuild module dependencies to make them explicit") - public static let explicitSwiftModuleMap: Option = Option("-explicit-swift-module-map-file", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "Specify a JSON file containing information of explicit Swift modules") + public static let explicitSwiftModuleMap: Option = Option("-explicit-swift-module-map-file", .separate, attributes: [.frontend, .noDriver, .argumentIsCASID], metaVar: "", helpText: "Specify a JSON file containing information of explicit Swift modules") public static let exportAs: Option = Option("-export-as", .separate, attributes: [.frontend, .moduleInterface], helpText: "Module name to use when referenced in clients module interfaces") public static let externalPassPipelineFilename: Option = Option("-external-pass-pipeline-filename", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Use the pass pipeline defined by ") public static let externalPluginPath: Option = Option("-external-plugin-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "#", helpText: "Add directory to the plugin search path with a plugin server executable", group: .pluginSearch) @@ -671,7 +671,7 @@ extension Option { public static let indexSystemModules: Option = Option("-index-system-modules", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Emit index data for imported serialized swift system modules") public static let indexUnitOutputPathFilelist: Option = Option("-index-unit-output-path-filelist", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], helpText: "Specify index unit output paths in a file rather than on the command line") public static let indexUnitOutputPath: Option = Option("-index-unit-output-path", .separate, attributes: [.frontend, .argumentIsPath, .cacheInvariant], metaVar: "", helpText: "Use as the output path in the produced index data.") - public static let inputFileKey: Option = Option("-input-file-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for input file") + public static let inputFileKey: Option = Option("-input-file-key", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], helpText: "Cache Key for input file") public static let inputPaths: Option = Option("-input-paths", .separate, attributes: [.noDriver, .argumentIsPath], metaVar: "", helpText: "The SDK contents under comparison") public static let inputPaths_: Option = Option("--input-paths", .separate, alias: Option.inputPaths, attributes: [.noDriver, .argumentIsPath], metaVar: "", helpText: "The SDK contents under comparison") public static let swiftinterfaceCompilerVersion: Option = Option("-interface-compiler-version", .separate, attributes: [.helpHidden, .frontend], metaVar: "", helpText: "The version of the Swift compiler used to generate a .swiftinterface file") @@ -959,7 +959,7 @@ extension Option { public static let suppressWarnings: Option = Option("-suppress-warnings", .flag, attributes: [.frontend], helpText: "Suppress all warnings") public static let swiftAsyncFramePointerEQ: Option = Option("-swift-async-frame-pointer=", .joined, attributes: [.helpHidden, .frontend, .noDriver], helpText: "One of 'auto', 'always' or 'never'") public static let swiftModuleCrossImport: Option = Option("-swift-module-cross-import", .multiArg, attributes: [.frontend, .noDriver], metaVar: " ", helpText: "Specify the cross import module", numArgs: 2) - public static let swiftModuleFile: Option = Option("-swift-module-file=", .joined, attributes: [.frontend, .noDriver], metaVar: "=", helpText: "Specify Swift module input explicitly built from textual interface") + public static let swiftModuleFile: Option = Option("-swift-module-file=", .joined, attributes: [.frontend, .noDriver, .argumentIsCASID], metaVar: "=", helpText: "Specify Swift module input explicitly built from textual interface") public static let swiftOnly: Option = Option("-swift-only", .flag, attributes: [.noDriver], helpText: "Only include APIs defined from Swift source") public static let swiftOnly_: Option = Option("--swift-only", .flag, alias: Option.swiftOnly, attributes: [.noDriver], helpText: "Only include APIs defined from Swift source") public static let swiftVersion: Option = Option("-swift-version", .separate, alias: Option.languageMode, attributes: [.helpHidden, .frontend, .synthesizeInterface, .moduleInterface], metaVar: "", helpText: "Interpret input according to a specific Swift language version number") diff --git a/Sources/makeOptions/makeOptions.cpp b/Sources/makeOptions/makeOptions.cpp index d5b576eae..1a86e0461 100644 --- a/Sources/makeOptions/makeOptions.cpp +++ b/Sources/makeOptions/makeOptions.cpp @@ -346,6 +346,7 @@ int makeOptions_main() { emitFlagIf(swift::options::SupplementaryOutput, ".supplementaryOutput"); emitFlagIf(swift::options::ArgumentIsFileList, ".argumentIsFileList"); emitFlagIf(swift::options::CacheInvariant, ".cacheInvariant"); + emitFlagIf(swift::options::ArgumentIsCASID, ".argumentIsCASID"); out << "]"; } From 60827e8a92c529fedf75454da2232cd4b6dfbe50 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 8 Oct 2026 10:10:15 -0700 Subject: [PATCH 2/2] [Caching] Compute the cache keys for a compile job in one call Use `swiftscan_cache_compute_keys_from_input_indices` when available to compute the cache keys for all the output producing inputs of a compile job at once, so the command-line is only processed once for a batch job. Fall back to computing the key for each input otherwise. rdar://189417668 --- Sources/CSwiftScan/include/swiftscan_header.h | 4 ++ .../InterModuleDependencyOracle.swift | 7 ++++ .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 7 ++-- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 6 +++ .../SwiftDriver/SwiftScan/SwiftScanCAS.swift | 26 ++++++++++++ .../SwiftDriverTests/CachingBuildTests.swift | 40 +++++++++++++++++++ .../Helpers/TestBuildConfig.swift | 6 +++ 7 files changed, 93 insertions(+), 3 deletions(-) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 1fe9ce390..8ff8b7b7e 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -328,6 +328,10 @@ typedef struct { swiftscan_string_ref_t (*swiftscan_cache_compute_key_from_input_index)( swiftscan_cas_t cas, int argc, const char **argv, unsigned input_index, swiftscan_string_ref_t *error); + swiftscan_string_set_t *(*swiftscan_cache_compute_keys_from_input_indices)( + swiftscan_cas_t cas, int argc, const char **argv, + const unsigned *input_indices, size_t num_inputs, + swiftscan_string_ref_t *error); //=== Scanner Caching Query/Replay Operations -----------------------------===// swiftscan_cached_compilation_t (*swiftscan_cache_query)( diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index e8f57798f..adc2bf98c 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -97,6 +97,13 @@ public class InterModuleDependencyOracle { return swiftScan.supportsBinaryModuleHeaderDependencies || swiftScan.supportsBinaryModuleHeaderDependency } + @_spi(Testing) public func supportsComputingMultipleCacheKeys() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsComputingMultipleCacheKeys + } + @_spi(Testing) public var supportsBridgingHeaderPCHCommand: Bool { guard let swiftScan = swiftScanLibInstance else { // If no scanner, feature is not supported. diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index bf5e902c8..a50ac41e5 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -1367,14 +1367,15 @@ extension Driver { public mutating func computeOutputCacheKeyForJob(commandLine: [Job.ArgTemplate], inputs: [(TypedVirtualPath, Int)]) throws -> [TypedVirtualPath: String] { // No caching setup, return empty dictionary. - guard let cas = self.cas else { + guard let cas = self.cas, !inputs.isEmpty else { return [:] } // Resolve command-line first. let arguments: [String] = try executor.resolver.resolveArgumentList(for: commandLine) - return try inputs.reduce(into: [:]) { keys, input in - keys[input.0] = try cas.computeCacheKey(commandLine: arguments, index: input.1) + let keys = try cas.computeCacheKeys(commandLine: arguments, indices: inputs.map { $0.1 }) + return zip(inputs, keys).reduce(into: [:]) { result, inputAndKey in + result[inputAndKey.0.0] = inputAndKey.1 } } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 56ea82634..c6ed38b2d 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -247,6 +247,11 @@ private extension String { return api.swiftscan_swift_binary_detail_get_header_dependency != nil } + @_spi(Testing) public var supportsComputingMultipleCacheKeys : Bool { + return api.swiftscan_cache_compute_keys_from_input_indices != nil && + api.swiftscan_string_set_dispose != nil + } + @_spi(Testing) public var supportsStringDispose : Bool { return api.swiftscan_string_dispose != nil } @@ -520,6 +525,7 @@ private extension swiftscan_functions_t { self.swiftscan_cas_dispose = loadOptional("swiftscan_cas_dispose") self.swiftscan_cache_compute_key = loadOptional("swiftscan_cache_compute_key") self.swiftscan_cache_compute_key_from_input_index = loadOptional("swiftscan_cache_compute_key_from_input_index") + self.swiftscan_cache_compute_keys_from_input_indices = loadOptional("swiftscan_cache_compute_keys_from_input_indices") self.swiftscan_cas_store = loadOptional("swiftscan_cas_store") self.swiftscan_cas_fs_builder_create = loadOptional("swiftscan_cas_fs_builder_create") diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift index cbc16850c..aeebf58c8 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift @@ -308,6 +308,32 @@ public final class SwiftScanCAS { return try scanner.toSwiftString(casid) } + /// Compute the cache keys for the inputs at `indices` with a single call if + /// supported by libSwiftScan. Otherwise, compute the key for each input. + public func computeCacheKeys(commandLine: [String], indices: [Int]) throws -> [String] { + guard scanner.supportsComputingMultipleCacheKeys else { + return try indices.map { try computeCacheKey(commandLine: commandLine, index: $0) } + } + let inputIndices = indices.map { UInt32($0) } + let keys = try scanner.handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + inputIndices.withUnsafeBufferPointer { indexArray in + scanner.api.swiftscan_cache_compute_keys_from_input_indices(cas, + Int32(commandLine.count), + commandArray, + indexArray.baseAddress, + indexArray.count, + &err_msg) + } + } + } + guard let keys else { + throw DependencyScanningError.casError("failed to compute cache keys") + } + defer { scanner.api.swiftscan_string_set_dispose(keys) } + return try scanner.toSwiftStringArray(keys.pointee) + } + public func createReplayInstance(commandLine: [String]) throws -> CacheReplayInstance { let instance = try scanner.handleCASError { err_msg in withArrayOfCStrings(commandLine) { commandArray in diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index a07885fa4..2f1be304a 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -1505,6 +1505,46 @@ struct CachingBuildTests { } } + @Test(.requireScannerSupportsComputingMultipleCacheKeys()) + func cacheBatchBuildMultipleCacheKeys() async throws { + try await withTemporaryDirectory { path in + let moduleCachePath = path.appending(component: "ModuleCache") + let casPath = path.appending(component: "cas") + try localFileSystem.createDirectory(moduleCachePath) + let inputs = (0..<4).map { path.appending(component: "file\($0).swift") } + for (index, input) in inputs.enumerated() { + try localFileSystem.writeFileContents(input) { + $0.send("public func foo\(index)() {}") + } + } + var driver = try TestDriver(args: [ + "swiftc", "-c", "-module-name", "Test", "-parse-stdlib", + "-disable-implicit-string-processing-module-import", + "-disable-implicit-concurrency-module-import", + "-explicit-module-build", "-enable-batch-mode", "-driver-batch-count", "2", + "-module-cache-path", moduleCachePath.nativePathString(escaped: false), + "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: false), + "-working-directory", path.nativePathString(escaped: false), + ] + inputs.map { $0.nativePathString(escaped: false) }) + let jobs = try await driver.planBuild() + let cas = try #require(driver.cas) + let resolver = try ArgsResolver(fileSystem: localFileSystem) + + // The keys computed for all the inputs of a job in one call are the same + // as the keys computed for each input. + let compileJobs = jobs.filter { $0.kind == .compile } + #expect(compileJobs.count == 2) + for job in compileJobs { + #expect(job.outputCacheKeys.count == job.primaryInputs.count) + let arguments: [String] = try resolver.resolveArgumentList(for: job.commandLine) + for (input, key) in job.outputCacheKeys { + let index = try #require(job.inputs.firstIndex(of: input)) + #expect(try cas.computeCacheKey(commandLine: arguments, index: index) == key) + } + } + } + } + @Test func cacheWholeModuleBuildPlan() async throws { try await withTemporaryDirectory { path in let moduleCachePath = path.appending(component: "ModuleCache") diff --git a/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift b/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift index cceb953ec..2f0dd9b75 100644 --- a/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift +++ b/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift @@ -178,6 +178,12 @@ extension Trait where Self == Testing.ConditionTrait { return enabled(if: supported, comment ?? "libSwiftScan does not support framework binary dependency reporting") } + /// Requires that libSwiftScan supports computing multiple cache keys in one call. + package static func requireScannerSupportsComputingMultipleCacheKeys(_ comment: Comment? = nil) -> Self { + let supported = (try? _scannerOracle?.supportsComputingMultipleCacheKeys()) ?? false + return enabled(if: supported, comment ?? "libSwiftScan does not support computing multiple cache keys") + } + /// Requires that libSwiftScan supports binary module header dependencies. package static func requireScannerSupportsBinaryModuleHeaderDependencies(_ comment: Comment? = nil) -> Self { let supported = (try? _scannerOracle?.supportsBinaryModuleHeaderDependencies()) ?? false