[Caching] Reduce the cost of computing compile job cache keys

Sources/CSwiftScan/include/swiftscan_header.h

@@ -328,6 +328,10 @@ typedef struct {
   swiftscan_string_ref_t (*swiftscan_cache_compute_key_from_input_index)(
       swiftscan_cas_t cas, int argc, const char **argv, unsigned input_index,
       swiftscan_string_ref_t *error);
+  swiftscan_string_set_t *(*swiftscan_cache_compute_keys_from_input_indices)(
+      swiftscan_cas_t cas, int argc, const char **argv,
+      const unsigned *input_indices, size_t num_inputs,
+      swiftscan_string_ref_t *error);
 
   //=== Scanner Caching Query/Replay Operations -----------------------------===//
   swiftscan_cached_compilation_t (*swiftscan_cache_query)(

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -97,6 +97,13 @@ public class InterModuleDependencyOracle {
     return swiftScan.supportsBinaryModuleHeaderDependencies || swiftScan.supportsBinaryModuleHeaderDependency
   }
 
+  @_spi(Testing) public func supportsComputingMultipleCacheKeys() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsComputingMultipleCacheKeys
+  }
+
   @_spi(Testing) public var supportsBridgingHeaderPCHCommand: Bool {
     guard let swiftScan = swiftScanLibInstance else {
       // If no scanner, feature is not supported.

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -1367,14 +1367,15 @@ extension Driver {
   public mutating func computeOutputCacheKeyForJob(commandLine: [Job.ArgTemplate],
                                                    inputs: [(TypedVirtualPath, Int)]) throws -> [TypedVirtualPath: String] {
     // No caching setup, return empty dictionary.
-    guard let cas = self.cas else {
+    guard let cas = self.cas, !inputs.isEmpty else {
       return [:]
     }
     // Resolve command-line first.
     let arguments: [String] = try executor.resolver.resolveArgumentList(for: commandLine)
 
-    return try inputs.reduce(into: [:]) { keys, input in
-      keys[input.0] = try cas.computeCacheKey(commandLine: arguments, index: input.1)
+    let keys = try cas.computeCacheKeys(commandLine: arguments, indices: inputs.map { $0.1 })
+    return zip(inputs, keys).reduce(into: [:]) { result, inputAndKey in
+      result[inputAndKey.0.0] = inputAndKey.1
     }
   }
 

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -247,6 +247,11 @@ private extension String {
     return api.swiftscan_swift_binary_detail_get_header_dependency != nil
   }
 
+  @_spi(Testing) public var supportsComputingMultipleCacheKeys : Bool {
+    return api.swiftscan_cache_compute_keys_from_input_indices != nil &&
+           api.swiftscan_string_set_dispose != nil
+  }
+
   @_spi(Testing) public var supportsStringDispose : Bool {
     return api.swiftscan_string_dispose != nil
   }
@@ -520,6 +525,7 @@ private extension swiftscan_functions_t {
     self.swiftscan_cas_dispose = loadOptional("swiftscan_cas_dispose")
     self.swiftscan_cache_compute_key = loadOptional("swiftscan_cache_compute_key")
     self.swiftscan_cache_compute_key_from_input_index = loadOptional("swiftscan_cache_compute_key_from_input_index")
+    self.swiftscan_cache_compute_keys_from_input_indices = loadOptional("swiftscan_cache_compute_keys_from_input_indices")
     self.swiftscan_cas_store = loadOptional("swiftscan_cas_store")
 
     self.swiftscan_cas_fs_builder_create = loadOptional("swiftscan_cas_fs_builder_create")

Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift

@@ -308,6 +308,32 @@ public final class SwiftScanCAS {
     return try scanner.toSwiftString(casid)
   }
 
+  /// Compute the cache keys for the inputs at `indices` with a single call if
+  /// supported by libSwiftScan. Otherwise, compute the key for each input.
+  public func computeCacheKeys(commandLine: [String], indices: [Int]) throws -> [String] {
+    guard scanner.supportsComputingMultipleCacheKeys else {
+      return try indices.map { try computeCacheKey(commandLine: commandLine, index: $0) }
+    }
+    let inputIndices = indices.map { UInt32($0) }
+    let keys = try scanner.handleCASError { err_msg in
+      withArrayOfCStrings(commandLine) { commandArray in
+        inputIndices.withUnsafeBufferPointer { indexArray in
+          scanner.api.swiftscan_cache_compute_keys_from_input_indices(cas,
+                                                                      Int32(commandLine.count),
+                                                                      commandArray,
+                                                                      indexArray.baseAddress,
+                                                                      indexArray.count,
+                                                                      &err_msg)
+        }
+      }
+    }
+    guard let keys else {
+      throw DependencyScanningError.casError("failed to compute cache keys")
+    }
+    defer { scanner.api.swiftscan_string_set_dispose(keys) }
+    return try scanner.toSwiftStringArray(keys.pointee)
+  }
+
   public func createReplayInstance(commandLine: [String]) throws -> CacheReplayInstance {
     let instance = try scanner.handleCASError { err_msg in
       withArrayOfCStrings(commandLine) { commandArray in

Sources/SwiftOptions/Option.swift

@@ -33,6 +33,7 @@ public struct OptionAttributes: OptionSet, Hashable {
   public static let supplementaryOutput           = OptionAttributes(rawValue: 0x800)
   public static let argumentIsFileList            = OptionAttributes(rawValue: 0x1000)
   public static let cacheInvariant                = OptionAttributes(rawValue: 0x2000)
+  public static let argumentIsCASID               = OptionAttributes(rawValue: 0x4000)
 }
 
 /// Describes a command-line option.

Sources/SwiftOptions/Options.swift

@@ -64,7 +64,7 @@ extension Option {
   public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver, .argumentIsPath], metaVar: "<path>", helpText: "The path to a blocklist configuration file")
   public static let breakageAllowlistPath: Option = Option("-breakage-allowlist-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "An allowlist of breakages to not complain about")
   public static let bridgingHeaderDirectoryForPrint: Option = Option("-bridging-header-directory-for-print", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<path>", helpText: "Directory for bridging header to be printed in compatibility header")
-  public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for bridging header pch")
+  public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], helpText: "Cache Key for bridging header pch")
   public static let bsdk: Option = Option("-bsdk", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "path to the baseline SDK to import frameworks")
   public static let buildIdEQ: Option = Option("-build-id=", .joined, alias: Option.buildId, attributes: [])
   public static let buildId: Option = Option("-build-id", .joinedOrSeparate, attributes: [], metaVar: "<build-id>", helpText: "Specify the build ID argument passed to the linker")
@@ -88,8 +88,8 @@ extension Option {
   public static let clangBuildSessionFile: Option = Option("-clang-build-session-file", .separate, attributes: [.frontend, .argumentIsPath, .cacheInvariant], helpText: "Use the last modification time of <file> as the underlying Clang build session timestamp")
   public static let clangHeaderExposeDecls: Option = Option("-clang-header-expose-decls=", .joined, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "all-public|has-expose-attr", helpText: "Which declarations should be exposed in the generated clang header.")
   public static let clangHeaderExposeModule: Option = Option("-clang-header-expose-module", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<imported-module-name>=<generated-header-name>", helpText: "Allow the compiler to assume that APIs from the specified module are exposed to C/C++/Objective-C in another generated header, so that APIs in the current module that depend on declarations from the specified module can be exposed in the generated header.")
-  public static let clangIncludeTreeFilelist: Option = Option("-clang-include-tree-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<cas-id>", helpText: "Clang Include Tree FileList CASID")
-  public static let clangIncludeTreeRoot: Option = Option("-clang-include-tree-root", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<cas-id>", helpText: "Clang Include Tree CASID")
+  public static let clangIncludeTreeFilelist: Option = Option("-clang-include-tree-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], metaVar: "<cas-id>", helpText: "Clang Include Tree FileList CASID")
+  public static let clangIncludeTreeRoot: Option = Option("-clang-include-tree-root", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], metaVar: "<cas-id>", helpText: "Clang Include Tree CASID")
   public static let clangScannerModuleCachePath: Option = Option("-clang-scanner-module-cache-path", .separate, attributes: [.frontend, .doesNotAffectIncrementalBuild, .argumentIsPath, .cacheInvariant], helpText: "Specifies the Clang dependency scanner module cache path")
   public static let clangTargetVariant: Option = Option("-clang-target-variant", .separate, attributes: [.frontend, .synthesizeInterface], helpText: "Separately set the target we should use for internal Clang instance for the 'zippered' code for macCatalyst")
   public static let clangTarget: Option = Option("-clang-target", .separate, attributes: [.frontend, .synthesizeInterface], helpText: "Separately set the target we should use for internal Clang instance")
@@ -139,8 +139,8 @@ extension Option {
   public static let debugInverseRequirements: Option = Option("-debug-inverse-requirements", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print real requirements in -debug-generic-signatures output")
   public static let debugMapping: Option = Option("-debug-mapping", .flag, attributes: [.noDriver], helpText: "Dumping information for debug purposes")
   public static let debugMapping_: Option = Option("--debug-mapping", .flag, alias: Option.debugMapping, attributes: [.noDriver], helpText: "Dumping information for debug purposes")
-  public static let debugModulePathEQ: Option = Option("-debug-module-path=", .joined, alias: Option.debugModulePath, attributes: [.frontend, .argumentIsPath])
-  public static let debugModulePath: Option = Option("-debug-module-path", .separate, attributes: [.frontend, .argumentIsPath], helpText: "Path or cache key to this module's binary swiftmodule artifact (required by debug info)")
+  public static let debugModulePathEQ: Option = Option("-debug-module-path=", .joined, alias: Option.debugModulePath, attributes: [.frontend, .argumentIsPath, .argumentIsCASID])
+  public static let debugModulePath: Option = Option("-debug-module-path", .separate, attributes: [.frontend, .argumentIsPath, .argumentIsCASID], helpText: "Path or cache key to this module's binary swiftmodule artifact (required by debug info)")
   public static let debugModuleSelfKey: Option = Option("-debug-module-self-key", .flag, attributes: [.frontend], helpText: "Embeded the key of current job in debug info generation (required by debug info)")
   public static let debugPrefixMap: Option = Option("-debug-prefix-map", .separate, attributes: [.frontend], metaVar: "<prefix=replacement>", helpText: "Remap source paths in debug info")
   public static let debugRequirementMachine: Option = Option("-debug-requirement-machine=", .joined, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Fine-grained debug output from the generics implementation")
@@ -593,7 +593,7 @@ extension Option {
   public static let explicitDependencyGraphFormat: Option = Option("-explicit-dependency-graph-format=", .joined, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Specify the explicit dependency graph output format to either 'json' or 'dot'")
   public static let explicitInterfaceModuleBuild: Option = Option("-explicit-interface-module-build", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Use the specified command-line to build the module from interface, instead of flags specified in the interface")
   public static let driverExplicitModuleBuild: Option = Option("-explicit-module-build", .flag, attributes: [], helpText: "Prebuild module dependencies to make them explicit")
-  public static let explicitSwiftModuleMap: Option = Option("-explicit-swift-module-map-file", .separate, attributes: [.frontend, .noDriver], metaVar: "<path>", helpText: "Specify a JSON file containing information of explicit Swift modules")
+  public static let explicitSwiftModuleMap: Option = Option("-explicit-swift-module-map-file", .separate, attributes: [.frontend, .noDriver, .argumentIsCASID], metaVar: "<path>", helpText: "Specify a JSON file containing information of explicit Swift modules")
   public static let exportAs: Option = Option("-export-as", .separate, attributes: [.frontend, .moduleInterface], helpText: "Module name to use when referenced in clients module interfaces")
   public static let externalPassPipelineFilename: Option = Option("-external-pass-pipeline-filename", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<pass_pipeline_file>", helpText: "Use the pass pipeline defined by <pass_pipeline_file>")
   public static let externalPluginPath: Option = Option("-external-plugin-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>#<plugin-server-path>", helpText: "Add directory to the plugin search path with a plugin server executable", group: .pluginSearch)
@@ -671,7 +671,7 @@ extension Option {
   public static let indexSystemModules: Option = Option("-index-system-modules", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Emit index data for imported serialized swift system modules")
   public static let indexUnitOutputPathFilelist: Option = Option("-index-unit-output-path-filelist", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], helpText: "Specify index unit output paths in a file rather than on the command line")
   public static let indexUnitOutputPath: Option = Option("-index-unit-output-path", .separate, attributes: [.frontend, .argumentIsPath, .cacheInvariant], metaVar: "<path>", helpText: "Use <path> as the output path in the produced index data.")
-  public static let inputFileKey: Option = Option("-input-file-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for input file")
+  public static let inputFileKey: Option = Option("-input-file-key", .separate, attributes: [.helpHidden, .frontend, .noDriver, .argumentIsCASID], helpText: "Cache Key for input file")
   public static let inputPaths: Option = Option("-input-paths", .separate, attributes: [.noDriver, .argumentIsPath], metaVar: "<path>", helpText: "The SDK contents under comparison")
   public static let inputPaths_: Option = Option("--input-paths", .separate, alias: Option.inputPaths, attributes: [.noDriver, .argumentIsPath], metaVar: "<path>", helpText: "The SDK contents under comparison")
   public static let swiftinterfaceCompilerVersion: Option = Option("-interface-compiler-version", .separate, attributes: [.helpHidden, .frontend], metaVar: "<intcvers>", helpText: "The version of the Swift compiler used to generate a .swiftinterface file")
@@ -959,7 +959,7 @@ extension Option {
   public static let suppressWarnings: Option = Option("-suppress-warnings", .flag, attributes: [.frontend], helpText: "Suppress all warnings")
   public static let swiftAsyncFramePointerEQ: Option = Option("-swift-async-frame-pointer=", .joined, attributes: [.helpHidden, .frontend, .noDriver], helpText: "One of 'auto', 'always' or 'never'")
   public static let swiftModuleCrossImport: Option = Option("-swift-module-cross-import", .multiArg, attributes: [.frontend, .noDriver], metaVar: "<moduleName> <crossImport.swiftoverlay>", helpText: "Specify the cross import module", numArgs: 2)
-  public static let swiftModuleFile: Option = Option("-swift-module-file=", .joined, attributes: [.frontend, .noDriver], metaVar: "<name>=<path>", helpText: "Specify Swift module input explicitly built from textual interface")
+  public static let swiftModuleFile: Option = Option("-swift-module-file=", .joined, attributes: [.frontend, .noDriver, .argumentIsCASID], metaVar: "<name>=<path>", helpText: "Specify Swift module input explicitly built from textual interface")
   public static let swiftOnly: Option = Option("-swift-only", .flag, attributes: [.noDriver], helpText: "Only include APIs defined from Swift source")
   public static let swiftOnly_: Option = Option("--swift-only", .flag, alias: Option.swiftOnly, attributes: [.noDriver], helpText: "Only include APIs defined from Swift source")
   public static let swiftVersion: Option = Option("-swift-version", .separate, alias: Option.languageMode, attributes: [.helpHidden, .frontend, .synthesizeInterface, .moduleInterface], metaVar: "<vers>", helpText: "Interpret input according to a specific Swift language version number")

Sources/makeOptions/makeOptions.cpp

@@ -346,6 +346,7 @@ int makeOptions_main() {
         emitFlagIf(swift::options::SupplementaryOutput, ".supplementaryOutput");
         emitFlagIf(swift::options::ArgumentIsFileList, ".argumentIsFileList");
         emitFlagIf(swift::options::CacheInvariant, ".cacheInvariant");
+        emitFlagIf(swift::options::ArgumentIsCASID, ".argumentIsCASID");
         out << "]";
       }
 

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -1505,6 +1505,46 @@ struct CachingBuildTests {
     }
   }
 
+  @Test(.requireScannerSupportsComputingMultipleCacheKeys())
+  func cacheBatchBuildMultipleCacheKeys() async throws {
+    try await withTemporaryDirectory { path in
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      let casPath = path.appending(component: "cas")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let inputs = (0..<4).map { path.appending(component: "file\($0).swift") }
+      for (index, input) in inputs.enumerated() {
+        try localFileSystem.writeFileContents(input) {
+          $0.send("public func foo\(index)() {}")
+        }
+      }
+      var driver = try TestDriver(args: [
+        "swiftc", "-c", "-module-name", "Test", "-parse-stdlib",
+        "-disable-implicit-string-processing-module-import",
+        "-disable-implicit-concurrency-module-import",
+        "-explicit-module-build", "-enable-batch-mode", "-driver-batch-count", "2",
+        "-module-cache-path", moduleCachePath.nativePathString(escaped: false),
+        "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: false),
+        "-working-directory", path.nativePathString(escaped: false),
+      ] + inputs.map { $0.nativePathString(escaped: false) })
+      let jobs = try await driver.planBuild()
+      let cas = try #require(driver.cas)
+      let resolver = try ArgsResolver(fileSystem: localFileSystem)
+
+      // The keys computed for all the inputs of a job in one call are the same
+      // as the keys computed for each input.
+      let compileJobs = jobs.filter { $0.kind == .compile }
+      #expect(compileJobs.count == 2)
+      for job in compileJobs {
+        #expect(job.outputCacheKeys.count == job.primaryInputs.count)
+        let arguments: [String] = try resolver.resolveArgumentList(for: job.commandLine)
+        for (input, key) in job.outputCacheKeys {
+          let index = try #require(job.inputs.firstIndex(of: input))
+          #expect(try cas.computeCacheKey(commandLine: arguments, index: index) == key)
+        }
+      }
+    }
+  }
+
   @Test func cacheWholeModuleBuildPlan() async throws {
     try await withTemporaryDirectory { path in
       let moduleCachePath = path.appending(component: "ModuleCache")