diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 1fe9ce390..43b67aea7 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -239,6 +239,11 @@ typedef struct { (*swiftscan_scan_invocation_get_argc)(swiftscan_scan_invocation_t); swiftscan_string_set_t * (*swiftscan_scan_invocation_get_argv)(swiftscan_scan_invocation_t); + // Optional: not present in older libSwiftScan versions. + void + (*swiftscan_scan_invocation_set_main_executable_path)(swiftscan_scan_invocation_t, const char *); + swiftscan_string_ref_t + (*swiftscan_scan_invocation_get_main_executable_path)(swiftscan_scan_invocation_t); //=== Cleanup Functions ---------------------------------------------------===// void diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index fc4456218..d481954fb 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -13,6 +13,7 @@ add_library(SwiftDriver "ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift" "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift" "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift" + "ExplicitModuleBuilds/InterModuleDependencies/TestHarnessInterModuleDependencyOracle.swift" SwiftScan/DependencyGraphBuilder.swift SwiftScan/Loader.swift diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index ac5c42900..30cf3bc1c 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -75,6 +75,7 @@ public struct Driver { case conditionalCompilationFlagIsNotValidIdentifier(String) case baselineGenerationRequiresTopLevelModule(String) case optionRequiresAnother(String, String) + case optionRequiresExplicitModuleBuild(String) case unableToCreateReproducer // Explicit Module Build Failures case malformedModuleDependency(String, String) @@ -150,6 +151,8 @@ public struct Driver { return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'" case .optionRequiresAnother(let first, let second): return "'\(first)' cannot be specified if '\(second)' is not present" + case .optionRequiresExplicitModuleBuild(let arg): + return "'\(arg)' requires explicit module builds, which are disabled for this compilation" case .unableToCreateReproducer: return "failed to create reproducer" } @@ -395,7 +398,7 @@ public struct Driver { /// If loaded module trace is emitted by scanner. lazy var loadedModuleTraceEmittedByScanner: Bool = { // check to see if -dependency-only-import flag is supported. - isFrontendArgSupported(.dependencyOnlyImport) && parsedOptions.hasArgument(.driverExplicitModuleBuild) + isFrontendArgSupported(.dependencyOnlyImport) && isExplicitModuleBuildEnabled }() /// Whether compile jobs are passed -debug-module-path. @@ -676,6 +679,14 @@ public struct Driver { /// is shared across many targets; otherwise, a new instance is created by the driver itself. @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle + /// Whether this driver invocation will build module dependencies explicitly. + /// + /// `swiftc` invocations default to explicit module builds. + /// Other driver kinds still require `-explicit-module-build` to enable + /// the mode. If `libSwiftScan` is not available, the driver falls back to + /// implicit module builds and emits a warning. + @_spi(Testing) public let isExplicitModuleBuildEnabled: Bool + /// A collection of all the flags the selected toolchain's `swift-frontend` supports public let supportedFrontendFlags: Set @@ -1075,6 +1086,17 @@ public struct Driver { let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem) self.inputFiles = inputFiles + // `-nonlib-dependency-scanner` asks for the out-of-process scanner instead + // of a libSwiftScan instance, so scanning remains possible without one. + let canScanDependencies = self.swiftScanLibInstance != nil + || parsedOptions.hasArgument(.driverScanDependenciesNonLib) + self.isExplicitModuleBuildEnabled = Self.computeIsExplicitModuleBuildEnabled( + &parsedOptions, + driverKind: self.driverKind, + inputFiles: inputFiles, + canScanDependencies: canScanDependencies, + diagnosticsEngine: diagnosticEngine) + let incrementalFileHashes = parsedOptions.hasFlag(positive: .enableIncrementalFileHashing, negative: .disableIncrementalFileHashing, default: false) @@ -1142,13 +1164,17 @@ public struct Driver { fileSystem: fileSystem, workingDirectory: workingDirectory, diagnosticEngine: diagnosticEngine) - Self.validateEmitDependencyGraphArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) + Self.validateEmitDependencyGraphArgs(&parsedOptions, + isExplicitModuleBuildEnabled: self.isExplicitModuleBuildEnabled, + diagnosticEngine: diagnosticEngine) Self.validateValidateClangModulesOnceOptions(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateParseableOutputArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateCompilationConditionArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateFrameworkSearchPathArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateCoverageArgs(&parsedOptions, diagnosticsEngine: diagnosticEngine) - Self.validateLinkArgs(&parsedOptions, diagnosticsEngine: diagnosticEngine) + Self.validateLinkArgs(&parsedOptions, + isExplicitModuleBuildEnabled: self.isExplicitModuleBuildEnabled, + diagnosticsEngine: diagnosticEngine) try toolchain.validateArgs(&parsedOptions, targetTriple: self.frontendTargetInfo.target.triple, targetVariantTriple: self.frontendTargetInfo.targetVariant?.triple, @@ -1228,7 +1254,7 @@ public struct Driver { // Caching options. let cachingEnabled = parsedOptions.hasArgument(.cacheCompileJob) || env.keys.contains("SWIFT_ENABLE_CACHING") if cachingEnabled { - if !parsedOptions.hasArgument(.driverExplicitModuleBuild) { + if !self.isExplicitModuleBuildEnabled { diagnosticsEngine.emit(.warning("-cache-compile-job cannot be used without explicit module build, turn off caching"), location: nil) self.enableCaching = false @@ -2115,6 +2141,7 @@ extension Driver { jobExecutionDelegate: JobExecutionDelegate, forceResponseFiles: Bool ) throws { + try createPrecompiledBridgingHeaderDirectories(for: allJobs) let continueBuildingAfterErrors = computeContinueBuildingAfterErrors() try executor.execute( workload: .init(allJobs, @@ -2126,6 +2153,24 @@ extension Driver { recordedInputMetadata: recordedInputMetadata) } + /// An implicit build passes `-pch-output-dir` to the frontend, which creates + /// the directory. An explicit build names the PCH itself and passes it with + /// `-o`, which the frontend opens without creating its directory. + private func createPrecompiledBridgingHeaderDirectories(for jobs: [Job]) throws { + guard isExplicitModuleBuildEnabled, parsedOptions.hasArgument(.pchOutputDir) else { return } + let base = workingDirectory ?? fileSystem.currentWorkingDirectory + for job in jobs where job.kind == .generatePCH { + for output in job.outputs where output.type == .pch { + var directory = output.file.parentDirectory + if let base = base { + directory = directory.resolvedRelativePath(base: base) + } + guard let path = directory.absolutePath, !fileSystem.exists(path) else { continue } + try fileSystem.createDirectory(path, recursive: true) + } + } + } + public func writeIncrementalBuildInformation(_ jobs: [Job]) { // In case the write fails, don't crash the build. // A mitigation to rdar://76359678. @@ -3429,12 +3474,12 @@ extension Driver { } static func validateEmitDependencyGraphArgs(_ parsedOptions: inout ParsedOptions, + isExplicitModuleBuildEnabled: Bool, diagnosticEngine: DiagnosticsEngine) { - // '-print-explicit-dependency-graph' requires '-explicit-module-build' + // '-print-explicit-dependency-graph' requires an explicit module build if parsedOptions.hasArgument(.printExplicitDependencyGraph) && - !parsedOptions.hasArgument(.driverExplicitModuleBuild) { - diagnosticEngine.emit(.error(Error.optionRequiresAnother(Option.printExplicitDependencyGraph.spelling, - Option.driverExplicitModuleBuild.spelling)), + !isExplicitModuleBuildEnabled { + diagnosticEngine.emit(.error(Error.optionRequiresExplicitModuleBuild(Option.printExplicitDependencyGraph.spelling)), location: nil) } // '-explicit-dependency-graph-format=' requires '-print-explicit-dependency-graph' @@ -3609,7 +3654,9 @@ extension Driver { } } - private static func validateLinkArgs(_ parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine) { + private static func validateLinkArgs(_ parsedOptions: inout ParsedOptions, + isExplicitModuleBuildEnabled: Bool, + diagnosticsEngine: DiagnosticsEngine) { if parsedOptions.hasArgument(.experimentalHermeticSealAtLink) { if parsedOptions.hasArgument(.enableLibraryEvolution) { diagnosticsEngine.emit(.error_hermetic_seal_cannot_have_library_evolution) @@ -3622,9 +3669,8 @@ extension Driver { } if parsedOptions.hasArgument(.explicitAutoLinking) { - if !parsedOptions.hasArgument(.driverExplicitModuleBuild) { - diagnosticsEngine.emit(.error(Error.optionRequiresAnother(Option.explicitAutoLinking.spelling, - Option.driverExplicitModuleBuild.spelling)), + if !isExplicitModuleBuildEnabled { + diagnosticsEngine.emit(.error(Error.optionRequiresExplicitModuleBuild(Option.explicitAutoLinking.spelling)), location: nil) } } @@ -3785,10 +3831,15 @@ extension Driver { } let swiftScanLibPath: AbsolutePath? = compilerIntegratedTooling ? nil : try toolchain.lookupSwiftScanLib() + let willFallBackToScanningOutOfProcess = + parsedOptions.hasFlag(positive: .driverExplicitModuleBuild, + negative: .driverNoExplicitModuleBuild) == true do { guard compilerIntegratedTooling || (swiftScanLibPath != nil && fileSystem.exists(swiftScanLibPath!)) else { - diagnosticsEngine.emit(.warn_scan_dylib_not_found()) + if willFallBackToScanningOutOfProcess { + diagnosticsEngine.emit(.warn_scan_dylib_not_found()) + } return nil } @@ -3797,11 +3848,73 @@ extension Driver { // The driver needs a reference to this for non-scanning tasks return interModuleDependencyOracle.getScannerInstance() } catch { - diagnosticsEngine.emit(.warn_scan_dylib_load_failed(swiftScanLibPath?.description ?? "built-in")) + if willFallBackToScanningOutOfProcess { + diagnosticsEngine.emit(.warn_scan_dylib_load_failed(swiftScanLibPath?.description ?? "built-in")) + } } return nil } + /// Whether the requested action only parses its inputs, and so never loads + /// modules. Mirrors `FrontendOptions::shouldActionOnlyParse`. + private static func isParseOnlyAction(_ parsedOptions: inout ParsedOptions) -> Bool { + guard let mode = parsedOptions.getLast(in: .modes) else { return false } + switch mode.option { + case .parse, .dumpParse, .dumpInterfaceHash: + // Purely syntactic: these never resolve an import. + return true + case .emitImportedModules, .scanDependencies: + // These report what a module imports rather than loading it, and the + // frontend discovers that itself. Scanning first would duplicate the + // frontend's own work and fail on imports it would merely have listed. + return true + default: + return false + } + } + + /// Determine whether this invocation should build module dependencies explicitly. + /// + /// Without an explicit `-explicit-module-build` or `-no-explicit-module-build`, + /// in `swiftc` batch mode, explicit module builds are enabled by default + /// unless the action only parses its inputs, there are no Swift source files + /// to scan, the language mode is Swift 4, or dependencies cannot be scanned + /// at all. + static func computeIsExplicitModuleBuildEnabled( + _ parsedOptions: inout ParsedOptions, + driverKind: DriverKind, + inputFiles: [TypedVirtualPath], + canScanDependencies: Bool, + diagnosticsEngine: DiagnosticsEngine + ) -> Bool { + if let requested = parsedOptions.hasFlag(positive: .driverExplicitModuleBuild, + negative: .driverNoExplicitModuleBuild) { + return requested + } + guard driverKind == .batch else { return false } + // Scanning for an action that never loads modules is wasted work, and it + // turns an unresolvable import into an error the action would not hit. + guard !isParseOnlyAction(&parsedOptions) else { return false } + // The scanner reads only Swift source files, so there is nothing to scan + // when compiling SIL or building and dumping Clang modules. Standard input + // must be left for the job that compiles it, since it can be read once. + guard inputFiles.contains(where: { $0.type == .swift && $0.file != .standardInput }) else { + return false + } + // An explicit build imports each SDK interface's Clang dependencies with + // this compilation's API notes version, and Swift 4 API notes rename APIs + // that interfaces written for later language modes use. `-swift-version` + // is an alias of `-language-mode`. + guard parsedOptions.getLastArgument(.languageMode)?.asSingle != "4" else { return false } + guard canScanDependencies else { + diagnosticsEngine.emit( + .warning("libSwiftScan is unavailable; disabling the default explicit module build for this swiftc invocation. Pass -explicit-module-build to force it on, or -no-explicit-module-build to silence this warning."), + location: nil) + return false + } + return true + } + static func computeToolchain( _ parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine, diff --git a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift index f25e4edc1..99518a8cb 100644 --- a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift +++ b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift @@ -131,7 +131,15 @@ import class TSCBasic.Process break case .regular, .verbose: - let output = (try? result.utf8Output() + result.utf8stderrOutput()) ?? "" + // A job whose output is standard output (`-o -`) writes its product + // there, which may be binary, so pass those bytes through unchanged. + // `-o -` reaches the job as an output path named `-`. + let writesProductToStdout = job.outputs.contains { $0.file.name == "-" } + if writesProductToStdout, let product = try? result.output.get(), !product.isEmpty { + stdoutStream.send(product) + stdoutStream.flush() + } + let output = (try? (writesProductToStdout ? "" : result.utf8Output()) + result.utf8stderrOutput()) ?? "" if !output.isEmpty { Driver.stdErrQueue.sync { stderrStream.send(output) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index e8f57798f..44ac449be 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -36,25 +36,34 @@ public class InterModuleDependencyOracle { /// Allow external clients to instantiate the oracle public init() {} + /// - Parameter compilerExecutablePath: The compiler executable that will run the + /// jobs of the modules this scan reports. The scanner finds tools and libraries + /// relative to it, so a client that knows it should pass it, or the scan and + /// the jobs can find different installations of, for example, the C++ standard + /// library. @_spi(Testing) public func getDependencies(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, moduleAliases: [String: String]? = nil, commandLine: [String], diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { precondition(hasScannerInstance) return try swiftScanLibInstance!.scanDependencies(workingDirectory: workingDirectory, + compilerExecutablePath: compilerExecutablePath, moduleAliases: moduleAliases, invocationCommand: commandLine, diagnostics: &diagnostics) } @_spi(Testing) public func getImports(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, moduleAliases: [String: String]? = nil, commandLine: [String], diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { precondition(hasScannerInstance) return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory, + compilerExecutablePath: compilerExecutablePath, moduleAliases: moduleAliases, invocationCommand: commandLine, diagnostics: &diagnostics) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/TestHarnessInterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/TestHarnessInterModuleDependencyOracle.swift new file mode 100644 index 000000000..46740880f --- /dev/null +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/TestHarnessInterModuleDependencyOracle.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import struct TSCBasic.AbsolutePath + +/// Answers dependency scans with a graph holding only the main module, so driver tests plan without scanning sources. +@_spi(Testing) public final class TestHarnessInterModuleDependencyOracle: InterModuleDependencyOracle { + @_spi(Testing) public override func getDependencies(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, + moduleAliases: [String: String]? = nil, + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) + throws -> InterModuleDependencyGraph { + guard let flagIndex = commandLine.firstIndex(of: "-module-name"), + flagIndex + 1 < commandLine.count else { + throw DependencyScanningError.dependencyScanFailed( + "TestHarnessInterModuleDependencyOracle: the scan command line has no -module-name") + } + let moduleName = commandLine[flagIndex + 1] + let modulePath = try VirtualPath.intern(path: "\(moduleName).swiftmodule") + let mainModule = ModuleInfo(modulePath: TextualVirtualPath(path: modulePath), + libraryLevel: nil, + sourceFiles: [], + directDependencies: [], + linkLibraries: [], + importInfos: [], + details: .swift(SwiftModuleDetails(bridgingPchCommandLine: nil))) + return InterModuleDependencyGraph(mainModuleName: moduleName, + modules: [.swift(moduleName): mainModule]) + } + + @_spi(Testing) public override func getImports(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, + moduleAliases: [String: String]? = nil, + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) + throws -> InterModuleDependencyImports { + InterModuleDependencyImports(imports: [], moduleAliases: moduleAliases) + } + + // With no scanner-built PCH command line in the graph, the driver must build the PCH command itself. + @_spi(Testing) public override var supportsBridgingHeaderPCHCommand: Bool { false } + + public override func getOrCreateCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, + pluginOptions: [(String, String)]) throws -> SwiftScanCAS { + throw DependencyScanningError.casError( + "TestHarnessInterModuleDependencyOracle does not support compilation caching; mark the test with .realDependencyScan") + } +} diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index dfe847c43..2c468b47d 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -272,8 +272,11 @@ public extension Driver { useResponseFiles: .disabled, using: executor.resolver) Self.sanitizeCommandForLibScanInvocation(&command) + // The scan finds tools and libraries relative to the compiler that runs the jobs. + let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path do { imports = try interModuleDependencyOracle.getImports(workingDirectory: cwd, + compilerExecutablePath: compilerExecutablePath, moduleAliases: moduleOutputInfo.aliases, commandLine: command, diagnostics: &scanDiagnostics) @@ -333,8 +336,11 @@ public extension Driver { useResponseFiles: .disabled, using: executor.resolver) Self.sanitizeCommandForLibScanInvocation(&command) + // The scan finds tools and libraries relative to the compiler that runs the jobs. + let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path do { dependencyGraph = try interModuleDependencyOracle.getDependencies(workingDirectory: cwd, + compilerExecutablePath: compilerExecutablePath, moduleAliases: moduleOutputInfo.aliases, commandLine: command, diagnostics: &scanDiagnostics) diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index 02a2cff99..6718a99d8 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -111,7 +111,7 @@ extension IncrementalCompilationState { default: true) { driver.diagnosticEngine.emit(.warning("ignoring obseleted option -disable-incremental-imports")) } - if driver.parsedOptions.contains(.driverExplicitModuleBuild) { + if driver.isExplicitModuleBuildEnabled { options.formUnion(.explicitModuleBuild) } if driver.parsedOptions.hasFlag(positive: .enableIncrementalFileHashing, diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index bf5e902c8..8ed68b9ce 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -83,7 +83,7 @@ extension Driver { break } - let isPlanJobForExplicitModule = parsedOptions.contains(.driverExplicitModuleBuild) && explicitModulePlanner != nil + let isPlanJobForExplicitModule = isExplicitModuleBuildEnabled && explicitModulePlanner != nil let jobNeedPathRemap: Bool // If in ExplicitModuleBuild mode and the dependency graph has been computed, add module // dependencies. @@ -206,7 +206,7 @@ extension Driver { // In explicit module build mode, the module cache and Clang build session // are only consulted by the dependency scanner. - if !parsedOptions.contains(.driverExplicitModuleBuild) || kind == .scanDependencies { + if !isExplicitModuleBuildEnabled || kind == .scanDependencies { try commandLine.appendLast(.moduleCachePath, from: &parsedOptions) if isFrontendArgSupported(.validateClangModulesOnce), isFrontendArgSupported(.clangBuildSessionFile) { @@ -317,7 +317,9 @@ extension Driver { try commandLine.appendLast(.lto, from: &parsedOptions) try commandLine.appendLast(.accessNotesPath, from: &parsedOptions) try commandLine.appendLast(.enableActorDataRaceChecks, .disableActorDataRaceChecks, from: &parsedOptions) - if isFrontendArgSupported(.dumpAstFormat) { + // Only the job that dumps the AST takes a format for it; the dependency + // scanner rejects the flag. + if kind == .compile && isFrontendArgSupported(.dumpAstFormat) { try commandLine.appendLast(.dumpAstFormat, from: &parsedOptions) } try commandLine.appendAll(.D, from: &parsedOptions) @@ -516,7 +518,7 @@ extension Driver { // the pch in the pchOutputDir and can start an implicit build in case // of a lookup failure. if parsedOptions.contains(.pchOutputDir) && - !parsedOptions.contains(.driverExplicitModuleBuild) { + !isExplicitModuleBuildEnabled { commandLine.appendFlag(importBridgingHeaderFlag) try addPathArgument(VirtualPath.lookup(importedObjCHeader), to:&commandLine, remap: jobNeedPathRemap) try commandLine.appendLast(.pchOutputDir, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index 1fafd4e0f..fab11f0da 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -68,7 +68,7 @@ extension Driver { // New compute and add inputs and outputs. if parsedOptions.hasArgument(.pchOutputDir) && - !parsedOptions.contains(.driverExplicitModuleBuild) { + !isExplicitModuleBuildEnabled { try commandLine.appendLast(.pchOutputDir, from: &parsedOptions) } else { commandLine.appendFlag(.o) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index b9eecec65..885ab8e2a 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -110,7 +110,7 @@ extension Driver { /// graph already, it is safe to re-use without repeating the scan. private mutating func configureExplicitModulePlanner(forVariantModule: Bool = false) throws -> ExplicitDependencyBuildPlanner? { - if (parsedOptions.contains(.driverExplicitModuleBuild) || + if (isExplicitModuleBuildEnabled || parsedOptions.contains(.explainModuleDependency)) && inputFiles.contains(where: { $0.type.isPartOfSwiftCompilation }) { let interModuleDependencyGraph = try scanModuleDependencies(forVariantModule: forVariantModule) @@ -230,7 +230,7 @@ extension Driver { // If asked, add jobs to precompile module dependencies. Otherwise exit. // We may have a dependency graph but not be required to add pre-compile jobs to the build plan, // for example when `-explain-dependency` is being used. - guard parsedOptions.contains(.driverExplicitModuleBuild) else { return } + guard isExplicitModuleBuildEnabled else { return } // Filter out the tasks for building module dependencies which are already up-to-date let mandatoryModuleCompileJobs: [Job] @@ -747,7 +747,7 @@ extension Driver { case .immediate: var jobs: [Job] = [] // Run the dependency scanner if this is an explicit module build - let explicitModulePlanner = parsedOptions.contains(.driverExplicitModuleBuild) ? try configureExplicitModulePlanner() : nil + let explicitModulePlanner = isExplicitModuleBuildEnabled ? try configureExplicitModulePlanner() : nil try addPrecompileModuleDependenciesJobs(explicitModulePlanner: explicitModulePlanner, incrementalRemarks: false, addJob: { jobs.append($0) }) diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index bfc00604e..a8ccb4a1b 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -59,7 +59,7 @@ extension Driver { outputs.append(TypedVirtualPath(file: outputPath, type: .diagnostics)) } - if parsedOptions.contains(.driverExplicitModuleBuild) && supportExplicitModuleVerifyInterface() { + if isExplicitModuleBuildEnabled && supportExplicitModuleVerifyInterface() { commandLine.appendFlag("-explicit-interface-module-build") if let key = try computeCacheKeyForInterface(emitModuleJob: emitModuleJob, interfaceKind: interfaceInput.type) { commandLine.appendFlag("-input-file-key") diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 56ea82634..393266921 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -163,7 +163,25 @@ private extension String { dylib.leak() } + /// Tell the scanner which compiler executable will run the jobs of this scan. + /// + /// The scanner finds tools and libraries, such as the C++ standard library, + /// relative to the compiler, as the compiler does. When the scanner and the + /// jobs it plans are not told the same executable, they can find different + /// installations. Libraries that predate this query ignore the path. + private func setMainExecutablePath(_ compilerExecutablePath: AbsolutePath?, + of invocation: swiftscan_scan_invocation_t?) { + guard let compilerExecutablePath, + let setMainExecutablePath = api.swiftscan_scan_invocation_set_main_executable_path else { + return + } + compilerExecutablePath.description.withCString { + setMainExecutablePath(invocation, $0) + } + } + func preScanImports(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, moduleAliases: [String: String]?, invocationCommand: [String], diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { @@ -174,6 +192,7 @@ private extension String { workingDirectory .description .cString(using: String.Encoding.utf8)) + setMainExecutablePath(compilerExecutablePath, of: invocation) withArrayOfCStrings(invocationCommand) { invocationStringArray in api.swiftscan_scan_invocation_set_argv(invocation, Int32(invocationCommand.count), @@ -198,6 +217,7 @@ private extension String { } func scanDependencies(workingDirectory: AbsolutePath, + compilerExecutablePath: AbsolutePath? = nil, moduleAliases: [String: String]?, invocationCommand: [String], diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { @@ -208,6 +228,7 @@ private extension String { workingDirectory .description .cString(using: String.Encoding.utf8)) + setMainExecutablePath(compilerExecutablePath, of: invocation) withArrayOfCStrings(invocationCommand) { invocationStringArray in api.swiftscan_scan_invocation_set_argv(invocation, Int32(invocationCommand.count), @@ -474,6 +495,12 @@ private extension swiftscan_functions_t { self.swiftscan_compiler_target_info_query_v2 = loadOptional("swiftscan_compiler_target_info_query_v2") + // Telling the scanner which compiler runs the scanned jobs + self.swiftscan_scan_invocation_set_main_executable_path = + loadOptional("swiftscan_scan_invocation_set_main_executable_path") + self.swiftscan_scan_invocation_get_main_executable_path = + loadOptional("swiftscan_scan_invocation_get_main_executable_path") + // Scanner diagnostic emission query self.swiftscan_diagnostic_get_message = loadOptional("swiftscan_diagnostic_get_message") diff --git a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift index 9a0e57092..540f5a6e3 100644 --- a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift @@ -440,7 +440,7 @@ public final class DarwinToolchain: Toolchain { // If not specified otherwise, we should use the same triple as -target if !driver.parsedOptions.hasArgument(.disableClangTarget) && driver.isFrontendArgSupported(.clangTarget) && - driver.parsedOptions.contains(.driverExplicitModuleBuild) { + driver.isExplicitModuleBuildEnabled { // The common target triple for all Clang dependencies of this compilation, // both direct and transitive is computed as: // 1. An explicitly-specified `-clang-target` argument to this driver invocation diff --git a/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift b/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift index e1c67be7a..7964eda15 100644 --- a/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift +++ b/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift @@ -191,6 +191,10 @@ public func getSingleFrontendInvocationFromDriverArgumentsV5(driverPath: String, // frontend command. args.append("-whole-module-optimization") + // For now, disable explicitly-built modules since that would require + // additional tasks beyond the single frontend command. + args.append("-no-explicit-module-build") + // Explicitly disable batch mode to avoid a spurious warning when combining // -enable-batch-mode with -whole-module-optimization. This is an // implementation detail. diff --git a/Tests/IncrementalTestFramework/CompiledSourceCollector.swift b/Tests/IncrementalTestFramework/CompiledSourceCollector.swift index a9bd47628..0ba8282aa 100644 --- a/Tests/IncrementalTestFramework/CompiledSourceCollector.swift +++ b/Tests/IncrementalTestFramework/CompiledSourceCollector.swift @@ -26,20 +26,36 @@ struct CompiledSourceCollector { private var collectedCompiledBasenames = [String]() private var collectedReadDependencies = Set() + /// Lifecycle messages for jobs that build a module rather than compile + /// sources. They share the `Compiling ` prefix with a source compile, so the + /// text after that prefix is a module description, not a list of files. + private static let moduleBuildDescriptions = [ + "Swift module", + "Clang module", + "bridging header", + ] + private func getCompiledBasenames(from d: Diagnostic) -> [String] { let dd = d.description guard let startOfSources = dd.range(of: "Starting Compiling ")?.upperBound else { return [] } - return dd.suffix(from: startOfSources) + let sources = dd.suffix(from: startOfSources) + guard !Self.moduleBuildDescriptions.contains(where: { sources.hasPrefix($0) }) + else { + return [] + } + return sources .split(separator: ",") .map {$0.drop(while: {$0 == " "})} - .map { (s: Substring) -> Substring in - assert(s.hasSuffix(".swift")) - return s + .compactMap { (s: Substring) -> String? in + guard s.hasSuffix(".swift") else { + Issue.record("Job lifecycle reported compiling '\(s)', which is not a Swift source") + return nil + } + return String(s) } - .compactMap {String($0)} } private func getReadDependencies(from d: Diagnostic) -> String? { diff --git a/Tests/IncrementalTestFramework/Module.swift b/Tests/IncrementalTestFramework/Module.swift index d55f990ee..3a1b52b8c 100644 --- a/Tests/IncrementalTestFramework/Module.swift +++ b/Tests/IncrementalTestFramework/Module.swift @@ -143,7 +143,12 @@ extension Module { let diagnosticsEngine = DiagnosticsEngine(handlers: handlers) let args = try arguments(in: context) - var driver = try TestDriver(args: args, diagnosticsEngine: diagnosticsEngine) + // These builds run their compiles against the real toolchain, so they get + // the real oracle rather than the test-harness one. It goes unqueried while + // `-no-explicit-module-build` is in `boilerPlateArgs`, but it is the right + // oracle for a build that executes. + var driver = try TestDriver(args: args, diagnosticsEngine: diagnosticsEngine, + interModuleDependencyOracle: InterModuleDependencyOracle()) let jobs = try await driver.planBuild() try await driver.run(jobs: jobs) @@ -157,7 +162,11 @@ extension Module { "-no-color-diagnostics", "-incremental", "-driver-show-incremental", - "-driver-show-job-lifecycle"] + "-driver-show-job-lifecycle", + // These tests count which sources recompiled. Explicit module builds + // would add module-building jobs only when the module cache is cold, so + // the job set would vary between runs on the same machine. + "-no-explicit-module-build"] var searchPaths: [String] { let swiftModules = self.imports.map { diff --git a/Tests/SwiftDriverTests/APIDigesterTests.swift b/Tests/SwiftDriverTests/APIDigesterTests.swift index 8afe26f70..edaf919a7 100644 --- a/Tests/SwiftDriverTests/APIDigesterTests.swift +++ b/Tests/SwiftDriverTests/APIDigesterTests.swift @@ -333,7 +333,7 @@ import Testing } } -@Suite struct APIDigesterBuildTests { +@Suite(.realDependencyScan) struct APIDigesterBuildTests { @Test(.disabled("rdar://82302797")) func baselineGenerationEndToEnd() async throws { try await withTemporaryDirectory { path in let source = path.appending(component: "foo.swift") diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index a07885fa4..df2922464 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -192,7 +192,7 @@ private func checkCachingBuildJobDependencies( } } -@Suite(.enabled(if: cachingFeatureSupported, "caching not supported")) +@Suite(.enabled(if: cachingFeatureSupported, "caching not supported"), .realDependencyScan) struct CachingBuildTests { /// A scenario exercising `-supplementary-output-file-map` emission with diff --git a/Tests/SwiftDriverTests/CompileJobTests.swift b/Tests/SwiftDriverTests/CompileJobTests.swift index df074493c..376968a8a 100644 --- a/Tests/SwiftDriverTests/CompileJobTests.swift +++ b/Tests/SwiftDriverTests/CompileJobTests.swift @@ -373,7 +373,7 @@ import Testing let plannedJobs = try await driver.planBuild().removingAutolinkExtractJobs() #expect(plannedJobs.count == 1) #expect(plannedJobs[0].kind == .compile) - expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs.count, 3) try expectEqual(plannedJobs[0].inputs[0].file, try toPath("main.swift")) try expectEqual(plannedJobs[0].inputs[1].file, try toPath("multi-threaded.swift")) #expect(plannedJobs[0].outputs.count == (driver.targetTriple.isDarwin ? 4 : 3)) @@ -976,11 +976,12 @@ import Testing var driver = try TestDriver(args: ["swiftc", "-typecheck", importHeaderFlag, "TestInputHeader.h", "foo.swift"]) let plannedJobs = try await driver.planBuild() #expect(plannedJobs.count == 2) - #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) + #expect(plannedJobs[0].outputs.count == 1) #expect(matchTemporary(plannedJobs[0].outputs[0].file, "TestInputHeader.pch")) expectEqual(plannedJobs[0].outputs[0].type, .pch) @@ -990,7 +991,7 @@ import Testing #expect(commandContainsTemporaryPath(plannedJobs[0].commandLine, "TestInputHeader.pch")) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) #expect(plannedJobs[1].commandLine.contains(.flag(importHeaderFlag))) #expect(commandContainsTemporaryPath(plannedJobs[1].commandLine, "TestInputHeader.pch")) @@ -1004,8 +1005,9 @@ import Testing #expect(plannedJobs.count == 1) #expect(plannedJobs[0].kind == .compile) - expectEqual(plannedJobs[0].inputs.count, 1) + expectEqual(plannedJobs[0].inputs.count, 2) try expectEqual(plannedJobs[0].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[0].inputs[1].type, .jsonSwiftArtifacts) #expect(plannedJobs[0].commandLine.contains(.flag(importHeaderFlag))) } @@ -1017,9 +1019,10 @@ import Testing #expect(plannedJobs.count == 2) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 1) #expect(matchTemporary(plannedJobs[0].outputs[0].file, "TestInputHeader.pch")) expectEqual(plannedJobs[0].outputs[0].type, .pch) @@ -1031,8 +1034,10 @@ import Testing #expect(commandContainsTemporaryPath(plannedJobs[0].commandLine, "TestInputHeader.pch")) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[1].inputs[1].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[1].inputs[2].type, .pch) } do { @@ -1043,9 +1048,10 @@ import Testing #expect(plannedJobs.count == 2) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 1) try expectEqual( plannedJobs[0].outputs[0].file.nativePathString(escaped: false), @@ -1054,13 +1060,16 @@ import Testing expectEqual(plannedJobs[0].outputs[0].type, .pch) #expect(plannedJobs[0].commandLine.contains(.flag("-frontend"))) #expect(plannedJobs[0].commandLine.contains(.flag("-emit-pch"))) - #expect(plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) - #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch")))) + #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) + #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) - #expect(plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) + expectEqual(plannedJobs[1].inputs[1].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[1].inputs[2].type, .pch) + #expect(plannedJobs[1].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) + #expect(!plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) } do { @@ -1072,8 +1081,9 @@ import Testing #expect(plannedJobs.count == 1) #expect(plannedJobs[0].kind == .compile) - expectEqual(plannedJobs[0].inputs.count, 1) + expectEqual(plannedJobs[0].inputs.count, 2) try expectEqual(plannedJobs[0].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[0].inputs[1].type, .jsonSwiftArtifacts) #expect(plannedJobs[0].commandLine.contains(.flag(importHeaderFlag))) #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) } @@ -1087,8 +1097,9 @@ import Testing #expect(plannedJobs.count == 1) #expect(plannedJobs[0].kind == .compile) - expectEqual(plannedJobs[0].inputs.count, 1) + expectEqual(plannedJobs[0].inputs.count, 2) try expectEqual(plannedJobs[0].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[0].inputs[1].type, .jsonSwiftArtifacts) #expect(plannedJobs[0].commandLine.contains(.flag(importHeaderFlag))) #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) } @@ -1102,9 +1113,10 @@ import Testing #expect(plannedJobs.count == 2) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 2) #expect(matchTemporary(plannedJobs[0].outputs[0].file, "TestInputHeader.dia")) expectEqual(plannedJobs[0].outputs[0].type, .diagnostics) @@ -1117,13 +1129,16 @@ import Testing #expect(commandContainsTemporaryPath(plannedJobs[0].commandLine, "TestInputHeader.dia")) #expect(plannedJobs[0].commandLine.contains(.flag("-frontend"))) #expect(plannedJobs[0].commandLine.contains(.flag("-emit-pch"))) - #expect(plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) - #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch")))) + #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) + #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) - #expect(plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) + expectEqual(plannedJobs[1].inputs[1].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[1].inputs[2].type, .pch) + #expect(plannedJobs[1].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) + #expect(!plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) } do { @@ -1135,9 +1150,10 @@ import Testing #expect(plannedJobs.count == 3) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 2) #expect( plannedJobs[0].outputs[0].file.name.range( @@ -1160,13 +1176,16 @@ import Testing ) #expect(plannedJobs[0].commandLine.contains(.flag("-frontend"))) #expect(plannedJobs[0].commandLine.contains(.flag("-emit-pch"))) - #expect(plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) - #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch")))) + #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) + #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) #expect(plannedJobs[1].kind == .emitModule) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) - #expect(plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) + expectEqual(plannedJobs[1].inputs[1].type, .pch) + expectEqual(plannedJobs[1].inputs[2].type, .jsonSwiftArtifacts) + #expect(plannedJobs[1].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) + #expect(!plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) // FIXME: validate that merge module is correct job and that it has correct inputs and flags } @@ -1180,9 +1199,10 @@ import Testing #expect(plannedJobs.count == 2) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 1) try expectEqual( plannedJobs[0].outputs[0].file.nativePathString(escaped: false), @@ -1191,12 +1211,15 @@ import Testing expectEqual(plannedJobs[0].outputs[0].type, .pch) #expect(plannedJobs[0].commandLine.contains(.flag("-frontend"))) #expect(plannedJobs[0].commandLine.contains(.flag("-emit-pch"))) - #expect(plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) - #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch")))) + #expect(!plannedJobs[0].commandLine.contains(.flag("-pch-output-dir"))) + #expect(plannedJobs[0].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[1].inputs[1].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[1].inputs[2].type, .pch) + #expect(plannedJobs[1].commandLine.contains(.path(try VirtualPath(path: "/pch/TestInputHeader.pch")))) #expect(!plannedJobs[1].commandLine.contains(.flag("-pch-disable-validation"))) } @@ -1208,9 +1231,10 @@ import Testing #expect(plannedJobs.count == 2) #expect(plannedJobs[0].kind == .generatePCH) - expectEqual(plannedJobs[0].inputs.count, 1) - try expectEqual(plannedJobs[0].inputs[0].file, try toPath("TestInputHeader.h")) - expectEqual(plannedJobs[0].inputs[0].type, .objcHeader) + expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs[0].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[0].inputs[1].type, .objcHeader) + try expectEqual(plannedJobs[0].inputs[1].file, try toPath("TestInputHeader.h")) #expect(plannedJobs[0].outputs.count == 1) #expect(matchTemporary(plannedJobs[0].outputs[0].file, "TestInputHeader.pch")) expectEqual(plannedJobs[0].outputs[0].type, .pch) @@ -1221,8 +1245,10 @@ import Testing #expect(commandContainsTemporaryPath(plannedJobs[0].commandLine, "TestInputHeader.pch")) #expect(plannedJobs[1].kind == .compile) - expectEqual(plannedJobs[1].inputs.count, 2) + expectEqual(plannedJobs[1].inputs.count, 3) try expectEqual(plannedJobs[1].inputs[0].file, try toPath("foo.swift")) + expectEqual(plannedJobs[1].inputs[1].type, .jsonSwiftArtifacts) + expectEqual(plannedJobs[1].inputs[2].type, .pch) } // Immediate mode doesn't generate a pch @@ -1379,8 +1405,9 @@ import Testing #expect(plannedJobs.count == 3) #expect(plannedJobs[0].kind == .generatePCH) #expect(plannedJobs[1].kind == .compile) - #expect(plannedJobs[1].inputs[0].file.extension == "swift") - #expect(plannedJobs[1].inputs[1].file.extension == "pch") + #expect(plannedJobs[1].inputs[0].type == .swift) + #expect(plannedJobs[1].inputs[1].type == .jsonSwiftArtifacts) + #expect(plannedJobs[1].inputs[2].type == .pch) } @Test func internalPCHasCompileInput() async throws { @@ -1398,8 +1425,9 @@ import Testing #expect(plannedJobs.count == 3) #expect(plannedJobs[0].kind == .generatePCH) #expect(plannedJobs[1].kind == .compile) - #expect(plannedJobs[1].inputs[0].file.extension == "swift") - #expect(plannedJobs[1].inputs[1].file.extension == "pch") + #expect(plannedJobs[1].inputs[0].type == .swift) + #expect(plannedJobs[1].inputs[1].type == .jsonSwiftArtifacts) + #expect(plannedJobs[1].inputs[2].type == .pch) } @Test func cxxInteropOptions() async throws { @@ -1629,9 +1657,10 @@ import Testing #expect(!driver.diagnosticEngine.hasErrors) #expect(plannedJobs.count == 1) #expect(plannedJobs[0].kind == .compile) - expectEqual(plannedJobs[0].inputs.count, 2) + expectEqual(plannedJobs[0].inputs.count, 3) try expectEqual(plannedJobs[0].inputs[0].file, try toPath("main.swift")) try expectEqual(plannedJobs[0].inputs[1].file, try toPath("multi-threaded.swift")) + expectEqual(plannedJobs[0].inputs[2].type, .jsonSwiftArtifacts) #expect(plannedJobs[0].outputs.count == (driver.targetTriple.isDarwin ? 4 : 3)) #expect(try plannedJobs[0].outputs[0].file == toPath("test.swiftmodule")) } diff --git a/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift b/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift index b570d510a..395a1e222 100644 --- a/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift +++ b/Tests/SwiftDriverTests/CrossModuleIncrementalBuildTests.swift @@ -19,7 +19,7 @@ import TSCBasic import TestUtilities import Testing -@Suite(.enabled(if: sdkArgumentsAvailable)) struct CrossModuleIncrementalBuildTests { +@Suite(.enabled(if: sdkArgumentsAvailable), .realDependencyScan) struct CrossModuleIncrementalBuildTests { func makeOutputFileMap( in workingDirectory: AbsolutePath, module: String, diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 88f7a5482..4ca9a3f3e 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -189,7 +189,133 @@ func getStdlibShimsPaths(_ driver: Driver) throws -> (AbsolutePath, AbsolutePath } /// Test that for the given JSON module dependency graph, valid jobs are generated -@Suite struct ExplicitModuleBuildTests { +@Suite(.realDependencyScan) struct ExplicitModuleBuildTests { + /// `-nonlib-dependency-scanner` selects the out-of-process scanner rather + /// than making scanning unavailable, so the default must stay on. + @Test func nonlibDependencyScannerKeepsExplicitModuleBuildDefault() async throws { + try await assertNoDriverDiagnostics(args: "swiftc", "-c", "foo.swift", "-nonlib-dependency-scanner") { + driver in + #expect(driver.isExplicitModuleBuildEnabled) + } + } + + /// A missing scanner library downgrades the default; it must not also + /// promise an out-of-process fallback that will not happen. + @Test func missingScannerLibraryWarnsOnlyAboutTheDowngrade() async throws { + var env = ProcessEnv.block + env["SWIFT_DRIVER_SWIFTSCAN_LIB"] = "/nonexistent/lib_InternalSwiftScan.dylib" + try await assertDriverDiagnostics(args: "swiftc", "-c", "foo.swift", env: env) { driver, verifier in + verifier.expect(.warning("libSwiftScan is unavailable; disabling the default explicit module build")) + #expect(!driver.isExplicitModuleBuildEnabled) + } + } + + /// Parse-only actions never load modules, so the default explicit module + /// build must not scan their dependencies. + @Test func parseOnlyActionsSkipDependencyScanning() async throws { + try await withTemporaryDirectory { path in + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main, bytes: "import DoesNotExist\nlet x = 1\n") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + for action in ["-parse", "-dump-parse", "-emit-imported-modules"] { + var driver = try TestDriver( + args: ["swiftc", action, main.nativePathString(escaped: false)] + sdkArgumentsForTesting + ) + let jobs = try await driver.planBuild() + #expect(!driver.diagnosticEngine.hasErrors, "\(action) should not resolve module dependencies") + #expect( + !jobs.contains { $0.commandLine.contains(.flag("-explicit-swift-module-map-file")) }, + "\(action) should not be planned as an explicit module build" + ) + } + } + } + + /// The scanner reads only Swift sources, so an invocation without any keeps + /// the default off, such as compiling SIL or building a Clang module. + @Test func inputsWithoutSwiftSourcesSkipDependencyScanning() async throws { + try await withTemporaryDirectory { path in + let sil = path.appending(component: "main.sil") + try localFileSystem.writeFileContents(sil, bytes: "sil_stage canonical\n") + let moduleMap = path.appending(component: "module.modulemap") + try localFileSystem.writeFileContents(moduleMap, bytes: "module M { header \"M.h\" }\n") + try localFileSystem.writeFileContents(path.appending(component: "M.h"), bytes: "") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + for args in [ + ["-emit-ir", sil.nativePathString(escaped: false)], + ["-emit-pcm", "-module-name", "M", moduleMap.nativePathString(escaped: false)], + ] { + var driver = try TestDriver(args: ["swiftc"] + args + sdkArgumentsForTesting) + let jobs = try await driver.planBuild() + #expect(!driver.diagnosticEngine.hasErrors, "\(args[0]) should plan without scanning") + #expect(!driver.isExplicitModuleBuildEnabled, "\(args[0]) has no Swift sources to scan") + #expect( + !jobs.contains { $0.commandLine.contains(.flag("-clang-target")) }, + "\(args[0]) should not take the explicit build's Clang target" + ) + } + + // Standard input can be read only once, by the job that compiles it, so + // the scanner must not read it first. Planning would scan, so only check + // the mode, which is decided before any input is read. + let stdinDriver = try TestDriver( + args: [ + "swiftc", "-emit-module", "-module-name", "M", + "-o", path.appending(component: "M.swiftmodule").nativePathString(escaped: false), "-", + ] + sdkArgumentsForTesting + ) + #expect(!stdinDriver.isExplicitModuleBuildEnabled, "standard input has no source file to scan") + } + } + + /// Only the job that dumps the AST takes a format for it. The dependency + /// scanner rejects `-dump-ast-format`, so it must not be forwarded there. + @Test func dumpASTFormatIsNotForwardedToTheScanner() async throws { + try await withTemporaryDirectory { path in + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main, bytes: "let x = 1\n") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + var driver = try TestDriver( + args: ["swiftc", "-dump-ast", "-dump-ast-format", "json", main.nativePathString(escaped: false)] + + sdkArgumentsForTesting + ) + guard driver.isFrontendArgSupported(.dumpAstFormat) else { return } + let scannerCommand = try driver.dependencyScannerInvocationCommand().1 + #expect(!scannerCommand.contains(.flag("-dump-ast-format"))) + let jobs = try await driver.planBuild() + #expect(!driver.diagnosticEngine.hasErrors) + let compileJob = try #require(jobs.first { $0.kind == .compile }) + #expect(compileJob.commandLine.contains(.flag("-dump-ast-format"))) + } + } + + /// An explicit build imports each SDK interface's Clang dependencies with + /// this compilation's API notes version. Swift 4 API notes rename APIs that + /// interfaces written for later modes use, so Swift 4 keeps the default off. + @Test func swift4LanguageModeKeepsImplicitModuleBuild() throws { + try withTemporaryDirectory { path in + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main, bytes: "let x = 1\n") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + for (version, expectExplicit) in [("4", false), ("4.2", true), ("5", true)] { + let driver = try TestDriver( + args: ["swiftc", "-swift-version", version, main.nativePathString(escaped: false)] + + sdkArgumentsForTesting + ) + #expect(driver.isExplicitModuleBuildEnabled == expectExplicit, "-swift-version \(version)") + } + let requested = try TestDriver( + args: ["swiftc", "-swift-version", "4", "-explicit-module-build", main.nativePathString(escaped: false)] + + sdkArgumentsForTesting + ) + #expect(requested.isExplicitModuleBuildEnabled, "an explicit request is still honored") + } + } + @Test func moduleDependencyBuildCommandGeneration() async throws { do { let driver = try TestDriver(args: [ @@ -622,7 +748,7 @@ func getStdlibShimsPaths(_ driver: Driver) throws -> (AbsolutePath, AbsolutePath // An implicit build has nothing else recording where the module came // from, so it still needs the serialized AST. - #expect(try await planLinkJob([]).passesASTPath) + #expect(try await planLinkJob(["-no-explicit-module-build"]).passesASTPath) // Dropping the AST is tied to the frontend recording this module's path // via -debug-module-path. A frontend too old to do so still needs it. @@ -1095,6 +1221,87 @@ func getStdlibShimsPaths(_ driver: Driver) throws -> (AbsolutePath, AbsolutePath } } + /// An implicit build hands `-pch-output-dir` to the frontend, which creates the + /// directory. An explicit build names the PCH itself and passes it with `-o`, + /// so the directory must exist before the PCH job runs. + @Test func explicitBuildCreatesPCHOutputDirectory() async throws { + let (stdlibPath, shimsPath, _, _) = try getDriverArtifactsForScanning() + try await withTemporaryDirectory { path in + let moduleCachePath = path.appending(component: "ModuleCache") + try localFileSystem.createDirectory(moduleCachePath) + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main, bytes: "let x = bridgedValue") + let bridgingHeader = path.appending(component: "Bridging.h") + try localFileSystem.writeFileContents(bridgingHeader, bytes: "static const int bridgedValue = 1;") + let pchOutputDir = path.appending(components: "not", "yet", "created") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + var driver = try TestDriver( + args: [ + "swiftc", + "-I", stdlibPath.nativePathString(escaped: false), + "-I", shimsPath.nativePathString(escaped: false), + "-explicit-module-build", + "-module-cache-path", moduleCachePath.nativePathString(escaped: false), + "-working-directory", path.nativePathString(escaped: false), + "-disable-implicit-concurrency-module-import", + "-disable-implicit-string-processing-module-import", + "-import-objc-header", bridgingHeader.nativePathString(escaped: false), + "-pch-output-dir", pchOutputDir.nativePathString(escaped: false), + "-c", main.nativePathString(escaped: false), + ] + sdkArgumentsForTesting + ) + let jobs = try await driver.planBuild() + let pchJob = try #require(jobs.first { $0.kind == .generatePCH }) + #expect(!pchJob.commandLine.contains(.flag("-pch-output-dir"))) + // Planning alone must not touch the file system. + #expect(!localFileSystem.exists(pchOutputDir)) + + try await driver.run(jobs: jobs) + #expect(!driver.diagnosticEngine.hasErrors) + let pchOutput = try #require(pchJob.outputs.first { $0.type == .pch }) + #expect(localFileSystem.exists(try #require(pchOutput.file.absolutePath))) + } + } + + /// With `-o -` the compile job's product is its standard output. Once module + /// jobs run alongside it, the compile runs as a child of the driver, and its + /// standard output must still reach the driver's. + @Test func explicitBuildWritesStandardOutputProductToStandardOutput() async throws { + let (stdlibPath, shimsPath, _, _) = try getDriverArtifactsForScanning() + try await withTemporaryDirectory { path in + let moduleCachePath = path.appending(component: "ModuleCache") + try localFileSystem.createDirectory(moduleCachePath) + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main, bytes: "let x = 1") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + var driver = try TestDriver( + args: [ + "swiftc", + "-I", stdlibPath.nativePathString(escaped: false), + "-I", shimsPath.nativePathString(escaped: false), + "-explicit-module-build", + "-module-cache-path", moduleCachePath.nativePathString(escaped: false), + "-working-directory", path.nativePathString(escaped: false), + "-disable-implicit-concurrency-module-import", + "-disable-implicit-string-processing-module-import", + "-emit-ir", "-o", "-", main.nativePathString(escaped: false), + ] + sdkArgumentsForTesting, + // The integrated driver discards job output; a standalone one relays it. + integratedDriver: false + ) + let jobs = try await driver.planBuild() + // A single job would run in place, replacing this process. + try #require(jobs.count > 1, "the empty module cache should add module jobs") + + try await driver.run(jobs: jobs) + #expect(!driver.diagnosticEngine.hasErrors) + #expect(driver.capturedStdout.contains("; ModuleID")) + #expect(!driver.capturedStderr.contains("; ModuleID")) + } + } + @Test func registerModuleDependencyFlag() async throws { let (stdlibPath, shimsPath, _, _) = try getDriverArtifactsForScanning() try await withTemporaryDirectory { path in @@ -3206,14 +3413,14 @@ func getStdlibShimsPaths(_ driver: Driver) throws -> (AbsolutePath, AbsolutePath do { let diagnosticEngine = DiagnosticsEngine() var driver = try TestDriver( - args: baseCommandLine + ["-print-explicit-dependency-graph"], + args: baseCommandLine + ["-print-explicit-dependency-graph", "-no-explicit-module-build"], diagnosticsEngine: diagnosticEngine ) let _ = try await driver.planBuild() #expect(diagnosticEngine.hasErrors) #expect( diagnosticEngine.diagnostics.first?.message.data.description - == "'-print-explicit-dependency-graph' cannot be specified if '-explicit-module-build' is not present" + == "'-print-explicit-dependency-graph' requires explicit module builds, which are disabled for this compilation" ) } do { diff --git a/Tests/SwiftDriverTests/Helpers/IncrementalTestHarness.swift b/Tests/SwiftDriverTests/Helpers/IncrementalTestHarness.swift index 03e47140a..b2925d442 100644 --- a/Tests/SwiftDriverTests/Helpers/IncrementalTestHarness.swift +++ b/Tests/SwiftDriverTests/Helpers/IncrementalTestHarness.swift @@ -83,6 +83,8 @@ final class IncrementalTestHarness { // "-v", "-save-temps", "-incremental", + // Explicit configs append -explicit-module-build, which takes precedence over this. + "-no-explicit-module-build", "-no-color-diagnostics", "-swift-version", "5", // Keep implicitly imported modules out of the incremental remarks; on diff --git a/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift b/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift index cceb953ec..fa49cd9b3 100644 --- a/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift +++ b/Tests/SwiftDriverTests/Helpers/TestBuildConfig.swift @@ -227,6 +227,27 @@ extension Trait where Self == KnownIssueTestTrait { } } +package struct RealDependencyScanTrait: TestTrait & SuiteTrait & TestScoping { + package var isRecursive: Bool { + true + } + + package func provideScope( + for test: Testing.Test, + testCase: Testing.Test.Case?, + performing function: @Sendable () async throws -> Void + ) async throws { + try await TestDriver.$usesRealDependencyScanner.withValue(true) { + try await function() + } + } +} + +extension Trait where Self == RealDependencyScanTrait { + /// Makes `TestDriver` use the real dependency scanner instead of `TestHarnessInterModuleDependencyOracle`. + package static var realDependencyScan: Self { Self() } +} + // MARK: - Feature availability /// A shared Driver instance used for checking feature support at test discovery time. diff --git a/Tests/SwiftDriverTests/IncrementalBasicTests.swift b/Tests/SwiftDriverTests/IncrementalBasicTests.swift index b48120f49..496a98b92 100644 --- a/Tests/SwiftDriverTests/IncrementalBasicTests.swift +++ b/Tests/SwiftDriverTests/IncrementalBasicTests.swift @@ -19,7 +19,7 @@ import TSCBasic import TestUtilities import Testing -@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available")) +@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available"), .realDependencyScan) struct IncrementalBasicTests: DiagVerifiable { @Test func incrementalDiagnostics() async throws { diff --git a/Tests/SwiftDriverTests/IncrementalExplicitBuildTests.swift b/Tests/SwiftDriverTests/IncrementalExplicitBuildTests.swift index 6b4919dee..7cd6efa97 100644 --- a/Tests/SwiftDriverTests/IncrementalExplicitBuildTests.swift +++ b/Tests/SwiftDriverTests/IncrementalExplicitBuildTests.swift @@ -19,7 +19,7 @@ import TSCBasic import TestUtilities import Testing -@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available")) +@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available"), .realDependencyScan) struct IncrementalExplicitBuildTests: DiagVerifiable { // MARK: - Simple builds diff --git a/Tests/SwiftDriverTests/IncrementalInputModificationTests.swift b/Tests/SwiftDriverTests/IncrementalInputModificationTests.swift index f9775160d..e7f46e034 100644 --- a/Tests/SwiftDriverTests/IncrementalInputModificationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalInputModificationTests.swift @@ -19,7 +19,7 @@ import TSCBasic import TestUtilities import Testing -@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available")) +@Suite(.enabled(if: sdkArgumentsAvailable, "SDK not available"), .realDependencyScan) struct IncrementalInputModificationTests: DiagVerifiable { @Test func optionsParsing() async throws { diff --git a/Tests/SwiftDriverTests/JobExecutorTests.swift b/Tests/SwiftDriverTests/JobExecutorTests.swift index e5839c222..d7eab62e5 100644 --- a/Tests/SwiftDriverTests/JobExecutorTests.swift +++ b/Tests/SwiftDriverTests/JobExecutorTests.swift @@ -118,7 +118,7 @@ extension DarwinToolchain { } } -@Suite(.serialized) struct JobExecutorTests { +@Suite(.serialized, .realDependencyScan) struct JobExecutorTests { @Test(.requireHostOS(.macosx)) func darwinBasic() async throws { let hostTriple = try TestDriver(args: ["swiftc", "test.swift"]).hostTriple let executor = try SwiftDriverExecutor( @@ -558,6 +558,7 @@ extension DarwinToolchain { "swiftc", main.pathString, "-driver-filelist-threshold", "0", "-o", outputPath.pathString, + "-no-explicit-module-build", ] + getHostToolchainSdkArg(executor), diagnosticsEngine: diags, executor: executor @@ -602,6 +603,7 @@ extension DarwinToolchain { "-ir-output-dir", path.pathString, "-driver-filelist-threshold", "0", "-o", outputPath.pathString, + "-no-explicit-module-build", ] + getHostToolchainSdkArg(executor), diagnosticsEngine: diags, executor: executor @@ -646,6 +648,7 @@ extension DarwinToolchain { "-driver-filelist-threshold", "0", "-Xfrontend", "-debug-crash-immediately", "-o", outputPath.pathString, + "-no-explicit-module-build", ] + getHostToolchainSdkArg(executor), diagnosticsEngine: diags, executor: executor diff --git a/Tests/SwiftDriverTests/LinkJobTests.swift b/Tests/SwiftDriverTests/LinkJobTests.swift index 9475cb05a..3fb8937d7 100644 --- a/Tests/SwiftDriverTests/LinkJobTests.swift +++ b/Tests/SwiftDriverTests/LinkJobTests.swift @@ -519,7 +519,11 @@ import Testing do { var driver = try TestDriver( - args: commonArgs + ["-emit-executable", "-Onone", "-emit-module", "-g", "-target", "x86_64-apple-macosx10.15"], + args: commonArgs + [ + "-emit-executable", "-Onone", "-emit-module", "-g", "-target", "x86_64-apple-macosx10.15", + // -add_ast_path is only emitted for implicit module builds. + "-no-explicit-module-build", + ], env: env ) let plannedJobs = try await driver.planBuild() @@ -1345,7 +1349,9 @@ import Testing @Test func emitModuleTrace() async throws { do { - var driver = try TestDriver(args: ["swiftc", "-typecheck", "-emit-loaded-module-trace", "foo.swift"]) + var driver = try TestDriver(args: [ + "swiftc", "-typecheck", "-emit-loaded-module-trace", "foo.swift", "-no-explicit-module-build", + ]) let plannedJobs = try await driver.planBuild() #expect(plannedJobs.count == 1) let job = plannedJobs[0] @@ -1360,7 +1366,7 @@ import Testing var driver = try TestDriver(args: [ "swiftc", "-typecheck", "-emit-loaded-module-trace", - "foo.swift", "bar.swift", "baz.swift", + "foo.swift", "bar.swift", "baz.swift", "-no-explicit-module-build", ]) let plannedJobs = try await driver.planBuild() let tracedJobs = try plannedJobs.filter { @@ -1376,7 +1382,7 @@ import Testing // opposed to the first input. var driver = try TestDriver(args: [ "swiftc", "-emit-loaded-module-trace", - "foo.o", "bar.swift", "baz.o", + "foo.o", "bar.swift", "baz.o", "-no-explicit-module-build", ]) let plannedJobs = try await driver.planBuild() let tracedJobs = try plannedJobs.filter { @@ -1394,7 +1400,7 @@ import Testing var driver = try TestDriver( args: [ "swiftc", "-typecheck", - "-emit-loaded-module-trace", "foo.swift", + "-emit-loaded-module-trace", "foo.swift", "-no-explicit-module-build", ], env: env ) @@ -1525,7 +1531,7 @@ import Testing #expect(linkJob.commandLine.contains(.flag("--gcc-toolchain=/foo/as/blarpy"))) } - @Test(.requireHostOS(.macosx, comment: "sdkArguments does not work on Linux")) func cleaningUpOldCompilationOutputs() + @Test(.requireHostOS(.macosx, comment: "sdkArguments does not work on Linux"), .realDependencyScan) func cleaningUpOldCompilationOutputs() async throws { // Build something, create an error, see if the .o and .swiftdeps files get cleaned up diff --git a/Tests/SwiftDriverTests/MiscDriverTests.swift b/Tests/SwiftDriverTests/MiscDriverTests.swift index 53b15d838..fc5e1a4e9 100644 --- a/Tests/SwiftDriverTests/MiscDriverTests.swift +++ b/Tests/SwiftDriverTests/MiscDriverTests.swift @@ -923,6 +923,7 @@ import CRT let objName = compileJob.outputs[0].file.basename let autolinkJob = plannedJobs.first { $0.kind == .autolinkExtract } let autolinkName = autolinkJob?.outputs[0].file.basename + let moduleMapName = try #require(compileJob.inputs.first(where: { $0.type == .jsonSwiftArtifacts })).file.name var serializer = DOTJobGraphSerializer(jobs: plannedJobs) var output = "" @@ -938,12 +939,15 @@ import CRT "emitModule (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" [fontsize=12]; "\(rebase("test.swift"))" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" [fontsize=12]; + "\(moduleMapName)" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; "\(rebase("test.swiftmodule"))" [fontsize=12]; "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.swiftmodule"))" [color=green]; "\(rebase("test.swiftdoc"))" [fontsize=12]; "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.swiftdoc"))" [color=green]; "compile (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" -> "compile (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" -> "compile (\(executableName("swift-frontend")))" [color=blue]; "\(objName)" [fontsize=12]; "compile (\(executableName("swift-frontend")))" -> "\(objName)" [color=green]; "autolinkExtract (\(executableName("swift-autolink-extract")))" [style=bold]; @@ -967,6 +971,8 @@ import CRT "emitModule (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" [fontsize=12]; "\(rebase("test.swift"))" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" [fontsize=12]; + "\(moduleMapName)" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; "\(rebase("test.swiftmodule"))" [fontsize=12]; "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.swiftmodule"))" [color=green]; "\(rebase("test.swiftdoc"))" [fontsize=12]; @@ -975,6 +981,7 @@ import CRT "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.abi.json"))" [color=green]; "compile (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" -> "compile (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" -> "compile (\(executableName("swift-frontend")))" [color=blue]; "\(objName)" [fontsize=12]; "compile (\(executableName("swift-frontend")))" -> "\(objName)" [color=green]; "link (\(linkerDriver))" [style=bold]; @@ -993,12 +1000,15 @@ import CRT "emitModule (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" [fontsize=12]; "\(rebase("test.swift"))" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" [fontsize=12]; + "\(moduleMapName)" -> "emitModule (\(executableName("swift-frontend")))" [color=blue]; "\(rebase("test.swiftmodule"))" [fontsize=12]; "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.swiftmodule"))" [color=green]; "\(rebase("test.swiftdoc"))" [fontsize=12]; "emitModule (\(executableName("swift-frontend")))" -> "\(rebase("test.swiftdoc"))" [color=green]; "compile (\(executableName("swift-frontend")))" [style=bold]; "\(rebase("test.swift"))" -> "compile (\(executableName("swift-frontend")))" [color=blue]; + "\(moduleMapName)" -> "compile (\(executableName("swift-frontend")))" [color=blue]; "\(objName)" [fontsize=12]; "compile (\(executableName("swift-frontend")))" -> "\(objName)" [color=green]; "link (\(linkerDriver))" [style=bold]; @@ -1535,6 +1545,8 @@ import CRT var driver = try TestDriver(args: [ "swiftc", "-validate-clang-modules-once", "-clang-build-session-file", "testClangModuleValidateOnce.session", + // Under an explicit module build these are passed only to the scanner. + "-no-explicit-module-build", "foo.swift", ]) let jobs = try await driver.planBuild().removingAutolinkExtractJobs() @@ -1725,8 +1737,9 @@ import CRT 1, "Emit module job should only have one swiftinterface output" ) - expectEqual(verifyJob.inputs.count, 1) + expectEqual(verifyJob.inputs.count, 2) expectEqual(verifyJob.inputs[0], emitInterfaceOutput[0]) + expectEqual(verifyJob.inputs[1].type, .jsonSwiftArtifacts) expectJobInvocationMatches(verifyJob, .path(emitInterfaceOutput[0].file)) #expect(!verifyJob.commandLine.contains("-downgrade-typecheck-interface-error")) #expect(!emitJob.commandLine.contains("-no-verify-emitted-module-interface")) @@ -1756,8 +1769,9 @@ import CRT "Emit module job should only have one swiftinterface output" ) expectEqual(verifyJob.kind, .verifyModuleInterface) - expectEqual(verifyJob.inputs.count, 1) + expectEqual(verifyJob.inputs.count, 2) expectEqual(verifyJob.inputs[0], emitInterfaceOutput[0]) + expectEqual(verifyJob.inputs[1].type, .jsonSwiftArtifacts) expectJobInvocationMatches(verifyJob, .path(emitInterfaceOutput[0].file)) #expect(!verifyJob.commandLine.contains("-downgrade-typecheck-interface-error")) #expect(!verifyJob.commandLine.contains("-no-downgrade-typecheck-interface-error")) @@ -1862,7 +1876,6 @@ import CRT ]) let plannedJobs = try await driver.planBuild() #expect(plannedJobs.map(\.kind) == [.emitModule, .verifyModuleInterface]) - let emitJob = try plannedJobs.findJob(.emitModule) expectJobInvocationMatches(emitJob, .flag("-check-api-availability-only")) @@ -1914,8 +1927,9 @@ import CRT 1, "There should be one public swiftinterface output" ) - expectEqual(verifyJob.inputs.count, 1) + expectEqual(verifyJob.inputs.count, 2) expectEqual(verifyJob.inputs[0], publicOutputs[0]) + expectEqual(verifyJob.inputs[1].type, .jsonSwiftArtifacts) #expect(verifyJob.outputs.isEmpty) } @@ -1961,8 +1975,9 @@ import CRT 1, "There should be one public swiftinterface output" ) - expectEqual(verifyJob.inputs.count, 1) + expectEqual(verifyJob.inputs.count, 2) expectEqual(verifyJob.inputs[0], publicOutputs[0]) + expectEqual(verifyJob.inputs[1].type, .jsonSwiftArtifacts) #expect(verifyJob.outputs.isEmpty) } } @@ -2025,8 +2040,10 @@ import CRT } } - @Test func cachingBuildOptions() async throws { - try await assertDriverDiagnostics(args: "swiftc", "foo.swift", "-emit-module", "-cache-compile-job") { + @Test(.realDependencyScan) func cachingBuildOptions() async throws { + try await assertDriverDiagnostics( + args: "swiftc", "foo.swift", "-emit-module", "-cache-compile-job", "-no-explicit-module-build" + ) { $1.expect(.warning("-cache-compile-job cannot be used without explicit module build, turn off caching")) } try await assertNoDriverDiagnostics( diff --git a/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift b/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift index e63896c49..5977ea7dd 100644 --- a/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/NonincrementalCompilationTests.swift @@ -253,6 +253,8 @@ import Testing try await assertDriverDiagnostics( args: [ "swiftc", "-module-name", "theModule", "-working-directory", path.pathString, + // These builds run their compiles; none of them exercise explicit modules. + "-no-explicit-module-build", main.pathString, other.pathString, ] + otherArgs + sdkArguments ) { driver, verifier in diff --git a/Tests/SwiftDriverTests/OutputJobTests.swift b/Tests/SwiftDriverTests/OutputJobTests.swift index bebe9bf8e..48eceff57 100644 --- a/Tests/SwiftDriverTests/OutputJobTests.swift +++ b/Tests/SwiftDriverTests/OutputJobTests.swift @@ -1216,7 +1216,7 @@ import Testing } } - @Test func supplementaryOutputFileMapUsage() async throws { + @Test(.realDependencyScan) func supplementaryOutputFileMapUsage() async throws { // Ensure filenames are escaped properly when using a supplementary output file map try await withTemporaryDirectory { path in let moduleCachePath = path.appending(component: "ModuleCache") diff --git a/Tests/SwiftDriverTests/SwiftDriverToolingInterfaceTests.swift b/Tests/SwiftDriverTests/SwiftDriverToolingInterfaceTests.swift index b9840ad74..54570c17b 100644 --- a/Tests/SwiftDriverTests/SwiftDriverToolingInterfaceTests.swift +++ b/Tests/SwiftDriverTests/SwiftDriverToolingInterfaceTests.swift @@ -23,6 +23,10 @@ import Testing try withTemporaryDirectory { path in let inputFile = path.appending(components: "test.swift") try localFileSystem.writeFileContents(inputFile) { $0.send("public func foo()") } + let mainFile = path.appending(components: "main.swift") + try localFileSystem.writeFileContents(mainFile) { $0.send("") } + let libFile = path.appending(components: "lib.swift") + try localFileSystem.writeFileContents(libFile) { $0.send("") } // Expected success scenarios: do { @@ -41,7 +45,7 @@ import Testing do { let testCommand = "-emit-executable " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV2( @@ -56,7 +60,7 @@ import Testing do { let testCommand = "-c " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV2( @@ -130,6 +134,10 @@ import Testing try withTemporaryDirectory { path in let inputFile = path.appending(components: "test.swift") try localFileSystem.writeFileContents(inputFile) { $0.send("public func foo()") } + let mainFile = path.appending(components: "main.swift") + try localFileSystem.writeFileContents(mainFile) { $0.send("") } + let libFile = path.appending(components: "lib.swift") + try localFileSystem.writeFileContents(libFile) { $0.send("") } let envBlock = ProcessEnv.block let env = envBlock.legacyVars @@ -155,7 +163,7 @@ import Testing do { let testCommand = "-emit-executable " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV3( @@ -172,7 +180,7 @@ import Testing do { let testCommand = "-c " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV3( @@ -254,6 +262,10 @@ import Testing try withTemporaryDirectory { path in let inputFile = path.appending(components: "test.swift") try localFileSystem.writeFileContents(inputFile) { $0.send("public func foo()") } + let mainFile = path.appending(components: "main.swift") + try localFileSystem.writeFileContents(mainFile) { $0.send("") } + let libFile = path.appending(components: "lib.swift") + try localFileSystem.writeFileContents(libFile) { $0.send("") } let envBlock = ProcessEnv.block let env = envBlock.legacyVars @@ -279,7 +291,7 @@ import Testing do { let testCommand = "-emit-executable " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV4( @@ -296,7 +308,7 @@ import Testing do { let testCommand = "-c " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV4( @@ -378,6 +390,10 @@ import Testing try withTemporaryDirectory { path in let inputFile = path.appending(components: "test.swift") try localFileSystem.writeFileContents(inputFile) { $0.send("public func foo()") } + let mainFile = path.appending(components: "main.swift") + try localFileSystem.writeFileContents(mainFile) { $0.send("") } + let libFile = path.appending(components: "lib.swift") + try localFileSystem.writeFileContents(libFile) { $0.send("") } let env = ProcessEnv.block let resolver = try ArgsResolver(fileSystem: localFileSystem) @@ -402,7 +418,7 @@ import Testing do { let testCommand = "-emit-executable " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV5( @@ -419,7 +435,7 @@ import Testing do { let testCommand = "-c " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header" var emittedDiagnostics: [Diagnostic] = [] #expect( !getSingleFrontendInvocationFromDriverArgumentsV5( @@ -501,13 +517,17 @@ import Testing try withTemporaryDirectory { path in let inputFile = path.appending(components: "test.swift") try localFileSystem.writeFileContents(inputFile) { $0.send("public func foo()") } + let mainFile = path.appending(components: "main.swift") + try localFileSystem.writeFileContents(mainFile) { $0.send("") } + let libFile = path.appending(components: "lib.swift") + try localFileSystem.writeFileContents(libFile) { $0.send("") } let driverPath = "swiftc" // Basic compilation test do { let testCommandStr = "-emit-executable " + inputFile.description - + " main.swift lib.swift -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" + + " \(mainFile.description) \(libFile.description) -module-name createCompilerInvocation -emit-module -emit-objc-header -o t.out" let testCommand = testCommandStr.split(separator: " ").compactMap { String($0) } // Invoke the C shim from CToolingTestShim which calls the `getSingleFrontendInvocationFromDriverArgumentsV2` diff --git a/Tests/SwiftDriverTests/TargetTests.swift b/Tests/SwiftDriverTests/TargetTests.swift index 65c10fb0c..38591c68e 100644 --- a/Tests/SwiftDriverTests/TargetTests.swift +++ b/Tests/SwiftDriverTests/TargetTests.swift @@ -550,6 +550,7 @@ import CRT "swiftc", "-target", "arm64-apple-ios12.0", "foo.swift", "-sdk", sdkRoot.pathString, + "-no-explicit-module-build", ], env: envVars ) diff --git a/Tests/SwiftDriverTests/ToolchainTests.swift b/Tests/SwiftDriverTests/ToolchainTests.swift index 12c25be04..1467eb66e 100644 --- a/Tests/SwiftDriverTests/ToolchainTests.swift +++ b/Tests/SwiftDriverTests/ToolchainTests.swift @@ -610,7 +610,7 @@ import CRT args: [ "swiftc", "-typecheck", "foo.swift", "-sdk", VirtualPath.absolute(sdkRoot).name, "-plugin-path", "PluginA", "-external-plugin-path", "Plugin~B#Bexe", "-load-plugin-library", "PluginB2", "-plugin-path", "PluginC", - "-working-directory", workingDirectory.nativePathString(escaped: false), + "-working-directory", workingDirectory.nativePathString(escaped: false), "-no-explicit-module-build", ], env: env ) @@ -760,6 +760,7 @@ import CRT "-enable-experimental-feature", "Embedded", "-sdk", VirtualPath.absolute(sdkRoot).name, "-resource-dir", resourceDir.pathString, + "-no-explicit-module-build", ]) guard driver.isFrontendArgSupported(.pluginPath) else { return } diff --git a/Tests/TestUtilities/TestDriver.swift b/Tests/TestUtilities/TestDriver.swift index 80b910091..0a6ef03d9 100644 --- a/Tests/TestUtilities/TestDriver.swift +++ b/Tests/TestUtilities/TestDriver.swift @@ -168,6 +168,9 @@ private final class NoInPlaceExecutionExecutor: DriverExecutor { /// try await driver.run(jobs: jobs) /// ``` package struct TestDriver { + /// When false, a driver created without an explicit oracle gets a `TestHarnessInterModuleDependencyOracle`. + @TaskLocal package static var usesRealDependencyScanner = false + private var driver: Driver /// Per-instance stdout stream. Defaults to an in-memory buffer so tests can read it via @@ -211,7 +214,8 @@ package struct TestDriver { fileSystem: fs, executor: exec, integratedDriver: integratedDriver, - interModuleDependencyOracle: interModuleDependencyOracle, + interModuleDependencyOracle: interModuleDependencyOracle + ?? (Self.usesRealDependencyScanner ? nil : TestHarnessInterModuleDependencyOracle()), stdoutStream: stdout, stderrStream: stderr ) @@ -286,6 +290,7 @@ package struct TestDriver { package var numThreads: Int { driver.numThreads } package var packageName: String? { driver.packageName } package var interModuleDependencyOracle: InterModuleDependencyOracle { driver.interModuleDependencyOracle } + package var isExplicitModuleBuildEnabled: Bool { driver.isExplicitModuleBuildEnabled } package var absoluteSDKPath: AbsolutePath? { driver.absoluteSDKPath } package var isAutolinkExtractJobNeeded: Bool { mutating get { driver.isAutolinkExtractJobNeeded }