Support -serialize-diagnostics=<format> in swift-driver

Sources/SwiftDriver/Driver/Driver.swift

@@ -419,6 +419,9 @@ public struct Driver {
   /// Path to the serialized diagnostics file.
   let serializedDiagnosticsFilePath: VirtualPath.Handle?
 
+  /// The format diagnostics are serialized in.
+  let serializedDiagnosticsFormat: FileType
+
   /// Path to the serialized diagnostics file of the emit-module task.
   let emitModuleSerializedDiagnosticsFilePath: VirtualPath.Handle?
 
@@ -1337,24 +1340,29 @@ public struct Driver {
         emitModuleSeparately: emitModuleSeparately,
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name)
+    self.serializedDiagnosticsFormat = Self.serializedDiagnosticsFormat(&parsedOptions,
+                                                                        diagnosticEngine: diagnosticsEngine)
     self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(
-        &parsedOptions, type: .diagnostics, isOutputOptions: [.serializeDiagnostics],
+        &parsedOptions, type: self.serializedDiagnosticsFormat,
+        isOutputOptions: [.serializeDiagnostics, .serializeDiagnosticsEQ],
         outputPath: .serializeDiagnosticsPath,
         compilerOutputType: compilerOutputType,
         compilerMode: compilerMode,
         emitModuleSeparately: emitModuleSeparately,
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name)
     self.emitModuleSerializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath(
-        &parsedOptions, type: .emitModuleDiagnostics, isOutputOptions: [.serializeDiagnostics],
+        &parsedOptions, type: Self.emitModuleSerializedDiagnosticsFormat(for: self.serializedDiagnosticsFormat),
+        isOutputOptions: [.serializeDiagnostics, .serializeDiagnosticsEQ],
         outputPath: .emitModuleSerializeDiagnosticsPath,
         compilerOutputType: compilerOutputType,
         compilerMode: compilerMode,
         emitModuleSeparately: emitModuleSeparately,
         outputFileMap: self.outputFileMap,
         moduleName: moduleOutputInfo.name)
     self.dependencyScanSerializedDiagnosticsPath = try Self.computeSupplementaryOutputPath(
-        &parsedOptions, type: .dependencyScanDiagnostics, isOutputOptions: [.serializeDiagnostics],
+        &parsedOptions, type: Self.dependencyScanSerializedDiagnosticsFormat(for: self.serializedDiagnosticsFormat),
+        isOutputOptions: [.serializeDiagnostics, .serializeDiagnosticsEQ],
         outputPath: .dependencyScanSerializeDiagnosticsPath,
         compilerOutputType: compilerOutputType,
         compilerMode: compilerMode,
@@ -3986,6 +3994,42 @@ extension Driver {
 
 // Supplementary outputs.
 extension Driver {
+  /// The format diagnostics are serialized in.
+  static func serializedDiagnosticsFormat(
+    _ parsedOptions: inout ParsedOptions,
+    diagnosticEngine: DiagnosticsEngine
+  ) -> FileType {
+    // The last of the bare flag, which means 'dia', and '=<format>' wins.
+    var format = FileType.diagnostics
+    for parsed in parsedOptions.arguments(for: .serializeDiagnostics, .serializeDiagnosticsEQ) {
+      guard parsed.option == .serializeDiagnosticsEQ else {
+        format = .diagnostics
+        continue
+      }
+      switch parsed.argument.asSingle {
+      case "sarif":
+        format = .sarifDiagnostics
+      case "dia":
+        format = .diagnostics
+      case let name:
+        diagnosticEngine.emit(.error_unsupported_argument(argument: name, option: .serializeDiagnosticsEQ))
+      }
+    }
+    return format
+  }
+
+  /// The format the emit-module job's diagnostics are serialized in, the same
+  /// as the rest of the compilation.
+  static func emitModuleSerializedDiagnosticsFormat(for format: FileType) -> FileType {
+    format == .sarifDiagnostics ? .emitModuleSARIFDiagnostics : .emitModuleDiagnostics
+  }
+
+  /// The format the dependency scan's diagnostics are serialized in, the same
+  /// as the rest of the compilation.
+  static func dependencyScanSerializedDiagnosticsFormat(for format: FileType) -> FileType {
+    format == .sarifDiagnostics ? .dependencyScanSARIFDiagnostics : .dependencyScanDiagnostics
+  }
+
   /// Determine the output path for a supplementary output.
   static func computeSupplementaryOutputPath(
     _ parsedOptions: inout ParsedOptions,
@@ -4031,7 +4075,7 @@ extension Driver {
 
     // Emit-module serialized diagnostics are always specified as a single-output
     // file
-    if type == .emitModuleDiagnostics,
+    if type == .emitModuleDiagnostics || type == .emitModuleSARIFDiagnostics,
        let singleOutputPath = try outputFileMap?.existingOutputForSingleInput(
            outputType: type) {
       return singleOutputPath

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -232,6 +232,10 @@ public extension Driver {
     if let depScanSerializedDiagnosticsPath = dependencyScanSerializedDiagnosticsPath {
       commandLine.appendFlag("-serialize-diagnostics-path")
       commandLine.appendPath(VirtualPath.lookup(depScanSerializedDiagnosticsPath))
+      // The format goes with the path, as for compile jobs.
+      if serializedDiagnosticsFormat == .sarifDiagnostics {
+        commandLine.appendFlag("-serialize-diagnostics=sarif")
+      }
     }
 
     // loadedModuleTrace file.

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -100,6 +100,7 @@ extension Driver {
          .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
          .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline,
          .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .dependencyScanDiagnostics,
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics, .dependencyScanSARIFDiagnostics,
          .cachedDiagnostics, .jsonSupportedFeatures, nil:
       return false
     }
@@ -551,12 +552,13 @@ extension FileType {
 
     case .swift, .dSYM, .autolink, .dependencies, .emitModuleDependencies,
          .swiftDocumentation, .pcm, .diagnostics, .emitModuleDiagnostics,
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics,
          .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord,
          .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface,
          .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
          .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline,
          .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo,
-         .cachedDiagnostics, .dependencyScanDiagnostics:
+         .cachedDiagnostics, .dependencyScanDiagnostics, .dependencyScanSARIFDiagnostics:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Jobs/EmitModuleJob.swift

@@ -40,7 +40,13 @@ extension Driver {
     addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
     addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)
     addSupplementalOutput(path: moduleOutputPaths.apiDescriptorFilePath, flag: "-emit-api-descriptor-path", type: .jsonAPIDescriptor)
-    addSupplementalOutput(path: emitModuleSerializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .emitModuleDiagnostics)
+    let emitModuleDiagnosticsFormat = Self.emitModuleSerializedDiagnosticsFormat(for: serializedDiagnosticsFormat)
+    addSupplementalOutput(path: emitModuleSerializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path",
+                          type: emitModuleDiagnosticsFormat)
+    // The format goes with the path, as for compile jobs.
+    if emitModuleSerializedDiagnosticsFilePath != nil && emitModuleDiagnosticsFormat == .emitModuleSARIFDiagnostics {
+      commandLine.appendFlag("-serialize-diagnostics=sarif")
+    }
     addSupplementalOutput(path: emitModuleDependenciesFilePath, flag: "-emit-dependencies-path", type: .emitModuleDependencies)
 
     // Skip files created by other jobs when emitting a module and building at the same time

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -935,7 +935,7 @@ extension Driver {
         flag: "-emit-reference-dependencies-path")
 
       try addOutputOfType(
-        outputType: .diagnostics,
+        outputType: serializedDiagnosticsFormat,
         finalOutputPath: serializedDiagnosticsFilePath,
         input: input,
         flag: "-serialize-diagnostics-path")
@@ -1179,6 +1179,11 @@ extension Driver {
       }
     }
 
+    // The format flag enables serialization, so pass it only with a SARIF output.
+    if flaggedInputOutputPairs.contains(where: { $0.output.type == .sarifDiagnostics }) {
+      commandLine.appendFlag("-serialize-diagnostics=sarif")
+    }
+
     return flaggedInputOutputPairs.map { $0.output }
   }
 

Sources/SwiftDriver/Jobs/GeneratePCHJob.swift

@@ -47,23 +47,32 @@ extension Driver {
     try addRuntimeLibraryFlags(commandLine: &commandLine)
 
     // TODO: Should this just be pch output with extension changed?
-    if parsedOptions.hasArgument(.serializeDiagnostics), let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle {
+    if parsedOptions.hasArgument(.serializeDiagnostics, .serializeDiagnosticsEQ),
+       let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle {
       commandLine.appendFlag(.serializeDiagnosticsPath)
       let path: VirtualPath
-      if let outputPath = try outputFileMap?.existingOutput(inputFile: input.fileHandle, outputType: .diagnostics) {
+      if let outputPath = try outputFileMap?.existingOutput(inputFile: input.fileHandle,
+                                                            outputType: serializedDiagnosticsFormat) {
         path = VirtualPath.lookup(outputPath)
       } else if let modulePath = parsedOptions.getLastArgument(.emitModulePath) {
         // TODO: does this hash need to be persistent?
         let code = UInt(bitPattern: modulePath.asSingle.hashValue)
         let outputName = input.file.basenameWithoutExt + "-" + String(code, radix: 36)
-        path = try VirtualPath(path: outputDirectory).appending(component: outputName.appendingFileTypeExtension(.diagnostics))
+        path = try VirtualPath(path: outputDirectory).appending(
+          component: outputName.appendingFileTypeExtension(serializedDiagnosticsFormat))
       } else {
         path =
           try VirtualPath.createUniqueTemporaryFile(
-            RelativePath(validating: input.file.basenameWithoutExt.appendingFileTypeExtension(.diagnostics)))
+            RelativePath(
+              validating: input.file.basenameWithoutExt.appendingFileTypeExtension(serializedDiagnosticsFormat)))
       }
       commandLine.appendPath(path)
-      outputs.append(.init(file: path.intern(), type: .diagnostics))
+      outputs.append(.init(file: path.intern(), type: serializedDiagnosticsFormat))
+
+      // The format goes with the path, as for compile jobs.
+      if serializedDiagnosticsFormat == .sarifDiagnostics {
+        commandLine.appendFlag("-serialize-diagnostics=sarif")
+      }
     }
 
     // New compute and add inputs and outputs.

Sources/SwiftDriver/Utilities/FileType.swift

@@ -86,12 +86,21 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Clang/Swift serialized diagnostics
   case diagnostics = "dia"
 
+  /// Diagnostics serialized to SARIF
+  case sarifDiagnostics = "sarif"
+
   /// Serialized diagnostics produced by module-generation
   case emitModuleDiagnostics = "emit-module.dia"
 
+  /// Diagnostics produced by module-generation, serialized to SARIF
+  case emitModuleSARIFDiagnostics = "emit-module.sarif"
+
   /// Serialized diagnostics produced by dependency scanning
   case dependencyScanDiagnostics = "dependency-scan.dia"
 
+  /// Diagnostics produced by dependency scanning, serialized to SARIF
+  case dependencyScanSARIFDiagnostics = "dependency-scan.sarif"
+
   /// Serialized diagnostics produced by module-generation
   case emitModuleDependencies = "emit-module.d"
 
@@ -256,12 +265,21 @@ extension FileType: CustomStringConvertible {
     case .diagnostics:
       return "diagnostics"
 
+    case .sarifDiagnostics:
+      return "sarif-diagnostics"
+
     case .emitModuleDiagnostics:
       return "emit-module-diagnostics"
 
+    case .emitModuleSARIFDiagnostics:
+      return "emit-module-sarif-diagnostics"
+
     case .dependencyScanDiagnostics:
       return "dependency-scan-diagnostics"
 
+    case .dependencyScanSARIFDiagnostics:
+      return "dependency-scan-sarif-diagnostics"
+
     case .jsonAPIBaseline:
       return "api-baseline-json"
 
@@ -300,6 +318,7 @@ extension FileType {
          .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies,
          .emitModuleDependencies, .swiftDocumentation, .pcm, .diagnostics,
          .emitModuleDiagnostics, .dependencyScanDiagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace,
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics, .dependencyScanSARIFDiagnostics,
          .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
          .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile,
          .jsonDependencies, .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures,
@@ -428,10 +447,16 @@ extension FileType {
       return "bitstream-opt-record"
     case .diagnostics:
       return "diagnostics"
+    case .sarifDiagnostics:
+      return "sarif-diagnostics"
     case .emitModuleDiagnostics:
       return "emit-module-diagnostics"
+    case .emitModuleSARIFDiagnostics:
+      return "emit-module-sarif-diagnostics"
     case .dependencyScanDiagnostics:
       return "dependency-scan-diagnostics"
+    case .dependencyScanSARIFDiagnostics:
+      return "dependency-scan-sarif-diagnostics"
     case .indexUnitOutputPath:
       return "index-unit-output-path"
     case .jsonAPIBaseline:
@@ -463,7 +488,8 @@ extension FileType {
          .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo,
          .jsonSwiftArtifacts, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues,
          .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics,
-         .raw_llvmIr, .jsonSupportedFeatures:
+         .raw_llvmIr, .jsonSupportedFeatures, .sarifDiagnostics, .emitModuleSARIFDiagnostics,
+         .dependencyScanSARIFDiagnostics:
       return true
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
@@ -483,6 +509,7 @@ extension FileType {
          .autolink, .swiftModule, .swiftDocumentation, .swiftInterface,
          .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .raw_sil, .raw_sib,
          .diagnostics, .emitModuleDiagnostics, .objcHeader, .swiftDeps, .remap,
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics, .dependencyScanSARIFDiagnostics,
          .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
          .modDepCache, .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies,
          .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
@@ -498,7 +525,8 @@ extension FileType {
     switch self {
     case .swift, .ast, .indexData, .indexUnitOutputPath, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSupportedFeatures:
       return false
-    case .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .swiftConstValues, .assembly, .raw_sil, .raw_sib, .llvmIR, .llvmBitcode, .diagnostics, .emitModuleDiagnostics, .emitModuleDependencies, .objcHeader, .swiftDeps, .modDepCache, .remap, .importedModules, .tbd, .jsonDependencies, .jsonSwiftArtifacts, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonAPIBaseline, .jsonABIBaseline, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr, .dependencyScanDiagnostics:
+    case .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .swiftConstValues, .assembly, .raw_sil, .raw_sib, .llvmIR, .llvmBitcode, .diagnostics, .emitModuleDiagnostics, .emitModuleDependencies, .objcHeader, .swiftDeps, .modDepCache, .remap, .importedModules, .tbd, .jsonDependencies, .jsonSwiftArtifacts, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonAPIBaseline, .jsonABIBaseline, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr, .dependencyScanDiagnostics,
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics, .dependencyScanSARIFDiagnostics:
       return true
     }
   }
@@ -507,6 +535,7 @@ extension FileType {
   var supportCaching: Bool {
     switch self {
     case .diagnostics, .emitModuleDiagnostics, // diagnostics are cached using cached diagnostics.
+         .sarifDiagnostics, .emitModuleSARIFDiagnostics,
          // Those are by-product from swift-driver and not considered outputs need caching.
          .jsonSwiftArtifacts, .remap, .indexUnitOutputPath, .modDepCache,
          // the remaining should not be an output from a caching swift job.
@@ -515,6 +544,7 @@ extension FileType {
       return false
     case .assembly, .llvmIR, .llvmBitcode, .object, .sil, .sib, .ast,
          .dependencies, .emitModuleDependencies, .swiftModule, .dependencyScanDiagnostics,
+         .dependencyScanSARIFDiagnostics,
          .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface,
          .swiftSourceInfoFile, .raw_sil, .raw_sib, .objcHeader, .swiftDeps, .tbd,
          .moduleTrace, .indexData, .yamlOptimizationRecord,

Sources/SwiftDriverExecution/MultiJobExecutor.swift

@@ -707,7 +707,7 @@ fileprivate extension Job {
   /// Don't leave incorrect compiler outputs lying around, but don't remove diagnostics!
   func removeOutputsOfFailedCompilation(from fileSystem: TSCBasic.FileSystem) {
     guard kind.isCompile else {return}
-    for output in outputs where output.type != .diagnostics {
+    for output in outputs where output.type != .diagnostics && output.type != .sarifDiagnostics {
       guard let absolutePath = output.file.absolutePath else { continue }
       try? fileSystem.removeFileTree(absolutePath)
     }

Tests/SwiftDriverTests/JobExecutorTests.swift

@@ -241,6 +241,46 @@ extension DarwinToolchain {
     }
   }
 
+  /// A failed compile job loses its outputs, except for its diagnostics, which
+  /// explain the failure.
+  @Test(.skipHostOS(.win32, comment: "Uses /bin/sh"),
+        arguments: [FileType.diagnostics, .sarifDiagnostics])
+  func failedCompilationKeepsDiagnostics(diagnosticsType: FileType) throws {
+    try withTemporaryDirectory { path in
+      let object = path.appending(component: "main.o")
+      let diagnostics = path.appending(component: "main.\(diagnosticsType.rawValue)")
+
+      // Stands in for a frontend that writes its outputs and then fails.
+      let compile = Job(
+        moduleName: "main",
+        kind: .compile,
+        tool: ResolvedTool(path: try AbsolutePath(validating: "/bin/sh"), supportsResponseFiles: false),
+        commandLine: [
+          "-c", "touch \"$0\" \"$1\"; exit 1",
+          .path(.absolute(object)), .path(.absolute(diagnostics)),
+        ],
+        inputs: [],
+        primaryInputs: [],
+        outputs: [
+          .init(file: VirtualPath.absolute(object).intern(), type: .object),
+          .init(file: VirtualPath.absolute(diagnostics).intern(), type: diagnosticsType),
+        ]
+      )
+      let executor = MultiJobExecutor(
+        workload: .all([compile]),
+        resolver: try ArgsResolver(fileSystem: localFileSystem),
+        executorDelegate: JobCollectingDelegate(),
+        diagnosticsEngine: DiagnosticsEngine()
+      )
+      #expect(throws: (any Error).self) {
+        try executor.execute(env: ProcessEnv.block, fileSystem: localFileSystem)
+      }
+
+      #expect(!localFileSystem.exists(object))
+      #expect(localFileSystem.exists(diagnostics))
+    }
+  }
+
   /// Ensure the executor is capable of forwarding its standard input to the compile job that requires it.
   @Test(.requireHostOS(.macosx)) func inputForwarding() async throws {
     let hostTriple = try TestDriver(args: ["swiftc", "test.swift"]).hostTriple