From 46fe04cf7456a6897713b8a91db2ae34276ad37e Mon Sep 17 00:00:00 2001 From: Volodymyr Sapsai Date: Mon, 28 Apr 2025 17:42:59 -0700 Subject: [PATCH] [Explicit Module Builds] Add support for creating a reproducer when clang process crashes. For the explicitly built modules generated script contains commands how to build all necessary .pcm files and how to compile the final translation unit. rdar://59743925 --- Sources/SWBCSupport/CLibclang.cpp | 27 +++++++++++++++ Sources/SWBCSupport/CLibclang.h | 6 ++++ .../SWBCore/LibclangVendored/Libclang.swift | 20 +++++++++++ .../ClangModuleDependencyGraph.swift | 32 +++++++++++++++-- .../TaskActions/ClangCompileTaskAction.swift | 13 +++++++ .../PrecompileClangModuleTaskAction.swift | 34 +++++++++++++------ 6 files changed, 119 insertions(+), 13 deletions(-) diff --git a/Sources/SWBCSupport/CLibclang.cpp b/Sources/SWBCSupport/CLibclang.cpp index 5a0ea7b4..d6640681 100644 --- a/Sources/SWBCSupport/CLibclang.cpp +++ b/Sources/SWBCSupport/CLibclang.cpp @@ -859,6 +859,12 @@ extern "C" { typedef struct CXOpaqueDependencyScannerServiceOptions *CXDependencyScannerServiceOptions; + enum CXErrorCode + (*clang_experimental_DependencyScanner_generateReproducer)( + int argc, const char *const *argv, const char *WorkingDirectory, + const char *ReproducerLocation, bool UseUniqueReproducerName, + CXString *messageOut); + /** * Creates a default set of service options. * Must be disposed with \c @@ -1423,6 +1429,7 @@ struct LibclangWrapper { LOOKUP_OPTIONAL(clang_experimental_cas_replayCompilation); LOOKUP_OPTIONAL(clang_experimental_cas_ReplayResult_dispose); LOOKUP_OPTIONAL(clang_experimental_cas_ReplayResult_getStderr); + LOOKUP_OPTIONAL(clang_experimental_DependencyScanner_generateReproducer); LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_create); LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_dispose); LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_setDependencyMode); @@ -1775,6 +1782,10 @@ extern "C" { lib->wrapper->fns.clang_experimental_DependencyScannerServiceOptions_setCWDOptimization; } + bool libclang_has_reproducer_feature(libclang_t lib) { + return lib->wrapper->fns.clang_experimental_DependencyScanner_generateReproducer; + } + libclang_casoptions_t libclang_casoptions_create(libclang_t lib) { auto opts = lib->wrapper->fns.clang_experimental_cas_Options_create(); return new libclang_casoptions_t_{{lib->wrapper, opts}}; @@ -2159,6 +2170,22 @@ extern "C" { return depGraph != nullptr; } + bool libclang_scanner_generate_reproducer(libclang_scanner_t scanner, + int argc, char *const *argv, + const char *workingDirectory, + const char **message) { + auto lib = scanner->scanner->lib; + LibclangFunctions::CXString messageString; + auto result = lib->fns.clang_experimental_DependencyScanner_generateReproducer( + argc, const_cast(argv), workingDirectory, + /*ReproducerLocation=*/NULL, /*UseUniqueReproducerName=*/true, &messageString); + if (message) { + *message = strdup(lib->fns.clang_getCString(messageString)); + } + lib->fns.clang_disposeString(messageString); + return result == LibclangFunctions::CXError_Success; + } + bool libclang_driver_get_actions(libclang_t wrapped_lib, int argc, char* const* argv, diff --git a/Sources/SWBCSupport/CLibclang.h b/Sources/SWBCSupport/CLibclang.h index 04701ae2..4161020b 100644 --- a/Sources/SWBCSupport/CLibclang.h +++ b/Sources/SWBCSupport/CLibclang.h @@ -126,6 +126,8 @@ bool libclang_has_cas_up_to_date_checks_feature(libclang_t lib); /// Whether the libclang has current working directory optimization support. bool libclang_has_current_working_directory_optimization(libclang_t lib); +bool libclang_has_reproducer_feature(libclang_t lib); + /// Create the CAS options object. libclang_casoptions_t libclang_casoptions_create(libclang_t lib); @@ -201,6 +203,10 @@ bool libclang_scanner_scan_dependencies( void (^diagnostics_callback)(const libclang_diagnostic_set_t), void (^error_callback)(const char *)); +bool libclang_scanner_generate_reproducer( + libclang_scanner_t scanner, int argc, char *const *argv, const char *workingDirectory, + const char **message); + /// Get the list of commands invoked by the given Clang driver command line. /// /// \param argc - The number of arguments. diff --git a/Sources/SWBCore/LibclangVendored/Libclang.swift b/Sources/SWBCore/LibclangVendored/Libclang.swift index de0c6a89..b1557974 100644 --- a/Sources/SWBCore/LibclangVendored/Libclang.swift +++ b/Sources/SWBCore/LibclangVendored/Libclang.swift @@ -102,6 +102,10 @@ public final class Libclang { public var supportsCurrentWorkingDirectoryOptimization: Bool { libclang_has_current_working_directory_optimization(lib) } + + public var supportsReproducerGeneration: Bool { + libclang_has_reproducer_feature(lib) + } } enum DependencyScanningError: Error { @@ -269,6 +273,22 @@ public final class DependencyScanner { } return fileDeps } + + public func generateReproducer( + commandLine: [String], + workingDirectory: String + ) throws -> String { + let args = CStringArray(commandLine) + var messageUnsafe: UnsafePointer! + defer { messageUnsafe?.deallocate() } + // The count is `- 1` here, because CStringArray appends a trailing nullptr. + let success = libclang_scanner_generate_reproducer(scanner, CInt(args.cArray.count - 1), args.cArray, workingDirectory, &messageUnsafe); + let message = String(cString: messageUnsafe) + guard success else { + throw Error.dependencyScanErrorString(message) + } + return message + } } fileprivate struct ClangDiagnosticSet { diff --git a/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift b/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift index 8cb06e21..a8dcc71d 100644 --- a/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift +++ b/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift @@ -109,6 +109,8 @@ package final class ClangModuleDependencyGraph { /// for example, when using `-save-temps`. package let commands: [CompileCommand] + package let scanningCommandLine: [String] + package let transitiveIncludeTreeIDs: OrderedSet package let transitiveCompileCommandCacheKeys: OrderedSet @@ -121,6 +123,7 @@ package final class ClangModuleDependencyGraph { moduleDependencies: OrderedSet, workingDirectory: Path, commands: [CompileCommand], + scanningCommandLine: [String], transitiveIncludeTreeIDs: OrderedSet, transitiveCompileCommandCacheKeys: OrderedSet, usesSerializedDiagnostics: Bool @@ -131,6 +134,7 @@ package final class ClangModuleDependencyGraph { self.modules = moduleDependencies self.workingDirectory = workingDirectory self.commands = commands + self.scanningCommandLine = scanningCommandLine self.transitiveIncludeTreeIDs = transitiveIncludeTreeIDs self.transitiveCompileCommandCacheKeys = transitiveCompileCommandCacheKeys self.usesSerializedDiagnostics = usesSerializedDiagnostics @@ -143,6 +147,7 @@ package final class ClangModuleDependencyGraph { moduleDependencies: OrderedSet, workingDirectory: Path, command: CompileCommand, + scanningCommandLine: [String], transitiveIncludeTreeIDs: OrderedSet, transitiveCompileCommandCacheKeys: OrderedSet, usesSerializedDiagnostics: Bool @@ -153,19 +158,21 @@ package final class ClangModuleDependencyGraph { self.modules = moduleDependencies self.workingDirectory = workingDirectory self.commands = [command] + self.scanningCommandLine = scanningCommandLine self.transitiveIncludeTreeIDs = transitiveIncludeTreeIDs self.transitiveCompileCommandCacheKeys = transitiveCompileCommandCacheKeys self.usesSerializedDiagnostics = usesSerializedDiagnostics } package func serialize(to serializer: T) where T : Serializer { - serializer.serializeAggregate(9) { + serializer.serializeAggregate(10) { serializer.serialize(kind) serializer.serialize(files) serializer.serialize(includeTreeID) serializer.serialize(modules) serializer.serialize(workingDirectory) serializer.serialize(commands) + serializer.serialize(scanningCommandLine) serializer.serialize(transitiveIncludeTreeIDs) serializer.serialize(transitiveCompileCommandCacheKeys) serializer.serialize(usesSerializedDiagnostics) @@ -173,13 +180,14 @@ package final class ClangModuleDependencyGraph { } package init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(9) + try deserializer.beginAggregate(10) self.kind = try deserializer.deserialize() self.files = try deserializer.deserialize() self.includeTreeID = try deserializer.deserialize() self.modules = try deserializer.deserialize() self.workingDirectory = try deserializer.deserialize() self.commands = try deserializer.deserialize() + self.scanningCommandLine = try deserializer.deserialize() self.transitiveIncludeTreeIDs = try deserializer.deserialize() self.transitiveCompileCommandCacheKeys = try deserializer.deserialize() self.usesSerializedDiagnostics = try deserializer.deserialize() @@ -334,12 +342,13 @@ package final class ClangModuleDependencyGraph { var moduleTransitiveCacheKeys: [String: OrderedSet] = [:] let fileDeps: DependencyScanner.FileDependencies + let scanningCommandLine = [compiler] + originalFileArgs let modulesCallbackErrors = LockedValue<[any Error]>([]) let dependencyPaths = LockedValue>([]) let requiredTargetDependencies = LockedValue>([]) do { fileDeps = try clangWithScanner.scanner.scanDependencies( - commandLine: [compiler] + originalFileArgs, + commandLine: scanningCommandLine, workingDirectory: workingDirectory.str, lookupOutput: { name, contextHash, kind in let moduleOutputPath = outputPathForModule(name, contextHash) @@ -432,6 +441,7 @@ package final class ClangModuleDependencyGraph { // Cached builds do not rely on the process working directory, and different scanner working directories should not inhibit task deduplication. The same is true if the scanner reports the working directory can be ignored. workingDirectory: module.cache_key != nil || module.is_cwd_ignored ? Path.root : workingDirectory, command: DependencyInfo.CompileCommand(cacheKey: module.cache_key, arguments: commandLine), + scanningCommandLine: scanningCommandLine, transitiveIncludeTreeIDs: transitiveIncludeTreeIDs, transitiveCompileCommandCacheKeys: transitiveCommandCacheKeys, usesSerializedDiagnostics: usesSerializedDiagnostics) @@ -513,6 +523,7 @@ package final class ClangModuleDependencyGraph { // Cached builds do not rely on the process working directory, and different scanner working directories should not inhibit task deduplication workingDirectory: fileDeps.commands.allSatisfy { $0.cache_key != nil } ? Path.root : workingDirectory, commands: commands, + scanningCommandLine: scanningCommandLine, transitiveIncludeTreeIDs: transitiveIncludeTreeIDs, transitiveCompileCommandCacheKeys: transitiveCommandCacheKeys, usesSerializedDiagnostics: usesSerializedDiagnostics) @@ -549,6 +560,21 @@ package final class ClangModuleDependencyGraph { return clangWithScanner.casDBs } + package func generateReproducer(forFailedDependency dependency: DependencyInfo, + libclangPath: Path, casOptions: CASOptions?) throws -> String? { + let clangWithScanner = try libclangWithScanner( + forPath: libclangPath, + casOptions: casOptions, + cacheFallbackIfNotAvailable: false, + core: core + ) + guard clangWithScanner.libclang.supportsReproducerGeneration else { + return nil + } + return try clangWithScanner.scanner.generateReproducer( + commandLine: dependency.scanningCommandLine, workingDirectory: dependency.workingDirectory.str) + } + package var isEmpty: Bool { recordedDependencyInfoRegistry.isEmpty } diff --git a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift index 5f26bde7..33d5f9d7 100644 --- a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift @@ -301,6 +301,19 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA outputDelegate.emitOutput("Failed frontend command:\n") outputDelegate.emitOutput(ByteString(encodingAsUTF8: commandString) + "\n") } + + if case .some(.exit(.uncaughtSignal, _)) = outputDelegate.result { + do { + if let reproducerMessage = try clangModuleDependencyGraph.generateReproducer( + forFailedDependency: dependencyInfo, + libclangPath: explicitModulesPayload.libclangPath, + casOptions: explicitModulesPayload.casOptions) { + outputDelegate.emitOutput(ByteString(encodingAsUTF8: reproducerMessage) + "\n") + } + } catch { + outputDelegate.error(error.localizedDescription) + } + } return lastResult ?? .failed } } diff --git a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift index 3c308b9e..7dc108f4 100644 --- a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift @@ -215,16 +215,30 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida enableStrictCASErrors: key.casOptions!.enableStrictCASErrors ) } - } else if result == .failed && !executionDelegate.userPreferences.enableDebugActivityLogs && !executionDelegate.emitFrontendCommandLines { - let commandString = UNIXShellCommandCodec( - encodingStrategy: .backslashes, - encodingBehavior: .fullCommandLine - ).encode(commandLine) - - // We need to find a way to use the generic infrastructure for displaying the command line in - // the build log. - outputDelegate.emitOutput("Failed frontend command:\n") - outputDelegate.emitOutput(ByteString(encodingAsUTF8: commandString) + "\n") + } else if result == .failed { + if !executionDelegate.userPreferences.enableDebugActivityLogs && !executionDelegate.emitFrontendCommandLines { + let commandString = UNIXShellCommandCodec( + encodingStrategy: .backslashes, + encodingBehavior: .fullCommandLine + ).encode(commandLine) + + // We need to find a way to use the generic infrastructure for displaying the command line in + // the build log. + outputDelegate.emitOutput("Failed frontend command:\n") + outputDelegate.emitOutput(ByteString(encodingAsUTF8: commandString) + "\n") + } + if case .some(.exit(.uncaughtSignal, _)) = outputDelegate.result { + do { + if let reproducerMessage = try clangModuleDependencyGraph.generateReproducer( + forFailedDependency: dependencyInfo, + libclangPath: key.libclangPath, + casOptions: key.casOptions) { + outputDelegate.emitOutput(ByteString(encodingAsUTF8: reproducerMessage) + "\n") + } + } catch { + outputDelegate.error(error.localizedDescription) + } + } } return result } catch {