Support loading macros from a binary artifact

.gitignore

@@ -23,3 +23,4 @@ Package.resolved
 /stage/
 Utilities/InstalledSwiftPMConfiguration/config.json
 Fixtures/BinaryLibraries/Static/Package1/Simple.artifactbundle/build
+Packages/

Sources/Build/BuildDescription/SwiftModuleBuildDescription.swift

@@ -246,6 +246,18 @@ public final class SwiftModuleBuildDescription {
         }
     }
 
+    /// Binary dependencies that vend prebuilt macro plugin executables (an artifact bundle
+    /// containing a `macro`-typed artifact), as opposed to `.macro` targets built from source.
+    public var requiredBinaryMacros: [BinaryModule] {
+        get throws {
+            try self.target.recursiveModuleDependencies().compactMap {
+                guard $0.type == .binary, let binary = $0.underlying as? BinaryModule,
+                      binary.containsMacro else { return nil }
+                return binary
+            }
+        }
+    }
+
     /// ObservabilityScope with which to emit diagnostics
     private let observabilityScope: ObservabilityScope
 
@@ -470,6 +482,19 @@ public final class SwiftModuleBuildDescription {
         }
         #endif
 
+        // Prebuilt macro plugins shipped as binary artifact bundles. These are host tools, so
+        // their variants are selected against the host triple. The artifact key is
+        // used as the plugin module name, matching the `module:` of the `#externalMacro` declaration
+        try self.requiredBinaryMacros.forEach { binaryMacro in
+            let macros = try binaryMacro.parseMacroArtifactArchives(
+                for: macroBuildParameters.triple,
+                fileSystem: self.fileSystem
+            )
+            for macro in macros where !macro.supportedTriples.isEmpty {
+                args += ["-Xfrontend", "-load-plugin-executable", "-Xfrontend", "\(macro.executablePath.pathString)#\(macro.name)"]
+            }
+        }
+
         if self.shouldDisableSandbox {
             let toolchainSupportsDisablingSandbox = DriverSupport.checkSupportedFrontendFlags(
                 flags: ["-disable-sandbox"],

Sources/PackageModel/ArtifactsArchiveMetadata.swift

@@ -51,6 +51,9 @@ public struct ArtifactsArchiveMetadata: Equatable {
 
         // Can't be marked as formally deprecated as we still need to use this value for warning users.
         case crossCompilationDestination
+
+        // Plugin executable that implements a macro, keyed to the host triple, not build target
+        case macro
     }
 
     public struct Variant: Equatable {

Sources/PackageModel/Module/BinaryModule.swift

@@ -21,7 +21,7 @@ public final class BinaryModule: Module {
 
     /// The kind of binary artifact.
     public let kind: Kind
-    
+
     /// The original source of the binary artifact.
     public let origin: Origin
 
@@ -97,6 +97,17 @@ public final class BinaryModule: Module {
         }
     }
 
+    public var containsMacro: Bool {
+        switch self.kind {
+        case .xcframework:
+            return false
+        case .artifactsArchive(let types):
+            return types.contains(.macro)
+        case .unknown:
+            return false
+        }
+    }
+
     public enum Origin: Equatable {
 
         /// Represents an artifact that was downloaded from a remote URL.

Sources/SPMBuildCore/BinaryTarget+Extensions.swift

@@ -97,6 +97,30 @@ extension BinaryModule {
         }
     }
 
+    /// Parses the prebuilt macro plugin executables in this artifact bundle, matching the given host `triple`.
+    public func parseMacroArtifactArchives(for triple: Triple, fileSystem: any FileSystem) throws -> [ExecutableInfo] {
+        // The host triple might contain a version which we don't want to take into account here.
+        let versionLessTriple = try triple.withoutVersion()
+        let metadata = try ArtifactsArchiveMetadata.parse(fileSystem: fileSystem, rootPath: self.artifactPath)
+        // Filter out everything except macro plugins.
+        let macros = metadata.artifacts.filter { $0.value.type == .macro }
+        // Construct an ExecutableInfo for each matching variant.
+        return try macros.flatMap { entry in
+            try entry.value.variants.map {
+                guard let supportedTriples = $0.supportedTriples else {
+                    throw StringError("No \"supportedTriples\" found in the artifact metadata for \(entry.key) in \(self.artifactPath)")
+                }
+                let filteredSupportedTriples = try supportedTriples
+                    .filter { try $0.withoutVersion() == versionLessTriple }
+                return ExecutableInfo(
+                    name: entry.key,
+                    executablePath: self.artifactPath.appending($0.path),
+                    supportedTriples: filteredSupportedTriples
+                )
+            }
+        }
+    }
+
     public func parseWindowsDLLArtifactArchives(for triple: Triple, fileSystem: any FileSystem) throws -> [WindowsDLLInfo] {
         // The host triple might contain a version which we don't want to take into account here.
         let versionLessTriple = try triple.withoutVersion()

Sources/SwiftBuildSupport/PIFBuilder.swift

@@ -468,6 +468,7 @@ public final class PIFBuilder {
                 addLocalRpaths: self.parameters.addLocalRpaths,
                 packageDisplayVersion: package.manifest.displayName,
                 pkgConfigDirectories: self.parameters.pkgConfigDirectories,
+                hostTriple: try self.parameters.pluginScriptRunner.hostTriple,
                 fileSystem: self.fileSystem,
                 observabilityScope: self.observabilityScope,
             )

Sources/SwiftBuildSupport/PackagePIFBuilder.swift

@@ -16,6 +16,7 @@ import protocol TSCBasic.FileSystem
 
 import struct Basics.AbsolutePath
 import struct Basics.SourceControlURL
+import struct Basics.Triple
 import struct Basics.Diagnostic
 import struct Basics.ObservabilityMetadata
 import class Basics.ObservabilityScope
@@ -50,6 +51,8 @@ public final class PackagePIFBuilder {
     let modulesGraph: ModulesGraph
     private let package: ResolvedPackage
 
+    let hostTriple: Basics.Triple
+
     /// Contains the package declarative specification.
     let packageManifest: PackageModel.Manifest // FIXME: Can't we just use `package.manifest` instead? —— Paulo
 
@@ -236,6 +239,7 @@ public final class PackagePIFBuilder {
         addLocalRpaths: AddLocalRpaths = .always,
         packageDisplayVersion: String?,
         pkgConfigDirectories: [AbsolutePath],
+        hostTriple: Basics.Triple,
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope,
     ) {
@@ -249,6 +253,7 @@ public final class PackagePIFBuilder {
         self.createDynamicVariantsForLibraryProducts = createDynamicVariantsForLibraryProducts
         self.packageDisplayVersion = packageDisplayVersion
         self.pkgConfigDirectories = pkgConfigDirectories
+        self.hostTriple = hostTriple
         self.fileSystem = fileSystem
         self.observabilityScope = observabilityScope
         self.addLocalRpaths = addLocalRpaths
@@ -266,6 +271,7 @@ public final class PackagePIFBuilder {
         addLocalRpaths: AddLocalRpaths = .always,
         packageDisplayVersion: String?,
         pkgConfigDirectories: [AbsolutePath],
+        hostTriple: Basics.Triple,
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope,
     ) {
@@ -280,6 +286,7 @@ public final class PackagePIFBuilder {
         self.addLocalRpaths = addLocalRpaths
         self.packageDisplayVersion = packageDisplayVersion
         self.pkgConfigDirectories = pkgConfigDirectories
+        self.hostTriple = hostTriple
         self.fileSystem = fileSystem
         self.observabilityScope = observabilityScope
     }

Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift

@@ -31,6 +31,8 @@ import struct PackageGraph.ResolvedPackage
 
 import struct PackageLoading.GeneratedFiles
 
+import SPMBuildCore
+
 import enum SwiftBuild.ProjectModel
 
 /// Extension to create PIF **modules** for a given package.
@@ -722,6 +724,22 @@ extension PackagePIFProjectBuilder {
             settings[.SKIP_BUILDING_DOCUMENTATION] = "YES"
         }
 
+        for dependency in try sourceModule.recursiveModuleDependencies() {
+            guard dependency.type == .binary,
+                  let binaryModule = dependency.underlying as? BinaryModule,
+                  binaryModule.containsMacro else { continue }
+            let macros = try binaryModule.parseMacroArtifactArchives(
+                for: pifBuilder.hostTriple,
+                fileSystem: pifBuilder.fileSystem
+            )
+            for macro in macros where !macro.supportedTriples.isEmpty {
+                let entry = "\(macro.executablePath.pathString)#\(macro.name)"
+                var loadBinaryMacros = settings[.SWIFT_LOAD_BINARY_MACROS] ?? []
+                loadBinaryMacros.append(entry)
+                settings[.SWIFT_LOAD_BINARY_MACROS] = loadBinaryMacros
+            }
+        }
+
         sourceModule.addParseAsLibrarySettings(to: &settings, toolsVersion: package.manifest.toolsVersion, fileSystem: pifBuilder.fileSystem)
 
         // Handle the target's dependencies (but only link against them if needed).
@@ -763,6 +781,12 @@ extension PackagePIFProjectBuilder {
                         log(.error, "'\(moduleDependency.name)' is a binary dependency, but its underlying module was not")
                         break
                     }
+                    if binaryModule.containsMacro {
+                        // Don't add it as a build file so the SwiftBuild engine does
+                        // not treat the artifact bundle as a linkable/resource artifact
+                        log(.debug, indent: 1, "Loading prebuilt binary macro '\(moduleDependency.name)'")
+                        break
+                    }
                     let binaryReference = self.binaryGroup.addFileReference { id in
                         return Self.createBinaryModuleFileReference(binaryModule, id: id)
                     }

Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Products.swift

@@ -19,6 +19,7 @@ import class Basics.ObservabilitySystem
 import struct Basics.SourceControlURL
 
 import PackageLoading
+import SPMBuildCore
 
 import class PackageModel.BinaryModule
 import class PackageModel.Manifest
@@ -203,6 +204,22 @@ extension PackagePIFProjectBuilder {
         settings[.CLANG_CXX_LANGUAGE_STANDARD] = mainModule.cxxLanguageStandard
         settings[.SWIFT_ENABLE_BARE_SLASH_REGEX] = "NO"
 
+        // Load any prebuilt macro plugins
+        for dependency in try mainModule.recursiveModuleDependencies() {
+            guard dependency.type == .binary,
+                  let binaryModule = dependency.underlying as? BinaryModule,
+                  binaryModule.containsMacro else { continue }
+            let macros = try binaryModule.parseMacroArtifactArchives(
+                for: pifBuilder.hostTriple,
+                fileSystem: pifBuilder.fileSystem
+            )
+            for macro in macros where !macro.supportedTriples.isEmpty {
+                var loadBinaryMacros = settings[.SWIFT_LOAD_BINARY_MACROS] ?? []
+                loadBinaryMacros.append("\(macro.executablePath.pathString)#\(macro.name)")
+                settings[.SWIFT_LOAD_BINARY_MACROS] = loadBinaryMacros
+            }
+        }
+
         // Create a group for the source files of the main module
         // For now we use an absolute path for it, but we should really make it
         // container-relative, since it's always inside the package directory.
@@ -383,6 +400,11 @@ extension PackagePIFProjectBuilder {
                         log(.error, "'\(moduleDependency.name)' is a binary dependency, but its underlying module was not")
                         break
                     }
+                    if binaryModule.containsMacro {
+                        // Skip adding prebuilt macro plugin as a build file
+                        log(.debug, indent: 1, "Loading prebuilt binary macro '\(moduleDependency.name)'")
+                        break
+                    }
                     let binaryFileRef = self.binaryGroup.addFileReference { id in
                         Self.createBinaryModuleFileReference(binaryModule, id: id)
                     }
@@ -676,6 +698,11 @@ extension PackagePIFProjectBuilder {
         for module in product.modules {
             // Binary targets are special in that they are just linked, not built.
             if let binaryTarget = module.underlying as? BinaryModule {
+                if binaryTarget.containsMacro {
+                    // Prebuilt macro plugin: loaded via SWIFT_LOAD_BINARY_MACROS, not linked.
+                    log(.debug, indent: 1, "Loading prebuilt binary macro '\(binaryTarget.name)'")
+                    continue
+                }
                 let binaryFileRef = self.binaryGroup.addFileReference { id in
                     FileReference(id: id, path: binaryTarget.artifactPath.pathString)
                 }
@@ -793,6 +820,11 @@ extension PackagePIFProjectBuilder {
                 }
 
                 if let binaryTarget = moduleDependency.underlying as? BinaryModule {
+                    if binaryTarget.containsMacro {
+                        // Prebuilt macro plugin: loaded via SWIFT_LOAD_BINARY_MACROS, not linked.
+                        log(.debug, indent: 1, "Loading prebuilt binary macro '\(binaryTarget.name)'")
+                        return
+                    }
                     let binaryFileRef = self.binaryGroup.addFileReference { id in
                         FileReference(id: id, path: binaryTarget.artifactPath.pathString)
                     }

Tests/BuildTests/BuildPlanTests.swift

@@ -6814,6 +6814,80 @@ class BuildPlanTestCase: BuildSystemProviderTestCase {
         try await self.sanitizerTest(.address, expectedName: "address")
     }
 
+    func testBinaryMacroLoadsPluginExecutable() async throws {
+        // A Swift module that depends on a `macro`-typed artifact bundle should be compiled
+        // with `-load-plugin-executable <host-variant>#<artifact-name>`, where the artifact
+        // name matches the `module:` of the consumer's `#externalMacro` declaration. This is
+        // the prebuilt counterpart to building a `.macro` target from source.
+        let fs = InMemoryFileSystem(emptyFiles: "/Pkg/Sources/MyLib/MyLib.swift")
+
+        let toolPath = AbsolutePath("/Pkg/MyMacros.artifactbundle")
+        try fs.createDirectory(toolPath, recursive: true)
+        try fs.writeFileContents(
+            toolPath.appending("info.json"),
+            string: """
+                {
+                    "schemaVersion": "1.0",
+                    "artifacts": {
+                        "MyMacros": {
+                            "type": "macro",
+                            "version": "1.0.0",
+                            "variants": [
+                                {
+                                    "path": "x86_64-apple-macosx/MyMacros",
+                                    "supportedTriples": ["x86_64-apple-macosx"]
+                                },
+                                {
+                                    "path": "x86_64-unknown-linux-gnu/MyMacros",
+                                    "supportedTriples": ["x86_64-unknown-linux-gnu"]
+                                }
+                            ]
+                        }
+                    }
+                }
+            """
+        )
+
+        let observability = ObservabilitySystem.makeForTesting()
+        let graph = try loadModulesGraph(
+            fileSystem: fs,
+            manifests: [
+                Manifest.createRootManifest(
+                    displayName: "Pkg",
+                    path: "/Pkg",
+                    targets: [
+                        TargetDescription(name: "MyLib", dependencies: ["MyMacros"]),
+                        TargetDescription(name: "MyMacros", path: "MyMacros.artifactbundle", type: .binary),
+                    ]
+                ),
+            ],
+            binaryArtifacts: [
+                .plain("pkg"): [
+                    "MyMacros": .init(kind: .artifactsArchive(types: [.macro]), originURL: nil, path: toolPath),
+                ],
+            ],
+            observabilityScope: observability.topScope
+        )
+        XCTAssertNoDiagnostics(observability.diagnostics)
+
+        let result = try await BuildPlanResult(plan: mockBuildPlan(
+            triple: .x86_64MacOS,
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: observability.topScope
+        ))
+        XCTAssertNoDiagnostics(observability.diagnostics)
+
+        let lib = try result.moduleBuildDescription(for: "MyLib").swift()
+        let args = try lib.compileArguments().joined(separator: " ")
+
+        // Only the host-matching variant (x86_64-apple-macosx) should be loaded, keyed by the
+        // artifact name; the foreign Linux variant must not be passed to the host compiler.
+        XCTAssertMatch(args, .contains("-load-plugin-executable"))
+        XCTAssertMatch(args, .contains("x86_64-apple-macosx/MyMacros#MyMacros"))
+        XCTAssertNoMatch(args, .contains("x86_64-unknown-linux-gnu/MyMacros"))
+    }
+
     func testThreadSanitizer() async throws {
         try await self.sanitizerTest(.thread, expectedName: "thread")
     }